diff --git a/.gitignore b/.gitignore index 8c960ec..d7516f0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ *.jl.cov *.jl.*.cov *.jl.mem +deps/build.log +deps/deps.jl +Manifest.toml diff --git a/.travis.yml b/.travis.yml index c3394ff..2220295 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,16 +1,33 @@ +## Documentation: http://docs.travis-ci.com/user/languages/julia/ language: julia os: - linux - osx julia: - - 0.6 + - 0.7 - nightly notifications: email: false git: - depth: 999999 -#script: # use the default script which is equivalent to the following -# - if [[ -a .git/shallow ]]; then git fetch --unshallow; fi -# - julia -e 'Pkg.clone(pwd()); Pkg.build("PkgDev"); Pkg.test("PkgDev"; coverage=true)' + depth: 99999999 + +## uncomment the following lines to allow failures on nightly julia +## (tests will run but not make your overall status red) +#matrix: +# allow_failures: +# - julia: nightly + +## uncomment and modify the following lines to manually install system packages +#addons: +# apt: # apt-get for linux +# packages: +# - gfortran +#before_script: # homebrew for mac +# - if [ $TRAVIS_OS_NAME = osx ]; then brew install gcc; fi + +## uncomment the following lines to override the default test script +#script: +# - julia -e 'Pkg.build(); Pkg.test(; coverage=true)' after_success: - - julia -e 'cd(Pkg.dir("PkgDev")); Pkg.add("Coverage"); using Coverage; Codecov.submit(process_folder())' + # push coverage results to Codecov + - julia -e 'Pkg.add("Coverage"); using Coverage; Codecov.submit(Codecov.process_folder())' diff --git a/Project.toml b/Project.toml new file mode 100644 index 0000000..82343e6 --- /dev/null +++ b/Project.toml @@ -0,0 +1,19 @@ +name = "PkgDev" +uuid = "149e707d-584d-56d3-88ec-740c18e106ff" +version = "0.2.1" + +[deps] +Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" +JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +LibGit2 = "76f85450-5226-5b5a-8eaa-529ad045b433" +Nullables = "4d1e1d77-625e-5b40-9113-a560ec7a8ecd" +Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +URIParser = "30578b45-9adc-5946-b283-645ec420af67" +UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" + +[extras] +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[targets] +test = ["Test"] diff --git a/README.md b/README.md index 8990a1d..0caff4e 100644 --- a/README.md +++ b/README.md @@ -1,54 +1,69 @@ -# Julia Package Development Kit (PDK) +# Julia Package Development Kit [![Build Status](https://travis-ci.org/JuliaLang/PkgDev.jl.svg?branch=master)](https://travis-ci.org/JuliaLang/PkgDev.jl)[![Build status](https://ci.appveyor.com/api/projects/status/gnd6dqbdaxcx1c23/branch/master?svg=true)](https://ci.appveyor.com/project/wildart/pkgdev-jl/branch/master) -PkgDev.jl provides a set of tools for a developer to create, maintain and register packages in Julia package repository, a.k.a. [METADATA](https://github.com/JuliaLang/METADATA.jl). +PkgDev.jl provides a set of tools for a developer to create, maintain and register packages in a Julia package registry, for example (but not limited to) [METADATA](https://github.com/JuliaLang/METADATA.jl). ## Requirements + For closer integration with GitHub API, PkgDev.jl requires `curl` to be installed. ## Usage -### register(pkg, [url]) -Register `pkg` at the git URL `url`, defaulting to the configured origin URL of the git repository `Pkg.dir(pkg)`. +### Generating a `Project.toml` based on `REQUIRE` + +In julia 0.7 and later, the file used to describe the dependnecies of a package was changed from a file +called `REQUIRE` to `Project.toml`. Use `PkgDev.add_project(pkgdir)` to add add a `Project.toml` file based on +the `REQUIRE` file. + +### `generate(pkgdir, license)` + +Generate a new package named `dirname(pkgdir)` at `pkgdir` with one of the bundled license: `"MIT"`, `"BSD"`, `CC0`, `"ISC"`, `"ASL"`, `"MPL"`, `"GPL-2.0+"`, `"GPL-3.0+"`, `"LGPL-2.1+"`, `"LGPL-3.0+"`. +If you want to make a package with a different license, you can edit it afterwards. `generate` creates a git repository at `pkgdir` for the package and inside it `LICENSE.md`, `README.md`, `Project.toml`, the julia entrypoint `$pkg/src/$pkg.jl`, and Travis and AppVeyor CI configuration files `.travis.yml` and `appveyor.yml`. + +> *Warning*: If you release code for under the GPL, you may discourage collaboration from members of the Julia community who work on non-GPL packages. For example, if a user works on Package Y, which is licensed under the MIT license that is used in many community projects, other developers might not feel safe to read the code you contribute to Package Y because any indication that their work is derivative could lead to litigation. In effect, you create a situation in which your source code is percieved as being closed to anyone who is working on a non-GPL project. + +Keyword parameters: + +* `travis` - enables generation of the `.travis.yml` configuration for [Travis CI](https://travis-ci.org/) service, the default value is `true`. +* `appveyor` - enables generation of the `appveyor.yml` configuration for [Appveyor](http://www.appveyor.com/) service, the default value is `true`. +* `coverage` - enables generation of a code coverage reporting to [Codecov](https://codecov.io) services, default value is `true`. + +### `register(pkgdir; [commit, registry, url]) + +Register the package at `pkgdir` at the git URL `url` to the registry at `registry` at `commit`, +defaulting to the configured origin URL of the git repository at `pkgdir`. +The `registry` defaults to the General registry, typically located at `~/.julia/registries/General` and +`commit` defaults to the current commit of the repo. -### tag(pkg, [ver, [commit]]) -Tag `commit` as version `ver` of package `pkg` and create a version entry in `METADATA`. If not provided, `commit` defaults to the current commit of the `pkg` repository. If `ver` is one of the symbols `:patch`, `:minor`, `:major` the next patch, minor or major version is used. If `ver` is not provided, it defaults to `:patch`. +### `tag(pkgdir; [commit, registry])` + +Tag the package at `pkgdir` on `commit` to the registry at `registry`, defaulting to the configured origin URL of the git repository at `pkgdir`. The `registry` defaults to the General registry, typically located at `~/.julia/registries/General` and `commit` defaults to the current commit of the repo. +The version used is the `version` entry in the package's project file (typically `Project.toml`). You are strongly encouraged to update the version numbers in accordance with the [semver standard](http://semver.org/): -* Use `tag(pkg, :major)` when you make *backwards-incompatible* API changes (i.e. changes that will break existing user code). -* Use `tag(pkg, :minor)` when you *add functionality* in a backwards-compatible way (i.e. existing user code will still work, but code using the *new* functionality will not work with *older* versions of your package). -* Use `tag(pkg, :patch)` when you make bug fixes and other improvements that *don't change the API* (i.e. user code is unchanged). + +If your version is above 1.0 use the following scheme for bumping version numbers: + +1. **major tag** when you make *backwards-incompatible* API changes (i.e. changes that will break existing user code). +2. **minor tag** when you *add functionality* in a backwards-compatible way (i.e. existing user code will still work, but code using the *new* functionality will not work with *older* versions of your package). +3. **patch tag** when you make bug fixes and other improvements that *don't change the API* (i.e. user code is unchanged). + +If your version is below 1.0, use minor tags for case 1 and patch tags for both case 2 and 3. + The key question is not how "small" the change is, but how it affects the API and user code. (Don't be reluctant to bump the minor version when you add new features to the API, no matter how trivial — version numbers are cheap!) If you drop support for an older version of Julia, you should make at least a minor version bump even if there were no API changes. -### publish() -For each new package version tagged in `METADATA` not already published, make sure that the tagged package commits have been pushed to the repository at the registered URL for the package and if they all have, open a pull request to `METADATA`. +### `publish(registry)` -### generate(pkg, license) -Generate a new package named `pkg` with one of the bundled license: `"MIT"`, `"BSD"`, `CC0`, `"ISC"`, `"ASL"`, `"MPL"`, `"GPL-2.0+"`, `"GPL-3.0+"`, `"LGPL-2.1+"`, `"LGPL-3.0+"`. If you want to make a package with a different license, you can edit it afterwards. Generate creates a git repository at `Pkg.dir(pkg)` for the package and inside it `LICENSE.md`, `README.md`, `REQUIRE`, the julia entrypoint `$pkg/src/$pkg.jl`, and Travis and AppVeyor CI configuration files `.travis.yml` and `appveyor.yml`. +For each new package version tagged in the registry not already published, make sure that the tagged package commits have been pushed to the repository at the registered URL for the package and if they all have, open a pull request to the registry if it resides on GitHub. -> *Warning*: If you release code for Package X under the GPL, you may discourage collaboration from members of the Julia community who work on non-GPL packages. For example, if a user works on Package Y, which is licensed under the MIT license that is used in many community projects, other developers might not feel safe to read the code you contribute to Package Y because any indication that their work is derivative could lead to litigation. In effect, you create a situation in which your source code is percieved as being closed to anyone who is working on a non-GPL project. -Keyword parameters: +### `create_registry` -* `path` - a location where the package will be generated, the default location is `Pkg.dir()` -* `travis` - enables generation of the `.travis.yml` configuration for [Travis CI](https://travis-ci.org/) service, the default value is `true`. -* `appveyor` - enables generation of the `appveyor.yml` configuration for [Appveyor](http://www.appveyor.com/) service, the default value is `true`. -* `coverage` - enables generation of a code coverage reporting to [Coveralls](https://coveralls.io) and [Codecov](https://codecov.io) services, default value is `true`. -### license([lic]) -List all bundled licenses. If a license label specified as a parameter then a full text of the license will be printed. -### freeable([io]) -Returns a list of packages which are good candidates for -`Pkg.free`. These are packages for which you are not tracking the -tagged release, but for which a tagged release is equivalent to the -current version. You can use `Pkg.free(PkgDev.freeable())` to -automatically free all such packages. - -This also prints (to `io`, defaulting to standard output) a list of -packages that are ahead of a tagged release, and prints the number of -commits that separate them. It can help discover packages that may be -due for tagging. +### `license([lic])` + +List all bundled licenses. If a license label specified as a parameter then a full text of the license will be printed. diff --git a/REQUIRE b/REQUIRE index 8429fdb..9691681 100644 --- a/REQUIRE +++ b/REQUIRE @@ -1,5 +1,4 @@ -julia 0.6 +julia 0.7-beta JSON URIParser -Compat 0.62.0 Nullables diff --git a/appveyor.yml b/appveyor.yml index a085201..65a883e 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,13 +1,22 @@ environment: + JULIA_PROJECT: "@." matrix: - - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x86/0.6/julia-0.6-latest-win32.exe" - - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x64/0.6/julia-0.6-latest-win64.exe" + - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x86/0.7/julia-0.7-latest-win32.exe" + - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x64/0.7/julia-0.7-latest-win64.exe" - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x86/julia-latest-win32.exe" - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x64/julia-latest-win64.exe" +## uncomment the following lines to allow failures on nightly julia +## (tests will run but not make your overall status red) +#matrix: +# allow_failures: +# - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x86/julia-latest-win32.exe" +# - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x64/julia-latest-win64.exe" + branches: only: - master + - /release-.*/ notifications: - provider: Email @@ -30,10 +39,8 @@ install: - C:\projects\julia-binary.exe /S /D=C:\projects\julia build_script: -# Need to convert from shallow to complete for Pkg.clone to work - - IF EXIST .git\shallow (git fetch --unshallow) - - C:\projects\julia\bin\julia -e "versioninfo(); - Pkg.clone(pwd(), \"PkgDev\"); Pkg.build(\"PkgDev\")" + - C:\projects\julia\bin\julia -e "import InteractiveUtils; versioninfo(); + Pkg.build()" test_script: - - C:\projects\julia\bin\julia --check-bounds=yes -e "Pkg.test(\"PkgDev\")" + - C:\projects\julia\bin\julia -e "Pkg.test()" diff --git a/src/PkgDev.jl b/src/PkgDev.jl index e93121d..fac34f1 100644 --- a/src/PkgDev.jl +++ b/src/PkgDev.jl @@ -2,66 +2,51 @@ __precompile__() module PkgDev -using Compat, Compat.Pkg, Compat.LibGit2 +using LibGit2 +import Pkg -export Entry, Generate, GitHub +struct PkgDevError + msg::String +end + +Base.show(io::IO, err::PkgDevError) = print(io, err.msg) + +# remove extension .jl +splitjl(pkg::AbstractString) = endswith(pkg, ".jl") ? pkg[1:end-3] : pkg include("utils.jl") include("github.jl") -include("entry.jl") +include("Registry.jl") include("license.jl") include("generate.jl") -const cd = Pkg.Dir.cd +add_project(pkgdir::String) = Generate.create_project_from_require(pkgdir) -# remove extension .jl -const PKGEXT = ".jl" -splitjl(pkg::AbstractString) = endswith(pkg, PKGEXT) ? pkg[1:end-length(PKGEXT)] : pkg +defaut_reg = joinpath(Pkg.depots1(), "registries", "General") """ - dir(pkg, [paths...]) + register(pkgdir; [commit, registry, url]) -Return package `pkg` directory location through search. Additional `paths` are appended. +Register package at `pkgdir` at `commit` into the registry at `registry`. If +not provided, `commit` defaults to the current commit of the `pkg` repo, +`registry` defaults to the General registry and `url` defaults to the url of the +`origin` remote. """ -function dir(pkg::AbstractString) - pkg = splitjl(pkg) - if isdefined(Base, :find_in_path) - pkgsrc = Base.find_in_path(pkg, Pkg.dir()) - else - pkgsrc = Base.find_package(pkg) - end - pkgsrc === nothing && return "" - abspath(dirname(pkgsrc), "..") |> realpath -end -dir(pkg::AbstractString, args...) = normpath(dir(pkg),args...) +register(pkgdir::AbstractString; commit=nothing, registry=default_reg(), url=nothing ) = + Registry.register(registry, pkgdir; commit=commit, url=url) """ - register(pkg, [url]) + tag(pkgdir; [commit, registry]) -Register `pkg` at the git URL `url`, defaulting to the configured origin URL of the git -repo `Pkg.dir(pkg)`. +Tag `commit` of package at `pkgdir` into the registry at `registry`. If +not provided, `commit` defaults to the current commit of the `pkg` repo +and `registry` defaults to the General registry. """ -register(pkg::AbstractString) = Entry.register(splitjl(pkg)) -register(pkg::AbstractString, url::AbstractString) = Entry.register(splitjl(pkg),url) +tag(pkgdir::AbstractString; commit = nothing, registry=default_reg()) = + Registry.tag(registry, pkgdir; commit=commit) -""" - tag(pkg, [ver, [commit]]) - -Tag `commit` as version `ver` of package `pkg` and create a version entry in `METADATA`. If -not provided, `commit` defaults to the current commit of the `pkg` repo. If `ver` is one of -the symbols `:patch`, `:minor`, `:major` the next patch, minor or major version is used. If -`ver` is not provided, it defaults to `:patch`. -""" -tag(pkg::AbstractString, sym::Symbol=:patch) = cd(Entry.tag,splitjl(pkg),sym) -tag(pkg::AbstractString, sym::Symbol, commit::AbstractString) = cd(Entry.tag,splitjl(pkg),sym,false,commit) - -tag(pkg::AbstractString, ver::VersionNumber; force::Bool=false) = cd(Entry.tag,splitjl(pkg),ver,force) -tag(pkg::AbstractString, ver::VersionNumber, commit::AbstractString; force::Bool=false) = - cd(Entry.tag,splitjl(pkg),ver,force,commit) - -submit(pkg::AbstractString) = cd(Entry.submit, splitjl(pkg)) -submit(pkg::AbstractString, commit::AbstractString) = cd(Entry.submit,splitjl(pkg),commit) +#= """ publish() @@ -76,25 +61,26 @@ Optionally, function accepts a name for a pull request branch. If it isn't prov will be automatically generated. """ publish(prbranch::AbstractString="") = Entry.publish(Pkg.Dir.getmetabranch(), prbranch) +=# @doc raw""" - generate(pkg,license) + generate(pkgdir, license) Generate a new package named `pkg` with one of these license keys: `"MIT"`, `"BSD"`, `"ASL"`, `"MPL"`, `"GPL-2.0+"`, `"GPL-3.0+"`, `"LGPL-2.1+"`, `"LGPL-3.0+"`. If you want to make a package with a different license, you can edit it afterwards. -Generate creates a git repo at `Pkg.dir(pkg)` for the package and inside it `LICENSE.md`, -`README.md`, `REQUIRE`, and the julia entrypoint `$pkg/src/$pkg.jl`. Travis, AppVeyor CI +Generate creates a git repo at `pkgdir` for the package and inside it `LICENSE.md`, +`README.md`, `Project.toml`, and the julia entrypoint `$pkg/src/$pkg.jl`. Travis, AppVeyor CI configuration files `.travis.yml` and `appveyor.yml` with code coverage statistics using -Coveralls or Codecov are created by default, but each can be disabled individually by +Codecov is created by default, but each can be disabled individually by setting `travis`, `appveyor` or `coverage` to `false`. """ generate(pkg::AbstractString, license::AbstractString; force::Bool=false, authors::Union{AbstractString,Array} = [], - config::Dict=Dict(), path::AbstractString = Pkg.Dir.path(), - travis::Bool = true, appveyor::Bool = true, coverage::Bool = true) = - Generate.package(splitjl(pkg), license, force=force, authors=authors, config=config, path=path, + config::Dict=Dict(), travis::Bool = true, appveyor::Bool = true, + coverage::Bool = true) = + Generate.package(splitjl(pkg), license, force=force, authors=authors, config=config, travis=travis, appveyor=appveyor, coverage=coverage) """ @@ -113,7 +99,7 @@ function config(force::Bool=false) username = LibGit2.get(cfg, "user.name", "") if isempty(username) || force - username = LibGit2.prompt("Enter user name", default=username) + username = Base.prompt("Enter user name", default=username) LibGit2.set!(cfg, "user.name", username) else println("User name: $username") @@ -121,7 +107,7 @@ function config(force::Bool=false) useremail = LibGit2.get(cfg, "user.email", "") if isempty(useremail) || force - useremail = LibGit2.prompt("Enter user email", default=useremail) + useremail = Base.prompt("Enter user email", default=useremail) LibGit2.set!(cfg, "user.email", useremail) else println("User email: $useremail") @@ -130,7 +116,7 @@ function config(force::Bool=false) # setup github account ghuser = LibGit2.get(cfg, "github.user", "") if isempty(ghuser) || force - ghuser = LibGit2.prompt("Enter GitHub user", default=(isempty(ghuser) ? username : ghuser)) + ghuser = Base.prompt("Enter GitHub user", default=(isempty(ghuser) ? username : ghuser)) LibGit2.set!(cfg, "github.user", ghuser) else println("GitHub user: $ghuser") @@ -138,32 +124,18 @@ function config(force::Bool=false) finally finalize(cfg) end - lowercase(LibGit2.prompt("Do you want to change this configuration?", default="N")) == "y" && config(true) + lowercase(Base.prompt("Do you want to change this configuration?", default="N")) == "y" && config(true) return end -""" - freeable([io::IO=STDOUT]) - -Return a list of packages which are good candidates for `Pkg.free`. These are packages for -which you are not tracking the tagged release, but for which a tagged release is equivalent -to the current version. You can use `Pkg.free(PkgDev.freeable())` to automatically free all -such packages. - -This also prints (to `io`, defaulting to standard output) a list of packages that are ahead -of a tagged release, and prints the number of commits that separate them. It can help -discover packages that may be due for tagging. -""" -freeable(args...) = cd(Entry.freeable, args...) - function __init__() # Check if git configuration exists cfg = LibGit2.GitConfig(LibGit2.Consts.CONFIG_LEVEL_GLOBAL) try username = LibGit2.get(cfg, "user.name", "") if isempty(username) - Compat.@warn("PkgDev.jl is not configured. Please, run `PkgDev.config()` " * - "before performing any operations.") + @warn("PkgDev.jl is not configured. Please, run `PkgDev.config()` " * + "before performing any operations.") end finally finalize(cfg) diff --git a/src/Registry.jl b/src/Registry.jl new file mode 100644 index 0000000..31b917d --- /dev/null +++ b/src/Registry.jl @@ -0,0 +1,585 @@ +module Registry + +import Base: thispatch, nextpatch, nextminor, nextmajor, check_new_version +using Pkg, LibGit2 +import ..PkgDev +import ..PkgDev.GitHub +using ..PkgDev: getrepohttpurl, PkgDevError + +import UUIDs + +import Pkg.TOML, Pkg.Operations, Pkg.API +using Pkg.Types + +function write_toml(f::Function, names::String...) + path = joinpath(names...) * ".toml" + mkpath(dirname(path)) + open(path, "w") do io + f(io) + end +end + +function create_registry(path; repo::Union{Nothing, String} = nothing, uuid = UUIDs.uuid1(), description = nothing) + isdir(path) && error("$(abspath(path)) already exists") + mkpath(path) + write_mainfile(path, uuid, repo, description) + LibGit2.with(LibGit2.init(path)) do repo + LibGit2.add!(repo, "*") + LibGit2.commit(repo, "initial commit for registry $(basename(path))") + end + return +end + +function write_mainfile(path, uuid, repo, description) + d = Dict{String, Any}() + d["name"] = basename(path) + d["uuid"] = string(uuid) + if repo !== nothing + d["repo"]= repo + end + if description !== nothing + d["description"] = description + end + write_mainfile(path, d) +end + +is_stdlib(ctx::Context, uuid::UUID) = haskey(ctx.stdlibs, uuid) + +function write_mainfile(path::String, data::Dict) + open(joinpath(path, "Registry.toml"), "w") do io + println(io, "name = ", repr(data["name"])) + println(io, "uuid = ", repr(data["uuid"])) + if haskey(data, "repo") + println(io, "repo = ", repr(data["repo"])) + end + println(io) + + if haskey(data, "description") + print(io, """ + description = \"\"\" + $(data["description"])\"\"\" + """ + ) + end + + println(io) + println(io, "[packages]") + if haskey(data, "packages") + for (uuid, data) in sort!(collect(data["packages"]), by=first) + println(io, uuid, " = { name = ", repr(data["name"]), ", path = ", repr(data["path"]), " }") + end + end + end +end + +struct PackageReg + uuid::UUID + name::String + url::Union{String, Nothing} + version::VersionNumber + git_tree_sha::SHA1 + deps::Dict{UUID, VersionSpec} +end + +const JULIA_UUID = UUID("1222c4b2-2114-5bfd-aeef-88e4692bbb3e") + +function collect_package_info(pkgpath::String; url, commit) + pkgpath = abspath(pkgpath) + if !isdir(pkgpath) + pkgerror("directory $(repr(pkgpath)) not found") + end + if !isdir(joinpath(pkgpath, ".git")) + pkgerror("can only register git repositories as packages") + end + project_file = projectfile_path(pkgpath) + if project_file === nothing + pkgerror("package needs a \"[Julia]Project.toml\" file") + end + local git_tree_sha + LibGit2.with(LibGit2.GitRepo(pkgpath)) do repo + if commit == nothing + if LibGit2.isdirty(repo) + pkgerror("git repo at $(repr(pkgpath)) is dirty") + end + commit = LibGit2.head(repo) + end + git_tree_sha = begin + LibGit2.with(LibGit2.peel(LibGit2.GitTree, commit)) do tree + SHA1(string(LibGit2.GitHash(tree))) + end + end + end + if url === nothing + try + url = LibGit2.getconfig(pkgpath, "remote.origin.url", "") + catch err + pkgerror("$pkg: $err") + end + end + + f = project_file + entry = joinpath(dirname(f), "src", "ads.jl") + + project = read_package(project_file) + if !haskey(project, "version") + pkgerror("project file did not contain a version entry") + end + vers = VersionNumber(project["version"]) + vers = VersionNumber(vers.major, vers.minor, vers.patch) + name = project["name"] + uuid = UUID(project["uuid"]) + + name_uuid = Dict{String, UUID}() + deps = Dict{UUID, VersionSpec}() + deps[JULIA_UUID] = VersionSpec() # every package depends on julia + for (pkg, dep_uuid) in project["deps"] + name_uuid[pkg] = UUID(dep_uuid) + deps[UUID(dep_uuid)] = VersionSpec() + end + + for (pkg, verspec) in get(project, "compat", []) + if pkg == "julia" + dep_uuid = JULIA_UUID + else + if !haskey(name_uuid, pkg) + pkgerror("package $pkg in compat section does not exist in deps section") + end + dep_uuid = name_uuid[pkg] + end + deps[dep_uuid] = Types.semver_spec(verspec) + end + + return PackageReg( + uuid, + name, + url, + vers, + git_tree_sha, + deps + ) +end + +register(registry::String, pkgpath; commit, url) = + register(registry, collect_package_info(pkgpath; url=url, commit=commit); registering_new=true) +tag(registry::String, pkgpath; commit) = + register(registry, collect_package_info(pkgpath; url=nothing, commit=commit); registering_new=false) + +function register(registry::String, pkg::PackageReg; registering_new::Bool) + !isdir(registry) && error(abspath(registry), " does not exist") + registry_main_file = joinpath(registry, "Registry.toml") + !isfile(registry_main_file) && error(abspath(registry_main_file), " does not exist") + registry_data = TOML.parsefile(joinpath(registry, "Registry.toml")) + + registry_packages = get(registry_data, "packages", Dict{String, Any}()) + + bin = string(first(pkg.name)) + + if haskey(registry_packages, string(pkg.uuid)) + registering_new == false || pkgerror("package $(pkg.name) with uuid $(pkg.uuid) already registered") + reldir = registry_packages[string(pkg.uuid)]["path"] + else + registering_new == true || pkgerror("package $(pkg.name) with uuid $(pkg.uuid) not registered") + pkg.url === nothing && pkgerror("no URL configured for package") + binpath = joinpath(registry, bin) + mkpath(binpath) + # store the package in $name__$i where i is the no. of pkgs with the same name + # unless i == 0, then store in $name + candidates = filter(x -> startswith(x, pkg.name), readdir(binpath)) + r = Regex("$(pkg.name)(__)?[0-9]*?\$") + offset = count(x -> occursin(r, x), candidates) + if offset == 0 + reldir = joinpath(string(first(pkg.name)), pkg.name) + else + reldir = joinpath(string(first(pkg.name)), "$(pkg.name)__$(offset+1)") + end + end + + registry_packages[string(pkg.uuid)] = Dict("name" => pkg.name, "path" => reldir) + pkg_registry_path = joinpath(registry, reldir) + + LibGit2.transact(LibGit2.GitRepo(registry)) do repo + mkpath(pkg_registry_path) + for f in ("Versions.toml", "Deps.toml", "Compat.toml") + isfile(joinpath(pkg_registry_path, f)) || touch(joinpath(pkg_registry_path, f)) + end + + version_data = Operations.load_versions(pkg_registry_path) + if haskey(version_data, pkg.version) + pkgerror("version $(pkg.version) already registered") + end + version_data[pkg.version] = pkg.git_tree_sha + + ctx = Context() + for (uuid, v) in pkg.deps + if !(is_stdlib(ctx, uuid) || string(uuid) in keys(registry_packages) || uuid == JULIA_UUID) + pkgerror("dependency with uuid $(uuid) not an stdlib nor registered package") + end + end + + deps_data = Operations.load_package_data_raw(UUID, joinpath(pkg_registry_path, "Deps.toml")) + compat_data = Operations.load_package_data_raw(VersionSpec, joinpath(pkg_registry_path, "Compat.toml")) + + new_deps = Dict{String, Any}() + new_compat = Dict{String, Any}() + for (uuid, v) in pkg.deps + if uuid == JULIA_UUID + new_compat["julia"] = v + # We don't put julia in deps + elseif is_stdlib(ctx, uuid) + name = ctx.stdlibs[uuid] + new_deps[name] = uuid + else + name = registry_packages[string(uuid)]["name"] + new_compat[name] = v + new_deps[name] = uuid + end + end + deps_data[VersionRange(pkg.version)] = new_deps + compat_data[VersionRange(pkg.version)] = new_compat + + # TODO: compression + + # Package.toml + write_toml(joinpath(pkg_registry_path, "Package")) do io + println(io, "name = ", repr(pkg.name)) + println(io, "uuid = ", repr(string(pkg.uuid))) + println(io, "repo = ", repr(pkg.url)) + end + + # Versions.toml + versionfile = joinpath(pkg_registry_path, "Versions.toml") + isfile(versionfile) || touch(versionfile) + write_toml(joinpath(pkg_registry_path, "Versions")) do io + for (i, (ver, v)) in enumerate(sort!(collect(version_data), by=first)) + i > 1 && println(io) + println(io, "[", toml_key(string(ver)), "]") + println(io, "git-tree-sha1 = ", repr(string(pkg.git_tree_sha))) + end + end + + function write_version_data(f::String, d::Dict) + write_toml(f) do io + for (i, (ver, v)) in enumerate(sort!(collect(d), lt = Pkg.Types.isless_ll, + by=x->first(x).lower)) + i > 1 && println(io) + println(io, "[", toml_key(string(ver)), "]") + for (key, val) in collect(v) # TODO sort? + println(io, key, " = \"$val\"") + end + end + end + end + + # Compat.toml + write_version_data(joinpath(pkg_registry_path, "Compat"), compat_data) + + # Deps.toml + write_version_data(joinpath(pkg_registry_path, "Deps"), deps_data) + + # Registry.toml + if registering_new + write_mainfile(joinpath(registry), registry_data) + LibGit2.add!(repo, "Registry.toml") + end + LibGit2.add!(repo, reldir) + # Commit it + prefix = registering_new ? "Register" : "Tag v$(pkg.version)" + LibGit2.commit(repo, "$prefix $(pkg.name) [$(pkg.url)]") + end + return +end + + +toml_key(str::String) = occursin(r"[^\w-]", str) ? repr(str) : str +toml_key(strs::String...) = join(map(toml_key, [strs...]), '.') + + +#= +function pull_request(dir::AbstractString; commit::AbstractString="", url::AbstractString="", branch::AbstractString="") + with(LibGit2.GitRepo, dir) do repo + if isempty(commit) + commit = string(LibGit2.head_oid(repo)) + else + !LibGit2.iscommit(commit, repo) && throw(PkgDevError("Cannot find pull commit: $commit")) + end + if isempty(url) + url = LibGit2.getconfig(repo, "remote.origin.url", "") + end + force_branch = !isempty(branch) + if !force_branch + branch = "pull-request/$(commit[1:8])" + end + + m = match(LibGit2.GITHUB_REGEX, url) + m === nothing && throw(PkgDevError("not a GitHub repo URL, can't make a pull request: $url")) + owner, owner_repo = m.captures[2:3] + user = GitHub.user() + @info("Forking $owner/$owner_repo to $user") + response = GitHub.fork(owner, owner_repo) + fork = response["clone_url"] + @info("Pushing changes as branch $branch") + refspecs = ["HEAD:refs/heads/$branch"] # workaround for $commit:refs/heads/$branch + fork, payload = push_url_and_credentials(fork) + LibGit2.push(repo, remoteurl=fork, refspecs=refspecs, force=force_branch, payload=payload) + pr_url = "$(response["html_url"])/compare/$branch" + @info("To create a pull-request, open:\n\n $pr_url\n") + end +end + +function submit(pkg::AbstractString, registry::AbstractString, commit::AbstractString="") + urlpath = Pkg.dir("METADATA",pkg,"url") + url = ispath(urlpath) ? readchomp(urlpath) : "" + pull_request(PkgDev.dir(pkg), commit=commit, url=url) +end + +function push_url_and_credentials(url) + payload = Nullable{LibGit2.AbstractCredentials}() + m = match(LibGit2.GITHUB_REGEX,url) + if m !== nothing + url = "https://github.com/$(m.captures[1]).git" + payload = GitHub.credentials() + end + url, payload +end + +function publish(branch::AbstractString, prbranch::AbstractString="") + tags = Dict{String,Vector{String}}() + metapath = Pkg.dir("METADATA") + with(LibGit2.GitRepo, metapath) do repo + LibGit2.branch(repo) == branch || + throw(PkgDevError("METADATA must be on $branch to publish changes")) + LibGit2.fetch(repo) + + ahead_remote, ahead_local = LibGit2.revcount(repo, "origin/$branch", branch) + rcount = min(ahead_remote, ahead_local) + ahead_remote - rcount > 0 && throw(PkgDevError("METADATA is behind origin/$branch – run `Pkg.update()` before publishing")) + ahead_local - rcount == 0 && throw(PkgDevError("There are no METADATA changes to publish")) + + # get changed files + for path in LibGit2.diff_files(repo, "origin/$branch", LibGit2.Consts.HEAD_FILE) + m = match(r"^(.+?)/versions/([^/]+)/sha1$", path) + m !== nothing && occursin(Base.VERSION_REGEX, m.captures[2]) || continue + pkg, ver = m.captures; ver = VersionNumber(ver) + sha1 = readchomp(joinpath(metapath,path)) + try + old = LibGit2.content(LibGit2.GitBlob(repo, "origin/$branch:$path")) + if old != sha1 + throw(PkgDevError("$pkg v$ver SHA1 changed in METADATA – refusing to publish")) + end + catch e + if !(e isa LibGit2.GitError && e.code == LibGit2.Error.ENOTFOUND) + rethrow(e) + end + end + + with(LibGit2.GitRepo, PkgDev.dir(pkg)) do pkg_repo + tag_name = "v$ver" + tag_commit = LibGit2.revparseid(pkg_repo, "$(tag_name)^{commit}") + LibGit2.iszero(tag_commit) || string(tag_commit) == sha1 || return false + haskey(tags,pkg) || (tags[pkg] = String[]) + push!(tags[pkg], tag_name) + return true + end || throw(PkgDevError("$pkg v$ver is incorrectly tagged – $sha1 expected")) + end + isempty(tags) && @info("No new package versions to publish") + @info("Validating METADATA") + check_metadata(Set(keys(tags))) + end + + for pkg in sort!(collect(keys(tags))) + with(LibGit2.GitRepo, PkgDev.dir(pkg)) do pkg_repo + forced = String[] + unforced = String[] + for tag in tags[pkg] + ver = VersionNumber(tag) + push!(isrewritable(ver) ? forced : unforced, tag) + end + remoteurl, payload = push_url_and_credentials( + LibGit2.url(LibGit2.get(LibGit2.GitRemote, pkg_repo, "origin"))) + if !isempty(forced) + @info("Pushing $pkg temporary tags: ", join(forced,", ")) + LibGit2.push(pkg_repo, remote="origin", remoteurl=remoteurl, force=true, + refspecs=["refs/tags/$tag:refs/tags/$tag" for tag in forced], + payload=payload) + end + if !isempty(unforced) + @info("Pushing $pkg permanent tags: ", join(unforced,", ")) + LibGit2.push(pkg_repo, remote="origin", remoteurl=remoteurl, + refspecs=["refs/tags/$tag:refs/tags/$tag" for tag in unforced], + payload=payload) + end + end + end + @info("Submitting METADATA changes") + pull_request(metapath, branch=prbranch) +end + +function write_tag_metadata(repo::LibGit2.GitRepo, pkg::AbstractString, ver::VersionNumber, commit::AbstractString, force::Bool=false) + pkgdir = PkgDev.dir(pkg) + content = with(LibGit2.GitRepo, pkgdir) do pkg_repo + LibGit2.content(LibGit2.GitBlob(pkg_repo, "$commit:REQUIRE")) + end + reqs = content !== nothing ? Pkg.Reqs.read(split(content, '\n', keep=false)) : Pkg.Reqs.Line[] + cd(Pkg.dir("METADATA")) do + # work around julia#18724 and PkgDev#28 + d = join([pkg, "versions", string(ver)], '/') + mkpath(d) + sha1file = join([d, "sha1"], '/') + if !force && ispath(sha1file) + current = readchomp(sha1file) + current == commit || + throw(PkgDevError("$pkg v$ver is already registered as $current, bailing")) + end + open(io->println(io,commit), sha1file, "w") + LibGit2.add!(repo, sha1file) + reqsfile = join([d, "requires"], '/') + if isempty(reqs) + ispath(reqsfile) && LibGit2.remove!(repo, reqsfile) + else + Pkg.Reqs.write(reqsfile,reqs) + LibGit2.add!(repo, reqsfile) + end + end + return nothing +end + +function register(pkgdir::AbstractString, registrypath::AbstractString, url::AbstractString="") + isempty(pkgdir) && throw(PkgDevError("$pkg does not exist")) + metapath = Pkg.dir("METADATA") + ispath(pkgdir, ".git") || throw(PkgDevError("$pkg is not a git repo")) + isfile(metapath, pkg, "url") && throw(PkgDevError("$pkg already registered")) + LibGit2.transact(LibGit2.GitRepo(metapath)) do repo + # Get versions from package repo + versions = with(LibGit2.GitRepo, pkgdir) do pkg_repo + tags = filter(t->startswith(t,"v"), LibGit2.tag_list(pkg_repo)) + filter!(tag->occursin(Base.VERSION_REGEX, tag), tags) + Dict( + VersionNumber(tag) => string(LibGit2.revparseid(pkg_repo, "$tag^{commit}")) + for tag in tags + ) + end + # Register package url in METADATA + cd(metapath) do + @info("Registering $pkg at $url") + mkdir(pkg) + # work around julia#18724 and PkgDev#28 + path = join([pkg, "url"], '/') + open(io->println(io,url), path, "w") + LibGit2.add!(repo, path) + end + # Register package version in METADATA + vers = sort!(collect(keys(versions))) + for ver in vers + @info("Tagging $pkg v$ver") + write_tag_metadata(repo, pkg,ver,versions[ver]) + end + # Commit changes in METADATA + if LibGit2.isdirty(repo) + @info("Committing METADATA for $pkg") + msg = "Register $pkg [$url]" + if !isempty(versions) + msg *= ": $(join(map(v->"v$v", vers),", "))" + end + LibGit2.commit(repo, msg) + else + @info("No METADATA changes to commit") + end + end + return +end + +function register(pkg::AbstractString) + pkgdir = PkgDev.dir(pkg) + isempty(pkgdir) && throw(PkgDevError("$pkg does not exist")) + url = "" + try + url = LibGit2.getconfig(pkgdir, "remote.origin.url", "") + catch err + throw(PkgDevError("$pkg: $err")) + end + !isempty(url) || throw(PkgDevError("$pkg: no URL configured")) + register(pkg, Pkg.Cache.normalize_url(url)) +end + +function isrewritable(v::VersionNumber) + thispatch(v)==v"0" || + length(v.prerelease)==1 && isempty(v.prerelease[1]) || + length(v.build)==1 && isempty(v.build[1]) +end + +function tag(pkg::AbstractString; force::Bool=false, commitish::AbstractString="HEAD") + pkgdir = PkgDev.dir(pkg) + ispath(pkgdir,".git") || throw(PkgDevError("$pkg is not a git repo")) + metapath = Pkg.dir("METADATA") + with(LibGit2.GitRepo,metapath) do repo + LibGit2.isdirty(repo, pkg) && throw(PkgDevError("METADATA/$pkg is dirty – commit or stash changes to tag")) + end + with(LibGit2.GitRepo, pkgdir) do repo + LibGit2.isdirty(repo) && throw(PkgDevError("$pkg is dirty – commit or stash changes to tag")) + commit = string(LibGit2.revparseid(repo, commitish)) + + + urlfile = joinpath(metapath,pkg,"url") + registered = isfile(urlfile) + + if registered + avail = Pkg.cd(Pkg.Read.available, pkg) + existing = [VersionNumber(x) for x in keys(avail)] + ancestors = filter(v->LibGit2.is_ancestor_of(avail[v].sha1, commit, repo), existing) + else + tags = filter(t->startswith(t,"v"), LibGit2.tag_list(repo)) + filter!(tag->occursin(Base.VERSION_REGEX, tag), tags) + existing = [VersionNumber(x) for x in tags] + filter!(tags) do tag + sha1 = string(LibGit2.revparseid(repo, "$tag^{commit}")) + LibGit2.is_ancestor_of(sha1, commit, repo) + end + ancestors = [VersionNumber(x) for x in tags] + end + sort!(existing) + if !force + isrewritable(ver) && filter!(v->v!=ver,existing) + check_new_version(existing,ver) + end + # TODO: check that SHA1 isn't the same as another version + @info("Tagging $pkg v$ver") + LibGit2.tag_create(repo, "v$ver", commit, + msg=(!isrewritable(ver) ? "$pkg v$ver [$(commit[1:10])]" : ""), + force=(force || isrewritable(ver)) ) + registered || return + try + LibGit2.transact(LibGit2.GitRepo(metapath)) do repo + write_tag_metadata(repo, pkg, ver, commit, force) + if LibGit2.isdirty(repo) + @info("Committing METADATA for $pkg") + # Convert repo url into proper http url + repourl = getrepohttpurl(readchomp(urlfile)) + tagmsg = "Tag $pkg v$ver [$repourl]" + prev_ver_idx = isa(ver, Symbol) ? 0 : coalesce(findlast(v -> v < ver, existing), 0) + if prev_ver_idx != 0 + prev_ver = string(existing[prev_ver_idx]) + prev_sha = readchomp(joinpath(metapath,pkg,"versions",prev_ver,"sha1")) + if occursin(LibGit2.GITHUB_REGEX, repourl) + tagmsg *= "\n\nDiff vs v$prev_ver: $repourl/compare/$prev_sha...$commit" + end + end + LibGit2.commit(repo, tagmsg) + else + @info("No METADATA changes to commit") + end + end + catch + LibGit2.tag_delete(repo, "v$ver") + rethrow() + end + end + return +end + +# TODO +function check_registry end + +=# + + +end diff --git a/src/entry.jl b/src/entry.jl deleted file mode 100644 index 6548139..0000000 --- a/src/entry.jl +++ /dev/null @@ -1,363 +0,0 @@ -module Entry - -import Base: thispatch, nextpatch, nextminor, nextmajor, check_new_version -using Compat, Compat.Pkg, Compat.LibGit2 -using Nullables -import ..PkgDev -import ..PkgDev.GitHub -using ..PkgDev: getrepohttpurl - -function pull_request(dir::AbstractString; commit::AbstractString="", url::AbstractString="", branch::AbstractString="") - with(LibGit2.GitRepo, dir) do repo - if isempty(commit) - commit = string(LibGit2.head_oid(repo)) - else - !LibGit2.iscommit(commit, repo) && throw(Pkg.PkgError("Cannot find pull commit: $commit")) - end - if isempty(url) - url = LibGit2.getconfig(repo, "remote.origin.url", "") - end - force_branch = !isempty(branch) - if !force_branch - branch = "pull-request/$(commit[1:8])" - end - - m = match(LibGit2.GITHUB_REGEX, url) - m === nothing && throw(Pkg.PkgError("not a GitHub repo URL, can't make a pull request: $url")) - owner, owner_repo = m.captures[2:3] - user = GitHub.user() - Compat.@info("Forking $owner/$owner_repo to $user") - response = GitHub.fork(owner,owner_repo) - fork = response["clone_url"] - Compat.@info("Pushing changes as branch $branch") - refspecs = ["HEAD:refs/heads/$branch"] # workaround for $commit:refs/heads/$branch - fork, payload = push_url_and_credentials(fork) - LibGit2.push(repo, remoteurl=fork, refspecs=refspecs, force=force_branch, payload=payload) - pr_url = "$(response["html_url"])/compare/$branch" - Compat.@info("To create a pull-request, open:\n\n $pr_url\n") - end -end - -function submit(pkg::AbstractString, commit::AbstractString="") - urlpath = Pkg.dir("METADATA",pkg,"url") - url = ispath(urlpath) ? readchomp(urlpath) : "" - pull_request(PkgDev.dir(pkg), commit=commit, url=url) -end - -function push_url_and_credentials(url) - payload = Nullable{LibGit2.AbstractCredentials}() - m = match(LibGit2.GITHUB_REGEX,url) - if m !== nothing - url = "https://github.com/$(m.captures[1]).git" - payload = GitHub.credentials() - end - url, payload -end - -function publish(branch::AbstractString, prbranch::AbstractString="") - tags = Dict{String,Vector{String}}() - metapath = Pkg.dir("METADATA") - with(LibGit2.GitRepo, metapath) do repo - LibGit2.branch(repo) == branch || - throw(Pkg.PkgError("METADATA must be on $branch to publish changes")) - LibGit2.fetch(repo) - - ahead_remote, ahead_local = LibGit2.revcount(repo, "origin/$branch", branch) - rcount = min(ahead_remote, ahead_local) - ahead_remote-rcount > 0 && throw(Pkg.PkgError("METADATA is behind origin/$branch – run `Pkg.update()` before publishing")) - ahead_local-rcount == 0 && throw(Pkg.PkgError("There are no METADATA changes to publish")) - - # get changed files - for path in LibGit2.diff_files(repo, "origin/$branch", LibGit2.Consts.HEAD_FILE) - m = match(r"^(.+?)/versions/([^/]+)/sha1$", path) - m !== nothing && occursin(Base.VERSION_REGEX, m.captures[2]) || continue - pkg, ver = m.captures; ver = VersionNumber(ver) - sha1 = readchomp(joinpath(metapath,path)) - try - old = LibGit2.content(LibGit2.GitBlob(repo, "origin/$branch:$path")) - if old != sha1 - throw(Pkg.PkgError("$pkg v$ver SHA1 changed in METADATA – refusing to publish")) - end - catch e - if !(e isa LibGit2.GitError && e.code == LibGit2.Error.ENOTFOUND) - rethrow(e) - end - end - - with(LibGit2.GitRepo, PkgDev.dir(pkg)) do pkg_repo - tag_name = "v$ver" - tag_commit = LibGit2.revparseid(pkg_repo, "$(tag_name)^{commit}") - LibGit2.iszero(tag_commit) || string(tag_commit) == sha1 || return false - haskey(tags,pkg) || (tags[pkg] = String[]) - push!(tags[pkg], tag_name) - return true - end || throw(Pkg.PkgError("$pkg v$ver is incorrectly tagged – $sha1 expected")) - end - isempty(tags) && Compat.@info("No new package versions to publish") - Compat.@info("Validating METADATA") - check_metadata(Set(keys(tags))) - end - - for pkg in sort!(collect(keys(tags))) - with(LibGit2.GitRepo, PkgDev.dir(pkg)) do pkg_repo - forced = String[] - unforced = String[] - for tag in tags[pkg] - ver = VersionNumber(tag) - push!(isrewritable(ver) ? forced : unforced, tag) - end - remoteurl, payload = push_url_and_credentials( - LibGit2.url(LibGit2.get(LibGit2.GitRemote, pkg_repo, "origin"))) - if !isempty(forced) - Compat.@info("Pushing $pkg temporary tags: ", join(forced,", ")) - LibGit2.push(pkg_repo, remote="origin", remoteurl=remoteurl, force=true, - refspecs=["refs/tags/$tag:refs/tags/$tag" for tag in forced], - payload=payload) - end - if !isempty(unforced) - Compat.@info("Pushing $pkg permanent tags: ", join(unforced,", ")) - LibGit2.push(pkg_repo, remote="origin", remoteurl=remoteurl, - refspecs=["refs/tags/$tag:refs/tags/$tag" for tag in unforced], - payload=payload) - end - end - end - Compat.@info("Submitting METADATA changes") - pull_request(metapath, branch=prbranch) -end - -function write_tag_metadata(repo::LibGit2.GitRepo, pkg::AbstractString, ver::VersionNumber, commit::AbstractString, force::Bool=false) - pkgdir = PkgDev.dir(pkg) - content = with(LibGit2.GitRepo, pkgdir) do pkg_repo - LibGit2.content(LibGit2.GitBlob(pkg_repo, "$commit:REQUIRE")) - end - reqs = content !== nothing ? Pkg.Reqs.read(split(content, '\n', keep=false)) : Pkg.Reqs.Line[] - cd(Pkg.dir("METADATA")) do - # work around julia#18724 and PkgDev#28 - d = join([pkg, "versions", string(ver)], '/') - mkpath(d) - sha1file = join([d, "sha1"], '/') - if !force && ispath(sha1file) - current = readchomp(sha1file) - current == commit || - throw(Pkg.PkgError("$pkg v$ver is already registered as $current, bailing")) - end - open(io->println(io,commit), sha1file, "w") - LibGit2.add!(repo, sha1file) - reqsfile = join([d, "requires"], '/') - if isempty(reqs) - ispath(reqsfile) && LibGit2.remove!(repo, reqsfile) - else - Pkg.Reqs.write(reqsfile,reqs) - LibGit2.add!(repo, reqsfile) - end - end - return nothing -end - -function register(pkg::AbstractString, url::AbstractString) - pkgdir = PkgDev.dir(pkg) - isempty(pkgdir) && throw(Pkg.PkgError("$pkg does not exist")) - metapath = Pkg.dir("METADATA") - ispath(pkgdir,".git") || throw(Pkg.PkgError("$pkg is not a git repo")) - isfile(metapath,pkg,"url") && throw(Pkg.PkgError("$pkg already registered")) - LibGit2.transact(LibGit2.GitRepo(metapath)) do repo - # Get versions from package repo - versions = with(LibGit2.GitRepo, pkgdir) do pkg_repo - tags = filter(t->startswith(t,"v"), LibGit2.tag_list(pkg_repo)) - filter!(tag->occursin(Base.VERSION_REGEX, tag), tags) - Dict( - VersionNumber(tag) => string(LibGit2.revparseid(pkg_repo, "$tag^{commit}")) - for tag in tags - ) - end - # Register package url in METADATA - cd(metapath) do - Compat.@info("Registering $pkg at $url") - mkdir(pkg) - # work around julia#18724 and PkgDev#28 - path = join([pkg, "url"], '/') - open(io->println(io,url), path, "w") - LibGit2.add!(repo, path) - end - # Register package version in METADATA - vers = sort!(collect(keys(versions))) - for ver in vers - Compat.@info("Tagging $pkg v$ver") - write_tag_metadata(repo, pkg,ver,versions[ver]) - end - # Commit changes in METADATA - if LibGit2.isdirty(repo) - Compat.@info("Committing METADATA for $pkg") - msg = "Register $pkg [$url]" - if !isempty(versions) - msg *= ": $(join(map(v->"v$v", vers),", "))" - end - LibGit2.commit(repo, msg) - else - Compat.@info("No METADATA changes to commit") - end - end - return -end - -function register(pkg::AbstractString) - pkgdir = PkgDev.dir(pkg) - isempty(pkgdir) && throw(Pkg.PkgError("$pkg does not exist")) - url = "" - try - url = LibGit2.getconfig(pkgdir, "remote.origin.url", "") - catch err - throw(Pkg.PkgError("$pkg: $err")) - end - !isempty(url) || throw(Pkg.PkgError("$pkg: no URL configured")) - register(pkg, Pkg.Cache.normalize_url(url)) -end - -function isrewritable(v::VersionNumber) - thispatch(v)==v"0" || - length(v.prerelease)==1 && isempty(v.prerelease[1]) || - length(v.build)==1 && isempty(v.build[1]) -end - -nextbump(v::VersionNumber) = isrewritable(v) ? v : nextpatch(v) - -function tag(pkg::AbstractString, ver::Union{Symbol,VersionNumber}, force::Bool=false, commitish::AbstractString="HEAD") - pkgdir = PkgDev.dir(pkg) - ispath(pkgdir,".git") || throw(Pkg.PkgError("$pkg is not a git repo")) - metapath = Pkg.dir("METADATA") - with(LibGit2.GitRepo,metapath) do repo - LibGit2.isdirty(repo, pkg) && throw(Pkg.PkgError("METADATA/$pkg is dirty – commit or stash changes to tag")) - end - with(LibGit2.GitRepo, pkgdir) do repo - LibGit2.isdirty(repo) && throw(Pkg.PkgError("$pkg is dirty – commit or stash changes to tag")) - commit = string(LibGit2.revparseid(repo, commitish)) - urlfile = joinpath(metapath,pkg,"url") - registered = isfile(urlfile) - - if registered - avail = Pkg.cd(Pkg.Read.available, pkg) - existing = [VersionNumber(x) for x in keys(avail)] - ancestors = filter(v->LibGit2.is_ancestor_of(avail[v].sha1, commit, repo), existing) - else - tags = filter(t->startswith(t,"v"), LibGit2.tag_list(repo)) - filter!(tag->occursin(Base.VERSION_REGEX, tag), tags) - existing = [VersionNumber(x) for x in tags] - filter!(tags) do tag - sha1 = string(LibGit2.revparseid(repo, "$tag^{commit}")) - LibGit2.is_ancestor_of(sha1, commit, repo) - end - ancestors = [VersionNumber(x) for x in tags] - end - sort!(existing) - if !force - if isa(ver,Symbol) - prv = isempty(existing) ? v"0" : - isempty(ancestors) ? maximum(existing) : maximum(ancestors) - ver = (ver == :bump ) ? nextbump(prv) : - (ver == :patch) ? nextpatch(prv) : - (ver == :minor) ? nextminor(prv) : - (ver == :major) ? nextmajor(prv) : - throw(Pkg.PkgError("invalid version selector: $ver")) - end - isrewritable(ver) && filter!(v->v!=ver,existing) - check_new_version(existing,ver) - end - # TODO: check that SHA1 isn't the same as another version - Compat.@info("Tagging $pkg v$ver") - LibGit2.tag_create(repo, "v$ver", commit, - msg=(!isrewritable(ver) ? "$pkg v$ver [$(commit[1:10])]" : ""), - force=(force || isrewritable(ver)) ) - registered || return - try - LibGit2.transact(LibGit2.GitRepo(metapath)) do repo - write_tag_metadata(repo, pkg, ver, commit, force) - if LibGit2.isdirty(repo) - Compat.@info("Committing METADATA for $pkg") - # Convert repo url into proper http url - repourl = getrepohttpurl(readchomp(urlfile)) - tagmsg = "Tag $pkg v$ver [$repourl]" - prev_ver_idx = isa(ver, Symbol) ? 0 : coalesce(findlast(v -> v < ver, existing), 0) - if prev_ver_idx != 0 - prev_ver = string(existing[prev_ver_idx]) - prev_sha = readchomp(joinpath(metapath,pkg,"versions",prev_ver,"sha1")) - if occursin(LibGit2.GITHUB_REGEX, repourl) - tagmsg *= "\n\nDiff vs v$prev_ver: $repourl/compare/$prev_sha...$commit" - end - end - LibGit2.commit(repo, tagmsg) - else - Compat.@info("No METADATA changes to commit") - end - end - catch - LibGit2.tag_delete(repo, "v$ver") - rethrow() - end - end - return -end - -function check_metadata(pkgs::Set{String} = Set{String}()) - avail = Pkg.cd(Pkg.Read.available) - deps, conflicts = Pkg.Query.dependencies(avail) - - for (dp,dv) in deps, (v,a) in dv, p in keys(a.requires) - haskey(deps, p) || throw(Pkg.PkgError("package $dp v$v requires a non-registered package: $p")) - end - - problematic = Pkg.Resolve.sanity_check(deps, pkgs) - if !isempty(problematic) - msg = "packages with unsatisfiable requirements found:\n" - for (p, vn, rp) in problematic - msg *= " $p v$vn – no valid versions exist for package $rp\n" - end - throw(Pkg.PkgError(msg)) - end - return -end - -function freeable(io::IO = STDOUT) - function latest_tag(pkgname) - avail = Pkg.Read.available(pkgname) - k = sort(collect(keys(avail))) - isempty(k) ? Nullable{keytype(avail)}() : Nullable(avail[k[end]]) - end - freelist = Any[] - firstprint = true - cd(Pkg.dir()) do - for (pkg, status) in sort!(collect(Pkg.Read.installed()), by=first) - ver, fix = status - if fix - LibGit2.with(LibGit2.GitRepo(pkg)) do repo - LibGit2.isdirty(repo) && return - head = string(LibGit2.head_oid(repo)) - tag = latest_tag(pkg) - isnull(tag) && return - taggedsha = get(tag).sha1 - if head != taggedsha - local vrs - try - vrs = LibGit2.revcount(repo, taggedsha, head) - catch - Compat.@warn("skipping $pkg because the tagged commit $taggedsha " * - "was not found in the git revision history") - return - end - n = vrs[2] - vrs[1] - if firstprint - println(io, "Packages with a gap between HEAD and the most recent tag:") - firstprint = false - end - println(io, pkg, ": ", n) - else - push!(freelist, pkg) - end - end - end - end - end - freelist -end - -end diff --git a/src/generate.jl b/src/generate.jl index 2f5a576..7f850a1 100644 --- a/src/generate.jl +++ b/src/generate.jl @@ -1,11 +1,13 @@ module Generate -using Compat, Compat.Pkg, Compat.LibGit2, Compat.Dates -import ..PkgDev: readlicense, LICENSES +using Pkg, LibGit2, Dates, UUIDs +using Pkg.Types +import ..PkgDev: readlicense, LICENSES, PkgDevError copyright_year() = string(Dates.year(Dates.today())) -copyright_name(repo::GitRepo) = LibGit2.getconfig(repo, "user.name", "") +copyright_name(repo::GitRepo) = (name = LibGit2.getconfig(repo, "user.name", ""), email = LibGit2.getconfig(repo, "user.email", "")) github_user() = LibGit2.getconfig("github.user", "") +author_str(author::NamedTuple{(:name, :email),Tuple{String,String}}) = string(author.name, isempty(author.email) ? "" : " <$(author.email)>") function git_contributors(repo::GitRepo, n::Int=typemax(Int)) contrib = Dict() @@ -18,78 +20,77 @@ function git_contributors(repo::GitRepo, n::Int=typemax(Int)) end names = Dict() - for (commits,name) in values(contrib) - names[name] = get(names,name,0) + commits + for (email, (commits, name)) in contrib + names[(name=name, email=email)] = get(names, name, 0) + commits end - names = sort!(collect(keys(names)),by=name->names[name],rev=true) - length(names) <= n ? names : [names[1:n]; "et al."] + names = sort!(collect(keys(names)), by=name -> names[name], rev=true) + l = length(names) <= n ? names : [names[1:n]; (name = "et al.", email = "")] + return length(l) == 1 ? l[1] : l end -function package( - pkg::AbstractString, +function package(pkg_path::AbstractString, license::AbstractString; - force::Bool = false, - authors::Union{AbstractString,Array} = "", - years::Union{Int,AbstractString} = copyright_year(), - user::AbstractString = github_user(), - config::Dict = Dict(), - path::AbstractString = Pkg.Dir.path(), - travis::Bool = true, - appveyor::Bool = true, - coverage::Bool = true, + force::Bool=false, + authors::Union{AbstractString,Array}="", + years::Union{Int,AbstractString}=copyright_year(), + user::AbstractString=github_user(), + config::Dict=Dict(), + travis::Bool=true, + appveyor::Bool=true, + coverage::Bool=true, ) - pkg_path = joinpath(path,pkg) + pkg = basename(pkg_path) + isnew = !ispath(pkg_path) try repo = if isnew url = isempty(user) ? "" : "https://github.com/$user/$pkg.jl.git" - Generate.init(pkg_path,url,config=config) + Generate.init(pkg_path, url, config=config) else repo = GitRepo(pkg_path) if LibGit2.isdirty(repo) finalize(repo) - throw(Pkg.PkgError("$pkg is dirty – commit or stash your changes")) + throw(PkgDevError("$pkg is dirty – commit or stash your changes")) end repo end LibGit2.transact(repo) do repo if isempty(authors) - authors = isnew ? copyright_name(repo) : git_contributors(repo,5) - end - - files = [Generate.license(pkg_path,license,years,authors,force=force), - Generate.readme(pkg_path,user,force=force,coverage=coverage), - Generate.entrypoint(pkg_path,force=force), - Generate.tests(pkg_path,force=force), - Generate.require(pkg_path,force=force), - Generate.gitignore(pkg_path,force=force) ] - - travis && push!(files, Generate.travis(pkg_path,force=force,coverage=coverage)) - appveyor && push!(files, Generate.appveyor(pkg_path,force=force)) + authors = isnew ? copyright_name(repo) : git_contributors(repo, 5) + end + files = [Generate.license(pkg_path, license, years, authors, force=force), + Generate.readme(pkg_path, user, force=force, coverage=coverage), + Generate.entrypoint(pkg_path, force=force), + Generate.tests(pkg_path, force=force), + Generate.project(pkg_path, authors, force=force), + Generate.gitignore(pkg_path, force=force) ] + + travis && push!(files, Generate.travis(pkg_path, force=force, coverage=coverage)) + appveyor && push!(files, Generate.appveyor(pkg_path, force=force)) coverage && push!(files, Generate.codecov(pkg_path, force=force)) msg = """ $pkg.jl $(isnew ? "generated" : "regenerated") files. license: $license - authors: $(join(vcat(authors),", ")) + authors: $(join(vcat(authors), ", ")) years: $years user: $user Julia Version $VERSION [$(Base.GIT_VERSION_INFO.commit_short)] """ - LibGit2.add!(repo, files..., flags = LibGit2.Consts.INDEX_ADD_FORCE) + LibGit2.add!(repo, files..., flags=LibGit2.Consts.INDEX_ADD_FORCE) if isnew - Compat.@info("Committing $pkg generated files") - LibGit2.commit(repo, msg) - elseif LibGit2.isdirty(repo) - LibGit2.remove!(repo, files...) - Compat.@info("Regenerated files left unstaged, use `git add -p` to select") - open(io->print(io,msg), joinpath(LibGit2.gitdir(repo),"MERGE_MSG"), "w") - else - Compat.@info("Regenerated files are unchanged") - end + @info("Committing $pkg generated files") + LibGit2.commit(repo, msg) + elseif LibGit2.isdirty(repo) + LibGit2.remove!(repo, files...) + @info("Regenerated files left unstaged, use `git add -p` to select") + open(io -> print(io, msg), joinpath(LibGit2.gitdir(repo), "MERGE_MSG"), "w") + else + @info("Regenerated files are unchanged") + end end catch isnew && Base.rm(pkg_path, recursive=true) @@ -101,37 +102,38 @@ end function init(pkg::AbstractString, url::AbstractString=""; config::Dict=Dict()) if !ispath(pkg) pkg_name = basename(pkg) - Compat.@info("Initializing $pkg_name repo: $pkg") + @info("Initializing $pkg_name repo: $pkg") repo = LibGit2.init(pkg) try with(GitConfig, repo) do cfg - for (key,val) in config - LibGit2.set!(cfg, key, val) - end + for (key, val) in config + LibGit2.set!(cfg, key, val) + end end LibGit2.commit(repo, "initial empty commit") catch err - throw(Pkg.PkgError("Unable to initialize $pkg_name package: $err")) + throw(PkgDevError("Unable to initialize $pkg_name package: $err")) end else repo = GitRepo(pkg) end try if !isempty(url) - Compat.@info("Origin: $url") + @info("Origin: $url") with(LibGit2.GitRemote, repo, "origin", url) do rmt LibGit2.save(rmt) end LibGit2.set_remote_url(repo, url) end + catch end return repo end function genfile(f::Function, pkg::AbstractString, file::AbstractString, force::Bool=false) - path = joinpath(pkg,file) + path = joinpath(pkg, file) if force || !ispath(path) - Compat.@info("Generating $file") + @info("Generating $file") mkpath(dirname(path)) open(f, path, "w") return file @@ -142,54 +144,47 @@ end function license(pkg::AbstractString, license::AbstractString, years::Union{Int,AbstractString}, - authors::Union{AbstractString,Array}; + authors::Union{NamedTuple{(:name, :email),Tuple{String,String}},Array}; force::Bool=false) pkg_name = basename(pkg) - file = genfile(pkg,"LICENSE.md",force) do io + file = genfile(pkg, "LICENSE.md", force) do io if !haskey(LICENSES, license) - licenses = join(sort!(collect(keys(LICENSES)), by=lowercase), ", ") - throw(Pkg.PkgError("$license is not a known license choice, choose one of: $licenses.")) - end + licenses = join(sort!(collect(keys(LICENSES)), by=lowercase), ", ") + throw(PkgDevError("$license is not a known license choice, choose one of: $licenses.")) + end println(io, "The $pkg_name.jl package is licensed under the $(LICENSES[license]):") println(io) - println(io, copyright(years,authors)) - lic=readlicense(license) - for l in split(lic,['\n','\r']) - println(io, ">", length(l) > 0 ? " " : "", l) - end + println(io, copyright(years, authors)) + lic = readlicense(license) + for l in split(lic, ['\n','\r']) + println(io, ">", length(l) > 0 ? " " : "", l) end - !isempty(file) || Compat.@info("License file exists, leaving unmodified; use `force=true` to overwrite") + end + !isempty(file) || @info("License file exists, leaving unmodified; use `force=true` to overwrite") file end function readme(pkg::AbstractString, user::AbstractString=""; force::Bool=false, coverage::Bool=true) pkg_name = basename(pkg) - genfile(pkg,"README.md",force) do io + genfile(pkg, "README.md", force) do io println(io, "# $pkg_name") isempty(user) && return url = "https://travis-ci.org/$user/$pkg_name.jl" println(io, "\n[![Build Status]($url.svg?branch=master)]($url)") if coverage - coveralls_badge = "https://coveralls.io/repos/$user/$pkg_name.jl/badge.svg?branch=master&service=github" - coveralls_url = "https://coveralls.io/github/$user/$pkg_name.jl?branch=master" - println(io, "\n[![Coverage Status]($coveralls_badge)]($coveralls_url)") - codecov_badge = "http://codecov.io/github/$user/$pkg_name.jl/coverage.svg?branch=master" - codecov_url = "http://codecov.io/github/$user/$pkg_name.jl?branch=master" - println(io, "\n[![codecov.io]($codecov_badge)]($codecov_url)") - end + codecov_badge = "http://codecov.io/github/$user/$pkg_name.jl/coverage.svg?branch=master" + codecov_url = "http://codecov.io/github/$user/$pkg_name.jl?branch=master" + println(io, "\n[![codecov.io]($codecov_badge)]($codecov_url)") + end end end function tests(pkg::AbstractString; force::Bool=false) pkg_name = basename(pkg) - genfile(pkg,"test/runtests.jl",force) do io + genfile(pkg, "test/runtests.jl", force) do io print(io, """ using $pkg_name - @static if VERSION < v"0.7.0-DEV.2005" - using Base.Test - else - using Test - end + using Test # write your own tests here @test 1 == 2 @@ -198,22 +193,148 @@ function tests(pkg::AbstractString; force::Bool=false) end function versionfloor(ver::VersionNumber) - # return "major.minor" for the most recent release version relative to ver - # for prereleases with ver.minor == ver.patch == 0, return "major-" since we - # don't know what the most recent minor version is for the previous major - if isempty(ver.prerelease) || ver.patch > 0 - return string(ver.major, '.', ver.minor) - elseif ver.minor > 0 - return string(ver.major, '.', ver.minor - 1) + return string(ver.major, ".", ver.minor) +end + +function create_project_from_require(path::String, force::Bool=false) + ctx = Context() + + # Package data + path = abspath(path) + m = match(Pkg.Types.reg_pkg, path) + m === nothing && cmderror("cannot determine package name from path: $(path)") + pkgname = m.captures[1] + + mainpkg = PackageSpec(pkgname) + registry_resolve!(ctx.env, [mainpkg]) + if !has_uuid(mainpkg) + uuid = UUIDs.uuid1() + @info "Unregistered package, giving it a new UUID: $uuid" + mainpkg.version = v"0.1.0" else - return string(ver.major, '-') + uuid = mainpkg.uuid + @info "Registered package, using already given UUID: $(mainpkg.uuid)" + Pkg.Operations.set_maximum_version_registry!(ctx.env, mainpkg) + v = mainpkg.version + # Remove the build + mainpkg.version = VersionNumber(v.major, v.minor, v.patch) + end + + # Dependency data + dep_pkgs = PackageSpec[] + test_pkgs = PackageSpec[] + compatibility = Pair{String, String}[] + + reqfiles = [joinpath(path, "REQUIRE"), joinpath(path, "test", "REQUIRE")] + for (reqfile, pkgs) in zip(reqfiles, [dep_pkgs, test_pkgs]) + if isfile(reqfile) + for r in Pkg.Pkg2.Reqs.read(reqfile) + r isa Pkg.Pkg2.Reqs.Requirement || continue + r.package == "julia" && continue + push!(pkgs, PackageSpec(r.package)) + intervals = r.versions.intervals + if length(intervals) != 1 + @warn "Project.toml creator cannot handle multiple requirements for $(r.package), ignoring" + else + l = intervals[1].lower + h = intervals[1].upper + if l != v"0.0.0-" + # no upper bound + if h == typemax(VersionNumber) + push!(compatibility, r.package => string(">=", VersionNumber(l.major, l.minor, l.patch))) + else # assume semver + push!(compatibility, r.package => string(">=", VersionNumber(l.major, l.minor, l.patch), ", ", + "<", VersionNumber(h.major, h.minor, h.patch))) + end + end + end + end + registry_resolve!(ctx.env, pkgs) + ensure_resolved(ctx.env, pkgs) + end + end + + stdlib_deps = Pkg.Operations.find_stdlib_deps(ctx, path) + for (stdlib_uuid, stdlib) in stdlib_deps + pkg = PackageSpec(stdlib, stdlib_uuid) + if stdlib == "Test" + push!(test_pkgs, pkg) + else + push!(dep_pkgs, pkg) + end + end + + # Write project + + project = Dict( + "name" => pkgname, + "uuid" => string(uuid), + "version" => string(mainpkg.version), + "deps" => Dict(pkg.name => string(pkg.uuid) for pkg in dep_pkgs) + ) + + if !isempty(compatibility) + project["compat"] = + Dict(name => ver for (name, ver) in compatibility) + end + + if !isempty(test_pkgs) + project["extras"] = Dict(pkg.name => string(pkg.uuid) for pkg in test_pkgs) + project["targets"] = Dict("test" => [pkg.name for pkg in test_pkgs]) + end + + genfile(path, "Project.toml", force) do io + Pkg.TOML.print(io, project, sorted=true, by=key -> (Types.project_key_order(key), key)) end end -function require(pkg::AbstractString; force::Bool=false) - genfile(pkg,"REQUIRE",force) do io +function project(pkg::AbstractString, authors::Union{NamedTuple{(:name, :email),Tuple{String,String}}, Array}=""; force::Bool=false) + if isfile(joinpath(pkg, "REQUIRE")) + return create_project_from_require(pkg, force=force) + end + authors isa NamedTuple && (authors = [authors]) + authors_str = join([string("\"", author_str(author), "\"") for author in authors], ",") + + proj = nothing + for file in Base.project_names + if isfile(joinpath(pkg, file)) + proj = file + break + end + end + uuid = nothing + if proj !== nothing + projname = proj + m = match(r"uuid = \"(.*?)\"($|(\r\n|\r|\n))", read(joinpath(pkg, proj), String)) + if m !== nothing + uuid = m.captures[1] + end + else + projname = "Project.toml" + end + + if uuid === nothing + uuid = UUIDs.uuid1() + end + + pkgname = basename(pkg) + genfile(pkg, projname, force) do io print(io, """ - julia $(versionfloor(VERSION)) + authors = [$authors_str] + name = "$pkgname" + uuid = "$uuid" + version = "0.1.0" + + [deps] + + [compat] + julia = "$(versionfloor(VERSION))" + + [extras] + Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + + [targets] + test = ["Test"] """) end end @@ -228,7 +349,7 @@ function travis(pkg::AbstractString; force::Bool=false, coverage::Bool=true) else release = " - $vf" end - genfile(pkg,".travis.yml",force) do io + genfile(pkg, ".travis.yml", force) do io print(io, """ ## Documentation: http://docs.travis-ci.com/user/languages/julia/ language: julia @@ -240,14 +361,11 @@ function travis(pkg::AbstractString; force::Bool=false, coverage::Bool=true) - nightly notifications: email: false - git: - depth: 99999999 - ## uncomment the following lines to allow failures on nightly julia - ## (tests will run but not make your overall status red) - #matrix: - # allow_failures: - # - julia: nightly + # comment the following lines to disallow failures on nightly julia + matrix: + allow_failures: + - julia: nightly ## uncomment and modify the following lines to manually install system packages #addons: @@ -259,78 +377,54 @@ function travis(pkg::AbstractString; force::Bool=false, coverage::Bool=true) ## uncomment the following lines to override the default test script #script: - # - julia -e 'Pkg.clone(pwd()); Pkg.build("$pkg_name"); Pkg.test("$pkg_name"; coverage=true)' + # - julia -e 'import Pkg; Pkg.build(); Pkg.test(; coverage=true)' + $(c)after_success: - $(c) # push coverage results to Coveralls - $(c) - julia -e 'cd(Pkg.dir("$pkg_name")); Pkg.add("Coverage"); using Coverage; Coveralls.submit(Coveralls.process_folder())' $(c) # push coverage results to Codecov - $(c) - julia -e 'cd(Pkg.dir("$pkg_name")); Pkg.add("Coverage"); using Coverage; Codecov.submit(Codecov.process_folder())' + $(c) - julia -e 'import Pkg; Pkg.add("Coverage"); using Coverage; Codecov.submit(Codecov.process_folder())' """) end end function appveyor(pkg::AbstractString; force::Bool=false) - pkg_name = basename(pkg) - vf = versionfloor(VERSION) - if vf[end] == '-' # don't know what previous release was - vf = string(VERSION.major, '.', VERSION.minor) - rel32 = "# - JULIA_URL: \"https://julialang-s3.julialang.org/bin/winnt/x86/$vf/julia-$vf-latest-win32.exe\"" - rel64 = "# - JULIA_URL: \"https://julialang-s3.julialang.org/bin/winnt/x64/$vf/julia-$vf-latest-win64.exe\"" - else - rel32 = " - JULIA_URL: \"https://julialang-s3.julialang.org/bin/winnt/x86/$vf/julia-$vf-latest-win32.exe\"" - rel64 = " - JULIA_URL: \"https://julialang-s3.julialang.org/bin/winnt/x64/$vf/julia-$vf-latest-win64.exe\"" - end - genfile(pkg,"appveyor.yml",force) do io - print(io, """ - environment: - matrix: - $rel32 - $rel64 - - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x86/julia-latest-win32.exe" - - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x64/julia-latest-win64.exe" - - ## uncomment the following lines to allow failures on nightly julia - ## (tests will run but not make your overall status red) - #matrix: - # allow_failures: - # - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x86/julia-latest-win32.exe" - # - JULIA_URL: "https://julialangnightlies-s3.julialang.org/bin/winnt/x64/julia-latest-win64.exe" - - branches: - only: - - master - - /release-.*/ - - notifications: - - provider: Email - on_build_success: false - on_build_failure: false - on_build_status_changed: false - - install: - - ps: "[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]::Tls12" - # If there's a newer build queued for the same PR, cancel this one - - ps: if (\$env:APPVEYOR_PULL_REQUEST_NUMBER -and \$env:APPVEYOR_BUILD_NUMBER -ne ((Invoke-RestMethod ` - https://ci.appveyor.com/api/projects/\$env:APPVEYOR_ACCOUNT_NAME/\$env:APPVEYOR_PROJECT_SLUG/history?recordsNumber=50).builds | ` - Where-Object pullRequestId -eq \$env:APPVEYOR_PULL_REQUEST_NUMBER)[0].buildNumber) { ` - throw "There are newer queued builds for this pull request, failing early." } - # Download most recent Julia Windows binary - - ps: (new-object net.webclient).DownloadFile( - \$env:JULIA_URL, - "C:\\projects\\julia-binary.exe") - # Run installer silently, output to C:\\projects\\julia - - C:\\projects\\julia-binary.exe /S /D=C:\\projects\\julia - - build_script: - # Need to convert from shallow to complete for Pkg.clone to work - - IF EXIST .git\\shallow (git fetch --unshallow) - - C:\\projects\\julia\\bin\\julia -e "versioninfo(); - Pkg.clone(pwd(), \\"$pkg_name\\"); Pkg.build(\\"$pkg_name\\")" - - test_script: - - C:\\projects\\julia\\bin\\julia -e "Pkg.test(\\"$pkg_name\\")" - """) + """ + environment: + matrix: + - julia_version: 1.0 + - julia_version: latest + + platform: + - x86 + - x64 + + # comment the following lines to disallow failures on nightly julia + matrix: + allow_failures: + - julia_version: latest + + branches: + only: + - master + - /release-.*/ + + notifications: + - provider: Email + on_build_success: false + on_build_failure: false + on_build_status_changed: false + + install: + - ps: iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/JuliaCI/Appveyor.jl/master/bin/install.ps1')) + + build_script: + - echo "%JL_BUILD_SCRIPT%" + - julia -e "%JL_BUILD_SCRIPT%" + + test_script: + - echo "%JL_TEST_SCRIPT%" + - julia -e "%JL_TEST_SCRIPT%" end + """ end function codecov(pkg::AbstractString; force::Bool=false) @@ -342,34 +436,35 @@ function codecov(pkg::AbstractString; force::Bool=false) end function gitignore(pkg::AbstractString; force::Bool=false) - genfile(pkg,".gitignore",force) do io + genfile(pkg, ".gitignore", force) do io print(io, """ *.jl.cov *.jl.*.cov *.jl.mem + /Manifest.toml """) end end function entrypoint(pkg::AbstractString; force::Bool=false) pkg_name = basename(pkg) - genfile(pkg,"src/$pkg_name.jl",force) do io + genfile(pkg, "src/$pkg_name.jl", force) do io print(io, """ module $pkg_name - # package code goes here + greet() = print("Hello World!") end # module """) end end -copyright(years::AbstractString, authors::AbstractString) = "> Copyright (c) $years: $authors." +copyright(years::AbstractString, author::NamedTuple{(:name, :email),Tuple{String,String}}) = "> Copyright (c) $years: $(author_str(author))" function copyright(years::AbstractString, authors::Array) text = "> Copyright (c) $years:" for author in authors - text *= "\n> * $author" + text *= "\n> * $(author_str(author))" end return text end diff --git a/src/github.jl b/src/github.jl index 0106e4a..929c6d0 100644 --- a/src/github.jl +++ b/src/github.jl @@ -1,7 +1,8 @@ module GitHub import JSON -using Compat, Compat.Pkg +import Pkg +import ..PkgDev: PkgDevError const AUTH_NOTE = "Julia Package Manager" const AUTH_DATA = Dict{Any,Any}( @@ -13,7 +14,7 @@ const AUTH_DATA = Dict{Any,Any}( function user() usr = LibGit2.getconfig("github.user", "") if isempty(usr) #TODO: add `config` command to Git REPL and change below info - throw(Pkg.PkgError(""" + throw(PkgDevError(""" no GitHub user name configured; please configure with: PkgDev.config() @@ -24,7 +25,7 @@ function user() end function curl(url::AbstractString, opts::Cmd=``) - success(`curl --version`) || throw(Pkg.PkgError("using the GitHub API requires having `curl` installed")) + success(`curl --version`) || throw(PkgDevError("using the GitHub API requires having `curl` installed")) out, proc = open(`curl -i -s -S $opts $url`,"r") head = readline(out) status = parse(Int,split(head,r"\s+";limit=3)[2]) @@ -37,7 +38,7 @@ function curl(url::AbstractString, opts::Cmd=``) end wait(proc); return status, header, String(read(out)) end - throw(Pkg.PkgError("strangely formatted HTTP response")) + throw(PkgDevError("strangely formatted HTTP response")) end curl(url::AbstractString, data::Nothing, opts::Cmd=``) = curl(url,opts) curl(url::AbstractString, data, opts::Cmd=``) = @@ -46,7 +47,7 @@ curl(url::AbstractString, data, opts::Cmd=``) = function delete_token() tokfile = Pkg.Dir.path(".github","token") Base.rm(tokfile) - Compat.@info("Could not authenticate with existing token. Deleting token and trying again.") + @info("Could not authenticate with existing token. Deleting token and trying again.") end readtoken(tokfile=Pkg.Dir.path(".github","token")) = isfile(tokfile) ? strip(readchomp(tokfile)) : "" @@ -56,7 +57,7 @@ function token(user::AbstractString=user()) tok = readtoken(tokfile) !isempty(tok) && return tok - Compat.@info(""" + @info(""" Creating a personal access token for Julia Package Manager on GitHub. \tYou will be asked to provide credentials to your GitHub account.""") params = merge(AUTH_DATA, Dict("fingerprint" => randstring(40))) @@ -66,17 +67,17 @@ Creating a personal access token for Julia Package Manager on GitHub. # Check for two-factor authentication if status == 401 && get(header, "X-GitHub-OTP", "") |> x->startswith(x, "required") && isinteractive() tfa = true - Compat.@info("Two-factor authentication in use. Enter auth code. (You may have to re-enter your password.)") - print(STDERR, "Authentication code: ") - code = readline(STDIN) |> chomp + @info("Two-factor authentication in use. Enter auth code. (You may have to re-enter your password.)") + print(stderr, "Authentication code: ") + code = readline(stdin) |> chomp status, header, content = curl("https://api.github.com/authorizations",params,`-H "X-GitHub-OTP: $code" -u $user`) end if status == 422 error_code = JSON.parse(content)["errors"][1]["code"] - throw(Pkg.PkgError("GitHub returned validation error (422): $error_code: $(JSON.parse(content)["message"])")) + throw(PkgDevError("GitHub returned validation error (422): $error_code: $(JSON.parse(content)["message"])")) else - (status != 401 && status != 403) || throw(Pkg.PkgError("$status: $(JSON.parse(content)["message"])")) + (status != 401 && status != 403) || throw(PkgDevError("$status: $(JSON.parse(content)["message"])")) tok = JSON.parse(content)["token"] end @@ -110,11 +111,11 @@ end function pushable(owner::AbstractString, repo::AbstractString, user::AbstractString=user()) status, response = HEAD("repos/$owner/$repo") - status == 404 && throw(Pkg.PkgError("repo $owner/$repo does not exist")) + status == 404 && throw(PkgDevError("repo $owner/$repo does not exist")) status, response = GET("repos/$owner/$repo/collaborators/$user") status == 204 && return true status == 404 && return false - throw(Pkg.PkgError("unexpected API status code: $status – $(response["message"])")) + throw(PkgDevError("unexpected API status code: $status – $(response["message"])")) end function fork(owner::AbstractString, repo::AbstractString) @@ -123,7 +124,7 @@ function fork(owner::AbstractString, repo::AbstractString) delete_token() status, response = POST("repos/$owner/$repo/forks") end - status == 202 || throw(Pkg.PkgError("forking $owner/$repo failed: $(response["message"])")) + status == 202 || throw(PkgDevError("forking $owner/$repo failed: $(response["message"])")) return response end diff --git a/test/generate.jl b/test/generate.jl new file mode 100644 index 0000000..e7e5dd6 --- /dev/null +++ b/test/generate.jl @@ -0,0 +1,22 @@ +module TestGenerate + +using Test +using PkgDev +using Pkg + +include("utils.jl") + +temp_pkg_dir() do tmp; cd(tmp) do + @testset "generating a package with .jl extension" begin + pkg = "PackageWithExtension" + PkgDev.generate(pkg*".jl", "MIT", config=Dict("user.name"=>"Julia Test", "user.email"=>"test@julialang.org")) + testfile = joinpath(pkg, "test", "runtests.jl") + s = replace(read(testfile, String), "@test 1 == 2" => "@test 1 == 1") + write(testfile, s) + Pkg.develop(PackageSpec(path="PackageWithExtension")) + @test "PackageWithExtension" in keys(Pkg.installed()) + Pkg.test("PackageWithExtension") + end +end end + +end diff --git a/test/registry.jl b/test/registry.jl new file mode 100644 index 0000000..08fdd31 --- /dev/null +++ b/test/registry.jl @@ -0,0 +1,73 @@ +module RegistryTest + +using Test +using Pkg +using Pkg.Types +using PkgDev +import LibGit2 + +include("utils.jl") + +replace_in_file(f, pat) = write(f, replace(read(f, String), pat)) + +temp_pkg_dir() do project_path; cd(project_path) do + # Create a registry + registry_path = joinpath(DEPOT_PATH[1], "registries", "CustomReg"); + PkgDev.Registry.create_registry(registry_path, repo = registry_path, description = "This is a reg") + pkgs = mktempdir() + pkgdir = joinpath(pkgs, "TheFirst") + PkgDev.generate(pkgdir, "MIT") + cd(pkgdir) do + run(`git remote remove origin`) + run(`git remote add origin $pkgdir`) + end + PkgDev.register(joinpath(pkgs, "TheFirst"); registry=registry_path) + # Version already installed + @test_throws PkgError PkgDev.register(joinpath(pkgs, "TheFirst"); registry=registry_path) + Pkg.add("TheFirst") + @test Pkg.installed()["TheFirst"] == v"0.1.0" + # Add an stdlib dep and update it to v0.2.0 + Pkg.activate(pkgdir) + Pkg.add("Random") + replace_in_file(joinpath(pkgdir, "Project.toml"), "version = \"0.1.0\"" => "version = \"0.2.0\"") + LibGit2.with(LibGit2.GitRepo(pkgdir)) do repo + LibGit2.add!(repo, "*") + LibGit2.commit(repo, "tag v0.2.0") + end + Pkg.activate() + PkgDev.tag(joinpath(pkgs, "TheFirst"); registry=registry_path) + Pkg.update() + @test Pkg.installed()["TheFirst"] == v"0.2.0" + Pkg.rm("TheFirst") + pkg"add TheFirst@0.1.0" + @test Pkg.installed()["TheFirst"] == v"0.1.0" + Pkg.rm("TheFirst") + # Add a new package that depends on TheFirst + pkgdir = joinpath(pkgs, "TheSecond") + PkgDev.generate(pkgdir, "MIT") + cd(pkgdir) do + run(`git remote remove origin`) + run(`git remote add origin $pkgdir`) + end + Pkg.activate(pkgdir) + Pkg.add("UUIDs") + Pkg.add("TheFirst") + # Add a compat to TheFirst + proj = joinpath(pkgdir, "Project.toml") + p = Pkg.Types.parse_toml(proj) + p["compat"]["TheFirst"] = "0.1.0" + open(proj, "w") do io + Pkg.TOML.print(io, p) + end + LibGit2.with(LibGit2.GitRepo(joinpath(pkgs, "TheSecond"))) do repo + LibGit2.add!(repo, "*") + LibGit2.commit(repo, "tag v0.1.0") + end + Pkg.activate() + PkgDev.register(joinpath(pkgs, "TheSecond"); registry=registry_path, url=pkgdir) + Pkg.add("TheSecond") + @test Pkg.installed()["TheSecond"] == v"0.1.0" + @test Pkg.API.__installed(PKGMODE_MANIFEST)["TheFirst"] == v"0.1.0" +end end + +end # module diff --git a/test/runtests.jl b/test/runtests.jl index 4f62ff2..2d5bd86 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1,153 +1,12 @@ using PkgDev -using Compat, Compat.Test, Compat.Pkg, Compat.Random, Compat.LibGit2 +using Test, Pkg, Random, LibGit2 -function temp_pkg_dir(fn::Function, remove_tmp_dir::Bool=true) - # Used in tests below to set up and tear down a sandboxed package directory - tmpdir = joinpath(tempdir(),Random.randstring()) - withenv("JULIA_PKGDIR" => tmpdir) do - @test !isdir(Pkg.dir()) - try - Pkg.init() - @test isdir(Pkg.dir()) - Pkg.resolve() - - fn(Pkg.Dir.path()) - finally - remove_tmp_dir && Base.rm(tmpdir, recursive=true) - end - end -end - -temp_pkg_dir() do pkgdir - - @testset "testing a package with test dependencies causes them to be installed for the duration of the test" begin - PkgDev.generate("PackageWithTestDependencies", "MIT", config=Dict("user.name"=>"Julia Test", "user.email"=>"test@julialang.org")) - @test [keys(Pkg.installed())...] == ["PackageWithTestDependencies"] - @test read(Pkg.dir("PackageWithTestDependencies","REQUIRE"), String) == "julia $(PkgDev.Generate.versionfloor(VERSION))\n" - - isdir(Pkg.dir("PackageWithTestDependencies","test")) || mkdir(Pkg.dir("PackageWithTestDependencies","test")) - open(Pkg.dir("PackageWithTestDependencies","test","REQUIRE"),"w") do f - println(f,"Example") - end - - open(Pkg.dir("PackageWithTestDependencies","test","runtests.jl"),"w") do f - println(f, "using " * (VERSION < v"0.7.0-DEV.2005" ? "Base." : "") * "Test") - if VERSION >= v"0.7.0-DEV.3656" - println(f, "using Pkg") - end - println(f, "@test haskey(Pkg.installed(), \"Example\")") - end - - Pkg.resolve() - @test [keys(Pkg.installed())...] == ["PackageWithTestDependencies"] - - Pkg.test("PackageWithTestDependencies") - - @test [keys(Pkg.installed())...] == ["PackageWithTestDependencies"] - - # trying to pin an unregistered package errors - try - Pkg.pin("PackageWithTestDependencies", v"1.0.0") - error("unexpected") - catch err - @test isa(err, Pkg.PkgError) - @test err.msg == "PackageWithTestDependencies cannot be pinned – not a registered package" - end - end - - @testset "generating a package with .jl extension" begin - PkgDev.generate("PackageWithExtension.jl", "MIT", config=Dict("user.name"=>"Julia Test", "user.email"=>"test@julialang.org")) - @show keys(Pkg.installed()) - @test "PackageWithExtension" in keys(Pkg.installed()) - end - - @testset "testing a package with no runtests.jl errors" begin - PkgDev.generate("PackageWithNoTests", "MIT", config=Dict("user.name"=>"Julia Test", "user.email"=>"test@julialang.org")) - - if isfile(Pkg.dir("PackageWithNoTests", "test", "runtests.jl")) - Base.rm(Pkg.dir("PackageWithNoTests", "test", "runtests.jl")) - end - - try - Pkg.test("PackageWithNoTests") - error("unexpected") - catch err - @test err.msg == "PackageWithNoTests did not provide a test/runtests.jl file" - end - end - - @testset "testing a package with failing tests errors" begin - PkgDev.generate("PackageWithFailingTests", "MIT", config=Dict("user.name"=>"Julia Test", "user.email"=>"test@julialang.org")) - - isdir(Pkg.dir("PackageWithFailingTests","test")) || mkdir(Pkg.dir("PackageWithFailingTests","test")) - open(Pkg.dir("PackageWithFailingTests", "test", "runtests.jl"),"w") do f - println(f, "using " * (VERSION < v"0.7.0-DEV.2005" ? "Base." : "") * "Test") - println(f, "@test false") - end - - try - Pkg.test("PackageWithFailingTests") - error("unexpected") - catch err - @test err.msg == "PackageWithFailingTests had test errors" - end - end - - # FIXME coverage is currently broken on windows? - Compat.Sys.isunix() && @testset "testing with code-coverage" begin - PkgDev.generate("PackageWithCodeCoverage", "MIT", config=Dict("user.name"=>"Julia Test", "user.email"=>"test@julialang.org")) - - src = """ -module PackageWithCodeCoverage - -export f1, f2, f3, untested - -f1(x) = 2x -f2(x) = f1(x) -function f3(x) - 3x -end -untested(x) = 7 - -end""" - linetested = [false, false, false, false, true, true, false, true, false, false] - open(Pkg.dir("PackageWithCodeCoverage", "src", "PackageWithCodeCoverage.jl"), "w") do f - println(f, src) - end - isdir(Pkg.dir("PackageWithCodeCoverage","test")) || mkdir(Pkg.dir("PackageWithCodeCoverage","test")) - open(Pkg.dir("PackageWithCodeCoverage", "test", "runtests.jl"),"w") do f - println(f, "using PackageWithCodeCoverage") - println(f, "using " * (VERSION < v"0.7.0-DEV.2005" ? "Base." : "") * "Test") - println(f, "@test f2(2) == 4") - println(f, "@test f3(5) == 15") - end - - let codecov_yml = Pkg.dir("PackageWithCodeCoverage", ".codecov.yml") - @test isfile(codecov_yml) - @test readchomp(codecov_yml) == "comment: false" - end - - Pkg.test("PackageWithCodeCoverage") - covdir = Pkg.dir("PackageWithCodeCoverage","src") - covfiles = filter!(x -> occursin("PackageWithCodeCoverage.jl", x) && occursin(".cov", x), readdir(covdir)) - @test isempty(covfiles) - Pkg.test("PackageWithCodeCoverage", coverage=true) - covfiles = filter!(x -> occursin("PackageWithCodeCoverage.jl", x) && occursin(".cov", x), readdir(covdir)) - @test !isempty(covfiles) - for file in covfiles - @test isfile(joinpath(covdir,file)) - covstr = read(joinpath(covdir,file), String) - srclines = split(src, '\n') - covlines = split(covstr, '\n') - for i = 1:length(linetested) - covline = (linetested[i] ? " 1 " : " - ")*srclines[i] - @test covlines[i] == covline - end - end - end +include("generate.jl") +include("registry.jl") +#= if haskey(ENV, "CI") && lowercase(ENV["CI"]) == "true" - Compat.@info("setting git global configuration") + @info("setting git global configuration") run(`git config --global user.name "Julia Test"`) run(`git config --global user.email test@julialang.org`) run(`git config --global github.user JuliaTest`) @@ -179,24 +38,16 @@ end""" end end - @testset "testing freeable" begin - Pkg.add("Example") - io = IOBuffer() - f = PkgDev.freeable(io) - @test !(any(f .== "Example") || occursin("Example", String(take!(io)))) - Pkg.checkout("Example") - f = PkgDev.freeable(io) - @test any(f .== "Example") || occursin("Example", String(take!(io))) - end @testset "testing package registration" begin PkgDev.generate("GreatNewPackage", "MIT", config=Dict("user.name"=>"Julia Test", "user.email"=>"test@julialang.org")) PkgDev.register("GreatNewPackage") @test !isempty(read(joinpath(pkgdir, "METADATA", "GreatNewPackage", "url"), String)) end -end +end end @testset "Testing package utils" begin @test PkgDev.getrepohttpurl("https://github.com/JuliaLang/PkgDev.jl.git") == "https://github.com/JuliaLang/PkgDev.jl" @test PkgDev.getrepohttpurl("git://github.com/JuliaLang/PkgDev.jl.git") == "https://github.com/JuliaLang/PkgDev.jl" end +=# diff --git a/test/utils.jl b/test/utils.jl new file mode 100644 index 0000000..fcfc96f --- /dev/null +++ b/test/utils.jl @@ -0,0 +1,33 @@ +function temp_pkg_dir(fn::Function) + local env_dir + local old_load_path + local old_depot_path + local old_home_project + local old_active_project + try + old_load_path = copy(LOAD_PATH) + old_depot_path = copy(DEPOT_PATH) + old_home_project = Base.HOME_PROJECT[] + old_active_project = Base.ACTIVE_PROJECT[] + empty!(LOAD_PATH) + empty!(DEPOT_PATH) + Base.HOME_PROJECT[] = nothing + Base.ACTIVE_PROJECT[] = nothing + withenv("JULIA_PROJECT" => nothing, "JULIA_LOAD_PATH" => nothing) do + mktempdir() do env_dir + mktempdir() do depot_dir + push!(LOAD_PATH, "@", "@v#.#", "@stdlib") + push!(DEPOT_PATH, depot_dir) + fn(env_dir) + end + end + end + finally + empty!(LOAD_PATH) + empty!(DEPOT_PATH) + append!(LOAD_PATH, old_load_path) + append!(DEPOT_PATH, old_depot_path) + Base.HOME_PROJECT[] = old_home_project + Base.ACTIVE_PROJECT[] = old_active_project + end +end