diff --git a/.github/workflows/DocPreviewCleanup.yml b/.github/workflows/DocPreviewCleanup.yml new file mode 100644 index 0000000..1568024 --- /dev/null +++ b/.github/workflows/DocPreviewCleanup.yml @@ -0,0 +1,28 @@ +name: Doc Preview Cleanup + +on: + pull_request: + types: [closed] + +jobs: + doc-preview-cleanup: + runs-on: ubuntu-latest + steps: + - name: Checkout gh-pages branch + uses: actions/checkout@v2 + with: + ref: gh-pages + + - name: Delete preview and history + run: | + git config user.name "Documenter.jl" + git config user.email "documenter@juliadocs.github.io" + git rm -rf "previews/PR$PRNUM" + git commit -m "delete preview" + git branch gh-pages-new $(echo "delete history" | git commit-tree HEAD^{tree}) + env: + PRNUM: ${{ github.event.number }} + + - name: Push changes + run: | + git push --force origin gh-pages-new:gh-pages diff --git a/bors.toml b/bors.toml index 5e4a4e3..4e0be23 100644 --- a/bors.toml +++ b/bors.toml @@ -12,6 +12,8 @@ status = [ "Documentation" ] delete_merged_branches = true +# Squash commits before merging. +use_squash_merge = true # Uncomment this to use a two hour timeout. # The default is one hour. #timeout_sec = 7200 diff --git a/docs/Manifest.toml b/docs/Manifest.toml index fe1dc94..41eb209 100644 --- a/docs/Manifest.toml +++ b/docs/Manifest.toml @@ -1,16 +1,208 @@ # This file is machine-generated - editing it directly is not advised +[[AbstractFFTs]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "8ed9de2f1b1a9b1dee48582ad477c6e67b83eb2c" +uuid = "621f4979-c628-5d54-868e-fcf4e3e8185c" +version = "1.0.0" + +[[AbstractPlotting]] +deps = ["Animations", "ColorBrewer", "ColorSchemes", "ColorTypes", "Colors", "Contour", "Distributions", "DocStringExtensions", "FFMPEG", "FileIO", "FixedPointNumbers", "Formatting", "FreeType", "FreeTypeAbstraction", "GeometryBasics", "GridLayoutBase", "ImageIO", "IntervalSets", "Isoband", "KernelDensity", "LinearAlgebra", "Markdown", "Match", "Observables", "Packing", "PlotUtils", "PolygonOps", "Printf", "Random", "Serialization", "Showoff", "SignedDistanceFields", "SparseArrays", "StaticArrays", "Statistics", "StatsBase", "StructArrays", "UnicodeFun"] +git-tree-sha1 = "5000fe26eaa31cdd63390e5764dfe13ef6e678ce" +uuid = "537997a7-5e4e-5d89-9595-2241ea00577e" +version = "0.15.1" + +[[Adapt]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "345a14764e43fe927d6f5c250fe4c8e4664e6ee8" +uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" +version = "2.4.0" + +[[Animations]] +deps = ["Colors"] +git-tree-sha1 = "e81c509d2c8e49592413bfb0bb3b08150056c79d" +uuid = "27a7e980-b3e6-11e9-2bcd-0b925532e340" +version = "0.4.1" + +[[Artifacts]] +deps = ["Pkg"] +git-tree-sha1 = "c30985d8821e0cd73870b17b0ed0ce6dc44cb744" +uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" +version = "1.3.0" + +[[AxisAlgorithms]] +deps = ["LinearAlgebra", "Random", "SparseArrays", "WoodburyMatrices"] +git-tree-sha1 = "a4d07a1c313392a77042855df46c5f534076fab9" +uuid = "13072b0f-2c55-5437-9ae7-d433b7a33950" +version = "1.0.0" + [[Base64]] uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" +[[BinaryProvider]] +deps = ["Libdl", "Logging", "SHA"] +git-tree-sha1 = "ecdec412a9abc8db54c0efc5548c64dfce072058" +uuid = "b99e7846-7c00-51b0-8f62-c81ae34c0232" +version = "0.5.10" + +[[Bzip2_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "c3598e525718abcc440f69cc6d5f60dda0a1b61e" +uuid = "6e34b625-4abd-537c-b88f-471c36dfa7a0" +version = "1.0.6+5" + +[[CEnum]] +git-tree-sha1 = "215a9aa4a1f23fbd05b92769fdd62559488d70e9" +uuid = "fa961155-64e5-5f13-b03f-caf6b980ea82" +version = "0.4.1" + +[[CSV]] +deps = ["CategoricalArrays", "DataFrames", "Dates", "Mmap", "Parsers", "PooledArrays", "SentinelArrays", "Tables", "Unicode"] +git-tree-sha1 = "b1f60ea404dc40a76e7f539672ce651079a5a190" +uuid = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" +version = "0.7.10" + +[[Cairo]] +deps = ["Cairo_jll", "Colors", "Glib_jll", "Graphics", "Libdl", "Pango_jll"] +git-tree-sha1 = "d0b3f8b4ad16cb0a2988c6788646a5e6a17b6b1b" +uuid = "159f3aea-2a34-519c-b102-8c37f9878175" +version = "1.0.5" + +[[CairoMakie]] +deps = ["AbstractPlotting", "Cairo", "Colors", "FFTW", "FileIO", "FreeType", "GeometryBasics", "LinearAlgebra", "StaticArrays"] +git-tree-sha1 = "0e4922f4e8cc892537b3c08f7b3daaaa822c6e41" +uuid = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" +version = "0.3.10" + +[[Cairo_jll]] +deps = ["Artifacts", "Bzip2_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "JLLWrappers", "LZO_jll", "Libdl", "Pixman_jll", "Pkg", "Xorg_libXext_jll", "Xorg_libXrender_jll", "Zlib_jll", "libpng_jll"] +git-tree-sha1 = "e2f47f6d8337369411569fd45ae5753ca10394c6" +uuid = "83423d85-b0ee-5818-9007-b63ccbeb887a" +version = "1.16.0+6" + +[[CalibrationErrors]] +deps = ["DataStructures", "Distances", "KernelFunctions", "LinearAlgebra", "Reexport", "Statistics", "StatsBase", "UnPack"] +path = ".." +uuid = "33913031-fe46-5864-950f-100836f47845" +version = "0.5.9" + +[[CategoricalArrays]] +deps = ["DataAPI", "Future", "JSON", "Missings", "Printf", "Statistics", "StructTypes", "Unicode"] +git-tree-sha1 = "2ac27f59196a68070e132b25713f9a5bbc5fa0d2" +uuid = "324d7699-5711-5eae-9e2f-1d82baa6b597" +version = "0.8.3" + +[[ChainRulesCore]] +deps = ["LinearAlgebra", "MuladdMacro", "SparseArrays"] +git-tree-sha1 = "15081c431bb25848ad9b0d172a65794f3a3e197a" +uuid = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" +version = "0.9.24" + +[[ColorBrewer]] +deps = ["Colors", "JSON", "Test"] +git-tree-sha1 = "61c5334f33d91e570e1d0c3eb5465835242582c4" +uuid = "a2cac450-b92f-5266-8821-25eda20663c8" +version = "0.4.0" + +[[ColorSchemes]] +deps = ["ColorTypes", "Colors", "FixedPointNumbers", "Random", "StaticArrays"] +git-tree-sha1 = "3141757b5832ee7a0386db87997ee5a23ff20f4d" +uuid = "35d6a980-a343-548e-a6ea-1d62b119f2f4" +version = "3.10.2" + +[[ColorTypes]] +deps = ["FixedPointNumbers", "Random"] +git-tree-sha1 = "4bffea7ed1a9f0f3d1a131bbcd4b925548d75288" +uuid = "3da002f7-5984-5a60-b8a6-cbb66c0b333f" +version = "0.10.9" + +[[ColorVectorSpace]] +deps = ["ColorTypes", "Colors", "FixedPointNumbers", "LinearAlgebra", "SpecialFunctions", "Statistics", "StatsBase"] +git-tree-sha1 = "4d17724e99f357bfd32afa0a9e2dda2af31a9aea" +uuid = "c3611d14-8923-5661-9e6a-0046d554d3a4" +version = "0.8.7" + +[[Colors]] +deps = ["ColorTypes", "FixedPointNumbers", "InteractiveUtils", "Reexport"] +git-tree-sha1 = "ac5f2213e56ed8a34a3dd2f681f4df1166b34929" +uuid = "5ae59095-9a9b-59fe-a467-6f913c188581" +version = "0.12.6" + +[[Compat]] +deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "SHA", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"] +git-tree-sha1 = "919c7f3151e79ff196add81d7f4e45d91bbf420b" +uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" +version = "3.25.0" + +[[CompilerSupportLibraries_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "8e695f735fca77e9708e795eda62afdb869cbb70" +uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" +version = "0.3.4+0" + +[[Contour]] +deps = ["StaticArrays"] +git-tree-sha1 = "9f02045d934dc030edad45944ea80dbd1f0ebea7" +uuid = "d38c429a-6771-53c6-b99e-75d170b6e991" +version = "0.5.7" + +[[Crayons]] +git-tree-sha1 = "3f71217b538d7aaee0b69ab47d9b7724ca8afa0d" +uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f" +version = "4.0.4" + +[[DataAPI]] +git-tree-sha1 = "ad84f52c0b8f05aa20839484dbaf01690b41ff84" +uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" +version = "1.4.0" + +[[DataDeps]] +deps = ["BinaryProvider", "HTTP", "Libdl", "Reexport", "SHA", "p7zip_jll"] +git-tree-sha1 = "9f69dd052eaf292edd42d4bed999dfbd291927a0" +uuid = "124859b0-ceae-595e-8997-d05f6a7a8dfe" +version = "0.7.6" + +[[DataFrames]] +deps = ["CategoricalArrays", "Compat", "DataAPI", "Future", "InvertedIndices", "IteratorInterfaceExtensions", "LinearAlgebra", "Markdown", "Missings", "PooledArrays", "PrettyTables", "Printf", "REPL", "Reexport", "SortingAlgorithms", "Statistics", "TableTraits", "Tables", "Unicode"] +git-tree-sha1 = "b46e1deb4592a5df7416b10dfcd6b01fb194ab9a" +uuid = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" +version = "0.22.2" + +[[DataStructures]] +deps = ["Compat", "InteractiveUtils", "OrderedCollections"] +git-tree-sha1 = "fb0aa371da91c1ff9dc7fbed6122d3e411420b9c" +uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" +version = "0.18.8" + +[[DataValueInterfaces]] +git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6" +uuid = "e2d170a0-9d28-54be-80f0-106bbe20a464" +version = "1.0.0" + [[Dates]] deps = ["Printf"] uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" +[[DelimitedFiles]] +deps = ["Mmap"] +uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab" + +[[Distances]] +deps = ["LinearAlgebra", "Statistics"] +git-tree-sha1 = "e8b13ba5f166e11df2de6fc283e5db7864245df0" +uuid = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7" +version = "0.10.0" + [[Distributed]] deps = ["Random", "Serialization", "Sockets"] uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" +[[Distributions]] +deps = ["FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "SparseArrays", "SpecialFunctions", "Statistics", "StatsBase", "StatsFuns"] +git-tree-sha1 = "5c6a645ea65af6aac4f96b5dedaae660b7b56267" +uuid = "31c24e10-a181-5473-b8eb-7969acd0382f" +version = "0.24.10" + [[DocStringExtensions]] deps = ["LibGit2", "Markdown", "Pkg", "Test"] git-tree-sha1 = "50ddf44c53698f5e784bbebb3f4b21c5807401b1" @@ -23,53 +215,535 @@ git-tree-sha1 = "b7715ae18be02110a8cf9cc8ed2ccdb1e3e3aba2" uuid = "e30172f5-a6a5-5a46-863b-614d45cd2de4" version = "0.26.1" +[[EarCut_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "92d8f9f208637e8d2d28c664051a00569c01493d" +uuid = "5ae413db-bbd1-5e63-b57d-d24a61df00f5" +version = "2.1.5+1" + +[[EllipsisNotation]] +git-tree-sha1 = "18ee049accec8763be17a933737c1dd0fdf8673a" +uuid = "da5c29d0-fa7d-589e-88eb-ea29b0a81949" +version = "1.0.0" + +[[Expat_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "1402e52fcda25064f51c77a9655ce8680b76acf0" +uuid = "2e619515-83b5-522b-bb60-26c02a35a201" +version = "2.2.7+6" + +[[FFMPEG]] +deps = ["FFMPEG_jll", "x264_jll"] +git-tree-sha1 = "9a73ffdc375be61b0e4516d83d880b265366fe1f" +uuid = "c87230d0-a227-11e9-1b43-d7ebe4e7570a" +version = "0.4.0" + +[[FFMPEG_jll]] +deps = ["Artifacts", "Bzip2_jll", "FreeType2_jll", "FriBidi_jll", "JLLWrappers", "LAME_jll", "LibVPX_jll", "Libdl", "Ogg_jll", "OpenSSL_jll", "Opus_jll", "Pkg", "Zlib_jll", "libass_jll", "libfdk_aac_jll", "libvorbis_jll", "x264_jll", "x265_jll"] +git-tree-sha1 = "3cc57ad0a213808473eafef4845a74766242e05f" +uuid = "b22a6f82-2f65-5046-a5b2-351ab43fb4e5" +version = "4.3.1+4" + +[[FFTW]] +deps = ["AbstractFFTs", "FFTW_jll", "IntelOpenMP_jll", "Libdl", "LinearAlgebra", "MKL_jll", "Reexport"] +git-tree-sha1 = "c31e446bf3b12aad2ec8fc500fe19528c148d811" +uuid = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" +version = "1.3.1" + +[[FFTW_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "5a0d4b6a22a34d17d53543bd124f4b08ed78e8b0" +uuid = "f5851436-0d7a-5f13-b9de-f02708fd171a" +version = "3.3.9+7" + +[[FileIO]] +deps = ["Pkg"] +git-tree-sha1 = "fee8955b9dfa7bec67117ef48085fb2b559b9c22" +uuid = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" +version = "1.4.5" + +[[FillArrays]] +deps = ["LinearAlgebra", "Random", "SparseArrays"] +git-tree-sha1 = "ff537e5a3cba92fb48f30fec46723510450f2c0e" +uuid = "1a297f60-69ca-5386-bcde-b61e274b549b" +version = "0.10.2" + +[[FixedPointNumbers]] +deps = ["Statistics"] +git-tree-sha1 = "335bfdceacc84c5cdf16aadc768aa5ddfc5383cc" +uuid = "53c48c17-4a7d-5ca2-90c5-79b7896eea93" +version = "0.8.4" + +[[Fontconfig_jll]] +deps = ["Artifacts", "Bzip2_jll", "Expat_jll", "FreeType2_jll", "JLLWrappers", "Libdl", "Libuuid_jll", "Pkg", "Zlib_jll"] +git-tree-sha1 = "35895cf184ceaab11fd778b4590144034a167a2f" +uuid = "a3f928ae-7b40-5064-980b-68af3947d34b" +version = "2.13.1+14" + +[[Formatting]] +deps = ["Printf"] +git-tree-sha1 = "8339d61043228fdd3eb658d86c926cb282ae72a8" +uuid = "59287772-0a20-5a39-b81b-1366585eb4c0" +version = "0.4.2" + +[[FreeType]] +deps = ["CEnum", "FreeType2_jll"] +git-tree-sha1 = "d5e296dea44f9cfb8cc83681aa3db367e0cbfd4f" +uuid = "b38be410-82b0-50bf-ab77-7b57e271db43" +version = "3.0.1" + +[[FreeType2_jll]] +deps = ["Artifacts", "Bzip2_jll", "JLLWrappers", "Libdl", "Pkg", "Zlib_jll"] +git-tree-sha1 = "cbd58c9deb1d304f5a245a0b7eb841a2560cfec6" +uuid = "d7e528f0-a631-5988-bf34-fe36492bcfd7" +version = "2.10.1+5" + +[[FreeTypeAbstraction]] +deps = ["ColorVectorSpace", "Colors", "FreeType", "GeometryBasics", "StaticArrays"] +git-tree-sha1 = "3d955cfbefd6645ced60091d8c9bfcc96fc32345" +uuid = "663a7486-cb36-511b-a19d-713bb74d65c9" +version = "0.8.4" + +[[FriBidi_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "0d20aed5b14dd4c9a2453c1b601d08e1149679cc" +uuid = "559328eb-81f9-559d-9380-de523a88c83c" +version = "1.0.5+6" + +[[Functors]] +deps = ["MacroTools"] +git-tree-sha1 = "f40adc6422f548176bb4351ebd29e4abf773040a" +uuid = "d9f16b24-f501-4c13-a1f2-28368ffc5196" +version = "0.1.0" + +[[Future]] +deps = ["Random"] +uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" + +[[GeometryBasics]] +deps = ["EarCut_jll", "IterTools", "LinearAlgebra", "StaticArrays", "StructArrays", "Tables"] +git-tree-sha1 = "876f77f0d3253e882ff588af1c95d0e4a86c9766" +uuid = "5c1252a2-5f33-56bf-86c9-59e7332b4326" +version = "0.3.5" + +[[Gettext_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Libiconv_jll", "Pkg", "XML2_jll"] +git-tree-sha1 = "8c14294a079216000a0bdca5ec5a447f073ddc9d" +uuid = "78b55507-aeef-58d4-861c-77aaff3498b1" +version = "0.20.1+7" + +[[Glib_jll]] +deps = ["Artifacts", "Gettext_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Libiconv_jll", "Libmount_jll", "PCRE_jll", "Pkg", "Zlib_jll"] +git-tree-sha1 = "04690cc5008b38ecbdfede949220bc7d9ba26397" +uuid = "7746bdde-850d-59dc-9ae8-88ece973131d" +version = "2.59.0+4" + +[[Graphics]] +deps = ["Colors", "LinearAlgebra", "NaNMath"] +git-tree-sha1 = "45d684ead5b65c043ad46bd5be750d61c39d7ef8" +uuid = "a2bd30eb-e257-5431-a919-1863eab51364" +version = "1.0.2" + +[[Graphite2_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "42adbc6fd39ba41138f894b8ac711146a2b0d986" +uuid = "3b182d85-2403-5c21-9c21-1e1f0cc25472" +version = "1.3.13+4" + +[[GridLayoutBase]] +deps = ["GeometryBasics", "InteractiveUtils", "Match", "Observables"] +git-tree-sha1 = "0702ac06d05e653996a37fea1d631bc7e74512ee" +uuid = "3955a311-db13-416c-9275-1d80ed98e5e9" +version = "0.4.1" + +[[Grisu]] +git-tree-sha1 = "03d381f65183cb2d0af8b3425fde97263ce9a995" +uuid = "42e2da0e-8278-4e71-bc24-59509adca0fe" +version = "1.0.0" + +[[HTTP]] +deps = ["Base64", "Dates", "IniFile", "MbedTLS", "Sockets", "URIs"] +git-tree-sha1 = "63055ee44b5c2b95ec1921edcf856c60124ff0c3" +uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3" +version = "0.9.2" + +[[HarfBuzz_jll]] +deps = ["Artifacts", "Cairo_jll", "Fontconfig_jll", "FreeType2_jll", "Gettext_jll", "Glib_jll", "Graphite2_jll", "ICU_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Pkg"] +git-tree-sha1 = "90bed5fc61d12d10832ebf988988104888eebaca" +uuid = "2e76f6c2-a576-52d4-95c1-20adfe4de566" +version = "2.6.1+10" + +[[ICU_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "6ce9cf3c5490b045710d60ac3fd2fe48188846b3" +uuid = "a51ab1cf-af8e-5615-a023-bc2c838bba6b" +version = "67.1.0+3" + [[IOCapture]] deps = ["Logging"] git-tree-sha1 = "377252859f740c217b936cebcd918a44f9b53b59" uuid = "b5f81e59-6552-4d32-b1f0-c071b021bf89" version = "0.1.1" +[[ImageCore]] +deps = ["AbstractFFTs", "Colors", "FixedPointNumbers", "Graphics", "MappedArrays", "MosaicViews", "OffsetArrays", "PaddedViews", "Reexport"] +git-tree-sha1 = "79badd979fbee9b8980cd995cd5a86a9e93b8ad7" +uuid = "a09fc81d-aa75-5fe9-8630-4744c3626534" +version = "0.8.20" + +[[ImageIO]] +deps = ["FileIO", "Netpbm", "PNGFiles"] +git-tree-sha1 = "0d6d09c28d67611c68e25af0c2df7269c82b73c7" +uuid = "82e4d734-157c-48bb-816b-45c225c6df19" +version = "0.4.1" + +[[IndirectArrays]] +git-tree-sha1 = "c2a145a145dc03a7620af1444e0264ef907bd44f" +uuid = "9b13fd28-a010-5f03-acff-a1bbcff69959" +version = "0.5.1" + +[[IniFile]] +deps = ["Test"] +git-tree-sha1 = "098e4d2c533924c921f9f9847274f2ad89e018b8" +uuid = "83e8ac13-25f8-5344-8a64-a9f2b223428f" +version = "0.5.0" + +[[IntelOpenMP_jll]] +deps = ["Libdl", "Pkg"] +git-tree-sha1 = "fb8e1c7a5594ba56f9011310790e03b5384998d6" +uuid = "1d5cc7b8-4909-519e-a0f8-d0f5ad9712d0" +version = "2018.0.3+0" + [[InteractiveUtils]] deps = ["Markdown"] uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" +[[Interpolations]] +deps = ["AxisAlgorithms", "LinearAlgebra", "OffsetArrays", "Random", "Ratios", "SharedArrays", "SparseArrays", "StaticArrays", "WoodburyMatrices"] +git-tree-sha1 = "eb1dd6d5b2275faaaa18533e0fc5f9171cec25fa" +uuid = "a98d9a8b-a2ab-59e6-89dd-64a1c18fca59" +version = "0.13.1" + +[[IntervalSets]] +deps = ["Dates", "EllipsisNotation", "Statistics"] +git-tree-sha1 = "93a6d78525feb0d3ee2a2ae83a7d04db1db5663f" +uuid = "8197267c-284f-5f27-9208-e0e47529a953" +version = "0.5.2" + +[[InvertedIndices]] +deps = ["Test"] +git-tree-sha1 = "15732c475062348b0165684ffe28e85ea8396afc" +uuid = "41ab1584-1d38-5bbf-9106-f11c6c58b48f" +version = "1.0.0" + +[[Isoband]] +deps = ["isoband_jll"] +git-tree-sha1 = "f9b6d97355599074dc867318950adaa6f9946137" +uuid = "f1662d9f-8043-43de-a69a-05efc1cc6ff4" +version = "0.1.1" + +[[IterTools]] +git-tree-sha1 = "05110a2ab1fc5f932622ffea2a003221f4782c18" +uuid = "c8e1da08-722c-5040-9ed9-7db0dc04731e" +version = "1.3.0" + +[[IteratorInterfaceExtensions]] +git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856" +uuid = "82899510-4779-5014-852e-03e436cf321d" +version = "1.0.0" + +[[JLLWrappers]] +git-tree-sha1 = "04b49c556240b62d5a799e94c63d5fc14d3c07cd" +uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" +version = "1.1.4" + [[JSON]] deps = ["Dates", "Mmap", "Parsers", "Unicode"] git-tree-sha1 = "81690084b6198a2e1da36fcfda16eeca9f9f24e4" uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" version = "0.21.1" +[[KernelDensity]] +deps = ["Distributions", "DocStringExtensions", "FFTW", "Interpolations", "StatsBase"] +git-tree-sha1 = "09aeec87bdc9c1fa70d0b508dfa94a21acd280d9" +uuid = "5ab0869b-81aa-558d-bb23-cbf5423bbe9b" +version = "0.6.2" + +[[KernelFunctions]] +deps = ["Compat", "Distances", "Functors", "InteractiveUtils", "LinearAlgebra", "Random", "Requires", "SpecialFunctions", "StatsBase", "StatsFuns", "Test", "ZygoteRules"] +git-tree-sha1 = "e9152dff713b5e17e458403ab67cec33a9ff00d3" +uuid = "ec8451be-7e33-11e9-00cf-bbf324bd1392" +version = "0.8.11" + +[[LAME_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "df381151e871f41ee86cee4f5f6fd598b8a68826" +uuid = "c1c5ebd0-6772-5130-a774-d5fcae4a789d" +version = "3.100.0+3" + +[[LZO_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "f128cd6cd05ffd6d3df0523ed99b90ff6f9b349a" +uuid = "dd4b983a-f0e5-5f8d-a1b7-129d4a5fb1ac" +version = "2.10.0+3" + [[LibGit2]] deps = ["Printf"] uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" +[[LibVPX_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "85fcc80c3052be96619affa2fe2e6d2da3908e11" +uuid = "dd192d2f-8180-539f-9fb4-cc70b1dcf69a" +version = "1.9.0+1" + [[Libdl]] uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" +[[Libffi_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "a2cd088a88c0d37eef7d209fd3d8712febce0d90" +uuid = "e9f186c6-92d2-5b65-8a66-fee21dc1b490" +version = "3.2.1+4" + +[[Libgcrypt_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Libgpg_error_jll", "Pkg"] +git-tree-sha1 = "b391a18ab1170a2e568f9fb8d83bc7c780cb9999" +uuid = "d4300ac3-e22c-5743-9152-c294e39db1e4" +version = "1.8.5+4" + +[[Libgpg_error_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "ec7f2e8ad5c9fa99fc773376cdbc86d9a5a23cb7" +uuid = "7add5ba3-2f88-524e-9cd5-f83b8a55f7b8" +version = "1.36.0+3" + +[[Libiconv_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "8e924324b2e9275a51407a4e06deb3455b1e359f" +uuid = "94ce4f54-9a6c-5748-9c1c-f9c7231a4531" +version = "1.16.0+7" + +[[Libmount_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "51ad0c01c94c1ce48d5cad629425035ad030bfd5" +uuid = "4b2f31a3-9ecc-558c-b454-b3730dcb73e9" +version = "2.34.0+3" + +[[Libuuid_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "f879ae9edbaa2c74c922e8b85bb83cc84ea1450b" +uuid = "38a345b3-de98-5d2b-a5d3-14cd9215e700" +version = "2.34.0+7" + +[[LinearAlgebra]] +deps = ["Libdl"] +uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" + [[Logging]] uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" +[[MKL_jll]] +deps = ["IntelOpenMP_jll", "Libdl", "Pkg"] +git-tree-sha1 = "eb540ede3aabb8284cb482aa41d00d6ca850b1f8" +uuid = "856f044c-d86e-5d09-b602-aeab76dc8ba7" +version = "2020.2.254+0" + +[[MacroTools]] +deps = ["Markdown", "Random"] +git-tree-sha1 = "6a8a2a625ab0dea913aba95c11370589e0239ff0" +uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" +version = "0.5.6" + +[[MappedArrays]] +deps = ["FixedPointNumbers"] +git-tree-sha1 = "b92bd220c95a8bbe89af28f11201fd080e0e3fe7" +uuid = "dbb5928d-eab1-5f90-85c2-b9b0edb7c900" +version = "0.3.0" + [[Markdown]] deps = ["Base64"] uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" +[[Match]] +git-tree-sha1 = "5cf525d97caf86d29307150fcba763a64eaa9cbe" +uuid = "7eb4fadd-790c-5f42-8a69-bfa0b872bfbf" +version = "1.1.0" + +[[MbedTLS]] +deps = ["Dates", "MbedTLS_jll", "Random", "Sockets"] +git-tree-sha1 = "1c38e51c3d08ef2278062ebceade0e46cefc96fe" +uuid = "739be429-bea8-5141-9913-cc70e7f3736d" +version = "1.0.3" + +[[MbedTLS_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "0eef589dd1c26a3ac9d753fe1a8bcad63f956fa6" +uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" +version = "2.16.8+1" + +[[Missings]] +deps = ["DataAPI"] +git-tree-sha1 = "ed61674a0864832495ffe0a7e889c0da76b0f4c8" +uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28" +version = "0.4.4" + [[Mmap]] uuid = "a63ad114-7e13-5084-954f-fe012c677804" +[[MosaicViews]] +deps = ["MappedArrays", "OffsetArrays", "PaddedViews"] +git-tree-sha1 = "614e8d77264d20c1db83661daadfab38e8e4b77e" +uuid = "e94cdb99-869f-56ef-bcf0-1ae2bcbe0389" +version = "0.2.4" + +[[MuladdMacro]] +git-tree-sha1 = "c6190f9a7fc5d9d5915ab29f2134421b12d24a68" +uuid = "46d2c3a1-f734-5fdb-9937-b9b9aeba4221" +version = "0.2.2" + +[[NaNMath]] +git-tree-sha1 = "bfe47e760d60b82b66b61d2d44128b62e3a369fb" +uuid = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3" +version = "0.3.5" + +[[Netpbm]] +deps = ["ColorVectorSpace", "FileIO", "ImageCore"] +git-tree-sha1 = "03472ec5ea8884b791c6340712f3261c0f393691" +uuid = "f09324ee-3d7c-5217-9330-fc30815ba969" +version = "1.0.0" + +[[Observables]] +git-tree-sha1 = "3469ef96607a6b9a1e89e54e6f23401073ed3126" +uuid = "510215fc-4207-5dde-b226-833fc4488ee2" +version = "0.3.3" + +[[OffsetArrays]] +deps = ["Adapt"] +git-tree-sha1 = "5b9ed21d50a91bb783c87f37c8da9266f0265504" +uuid = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" +version = "1.4.4" + +[[Ogg_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "a42c0f138b9ebe8b58eba2271c5053773bde52d0" +uuid = "e7412a2a-1a6e-54c0-be00-318e2571c051" +version = "1.3.4+2" + +[[OpenSSL_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "71bbbc616a1d710879f5a1021bcba65ffba6ce58" +uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" +version = "1.1.1+6" + +[[OpenSpecFun_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "9db77584158d0ab52307f8c04f8e7c08ca76b5b3" +uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e" +version = "0.5.3+4" + +[[Opus_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "f9d57f4126c39565e05a2b0264df99f497fc6f37" +uuid = "91d4177d-7536-5919-b921-800302f37372" +version = "1.3.1+3" + +[[OrderedCollections]] +git-tree-sha1 = "cf59cfed2e2c12e8a2ff0a4f1e9b2cd8650da6db" +uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +version = "1.3.2" + +[[PCRE_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "1b556ad51dceefdbf30e86ffa8f528b73c7df2bb" +uuid = "2f80f16e-611a-54ab-bc61-aa92de5b98fc" +version = "8.42.0+4" + +[[PDMats]] +deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse", "Test"] +git-tree-sha1 = "95a4038d1011dfdbde7cecd2ad0ac411e53ab1bc" +uuid = "90014a1f-27ba-587c-ab20-58faa44d9150" +version = "0.10.1" + +[[PNGFiles]] +deps = ["Base64", "CEnum", "ImageCore", "IndirectArrays", "OffsetArrays", "libpng_jll"] +git-tree-sha1 = "aa6e87a2361c2fe5a63b1a6a4b567f13aa108991" +uuid = "f57f5aa1-a3ce-4bc8-8ab9-96f992907883" +version = "0.3.5" + +[[Packing]] +deps = ["GeometryBasics"] +git-tree-sha1 = "f4049d379326c2c7aa875c702ad19346ecb2b004" +uuid = "19eb6ba3-879d-56ad-ad62-d5c202156566" +version = "0.4.1" + +[[PaddedViews]] +deps = ["OffsetArrays"] +git-tree-sha1 = "91d229e113e8975a399e40d7c0b1ddf4da6d3c59" +uuid = "5432bcbf-9aad-5242-b902-cca2824c8663" +version = "0.5.7" + +[[PalmerPenguins]] +deps = ["CSV", "DataDeps"] +git-tree-sha1 = "d44224354c5d31262b4402d6f9fef79a715b1dce" +uuid = "8b842266-38fa-440a-9b57-31493939ab85" +version = "0.1.1" + +[[Pango_jll]] +deps = ["Artifacts", "Cairo_jll", "Fontconfig_jll", "FreeType2_jll", "FriBidi_jll", "Glib_jll", "HarfBuzz_jll", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "9a336dee51d20d1ed890c4a8dca636e86e2b76ca" +uuid = "36c8627f-9965-5494-a995-c6b170f724f3" +version = "1.42.4+10" + [[Parsers]] deps = ["Dates"] git-tree-sha1 = "50c9a9ed8c714945e01cd53a21007ed3865ed714" uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" version = "1.0.15" +[[Pixman_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "6a20a83c1ae86416f0a5de605eaea08a552844a3" +uuid = "30392449-352a-5448-841d-b1acce4e97dc" +version = "0.40.0+0" + [[Pkg]] deps = ["Dates", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"] uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +[[PlotUtils]] +deps = ["ColorSchemes", "Colors", "Dates", "Printf", "Random", "Reexport", "Statistics"] +git-tree-sha1 = "6a59c282058f46967069972d74526a68d27a9012" +uuid = "995b91a9-d308-5afd-9ec6-746e21dbc043" +version = "1.0.8" + +[[PolygonOps]] +git-tree-sha1 = "c031d2332c9a8e1c90eca239385815dc271abb22" +uuid = "647866c9-e3ac-4575-94e7-e3d426903924" +version = "0.1.1" + +[[PooledArrays]] +deps = ["DataAPI"] +git-tree-sha1 = "b1333d4eced1826e15adbdf01a4ecaccca9d353c" +uuid = "2dfb63ee-cc39-5dd5-95bd-886bf059d720" +version = "0.5.3" + +[[PrettyTables]] +deps = ["Crayons", "Formatting", "Markdown", "Reexport", "Tables"] +git-tree-sha1 = "237170206bf38a66fee4d845f4ae57f63788eeb0" +uuid = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" +version = "0.10.1" + [[Printf]] deps = ["Unicode"] uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" +[[QuadGK]] +deps = ["DataStructures", "LinearAlgebra"] +git-tree-sha1 = "12fbe86da16df6679be7521dfb39fbc861e1dc7b" +uuid = "1fd47b50-473d-5c70-9696-f719f8f3bcdc" +version = "2.4.1" + [[REPL]] deps = ["InteractiveUtils", "Markdown", "Sockets"] uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" @@ -78,22 +752,281 @@ uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" deps = ["Serialization"] uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +[[Ratios]] +git-tree-sha1 = "37d210f612d70f3f7d57d488cb3b6eff56ad4e41" +uuid = "c84ed2f1-dad5-54f0-aa8e-dbefe2724439" +version = "0.4.0" + +[[Reexport]] +deps = ["Pkg"] +git-tree-sha1 = "7b1d07f411bc8ddb7977ec7f377b97b158514fe0" +uuid = "189a3867-3050-52da-a836-e630ba90ab69" +version = "0.2.0" + +[[Requires]] +deps = ["UUIDs"] +git-tree-sha1 = "cfbac6c1ed70c002ec6361e7fd334f02820d6419" +uuid = "ae029012-a4dd-5104-9daa-d747884805df" +version = "1.1.2" + +[[Rmath]] +deps = ["Random", "Rmath_jll"] +git-tree-sha1 = "86c5647b565873641538d8f812c04e4c9dbeb370" +uuid = "79098fc4-a85e-5d69-aa6a-4863f24498fa" +version = "0.6.1" + +[[Rmath_jll]] +deps = ["Libdl", "Pkg"] +git-tree-sha1 = "d76185aa1f421306dec73c057aa384bad74188f0" +uuid = "f50d1b31-88e8-58de-be2c-1cc44531875f" +version = "0.2.2+1" + [[SHA]] uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +[[SentinelArrays]] +deps = ["Dates", "Random"] +git-tree-sha1 = "6ccde405cf0759eba835eb613130723cb8f10ff9" +uuid = "91c51154-3ec4-41a3-a24f-3f23e20d615c" +version = "1.2.16" + [[Serialization]] uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" +[[SharedArrays]] +deps = ["Distributed", "Mmap", "Random", "Serialization"] +uuid = "1a1011a3-84de-559e-8e89-a11a2f7dc383" + +[[Showoff]] +deps = ["Dates", "Grisu"] +git-tree-sha1 = "ee010d8f103468309b8afac4abb9be2e18ff1182" +uuid = "992d4aef-0814-514b-bc4d-f2e9a6c4116f" +version = "0.3.2" + +[[SignedDistanceFields]] +deps = ["Random", "Statistics", "Test"] +git-tree-sha1 = "d263a08ec505853a5ff1c1ebde2070419e3f28e9" +uuid = "73760f76-fbc4-59ce-8f25-708e95d2df96" +version = "0.4.0" + [[Sockets]] uuid = "6462fe0b-24de-5631-8697-dd941f90decc" +[[SortingAlgorithms]] +deps = ["DataStructures", "Random", "Test"] +git-tree-sha1 = "03f5898c9959f8115e30bc7226ada7d0df554ddd" +uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c" +version = "0.3.1" + +[[SparseArrays]] +deps = ["LinearAlgebra", "Random"] +uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" + +[[SpecialFunctions]] +deps = ["ChainRulesCore", "OpenSpecFun_jll"] +git-tree-sha1 = "75394dbe2bd346beeed750fb02baa6445487b862" +uuid = "276daf66-3868-5448-9aa4-cd146d93841b" +version = "1.2.1" + +[[StaticArrays]] +deps = ["LinearAlgebra", "Random", "Statistics"] +git-tree-sha1 = "9da72ed50e94dbff92036da395275ed114e04d49" +uuid = "90137ffa-7385-5640-81b9-e52037218182" +version = "1.0.1" + +[[Statistics]] +deps = ["LinearAlgebra", "SparseArrays"] +uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" + +[[StatsBase]] +deps = ["DataAPI", "DataStructures", "LinearAlgebra", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics"] +git-tree-sha1 = "7bab7d4eb46b225b35179632852b595a3162cb61" +uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" +version = "0.33.2" + +[[StatsFuns]] +deps = ["Rmath", "SpecialFunctions"] +git-tree-sha1 = "3b9f665c70712af3264b61c27a7e1d62055dafd1" +uuid = "4c63d2b9-4356-54db-8cca-17b64c39e42c" +version = "0.9.6" + +[[StructArrays]] +deps = ["Adapt", "DataAPI", "Tables"] +git-tree-sha1 = "8099ed9fb90b6e754d6ba8c6ed8670f010eadca0" +uuid = "09ab397b-f2b6-538f-b94a-2f83cf4a842a" +version = "0.4.4" + +[[StructTypes]] +deps = ["Dates", "UUIDs"] +git-tree-sha1 = "d94235fcdc4a09649f263365c5f7e4ed4ba6ed34" +uuid = "856f2bd8-1eba-4b0a-8007-ebc267875bd4" +version = "1.2.1" + +[[SuiteSparse]] +deps = ["Libdl", "LinearAlgebra", "Serialization", "SparseArrays"] +uuid = "4607b0f0-06f3-5cda-b6b1-a6196a1729e9" + +[[TableTraits]] +deps = ["IteratorInterfaceExtensions"] +git-tree-sha1 = "b1ad568ba658d8cbb3b892ed5380a6f3e781a81e" +uuid = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c" +version = "1.0.0" + +[[Tables]] +deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "LinearAlgebra", "TableTraits", "Test"] +git-tree-sha1 = "240d19b8762006ff04b967bdd833269ad642d550" +uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" +version = "1.2.2" + [[Test]] deps = ["Distributed", "InteractiveUtils", "Logging", "Random"] uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +[[URIs]] +git-tree-sha1 = "bc331715463c41d601cf8bfd38ca70a490af5c5b" +uuid = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" +version = "1.1.0" + [[UUIDs]] deps = ["Random", "SHA"] uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" +[[UnPack]] +git-tree-sha1 = "387c1f73762231e86e0c9c5443ce3b4a0a9a0c2b" +uuid = "3a884ed6-31ef-47d7-9d2a-63182c4928ed" +version = "1.0.2" + [[Unicode]] uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" + +[[UnicodeFun]] +deps = ["REPL"] +git-tree-sha1 = "53915e50200959667e78a92a418594b428dffddf" +uuid = "1cfade01-22cf-5700-b092-accc4b62d6e1" +version = "0.4.1" + +[[WoodburyMatrices]] +deps = ["LinearAlgebra", "SparseArrays"] +git-tree-sha1 = "59e2ad8fd1591ea019a5259bd012d7aee15f995c" +uuid = "efce3f68-66dc-5838-9240-27a6d6f5f9b6" +version = "0.5.3" + +[[XML2_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Libiconv_jll", "Pkg", "Zlib_jll"] +git-tree-sha1 = "be0db24f70aae7e2b89f2f3092e93b8606d659a6" +uuid = "02c8fc9c-b97f-50b9-bbe4-9be30ff0a78a" +version = "2.9.10+3" + +[[XSLT_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Libgcrypt_jll", "Pkg", "XML2_jll"] +git-tree-sha1 = "2b3eac39df218762d2d005702d601cd44c997497" +uuid = "aed1982a-8fda-507f-9586-7b0439959a61" +version = "1.1.33+4" + +[[Xorg_libX11_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libxcb_jll", "Xorg_xtrans_jll"] +git-tree-sha1 = "5be649d550f3f4b95308bf0183b82e2582876527" +uuid = "4f6342f7-b3d2-589e-9d20-edeb45f2b2bc" +version = "1.6.9+4" + +[[Xorg_libXau_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "4e490d5c960c314f33885790ed410ff3a94ce67e" +uuid = "0c0b7dd1-d40b-584c-a123-a41640f87eec" +version = "1.0.9+4" + +[[Xorg_libXdmcp_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "4fe47bd2247248125c428978740e18a681372dd4" +uuid = "a3789734-cfe1-5b06-b2d0-1dd0d9d62d05" +version = "1.1.3+4" + +[[Xorg_libXext_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll"] +git-tree-sha1 = "b7c0aa8c376b31e4852b360222848637f481f8c3" +uuid = "1082639a-0dae-5f34-9b06-72781eeb8cb3" +version = "1.3.4+4" + +[[Xorg_libXrender_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll"] +git-tree-sha1 = "19560f30fd49f4d4efbe7002a1037f8c43d43b96" +uuid = "ea2f1a96-1ddc-540d-b46f-429655e07cfa" +version = "0.9.10+4" + +[[Xorg_libpthread_stubs_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "6783737e45d3c59a4a4c4091f5f88cdcf0908cbb" +uuid = "14d82f49-176c-5ed1-bb49-ad3f5cbd8c74" +version = "0.1.0+3" + +[[Xorg_libxcb_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "XSLT_jll", "Xorg_libXau_jll", "Xorg_libXdmcp_jll", "Xorg_libpthread_stubs_jll"] +git-tree-sha1 = "daf17f441228e7a3833846cd048892861cff16d6" +uuid = "c7cfdc94-dc32-55de-ac96-5a1b8d977c5b" +version = "1.13.0+3" + +[[Xorg_xtrans_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "79c31e7844f6ecf779705fbc12146eb190b7d845" +uuid = "c5fb5394-a638-5e4d-96e5-b29de1b5cf10" +version = "1.4.0+3" + +[[Zlib_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "320228915c8debb12cb434c59057290f0834dbf6" +uuid = "83775a58-1f1d-513f-b197-d71354ab007a" +version = "1.2.11+18" + +[[ZygoteRules]] +deps = ["MacroTools"] +git-tree-sha1 = "9e7a1e8ca60b742e508a315c17eef5211e7fbfd7" +uuid = "700de1a5-db45-46bc-99cf-38207098b444" +version = "0.2.1" + +[[isoband_jll]] +deps = ["Libdl", "Pkg"] +git-tree-sha1 = "a1ac99674715995a536bbce674b068ec1b7d893d" +uuid = "9a68df92-36a6-505f-a73e-abb412b6bfb4" +version = "0.2.2+0" + +[[libass_jll]] +deps = ["Artifacts", "Bzip2_jll", "FreeType2_jll", "FriBidi_jll", "JLLWrappers", "Libdl", "Pkg", "Zlib_jll"] +git-tree-sha1 = "acc685bcf777b2202a904cdcb49ad34c2fa1880c" +uuid = "0ac62f75-1d6f-5e53-bd7c-93b484bb37c0" +version = "0.14.0+4" + +[[libfdk_aac_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "7a5780a0d9c6864184b3a2eeeb833a0c871f00ab" +uuid = "f638f0a6-7fb0-5443-88ba-1cc74229b280" +version = "0.1.6+4" + +[[libpng_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Zlib_jll"] +git-tree-sha1 = "6abbc424248097d69c0c87ba50fcb0753f93e0ee" +uuid = "b53b4c65-9356-5827-b1ea-8c7a1a84506f" +version = "1.6.37+6" + +[[libvorbis_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Ogg_jll", "Pkg"] +git-tree-sha1 = "fa14ac25af7a4b8a7f61b287a124df7aab601bcd" +uuid = "f27f6e37-5d2b-51aa-960f-b287f2bc3b7a" +version = "1.3.6+6" + +[[p7zip_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "ee65cfa19bea645698a0224bfa216f2b1c8b559f" +uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" +version = "16.2.0+3" + +[[x264_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "d713c1ce4deac133e3334ee12f4adff07f81778f" +uuid = "1270edf5-f2f9-52d2-97e9-ab00b5d0237a" +version = "2020.7.14+2" + +[[x265_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "487da2f8f2f0c8ee0e83f39d13037d6bbf0a45ab" +uuid = "dfaa095f-4041-5dcd-9319-2fabd8486b76" +version = "3.0.0+3" diff --git a/docs/Project.toml b/docs/Project.toml index 101c2b6..4093589 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -1,5 +1,15 @@ [deps] +AbstractPlotting = "537997a7-5e4e-5d89-9595-2241ea00577e" +CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" +DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" +JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +PalmerPenguins = "8b842266-38fa-440a-9b57-31493939ab85" [compat] +AbstractPlotting = "0.15" +CairoMakie = "0.3.10" +DataFrames = "0.22" Documenter = "0.26" +JSON = "0.21" +PalmerPenguins = "0.1" diff --git a/docs/make.jl b/docs/make.jl index 403d381..e506579 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -1,5 +1,19 @@ -using CalibrationErrors using Documenter +import JSON + +if haskey(ENV, "GITHUB_ACTIONS") + # Print `@debug` statements (https://github.com/JuliaDocs/Documenter.jl/issues/955) + ENV["JULIA_DEBUG"] = "Documenter" + # Bypass the accept download prompt + ENV["DATADEPS_ALWAYS_ACCEPT"] = "true" +end + +using CalibrationErrors + +# Avoid font caching warning in docs +using CairoMakie +CairoMakie.activate!() +scatter(rand(10), rand(10)) makedocs(; modules = [CalibrationErrors], @@ -12,11 +26,50 @@ makedocs(; assets=String[], ), pages = [ - "Home" => "index.md", + "index.md", + "introduction.md", + "ece.md", + "kce.md", ], ) +# Obtain PR from commit message +function pr_from_commit() + # Check if action is caused by bors on the "trying" branch + get(ENV, "GITHUB_ACTOR", "") == "bors[bot]" || return + get(ENV, "GITHUB_REF", "") == "refs/heads/trying" || return + + # Parse event payload + event_path = get(ENV, "GITHUB_EVENT_PATH", nothing) + event_path === nothing && return + event = JSON.parsefile(event_path) + if haskey(event, "head_commit") && haskey(event["head_commit"], "message") + m = match(r"^Try #(.*):$", event["head_commit"]["message"]) + m === nothing && return + return m.captures[1] + end + + return +end + +# Enable preview on the trying branch +function deploy_config() + if haskey(ENV, "GITHUB_ACTIONS") + repo = get(ENV, "GITHUB_REPOSITORY", "") + if repo == "devmotion/CalibrationErrors.jl" && (pr = pr_from_commit()) !== nothing + return Documenter.GitHubActions( + "devmotion/CalibrationErrors.jl", + "pull_request", + "refs/pull/$pr/merge", + ) + end + end + + return Documenter.auto_detect_deploy_system() +end + deploydocs(; repo = "github.com/devmotion/CalibrationErrors.jl.git", + deploy_config = deploy_config(), push_preview = true, ) diff --git a/docs/src/background.md b/docs/src/background.md deleted file mode 100644 index de2f393..0000000 --- a/docs/src/background.md +++ /dev/null @@ -1,38 +0,0 @@ -# Probabilistic setting - -## Probabilistic model - -A probabilistic model predicts a probability distribution of possible outputs -for a given input. - -A very simple probabilistic model is a model that predicts a uniform -distribution for a dice roll; there is no input and the possible outputs are -the numbers $1,2,3,4,5,6$. A probably more complicated probabilistic model -would be a model that predicts the distribution of stock price changes from -the stock prices of last week; here the input are the stock prices of last week -and the possible outputs are formed by the set of possible stock price changes. - -If we assume that for each input there exists an underlying -unknown probability distribution that captures the relation between the given -input and the possible outputs, we can aim for a model whose predicted -distributions are, in some sense, close to these unknown probability distributions. - -More mathematically, if we view inputs and outputs as random variables $X$ and $Y$ -on some probability space, we can strive for a model $g$ that satisfies -```math - g(X) \approx \mu_{Y|X}(\cdot|X), -``` -where $\mu_{Y|X}$ is a version of the conditional distribution of $Y$ given $X$. - -Closeness of $g(X)$ and $\mu_{Y|X}(\cdot|X)$ can be measured with distance measures -of probability distributions. Two prominent families of distances are -[$\phi$-divergences](https://en.wikipedia.org/wiki/F-divergence) -and [integral probability metrics](https://arxiv.org/pdf/0901.2698.pdf). - -## Classification model - -Here we restrict ourselves to classification models, i.e., models for which output -$Y$ takes only values from a finite set. - -The dice roll model above is a classification model, whereas the model that -predicts stock price changes is not. diff --git a/docs/src/calibration.md b/docs/src/calibration.md deleted file mode 100644 index b272aa7..0000000 --- a/docs/src/calibration.md +++ /dev/null @@ -1,146 +0,0 @@ -# Calibration - -## Motivation - -Ideally one would like to have a model that predicts the underlying probability -distribution for almost every input, i.e., a model $g$ such that almost always -```math - g(X) = \mu_{Y|X}(\cdot|X). -``` -Unfortunately, if we try to infer the model from a finite data set of inputs and -outputs that is usually not possible. - -In safety-critical applications such as medical decision-making or autonomous -driving, however, important decisions are based on the predictions of a model. -Since we are not able to obtain the perfect model, the model has to satisfy -other properties such that it is deemed trustworthy. - -## Definition - -One such property is calibration, which is also called reliability. Loosely speaking, -if we repeatedly predict the same distribution for different pairs of inputs and -outputs, we would like that in the long run the empirical distribution of the -observed outputs is similar to the predicted probability distribution. This property -guarantees that the predicted distribution is not only an arbitrary probability -distribution but actually makes sense from a frequentist point of view. - -A [classic example from the literature](https://www.jstor.org/stable/2987588) is a -weather forecaster who each morning predicts the probability that it will rain during -the day. If we assume that the forecaster's predictions are observed for a long time, -the forecaster is called calibrated "if among those days for which his prediction -is $x$, the long-run relative frequency of rain is also $x$". - -### Common notion - -Commonly (see, e.g, -[Guo et al. (2017)](http://proceedings.mlr.press/v70/guo17a/guo17a.pdf)), only -calibration of the most-confident predictions $\max_y g_y(x)$ of a model $g$ is -considered. According to this common notion a model is calibrated if almost -always -```math - \mathbb{P}[Y = \textrm{arg} \, \max_y g_y(X) \,|\, \max_y g_y(X)] = \max_y g_y(X). -``` - -### Strong notion - -According to the more general definition by -[Bröcker (2009)](https://rmets.onlinelibrary.wiley.com/doi/pdf/10.1002/qj.456) -and [Vaicenavicius et al. (2019)](http://proceedings.mlr.press/v89/vaicenavicius19a/vaicenavicius19a.pdf), -a probabilistic model $g$ is calibrated if almost always -```math - \mathbb{P}[Y = y \,|\, g(X)] = g_y(X) -``` -for all classes $y$. - -For classification problems with more than two classes, this definition of -calibration is stronger than the more common one above. By reducing the model -and applying the strong notion to the simplified model, however, this definition -still allows to investigate the calibration of the model with respect to only -certain aspects of interest such as the calibration of the most-confident -predictions. - -Thus in this Julia package and its documentation, we always refer to the strong -notion of calibration. - -Let $y_1, \ldots, y_m$ be the possible outputs. Then we can also define -calibration in a vectorized form. Equivalently to the definition above, a model -$g$ is calibrated if and only if -```math - r(g(X)) - g(X) = 0 -``` -holds almost always, where -```math - r(\xi) := (\mathbb{P}[Y = y_1 \,|\, g(X) = \xi], \ldots, \mathbb{P}[Y = y_m \,|\, g(X) = \xi]) -``` -denotes the so-called calibration function. - -## Measures - -Calibration measures allow a more fine-tuned analysis of calibration and enable -comparisons of calibration of different models. Intuitively, calibration -measures quantify the deviation of the left and right hand side in the -definitions above. - -### Expected calibration error (ECE) - -The most common calibration measure is the so-called expected calibration error -(ECE) (see, e.g., -[Guo et al. (2017)](http://proceedings.mlr.press/v70/guo17a/guo17a.pdf)). -Informally, it is defined as the average distance between the left and right -hand side of the definition above with respect to some metric. Mathematically, -the expected calibration of model $g$ with respect to distance measure $d$ is -defined as -```math - \mathrm{ECE}[d, g] := \mathbb{E}[d(r(g(X)), g(X))]. -``` -Here $d$ could be, e.g., the cityblock distance, the total variation distance, -or the squared Euclidean distance. - -If $d(p, q) = 0$ if and only if $p = q$, then the ECE of model $g$ with respect -to distance measure $d$ is zero if and only if $g$ is calibrated. - -### Calibration error (CE) - -More generally, Widmann et al. (2019) define the calibration error (CE) of -a model $g$ with respect to a function class $\mathcal{F} \subset \{f \colon -\Delta^m \to \mathbb{R}^m\}$ as -```math - \mathrm{CE}[\mathcal{F}, g] := \sup_{f \in \mathcal{F}} \mathbb{E}[(r(g(X)) - g(X))^\intercal f(g(X))]. -``` - -If model $g$ is calibrated, then the CE is zero, regardless of the choice of -$\mathcal{F}$. However, for some function spaces (e.g., for -$\mathcal{F} = \{0\}$) the CE is zero even if $g$ is not calibrated. - -Interestingly, the ECE with respect to the cityblock distance, the total -variation distance, and the squared Euclidean distance are all special cases -of the CE (Widmann et al. (2019)). - -### Kernel calibration error (KCE) - -The kernel calibration error (KCE) is another special case of the CE, in which -the unit ball of a reproducing kernel Hilbert space (RKHS) of vector-valued -functions is chosen as function space $\mathcal{F}$. - -A RKHS of vector-valued functions $f \colon \Delta^m \to \mathbb{R}^m$ can be -identified with a unique matrix-valued kernel $k \colon \Delta^m \times -\Delta^m \to \mathbb{R}^{m \times m}$. Then the KCE of a model $g$ with respect -to kernel $k$ is defined as -```math - \mathrm{KCE}[k, g] := \mathrm{CE}[\mathcal{F}, g], -``` -where $\mathcal{F}$ is the unit ball of the RKHS corresponding to kernel $k$. - -As Widmann et al. (2019) show, for a large class of kernels (so-called universal -kernels) the KCE is zero if and only if the model $g$ is calibrated. Moreover, -the KCE can be formulated in terms of the kernel $k$ as -```math - \mathrm{KCE}[k, g] := {\left(\mathbb{E}[(e_Y - g(X))^{\intercal} k(g(X), g(X')) (e_{Y'} - g(X'))]\right)}^{1/2}, -``` -where $(X',Y')$ is an independent copy of $(X,Y)$ and $e_i$ denotes the $i$th -unit vector. - -The so-called maximum mean calibration error (MMCE), proposed by -[Kumar et al. (2018)](http://proceedings.mlr.press/v80/kumar18a/kumar18a.pdf), -can be viewed as a special case of the KCE, in which only the most-confident -predictions are considered (Widmann et al. (2019)). diff --git a/docs/src/ece.md b/docs/src/ece.md new file mode 100644 index 0000000..453313d --- /dev/null +++ b/docs/src/ece.md @@ -0,0 +1,49 @@ +# [Expected calibration error (ECE)](@id ece) + +## Definition + +A common calibration measure is the so-called expected calibration error (ECE). +In its most general form (see [Widmann et al.](https://openreview.net/pdf?id=-bxf89v3Nx)), +the ECE with respect to distance measure $d(p, p')$ is defined as +```math +\mathrm{ECE}_d := \mathbb{E} d\big(P_X, \mathcal{L}(Y \,|\, P_X)\big). +``` + +As implied by its name, the ECE is the expected distance between the left and +right hand side of the calibration definition with respect to $d$. + +Usually, the ECE is used to analyze classification models (see, e.g., +[Guo et al. (2017)](http://proceedings.mlr.press/v70/guo17a/guo17a.pdf) and +[Vaicenavicius et al. (2019)](http://proceedings.mlr.press/v89/vaicenavicius19a/vaicenavicius19a.pdf)). +In this case, $P_X$ and $\mathcal{L}(Y \,|\, P_X)$ can be identified with vectors +in the probability simplex and $d$ can be chosen as a the cityblock distance, +the total variation distance, or the squared Euclidean distance. + +For other probabilistic predictive models such as regression models, one has to +choose a more general distance measure $d$ between probability distributions on the +target space since the conditional distributions $\mathcal{L}(Y \,|\, P_X)$ can be +arbitrarily complex in general. + +## Estimators + +The main challenge in the estimation of the ECE is the estimation of the conditional +distribution $\mathcal{L}(Y \,|\, P_X)$ from a finite data set of predictions and +corresponding targets. Typically, predictions are binned and empirical estimates of +the conditional distributions are calculated for each bin. You can construct such +estimators with [`ECE`](@ref). + +```@docs +ECE +``` + +### Binning algorithms + +Currently, two binning algorithms are supported. [`UniformBinning`](@ref) is a binning +schemes with bins of fixed bins of uniform size whereas [`MedianVarianceBinning`](@ref) +splits the validation data set of predictions and targets dynamically to reduce the +variance of the predictions. + +```@docs +UniformBinning +MedianVarianceBinning +``` diff --git a/docs/src/estimators.md b/docs/src/estimators.md deleted file mode 100644 index 08bcbe8..0000000 --- a/docs/src/estimators.md +++ /dev/null @@ -1 +0,0 @@ -# Calibration error estimators diff --git a/docs/src/index.md b/docs/src/index.md index 3e8079c..930df28 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -3,10 +3,3 @@ *Estimation of calibration errors.* A package for estimating calibration errors from predictions and targets. - -```@index -``` - -```@autodocs -Modules = [CalibrationErrors] -``` diff --git a/docs/src/introduction.md b/docs/src/introduction.md new file mode 100644 index 0000000..8bd4ee0 --- /dev/null +++ b/docs/src/introduction.md @@ -0,0 +1,129 @@ +# Introduction + +## Probabilistic predictive models + +A probabilistic predictive model predicts a probability distribution over a set +of targets for a given feature. By predicting a distribution, one can express +the uncertainty in the prediction, which might be inherent to the prediction +task (e.g., if the feature does not contain enough information to determine the +target with absolute certainty) or caused by insufficient knowledge of the +underlying relation between feature and target (e.g., if only a small number +of observations of features and corresponding targets are available).[^1] + +The following plot shows measurements of the bill length and the flipper length +for three different penguin species in the +[Palmer penguin dataset](https://github.com/allisonhorst/palmerpenguins). + +```@setup palmerpenguins +using CairoMakie +using DataFrames +using PalmerPenguins +using AbstractPlotting.ColorSchemes + +penguins = dropmissing(DataFrame(PalmerPenguins.load())) + +f = Figure() +f[1,1] = Axis(f; xlabel = "bill length [mm]", ylabel = "flipper length [mm]") +scs = map(enumerate(groupby(penguins, :species))) do (i, df) + scatter!( + df[!, :bill_length_mm], df[!, :flipper_length_mm]; + color = (ColorSchemes.Dark2_8[i], 0.8), + ) +end +f[1,2] = Legend(f, scs, string.(levels(penguins.species)), "species") +AbstractPlotting.save("penguins.svg", f) +``` + +![](penguins.svg) + +There exist many different probabilistic predictive models for predicting the +probability of the penguin species (*target*) given the bill and flipper length +(*feature*). We denote the feature by $X$ and the target by $Y$, and let +$P_X$ be the prediction of a specific model $P$ for a feature $X$. Ideally, we would +like that +```math +P_X = \mathcal{L}(Y \,|\, X) \qquad \text{almost surely}, +``` +i.e., the model should predict the law of target $Y$ given features $X$.[^2] Of course, +usually it is not possible to achieve this in practice. + +A very simple class of models are models that yield the same prediction for all features, +i.e., they return the same probabilities for the penguin species regardless of the bill and +flipper length. Clearly, more complicated take into account also the features and might output +different predictions for different features. + +In contrast to probabilistic predictive models, non-probabilistic predictive models +predict a single target instead of a distribution over targets. In fact, such models can be +viewed as a special class of probabilistic predictive models that output only Dirac +distributions, i.e., that always predict 100% probability for one penguin species and +0% probability for all others. + +Some other prediction models output a single target together with a confidence score +between 0 and 1. Even these models can be reformulated as probabilistic predictive models, +arguably in a slightly unconventional way: they correspond to a probabilistic model for a +a binary classification problem whose feature space is extended with the predicted target +and whose target is the predicted confidence score. + +[^1]: It does not matter how the model is obtained. In particular, both Bayesian and frequentist approaches can be used. + +[^2]: In classification problems, the law $\mathcal{L}(Y \,|\, X)$ can be identified with a vector in the probability simplex. Therefore often we just consider this equivalent formulation, both for the predictions $P_X$ and the law $\mathcal{L}(Y \,|\, X)$. + +## Calibration + +The main motivation for using a probabilistic model is that it provides additional +information about the uncertainty of the predictions, which is valuable for decision +making. A [classic example are weather forecasts](https://www.jstor.org/stable/2987588) +that also report the "probability of rain" instead of only if it will rain or not. + +Therefore it is not sufficient if the model predicts an arbitrary distribution. +Instead the predictions should actualy express the involved uncertainties "correctly". +One desired property is that the predictions are consistent: if the forecasts predict +an 80% probability of rain for an infinite sequence of days, then ideally on 80% of the +days it rains. + +More generally, mathematically we would like +```math +P_X = \mathcal{L}(Y \,|\, P_X) \quad \text{almost surely}, +``` +i.e., the predicted distribution of targets should be equal to the distribution of targets +conditioned on the predicted distribution.[^3] + +This statistical property is called *calibration*. If it is satisfied, a model is +*calibrated*. + +Obviously, the ideal model $P_X = \mathcal{L}(Y \,|\, X)$ is calibrated. However, +also the naive model $P_X = \mathcal{L}(Y)$ that always predicts the marginal +distribution of $Y$ independent of the features is calibrated.[^4] In fact, any model of +the form +```math +P_X = \mathcal{L}(Y \,|\, \phi(X)) \quad \text{almost surely}, +``` +where $\phi$ is some measurable function, is calibrated. + +[^3]: The general formulation applies not only to classification but to any prediction task with arbitrary target spaces, including regression. + +[^4]: In meterology, this model is called the *climatology*. + +## Calibration error + +Calibration errors such as the [expected calibration error](@ref ece) and the +[kernel calibration error](@ref kce) measure the calibration, or rather the +degree of miscalibration, of probabilistic predictive models. They allow a more +fine-tuned analysis of calibration and enable comparisons of calibration of +different models. Intuitively, calibration measures quantify the deviation of +$P_X$ and $\mathcal{L}(Y \,|\, P_X)$, i.e., the left and right hand side in +the calibration definition. + +### Estimation + +The calibration error of a model depends on the true conditional distribution of +targets which is unknown in practice. Therefore calibration errors have to be +estimated from a validation data set. + +Various estimators of different calibration errors such as the +[expected calibration error](@ref ece) and the [kernel calibration error](@ref kce) +can be evaluated with [`calibrationerror`](@ref). + +```@docs +calibrationerror +``` diff --git a/docs/src/kce.md b/docs/src/kce.md new file mode 100644 index 0000000..45e3274 --- /dev/null +++ b/docs/src/kce.md @@ -0,0 +1,86 @@ +# [Kernel calibration error (KCE)](@id kce) + +## Definition + +The kernel calibration error (KCE) is another calibration error. The KCE with +respect to a real-valued kernel +$k \colon (\mathcal{P} \times \mathcal{Y}) \times (\mathcal{P} \times \mathcal{Y}) \to \mathbb{R}$ +on the product space $\mathcal{P} \times \mathcal{Y}$ of predictions and targets +is defined as +as +```math +\mathrm{KCE}_k := \sup_{f \in \mathcal{B}_k} \bigg| \mathbb{E}_{Y,P_X} f(P_X, Y) - \mathbb{E}_{Z_X,P_X} f(P_X, Z_X)\bigg|, +``` +where $\mathcal{B}_{k}$ is the unit ball in the +[reproducing kernel Hilbert space (RKHS)](https://en.wikipedia.org/wiki/Reproducing_kernel_Hilbert_space) +of $k$, consisting of functions $f \colon \mathcal{P} \times \mathcal{Y} \to \mathbb{R}$, +and $Z_X$ is a random variable on the target space $\mathcal{Y}$ with conditional distribution +$P_X$ given $P_X$, i.e., $Z_X \,|\, P_X = \mu \sim \mu$ +([Widmann et al., 2021](https://openreview.net/pdf?id=-bxf89v3Nx)).[^1] + +For classification models with $m$ classes, there exists an equivalent formulation of the +KCE based on matrix-valued kernel +$\tilde{k} \colon \mathcal{P} \times \mathcal{P} \to \mathbb{R}^{m \times m}$ on +the space $\mathcal{P}$ of predictions +([Widmann et al., 2019](https://proceedings.neurips.cc/paper/2019/file/1c336b8080f82bcc2cd2499b4c57261d-Paper.pdf)). The definition above can be rewritten as +```math +\mathrm{KCE}_{\tilde{k}} := \sup_{f \in \mathcal{B}_{\tilde{k}}} \bigg| \mathbb{E}_{P_X} \big(\mathcal{L}(Y \,|\, P_X) - P_X\big)^\mathsf{T} f(P_X) \bigg|, +``` +where the matrix-valued kernel $\tilde{k}$ is given by +```math +\tilde{k}_{i,j}(p, q) = k((p, i), (q, j)) \quad (i,j=1,\ldots,m), +``` +and $\mathcal{B}_{\tilde{k}}$ is the unit ball in the RKHS of $\tilde{k}$, consisting +of vector-valued functions $f \colon \mathcal{P} \to \mathbb{R}^m$.[^2] However, +this formulation applies only to classification models whereas the general +definition above covers all probabilistic predictive models. + +For a large class of kernels the KCE is zero if and only if the model is +calibrated ([Widmann et al., 2021](https://openreview.net/pdf?id=-bxf89v3Nx)). +Moreover, the squared KCE (SKCE) can be formulated in terms of the kernel $k$ as +```math +\begin{aligned} +\mathrm{SKCE}_{k} := \mathrm{KCE}_k^2 &= \int k(u, v) \, \big(\mathcal{L}(P_X, Y) - \mathcal{L}(P_X, Z_X)\big)(u) \big(\mathcal{L}(P_X, Y) - \mathcal{L}(P_X, Z_X)\big)(v) \\ +&= \mathbb{E} h_k\big((P_X, Y), (P_{X'}, Y')\big), +\end{aligned} +``` +where $(X',Y')$ is an independent copy of $(X,Y)$ and +```math +\begin{aligned} +h_k\big((\mu, y), (\mu', y')\big) :={}& k\big((\mu, y), (\mu', y')\big) - \mathbb{E}_{Z \sim \mu} k\big((\mu, Z), (\mu', y')\big) \\ +&- \mathbb{E}_{Z' \sim \mu'} k\big((\mu, y), (\mu', Z')\big) + \mathbb{E}_{Z \sim \mu, Z' \sim \mu'} k\big((\mu, Z), (\mu', Z')\big). +\end{aligned} +``` + +[^1]: The KCE is a special case of calibration errors of the form $\sup_{f \in \mathcal{F}} \big| \mathbb{E}_{Y,P_X} f(P_X, Y) - \mathbb{E}_{Z_X,P_X} f(P_X, Z_X)\big|$, where $\mathcal{F}$ is some space of functions $f \colon \mathcal{P} \times \mathcal{Y} \to \mathbb{R}$ ([Widmann et al., 2021](https://openreview.net/pdf?id=-bxf89v3Nx)). For classification models, the [ECE](@ref ece) with respect to common distances such as the total variation distance or the squared Euclidean distance can be formulated in this way ([Widmann et al., 2019](https://proceedings.neurips.cc/paper/2019/file/1c336b8080f82bcc2cd2499b4c57261d-Paper.pdf)). + +[^2]: The maximum mean calibration error (MMCE) ([Kumar et al., 2018](http://proceedings.mlr.press/v80/kumar18a/kumar18a.pdf)) can be viewed as a special case of the KCE, in which only the most-confident predictions are considered ([Widmann et al., 2019](https://proceedings.neurips.cc/paper/2019/file/1c336b8080f82bcc2cd2499b4c57261d-Paper.pdf)). + +## Estimators + +For the SKCE biased and unbiased estimators exist. In CalibrationErrors.jl +three types of estimators are available, namely [`BiasedSKCE`](@ref), +[`UnbiasedSKCE`](@ref), and [`BlockUnbiasedSKCE`](@ref). Unsurprisingly, +[`BiasedSKCE`](@ref) is a biased estimator whereas the other two +estimators are unbiased. [`BiasedSKCE`](@ref) and [`UnbiasedSKCE`](@ref) +have quadratic sample complexity whereas [`BlockUnbiasedSKCE`](@ref) +is an estimator with linear sample complexity. + +### Biased estimators + +```@docs +BiasedSKCE +BiasedSKCE(::Kernel, ::Kernel) +``` + +### Unbiased estimators + +```@docs +UnbiasedSKCE +UnbiasedSKCE(::Kernel, ::Kernel) +``` + +```@docs +BlockUnbiasedSKCE +BlockUnbiasedSKCE(::Kernel, ::Kernel) +``` diff --git a/src/binning/generic.jl b/src/binning/generic.jl index 184b776..f8360f1 100644 --- a/src/binning/generic.jl +++ b/src/binning/generic.jl @@ -81,15 +81,3 @@ function adddata!(bin::Bin, prediction::AbstractVector{<:Real}, target::Integer) nothing end - -# distance computations - -""" - evaluate(distance, bin::Bin) - -Evaluate the `distance` between the average prediction and the distribution of targets in -the `bin`. -""" -function Distances.evaluate(distance::SemiMetric, bin::Bin) - evaluate(distance, bin.mean_predictions, bin.proportions_targets) -end \ No newline at end of file diff --git a/src/binning/medianvariance.jl b/src/binning/medianvariance.jl index fc032a0..1e65db4 100644 --- a/src/binning/medianvariance.jl +++ b/src/binning/medianvariance.jl @@ -1,3 +1,14 @@ +""" + MedianVarianceBinning([minsize::Int = 10, maxbins::Int = typemax(Int)]) + +Dynamic binning scheme of the probability simplex with at most `maxbins` bins that each +contain at least `minsize` samples. + +The data set is split recursively as long as it is possible to split the bins while +satisfying these conditions. In each step, the bin with the maximum variance of predicted +probabilities for any component is selected and split at the median of the predicted +probability of the component with the largest variance. +""" struct MedianVarianceBinning <: AbstractBinningAlgorithm minsize::Int maxbins::Int diff --git a/src/binning/uniform.jl b/src/binning/uniform.jl index 9c710b3..58635a1 100644 --- a/src/binning/uniform.jl +++ b/src/binning/uniform.jl @@ -1,3 +1,9 @@ +""" + UniformBinning(nbins::Int) + +Binning scheme of the probability simplex with `nbins` bins of uniform width for +each component. +""" struct UniformBinning <: AbstractBinningAlgorithm nbins::Int diff --git a/src/ece.jl b/src/ece.jl index 4d53375..3890e06 100644 --- a/src/ece.jl +++ b/src/ece.jl @@ -8,8 +8,25 @@ end """ ECE(binning[, distance = TotalVariation()]) -Create an estimator of the expected calibration error (ECE) with the given `binning` -algorithm and `distance` function. +Estimator of the expected calibration error (ECE) for a classification model with +respect to the given `distance` function using the `binning` algorithm. + +For classification models, the predictions ``P_{X_i}`` and targets ``Y_i`` are identified +with vectors in the probability simplex. The estimator of the ECE is defined as +```math +\\frac{1}{B} \\sum_{i=1}^B d\\big(\\overline{P}_i, \\overline{Y}_i\\big), +``` +where ``B`` is the number of non-empty bins, ``d`` is the distance function, and +``\\overline{P}_i`` and ``\\overline{Y}_i`` are the average vector of the predictions and +the average vector of targets in the ``i``th bin. By default, the total variation distance +is used. + +The `distance` has to be a function of the form +```julia +distance(pbar::Vector{<:Real}, ybar::Vector{<:Real}). +``` +In particular, distance measures of the package +[Distances.jl](https://github.com/JuliaStats/Distances.jl) are supported. """ ECE(binning::AbstractBinningAlgorithm) = ECE(binning, TotalVariation()) @@ -33,7 +50,7 @@ function _calibrationerror(ece::ECE, predictions::AbstractVector{<:AbstractVecto # evaluate the distance in the first bin @inbounds begin bin = bins[1] - x = evaluate(distance, bin) + x = distance(bin.mean_predictions, bin.proportions_targets) # initialize the estimate estimate = x / 1 @@ -43,7 +60,7 @@ function _calibrationerror(ece::ECE, predictions::AbstractVector{<:AbstractVecto for i in 2:nbins # evaluate the distance bin = bins[i] - x = evaluate(distance, bin) + x = distance(bin.mean_predictions, bin.proportions_targets) # update the estimate m = bin.nsamples diff --git a/src/mkce/unnormalized.jl b/src/mkce/unnormalized.jl new file mode 100644 index 0000000..17e7896 --- /dev/null +++ b/src/mkce/unnormalized.jl @@ -0,0 +1,78 @@ +struct UnnormalizedMKCE{K<:MatrixKernel,V<:AbstractArray} <: CalibrationErrorEstimator + kernel::K + witness::V +end + +function _calibrationerror(estimator::UnnormalizedMKCE, predictions::AbstractMatrix{<:Real}, + labels::AbstractVector{<:Integer}) + @unpack kernel, witness = estimator + + # obtain number of samples + nclasses, nsamples = size(predictions) + nwitness = length(witness) + + T = float(eltype(predictions)) + cache = Vector{T}(undef, nclasses) + meancache = Vector{T}(undef, nclasses) + + s = zero(T) + for i in 1:nwitness + v = witness[i] + fill!(meancache, zero(T)) + + for j in 1:nsamples + mkce_kernel!(cache, kernel, v, view(predictions, :, j), labels[j]) + meancache .+= cache + end + meancache ./= nsamples + + s += sum(abs2, meancache) + end + + s / nwitness +end + +""" + mkce_kernel!(out, k, v, p, y) + +Evaluate kernel function +```math +k(v, p) (p - e_y) +``` +of the mean kernel calibration error for the matrix-valued kernel `k`, witness point `v` +and prediction `p` with corresponding label `y`. + +This method assumes that `v`, `p`, and `y` are valid and specified correctly, and +does not perform any checks. +""" +function mkce_kernel!(out, kernel::MatrixKernel, v::AbstractVector{<:Real}, + p::AbstractVector{<:Real}, y::Integer) + copyto!(out, p) + @inbounds out[y] -= 1 + lmul!(kernel(v, p), out) + + out +end + +function mkce_kernel!(out, kernel::UniformScalingKernel, v::AbstractVector{<:Real}, + p::AbstractVector{<:Real}, y::Integer) + @unpack λ = kernel + + copyto!(out, p) + @inbounds out[y] -= 1 + lmul!(λ * kernel.kernel(v, p), out) + + out +end + +function mkce_kernel!(out, kernel::DiagonalKernel, v::AbstractVector{<:Real}, + p::AbstractVector{<:Real}, y::Integer) + @unpack diag = kernel + + copyto!(out, p) + @inbounds out[y] -= 1 + k = kernel.kernel(v, p) + @. out = k * diag + + out +end \ No newline at end of file diff --git a/src/skce/biased.jl b/src/skce/biased.jl index 29cd751..34dd65a 100644 --- a/src/skce/biased.jl +++ b/src/skce/biased.jl @@ -1,8 +1,57 @@ +@doc raw""" + BiasedSKCE(k) + +Biased estimator of the squared kernel calibration error (SKCE) with kernel `k`. + +Kernel `k` on the product space of predictions and targets has to be a `Kernel` from the +Julia package +[KernelFunctions.jl](https://github.com/JuliaGaussianProcesses/KernelFunctions.jl) +that can be evaluated for inputs that are tuples of predictions and targets. + +# Details + +The estimator is biased and guaranteed to be non-negative. Its sample complexity is +``O(n^2)``, where ``n`` is the total number of samples. + +Let ``(P_{X_i}, Y_i)_{i=1,\ldots,n}`` be a data set of predictions and corresponding +targets. The estimator is defined as +```math +\frac{1}{n^2} \sum_{i,j=1}^n h_k\big((P_{X_i}, Y_i), (P_{X_j}, Y_j)\big) +``` +where +```math +\begin{aligned} +h_k\big((μ, y), (μ', y')\big) ={}& k\big((μ, y), (μ', y')\big) + - 𝔼_{Z ∼ μ} k\big((μ, Z), (μ', y')\big) \\ + & - 𝔼_{Z' ∼ μ'} k\big((μ, y), (μ', Z')\big) + + 𝔼_{Z ∼ μ, Z' ∼ μ'} k\big((μ, Z), (μ', Z')\big). +\end{aligned} +``` + +# References + +Widmann, D., Lindsten, F., & Zachariah, D. (2019). [Calibration tests in multi-class +classification: A unifying framework](https://proceedings.neurips.cc/paper/2019/hash/1c336b8080f82bcc2cd2499b4c57261d-Abstract.html). +In: Advances in Neural Information Processing Systems (NeurIPS 2019) (pp. 12257–12267). + +Widmann, D., Lindsten, F., & Zachariah, D. (2021). [Calibration tests beyond +classification](https://openreview.net/forum?id=-bxf89v3Nx). + +See also: [`UnbiasedSKCE`](@ref), [`BlockUnbiasedSKCE`](@ref) +""" struct BiasedSKCE{K<:Kernel} <: SKCE """Kernel of estimator.""" kernel::K end +""" + BiasedSKCE(k₁, k₂) + +Biased estimator of the squared kernel calibration error (SKCE) with a tensor product +kernel ``k = k_1 \\otimes k_2``. + +See also: [`BiasedSKCE`](@ref) +""" BiasedSKCE(kernel1::Kernel, kernel2::Kernel) = BiasedSKCE(TensorProduct(kernel1, kernel2)) function _calibrationerror(skce::BiasedSKCE, diff --git a/src/skce/unbiased.jl b/src/skce/unbiased.jl index 19ae637..8e223e9 100644 --- a/src/skce/unbiased.jl +++ b/src/skce/unbiased.jl @@ -1,8 +1,57 @@ +@doc raw""" + UnbiasedSKCE(k) + +Unbiased estimator of the squared kernel calibration error (SKCE) with kernel `k`. + +Kernel `k` on the product space of predictions and targets has to be a `Kernel` from the +Julia package +[KernelFunctions.jl](https://github.com/JuliaGaussianProcesses/KernelFunctions.jl) +that can be evaluated for inputs that are tuples of predictions and targets. + +# Details + +The estimator is unbiased and not guaranteed to be non-negative. Its sample complexity +is ``O(n^2)``, where ``n`` is the total number of samples. + +Let ``(P_{X_i}, Y_i)_{i=1,\ldots,n}`` be a data set of predictions and corresponding +targets. The estimator is defined as +```math +\frac{2}{n(n-1)} \sum_{1 \leq i < j \leq n} h_k\big((P_{X_i}, Y_i), (P_{X_j}, Y_j)\big), +``` +where +```math +\begin{aligned} +h_k\big((μ, y), (μ', y')\big) ={}& k\big((μ, y), (μ', y')\big) + - 𝔼_{Z ∼ μ} k\big((μ, Z), (μ', y')\big) \\ + & - 𝔼_{Z' ∼ μ'} k\big((μ, y), (μ', Z')\big) + + 𝔼_{Z ∼ μ, Z' ∼ μ'} k\big((μ, Z), (μ', Z')\big). +\end{aligned} +``` + +# References + +Widmann, D., Lindsten, F., & Zachariah, D. (2019). [Calibration tests in multi-class +classification: A unifying framework](https://proceedings.neurips.cc/paper/2019/hash/1c336b8080f82bcc2cd2499b4c57261d-Abstract.html). +In: Advances in Neural Information Processing Systems (NeurIPS 2019) (pp. 12257–12267). + +Widmann, D., Lindsten, F., & Zachariah, D. (2021). [Calibration tests beyond +classification](https://openreview.net/forum?id=-bxf89v3Nx). + +See also: [`BiasedSKCE`](@ref), [`BlockUnbiasedSKCE`](@ref) +""" struct UnbiasedSKCE{K<:Kernel} <: SKCE """Kernel of estimator.""" kernel::K end +""" + UnbiasedSKCE(k₁, k₂) + +Unbiased estimator of the squared kernel calibration error (SKCE) with a tensor product +kernel ``k = k_1 \\otimes k_2``. + +See also: [`UnbiasedSKCE`](@ref) +""" function UnbiasedSKCE(kernel1::Kernel, kernel2::Kernel) return UnbiasedSKCE(TensorProduct(kernel1, kernel2)) end @@ -24,6 +73,52 @@ function _calibrationerror( return estimate end +@doc raw""" + BlockUnbiasedSKCE(k[, blocksize = 2]) + +Unbiased estimator of the squared kernel calibration error (SKCE) with kernel `k` that +uses blocks with `blocksize` samples. + +Kernel `k` on the product space of predictions and targets has to be a `Kernel` from the +Julia package +[KernelFunctions.jl](https://github.com/JuliaGaussianProcesses/KernelFunctions.jl) +that can be evaluated for inputs that are tuples of predictions and targets. + +The number of samples per block must be at least 2 and at most the total number of samples. +Note that samples in the last block are discarded if it is incomplete (see details below). + +# Details + +The estimator is unbiased and not guaranteed to be non-negative. Its sample complexity +is ``O(Bn)``, where ``B`` is the block size and ``n`` is the total number of samples. + +Let ``(P_{X_i}, Y_i)_{i=1,\ldots,n}`` be a data set of predictions and corresponding +targets. The estimator with block size ``B`` is defined as +```math +{\bigg\lfloor \frac{n}{B} \bigg\rfloor}^{-1} \sum_{b=1}^{\lfloor n/B \rfloor} +\frac{2}{B(B-1)} \sum_{(b - 1) B < i < j \leq bB} h_k\big((P_{X_i}, Y_i), (P_{X_j}, Y_j)\big), +``` +where +```math +\begin{aligned} +h_k\big((μ, y), (μ', y')\big) ={}& k\big((μ, y), (μ', y')\big) + - 𝔼_{Z ∼ μ} k\big((μ, Z), (μ', y')\big) \\ + & - 𝔼_{Z' ∼ μ'} k\big((μ, y), (μ', Z')\big) + + 𝔼_{Z ∼ μ, Z' ∼ μ'} k\big((μ, Z), (μ', Z')\big). +\end{aligned} +``` + +# References + +Widmann, D., Lindsten, F., & Zachariah, D. (2019). [Calibration tests in multi-class +classification: A unifying framework](https://proceedings.neurips.cc/paper/2019/hash/1c336b8080f82bcc2cd2499b4c57261d-Abstract.html). +In: Advances in Neural Information Processing Systems (NeurIPS 2019) (pp. 12257–12267). + +Widmann, D., Lindsten, F., & Zachariah, D. (2021). [Calibration tests beyond +classification](https://openreview.net/forum?id=-bxf89v3Nx). + +See also: [`BiasedSKCE`](@ref), [`UnbiasedSKCE`](@ref) +""" struct BlockUnbiasedSKCE{K<:Kernel} <: SKCE """Kernel of estimator.""" kernel::K @@ -36,6 +131,14 @@ struct BlockUnbiasedSKCE{K<:Kernel} <: SKCE end end +""" + BlockUnbiasedSKCE(k₁, k₂) + +Unbiased estimator of the squared kernel calibration error (SKCE) with a tensor product +kernel ``k = k_1 \\otimes k_2`` that uses blocks with `blocksize` samples. + +See also: [`BlockUnbiasedSKCE`](@ref) +""" function BlockUnbiasedSKCE(kernel1::Kernel, kernel2::Kernel, blocksize::Int = 2) return BlockUnbiasedSKCE(TensorProduct(kernel1, kernel2), blocksize) end diff --git a/test/binning/generic.jl b/test/binning/generic.jl index 3e7dd03..4337957 100644 --- a/test/binning/generic.jl +++ b/test/binning/generic.jl @@ -24,12 +24,6 @@ Random.seed!(1234) @test sum(bin.mean_predictions) ≈ 1 @test sum(bin.proportions_targets) ≈ 1 - # check distance calculations - for distance in (TotalVariation(), Cityblock(), Euclidean(), SqEuclidean()) - @test evaluate(distance, bin) == evaluate(distance, mean(predictions), - proportions(targets, nclasses)) - end - # compare with adding data bin2 = Bin(predictions[1], targets[1]) for i in 2:nsamples