Merge bcac074 into 2361c93

theogf · Mar 29, 2021 · 8ee966e · 8ee966e
2 parents 2361c93 + bcac074
commit 8ee966e
Show file tree

Hide file tree

Showing 70 changed files with 904 additions and 1,485 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -14,7 +14,7 @@ jobs:
       matrix:
         version:
           - '1'
-          - '1.3'
+          - '1.5'
           - 'nightly'
         os:
           - ubuntu-latest

diff --git a/Project.toml b/Project.toml
@@ -5,6 +5,7 @@ version = "0.9.4"
 
 [deps]
 AdvancedHMC = "0bf59076-c3b1-5ca4-86bd-e02cd72cde3d"
+ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
 Clustering = "aaaa29a8-35af-508c-8bc3-b662a17a0fe5"
 DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
@@ -14,11 +15,10 @@ Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
 FastGaussQuadrature = "442a2c76-b920-505d-bb47-c5924d526838"
 Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c"
 ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210"
+InducingPoints = "b4bd816d-b975-4295-ac05-5f2992945579"
 KernelFunctions = "ec8451be-7e33-11e9-00cf-bbf324bd1392"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 MCMCChains = "c7f686f2-ff18-58e9-bc7b-31028e88f75d"
-MLDataUtils = "cc2ba9b6-d476-5e6d-8eaf-a92d5412d41d"
-PDMats = "90014a1f-27ba-587c-ab20-58faa44d9150"
 ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 RecipesBase = "3cdcf5f2-1ef4-517c-9805-6587b60abb01"
@@ -38,18 +38,16 @@ DeterminantalPointProcesses = "0.1.0"
 Distances = "0.8, 0.9, 0.10"
 Distributions = "0.21.5, 0.22, 0.23, 0.24"
 FastGaussQuadrature = "0.4"
-Flux = "0.10, 0.11"
+Flux = "0.10, 0.11, 0.12"
 ForwardDiff = "0.10"
-KernelFunctions = "0.5, 0.6, 0.7, 0.8"
+KernelFunctions = "0.8, 0.9"
 MCMCChains = "0.3.15, 2.0, 3.0, 4.0"
-MLDataUtils = "0.5"
-PDMats = "0.10, 0.11"
 ProgressMeter = "1"
 RecipesBase = "1.0, 1.1"
 Reexport = "0.2, 1"
 SimpleTraits = "0.9"
 SpecialFunctions = "0.9, 0.10, 1"
 StatsBase = "0.32, 0.33"
 StatsFuns = "0.8, 0.9"
-Zygote = "0.5, 0.6"
-julia = "1.3"
+Zygote = "0.6.7"
+julia = "1.5"
diff --git a/src/AugmentedGaussianProcesses.jl b/src/AugmentedGaussianProcesses.jl
@@ -4,102 +4,97 @@ General Framework for the data augmented Gaussian Processes
 
 """
 module AugmentedGaussianProcesses
-    const AGP = AugmentedGaussianProcesses; export AGP
-    export AbstractGP, GP, VGP, SVGP, VStP, MCGP, MOVGP, MOSVGP, MOARGP,OnlineSVGP
-    export Likelihood,  RegressionLikelihood, ClassificationLikelihood, MultiClassLikelihood
-    export GaussianLikelihood, StudentTLikelihood, LaplaceLikelihood, HeteroscedasticLikelihood
-    export LogisticLikelihood, BayesianSVM
-    export SoftMaxLikelihood, LogisticSoftMaxLikelihood
-    export PoissonLikelihood, NegBinomialLikelihood
-    export Inference, Analytic, AnalyticVI, AnalyticSVI, GibbsSampling, HMCSampling, MCIntegrationVI, MCIntegrationSVI, QuadratureVI, QuadratureSVI
-    export NumericalVI, NumericalSVI
-    export PriorMean, ZeroMean, ConstantMean, EmpiricalMean, AffineMean
-    #Useful functions
-    export train!, sample
-    export predict_f, predict_y, proba_y
-    export fstar, ELBO
-    export covariance, diag_covariance, prior_mean
-    export @augmodel
 
-    #General modules
-    using Reexport
-    using LinearAlgebra
-    using Random
-    @reexport using KernelFunctions
-    using KernelFunctions: ColVecs, RowVecs
-    using Zygote, ForwardDiff
-    using Flux: params, destructure
-    @reexport using Flux.Optimise
-    using PDMats: PDMat, invquad
-    using AdvancedHMC
-    using MCMCChains
-    using StatsBase
-    using StatsFuns
-    using SpecialFunctions
-    using Distributions
-    using FastGaussQuadrature: gausshermite
-    using ProgressMeter, SimpleTraits
-    #Exported modules
-    # export KMeansModule
-    export KMeansInducingPoints
+const AGP = AugmentedGaussianProcesses
+export AGP
+export AbstractGP, GP, VGP, SVGP, VStP, MCGP, MOVGP, MOSVGP, MOARGP, OnlineSVGP # All models
+export Likelihood,  RegressionLikelihood, ClassificationLikelihood, MultiClassLikelihood, EventLikelihood # All categories of likelihoods
+export GaussianLikelihood, StudentTLikelihood, LaplaceLikelihood, HeteroscedasticLikelihood # Regression Likelihoods
+export LogisticLikelihood, BayesianSVM # Classification Likelihoods
+export SoftMaxLikelihood, LogisticSoftMaxLikelihood # Multiclass Classification Likelihoods
+export PoissonLikelihood, NegBinomialLikelihood # Event Likelihoods
+export Inference, Analytic, AnalyticVI, AnalyticSVI # Inference objects
+export GibbsSampling, HMCSampling # Sampling inference
+export NumericalVI, NumericalSVI, MCIntegrationVI, MCIntegrationSVI, QuadratureVI, QuadratureSVI # Numerical inference
+export PriorMean, ZeroMean, ConstantMean, EmpiricalMean, AffineMean # Prior means
+#Useful functions
+export train!, sample
+export predict_f, predict_y, proba_y
+export fstar
+export ELBO
+export covariance, diag_covariance, prior_mean
+export @augmodel
 
-    #Useful functions and module
-    include(joinpath("functions", "PGSampler.jl"))
-    include(joinpath("functions", "GIGSampler.jl"))
-    include(joinpath("functions", "lap_transf_dist.jl"))
-    #include("functions/PerturbativeCorrection.jl")
-    # include("functions/GPAnalysisTools.jl")
-    # include("functions/IO_model.jl")
-    #Custom modules
-    using .PGSampler
-    using .GIGSampler
+#General modules
+using Reexport
+using LinearAlgebra
+using Random
+@reexport using KernelFunctions
+using KernelFunctions: ColVecs, RowVecs
+using Zygote, ForwardDiff
+using ChainRulesCore: ChainRulesCore, NO_FIELDS, DoesNotExist
+using Flux: params, destructure
+@reexport using Flux.Optimise
+using AdvancedHMC
+using MCMCChains
+using StatsBase
+@reexport using InducingPoints
+using StatsFuns
+using SpecialFunctions
+using Distributions: 
+                Distributions, Distribution,
+                dim, cov, mean, var, 
+                pdf, logpdf, loglikelihood,
+                Normal, Poisson, NegativeBinomial, InverseGamma, Laplace, MvNormal, Gamma
+using FastGaussQuadrature: gausshermite
+using ProgressMeter, SimpleTraits
 
-    include(joinpath("inducingpoints" , "InducingPoints.jl"))
-    @reexport using .InducingPoints
+#Include custom module for additional distributions
+include(joinpath("ComplementaryDistributions", "ComplementaryDistributions.jl"))
+using .ComplementaryDistributions
 
-    # using .PerturbativeCorrection
-    # using .GPAnalysisTools
-    # using .IO_model
 
 
-    # Main classes
-    abstract type Inference{T<:Real} end
-    abstract type VariationalInference{T} <: Inference{T} end
-    abstract type SamplingInference{T} <: Inference{T} end
-    abstract type Likelihood{T<:Real} end
-    abstract type AbstractLatent{T<:Real,Tpr,Tpo} end
 
-    include(joinpath("mean", "priormean.jl"))
-    include(joinpath("data", "datacontainer.jl"))
-    include(joinpath("functions", "utils.jl"))
+# Main classes
+abstract type Inference{T<:Real} end
+abstract type VariationalInference{T} <: Inference{T} end
+abstract type SamplingInference{T} <: Inference{T} end
+abstract type Likelihood{T<:Real} end
+abstract type AbstractLatent{T<:Real,Tpr,Tpo} end
 
-    # Models
-    include(joinpath("models", "AbstractGP.jl"))
-    include(joinpath("gpblocks", "latentgp.jl"))
-    include(joinpath("models", "GP.jl"))
-    include(joinpath("models", "VGP.jl"))
-    include(joinpath("models", "MCGP.jl"))
-    include(joinpath("models", "SVGP.jl"))
-    include(joinpath("models", "VStP.jl"))
-    include(joinpath("models", "MOSVGP.jl"))
-    include(joinpath("models", "MOVGP.jl"))
-    include(joinpath("models", "OnlineSVGP.jl"))
-    include(joinpath("models", "single_output_utils.jl"))
-    include(joinpath("models", "multi_output_utils.jl"))
+include(joinpath("mean", "priormean.jl"))
+include(joinpath("data", "datacontainer.jl"))
+include(joinpath("functions", "utils.jl"))
 
-    include(joinpath("inference", "inference.jl"))
-    include(joinpath("likelihood", "likelihood.jl"))
+# Models
+include(joinpath("models", "AbstractGP.jl"))
+include(joinpath("gpblocks", "latentgp.jl"))
+include(joinpath("models", "GP.jl"))
+include(joinpath("models", "VGP.jl"))
+include(joinpath("models", "MCGP.jl"))
+include(joinpath("models", "SVGP.jl"))
+include(joinpath("models", "VStP.jl"))
+include(joinpath("models", "MOSVGP.jl"))
+include(joinpath("models", "MOVGP.jl"))
+include(joinpath("models", "OnlineSVGP.jl"))
+include(joinpath("models", "single_output_utils.jl"))
+include(joinpath("models", "multi_output_utils.jl"))
 
-    include(joinpath("likelihood", "generic_likelihood.jl"))
+include(joinpath("inference", "inference.jl"))
+include(joinpath("likelihood", "likelihood.jl"))
+include(joinpath("likelihood", "generic_likelihood.jl"))
 
-    include(joinpath("functions", "KLdivergences.jl"))
-    include(joinpath("data", "utils.jl"))
-    include(joinpath("functions", "plotting.jl"))
+include(joinpath("functions", "KLdivergences.jl"))
+include(joinpath("functions", "ELBO.jl"))
+include(joinpath("data", "utils.jl"))
+include(joinpath("functions", "plotting.jl"))
+
+# Training and prediction functions
+include(joinpath("training", "training.jl"))
+include(joinpath("training", "onlinetraining.jl"))
+include(joinpath("hyperparameter", "autotuning.jl"))
+include(joinpath("training", "predictions.jl"))
+include("ar_predict.jl")
 
-    # Training and prediction functions
-    include(joinpath("training", "training.jl"))
-    include(joinpath("training", "onlinetraining.jl"))
-    include(joinpath("hyperparameter", "autotuning.jl"))
-    include(joinpath("training", "predictions.jl"))
-    include("ar_predict.jl")
 end #End Module
diff --git a/src/ComplementaryDistributions/ComplementaryDistributions.jl b/src/ComplementaryDistributions/ComplementaryDistributions.jl
@@ -0,0 +1,13 @@
+module ComplementaryDistributions
+
+using Distributions
+using Random
+using SpecialFunctions
+using StatsFuns: twoπ
+
+export GeneralizedInverseGaussian, PolyaGamma, LaplaceTransformDistribution
+include("generalizedinversegaussian.jl")
+include("polyagamma.jl")
+include("lap_transf_dist.jl")
+
+end
diff --git a/src/functions/GIGSampler.jl → ...stributions/generalizedinversegaussian.jl b/src/functions/GIGSampler.jl → ...stributions/generalizedinversegaussian.jl
@@ -1,10 +1,3 @@
-"""Module for a Generalized Inverse Gaussian Sampler"""
-module GIGSampler
-
-using Distributions
-using SpecialFunctions
-
-export GeneralizedInverseGaussian
 
 """Sampler object"""
 struct GeneralizedInverseGaussian{T<:Real} <: Distributions.ContinuousUnivariateDistribution
@@ -22,7 +15,7 @@ function GeneralizedInverseGaussian(a::T, b::T, p::T) where T
 end
 
 Distributions.params(d::GeneralizedInverseGaussian) = (d.a, d.b, d.p)
-@inline Distributions.partype(d::GeneralizedInverseGaussian{T}) where T <: Real = T
+@inline Distributions.partype(::GeneralizedInverseGaussian{T}) where T <: Real = T
 
 
 function Distributions.mean(d::GeneralizedInverseGaussian)
@@ -40,7 +33,6 @@ end
 
 Distributions.mode(d::GeneralizedInverseGaussian) = ((d.p - 1) + sqrt((d.p - 1)^2 + d.a * d.b)) / d.a
 
-
 function Distributions.pdf(d::GeneralizedInverseGaussian{T}, x::Real) where T <: Real
     if x > 0
         a, b, p = params(d)
@@ -165,6 +157,4 @@ function _rou_shift(λ::Real, β::Real)
             return x
         end
     end
-end
-
-end #module GIGSampler
+end
diff --git a/src/functions/lap_transf_dist.jl → ...lementaryDistributions/lap_transf_dist.jl b/src/functions/lap_transf_dist.jl → ...lementaryDistributions/lap_transf_dist.jl
@@ -1,17 +1,14 @@
-using Random
-
 ###
 # Distribution only based on its laplace transform function `f` and exponential tilting `c²`.
 # The sampling is from Ridout, M. S. (2009). Generating random numbers from a distribution specified by its laplace transform. Statistics and Computing, 19(4):439.
 # And the inverse laplace operation is done via Grassmann, W. K. (Ed.). (2000). Computational Probability. International Series in #Operations Research & Management Science. doi:10.1007/978-1-4757-4828-4
-
 struct LaplaceTransformDistribution{T,TAlg} <: Distributions.ContinuousUnivariateDistribution
     f::Function # Laplace transform of the pdf
     c²::T # Exponential tilting parameter
     alg::TAlg # Algorithm to compute the inverse Laplace transform
     function LaplaceTransformDistribution{T,TAlg}(f::Function,c²::T,alg::TAlg) where {T<:Real,TAlg}
-        @assert _check_f(f) "The function passed is not valid"# Do series of check on f
-        @assert c²>=0 "c² has to a be non-negative real"
+        _check_f(f) || error("The function passed is not valid") # Do a series of checks on f
+        c² >= 0 || error("c² has to a be non-negative real")
         new{T,TAlg}(f,c²,alg)
     end
 end
@@ -21,13 +18,13 @@ LaplaceTransformDistribution(f::Function,c²::T=0.0,alg::TAlg=BromwichInverseLap
 function _check_f(f)
     return true # TODO Add tests for complete monotonicity / PDR
 end
-_gradf(d::LaplaceTransformDistribution,x::Real) = ForwardDiff.gradient(dist.f,[x])[1]
-_gradlogf(d::LaplaceTransformDistribution,x::Real) = ForwardDiff.gradient(log∘dist.f,[x])[1]
-_hessianlogf(d::LaplaceTransformDistribution,x::Real) = ForwardDiff.hessian(log∘dist.f,[x])[1]
+_gradf(d::LaplaceTransformDistribution, x::Real) = first(ForwardDiff.gradient(d.f,[x]))
+_gradlogf(d::LaplaceTransformDistribution, x::Real) = first(ForwardDiff.gradient(log ∘ d.f, [x]))
+_hessianlogf(d::LaplaceTransformDistribution, x::Real) = first(ForwardDiff.hessian(log ∘ d.f, [x]))
 
-Distributions.pdf(dist::LaplaceTransformDistribution,x::Real) = apply_f(dist,x)
-Distributions.mean(dist::LaplaceTransformDistribution) = _gradf(dist,dist.c²)/dist.f(dist.c²)
-Distributions.var(dist::LaplaceTransformDistribution) = _hessianlogf(dist,dist.c²)/dist.f(dist.c²)-mean(dist)^2
+Distributions.pdf(dist::LaplaceTransformDistribution,x::Real) = apply_f(dist, x)
+Distributions.mean(dist::LaplaceTransformDistribution) = _gradf(dist, dist.c²) / dist.f(dist.c²)
+Distributions.var(dist::LaplaceTransformDistribution) = _hessianlogf(dist, dist.c²) / dist.f(dist.c²) - mean(dist)^2
 
 function Random.rand(dist::LaplaceTransformDistribution)
     first(rand(dist,1))