diff --git a/Project.toml b/Project.toml index 6dea764..b5f833e 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "LearnTestAPI" uuid = "3111ed91-c4f2-40e7-bb19-7f6c618409b8" authors = ["Anthony D. Blaom "] -version = "0.2.4" +version = "0.3.0" [deps] CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597" @@ -25,13 +25,13 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" UnPack = "3a884ed6-31ef-47d7-9d2a-63182c4928ed" [compat] -CategoricalArrays = "0.10.8" -CategoricalDistributions = "0.1.15" +CategoricalArrays = "1" +CategoricalDistributions = "0.2" Distributions = "0.25" InteractiveUtils = "<0.0.1, 1" IsURL = "0.2.0" -LearnAPI = "0.2.0,1" -LearnDataFrontEnds = "0.1" +LearnAPI = "2" +LearnDataFrontEnds = "0.2" LinearAlgebra = "<0.0.1, 1" MLCore = "1.0.0" MacroTools = "0.5" diff --git a/src/learners/classification.jl b/src/learners/classification.jl index 93e214d..ba66c12 100644 --- a/src/learners/classification.jl +++ b/src/learners/classification.jl @@ -24,7 +24,7 @@ struct ConstantClassifierFitted learner::ConstantClassifier probabilities names::Vector{Symbol} - classes_seen + levels_seen codes_seen decoder end @@ -44,10 +44,15 @@ LearnAPI.features(learner::ConstantClassifier, data) = LearnAPI.target(learner::ConstantClassifier, data) = LearnAPI.target(learner, data, front_end) -function LearnAPI.fit(learner::ConstantClassifier, observations::FrontEnds.Obs; verbosity=1) +function LearnAPI.fit( + learner::ConstantClassifier, + observations::FrontEnds.Obs; + verbosity=LearnAPI.default_verbosity(), + ) + y = observations.target # integer "codes" names = observations.names - classes_seen = observations.classes_seen + levels_seen = observations.levels_seen codes_seen = sort(unique(y)) decoder = observations.decoder @@ -59,7 +64,7 @@ function LearnAPI.fit(learner::ConstantClassifier, observations::FrontEnds.Obs; learner, probabilities, names, - classes_seen, + levels_seen, codes_seen, decoder, ) @@ -89,7 +94,7 @@ function LearnAPI.predict( probs = model.probabilities # repeat vertically to get rows of a matrix: probs_matrix = reshape(repeat(probs, n), (length(probs), n))' - return CategoricalDistributions.UnivariateFinite(model.classes_seen, probs_matrix) + return CategoricalDistributions.UnivariateFinite(model.levels_seen, probs_matrix) end LearnAPI.predict(model::ConstantClassifierFitted, ::Distribution, data) = predict(model, Distribution(), obs(model, data)) diff --git a/src/learners/dimension_reduction.jl b/src/learners/dimension_reduction.jl index c9d577d..6bb7f54 100644 --- a/src/learners/dimension_reduction.jl +++ b/src/learners/dimension_reduction.jl @@ -65,7 +65,11 @@ LearnAPI.obs(model::TruncatedSVDFitted, data) = LearnAPI.features(learner::TruncatedSVD, data) = LearnAPI.features(learner, data, FrontEnds.Tarragon()) -function LearnAPI.fit(learner::TruncatedSVD, observations::FrontEnds.Obs; verbosity=1) +function LearnAPI.fit( + learner::TruncatedSVD, + observations::FrontEnds.Obs; + verbosity=LearnAPI.default_verbosity(), + ) # unpack hyperparameters: codim = learner.codim diff --git a/src/learners/ensembling.jl b/src/learners/ensembling.jl index b634399..5137423 100644 --- a/src/learners/ensembling.jl +++ b/src/learners/ensembling.jl @@ -69,7 +69,7 @@ LearnAPI.obs(model::EnsembleFitted, data) = LearnAPI.obs(first(model.models), da LearnAPI.target(learner::Ensemble, data) = LearnAPI.target(learner.atom, data) LearnAPI.features(learner::Ensemble, data) = LearnAPI.features(learner.atom, data) -function LearnAPI.fit(learner::Ensemble, data; verbosity=1) +function LearnAPI.fit(learner::Ensemble, data; verbosity=LearnAPI.default_verbosity()) # unpack hyperparameters: atom = learner.atom @@ -112,7 +112,7 @@ function LearnAPI.update( model::EnsembleFitted, data, replacements::Pair{Symbol}...; - verbosity=1, + verbosity=LearnAPI.default_verbosity(), ) learner_old = LearnAPI.learner(model) learner = LearnAPI.clone(learner_old, replacements...) @@ -205,7 +205,7 @@ LearnAPI.components(model::EnsembleFitted) = [:atom => model.models,] # - `update` # - `predict` (`Point` predictions) # - `predictions` (returns predictions on all supplied data) -# - `out_of_sample_indices` (articluates which data is the internal validation data) +# - `out_of_sample_indices` (articulates which data is the internal validation data) # - `trees` # - `training_losses` # - `out_of_sample_losses` @@ -361,7 +361,7 @@ struct StumpRegressorFitted rng end -function LearnAPI.fit(learner::StumpRegressor, data; verbosity=1) +function LearnAPI.fit(learner::StumpRegressor, data; verbosity=LearnAPI.default_verbosity()) x, y = data rng = deepcopy(learner.rng) @@ -426,7 +426,7 @@ function LearnAPI.update( model::StumpRegressorFitted, data, # ignored as cached replacements::Pair{Symbol}...; - verbosity=1, + verbosity=LearnAPI.default_verbosity(), ) learner_old = LearnAPI.learner(model) @@ -490,8 +490,9 @@ function LearnAPI.update( end -# needed, because model is supervised: -LearnAPI.target(learner::StumpRegressor, observations) = last(observations) +# training data deconstructors: +LearnAPI.features(learner::StumpRegressor, data) = first(data) +LearnAPI.target(learner::StumpRegressor, data) = last(data) LearnAPI.predict(model::StumpRegressorFitted, ::Point, x) = _predict(model.forest, x) diff --git a/src/learners/gradient_descent.jl b/src/learners/gradient_descent.jl index f745e79..e7b7e6a 100644 --- a/src/learners/gradient_descent.jl +++ b/src/learners/gradient_descent.jl @@ -10,6 +10,7 @@ using StableRNGs import Optimisers import Zygote import NNlib +import CategoricalArrays import CategoricalDistributions import CategoricalDistributions: pdf, mode import ComponentArrays @@ -55,7 +56,7 @@ for the specified number of `epochs`. - `perceptron`: component array with components `weights` and `bias` - `optimiser`: optimiser from Optimiser.jl - `X`: feature matrix, of size `(p, n)` -- `y_hot`: one-hot encoded target, of size `(nclasses, n)` +- `y_hot`: one-hot encoded target, of size `(nlevels, n)` - `epochs`: number of epochs - `state`: optimiser state @@ -108,7 +109,7 @@ point predictions with `predict(model, Point(), Xnew)`. # Warm restart options - update(model, newdata, :epochs=>n, other_replacements...; verbosity=1) + update(model, newdata, :epochs=>n, other_replacements...) If `Δepochs = n - perceptron.epochs` is non-negative, then return an updated model, with the weights and bias of the previously learned perceptron used as the starting state in @@ -117,7 +118,7 @@ instead of the previous training data. Any other hyperparaameter `replacements` adopted. If `Δepochs` is negative or not specified, instead return `fit(learner, newdata)`, where `learner=LearnAPI.clone(learner; epochs=n, replacements....)`. - update_observations(model, newdata, replacements...; verbosity=1) + update_observations(model, newdata, replacements...) Return an updated model, with the weights and bias of the previously learned perceptron used as the starting state in new gradient descent updates. Adopt any specified @@ -132,23 +133,23 @@ PerceptronClassifier(; epochs=50, optimiser=Optimisers.Adam(), rng=Random.defaul struct PerceptronClassifierObs X::Matrix{Float32} y_hot::BitMatrix # one-hot encoded target - classes # the (ordered) pool of `y`, as `CategoricalValue`s + levels # the (ordered) pool of `y`, as `CategoricalValue`s end # For pre-processing the training data: function LearnAPI.obs(::PerceptronClassifier, data::Tuple) X, y = data - classes = CategoricalDistributions.classes(y) - y_hot = classes .== permutedims(y) # one-hot encoding - return PerceptronClassifierObs(X, y_hot, classes) + levels = CategoricalArrays.levels(y) + y_hot = levels .== permutedims(y) # one-hot encoding + return PerceptronClassifierObs(X, y_hot, levels) end LearnAPI.obs(::PerceptronClassifier, observations::PerceptronClassifierObs) = observations # involutivity # helper: -function decode(y_hot, classes) +function decode(y_hot, levels) n = size(y_hot, 2) - [only(classes[y_hot[:,i]]) for i in 1:n] + [only(levels[y_hot[:,i]]) for i in 1:n] end # implement `RadomAccess()` interface for output of `obs`: @@ -156,14 +157,14 @@ Base.length(observations::PerceptronClassifierObs) = size(observations.y_hot, 2) Base.getindex(observations::PerceptronClassifierObs, I) = PerceptronClassifierObs( observations.X[:, I], observations.y_hot[:, I], - observations.classes, + observations.levels, ) # training data deconstructors: LearnAPI.target( learner::PerceptronClassifier, observations::PerceptronClassifierObs, -) = decode(observations.y_hot, observations.classes) +) = decode(observations.y_hot, observations.levels) LearnAPI.target(learner::PerceptronClassifier, data) = LearnAPI.target(learner, obs(learner, data)) LearnAPI.features( @@ -184,7 +185,7 @@ struct PerceptronClassifierFitted learner::PerceptronClassifier perceptron # component array storing weights and bias state # optimiser state - classes # target classes + levels # target levels losses end @@ -194,7 +195,7 @@ LearnAPI.learner(model::PerceptronClassifierFitted) = model.learner function LearnAPI.fit( learner::PerceptronClassifier, observations::PerceptronClassifierObs; - verbosity=1, + verbosity=LearnAPI.default_verbosity(), ) # unpack hyperparameters: @@ -205,12 +206,12 @@ function LearnAPI.fit( # unpack data: X = observations.X y_hot = observations.y_hot - classes = observations.classes - nclasses = length(classes) + levels = observations.levels + nlevels = length(levels) # initialize bias and weights: - weights = randn(rng, Float32, nclasses, p) - bias = zeros(Float32, nclasses) + weights = randn(rng, Float32, nlevels, p) + bias = zeros(Float32, nlevels) perceptron = (; weights, bias) |> ComponentArrays.ComponentArray # initialize optimiser: @@ -218,7 +219,7 @@ function LearnAPI.fit( perceptron, state, losses = corefit(perceptron, X, y_hot, epochs, state, verbosity) - return PerceptronClassifierFitted(learner, perceptron, state, classes, losses) + return PerceptronClassifierFitted(learner, perceptron, state, levels, losses) end # `fit` for unprocessed data: @@ -230,16 +231,16 @@ function LearnAPI.update_observations( model::PerceptronClassifierFitted, observations_new::PerceptronClassifierObs, replacements...; - verbosity=1, + verbosity=LearnAPI.default_verbosity(), ) # unpack data: X = observations_new.X y_hot = observations_new.y_hot - classes = observations_new.classes - nclasses = length(classes) + levels = observations_new.levels + nlevels = length(levels) - classes == model.classes || error("New training target has incompatible classes.") + levels == model.levels || error("New training target has incompatible levels.") learner_old = LearnAPI.learner(model) learner = LearnAPI.clone(learner_old, replacements...) @@ -252,7 +253,7 @@ function LearnAPI.update_observations( perceptron, state, losses_new = corefit(perceptron, X, y_hot, epochs, state, verbosity) losses = vcat(losses, losses_new) - return PerceptronClassifierFitted(learner, perceptron, state, classes, losses) + return PerceptronClassifierFitted(learner, perceptron, state, levels, losses) end LearnAPI.update_observations(model::PerceptronClassifierFitted, data, args...; kwargs...) = update_observations(model, obs(LearnAPI.learner(model), data), args...; kwargs...) @@ -262,16 +263,16 @@ function LearnAPI.update( model::PerceptronClassifierFitted, observations::PerceptronClassifierObs, replacements...; - verbosity=1, + verbosity=LearnAPI.default_verbosity(), ) # unpack data: X = observations.X y_hot = observations.y_hot - classes = observations.classes - nclasses = length(classes) + levels = observations.levels + nlevels = length(levels) - classes == model.classes || error("New training target has incompatible classes.") + levels == model.levels || error("New training target has incompatible levels.") learner_old = LearnAPI.learner(model) learner = LearnAPI.clone(learner_old, replacements...) @@ -289,7 +290,7 @@ function LearnAPI.update( corefit(perceptron, X, y_hot, Δepochs, state, verbosity) losses = vcat(losses, losses_new) - return PerceptronClassifierFitted(learner, perceptron, state, classes, losses) + return PerceptronClassifierFitted(learner, perceptron, state, levels, losses) end LearnAPI.update(model::PerceptronClassifierFitted, data, args...; kwargs...) = update(model, obs(LearnAPI.learner(model), data), args...; kwargs...) @@ -299,9 +300,9 @@ LearnAPI.update(model::PerceptronClassifierFitted, data, args...; kwargs...) = function LearnAPI.predict(model::PerceptronClassifierFitted, ::Distribution, Xnew) perceptron = model.perceptron - classes = model.classes + levels = model.levels probs = perceptron.weights*Xnew .+ perceptron.bias |> NNlib.softmax - return CategoricalDistributions.UnivariateFinite(classes, probs') + return CategoricalDistributions.UnivariateFinite(levels, probs') end LearnAPI.predict(model::PerceptronClassifierFitted, ::Point, Xnew) = diff --git a/src/learners/incremental_algorithms.jl b/src/learners/incremental_algorithms.jl index 8b2a3ee..0d3a63f 100644 --- a/src/learners/incremental_algorithms.jl +++ b/src/learners/incremental_algorithms.jl @@ -48,7 +48,7 @@ end LearnAPI.learner(::NormalEstimatorFitted) = NormalEstimator() -function LearnAPI.fit(::NormalEstimator, y; verbosity=1) +function LearnAPI.fit(::NormalEstimator, y; verbosity=LearnAPI.default_verbosity()) n = length(y) Σy = sum(y) ȳ = Σy/n @@ -56,7 +56,11 @@ function LearnAPI.fit(::NormalEstimator, y; verbosity=1) return NormalEstimatorFitted(Σy, ȳ, ss, n) end -function LearnAPI.update_observations(model::NormalEstimatorFitted, ynew; verbosity=1) +function LearnAPI.update_observations( + model::NormalEstimatorFitted, + ynew; + verbosity=LearnAPI.default_verbosity(), + ) m = length(ynew) n = model.n + m Σynew = sum(ynew) @@ -67,28 +71,28 @@ function LearnAPI.update_observations(model::NormalEstimatorFitted, ynew; verbos return NormalEstimatorFitted(Σy, ȳ, ss, n) end -LearnAPI.features(::NormalEstimator, y) = nothing LearnAPI.target(::NormalEstimator, y) = y -LearnAPI.predict(model::NormalEstimatorFitted, ::SingleDistribution) = +LearnAPI.predict(model::NormalEstimatorFitted, ::Distribution) = Distributions.Normal(model.ȳ, sqrt(model.ss/model.n)) LearnAPI.predict(model::NormalEstimatorFitted, ::Point) = model.ȳ function LearnAPI.predict(model::NormalEstimatorFitted, ::ConfidenceInterval) - d = predict(model, SingleDistribution()) + d = predict(model, Distribution()) return (quantile(d, 0.025), quantile(d, 0.975)) end # for fit and predict in one line: LearnAPI.predict(::NormalEstimator, k::LearnAPI.KindOfProxy, y) = predict(fit(NormalEstimator(), y), k) -LearnAPI.predict(::NormalEstimator, y) = predict(NormalEstimator(), SingleDistribution(), y) +LearnAPI.predict(::NormalEstimator, y) = predict(NormalEstimator(), Distribution(), y) LearnAPI.extras(model::NormalEstimatorFitted) = (μ=model.ȳ, σ=sqrt(model.ss/model.n)) @trait( NormalEstimator, constructor = NormalEstimator, - kinds_of_proxy = (SingleDistribution(), Point(), ConfidenceInterval()), + kind_of = LearnAPI.Generative(), + kinds_of_proxy = (Distribution(), Point(), ConfidenceInterval()), tags = ("density estimation", "incremental algorithms"), is_pure_julia = true, human_name = "normal distribution estimator", @@ -98,7 +102,6 @@ LearnAPI.extras(model::NormalEstimatorFitted) = (μ=model.ȳ, σ=sqrt(model.ss/ :(LearnAPI.clone), :(LearnAPI.strip), :(LearnAPI.obs), - :(LearnAPI.features), :(LearnAPI.target), :(LearnAPI.predict), :(LearnAPI.update_observations), diff --git a/src/learners/regression.jl b/src/learners/regression.jl index 39f6ed6..ce330ff 100644 --- a/src/learners/regression.jl +++ b/src/learners/regression.jl @@ -1,7 +1,7 @@ # This file defines: -# - `Ridge(; lambda=0.1)` -# - `BabyRidge(; lambda=0.1)` +# - `Ridge(; lambda=0.1)` (uses canned data front end) +# - `BabyRidge(; lambda=0.1)` (no data front end) using LearnAPI using Tables @@ -46,7 +46,11 @@ LearnAPI.obs(model::RidgeFitted, data) = obs(model, data, frontend) LearnAPI.features(learner::Ridge, data) = LearnAPI.features(learner, data, frontend) LearnAPI.target(learner::Ridge, data) = LearnAPI.target(learner, data, frontend) -function LearnAPI.fit(learner::Ridge, observations::FrontEnds.Obs; verbosity=1) +function LearnAPI.fit( + learner::Ridge, + observations::FrontEnds.Obs; + verbosity=LearnAPI.default_verbosity(), + ) # unpack hyperparameters and data: lambda = learner.lambda @@ -130,7 +134,7 @@ struct BabyRidgeFitted{T,F} feature_importances::F end -function LearnAPI.fit(learner::BabyRidge, data; verbosity=1) +function LearnAPI.fit(learner::BabyRidge, data; verbosity=LearnAPI.default_verbosity()) X, y = data @@ -150,12 +154,17 @@ end LearnAPI.learner(model::BabyRidgeFitted) = model.learner +# training data deconstructors: +LearnAPI.features(learner::BabyRidge, (X, y)) = X +LearnAPI.target(learner::BabyRidge, (X, y)) = y + LearnAPI.predict(model::BabyRidgeFitted, ::Point, Xnew) = Tables.matrix(Xnew)*model.coefficients LearnAPI.strip(model::BabyRidgeFitted) = BabyRidgeFitted(model.learner, model.coefficients, nothing) + @trait( BabyRidge, constructor = BabyRidge, diff --git a/src/learners/static_algorithms.jl b/src/learners/static_algorithms.jl index 14f43db..8ef9a4d 100644 --- a/src/learners/static_algorithms.jl +++ b/src/learners/static_algorithms.jl @@ -36,7 +36,7 @@ Selector(; names=Symbol[]) = Selector(names) # LearnAPI.constructor defined lat # `fit` consumes no observational data, does no "learning", and just returns a thinly # wrapped `learner` (to distinguish it from the learner in dispatch): -LearnAPI.fit(learner::Selector; verbosity=1) = Ref(learner) +LearnAPI.fit(learner::Selector; verbosity=LearnAPI.default_verbosity()) = Ref(learner) LearnAPI.learner(model::Base.RefValue{Selector}) = model[] function LearnAPI.transform(model::Base.RefValue{Selector}, X) @@ -55,12 +55,12 @@ function LearnAPI.transform(learner::Selector, X) transform(model, X) end -# note the necessity of overloading `is_static` (`fit` consumes no data): +# note the necessity of overloading `kind_of` (because `fit` consumes no data): @trait( Selector, constructor = Selector, + kind_of = LearnAPI.Static(), tags = ("feature engineering",), - is_static = true, functions = ( :(LearnAPI.fit), :(LearnAPI.learner), @@ -107,7 +107,8 @@ LearnAPI.learner(model::FancySelectorFitted) = model.learner rejected(model::FancySelectorFitted) = model.rejected # Here we are wrapping `learner` with a place-holder for the `rejected` feature names. -LearnAPI.fit(learner::FancySelector; verbosity=1) = FancySelectorFitted(learner) +LearnAPI.fit(learner::FancySelector; verbosity=LearnAPI.default_verbosity()) = + FancySelectorFitted(learner) # output the filtered table and add `rejected` field to model (mutatated!) function LearnAPI.transform(model::FancySelectorFitted, X) @@ -127,11 +128,11 @@ function LearnAPI.transform(learner::FancySelector, X) transform(model, X) end -# note the necessity of overloading `is_static` (`fit` consumes no data): +# note the necessity of overloading `kind_of` (because `fit` consumes no data): @trait( FancySelector, constructor = FancySelector, - is_static = true, + kind_of = LearnAPI.Static(), tags = ("feature engineering",), functions = ( :(LearnAPI.fit), diff --git a/src/logging.jl b/src/logging.jl index 346bb89..3f91227 100644 --- a/src/logging.jl +++ b/src/logging.jl @@ -8,7 +8,7 @@ const QUIET = "- specify `verbosity=1` if debugging" const CONSTRUCTOR = """ - Testing that learner can be reconstructed from its constructors. + Testing that learner can be reconstructed from its constructor. [Reference](https://juliaai.github.io/LearnAPI.jl/dev/reference/#learners). """ @@ -34,23 +34,53 @@ const FUNCTIONS = """ """ const ERR_MISSINNG_OBLIGATORIES = "These obligatory functions are missing from the return value of "* - "`LearnAPI.functions(learner)`; " + "`LearnAPI.functions(learner)`: " + + + +const DECONSTRUCTORS = """ + + Checking that the data deconsructors (`features`, `target` and `weights`) have only + been implemented where appropriate, and looking for clues that some desirable + implementations have been forgotten. + + """ +const WARN_GENERATIVE_NO_TARGET = """ + + Typically, when `LearnAPI.kind_of(learner)==LearnAPI.Generative()`, we expect + `LearnAPI.target` to be implemented. If you have implemented it, check you added + `:(LearnAPI.target)` to the tuple returned by `LearnAPI.functions(learner)`. If you + intentionally left it unimplemented, ignore this warning. + + """ +const NO_DECONSTRUCTORS_FOR_STATIC = """ + + Since `LearnAPI.kind_of(learner)==LearnAPI.Static()`, we are checking that none of the + following are in the tuple returned by `LearnAPI.functions(learner)`: + `:(LearnAPI.features)`, `:(LearnAPI.target)`, `:(LearnAPI.weights)`, because there + is never training data to deconstruct. -const FUNCTIONS3 = """ + """ +const WARN_DESCRIMINATIVE_NO_FEATURES = """ - Testing that `LearnAPI.functions(learner)` includes `:(LearnAPI.features).` + Typically, when `LearnAPI.kind_of(learner)==LearnAPI.Descriminative()`, we expect + `LearnAPI.features` to be implemented. If you have implemented it, check you added + `:(LearnAPI.features)` to the tuple returned by `LearnAPI.functions(learner)`. If you + intentionally left it unimplemented, ignore this warning. """ -const FUNCTIONS4 = """ +const WARN_DESCRIMINATIVE_NO_TARGET = """ - Testing that `LearnAPI.functions(learner)` excludes `:(LearnAPI.features)`, as - `LearnAPI.is_static(learner)` is `true`. + Frequently, when `LearnAPI.kind_of(learner)==LearnAPI.Descriminative()`, + `LearnAPI.target` is also implemented. If you have implemented it, check you added + `:(LearnAPI.features)` to the tuple returned by `LearnAPI.functions(learner)`. If you + intentionally left it unimplemented, ignore this warning. """ const TAGS = """ Testing that `LearnAPI.tags(learner)` has correct form. List allowed tags with - `LearnAPII.tags()`. + `LearnAPI.tags()`. """ const KINDS_OF_PROXY = """ @@ -62,14 +92,14 @@ const KINDS_OF_PROXY = """ """ const FIT_IS_STATIC = """ - `LearnAPI.is_static(learner)` is `true`. Therefore attempting to call - `fit(learner)`. + `LearnAPI.kind_of(learner)==LearnAPI.Static()`. Therefore attempting to define + `model = fit(learner)`. """ const FIT_IS_NOT_STATIC = """ - Attempting to call `fit(learner, data)`. If you implemented `fit(learner)` instead, - then you need to arrange `LearnAPI.is_static(learner) == true`. + Attempting to define `model = fit(learner, data)`. If you implemented `fit(learner)` + instead, then you need to arrange `LearnAPI.kind_of(learner)==LearnAPI.Static()`. """ const LEARNER = """ @@ -86,7 +116,7 @@ const FUNCTIONS2 = """ """ const ERR_MISSING_FUNCTIONS = - "The following overloaded functions are missing from the return value of"* + "The following implemented/overloaded functions are missing from the return value of"* "`LearnAPI.functions(learner)`: " const OBS = """ @@ -118,7 +148,9 @@ const PREDICT_HAS_NO_FEATURES = """ Attempting to call `predict(model, kind)` for each `kind` in `LearnAPI.kinds_of_proxy(learner)`. (We are not providing `predict` with a data - argument because `features(obs(learner, data)) == nothing`). + argument because either `LearnAPI.kind_of(learner)==LearnAPI.Generative()`, or because + we presume `LearnAPI.features` is not implemented, as `:(LearnAPI.features)` is not in + the return value of LearnAPI.functions(learner)`.) """ @@ -256,14 +288,12 @@ const TRANSFORM_ON_SELECTIONS2 = """ """ const TARGET0 = """ - Attempting to call `LearnAPI.target(learner, data)` (fallback returns - `last(data)`). + Attempting to call `LearnAPI.target(learner, data)`. """ const TARGET = """ - Attempting to call `LearnAPI.target(learner, observations)` (fallback returns - `last(observations)`). + Attempting to call `LearnAPI.target(learner, observations)` """ const TARGET_SELECTIONS = """ @@ -311,7 +341,7 @@ const UPDATE = """ """ const ERR_STATIC_UPDATE = ErrorException( "`(LearnAPI.update)` is in `LearnAPI.functions(learner)` but "* - "`LearnAPI.is_static(learner)` is `true`. You cannot implement `update` "* + "`LearnAPI.kind_of(learner)==LearnAPI.Static()`. You cannot implement `update` "* "for static learners. " ) const UPDATE_ITERATIONS = """ diff --git a/src/testapi.jl b/src/testapi.jl index 377584c..ed9c8b5 100644 --- a/src/testapi.jl +++ b/src/testapi.jl @@ -60,14 +60,17 @@ hyperparameter settings, are explicitly tested. Each `dataset` is used as follows. -If `LearnAPI.is_static(learner) == false`, then: +Assuming [`LearnAPI.kind_of(learner)`](@ref) returns [`LearnAPI.Descriminative()`](@ref) +or [`LearnAPI.Generative()`](@ref): - `dataset` is passed to `fit` and, if necessary, its `update` cousins -- If `X = LearnAPI.features(learner, dataset) == nothing`, then `predict` and/or - `transform` are called with no data. Otherwise, they are called with `X`. +- In the `Generative()` case, `predict` and/or `transform` are called without a data + argument; in the `Descriminative()` case these methods are called with the data argument + ` X = LearnAPI.features(learner, dataset)`, assuming `:features in + LearnAPI.functions(learner)`, and are otherwise not called. -If instead `LearnAPI.is_static(learner) == true`, then `fit` and its cousins are called +If instead `LearnAPI.kind_of(learner) == Static()`, then `fit` and its cousins are called without any data, and `dataset` is passed directly to `predict` and/or `transform`. """ @@ -87,7 +90,8 @@ macro testapi(learner, data...) verbosity=$verbosity _human_name = LearnAPI.human_name(learner) _data_interface = LearnAPI.data_interface(learner) - _is_static = LearnAPI.is_static(learner) + _is_static = LearnAPI.kind_of(learner) == LearnAPI.Static() + _is_generative = LearnAPI.kind_of(learner) == LearnAPI.Generative() if isnothing(verbosity) || verbosity > 0 @info "------ running @testapi - $_human_name "*$LOUD @@ -124,13 +128,21 @@ macro testapi(learner, data...) end end - if !_is_static - @logged_testset $FUNCTIONS3 verbosity begin - Test.@test :(LearnAPI.features) in _functions - end - else - @logged_testset $FUNCTIONS4 verbosity begin - Test.@test !(:(LearnAPI.features) in _functions) + _has_features = :(LearnAPI.features) in _functions + _has_target = :(LearnAPI.target) in _functions + _has_weights = :(LearnAPI.weights) in _functions + + @logged_testset $DECONSTRUCTORS verbosity begin + if _is_generative + !_has_target && verbosity > 0 && @warn $WARN_GENERATIVE_NO_TARGET + Test.@test !_has_features + elseif _is_static + @logged_testset $NO_DECONSTRUCTORS_FOR_STATIC verbosity begin + Test.@test !_has_target && !_has_features && !_has_weights + end + else + !_has_features && verbosity > 0 && @warn $WARN_DESCRIMINATIVE_NO_FEATURES + !_has_features && verbosity > 0 && @warn $WARN_DESCRIMINATIVE_NO_TARGET end end @@ -199,13 +211,15 @@ macro testapi(learner, data...) X = if _is_static data - else + elseif _has_features @logged_testset $FEATURES0 verbosity begin LearnAPI.features(learner, data) end @logged_testset $FEATURES verbosity begin LearnAPI.features(learner, observations) end + else + nothing end if !(isnothing(X)) @@ -467,14 +481,11 @@ macro testapi(learner, data...) # weights - _w = @logged_testset $WEIGHTS verbosity begin - LearnAPI.weights(learner, observations) - end - - if !(isnothing(_w)) - @logged_testset $WEIGHTS_IN_FUNCTIONS verbosity begin - Test.@test :(LearnAPI.weights) in _functions + if :(LearnAPI.weights) in _functions + _w = @logged_testset $WEIGHTS verbosity begin + LearnAPI.weights(learner, observations) end + w = @logged_testset $WEIGHTS_SELECTIONS verbosity begin LearnTestAPI.learner_get( learner, @@ -482,10 +493,6 @@ macro testapi(learner, data...) data->LearnAPI.weights(learner, data), ) end - else - @logged_testset $WEIGHTS_NOT_IN_FUNCTIONS verbosity begin - Test.@test !(:(LearnAPI.weights) in _functions) - end end # update @@ -612,7 +619,7 @@ macro testapi(learner, data...) # traits # `constructor`, `functions`, `kinds_of_proxy`, `tags`, `nonlearners`, - # `iteration_parameter`, `data_interface`, `is_static` tested already above + # `iteration_parameter`, `data_interface`, `kind_of` tested already above @logged_testset $PKG_NAME verbosity begin pkg_name = LearnAPI.pkg_name(learner) @@ -686,3 +693,4 @@ macro testapi(learner, data...) nothing end # quote end # macro + diff --git a/test/learners/classification.jl b/test/learners/classification.jl index 3c0eb1d..3d42be4 100644 --- a/test/learners/classification.jl +++ b/test/learners/classification.jl @@ -45,7 +45,7 @@ f = @formula(t ~ c + a) # # TESTS learner = LearnTestAPI.ConstantClassifier() -@testapi learner (X1, y) +@testapi learner (X1, y) verbosity=0 @testapi learner (X2, y) (X3, y) (X4, y) (T1, :t) (T2, :t) (T3, f) (T4, f) verbosity=0 @testset "extra tests for constant classifier" begin diff --git a/test/learners/dimension_reduction.jl b/test/learners/dimension_reduction.jl index a7ade73..6600eab 100644 --- a/test/learners/dimension_reduction.jl +++ b/test/learners/dimension_reduction.jl @@ -12,7 +12,7 @@ U, Vt = r.U, r.Vt X = U*diagm([1, 2, 3, 0.01, 0.01])*Vt learner = LearnTestAPI.TruncatedSVD(codim=2) -@testapi learner X verbosity=1 +@testapi learner X verbosity=0 @testset "extra test for truncated SVD" begin model = @test_logs( diff --git a/test/learners/ensembling.jl b/test/learners/ensembling.jl index d4db7e6..195a749 100644 --- a/test/learners/ensembling.jl +++ b/test/learners/ensembling.jl @@ -25,7 +25,7 @@ Xtest = Tables.subset(X, test) rng = StableRNG(123) atom = LearnTestAPI.Ridge() learner = LearnTestAPI.Ensemble(atom; n=4, rng) -@testapi learner data verbosity=1 +@testapi learner data verbosity=0 @testset "extra tests for ensemble" begin @test LearnAPI.clone(learner) == learner diff --git a/test/learners/gradient_descent.jl b/test/learners/gradient_descent.jl index c184e62..58c4991 100644 --- a/test/learners/gradient_descent.jl +++ b/test/learners/gradient_descent.jl @@ -44,7 +44,7 @@ learner = @testapi learner (X, y) verbosity=0 # use verbosity=1 to debug -@testset "extra tests for perceptron classfier" begin +@testset "extra tests for perceptron classifier" begin model40 = fit(learner, (Xtrain, ytrain); verbosity=0) # 40 epochs is sufficient for 90% accuracy in this case: diff --git a/test/learners/regression.jl b/test/learners/regression.jl index 1ee064c..dac24f3 100644 --- a/test/learners/regression.jl +++ b/test/learners/regression.jl @@ -20,7 +20,7 @@ data = (X, y) # # RIDGE learner = LearnTestAPI.Ridge(lambda=0.5) -@testapi learner data verbosity=1 +@testapi learner data verbosity=0 @testset "extra tests for ridge regressor" begin @test :(LearnAPI.obs) in LearnAPI.functions(learner) diff --git a/test/learners/static_algorithms.jl b/test/learners/static_algorithms.jl index 26e21c5..eeae3b6 100644 --- a/test/learners/static_algorithms.jl +++ b/test/learners/static_algorithms.jl @@ -9,7 +9,7 @@ import DataFrames learner = LearnTestAPI.Selector(names=[:x, :w]) X = DataFrames.DataFrame(rand(3, 4), [:x, :y, :z, :w]) -@testapi learner X verbosity=1 +@testapi learner X verbosity=0 @testset "test a static transformer" begin model = fit(learner) # no data arguments! @@ -23,7 +23,7 @@ end learner = LearnTestAPI.FancySelector(names=[:x, :w]) X = DataFrames.DataFrame(rand(3, 4), [:x, :y, :z, :w]) -@testapi learner X verbosity=1 +@testapi learner X verbosity=0 @testset "test a variation that reports byproducts" begin model = fit(learner) # no data arguments!