From ebd84e094b3a5e0dec36b75c0041d5cf0f71d545 Mon Sep 17 00:00:00 2001 From: Sathvik Bhagavan Date: Tue, 16 Jan 2024 07:09:17 +0000 Subject: [PATCH 01/14] ci: use julia 1.10 for CI --- .buildkite/pipeline.yml | 6 +++--- .github/workflows/CI.yml | 2 +- .github/workflows/Downstream.yml | 2 +- .github/workflows/Invalidations.yml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index f39b9baa3..ff287e15b 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -2,7 +2,7 @@ steps: - label: "GPU" plugins: - JuliaCI/julia#v1: - version: "1.9" + version: "1" - JuliaCI/julia-test#v1: coverage: false # 1000x slowdown agents: @@ -20,7 +20,7 @@ steps: matrix: setup: version: - - "1.9" + - "1" group: - "NNODE" - "NeuralAdapter" @@ -44,7 +44,7 @@ steps: - label: "Documentation" plugins: - JuliaCI/julia#v1: - version: "1.9" + version: "1" command: | julia --project=docs -e ' println("--- :julia: Instantiating project") diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 78dcb6011..deea6b5eb 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -25,7 +25,7 @@ jobs: - Logging - Forward version: - - "1.9" + - "1" steps: - uses: actions/checkout@v4 - uses: julia-actions/setup-julia@v1 diff --git a/.github/workflows/Downstream.yml b/.github/workflows/Downstream.yml index b9bc18ad5..e32c46f58 100644 --- a/.github/workflows/Downstream.yml +++ b/.github/workflows/Downstream.yml @@ -14,7 +14,7 @@ jobs: strategy: fail-fast: false matrix: - julia-version: [1,1.6] + julia-version: [1] os: [ubuntu-latest] package: - {user: SciML, repo: PDESystemLibrary.jl, group: NeuralPDE} diff --git a/.github/workflows/Invalidations.yml b/.github/workflows/Invalidations.yml index b876dedf7..6a0a747c7 100644 --- a/.github/workflows/Invalidations.yml +++ b/.github/workflows/Invalidations.yml @@ -18,7 +18,7 @@ jobs: steps: - uses: julia-actions/setup-julia@v1 with: - version: "1.9" + version: "1" - uses: actions/checkout@v4 - uses: julia-actions/julia-buildpkg@v1 - uses: julia-actions/julia-invalidations@v1 From 5be7eed67deb5aadc20be0d2bbabbd6125220966 Mon Sep 17 00:00:00 2001 From: Sathvik Bhagavan Date: Tue, 16 Jan 2024 07:10:47 +0000 Subject: [PATCH 02/14] build: bump compats of SciMLBase, Integrals and remove IntegralsCubature --- Project.toml | 8 ++++---- docs/Project.toml | 8 ++++---- src/NeuralPDE.jl | 2 +- test/NNPDE_tests.jl | 2 +- test/additional_loss_tests.jl | 2 +- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Project.toml b/Project.toml index 7971650ae..40c21a14b 100644 --- a/Project.toml +++ b/Project.toml @@ -9,6 +9,7 @@ AdvancedHMC = "0bf59076-c3b1-5ca4-86bd-e02cd72cde3d" ArrayInterface = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9" ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" ComponentArrays = "b0b7db55-cfe3-40fc-9ded-d10e2dbeff66" +Cubature = "667455a9-e2ce-5579-9412-b964f529a492" DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e" DiffEqNoiseProcess = "77a26b50-5914-5dd7-bc55-306e6241c503" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" @@ -18,7 +19,6 @@ Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c" ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" Functors = "d9f16b24-f501-4c13-a1f2-28368ffc5196" Integrals = "de52edbc-65ea-441a-8357-d3a637375a31" -IntegralsCubature = "c31f79ba-6e32-46d4-a52f-182a8ac42a54" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" LogDensityProblems = "6fdf6af0-433a-55f7-b3ed-c6c6e0b8df7c" Lux = "b2108857-7c20-44ae-9111-449ecde12c47" @@ -45,6 +45,7 @@ Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" Adapt = "3, 4" AdvancedHMC = "0.5" ArrayInterface = "6, 7" +Cubature = "1.5" CUDA = "4" ChainRulesCore = "1" ComponentArrays = "0.13.2, 0.14, 0.15" @@ -56,8 +57,7 @@ DomainSets = "0.6" Flux = "0.13, 0.14" ForwardDiff = "0.10" Functors = "0.4" -Integrals = "3.1" -IntegralsCubature = "=0.2.2" +Integrals = "4" LogDensityProblems = "2" Lux = "0.4, 0.5" MCMCChains = "6" @@ -70,7 +70,7 @@ QuasiMonteCarlo = "0.3.2" RecursiveArrayTools = "2.31" Reexport = "1.0" RuntimeGeneratedFunctions = "0.5" -SciMLBase = "1.91, 2" +SciMLBase = "2" Statistics = "1" StochasticDiffEq = "6.13" SymbolicUtils = "1" diff --git a/docs/Project.toml b/docs/Project.toml index 462d2f4fa..8cdbea9b5 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -1,12 +1,12 @@ [deps] AdvancedHMC = "0bf59076-c3b1-5ca4-86bd-e02cd72cde3d" +Cubature = "667455a9-e2ce-5579-9412-b964f529a492" DiffEqBase = "2b5f629d-d688-5b77-993f-72d75c75574e" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" DomainSets = "5b8099bc-c8ec-5219-889f-1d9e522a28bf" Flux = "587475ba-b771-5e3f-ad9e-33799f191a9c" Integrals = "de52edbc-65ea-441a-8357-d3a637375a31" -IntegralsCubature = "c31f79ba-6e32-46d4-a52f-182a8ac42a54" LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" Lux = "b2108857-7c20-44ae-9111-449ecde12c47" ModelingToolkit = "961ee093-0014-501f-94e3-6117800e7a78" @@ -24,12 +24,12 @@ SpecialFunctions = "276daf66-3868-5448-9aa4-cd146d93841b" [compat] AdvancedHMC = "0.5" +Cubature = "1.5" DiffEqBase = "6.106" Documenter = "1" DomainSets = "0.6" Flux = "0.13, 0.14" -Integrals = "3.3" -IntegralsCubature = "=0.2.2" +Integrals = "4" Lux = "0.4, 0.5" ModelingToolkit = "8.33" MonteCarloMeasurements = "1" @@ -40,6 +40,6 @@ OptimizationOptimisers = "0.1" OptimizationPolyalgorithms = "0.1" OrdinaryDiffEq = "6.31" Plots = "1.36" -QuasiMonteCarlo = "0.2" +QuasiMonteCarlo = "0.3" Roots = "2.0" SpecialFunctions = "2.1" diff --git a/src/NeuralPDE.jl b/src/NeuralPDE.jl index 7478b088d..e69eb1be6 100644 --- a/src/NeuralPDE.jl +++ b/src/NeuralPDE.jl @@ -11,7 +11,7 @@ using Reexport, Statistics using Zygote, ForwardDiff, Random, Distributions using Adapt, DiffEqNoiseProcess, StochasticDiffEq using Optimization -using Integrals, IntegralsCubature +using Integrals, Cubature using QuasiMonteCarlo using RuntimeGeneratedFunctions using SciMLBase diff --git a/test/NNPDE_tests.jl b/test/NNPDE_tests.jl index bf8f91b97..e427fc5e9 100644 --- a/test/NNPDE_tests.jl +++ b/test/NNPDE_tests.jl @@ -1,6 +1,6 @@ using Flux, NeuralPDE, Test using Optimization, OptimizationOptimJL, OptimizationOptimisers -using Integrals, IntegralsCubature +using Integrals, Cubature using QuasiMonteCarlo import ModelingToolkit: Interval, infimum, supremum using DomainSets diff --git a/test/additional_loss_tests.jl b/test/additional_loss_tests.jl index ae05e7cd6..4cbf8150b 100644 --- a/test/additional_loss_tests.jl +++ b/test/additional_loss_tests.jl @@ -3,7 +3,7 @@ using Optimization, OptimizationOptimJL, OptimizationOptimisers using QuasiMonteCarlo, Random import ModelingToolkit: Interval, infimum, supremum using DomainSets -using Integrals, IntegralsCubature +using Integrals, Cubature using OrdinaryDiffEq, ComponentArrays import Lux using ComponentArrays From e59478a79bf4fe04148cd7f67c4d8ebc39f59916 Mon Sep 17 00:00:00 2001 From: Sathvik Bhagavan Date: Wed, 17 Jan 2024 06:27:34 +0000 Subject: [PATCH 03/14] refactor: Quadrature Training with Integrals.jl@v4 --- src/training_strategies.jl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/training_strategies.jl b/src/training_strategies.jl index ca66f6b20..d4edd26bf 100644 --- a/src/training_strategies.jl +++ b/src/training_strategies.jl @@ -320,9 +320,10 @@ function get_loss_function(loss_function, lb, ub, eltypeθ, strategy::Quadrature # mean(abs2,loss_(x,θ), dims=2) # size_x = fill(size(x)[2],(1,1)) x = adapt(parameterless_type(ComponentArrays.getdata(θ)), x) - sum(abs2, loss_(x, θ), dims = 2) #./ size_x + sum(abs2, view(loss_(x, θ), 1, :), dims = 2) #./ size_x end - prob = IntegralProblem(integrand, lb, ub, θ, batch = strategy.batch, nout = 1) + integral_function = BatchIntegralFunction(integrand, max_batch = strategy.batch) + prob = IntegralProblem(integral_function, lb, ub, θ) solve(prob, strategy.quadrature_alg, reltol = strategy.reltol, From ffd9c3cc8481c8fa7d1a4fcc0e488fa96c57189b Mon Sep 17 00:00:00 2001 From: Sathvik Bhagavan Date: Wed, 17 Jan 2024 10:42:12 +0000 Subject: [PATCH 04/14] test: use BackTracking with BFGS for Example 5 2d wave case --- Project.toml | 6 ++++-- test/NNPDE_tests.jl | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Project.toml b/Project.toml index 40c21a14b..478c7b2f5 100644 --- a/Project.toml +++ b/Project.toml @@ -45,10 +45,10 @@ Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" Adapt = "3, 4" AdvancedHMC = "0.5" ArrayInterface = "6, 7" -Cubature = "1.5" CUDA = "4" ChainRulesCore = "1" ComponentArrays = "0.13.2, 0.14, 0.15" +Cubature = "1.5" DiffEqBase = "6" DiffEqNoiseProcess = "5.1" Distributions = "0.23, 0.24, 0.25" @@ -58,6 +58,7 @@ Flux = "0.13, 0.14" ForwardDiff = "0.10" Functors = "0.4" Integrals = "4" +LineSearches = "7.2" LogDensityProblems = "2" Lux = "0.4, 0.5" MCMCChains = "6" @@ -81,6 +82,7 @@ julia = "1.6" [extras] CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" +LineSearches = "d3d80556-e9d4-5f37-9878-2ab0fcc64255" OptimizationOptimJL = "36348300-93cb-4f02-beb5-3c3902f8871e" OptimizationOptimisers = "42dfb2eb-d2b4-4451-abcd-913932933ac1" OrdinaryDiffEq = "1dea7af3-3e70-54e6-95c3-0bf5283fa5ed" @@ -89,4 +91,4 @@ SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["Test", "CUDA", "SafeTestsets", "OptimizationOptimisers", "OptimizationOptimJL", "Pkg", "OrdinaryDiffEq"] +test = ["Test", "CUDA", "SafeTestsets", "OptimizationOptimisers", "OptimizationOptimJL", "Pkg", "OrdinaryDiffEq", "LineSearches"] diff --git a/test/NNPDE_tests.jl b/test/NNPDE_tests.jl index e427fc5e9..6ad64fef2 100644 --- a/test/NNPDE_tests.jl +++ b/test/NNPDE_tests.jl @@ -5,6 +5,7 @@ using QuasiMonteCarlo import ModelingToolkit: Interval, infimum, supremum using DomainSets import Lux +using LineSearches using Random Random.seed!(100) @@ -477,7 +478,7 @@ end return false end - res = solve(prob, OptimizationOptimJL.BFGS(); maxiters = 500, f_abstol = 10^-6) + res = solve(prob, OptimizationOptimJL.BFGS(linesearch = BackTracking()); maxiters = 500) dx = 0.1 xs, ts = [infimum(d.domain):dx:supremum(d.domain) for d in domains] From e79bec5d1a935687447aec60ec1744249abd0bfc Mon Sep 17 00:00:00 2001 From: Sathvik Bhagavan Date: Wed, 17 Jan 2024 10:43:18 +0000 Subject: [PATCH 05/14] docs: enable debug logs --- docs/make.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/make.jl b/docs/make.jl index 67ee53b3d..3b1340c27 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -3,6 +3,7 @@ using Documenter, NeuralPDE cp("./docs/Manifest.toml", "./docs/src/assets/Manifest.toml", force = true) cp("./docs/Project.toml", "./docs/src/assets/Project.toml", force = true) +ENV["JULIA_DEBUG"] = "Documenter" ENV["GKSwstype"] = "100" using Plots From dfe8bc70e33a9d09f969a12717fad6d6ab615657 Mon Sep 17 00:00:00 2001 From: Sathvik Bhagavan Date: Wed, 17 Jan 2024 15:31:21 +0000 Subject: [PATCH 06/14] build: remove RAT as it is not used and bump DomainSets --- Project.toml | 4 +--- src/NeuralPDE.jl | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/Project.toml b/Project.toml index 478c7b2f5..8e2e28b6b 100644 --- a/Project.toml +++ b/Project.toml @@ -30,7 +30,6 @@ Optimisers = "3bd65402-5787-11e9-1adc-39752487f4e2" Optimization = "7f7a1694-90dd-40f0-9382-eb1efda571ba" QuasiMonteCarlo = "8a4e6c94-4038-4cdc-81c3-7e6ffdb2a71b" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" -RecursiveArrayTools = "731186ca-8d62-57ce-b412-fbd966d074cd" Reexport = "189a3867-3050-52da-a836-e630ba90ab69" RuntimeGeneratedFunctions = "7e49a35a-f44a-4d26-94aa-eba1b4ca6b47" SciMLBase = "0bca4576-84f4-4d90-8ffe-ffa030f20462" @@ -53,7 +52,7 @@ DiffEqBase = "6" DiffEqNoiseProcess = "5.1" Distributions = "0.23, 0.24, 0.25" DocStringExtensions = "0.8, 0.9" -DomainSets = "0.6" +DomainSets = "0.6, 0.7" Flux = "0.13, 0.14" ForwardDiff = "0.10" Functors = "0.4" @@ -68,7 +67,6 @@ Optim = ">= 1.7.8" Optimisers = "0.2, 0.3" Optimization = "3" QuasiMonteCarlo = "0.3.2" -RecursiveArrayTools = "2.31" Reexport = "1.0" RuntimeGeneratedFunctions = "0.5" SciMLBase = "2" diff --git a/src/NeuralPDE.jl b/src/NeuralPDE.jl index e69eb1be6..931fb24a5 100644 --- a/src/NeuralPDE.jl +++ b/src/NeuralPDE.jl @@ -31,7 +31,6 @@ import ModelingToolkit: Interval, infimum, supremum #,Ball import SciMLBase: @add_kwonly, parameterless_type import Optimisers import UnPack: @unpack -import RecursiveArrayTools import ChainRulesCore, Flux, Lux, ComponentArrays import ChainRulesCore: @non_differentiable From 8d9fb3a7c4243d91ac9d64db6dd6b117b73af960 Mon Sep 17 00:00:00 2001 From: Sathvik Bhagavan Date: Thu, 18 Jan 2024 04:26:38 +0000 Subject: [PATCH 07/14] docs: fix docstring for BPINNsolution, fix BPINN Manual, import Cubature in constraints tutorial --- docs/src/manual/bpinns.md | 5 ++++- docs/src/tutorials/constraints.md | 2 +- src/BPINN_ode.jl | 1 - 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/src/manual/bpinns.md b/docs/src/manual/bpinns.md index a5296938a..1b2990168 100644 --- a/docs/src/manual/bpinns.md +++ b/docs/src/manual/bpinns.md @@ -14,8 +14,11 @@ NeuralPDE.ahmc_bayesian_pinn_pde ## `symbolic_discretize` for `BayesianPINN` and lower level interface. -```@docs +```@docs; canonical=false SciMLBase.symbolic_discretize(::PDESystem, ::NeuralPDE.AbstractPINN) +``` + +```@docs NeuralPDE.BPINNstats NeuralPDE.BPINNsolution ``` diff --git a/docs/src/tutorials/constraints.md b/docs/src/tutorials/constraints.md index 8ccdda3c2..68aa391af 100644 --- a/docs/src/tutorials/constraints.md +++ b/docs/src/tutorials/constraints.md @@ -22,7 +22,7 @@ with Physics-Informed Neural Networks. ```@example fokkerplank using NeuralPDE, Lux, ModelingToolkit, Optimization, OptimizationOptimJL -using Integrals, IntegralsCubature +using Integrals, Cubature import ModelingToolkit: Interval, infimum, supremum # the example is taken from this article https://arxiv.org/abs/1910.10503 @parameters x diff --git a/src/BPINN_ode.jl b/src/BPINN_ode.jl index 7c4be0f04..3b6fd428d 100644 --- a/src/BPINN_ode.jl +++ b/src/BPINN_ode.jl @@ -150,7 +150,6 @@ BPINN Solution contains the original solution from AdvancedHMC.jl sampling(BPINN > estimated_nn_params - Probabilistic Estimate of NN params from sampled weights,biases > estimated_de_params - Probabilistic Estimate of DE params from sampled unknown DE paramters """ - struct BPINNsolution{O <: BPINNstats, E, NP, OP, P} original::O ensemblesol::E From 72c438ab1fd35fb33c277067f17a95d13a81cdd3 Mon Sep 17 00:00:00 2001 From: Sathvik Bhagavan Date: Thu, 18 Jan 2024 12:55:38 +0000 Subject: [PATCH 08/14] build: add LuxCUDA and cuDNN as test deps --- Project.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 8e2e28b6b..5733e4e2f 100644 --- a/Project.toml +++ b/Project.toml @@ -81,12 +81,14 @@ julia = "1.6" [extras] CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" LineSearches = "d3d80556-e9d4-5f37-9878-2ab0fcc64255" +LuxCUDA = "d0bbae9a-e099-4d5b-a835-1c6931763bda" OptimizationOptimJL = "36348300-93cb-4f02-beb5-3c3902f8871e" OptimizationOptimisers = "42dfb2eb-d2b4-4451-abcd-913932933ac1" OrdinaryDiffEq = "1dea7af3-3e70-54e6-95c3-0bf5283fa5ed" Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" SafeTestsets = "1bc83da4-3b8d-516f-aca4-4fe02f6d838f" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +cuDNN = "02a925ec-e4fe-4b08-9a7e-0d78e3d38ccd" [targets] -test = ["Test", "CUDA", "SafeTestsets", "OptimizationOptimisers", "OptimizationOptimJL", "Pkg", "OrdinaryDiffEq", "LineSearches"] +test = ["Test", "CUDA", "SafeTestsets", "OptimizationOptimisers", "OptimizationOptimJL", "Pkg", "OrdinaryDiffEq", "LineSearches", "cuDNN", "LuxCUDA"] From dfda7e775eb2ae8939fe5459b7dd19949cdb216b Mon Sep 17 00:00:00 2001 From: Sathvik Bhagavan Date: Thu, 18 Jan 2024 12:56:21 +0000 Subject: [PATCH 09/14] test: use LuxCUDA and gpu_device for using GPUs --- test/NNPDE_tests_gpu.jl | 8 ++++---- test/NNPDE_tests_gpu_Lux.jl | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/test/NNPDE_tests_gpu.jl b/test/NNPDE_tests_gpu.jl index 8f33efe71..4a2896a84 100644 --- a/test/NNPDE_tests_gpu.jl +++ b/test/NNPDE_tests_gpu.jl @@ -25,7 +25,7 @@ eq = Dθ(u(θ)) ~ θ^3 + 2.0f0 * θ + (θ^2) * ((1.0f0 + 3 * (θ^2)) / (1.0f0 + u(θ) * (θ + ((1.0f0 + 3.0f0 * (θ^2)) / (1.0f0 + θ + θ^3))) # Initial and boundary conditions -bcs = [u(0.0) ~ 1.0f0] +bcs = [u(0.f0) ~ 1.0f0] # Space and time domains domains = [θ ∈ Interval(0.0f0, 1.0f0)] @@ -85,7 +85,7 @@ chain = Flux.Chain(Dense(2, inner, Flux.σ), Dense(inner, inner, Flux.σ), Dense(inner, inner, Flux.σ), Dense(inner, inner, Flux.σ), - Dense(inner, 1)) |> gpu + Dense(inner, 1)) |> gpu |> f64 strategy = NeuralPDE.StochasticTraining(500) discretization = NeuralPDE.PhysicsInformedNN(chain, @@ -138,7 +138,7 @@ chain = Flux.Chain(Dense(2, inner, Flux.σ), Dense(inner, inner, Flux.σ), Dense(inner, inner, Flux.σ), Dense(inner, inner, Flux.σ), - Dense(inner, 1)) |> gpu + Dense(inner, 1)) |> gpu |> f64 strategy = NeuralPDE.QuasiRandomTraining(500; #points sampling_alg = SobolSample(), @@ -205,7 +205,7 @@ chain = Flux.Chain(Dense(3, inner, Flux.σ), Dense(inner, inner, Flux.σ), Dense(inner, inner, Flux.σ), Dense(inner, inner, Flux.σ), - Dense(inner, 1)) |> gpu + Dense(inner, 1)) |> gpu |> f64 strategy = NeuralPDE.GridTraining(0.05) discretization = NeuralPDE.PhysicsInformedNN(chain, diff --git a/test/NNPDE_tests_gpu_Lux.jl b/test/NNPDE_tests_gpu_Lux.jl index 6055673a7..6dfbe3aac 100644 --- a/test/NNPDE_tests_gpu_Lux.jl +++ b/test/NNPDE_tests_gpu_Lux.jl @@ -1,7 +1,7 @@ using Lux, ComponentArrays, OptimizationOptimisers using Test, NeuralPDE using Optimization -using CUDA, QuasiMonteCarlo +using LuxCUDA, QuasiMonteCarlo import ModelingToolkit: Interval, infimum, supremum using Random @@ -12,7 +12,7 @@ callback = function (p, l) return false end CUDA.allowscalar(false) -#const gpuones = cu(ones(1)) +const gpud = gpu_device() ## ODE println("ode") @@ -41,7 +41,7 @@ chain = Chain(Dense(1, inner, Lux.σ), Dense(inner, 1)) strategy = NeuralPDE.GridTraining(dt) -ps = Lux.setup(Random.default_rng(), chain)[1] |> ComponentArray |> gpu .|> Float64 +ps = Lux.setup(Random.default_rng(), chain)[1] |> ComponentArray |> gpud discretization = NeuralPDE.PhysicsInformedNN(chain, strategy; init_params = ps) @@ -90,7 +90,7 @@ chain = Lux.Chain(Dense(2, inner, Lux.σ), Dense(inner, 1)) strategy = NeuralPDE.StochasticTraining(500) -ps = Lux.setup(Random.default_rng(), chain)[1] |> ComponentArray |> gpu .|> Float64 +ps = Lux.setup(Random.default_rng(), chain)[1] |> ComponentArray |> gpud .|> Float64 discretization = NeuralPDE.PhysicsInformedNN(chain, strategy; init_params = ps) @@ -148,7 +148,7 @@ strategy = NeuralPDE.QuasiRandomTraining(500; #points sampling_alg = SobolSample(), resampling = false, minibatch = 30) -ps = Lux.setup(Random.default_rng(), chain)[1] |> ComponentArray |> gpu .|> Float64 +ps = Lux.setup(Random.default_rng(), chain)[1] |> ComponentArray |> gpud .|> Float64 discretization = NeuralPDE.PhysicsInformedNN(chain, strategy; init_params = ps) @@ -213,7 +213,7 @@ chain = Lux.Chain(Dense(3, inner, Lux.σ), Dense(inner, 1)) strategy = NeuralPDE.GridTraining(0.05) -ps = Lux.setup(Random.default_rng(), chain)[1] |> ComponentArray |> gpu .|> Float64 +ps = Lux.setup(Random.default_rng(), chain)[1] |> ComponentArray |> gpud .|> Float64 discretization = NeuralPDE.PhysicsInformedNN(chain, strategy; init_params = ps) From 2ef31501f476664178c239fc9a7f9b5e70c66c97 Mon Sep 17 00:00:00 2001 From: Sathvik Bhagavan Date: Sat, 20 Jan 2024 14:29:16 +0000 Subject: [PATCH 10/14] build: compat bump DomainSets and add compat for Distributions --- docs/Project.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/Project.toml b/docs/Project.toml index 8cdbea9b5..1f29b7c91 100644 --- a/docs/Project.toml +++ b/docs/Project.toml @@ -26,8 +26,9 @@ SpecialFunctions = "276daf66-3868-5448-9aa4-cd146d93841b" AdvancedHMC = "0.5" Cubature = "1.5" DiffEqBase = "6.106" +Distributions = "0.23, 0.24, 0.25" Documenter = "1" -DomainSets = "0.6" +DomainSets = "0.6, 0.7" Flux = "0.13, 0.14" Integrals = "4" Lux = "0.4, 0.5" From 5df70a8878ab45a209f2292200427c390e7e3019 Mon Sep 17 00:00:00 2001 From: Christopher Rackauckas Date: Sat, 20 Jan 2024 10:29:46 -0500 Subject: [PATCH 11/14] Update NNODE_tests.jl --- test/NNODE_tests.jl | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/test/NNODE_tests.jl b/test/NNODE_tests.jl index 7e3dec522..0ea198526 100644 --- a/test/NNODE_tests.jl +++ b/test/NNODE_tests.jl @@ -10,7 +10,7 @@ linear = (u, p, t) -> cos(2pi * t) tspan = (0.0f0, 1.0f0) u0 = 0.0f0 prob = ODEProblem(linear, u0, tspan) -chain = Flux.Chain(Dense(1, 5, σ), Dense(5, 1)) +chain = Flux.Chain(Dense(1, 5, σ), Dense(5, 1)) |> Flux.f64 luxchain = Lux.Chain(Lux.Dense(1, 5, Lux.σ), Lux.Dense(5, 1)) opt = OptimizationOptimisers.Adam(0.1, (0.9, 0.95)) @@ -26,7 +26,7 @@ sol = solve(prob, NeuralPDE.NNODE(chain, opt), verbose = true, sol = solve(prob, NeuralPDE.NNODE(luxchain, opt), dt = 1 / 20.0f0, verbose = true, abstol = 1.0f-10, maxiters = 200) -@test_throws Any solve(prob, NeuralPDE.NNODE(luxchain, opt; autodiff = true), +Any solve(prob, NeuralPDE.NNODE(luxchain, opt; autodiff = true), dt = 1 / 20.0f0, verbose = true, abstol = 1.0f-10, maxiters = 200) @@ -51,14 +51,14 @@ linear = (u, p, t) -> [cos(2pi * t)] tspan = (0.0f0, 1.0f0) u0 = [0.0f0] prob = ODEProblem(linear, u0, tspan) -chain = Flux.Chain(Dense(1, 5, σ), Dense(5, 1)) +chain = Flux.Chain(Dense(1, 5, σ), Dense(5, 1)) |> Flux.f64 luxchain = Lux.Chain(Lux.Dense(1, 5, σ), Lux.Dense(5, 1)) opt = OptimizationOptimJL.BFGS() sol = solve(prob, NeuralPDE.NNODE(chain, opt), dt = 1 / 20.0f0, abstol = 1e-10, verbose = true, maxiters = 200) -@test_throws Any solve(prob, NeuralPDE.NNODE(chain, opt; autodiff = true), dt = 1 / 20.0f0, +solve(prob, NeuralPDE.NNODE(chain, opt; autodiff = true), dt = 1 / 20.0f0, abstol = 1e-10, verbose = true, maxiters = 200) sol = solve(prob, NeuralPDE.NNODE(chain, opt), abstol = 1.0f-6, @@ -67,7 +67,7 @@ sol = solve(prob, NeuralPDE.NNODE(chain, opt), abstol = 1.0f-6, sol = solve(prob, NeuralPDE.NNODE(luxchain, opt), dt = 1 / 20.0f0, abstol = 1e-10, verbose = true, maxiters = 200) -@test_throws Any solve(prob, NeuralPDE.NNODE(luxchain, opt; autodiff = true), +solve(prob, NeuralPDE.NNODE(luxchain, opt; autodiff = true), dt = 1 / 20.0f0, abstol = 1e-10, verbose = true, maxiters = 200) @@ -83,14 +83,14 @@ linear = (u, p, t) -> @. t^3 + 2 * t + (t^2) * ((1 + 3 * (t^2)) / (1 + t + (t^3) u * (t + ((1 + 3 * (t^2)) / (1 + t + t^3))) linear_analytic = (u0, p, t) -> [exp(-(t^2) / 2) / (1 + t + t^3) + t^2] prob = ODEProblem(ODEFunction(linear, analytic = linear_analytic), [1.0f0], (0.0f0, 1.0f0)) -chain = Flux.Chain(Dense(1, 128, σ), Dense(128, 1)) +chain = Flux.Chain(Dense(1, 128, σ), Dense(128, 1)) |> Flux.f64 luxchain = Lux.Chain(Lux.Dense(1, 128, σ), Lux.Dense(128, 1)) opt = OptimizationOptimisers.Adam(0.01) sol = solve(prob, NeuralPDE.NNODE(chain, opt), verbose = true, maxiters = 400) @test sol.errors[:l2] < 0.5 -@test_throws Any solve(prob, NeuralPDE.NNODE(chain, opt; batch = true), verbose = true, +solve(prob, NeuralPDE.NNODE(chain, opt; batch = true), verbose = true, maxiters = 400) sol = solve(prob, NeuralPDE.NNODE(luxchain, opt), verbose = true, maxiters = 400) @@ -142,7 +142,7 @@ sol = solve(prob, NeuralPDE.NNODE(luxchain, opt; batch = true), verbose = true, linear = (u, p, t) -> -u / 5 + exp(-t / 5) .* cos(t) linear_analytic = (u0, p, t) -> exp(-t / 5) * (u0 + sin(t)) prob = ODEProblem(ODEFunction(linear, analytic = linear_analytic), 0.0f0, (0.0f0, 1.0f0)) -chain = Flux.Chain(Dense(1, 5, σ), Dense(5, 1)) +chain = Flux.Chain(Dense(1, 5, σ), Dense(5, 1)) |> Flux.f64 luxchain = Lux.Chain(Lux.Dense(1, 5, σ), Lux.Dense(5, 1)) opt = OptimizationOptimisers.Adam(0.1) @@ -214,7 +214,7 @@ u0 = [0.0f0, -1.0f0 / 2pi] linear_analytic = (u0, p, t) -> [sin(2pi * t) / 2pi, -cos(2pi * t) / 2pi] odefunction = ODEFunction(linear, analytic = linear_analytic) prob = ODEProblem(odefunction, u0, tspan) -chain = Flux.Chain(Dense(1, 10, σ), Dense(10, 2)) +chain = Flux.Chain(Dense(1, 10, σ), Dense(10, 2)) |> Flux.f64 opt = OptimizationOptimisers.Adam(0.1) alg = NeuralPDE.NNODE(chain, opt; autodiff = false) @@ -313,7 +313,7 @@ sol1 = solve(prob, alg1, verbose = true, abstol = 1.0f-10, maxiters = 200) @test sol1.errors[:l2] < 0.5 # StochasticTraining(Flux Chain) -chain = Flux.Chain(Dense(1, 5, σ), Dense(5, 1)) +chain = Flux.Chain(Dense(1, 5, σ), Dense(5, 1)) |> Flux.f64 (u_, t_) = (u_analytical(ts), ts) function additional_loss(phi, θ) From 4f3a9ac7e6611a35245c4db5e73555844c4af8ca Mon Sep 17 00:00:00 2001 From: Sathvik Bhagavan Date: Sat, 20 Jan 2024 15:35:28 +0000 Subject: [PATCH 12/14] test: fix typo in NNODE tests --- test/NNODE_tests.jl | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/NNODE_tests.jl b/test/NNODE_tests.jl index 0ea198526..2cdc2f1e0 100644 --- a/test/NNODE_tests.jl +++ b/test/NNODE_tests.jl @@ -17,7 +17,7 @@ opt = OptimizationOptimisers.Adam(0.1, (0.9, 0.95)) sol = solve(prob, NeuralPDE.NNODE(chain, opt), dt = 1 / 20.0f0, verbose = true, abstol = 1.0f-10, maxiters = 200) -@test_throws Any solve(prob, NeuralPDE.NNODE(chain, opt; autodiff = true), dt = 1 / 20.0f0, +solve(prob, NeuralPDE.NNODE(chain, opt; autodiff = true), dt = 1 / 20.0f0, verbose = true, abstol = 1.0f-10, maxiters = 200) sol = solve(prob, NeuralPDE.NNODE(chain, opt), verbose = true, @@ -26,7 +26,7 @@ sol = solve(prob, NeuralPDE.NNODE(chain, opt), verbose = true, sol = solve(prob, NeuralPDE.NNODE(luxchain, opt), dt = 1 / 20.0f0, verbose = true, abstol = 1.0f-10, maxiters = 200) -Any solve(prob, NeuralPDE.NNODE(luxchain, opt; autodiff = true), +solve(prob, NeuralPDE.NNODE(luxchain, opt; autodiff = true), dt = 1 / 20.0f0, verbose = true, abstol = 1.0f-10, maxiters = 200) @@ -90,13 +90,13 @@ opt = OptimizationOptimisers.Adam(0.01) sol = solve(prob, NeuralPDE.NNODE(chain, opt), verbose = true, maxiters = 400) @test sol.errors[:l2] < 0.5 -solve(prob, NeuralPDE.NNODE(chain, opt; batch = true), verbose = true, +@test_throws AssertionError solve(prob, NeuralPDE.NNODE(chain, opt; batch = true), verbose = true, maxiters = 400) sol = solve(prob, NeuralPDE.NNODE(luxchain, opt), verbose = true, maxiters = 400) @test sol.errors[:l2] < 0.5 -@test_throws Any solve(prob, NeuralPDE.NNODE(luxchain, opt; batch = true), verbose = true, +@test_throws AssertionError solve(prob, NeuralPDE.NNODE(luxchain, opt; batch = true), verbose = true, maxiters = 400) sol = solve(prob, @@ -150,7 +150,7 @@ sol = solve(prob, NeuralPDE.NNODE(chain, opt), verbose = true, maxiters = 400, abstol = 1.0f-8) @test sol.errors[:l2] < 0.5 -@test_throws Any solve(prob, NeuralPDE.NNODE(chain, opt; batch = true), verbose = true, +@test_throws AssertionError solve(prob, NeuralPDE.NNODE(chain, opt; batch = true), verbose = true, maxiters = 400, abstol = 1.0f-8) @@ -158,7 +158,7 @@ sol = solve(prob, NeuralPDE.NNODE(luxchain, opt), verbose = true, maxiters = 400 abstol = 1.0f-8) @test sol.errors[:l2] < 0.5 -@test_throws Any solve(prob, NeuralPDE.NNODE(luxchain, opt; batch = true), verbose = true, +@test_throws AssertionError solve(prob, NeuralPDE.NNODE(luxchain, opt; batch = true), verbose = true, maxiters = 400, abstol = 1.0f-8) From 607d4f10d145cc8d211d903c36bfbba21397e1ac Mon Sep 17 00:00:00 2001 From: Sathvik Bhagavan Date: Mon, 22 Jan 2024 05:10:11 +0000 Subject: [PATCH 13/14] test: throw error when autodiff is true for all except QuadratureTraining --- test/NNODE_tests.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/NNODE_tests.jl b/test/NNODE_tests.jl index 2cdc2f1e0..3f7edf43c 100644 --- a/test/NNODE_tests.jl +++ b/test/NNODE_tests.jl @@ -17,7 +17,7 @@ opt = OptimizationOptimisers.Adam(0.1, (0.9, 0.95)) sol = solve(prob, NeuralPDE.NNODE(chain, opt), dt = 1 / 20.0f0, verbose = true, abstol = 1.0f-10, maxiters = 200) -solve(prob, NeuralPDE.NNODE(chain, opt; autodiff = true), dt = 1 / 20.0f0, +@test_throws ArgumentError solve(prob, NeuralPDE.NNODE(chain, opt; autodiff = true), dt = 1 / 20.0f0, verbose = true, abstol = 1.0f-10, maxiters = 200) sol = solve(prob, NeuralPDE.NNODE(chain, opt), verbose = true, @@ -26,7 +26,7 @@ sol = solve(prob, NeuralPDE.NNODE(chain, opt), verbose = true, sol = solve(prob, NeuralPDE.NNODE(luxchain, opt), dt = 1 / 20.0f0, verbose = true, abstol = 1.0f-10, maxiters = 200) -solve(prob, NeuralPDE.NNODE(luxchain, opt; autodiff = true), +@test_throws ArgumentError solve(prob, NeuralPDE.NNODE(luxchain, opt; autodiff = true), dt = 1 / 20.0f0, verbose = true, abstol = 1.0f-10, maxiters = 200) @@ -58,7 +58,7 @@ opt = OptimizationOptimJL.BFGS() sol = solve(prob, NeuralPDE.NNODE(chain, opt), dt = 1 / 20.0f0, abstol = 1e-10, verbose = true, maxiters = 200) -solve(prob, NeuralPDE.NNODE(chain, opt; autodiff = true), dt = 1 / 20.0f0, +@test_throws ArgumentError solve(prob, NeuralPDE.NNODE(chain, opt; autodiff = true), dt = 1 / 20.0f0, abstol = 1e-10, verbose = true, maxiters = 200) sol = solve(prob, NeuralPDE.NNODE(chain, opt), abstol = 1.0f-6, @@ -67,7 +67,7 @@ sol = solve(prob, NeuralPDE.NNODE(chain, opt), abstol = 1.0f-6, sol = solve(prob, NeuralPDE.NNODE(luxchain, opt), dt = 1 / 20.0f0, abstol = 1e-10, verbose = true, maxiters = 200) -solve(prob, NeuralPDE.NNODE(luxchain, opt; autodiff = true), +@test_throws ArgumentError solve(prob, NeuralPDE.NNODE(luxchain, opt; autodiff = true), dt = 1 / 20.0f0, abstol = 1e-10, verbose = true, maxiters = 200) From 8a0dccc7fe7779d7be943fa8fc849069aec6bd7c Mon Sep 17 00:00:00 2001 From: Sathvik Bhagavan Date: Mon, 22 Jan 2024 05:10:44 +0000 Subject: [PATCH 14/14] refactor: throw error when autodiff is true for all except QuadratureTraining --- src/ode_solve.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ode_solve.jl b/src/ode_solve.jl index 4ad843283..e724b93d7 100644 --- a/src/ode_solve.jl +++ b/src/ode_solve.jl @@ -272,8 +272,8 @@ end function generate_loss(strategy::GridTraining, phi, f, autodiff::Bool, tspan, p, batch) ts = tspan[1]:(strategy.dx):tspan[2] - # sum(abs2,inner_loss(t,θ) for t in ts) but Zygote generators are broken + autodiff && throw(ArgumentError("autodiff not supported for GridTraining.")) function loss(θ, _) if batch sum(abs2, inner_loss(phi, f, autodiff, ts, θ, p)) @@ -287,6 +287,7 @@ end function generate_loss(strategy::StochasticTraining, phi, f, autodiff::Bool, tspan, p, batch) # sum(abs2,inner_loss(t,θ) for t in ts) but Zygote generators are broken + autodiff && throw(ArgumentError("autodiff not supported for StochasticTraining.")) function loss(θ, _) ts = adapt(parameterless_type(θ), [(tspan[2] - tspan[1]) * rand() + tspan[1] for i in 1:(strategy.points)]) @@ -302,6 +303,7 @@ end function generate_loss(strategy::WeightedIntervalTraining, phi, f, autodiff::Bool, tspan, p, batch) + autodiff && throw(ArgumentError("autodiff not supported for WeightedIntervalTraining.")) minT = tspan[1] maxT = tspan[2]