From 5839f145c9518131bd388942e5bd0f7586dbfef7 Mon Sep 17 00:00:00 2001
From: Navid Constantinou <navidcy@gmail.com>
Date: Thu, 29 Aug 2019 07:14:04 +1000
Subject: [PATCH 01/24] updates julia versions + fixes deprecations

---
 .travis.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 0f7c8f8c..8a63fd60 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,20 +6,20 @@ os:
 
 julia:
   - 1.1
-  # - nightly
+  - nightly
 
 matrix:
   allow_failures:
     - julia: nightly
 
 after_success:
-  - julia --project -e 'using Pkg; Pkg.add("Coverage"); Pkg.dir("GeophysicalFlows"); using Coverage; Coveralls.submit(Coveralls.process_folder())'
-  - julia --project -e 'using Pkg; Pkg.add("Coverage"); Pkg.dir("GeophysicalFlows"); using Coverage; Codecov.submit(Codecov.process_folder())'
+  - julia --project -e 'using Pkg; Pkg.add("Coverage"); import GeophysicalFlows; joinpath(dirname(pathof(GeophysicalFlows)), ".."); using Coverage; Coveralls.submit(Coveralls.process_folder());'
+  - julia --project -e 'using Pkg; Pkg.add("Coverage"); import GeophysicalFlows; joinpath(dirname(pathof(GeophysicalFlows)), ".."); using Coverage; Codecov.submit(Codecov.process_folder())'
 
 jobs:
   include:
     - stage: "Documentation"
-      julia: 1.0
+      julia: 1.1
       os: linux
       script:
         - julia --project=docs/ -e 'using Pkg; Pkg.instantiate();

From 1107f0ee739c7d9388abd958a846e8a5f442b1d7 Mon Sep 17 00:00:00 2001
From: Navid Constantinou <navidcy@gmail.com>
Date: Thu, 29 Aug 2019 07:17:17 +1000
Subject: [PATCH 02/24] updates doc deployment to use Documenter v0.22

---
 docs/Project.toml | 3 ++-
 docs/make.jl      | 4 ++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/docs/Project.toml b/docs/Project.toml
index c2785383..c925d571 100644
--- a/docs/Project.toml
+++ b/docs/Project.toml
@@ -1,5 +1,6 @@
 [deps]
 Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
+GeophysicalFlows = "44ee3b1c-bc02-53fa-8355-8e347616e15e"
 
 [compat]
-Documenter = "~0.20"
+Documenter = "~0.22"
diff --git a/docs/make.jl b/docs/make.jl
index 5b1852e1..e02f1612 100644
--- a/docs/make.jl
+++ b/docs/make.jl
@@ -12,8 +12,8 @@ makedocs(
  doctest = false,
    clean = true,
 checkdocs = :all,
-  format = :html,
- authors = "Gregory L. Wagner and Navid C. Constantinou",
+  format = Documenter.HTML(prettyurls = get(ENV, "CI", nothing) == "true"),
+ authors = "Navid C. Constantinou and Gregory L. Wagner",
 sitename = "GeophysicalFlows.jl",
    pages = Any[
             "Home" => "index.md",

From d1d0571071bc79cc00aaf49d31191b3a06633465 Mon Sep 17 00:00:00 2001
From: Navid Constantinou <navidcy@gmail.com>
Date: Thu, 29 Aug 2019 08:44:16 +1000
Subject: [PATCH 03/24] test_utils now work on both CPU/GPU

---
 src/utils.jl       | 15 +++++++++------
 test/test_utils.jl |  4 ++--
 2 files changed, 11 insertions(+), 8 deletions(-)

diff --git a/src/utils.jl b/src/utils.jl
index b975afe6..5d51d384 100644
--- a/src/utils.jl
+++ b/src/utils.jl
@@ -22,20 +22,23 @@ Generate a real and random two-dimensional vorticity field q(x, y) with
 a Fourier spectrum peaked around a central non-dimensional wavenumber kpeak and
 normalized so that its total energy is E0.
 """
-function peakedisotropicspectrum(g::TwoDGrid, kpeak::Real, E0::Real; mask=ones(size(g.Krsq)), allones=false)
+function peakedisotropicspectrum(g::AbstractGrid{T, A}, kpeak::Real, E0::Real; mask=ones(size(g.Krsq)), allones=false) where {T, A}
   if g.Lx !== g.Ly
-      error("the domain is not square")
+    error("the domain is not square")
   else
     k0 = kpeak*2π/g.Lx
     modk = sqrt.(g.Krsq)
-    psik = zeros(g.nk, g.nl)
+    psik = A(zeros(T, (g.nk, g.nl)))
     psik = @. (modk^2 * (1 + (modk/k0)^4))^(-0.5)
     psik[1, 1] = 0.0
-    psih = @. randn(Complex{typeof(g.Lx)}, size(g.Krsq))*psik
+    phases = randn(Complex{T}, size(g.Krsq))
+    phases_real, phases_imag = real.(phases), imag.(phases)
+    phases = A(phases_real) + im*A(phases_imag)
+    psih = @. phases*psik
     if allones; psih = psik; end
-    psih = psih.*mask
+    psih = psih.*A(mask)
     Ein = real(sum(g.Krsq.*abs2.(psih)/(g.nx*g.ny)^2))
     psih = psih*sqrt(E0/Ein)
-    q = -irfft(g.Krsq.*psih, g.nx)
+    q = A(-irfft(g.Krsq.*psih, g.nx))
   end
 end
diff --git a/test/test_utils.jl b/test/test_utils.jl
index bcddae09..464337c2 100644
--- a/test/test_utils.jl
+++ b/test/test_utils.jl
@@ -1,9 +1,9 @@
 """
 Test the peakedisotropicspectrum function.
 """
-function testpeakedisotropicspectrum()
+function testpeakedisotropicspectrum(dev::Device=CPU())
   n, L = 128, 2π
-  gr = TwoDGrid(n, L)
+  gr = TwoDGrid(dev, n, L)
   k0, E0 = 6, 0.5
   qi = peakedisotropicspectrum(gr, k0, E0; allones=true)
   ρ, qhρ = FourierFlows.radialspectrum(rfft(qi).*gr.invKrsq, gr)

From 6898f57b0c838a1829885653ceab2671277a92de Mon Sep 17 00:00:00 2001
From: Navid Constantinou <navidcy@gmail.com>
Date: Fri, 30 Aug 2019 08:42:07 +1000
Subject: [PATCH 04/24] modifies lambdipole so that it can run on GPU

---
 src/utils.jl | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/utils.jl b/src/utils.jl
index 5d51d384..70fe83b7 100644
--- a/src/utils.jl
+++ b/src/utils.jl
@@ -4,14 +4,15 @@
 Return the 2D vorticity field of the Lamb dipole with strength `U` and radius `R`, centered on
 `center=(xc, yc)` and on the grid `g`. The default value of `center` is the middle of the grid.
 """
-function lambdipole(U, R, g; center=(mean(g.x), mean(g.y)))
+function lambdipole(U, R, g::AbstractGrid{T, A}; center=(mean(g.x), mean(g.y))) where {T, A}
   firstzero = 3.8317059702075123156
   k = firstzero/R # dipole wavenumber for radius R in terms of first zero of besselj
   q0 = -2U*k/besselj(0, k*R) # dipole amplitude for strength U and radius R
 
   xc, yc = center
   r = @. sqrt( (g.x-xc)^2 + (g.y-yc)^2 )
-  q = @. q0 * besselj(1, k*r) * (g.y-yc)/r
+  besselj1 = A([besselj(1, k*r[i, j]) for i=1:g.nx, j=1:g.ny])
+  q = @. q0 * besselj1 * (g.y-yc)/r
   @. q[r >= R] = 0
   q
 end

From af359e7728cc42d293375ecc01f3b3fec23aee08 Mon Sep 17 00:00:00 2001
From: Navid Constantinou <navidcy@gmail.com>
Date: Fri, 30 Aug 2019 08:42:38 +1000
Subject: [PATCH 05/24] adds CuArrays and updates Compat

---
 Manifest.toml | 70 ++++++++++++++++++++++++++++++---------------------
 Project.toml  | 13 ++++++----
 2 files changed, 49 insertions(+), 34 deletions(-)

diff --git a/Manifest.toml b/Manifest.toml
index b442dc8e..78f2dcb9 100644
--- a/Manifest.toml
+++ b/Manifest.toml
@@ -29,15 +29,15 @@ version = "0.5.6"
 
 [[CSTParser]]
 deps = ["Tokenize"]
-git-tree-sha1 = "376a39f1862000442011390f1edf5e7f4dcc7142"
+git-tree-sha1 = "c69698c3d4a7255bc1b4bc2afc09f59db910243b"
 uuid = "00ebfdb7-1f24-5e51-bd34-a7502290713f"
-version = "0.6.0"
+version = "0.6.2"
 
 [[CodecZlib]]
-deps = ["BinaryProvider", "Libdl", "Test", "TranscodingStreams"]
-git-tree-sha1 = "36bbf5374c661054d41410dc53ff752972583b9b"
+deps = ["BinaryProvider", "Libdl", "TranscodingStreams"]
+git-tree-sha1 = "05916673a2627dd91b4969ff8ba6941bc85a960e"
 uuid = "944b1d66-785c-5afd-91f1-9de20f533193"
-version = "0.5.2"
+version = "0.6.0"
 
 [[ColorTypes]]
 deps = ["FixedPointNumbers", "Random"]
@@ -46,10 +46,10 @@ uuid = "3da002f7-5984-5a60-b8a6-cbb66c0b333f"
 version = "0.8.0"
 
 [[Colors]]
-deps = ["ColorTypes", "FixedPointNumbers", "InteractiveUtils", "Printf", "Reexport", "Test"]
-git-tree-sha1 = "9f0a0210450acb91c730b730a994f8eef1d3d543"
+deps = ["ColorTypes", "FixedPointNumbers", "InteractiveUtils", "Printf", "Reexport"]
+git-tree-sha1 = "c9c1845d6bf22e34738bee65c357a69f416ed5d1"
 uuid = "5ae59095-9a9b-59fe-a467-6f913c188581"
-version = "0.9.5"
+version = "0.9.6"
 
 [[Compat]]
 deps = ["Base64", "Dates", "DelimitedFiles", "Distributed", "InteractiveUtils", "LibGit2", "Libdl", "LinearAlgebra", "Markdown", "Mmap", "Pkg", "Printf", "REPL", "Random", "Serialization", "SharedArrays", "Sockets", "SparseArrays", "Statistics", "Test", "UUIDs", "Unicode"]
@@ -88,10 +88,10 @@ deps = ["Random", "Serialization", "Sockets"]
 uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"
 
 [[FFTW]]
-deps = ["AbstractFFTs", "BinaryProvider", "Compat", "Conda", "Libdl", "LinearAlgebra", "Reexport", "Test"]
-git-tree-sha1 = "29cda58afbf62f35b1a094882ad6c745a47b2eaa"
+deps = ["AbstractFFTs", "BinaryProvider", "Conda", "Libdl", "LinearAlgebra", "Reexport", "Test"]
+git-tree-sha1 = "e1a479d3c972f20c9a70563eec740bbfc786f515"
 uuid = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341"
-version = "0.2.4"
+version = "0.3.0"
 
 [[FileIO]]
 deps = ["Pkg"]
@@ -105,16 +105,16 @@ uuid = "53c48c17-4a7d-5ca2-90c5-79b7896eea93"
 version = "0.6.1"
 
 [[FourierFlows]]
-deps = ["FFTW", "Interpolations", "JLD2", "LinearAlgebra", "Random", "Reexport", "Statistics"]
-git-tree-sha1 = "1addb960281c1cbd44156975ebb8e21a99f79232"
+deps = ["Coverage", "FFTW", "Interpolations", "JLD2", "LinearAlgebra", "Printf", "Random", "Reexport", "Requires", "Statistics"]
+git-tree-sha1 = "e430df633d39e02e772856e6cb745b99b069de19"
 uuid = "2aec4490-903f-5c70-9b11-9bed06a700e1"
-version = "0.3.0"
+version = "0.3.1"
 
 [[HTTP]]
 deps = ["Base64", "Dates", "IniFile", "MbedTLS", "Sockets"]
-git-tree-sha1 = "03ddc88af7f2d963fac5aa9f3ac8e11914d68a78"
+git-tree-sha1 = "c4a527dba1d26add0e85946e1a53f42a1b343acc"
 uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3"
-version = "0.8.4"
+version = "0.8.5"
 
 [[IniFile]]
 deps = ["Test"]
@@ -139,10 +139,10 @@ uuid = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
 version = "0.1.2"
 
 [[JSON]]
-deps = ["Dates", "Distributed", "Mmap", "Sockets", "Test", "Unicode"]
-git-tree-sha1 = "1f7a25b53ec67f5e9422f1f551ee216503f4a0fa"
+deps = ["Dates", "Mmap", "Parsers", "Unicode"]
+git-tree-sha1 = "b34d7cef7b337321e97d22242c3c2b91f476748e"
 uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
-version = "0.20.0"
+version = "0.21.0"
 
 [[LaTeXStrings]]
 deps = ["Compat"]
@@ -164,20 +164,20 @@ uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
 
 [[MacroTools]]
-deps = ["CSTParser", "Compat", "DataStructures", "Test"]
-git-tree-sha1 = "daecd9e452f38297c686eba90dba2a6d5da52162"
+deps = ["CSTParser", "Compat", "DataStructures", "Test", "Tokenize"]
+git-tree-sha1 = "d6e9dedb8c92c3465575442da456aec15a89ff76"
 uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
-version = "0.5.0"
+version = "0.5.1"
 
 [[Markdown]]
 deps = ["Base64"]
 uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
 
 [[MbedTLS]]
-deps = ["BinaryProvider", "Dates", "Distributed", "Libdl", "Random", "Sockets", "Test"]
-git-tree-sha1 = "2d94286a9c2f52c63a16146bb86fd6cdfbf677c6"
+deps = ["BinaryProvider", "Dates", "Libdl", "Random", "Sockets"]
+git-tree-sha1 = "85f5947b53c8cfd53ccfa3f4abae31faa22c2181"
 uuid = "739be429-bea8-5141-9913-cc70e7f3736d"
-version = "0.6.8"
+version = "0.7.0"
 
 [[Mmap]]
 uuid = "a63ad114-7e13-5084-954f-fe012c677804"
@@ -193,6 +193,12 @@ git-tree-sha1 = "c4c13474d23c60d20a67b217f1d7f22a40edf8f1"
 uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
 version = "1.1.0"
 
+[[Parsers]]
+deps = ["Dates", "Test"]
+git-tree-sha1 = "db2b35dedab3c0e46dc15996d170af07a5ab91c9"
+uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
+version = "0.3.6"
+
 [[Pkg]]
 deps = ["Dates", "LibGit2", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"]
 uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
@@ -233,6 +239,12 @@ git-tree-sha1 = "7b1d07f411bc8ddb7977ec7f377b97b158514fe0"
 uuid = "189a3867-3050-52da-a836-e630ba90ab69"
 version = "0.2.0"
 
+[[Requires]]
+deps = ["Test"]
+git-tree-sha1 = "f6fbf4ba64d295e146e49e021207993b6b48c7d1"
+uuid = "ae029012-a4dd-5104-9daa-d747884805df"
+version = "0.5.2"
+
 [[SHA]]
 uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
 
@@ -271,15 +283,15 @@ deps = ["Distributed", "InteractiveUtils", "Logging", "Random"]
 uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [[Tokenize]]
-git-tree-sha1 = "0de343efc07da00cd449d5b04e959ebaeeb3305d"
+git-tree-sha1 = "dfcdbbfb2d0370716c815cbd6f8a364efb6f42cf"
 uuid = "0796e94c-ce3b-5d07-9a54-7f471281c624"
-version = "0.5.4"
+version = "0.5.6"
 
 [[TranscodingStreams]]
 deps = ["Random", "Test"]
-git-tree-sha1 = "a25d8e5a28c3b1b06d3859f30757d43106791919"
+git-tree-sha1 = "7c53c35547de1c5b9d46a4797cf6d8253807108c"
 uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa"
-version = "0.9.4"
+version = "0.9.5"
 
 [[URIParser]]
 deps = ["Test", "Unicode"]
diff --git a/Project.toml b/Project.toml
index 316f3478..f91d6627 100644
--- a/Project.toml
+++ b/Project.toml
@@ -17,12 +17,15 @@ Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
 SpecialFunctions = "276daf66-3868-5448-9aa4-cd146d93841b"
 Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 
+[compat]
+FFTW = "≥ 0.2.4"
+FourierFlows = "≥ 0.3.1"
+JLD2 = "≥ 0.1.2"
+julia = "≥ 1.0.0"
+
 [extras]
+CuArrays = "3a865a2d-5b23-5a0f-bc46-62713ec82fae"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["Test"]
-
-[compat]
-FourierFlows = "≥ 0.3.0"
-julia = "≥ 1.0.0"
+test = ["Test", "CuArrays"]

From e35372950f8c221b7ef6045096510b52e7fe3a1d Mon Sep 17 00:00:00 2001
From: Navid Constantinou <navidcy@gmail.com>
Date: Fri, 30 Aug 2019 08:44:02 +1000
Subject: [PATCH 06/24] modifies TwoDTurb module so that it runs both on CPU
 and GPU

---
 src/twodturb.jl | 88 +++++++++++++++++++++++++++++--------------------
 1 file changed, 52 insertions(+), 36 deletions(-)

diff --git a/src/twodturb.jl b/src/twodturb.jl
index 7b6e9451..cebf26b5 100644
--- a/src/twodturb.jl
+++ b/src/twodturb.jl
@@ -22,11 +22,6 @@ using FourierFlows: getfieldspecs, structvarsexpr, parsevalsum, parsevalsum2
 
 abstract type TwoDTurbVars <: AbstractVars end
 
-const physicalvars = [:zeta, :u, :v]
-const transformvars = [ Symbol(var, :h) for var in physicalvars ]
-const forcedvars = [:Fh]
-const stochforcedvars = [:prevsol]
-
 nothingfunction(args...) = nothing
 
 """
@@ -50,14 +45,15 @@ function Problem(;
      stepper = "RK4",
        calcF = nothingfunction,
   stochastic = false,
-           T = Float64
+           T = Float64,
+         dev = CPU()
 )
 
-  gr = TwoDGrid(nx, Lx, ny, Ly)
+  gr = TwoDGrid(dev, nx, Lx, ny, Ly)
   pr = Params{T}(nu, nnu, mu, nmu, calcF)
-  vs = calcF == nothingfunction ? Vars(gr) : (stochastic ? StochasticForcedVars(gr) : ForcedVars(gr))
+  vs = calcF == nothingfunction ? Vars(dev, gr) : (stochastic ? StochasticForcedVars(dev, gr) : ForcedVars(dev, gr))
   eq = Equation(pr, gr)
-  FourierFlows.Problem(eq, stepper, dt, gr, vs, pr)
+  FourierFlows.Problem(eq, stepper, dt, gr, vs, pr, dev)
 end
 
 
@@ -100,50 +96,70 @@ end
 # Vars
 # ----
 
-varspecs = cat(
-  getfieldspecs(physicalvars, :(Array{T,2})),
-  getfieldspecs(transformvars, :(Array{Complex{T},2})),
-  dims=1)
+struct Vars{Aphys, Atrans} <: TwoDTurbVars
+   zeta :: Aphys
+      u :: Aphys
+      v :: Aphys
+  zetah :: Atrans
+     uh :: Atrans
+     vh :: Atrans
+end
 
-forcedvarspecs = cat(varspecs, getfieldspecs(forcedvars, :(Array{Complex{T},2})), dims=1)
-stochforcedvarspecs = cat(forcedvarspecs, getfieldspecs(stochforcedvars, :(Array{Complex{T},2})), dims=1)
+struct ForcedVars{Aphys, Atrans} <: TwoDTurbVars
+   zeta :: Aphys
+      u :: Aphys
+      v :: Aphys
+  zetah :: Atrans
+     uh :: Atrans
+     vh :: Atrans
+     Fh :: Atrans
+end
+
+struct StochasticForcedVars{Aphys, Atrans} <: TwoDTurbVars
+     zeta :: Aphys
+        u :: Aphys
+        v :: Aphys
+    zetah :: Atrans
+       uh :: Atrans
+       vh :: Atrans
+       Fh :: Atrans
+  prevsol :: Atrans
+end
 
-# Construct Vars types
-eval(structvarsexpr(:Vars, varspecs; parent=:TwoDTurbVars))
-eval(structvarsexpr(:ForcedVars, forcedvarspecs; parent=:TwoDTurbVars))
-eval(structvarsexpr(:StochasticForcedVars, stochforcedvarspecs; parent=:TwoDTurbVars))
 
 """
-    Vars(g)
+    Vars(dev, g)
 
-Returns the vars for unforced two-dimensional turbulence with grid g.
+Returns the vars for unforced two-dimensional turbulence on device dev and with 
+  grid g.
 """
-function Vars(g; T=typeof(g.Lx))
-  @createarrays T (g.nx, g.ny) zeta u v
-  @createarrays Complex{T} (g.nkr, g.nl) sol zetah uh vh
+function Vars(::Dev, g::AbstractGrid{T}) where {Dev, T}
+  @devzeros Dev T (g.nx, g.ny) zeta u v
+  @devzeros Dev Complex{T} (g.nkr, g.nl) zetah uh vh
   Vars(zeta, u, v, zetah, uh, vh)
 end
 
 """
-    ForcedVars(g)
+    ForcedVars(dev, g)
 
-Returns the vars for forced two-dimensional turbulence with grid g.
+Returns the vars for forced two-dimensional turbulence on device dev and with 
+  grid g.
 """
-function ForcedVars(g; T=typeof(g.Lx))
-  v = Vars(g; T=T)
-  Fh = zeros(Complex{T}, (g.nkr, g.nl))
+function ForcedVars(dev::Dev, g::AbstractGrid{T}) where {Dev, T}
+  v = Vars(dev, g)
+  @devzeros Dev Complex{T} (g.nkr, g.nl) Fh
   ForcedVars(getfield.(Ref(v), fieldnames(typeof(v)))..., Fh)
 end
 
 """
-    StochasticForcedVars(g; T)
+    StochasticForcedVars(dev, g)
 
-Returns the vars for stochastically forced two-dimensional turbulence with grid
-g.
+Returns the vars for stochastically forced two-dimensional turbulence on device
+  dev and with grid g.
 """
-function StochasticForcedVars(g; T=typeof(g.Lx))
-  v = ForcedVars(g; T=T)
-  prevsol = zeros(Complex{T}, (g.nkr, g.nl))
+function StochasticForcedVars(dev::Dev, g::AbstractGrid{T}) where {Dev, T}
+  v = ForcedVars(dev, g)
+  @devzeros Dev Complex{T} (g.nkr, g.nl) prevsol
   StochasticForcedVars(getfield.(Ref(v), fieldnames(typeof(v)))..., prevsol)
 end
 
@@ -281,7 +297,7 @@ Returns the domain-averaged rate of work of energy by the forcing Fh.
 end
 
 @inline function work(sol, v::StochasticForcedVars, g)
-  @. v.uh = g.invKrsq * (v.prevsol + sol)/2.0 * conj(v.Fh) # Stratonovich
+  @. v.uh = g.invKrsq * (v.prevsol + sol)/2 * conj(v.Fh) # Stratonovich
   # @. v.uh = g.invKrsq * v.prevsol * conj(v.Fh)           # Ito
   1/(g.Lx*g.Ly)*parsevalsum(v.uh, g)
 end

From 733833beabbc694bf3a7f49aa6948d5a77e722c1 Mon Sep 17 00:00:00 2001
From: Navid Constantinou <navidcy@gmail.com>
Date: Fri, 30 Aug 2019 08:44:49 +1000
Subject: [PATCH 07/24] the twodturb tests now run both on CPU and GPU

---
 test/runtests.jl      | 41 +++++++++++++++++++++-----------
 test/test_twodturb.jl | 54 +++++++++++++++++++++----------------------
 2 files changed, 53 insertions(+), 42 deletions(-)

diff --git a/test/runtests.jl b/test/runtests.jl
index fa7e873a..ced8d807 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -1,11 +1,10 @@
-#!/usr/bin/env julia
-
 using
   FourierFlows,
   Test,
   Statistics,
   Random,
-  FFTW
+  FFTW,
+  Test
 
 import # use 'import' rather than 'using' for submodules to keep namespace clean
   GeophysicalFlows.TwoDTurb,
@@ -16,6 +15,11 @@ import # use 'import' rather than 'using' for submodules to keep namespace clean
 using FourierFlows: parsevalsum, xmoment, ymoment
 using GeophysicalFlows: lambdipole, peakedisotropicspectrum
 
+# the devices on which tests will run
+devices = (CPU(),)
+@hascuda devices = (CPU(), GPU())
+@hascuda using CuArrays
+
 const rtol_lambdipole = 1e-2 # tolerance for lamb dipole tests
 const rtol_multilayerqg = 1e-13 # tolerance for multilayerqg forcing tests
 const rtol_twodturb = 1e-13 # tolerance for twodturb forcing tests
@@ -28,22 +32,31 @@ cfl(prob) = cfl(prob.vars.u, prob.vars.v, prob.clock.dt, prob.grid.dx)
 # Run tests
 testtime = @elapsed begin
 
-@testset "Utils" begin
-  include("test_utils.jl")
+for dev in devices
+  
+  println("testing on "*string(typeof(dev)))
+  
+  @testset "Utils" begin
+    include("test_utils.jl")
 
-  @test testpeakedisotropicspectrum()
-end
+    @test testpeakedisotropicspectrum(dev)
+  end
 
-@testset "TwoDTurb" begin
-  include("test_twodturb.jl")
+  @testset "TwoDTurb" begin
+    include("test_twodturb.jl")
+
+    @test test_twodturb_advection(0.0005, "ForwardEuler", dev)
+    @test test_twodturb_lambdipole(256, 1e-3, dev)
+    @test test_twodturb_stochasticforcingbudgets(dev)
+    @test test_twodturb_deterministicforcingbudgets(dev)
+    @test test_twodturb_energyenstrophy(dev)
+  end
 
-  @test test_twodturb_advection(0.0005, "ForwardEuler")
-  @test test_twodturb_lambdipole(256, 1e-3)
-  @test test_twodturb_stochasticforcingbudgets()
-  @test test_twodturb_deterministicforcingbudgets()
-  @test test_twodturb_energyenstrophy()
 end
 
+
+println("rest of tests only on CPU")
+
 @testset "BarotropicQG" begin
   include("test_barotropicqg.jl")
 
diff --git a/test/test_twodturb.jl b/test/test_twodturb.jl
index ec73a09b..fddc61ed 100644
--- a/test/test_twodturb.jl
+++ b/test/test_twodturb.jl
@@ -1,11 +1,11 @@
-function test_twodturb_lambdipole(n, dt; L=2π, Ue=1, Re=L/20, nu=0.0, nnu=1, ti=L/Ue*0.01, nm=3)
+function test_twodturb_lambdipole(n, dt, dev::Device=CPU(); L=2π, Ue=1, Re=L/20, nu=0.0, nnu=1, ti=L/Ue*0.01, nm=3)
   nt = round(Int, ti/dt)
-  prob = TwoDTurb.Problem(nx=n, Lx=L, nu=nu, nnu=nnu, dt=dt, stepper="FilteredRK4")
+  prob = TwoDTurb.Problem(nx=n, Lx=L, nu=nu, nnu=nnu, dt=dt, stepper="FilteredRK4", dev=dev)
   zeta₀ = lambdipole(Ue, Re, prob.grid)
   TwoDTurb.set_zeta!(prob, zeta₀)
 
-  xzeta = zeros(nm)   # centroid of abs(zeta)
-  Ue_m = zeros(nm) # measured dipole speed
+  xzeta = zeros(nm) # centroid of abs(zeta)
+  Ue_m = zeros(nm)  # measured dipole speed
   x, y = gridpoints(prob.grid)
   zeta = prob.vars.zeta
 
@@ -20,19 +20,19 @@ function test_twodturb_lambdipole(n, dt; L=2π, Ue=1, Re=L/20, nu=0.0, nnu=1, ti
   isapprox(Ue, mean(Ue_m[2:end]), rtol=rtol_lambdipole)
 end
 
-function test_twodturb_stochasticforcingbudgets(; n=256, L=2π, dt=0.005, nu=1e-7, nnu=2, mu=1e-1, nmu=0, tf=0.1/mu)
+function test_twodturb_stochasticforcingbudgets(dev::Device=CPU(); n=256, L=2π, dt=0.005, nu=1e-7, nnu=2, mu=1e-1, nmu=0, tf=0.1/mu)
   nt = round(Int, tf/dt)
 
-  # Forcing
+  # Forcing parameters
   kf, dkf = 12.0, 2.0
   ε = 0.1
 
-  gr  = TwoDGrid(n, L)
+  gr  = TwoDGrid(dev, n, L)
   x, y = gridpoints(gr)
 
-  Kr = [ gr.kr[i] for i=1:gr.nkr, j=1:gr.nl]
+  Kr = ArrayType(dev)([ gr.kr[i] for i=1:gr.nkr, j=1:gr.nl])
 
-  force2k = zero(gr.Krsq)
+  force2k = ArrayType(dev)(zero(gr.Krsq))
   @. force2k = exp(-(sqrt(gr.Krsq)-kf)^2/(2*dkf^2))
   @. force2k[gr.Krsq .< 2.0^2 ] = 0
   @. force2k[gr.Krsq .> 20.0^2 ] = 0
@@ -43,14 +43,14 @@ function test_twodturb_stochasticforcingbudgets(; n=256, L=2π, dt=0.005, nu=1e-
   Random.seed!(1234)
 
   function calcF!(Fh, sol, t, cl, v, p, g)
-    eta = exp.(2π*im*rand(Float64, size(sol)))/sqrt(cl.dt)
+    eta = ArrayType(dev)(exp.(2π*im*rand(Float64, size(sol)))/sqrt(cl.dt))
     eta[1, 1] = 0.0
     @. Fh = eta * sqrt(force2k)
     nothing
   end
 
   prob = TwoDTurb.Problem(nx=n, Lx=L, nu=nu, nnu=nnu, mu=mu, nmu=nmu, dt=dt,
-   stepper="RK4", calcF=calcF!, stochastic=true)
+   stepper="RK4", calcF=calcF!, stochastic=true, dev=dev)
 
   sol, cl, v, p, g = prob.sol, prob.clock, prob.vars, prob.params, prob.grid;
 
@@ -83,28 +83,26 @@ function test_twodturb_stochasticforcingbudgets(; n=256, L=2π, dt=0.005, nu=1e-
 end
 
 
-function test_twodturb_deterministicforcingbudgets(; n=256, dt=0.01, L=2π, nu=1e-7, nnu=2, mu=1e-1, nmu=0)
+function test_twodturb_deterministicforcingbudgets(dev::Device=CPU(); n=256, dt=0.01, L=2π, nu=1e-7, nnu=2, mu=1e-1, nmu=0)
   n, L  = 256, 2π
   nu, nnu = 1e-7, 2
   mu, nmu = 1e-1, 0
   dt, tf = 0.005, 0.1/mu
   nt = round(Int, tf/dt)
-  g = TwoDGrid(n, L)
-
-  gr  = TwoDGrid(n, L)
+  
+  gr  = TwoDGrid(dev, n, L)
   x, y = gridpoints(gr)
 
   # Forcing = 0.01cos(4x)cos(5y)cos(2t)
-
-  f = @. 0.01*cos(4*x)*cos(5*y)
+  f = @. 0.01cos(4x)*cos(5y)
   fh = rfft(f)
-  function calcF!(Fh, sol, t, cl, v, p, g)
-    @. Fh = fh*cos(2*t)
+  function calcF!(Fh, sol, t, cl, v, p, g::AbstractGrid{T, A}) where {T, A}
+    Fh = fh*cos(2t)
     nothing
   end
 
   prob = TwoDTurb.Problem(nx=n, Lx=L, nu=nu, nnu=nnu, mu=mu, nmu=nmu, dt=dt,
-   stepper="RK4", calcF=calcF!, stochastic=false)
+   stepper="RK4", calcF=calcF!, stochastic=false, dev=dev)
 
   sol, cl, v, p, g = prob.sol, prob.clock, prob.vars, prob.params, prob.grid
 
@@ -146,14 +144,14 @@ forcing Ff is derived according to Ff = ∂ζf/∂t + J(ψf, ζf) - nuΔζf. One
 to the vorticity equation forced by this Ff is then ζf. (This solution may not
 be realized, at least at long times, if it is unstable.)
 """
-function test_twodturb_advection(dt, stepper; n=128, L=2π, nu=1e-2, nnu=1, mu=0.0, nmu=0)
+function test_twodturb_advection(dt, stepper, dev::Device=CPU(); n=128, L=2π, nu=1e-2, nnu=1, mu=0.0, nmu=0)
   n, L  = 128, 2π
   nu, nnu = 1e-2, 1
   mu, nmu = 0.0, 0
   tf = 1.0
   nt = round(Int, tf/dt)
 
-  gr  = TwoDGrid(n, L)
+  gr  = TwoDGrid(dev, n, L)
   x, y = gridpoints(gr)
 
    psif = @.   sin(2x)*cos(2y) +  2sin(x)*cos(3y)
@@ -172,7 +170,7 @@ function test_twodturb_advection(dt, stepper; n=128, L=2π, nu=1e-2, nnu=1, mu=0
     nothing
   end
 
-  prob = TwoDTurb.Problem(nx=n, Lx=L, nu=nu, nnu=nnu, mu=mu, nmu=nmu, dt=dt, stepper=stepper, calcF=calcF!, stochastic=false)
+  prob = TwoDTurb.Problem(nx=n, Lx=L, nu=nu, nnu=nnu, mu=mu, nmu=nmu, dt=dt, stepper=stepper, calcF=calcF!, stochastic=false, dev=dev)
   sol, cl, p, v, g = prob.sol, prob.clock, prob.params, prob.vars, prob.grid
   TwoDTurb.set_zeta!(prob, zetaf)
 
@@ -182,10 +180,10 @@ function test_twodturb_advection(dt, stepper; n=128, L=2π, nu=1e-2, nnu=1, mu=0
   isapprox(prob.vars.zeta, zetaf, rtol=rtol_twodturb)
 end
 
-function test_twodturb_energyenstrophy()
+function test_twodturb_energyenstrophy(dev::Device=CPU())
   nx, Lx  = 128, 2π
   ny, Ly  = 128, 3π
-  gr = TwoDGrid(nx, Lx, ny, Ly)
+  gr = TwoDGrid(dev, nx, Lx, ny, Ly)
   k0, l0 = gr.k[2], gr.l[2] # fundamental wavenumbers
   x, y = gridpoints(gr)
 
@@ -195,7 +193,7 @@ function test_twodturb_energyenstrophy()
   energy_calc = 29/9
   enstrophy_calc = 2701/162
 
-  prob = TwoDTurb.Problem(nx=nx, Lx=Lx, ny=ny, Ly=Ly, stepper="ForwardEuler")
+  prob = TwoDTurb.Problem(nx=nx, Lx=Lx, ny=ny, Ly=Ly, stepper="ForwardEuler", dev=dev)
 
   TwoDTurb.set_zeta!(prob, zeta0)
   TwoDTurb.updatevars!(prob)
@@ -203,6 +201,6 @@ function test_twodturb_energyenstrophy()
   energyzeta0 = TwoDTurb.energy(prob)
   enstrophyzeta0 = TwoDTurb.enstrophy(prob)
 
-  (isapprox(energyzeta0, 29.0/9, rtol=rtol_twodturb) &&
-   isapprox(enstrophyzeta0, 2701.0/162, rtol=rtol_twodturb))
+  (isapprox(energyzeta0, energy_calc, rtol=rtol_twodturb) &&
+   isapprox(enstrophyzeta0, enstrophy_calc, rtol=rtol_twodturb))
 end

From 61c38bb1e5a910e3bb98fa79f68aa924a3a86b88 Mon Sep 17 00:00:00 2001
From: Navid Constantinou <navidcy@gmail.com>
Date: Fri, 30 Aug 2019 08:52:41 +1000
Subject: [PATCH 08/24] use julia v1.2 for tests and docs

---
 .travis.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 8a63fd60..c08c9cab 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -5,7 +5,7 @@ os:
   - linux
 
 julia:
-  - 1.1
+  - 1.2
   - nightly
 
 matrix:
@@ -19,7 +19,7 @@ after_success:
 jobs:
   include:
     - stage: "Documentation"
-      julia: 1.1
+      julia: 1.2
       os: linux
       script:
         - julia --project=docs/ -e 'using Pkg; Pkg.instantiate();

From 12827dbaf160eca587027e1f7e81046094ed3b24 Mon Sep 17 00:00:00 2001
From: Navid Constantinou <navidcy@gmail.com>
Date: Fri, 30 Aug 2019 10:21:14 +1000
Subject: [PATCH 09/24] bumps julia version to 1.2

---
 appveyor.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/appveyor.yml b/appveyor.yml
index 6681175c..9cd68255 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -1,6 +1,6 @@
 environment:
   matrix:
-  - julia_version: 1
+  - julia_version: 1.2
   - julia_version: nightly
 
 platform:

From eaa0f12ba824e8f82b14aea79e86e0da0f457f36 Mon Sep 17 00:00:00 2001
From: Navid Constantinou <navidcy@gmail.com>
Date: Fri, 30 Aug 2019 11:47:11 +1000
Subject: [PATCH 10/24] adds some more tests to increase coverage

---
 test/runtests.jl      | 1 +
 test/test_twodturb.jl | 9 +++++++--
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/test/runtests.jl b/test/runtests.jl
index ced8d807..7bfa376c 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -50,6 +50,7 @@ for dev in devices
     @test test_twodturb_stochasticforcingbudgets(dev)
     @test test_twodturb_deterministicforcingbudgets(dev)
     @test test_twodturb_energyenstrophy(dev)
+    @test TwoDTurb.nothingfunction() == nothing
   end
 
 end
diff --git a/test/test_twodturb.jl b/test/test_twodturb.jl
index fddc61ed..9af799cb 100644
--- a/test/test_twodturb.jl
+++ b/test/test_twodturb.jl
@@ -194,13 +194,18 @@ function test_twodturb_energyenstrophy(dev::Device=CPU())
   enstrophy_calc = 2701/162
 
   prob = TwoDTurb.Problem(nx=nx, Lx=Lx, ny=ny, Ly=Ly, stepper="ForwardEuler", dev=dev)
+  
+  sol, cl, v, p, g = prob.sol, prob.clock, prob.vars, prob.params, prob.grid;
 
   TwoDTurb.set_zeta!(prob, zeta0)
   TwoDTurb.updatevars!(prob)
 
   energyzeta0 = TwoDTurb.energy(prob)
   enstrophyzeta0 = TwoDTurb.enstrophy(prob)
+  
+  params = TwoDTurb.Params(p.nu, p.nnu)
 
   (isapprox(energyzeta0, energy_calc, rtol=rtol_twodturb) &&
-   isapprox(enstrophyzeta0, enstrophy_calc, rtol=rtol_twodturb))
-end
+   isapprox(enstrophyzeta0, enstrophy_calc, rtol=rtol_twodturb) &&
+   TwoDTurb.addforcing!(prob.timestepper.N, sol, cl.t, cl, v, p, g)==nothing && p == params)
+end
\ No newline at end of file

From d19d84fe286a445dc64e3297133734d6b9ab212e Mon Sep 17 00:00:00 2001
From: Navid Constantinou <navidcy@gmail.com>
Date: Fri, 30 Aug 2019 11:52:53 +1000
Subject: [PATCH 11/24] =?UTF-8?q?changes=20nu,=20mu=20-->=20=CE=BD,=20?=
 =?UTF-8?q?=CE=BC?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 examples/twodturb_mcwilliams1984.jl    | 12 ++++----
 examples/twodturb_randomdecay.jl       |  6 ++--
 examples/twodturb_stochasticforcing.jl | 28 +++++++++----------
 src/twodturb.jl                        | 38 +++++++++++++-------------
 test/test_twodturb.jl                  | 36 ++++++++++++------------
 5 files changed, 60 insertions(+), 60 deletions(-)

diff --git a/examples/twodturb_mcwilliams1984.jl b/examples/twodturb_mcwilliams1984.jl
index 3a4e3816..7e706d21 100644
--- a/examples/twodturb_mcwilliams1984.jl
+++ b/examples/twodturb_mcwilliams1984.jl
@@ -9,8 +9,8 @@ import GeophysicalFlows: peakedisotropicspectrum
 # Parameters
   n = 256
   L = 2π
-nnu = 2
- nu = 0.0
+ nν = 2
+  ν = 0.0
  dt = 1e-2
 nsteps = 5000
 nsubs = 200
@@ -26,7 +26,7 @@ if isfile(filename); rm(filename); end
 if !isdir(plotpath); mkdir(plotpath); end
 
 # Initialize problem
-prob = TwoDTurb.Problem(; nx=n, Lx=L, ny=n, Ly=L, nu=nu, nnu=nnu, dt=dt, stepper="FilteredRK4")
+prob = TwoDTurb.Problem(; nx=n, Lx=L, ny=n, Ly=L, ν=ν, nν=nν, dt=dt, stepper="FilteredRK4", dev=dev)
 
 sol, cl, vs, gr, filter = prob.sol, prob.clock, prob.vars, prob.grid, prob.timestepper.filter
 x, y = gridpoints(gr)
@@ -45,8 +45,8 @@ diags = [E, Z] # A list of Diagnostics types passed to "stepforward!" will
 # be updated every timestep.
 
 # Create Output
-get_sol(prob) = prob.sol # extracts the Fourier-transformed solution
-get_u(prob) = irfft(im*gr.l.*gr.invKrsq.*sol, gr.nx)
+get_sol(prob) = Array(prob.sol) # extracts the Fourier-transformed solution
+get_u(prob) = Array(irfft(im*gr.l.*gr.invKrsq.*sol, gr.nx))
 out = Output(prob, filename, (:sol, get_sol), (:u, get_u))
 saveproblem(out)
 
@@ -89,7 +89,7 @@ while cl.step < nsteps
   println(log)
   plot_output(prob, fig, axs; drawcolorbar=false)
 end
-
+println("finished")
 plot_output(prob, fig, axs; drawcolorbar=false)
 
 savename = @sprintf("%s_%09d.png", joinpath(plotpath, plotname), cl.step)
diff --git a/examples/twodturb_randomdecay.jl b/examples/twodturb_randomdecay.jl
index f6f1aa03..93c51477 100644
--- a/examples/twodturb_randomdecay.jl
+++ b/examples/twodturb_randomdecay.jl
@@ -10,14 +10,14 @@ import GeophysicalFlows.TwoDTurb
 
 nx = 256        # Resolution
 Lx = 2π         # Domain size
-nu = 1e-6       # Viscosity
-nnu = 1         # Order of (hyper-)viscosity. nnu=1 means Laplacian
+ ν = 1e-6       # Viscosity
+nν = 1          # Order of (hyper-)viscosity. nν=1 means Laplacian
 dt = 0.1        # Timestep
 nint = 200      # Number of steps between plots
 ntot = 10nint   # Number of total timesteps
 
 # Define problem
-prob = TwoDTurb.Problem(nx=nx, Lx=Lx, nu=nu, nnu=nnu, dt=dt, stepper="FilteredRK4")
+prob = TwoDTurb.Problem(nx=nx, Lx=Lx, ν=ν, nν=nν, dt=dt, stepper="FilteredRK4")
 TwoDTurb.set_zeta!(prob, rand(nx, nx))
 
 cl, vs, gr = prob.clock, prob.vars, prob.grid
diff --git a/examples/twodturb_stochasticforcing.jl b/examples/twodturb_stochasticforcing.jl
index 687f7bdb..83947533 100644
--- a/examples/twodturb_stochasticforcing.jl
+++ b/examples/twodturb_stochasticforcing.jl
@@ -7,9 +7,9 @@ import GeophysicalFlows.TwoDTurb
 import GeophysicalFlows.TwoDTurb: energy, enstrophy, dissipation, work, drag
 
   n, L  = 256, 2π
-nu, nnu = 1e-7, 2
-mu, nmu = 1e-1, 0
-dt, tf = 0.005, 0.2/mu
+ ν, nν = 1e-7, 2
+ μ, nμ = 1e-1, 0
+dt, tf = 0.005, 0.2/μ
 nt = round(Int, tf/dt)
 ns = 4
 
@@ -38,7 +38,7 @@ function calcF!(Fh, sol, t, cl, v, p, g)
   nothing
 end
 
-prob = TwoDTurb.Problem(nx=n, Lx=L, nu=nu, nnu=nnu, mu=mu, nmu=nmu, dt=dt, stepper="RK4",
+prob = TwoDTurb.Problem(nx=n, Lx=L, ν=ν, nν=nν, μ=μ, nμ=nμ, dt=dt, stepper="RK4",
                         calcF=calcF!, stochastic=true)
 
 sol, cl, v, p, g = prob.sol, prob.clock, prob.vars, prob.params, prob.grid
@@ -55,7 +55,7 @@ function makeplot(prob, diags)
   TwoDTurb.updatevars!(prob)
   E, D, W, R = diags
 
-  t = round(mu*cl.t, digits=2)
+  t = round(μ*cl.t, digits=2)
   sca(axs[1]); cla()
   pcolormesh(x, y, v.zeta)
   xlabel(L"$x$")
@@ -80,25 +80,25 @@ function makeplot(prob, diags)
 
   # If the Ito interpretation was used for the work
   # then we need to add the drift term: I[ii2] + ε
-  plot(mu*E.t[ii], W[ii2], label=L"work ($W$)")   # Ito
-  # plot(mu*E.t[ii], W[ii2] , label=L"work ($W$)")      # Stratonovich
-  plot(mu*E.t[ii], ε .+ 0*E.t[ii], "--", label=L"ensemble mean  work ($\langle W\rangle $)")
-  # plot(mu*E.t[ii], -D[ii], label="dissipation (\$D\$)")
-  plot(mu*E.t[ii], -R[ii], label=L"drag ($D=2\mu E$)")
-  plot(mu*E.t[ii], 0*E.t[ii], "k:", linewidth=0.5)
+  plot(μ*E.t[ii], W[ii2], label=L"work ($W$)")   # Ito
+  # plot(μ*E.t[ii], W[ii2] , label=L"work ($W$)")      # Stratonovich
+  plot(μ*E.t[ii], ε .+ 0*E.t[ii], "--", label=L"ensemble mean  work ($\langle W\rangle $)")
+  # plot(μ*E.t[ii], -D[ii], label="dissipation (\$D\$)")
+  plot(μ*E.t[ii], -R[ii], label=L"drag ($D=2\mu E$)")
+  plot(μ*E.t[ii], 0*E.t[ii], "k:", linewidth=0.5)
   ylabel("Energy sources and sinks")
   xlabel(L"$\mu t$")
   legend(fontsize=10)
 
   sca(axs[2]); cla()
-  plot(mu*E.t[ii], total[ii], label=L"computed $W-D$")
-  plot(mu*E.t[ii], dEdt, "--k", label=L"numerical $dE/dt$")
+  plot(μ*E.t[ii], total[ii], label=L"computed $W-D$")
+  plot(μ*E.t[ii], dEdt, "--k", label=L"numerical $dE/dt$")
   ylabel(L"$dE/dt$")
   xlabel(L"$\mu t$")
   legend(fontsize=10)
 
   sca(axs[4]); cla()
-  plot(mu*E.t[ii], residual, "c-", label=L"residual $dE/dt$ = computed $-$ numerical")
+  plot(μ*E.t[ii], residual, "c-", label=L"residual $dE/dt$ = computed $-$ numerical")
   xlabel(L"$\mu t$")
   legend(fontsize=10)
 
diff --git a/src/twodturb.jl b/src/twodturb.jl
index cebf26b5..2389f045 100644
--- a/src/twodturb.jl
+++ b/src/twodturb.jl
@@ -37,10 +37,10 @@ function Problem(;
           Ly = Lx,
           dt = 0.01,
     # Drag and/or hyper-/hypo-viscosity
-          nu = 0,
-         nnu = 1,
-          mu = 0,
-         nmu = 0,
+           ν = 0,
+          nν = 1,
+           μ = 0,
+          nμ = 0,
     # Timestepper and eqn options
      stepper = "RK4",
        calcF = nothingfunction,
@@ -50,7 +50,7 @@ function Problem(;
 )
 
   gr = TwoDGrid(dev, nx, Lx, ny, Ly)
-  pr = Params{T}(nu, nnu, mu, nmu, calcF)
+  pr = Params{T}(ν, nν, μ, nμ, calcF)
   vs = calcF == nothingfunction ? Vars(dev, gr) : (stochastic ? StochasticForcedVars(dev, gr) : ForcedVars(dev, gr))
   eq = Equation(pr, gr)
   FourierFlows.Problem(eq, stepper, dt, gr, vs, pr, dev)
@@ -62,18 +62,18 @@ end
 # ----------
 
 """
-    Params(nu, nnu, mu, nmu, calcF!)
+    Params(ν, nν, μ, nμ, calcF!)
 
 Returns the params for two-dimensional turbulence.
 """
 struct Params{T} <: AbstractParams
-  nu::T              # Vorticity viscosity
-  nnu::Int           # Vorticity hyperviscous order
-  mu::T              # Bottom drag or hypoviscosity
-  nmu::Int           # Order of hypodrag
-  calcF!::Function   # Function that calculates the forcing F
+  ν::T              # Vorticity viscosity
+  nν::Int           # Vorticity hyperviscous order
+  μ::T              # Bottom drag or hypoviscosity
+  nμ::Int           # Order of hypodrag
+  calcF!::Function  # Function that calculates the forcing F
 end
-Params(nu, nnu) = Params(nu, nnu, typeof(nu)(0), 0, nothingfunction)
+Params(ν, nν) = Params(ν, nν, typeof(ν)(0), 0, nothingfunction)
 
 
 # ---------
@@ -86,7 +86,7 @@ Params(nu, nnu) = Params(nu, nnu, typeof(nu)(0), 0, nothingfunction)
 Returns the equation for two-dimensional turbulence with params p and grid g.
 """
 function Equation(p::Params, g::AbstractGrid{T}) where T
-  L = @. - p.nu*g.Krsq^p.nnu - p.mu*g.Krsq^p.nmu
+  L = @. - p.ν*g.Krsq^p.nν - p.μ*g.Krsq^p.nμ
   L[1, 1] = 0
   FourierFlows.Equation(L, calcN!, g)
 end
@@ -276,13 +276,13 @@ end
 """
     dissipation(prob)
 
-Returns the domain-averaged dissipation rate. nnu must be >= 1.
+Returns the domain-averaged dissipation rate. nν must be >= 1.
 """
 @inline function dissipation(prob)
   sol, v, p, g = prob.sol, prob.vars, prob.params, prob.grid
-  @. v.uh = g.Krsq^(p.nnu-1) * abs2(sol)
+  @. v.uh = g.Krsq^(p.nν-1) * abs2(sol)
   v.uh[1, 1] = 0
-  p.nu/(g.Lx*g.Ly)*parsevalsum(v.uh, g)
+  p.ν/(g.Lx*g.Ly)*parsevalsum(v.uh, g)
 end
 
 """
@@ -307,13 +307,13 @@ end
 """
     drag(prob)
 
-Returns the extraction of domain-averaged energy by drag/hypodrag mu.
+Returns the extraction of domain-averaged energy by drag/hypodrag μ.
 """
 @inline function drag(prob)
   sol, v, p, g = prob.sol, prob.vars, prob.params, prob.grid
-  @. v.uh = g.Krsq^(p.nmu-1) * abs2(sol)
+  @. v.uh = g.Krsq^(p.nμ-1) * abs2(sol)
   v.uh[1, 1] = 0
-  p.mu/(g.Lx*g.Ly)*parsevalsum(v.uh, g)
+  p.μ/(g.Lx*g.Ly)*parsevalsum(v.uh, g)
 end
 
 end # module
diff --git a/test/test_twodturb.jl b/test/test_twodturb.jl
index 9af799cb..8a7918e5 100644
--- a/test/test_twodturb.jl
+++ b/test/test_twodturb.jl
@@ -1,6 +1,6 @@
-function test_twodturb_lambdipole(n, dt, dev::Device=CPU(); L=2π, Ue=1, Re=L/20, nu=0.0, nnu=1, ti=L/Ue*0.01, nm=3)
+function test_twodturb_lambdipole(n, dt, dev::Device=CPU(); L=2π, Ue=1, Re=L/20, ν=0.0, nν=1, ti=L/Ue*0.01, nm=3)
   nt = round(Int, ti/dt)
-  prob = TwoDTurb.Problem(nx=n, Lx=L, nu=nu, nnu=nnu, dt=dt, stepper="FilteredRK4", dev=dev)
+  prob = TwoDTurb.Problem(nx=n, Lx=L, ν=ν, nν=nν, dt=dt, stepper="FilteredRK4", dev=dev)
   zeta₀ = lambdipole(Ue, Re, prob.grid)
   TwoDTurb.set_zeta!(prob, zeta₀)
 
@@ -20,7 +20,7 @@ function test_twodturb_lambdipole(n, dt, dev::Device=CPU(); L=2π, Ue=1, Re=L/20
   isapprox(Ue, mean(Ue_m[2:end]), rtol=rtol_lambdipole)
 end
 
-function test_twodturb_stochasticforcingbudgets(dev::Device=CPU(); n=256, L=2π, dt=0.005, nu=1e-7, nnu=2, mu=1e-1, nmu=0, tf=0.1/mu)
+function test_twodturb_stochasticforcingbudgets(dev::Device=CPU(); n=256, L=2π, dt=0.005, ν=1e-7, nν=2, μ=1e-1, nμ=0, tf=0.1/μ)
   nt = round(Int, tf/dt)
 
   # Forcing parameters
@@ -49,7 +49,7 @@ function test_twodturb_stochasticforcingbudgets(dev::Device=CPU(); n=256, L=2π,
     nothing
   end
 
-  prob = TwoDTurb.Problem(nx=n, Lx=L, nu=nu, nnu=nnu, mu=mu, nmu=nmu, dt=dt,
+  prob = TwoDTurb.Problem(nx=n, Lx=L, ν=ν, nν=nν, μ=μ, nμ=nμ, dt=dt,
    stepper="RK4", calcF=calcF!, stochastic=true, dev=dev)
 
   sol, cl, v, p, g = prob.sol, prob.clock, prob.vars, prob.params, prob.grid;
@@ -65,7 +65,7 @@ function test_twodturb_stochasticforcingbudgets(dev::Device=CPU(); n=256, L=2π,
   TwoDTurb.updatevars!(prob)
 
   E, D, W, R = diags
-  t = round(mu*cl.t, digits=2)
+  t = round(μ*cl.t, digits=2)
 
   i₀ = 1
   dEdt = (E[(i₀+1):E.i] - E[i₀:E.i-1])/cl.dt
@@ -83,11 +83,11 @@ function test_twodturb_stochasticforcingbudgets(dev::Device=CPU(); n=256, L=2π,
 end
 
 
-function test_twodturb_deterministicforcingbudgets(dev::Device=CPU(); n=256, dt=0.01, L=2π, nu=1e-7, nnu=2, mu=1e-1, nmu=0)
+function test_twodturb_deterministicforcingbudgets(dev::Device=CPU(); n=256, dt=0.01, L=2π, ν=1e-7, nν=2, μ=1e-1, nμ=0)
   n, L  = 256, 2π
-  nu, nnu = 1e-7, 2
-  mu, nmu = 1e-1, 0
-  dt, tf = 0.005, 0.1/mu
+  ν, nν = 1e-7, 2
+  μ, nμ = 1e-1, 0
+  dt, tf = 0.005, 0.1/μ
   nt = round(Int, tf/dt)
   
   gr  = TwoDGrid(dev, n, L)
@@ -101,7 +101,7 @@ function test_twodturb_deterministicforcingbudgets(dev::Device=CPU(); n=256, dt=
     nothing
   end
 
-  prob = TwoDTurb.Problem(nx=n, Lx=L, nu=nu, nnu=nnu, mu=mu, nmu=nmu, dt=dt,
+  prob = TwoDTurb.Problem(nx=n, Lx=L, ν=ν, nν=nν, μ=μ, nμ=nμ, dt=dt,
    stepper="RK4", calcF=calcF!, stochastic=false, dev=dev)
 
   sol, cl, v, p, g = prob.sol, prob.clock, prob.vars, prob.params, prob.grid
@@ -119,7 +119,7 @@ function test_twodturb_deterministicforcingbudgets(dev::Device=CPU(); n=256, dt=
   TwoDTurb.updatevars!(prob)
 
   E, D, W, R = diags
-  t = round(mu*cl.t, digits=2)
+  t = round(μ*cl.t, digits=2)
 
   i₀ = 1
   dEdt = (E[(i₀+1):E.i] - E[i₀:E.i-1])/cl.dt
@@ -140,14 +140,14 @@ Tests the advection term in the twodturb module by timestepping a
 test problem with timestep dt and timestepper identified by the string stepper.
 The test problem is derived by picking a solution ζf (with associated
 streamfunction ψf) for which the advection term J(ψf, ζf) is non-zero. Next, a
-forcing Ff is derived according to Ff = ∂ζf/∂t + J(ψf, ζf) - nuΔζf. One solution
+forcing Ff is derived according to Ff = ∂ζf/∂t + J(ψf, ζf) - νΔζf. One solution
 to the vorticity equation forced by this Ff is then ζf. (This solution may not
 be realized, at least at long times, if it is unstable.)
 """
-function test_twodturb_advection(dt, stepper, dev::Device=CPU(); n=128, L=2π, nu=1e-2, nnu=1, mu=0.0, nmu=0)
+function test_twodturb_advection(dt, stepper, dev::Device=CPU(); n=128, L=2π, ν=1e-2, nν=1, μ=0.0, nμ=0)
   n, L  = 128, 2π
-  nu, nnu = 1e-2, 1
-  mu, nmu = 0.0, 0
+  ν, nν = 1e-2, 1
+  μ, nμ = 0.0, 0
   tf = 1.0
   nt = round(Int, tf/dt)
 
@@ -158,7 +158,7 @@ function test_twodturb_advection(dt, stepper, dev::Device=CPU(); n=128, L=2π, n
   zetaf = @. -8sin(2x)*cos(2y) - 20sin(x)*cos(3y)
 
   Ff = @. -(
-    nu*( 64sin(2x)*cos(2y) + 200sin(x)*cos(3y) )
+    ν*( 64sin(2x)*cos(2y) + 200sin(x)*cos(3y) )
     + 8*( cos(x)*cos(3y)*sin(2x)*sin(2y) - 3cos(2x)*cos(2y)*sin(x)*sin(3y) )
   )
 
@@ -170,7 +170,7 @@ function test_twodturb_advection(dt, stepper, dev::Device=CPU(); n=128, L=2π, n
     nothing
   end
 
-  prob = TwoDTurb.Problem(nx=n, Lx=L, nu=nu, nnu=nnu, mu=mu, nmu=nmu, dt=dt, stepper=stepper, calcF=calcF!, stochastic=false, dev=dev)
+  prob = TwoDTurb.Problem(nx=n, Lx=L, ν=ν, nν=nν, μ=μ, nμ=nμ, dt=dt, stepper=stepper, calcF=calcF!, stochastic=false, dev=dev)
   sol, cl, p, v, g = prob.sol, prob.clock, prob.params, prob.vars, prob.grid
   TwoDTurb.set_zeta!(prob, zetaf)
 
@@ -203,7 +203,7 @@ function test_twodturb_energyenstrophy(dev::Device=CPU())
   energyzeta0 = TwoDTurb.energy(prob)
   enstrophyzeta0 = TwoDTurb.enstrophy(prob)
   
-  params = TwoDTurb.Params(p.nu, p.nnu)
+  params = TwoDTurb.Params(p.ν, p.nν)
 
   (isapprox(energyzeta0, energy_calc, rtol=rtol_twodturb) &&
    isapprox(enstrophyzeta0, enstrophy_calc, rtol=rtol_twodturb) &&

From d79c8bb35e75e3f9476ad11794767324c123ff0e Mon Sep 17 00:00:00 2001
From: Gregory Wagner <wagner.greg@gmail.com>
Date: Sat, 31 Aug 2019 08:20:02 -0400
Subject: [PATCH 12/24] consolidates structs for vars, forcedvars, and
 stochasticforcedvars

---
 src/twodturb.jl | 47 +++++++++++++++--------------------------------
 1 file changed, 15 insertions(+), 32 deletions(-)

diff --git a/src/twodturb.jl b/src/twodturb.jl
index 2389f045..8a8a4089 100644
--- a/src/twodturb.jl
+++ b/src/twodturb.jl
@@ -96,71 +96,54 @@ end
 # Vars
 # ----
 
-struct Vars{Aphys, Atrans} <: TwoDTurbVars
-   zeta :: Aphys
-      u :: Aphys
-      v :: Aphys
-  zetah :: Atrans
-     uh :: Atrans
-     vh :: Atrans
-end
-
-struct ForcedVars{Aphys, Atrans} <: TwoDTurbVars
-   zeta :: Aphys
-      u :: Aphys
-      v :: Aphys
-  zetah :: Atrans
-     uh :: Atrans
-     vh :: Atrans
-     Fh :: Atrans
-end
-
-struct StochasticForcedVars{Aphys, Atrans} <: TwoDTurbVars
+struct Vars{Aphys, Atrans, F, P} <: TwoDTurbVars
      zeta :: Aphys
         u :: Aphys
         v :: Aphys
     zetah :: Atrans
        uh :: Atrans
        vh :: Atrans
-       Fh :: Atrans
-  prevsol :: Atrans
+       Fh :: F
+  prevsol :: P
 end
 
+const ForcedVars = Vars{<:AbstractArray, <:AbstractArray, <:AbstractArray, Nothing}
+const StochasticForcedVars = Vars{<:AbstractArray, <:AbstractArray, <:AbstractArray, <:AbstractArray}
 
 """
     Vars(dev, g)
 
 Returns the vars for unforced two-dimensional turbulence on device dev and with 
-  grid g.
+grid g.
 """
 function Vars(::Dev, g::AbstractGrid{T}) where {Dev, T}
   @devzeros Dev T (g.nx, g.ny) zeta u v
   @devzeros Dev Complex{T} (g.nkr, g.nl) zetah uh vh
-  Vars(zeta, u, v, zetah, uh, vh)
+  Vars(zeta, u, v, zetah, uh, vh, nothing, nothing)
 end
 
 """
     ForcedVars(dev, g)
 
 Returns the vars for forced two-dimensional turbulence on device dev and with 
-  grid g.
+grid g.
 """
 function ForcedVars(dev::Dev, g::AbstractGrid{T}) where {Dev, T}
-  v = Vars(dev, g)
-  @devzeros Dev Complex{T} (g.nkr, g.nl) Fh
-  ForcedVars(getfield.(Ref(v), fieldnames(typeof(v)))..., Fh)
+  @devzeros Dev T (g.nx, g.ny) zeta u v
+  @devzeros Dev Complex{T} (g.nkr, g.nl) zetah uh vh Fh
+  Vars(zeta, u, v, zetah, uh, vh, Fh, nothing)
 end
 
 """
     StochasticForcedVars(dev, g)
 
 Returns the vars for stochastically forced two-dimensional turbulence on device
-  dev and with grid g.
+dev and with grid g.
 """
 function StochasticForcedVars(dev::Dev, g::AbstractGrid{T}) where {Dev, T}
-  v = ForcedVars(dev, g)
-  @devzeros Dev Complex{T} (g.nkr, g.nl) prevsol
-  StochasticForcedVars(getfield.(Ref(v), fieldnames(typeof(v)))..., prevsol)
+  @devzeros Dev T (g.nx, g.ny) zeta u v
+  @devzeros Dev Complex{T} (g.nkr, g.nl) zetah uh vh Fh prevsol
+  Vars(zeta, u, v, zetah, uh, vh, Fh, prevsol)
 end
 
 

From bcf4257b1e683a62c54b88ccbe7d9c112f635eab Mon Sep 17 00:00:00 2001
From: Navid Constantinou <navidcy@gmail.com>
Date: Sun, 1 Sep 2019 07:24:30 +1000
Subject: [PATCH 13/24] adds CuArrays + CUDAapi as dependencies

---
 Manifest.toml | 81 ++++++++++++++++++++++++++++++++++++++++++++++++---
 Project.toml  |  4 +--
 2 files changed, 79 insertions(+), 6 deletions(-)

diff --git a/Manifest.toml b/Manifest.toml
index 78f2dcb9..e9e0561a 100644
--- a/Manifest.toml
+++ b/Manifest.toml
@@ -6,6 +6,12 @@ git-tree-sha1 = "380e36c66edfa099cd90116b24c1ce8cafccac40"
 uuid = "621f4979-c628-5d54-868e-fcf4e3e8185c"
 version = "0.4.1"
 
+[[Adapt]]
+deps = ["LinearAlgebra"]
+git-tree-sha1 = "82dab828020b872fa9efd3abec1152b075bc7cbf"
+uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e"
+version = "1.0.0"
+
 [[AxisAlgorithms]]
 deps = ["LinearAlgebra", "Random", "SparseArrays", "WoodburyMatrices"]
 git-tree-sha1 = "a4d07a1c313392a77042855df46c5f534076fab9"
@@ -27,12 +33,35 @@ git-tree-sha1 = "c7361ce8a2129f20b0e05a89f7070820cfed6648"
 uuid = "b99e7846-7c00-51b0-8f62-c81ae34c0232"
 version = "0.5.6"
 
+[[CEnum]]
+git-tree-sha1 = "62847acab40e6855a9b5905ccb99c2b5cf6b3ebb"
+uuid = "fa961155-64e5-5f13-b03f-caf6b980ea82"
+version = "0.2.0"
+
 [[CSTParser]]
 deps = ["Tokenize"]
 git-tree-sha1 = "c69698c3d4a7255bc1b4bc2afc09f59db910243b"
 uuid = "00ebfdb7-1f24-5e51-bd34-a7502290713f"
 version = "0.6.2"
 
+[[CUDAapi]]
+deps = ["Libdl", "Logging"]
+git-tree-sha1 = "9b2b4b71d6b7f946c9689bb4dea03ff92e3c7091"
+uuid = "3895d2a7-ec45-59b8-82bb-cfc6a382f9b3"
+version = "1.1.0"
+
+[[CUDAdrv]]
+deps = ["CUDAapi", "Libdl", "Printf"]
+git-tree-sha1 = "9ce99b5732c70e06ed97c042187baed876fb1698"
+uuid = "c5f51814-7f29-56b8-a69c-e4d8f6be1fde"
+version = "3.1.0"
+
+[[CUDAnative]]
+deps = ["Adapt", "CUDAapi", "CUDAdrv", "DataStructures", "InteractiveUtils", "LLVM", "Libdl", "Logging", "Printf", "TimerOutputs"]
+git-tree-sha1 = "0a00bef482b7c9127495c7f4a2a85e73b13b5af8"
+uuid = "be33ccc6-a3ff-5ff2-a52e-74243cff1e17"
+version = "2.3.0"
+
 [[CodecZlib]]
 deps = ["BinaryProvider", "Libdl", "TranscodingStreams"]
 git-tree-sha1 = "05916673a2627dd91b4969ff8ba6941bc85a960e"
@@ -69,6 +98,18 @@ git-tree-sha1 = "f9780daf3fea51ad2d7b7ed4f480ac34ab36587f"
 uuid = "a2441757-f6aa-5fb2-8edb-039e3f45d037"
 version = "0.9.2"
 
+[[Crayons]]
+deps = ["Test"]
+git-tree-sha1 = "f621b8ef51fd2004c7cf157ea47f027fdeac5523"
+uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f"
+version = "4.0.0"
+
+[[CuArrays]]
+deps = ["AbstractFFTs", "Adapt", "CUDAapi", "CUDAdrv", "CUDAnative", "GPUArrays", "LinearAlgebra", "MacroTools", "NNlib", "Printf", "Random", "Requires", "SparseArrays", "TimerOutputs"]
+git-tree-sha1 = "46b48742a84bb839e74215b7e468a4a1c6ba30f9"
+uuid = "3a865a2d-5b23-5a0f-bc46-62713ec82fae"
+version = "1.2.1"
+
 [[DataStructures]]
 deps = ["InteractiveUtils", "OrderedCollections"]
 git-tree-sha1 = "0809951a1774dc724da22d26e4289bbaab77809a"
@@ -99,17 +140,31 @@ git-tree-sha1 = "351f001a78aa1b7ad2696e386e110b5abd071c71"
 uuid = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"
 version = "1.0.7"
 
+[[FillArrays]]
+deps = ["LinearAlgebra", "Random", "SparseArrays"]
+git-tree-sha1 = "8fba6ddaf66b45dec830233cea0aae43eb1261ad"
+uuid = "1a297f60-69ca-5386-bcde-b61e274b549b"
+version = "0.6.4"
+
 [[FixedPointNumbers]]
 git-tree-sha1 = "d14a6fa5890ea3a7e5dcab6811114f132fec2b4b"
 uuid = "53c48c17-4a7d-5ca2-90c5-79b7896eea93"
 version = "0.6.1"
 
 [[FourierFlows]]
-deps = ["Coverage", "FFTW", "Interpolations", "JLD2", "LinearAlgebra", "Printf", "Random", "Reexport", "Requires", "Statistics"]
-git-tree-sha1 = "e430df633d39e02e772856e6cb745b99b069de19"
+deps = ["CUDAapi", "Coverage", "CuArrays", "FFTW", "Interpolations", "JLD2", "LinearAlgebra", "Printf", "Random", "Reexport", "Requires", "Statistics"]
+git-tree-sha1 = "8b58b99504aec93275be09ae5bb8a39817956d17"
+repo-rev = "ImplementCUDAapi"
+repo-url = "https://github.com/FourierFlows/FourierFlows.jl.git"
 uuid = "2aec4490-903f-5c70-9b11-9bed06a700e1"
 version = "0.3.1"
 
+[[GPUArrays]]
+deps = ["Adapt", "FFTW", "FillArrays", "LinearAlgebra", "Printf", "Random", "Serialization", "StaticArrays", "Test"]
+git-tree-sha1 = "dd169c636d1d3656a9faca772f5bd7c226a61254"
+uuid = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7"
+version = "1.0.1"
+
 [[HTTP]]
 deps = ["Base64", "Dates", "IniFile", "MbedTLS", "Sockets"]
 git-tree-sha1 = "c4a527dba1d26add0e85946e1a53f42a1b343acc"
@@ -144,6 +199,12 @@ git-tree-sha1 = "b34d7cef7b337321e97d22242c3c2b91f476748e"
 uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 version = "0.21.0"
 
+[[LLVM]]
+deps = ["CEnum", "Libdl", "Printf", "Unicode"]
+git-tree-sha1 = "52cfea426bd248a427aace7d88eb5d45b84ea297"
+uuid = "929cbde3-209d-540e-8aea-75f648917ca0"
+version = "1.2.0"
+
 [[LaTeXStrings]]
 deps = ["Compat"]
 git-tree-sha1 = "7ab9b8788cfab2bdde22adf9004bda7ad9954b6c"
@@ -182,6 +243,12 @@ version = "0.7.0"
 [[Mmap]]
 uuid = "a63ad114-7e13-5084-954f-fe012c677804"
 
+[[NNlib]]
+deps = ["Libdl", "LinearAlgebra", "Requires", "Statistics", "TimerOutputs"]
+git-tree-sha1 = "0c667371391fc6bb31f7f12f96a56a17098b3de8"
+uuid = "872c559c-99b0-510c-b3b7-b6c96a88d5cd"
+version = "0.6.0"
+
 [[OffsetArrays]]
 git-tree-sha1 = "1af2f79c7eaac3e019a0de41ef63335ff26a0a57"
 uuid = "6fe1bfb0-de20-5000-8ca7-80f57d26f881"
@@ -195,9 +262,9 @@ version = "1.1.0"
 
 [[Parsers]]
 deps = ["Dates", "Test"]
-git-tree-sha1 = "db2b35dedab3c0e46dc15996d170af07a5ab91c9"
+git-tree-sha1 = "ef0af6c8601db18c282d092ccbd2f01f3f0cd70b"
 uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
-version = "0.3.6"
+version = "0.3.7"
 
 [[Pkg]]
 deps = ["Dates", "LibGit2", "Markdown", "Printf", "REPL", "Random", "SHA", "UUIDs"]
@@ -282,6 +349,12 @@ uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 deps = ["Distributed", "InteractiveUtils", "Logging", "Random"]
 uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
+[[TimerOutputs]]
+deps = ["Crayons", "Printf", "Test", "Unicode"]
+git-tree-sha1 = "b80671c06f8f8bae08c55d67b5ce292c5ae2660c"
+uuid = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f"
+version = "0.5.0"
+
 [[Tokenize]]
 git-tree-sha1 = "dfcdbbfb2d0370716c815cbd6f8a364efb6f42cf"
 uuid = "0796e94c-ce3b-5d07-9a54-7f471281c624"
diff --git a/Project.toml b/Project.toml
index f91d6627..0134309c 100644
--- a/Project.toml
+++ b/Project.toml
@@ -6,6 +6,7 @@ version = "0.3.0"
 
 [deps]
 Coverage = "a2441757-f6aa-5fb2-8edb-039e3f45d037"
+CuArrays = "3a865a2d-5b23-5a0f-bc46-62713ec82fae"
 FFTW = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341"
 FourierFlows = "2aec4490-903f-5c70-9b11-9bed06a700e1"
 JLD2 = "033835bb-8acc-5ee8-8aae-3f567f8a3819"
@@ -24,8 +25,7 @@ JLD2 = "≥ 0.1.2"
 julia = "≥ 1.0.0"
 
 [extras]
-CuArrays = "3a865a2d-5b23-5a0f-bc46-62713ec82fae"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["Test", "CuArrays"]
+test = ["Test"]

From 003a2d9f619ad5db3d10004be1648c836948b343 Mon Sep 17 00:00:00 2001
From: Navid Constantinou <navidcy@gmail.com>
Date: Sun, 1 Sep 2019 07:24:35 +1000
Subject: [PATCH 14/24] use the has_cuda() variable from CUDAapi

---
 test/runtests.jl | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/runtests.jl b/test/runtests.jl
index a6fd2017..0e9e9a20 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -17,8 +17,8 @@ using GeophysicalFlows: lambdipole, peakedisotropicspectrum
 
 # the devices on which tests will run
 devices = (CPU(),)
-@hascuda devices = (CPU(), GPU())
-@hascuda using CuArrays
+@has_cuda devices = (CPU(), GPU())
+@has_cuda using CuArrays
 
 const rtol_lambdipole = 1e-2 # tolerance for lamb dipole tests
 const rtol_multilayerqg = 1e-13 # tolerance for multilayerqg forcing tests

From f9fc5afb9b4327b6a9ca3d64ddaf572cb9ea643b Mon Sep 17 00:00:00 2001
From: Navid Constantinou <navidcy@gmail.com>
Date: Sun, 1 Sep 2019 07:34:33 +1000
Subject: [PATCH 15/24] updates twodturb example to use dev=CPU()/GPU()

---
 examples/twodturb_mcwilliams1984.jl    |  4 +++-
 examples/twodturb_randomdecay.jl       |  5 +++--
 examples/twodturb_stochasticforcing.jl | 12 +++++++-----
 3 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/examples/twodturb_mcwilliams1984.jl b/examples/twodturb_mcwilliams1984.jl
index 7e706d21..08257597 100644
--- a/examples/twodturb_mcwilliams1984.jl
+++ b/examples/twodturb_mcwilliams1984.jl
@@ -6,6 +6,8 @@ import GeophysicalFlows.TwoDTurb
 import GeophysicalFlows.TwoDTurb: energy, enstrophy
 import GeophysicalFlows: peakedisotropicspectrum
 
+dev = CPU()     # Device (CPU/GPU)
+
 # Parameters
   n = 256
   L = 2π
@@ -13,7 +15,7 @@ import GeophysicalFlows: peakedisotropicspectrum
   ν = 0.0
  dt = 1e-2
 nsteps = 5000
-nsubs = 200
+ nsubs = 200
 
 # Files
 filepath = "."
diff --git a/examples/twodturb_randomdecay.jl b/examples/twodturb_randomdecay.jl
index 93c51477..f98ae128 100644
--- a/examples/twodturb_randomdecay.jl
+++ b/examples/twodturb_randomdecay.jl
@@ -7,6 +7,7 @@ using FFTW: rfft
 
 import GeophysicalFlows.TwoDTurb
 
+dev = CPU()     # Device (CPU/GPU)
 
 nx = 256        # Resolution
 Lx = 2π         # Domain size
@@ -15,9 +16,9 @@ nν = 1          # Order of (hyper-)viscosity. nν=1 means Laplacian
 dt = 0.1        # Timestep
 nint = 200      # Number of steps between plots
 ntot = 10nint   # Number of total timesteps
-
+ 
 # Define problem
-prob = TwoDTurb.Problem(nx=nx, Lx=Lx, ν=ν, nν=nν, dt=dt, stepper="FilteredRK4")
+prob = TwoDTurb.Problem(nx=nx, Lx=Lx, ν=ν, nν=nν, dt=dt, stepper="FilteredRK4", dev=dev)
 TwoDTurb.set_zeta!(prob, rand(nx, nx))
 
 cl, vs, gr = prob.clock, prob.vars, prob.grid
diff --git a/examples/twodturb_stochasticforcing.jl b/examples/twodturb_stochasticforcing.jl
index 83947533..8f479305 100644
--- a/examples/twodturb_stochasticforcing.jl
+++ b/examples/twodturb_stochasticforcing.jl
@@ -6,18 +6,20 @@ using Printf: @printf
 import GeophysicalFlows.TwoDTurb
 import GeophysicalFlows.TwoDTurb: energy, enstrophy, dissipation, work, drag
 
-  n, L  = 256, 2π
+   dev = CPU()     # Device (CPU/GPU)
+
+ n, L  = 256, 2π
  ν, nν = 1e-7, 2
  μ, nμ = 1e-1, 0
 dt, tf = 0.005, 0.2/μ
-nt = round(Int, tf/dt)
-ns = 4
+    nt = round(Int, tf/dt)
+    ns = 4
 
 # Forcing
 kf, dkf = 12.0, 2.0     # forcing central wavenumber, wavenumber width
 ε = 0.1                 # energy injection rate
 
-gr  = TwoDGrid(n, L)
+gr  = TwoDGrid(dev, n, L)
 x, y = gridpoints(gr)
 
 Kr = [ gr.kr[i] for i=1:gr.nkr, j=1:gr.nl]
@@ -39,7 +41,7 @@ function calcF!(Fh, sol, t, cl, v, p, g)
 end
 
 prob = TwoDTurb.Problem(nx=n, Lx=L, ν=ν, nν=nν, μ=μ, nμ=nμ, dt=dt, stepper="RK4",
-                        calcF=calcF!, stochastic=true)
+                        calcF=calcF!, stochastic=true, dev=dev)
 
 sol, cl, v, p, g = prob.sol, prob.clock, prob.vars, prob.params, prob.grid
 

From cfbcbae7fa65c4dfa3ce0dd7744b1c55906cd8b7 Mon Sep 17 00:00:00 2001
From: Navid Constantinou <navidcy@gmail.com>
Date: Sun, 1 Sep 2019 07:48:03 +1000
Subject: [PATCH 16/24] minor fixes in examples

---
 examples/twodturb_randomdecay.jl       | 18 +++++++++---------
 examples/twodturb_stochasticforcing.jl |  4 ++--
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/examples/twodturb_randomdecay.jl b/examples/twodturb_randomdecay.jl
index f98ae128..ee5f7c79 100644
--- a/examples/twodturb_randomdecay.jl
+++ b/examples/twodturb_randomdecay.jl
@@ -7,19 +7,19 @@ using FFTW: rfft
 
 import GeophysicalFlows.TwoDTurb
 
-dev = CPU()     # Device (CPU/GPU)
+ dev = CPU()   # Device (CPU/GPU)
 
-nx = 256        # Resolution
-Lx = 2π         # Domain size
- ν = 1e-6       # Viscosity
-nν = 1          # Order of (hyper-)viscosity. nν=1 means Laplacian
-dt = 0.1        # Timestep
-nint = 200      # Number of steps between plots
-ntot = 10nint   # Number of total timesteps
+  nx = 256     # Resolution
+  Lx = 2π      # Domain size
+   ν = 1e-6    # Viscosity
+  nν = 1       # Order of (hyper-)viscosity. nν=1 means Laplacian
+  dt = 0.1     # Timestep
+nint = 200     # Number of steps between plots
+ntot = 10nint  # Number of total timesteps
  
 # Define problem
 prob = TwoDTurb.Problem(nx=nx, Lx=Lx, ν=ν, nν=nν, dt=dt, stepper="FilteredRK4", dev=dev)
-TwoDTurb.set_zeta!(prob, rand(nx, nx))
+TwoDTurb.set_zeta!(prob, ArrayType(dev)(rand(nx, nx)))
 
 cl, vs, gr = prob.clock, prob.vars, prob.grid
 x, y = gridpoints(gr)
diff --git a/examples/twodturb_stochasticforcing.jl b/examples/twodturb_stochasticforcing.jl
index 8f479305..9eb1e37c 100644
--- a/examples/twodturb_stochasticforcing.jl
+++ b/examples/twodturb_stochasticforcing.jl
@@ -22,7 +22,7 @@ kf, dkf = 12.0, 2.0     # forcing central wavenumber, wavenumber width
 gr  = TwoDGrid(dev, n, L)
 x, y = gridpoints(gr)
 
-Kr = [ gr.kr[i] for i=1:gr.nkr, j=1:gr.nl]
+Kr = ArrayType(dev)([ gr.kr[i] for i=1:gr.nkr, j=1:gr.nl])
 
 force2k = @. exp(-(sqrt(gr.Krsq)-kf)^2/(2*dkf^2))
 force2k[gr.Krsq .< 2.0^2 ] .= 0
@@ -34,7 +34,7 @@ force2k .= ε/ε0 * force2k
 seed!(1234)
 
 function calcF!(Fh, sol, t, cl, v, p, g)
-  eta = exp.(2π*im*rand(Float64, size(sol)))/sqrt(cl.dt)
+  eta = ArrayType(dev)(exp.(2π*im*rand(typeof(gr.Lx), size(sol)))/sqrt(cl.dt))
   eta[1, 1] = 0
   @. Fh = eta*sqrt(force2k)
   nothing

From 2694bfaa71cf16ff51e0750b7f36579c7f9af328 Mon Sep 17 00:00:00 2001
From: "Navid C. Constantinou" <navidcy@users.noreply.github.com>
Date: Sun, 1 Sep 2019 14:45:41 +1000
Subject: [PATCH 17/24] minor beautification

---
 src/twodturb.jl | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/twodturb.jl b/src/twodturb.jl
index 8a8a4089..c4bc7df1 100644
--- a/src/twodturb.jl
+++ b/src/twodturb.jl
@@ -67,11 +67,11 @@ end
 Returns the params for two-dimensional turbulence.
 """
 struct Params{T} <: AbstractParams
-  ν::T              # Vorticity viscosity
-  nν::Int           # Vorticity hyperviscous order
-  μ::T              # Bottom drag or hypoviscosity
-  nμ::Int           # Order of hypodrag
-  calcF!::Function  # Function that calculates the forcing F
+      ν  :: T         # Vorticity viscosity
+      nν :: Int       # Vorticity hyperviscous order
+       μ :: T         # Bottom drag or hypoviscosity
+      nμ :: Int       # Order of hypodrag
+  calcF! :: Function  # Function that calculates the forcing F
 end
 Params(ν, nν) = Params(ν, nν, typeof(ν)(0), 0, nothingfunction)
 

From bdef2b90dd7776ea3b83c4f25e089a0dfee97656 Mon Sep 17 00:00:00 2001
From: Navid Constantinou <navidcy@gmail.com>
Date: Wed, 4 Sep 2019 06:48:19 +0400
Subject: [PATCH 18/24] updates Manifest

---
 Manifest.toml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Manifest.toml b/Manifest.toml
index e9e0561a..8da0dfc2 100644
--- a/Manifest.toml
+++ b/Manifest.toml
@@ -153,8 +153,8 @@ version = "0.6.1"
 
 [[FourierFlows]]
 deps = ["CUDAapi", "Coverage", "CuArrays", "FFTW", "Interpolations", "JLD2", "LinearAlgebra", "Printf", "Random", "Reexport", "Requires", "Statistics"]
-git-tree-sha1 = "8b58b99504aec93275be09ae5bb8a39817956d17"
-repo-rev = "ImplementCUDAapi"
+git-tree-sha1 = "60d0e58e32fcd19d4ceef13626c48f63887a2329"
+repo-rev = "master"
 repo-url = "https://github.com/FourierFlows/FourierFlows.jl.git"
 uuid = "2aec4490-903f-5c70-9b11-9bed06a700e1"
 version = "0.3.1"

From f5bfb7da8f2086dbd7023b40987955f13d7dd6a8 Mon Sep 17 00:00:00 2001
From: Navid Constantinou <navidcy@gmail.com>
Date: Wed, 4 Sep 2019 06:50:02 +0400
Subject: [PATCH 19/24] updates Manifest

---
 Manifest.toml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Manifest.toml b/Manifest.toml
index 8da0dfc2..f966518c 100644
--- a/Manifest.toml
+++ b/Manifest.toml
@@ -44,7 +44,6 @@ git-tree-sha1 = "c69698c3d4a7255bc1b4bc2afc09f59db910243b"
 uuid = "00ebfdb7-1f24-5e51-bd34-a7502290713f"
 version = "0.6.2"
 
-[[CUDAapi]]
 deps = ["Libdl", "Logging"]
 git-tree-sha1 = "9b2b4b71d6b7f946c9689bb4dea03ff92e3c7091"
 uuid = "3895d2a7-ec45-59b8-82bb-cfc6a382f9b3"

From 531b4e09f4ed4dd085d57617a72b0d27ba358424 Mon Sep 17 00:00:00 2001
From: Navid Constantinou <navidcy@gmail.com>
Date: Wed, 4 Sep 2019 06:56:27 +0400
Subject: [PATCH 20/24] updates Manifest

---
 Manifest.toml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/Manifest.toml b/Manifest.toml
index f966518c..de95b746 100644
--- a/Manifest.toml
+++ b/Manifest.toml
@@ -44,6 +44,7 @@ git-tree-sha1 = "c69698c3d4a7255bc1b4bc2afc09f59db910243b"
 uuid = "00ebfdb7-1f24-5e51-bd34-a7502290713f"
 version = "0.6.2"
 
+[[CUDAapi]]
 deps = ["Libdl", "Logging"]
 git-tree-sha1 = "9b2b4b71d6b7f946c9689bb4dea03ff92e3c7091"
 uuid = "3895d2a7-ec45-59b8-82bb-cfc6a382f9b3"
@@ -329,10 +330,10 @@ deps = ["LinearAlgebra", "Random"]
 uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
 
 [[SpecialFunctions]]
-deps = ["BinDeps", "BinaryProvider", "Libdl", "Test"]
-git-tree-sha1 = "0b45dc2e45ed77f445617b99ff2adf0f5b0f23ea"
+deps = ["BinDeps", "BinaryProvider", "Libdl"]
+git-tree-sha1 = "3bdd374b6fd78faf0119b8c5d538788dbf910c6e"
 uuid = "276daf66-3868-5448-9aa4-cd146d93841b"
-version = "0.7.2"
+version = "0.8.0"
 
 [[StaticArrays]]
 deps = ["LinearAlgebra", "Random", "Statistics"]

From babe8c4f23fe3075f997dcae3674ee70695d434f Mon Sep 17 00:00:00 2001
From: Navid Constantinou <navidcy@gmail.com>
Date: Wed, 4 Sep 2019 08:07:39 +0400
Subject: [PATCH 21/24] some beautifications

---
 examples/twodturb_mcwilliams1984.jl    |  2 +-
 examples/twodturb_randomdecay.jl       |  2 +-
 examples/twodturb_stochasticforcing.jl |  2 +-
 src/twodturb.jl                        | 12 ++++++------
 test/test_twodturb.jl                  |  4 ++--
 5 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/examples/twodturb_mcwilliams1984.jl b/examples/twodturb_mcwilliams1984.jl
index 08257597..71be770c 100644
--- a/examples/twodturb_mcwilliams1984.jl
+++ b/examples/twodturb_mcwilliams1984.jl
@@ -37,7 +37,7 @@ x, y = gridpoints(gr)
 # that reproduces the results of the paper by McWilliams (1984)
 seed!(1234)
 k0, E0 = 6, 0.5
-zetai = peakedisotropicspectrum(gr, k0, E0, mask=filter)
+zetai  = peakedisotropicspectrum(gr, k0, E0, mask=filter)
 TwoDTurb.set_zeta!(prob, zetai)
 
 # Create Diagnostic -- energy and enstrophy are functions imported at the top.
diff --git a/examples/twodturb_randomdecay.jl b/examples/twodturb_randomdecay.jl
index ee5f7c79..c7c696e1 100644
--- a/examples/twodturb_randomdecay.jl
+++ b/examples/twodturb_randomdecay.jl
@@ -50,7 +50,7 @@ while cl.step < ntot
 end
 
 # Plot the radial energy spectrum
-E = @. 0.5*(vs.u^2 + vs.v^2) # energy density
+E  = @. 0.5*(vs.u^2 + vs.v^2) # energy density
 Eh = rfft(E)
 kr, Ehr = FourierFlows.radialspectrum(Eh, gr, refinement=1)
 
diff --git a/examples/twodturb_stochasticforcing.jl b/examples/twodturb_stochasticforcing.jl
index 9eb1e37c..a3ca7566 100644
--- a/examples/twodturb_stochasticforcing.jl
+++ b/examples/twodturb_stochasticforcing.jl
@@ -19,7 +19,7 @@ dt, tf = 0.005, 0.2/μ
 kf, dkf = 12.0, 2.0     # forcing central wavenumber, wavenumber width
 ε = 0.1                 # energy injection rate
 
-gr  = TwoDGrid(dev, n, L)
+gr   = TwoDGrid(dev, n, L)
 x, y = gridpoints(gr)
 
 Kr = ArrayType(dev)([ gr.kr[i] for i=1:gr.nkr, j=1:gr.nl])
diff --git a/src/twodturb.jl b/src/twodturb.jl
index c4bc7df1..04ed2d42 100644
--- a/src/twodturb.jl
+++ b/src/twodturb.jl
@@ -157,8 +157,8 @@ end
 Calculates the advection term.
 """
 function calcN_advection!(N, sol, t, cl, v, p, g)
-  @. v.uh =  im * g.l  * g.invKrsq * sol
-  @. v.vh = -im * g.kr * g.invKrsq * sol
+  @. v.uh =   im * g.l  * g.invKrsq * sol
+  @. v.vh = - im * g.kr * g.invKrsq * sol
   @. v.zetah = sol
 
   ldiv!(v.u, g.rfftplan, v.uh)
@@ -171,7 +171,7 @@ function calcN_advection!(N, sol, t, cl, v, p, g)
   mul!(v.uh, g.rfftplan, v.u) # \hat{u*zeta}
   mul!(v.vh, g.rfftplan, v.v) # \hat{v*zeta}
 
-  @. N = -im*g.kr*v.uh - im*g.l*v.vh
+  @. N = - im*g.kr*v.uh - im*g.l*v.vh
   nothing
 end
 
@@ -210,9 +210,9 @@ Update the vars in v on the grid g with the solution in sol.
 """
 function updatevars!(prob)
   v, g, sol = prob.vars, prob.grid, prob.sol
-  v.zetah .= sol
-  @. v.uh =  im * g.l  * g.invKrsq * sol
-  @. v.vh = -im * g.kr * g.invKrsq * sol
+  @. v.zetah = sol
+  @. v.uh =   im * g.l  * g.invKrsq * sol
+  @. v.vh = - im * g.kr * g.invKrsq * sol
   ldiv!(v.zeta, g.rfftplan, deepcopy(v.zetah))
   ldiv!(v.u, g.rfftplan, deepcopy(v.uh))
   ldiv!(v.v, g.rfftplan, deepcopy(v.vh))
diff --git a/test/test_twodturb.jl b/test/test_twodturb.jl
index 8a7918e5..2c0f56cf 100644
--- a/test/test_twodturb.jl
+++ b/test/test_twodturb.jl
@@ -151,7 +151,7 @@ function test_twodturb_advection(dt, stepper, dev::Device=CPU(); n=128, L=2π, 
   tf = 1.0
   nt = round(Int, tf/dt)
 
-  gr  = TwoDGrid(dev, n, L)
+  gr   = TwoDGrid(dev, n, L)
   x, y = gridpoints(gr)
 
    psif = @.   sin(2x)*cos(2y) +  2sin(x)*cos(3y)
@@ -184,9 +184,9 @@ function test_twodturb_energyenstrophy(dev::Device=CPU())
   nx, Lx  = 128, 2π
   ny, Ly  = 128, 3π
   gr = TwoDGrid(dev, nx, Lx, ny, Ly)
-  k0, l0 = gr.k[2], gr.l[2] # fundamental wavenumbers
   x, y = gridpoints(gr)
 
+  k0, l0 = gr.k[2], gr.l[2] # fundamental wavenumbers
     psi0 = @. sin(2k0*x)*cos(2l0*y) + 2sin(k0*x)*cos(3l0*y)
    zeta0 = @. -((2k0)^2+(2l0)^2)*sin(2k0*x)*cos(2l0*y) - (k0^2+(3l0)^2)*2sin(k0*x)*cos(3l0*y)
 

From 9a911d6e64372f6e8d863cc731007aaf6150621a Mon Sep 17 00:00:00 2001
From: Navid Constantinou <navidcy@gmail.com>
Date: Thu, 5 Sep 2019 09:37:29 +0300
Subject: [PATCH 22/24] removes duplicate Test import

---
 test/runtests.jl | 1 -
 1 file changed, 1 deletion(-)

diff --git a/test/runtests.jl b/test/runtests.jl
index 0e9e9a20..75fe5fbb 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -1,6 +1,5 @@
 using
   FourierFlows,
-  Test,
   Statistics,
   Random,
   FFTW,

From 9e25e9814c674d3a03d30169435e8c37dd729bf0 Mon Sep 17 00:00:00 2001
From: Navid Constantinou <navidcy@gmail.com>
Date: Thu, 19 Sep 2019 12:18:20 +0300
Subject: [PATCH 23/24] fixes Float type instability in Problem

---
 src/twodturb.jl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/twodturb.jl b/src/twodturb.jl
index ea37d1f9..44063aa4 100644
--- a/src/twodturb.jl
+++ b/src/twodturb.jl
@@ -49,7 +49,7 @@ function Problem(;
          dev = CPU()
 )
 
-  gr = TwoDGrid(dev, nx, Lx, ny, Ly)
+  gr = TwoDGrid(dev, nx, Lx, ny, Ly; T=T)
   pr = Params{T}(ν, nν, μ, nμ, calcF)
   vs = calcF == nothingfunction ? Vars(dev, gr) : (stochastic ? StochasticForcedVars(dev, gr) : ForcedVars(dev, gr))
   eq = Equation(pr, gr)

From 9276b42b3f1d8a32c02573a87011e0a6e6b7e18c Mon Sep 17 00:00:00 2001
From: Navid Constantinou <navidcy@gmail.com>
Date: Thu, 19 Sep 2019 13:13:32 +0300
Subject: [PATCH 24/24] updates toml files

---
 Manifest.toml | 22 ++++++++++------------
 Project.toml  |  2 +-
 2 files changed, 11 insertions(+), 13 deletions(-)

diff --git a/Manifest.toml b/Manifest.toml
index de95b746..a4a68be2 100644
--- a/Manifest.toml
+++ b/Manifest.toml
@@ -58,9 +58,9 @@ version = "3.1.0"
 
 [[CUDAnative]]
 deps = ["Adapt", "CUDAapi", "CUDAdrv", "DataStructures", "InteractiveUtils", "LLVM", "Libdl", "Logging", "Printf", "TimerOutputs"]
-git-tree-sha1 = "0a00bef482b7c9127495c7f4a2a85e73b13b5af8"
+git-tree-sha1 = "52ae1ce10ebfa686e227655c47b19add89308623"
 uuid = "be33ccc6-a3ff-5ff2-a52e-74243cff1e17"
-version = "2.3.0"
+version = "2.3.1"
 
 [[CodecZlib]]
 deps = ["BinaryProvider", "Libdl", "TranscodingStreams"]
@@ -153,23 +153,21 @@ version = "0.6.1"
 
 [[FourierFlows]]
 deps = ["CUDAapi", "Coverage", "CuArrays", "FFTW", "Interpolations", "JLD2", "LinearAlgebra", "Printf", "Random", "Reexport", "Requires", "Statistics"]
-git-tree-sha1 = "60d0e58e32fcd19d4ceef13626c48f63887a2329"
-repo-rev = "master"
-repo-url = "https://github.com/FourierFlows/FourierFlows.jl.git"
+git-tree-sha1 = "153b295514c016e30b300b9e65b0615061976166"
 uuid = "2aec4490-903f-5c70-9b11-9bed06a700e1"
-version = "0.3.1"
+version = "0.3.2"
 
 [[GPUArrays]]
 deps = ["Adapt", "FFTW", "FillArrays", "LinearAlgebra", "Printf", "Random", "Serialization", "StaticArrays", "Test"]
-git-tree-sha1 = "dd169c636d1d3656a9faca772f5bd7c226a61254"
+git-tree-sha1 = "b5009ac44b141ded5e6f04c4db83807970f56e91"
 uuid = "0c68f7d7-f131-5f86-a1c3-88cf8149b2d7"
-version = "1.0.1"
+version = "1.0.2"
 
 [[HTTP]]
 deps = ["Base64", "Dates", "IniFile", "MbedTLS", "Sockets"]
-git-tree-sha1 = "c4a527dba1d26add0e85946e1a53f42a1b343acc"
+git-tree-sha1 = "f1e1b417a34cf73a70cbed919915bf8f8bad1806"
 uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3"
-version = "0.8.5"
+version = "0.8.6"
 
 [[IniFile]]
 deps = ["Test"]
@@ -201,9 +199,9 @@ version = "0.21.0"
 
 [[LLVM]]
 deps = ["CEnum", "Libdl", "Printf", "Unicode"]
-git-tree-sha1 = "52cfea426bd248a427aace7d88eb5d45b84ea297"
+git-tree-sha1 = "4a05f742837779a00bd8c9a18da6817367c4245d"
 uuid = "929cbde3-209d-540e-8aea-75f648917ca0"
-version = "1.2.0"
+version = "1.3.0"
 
 [[LaTeXStrings]]
 deps = ["Compat"]
diff --git a/Project.toml b/Project.toml
index 0134309c..29deccfe 100644
--- a/Project.toml
+++ b/Project.toml
@@ -20,7 +20,7 @@ Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 
 [compat]
 FFTW = "≥ 0.2.4"
-FourierFlows = "≥ 0.3.1"
+FourierFlows = "≥ 0.3.2"
 JLD2 = "≥ 0.1.2"
 julia = "≥ 1.0.0"