From 8b51f80a616f7471e4d9c84c2b6c47ce593549ad Mon Sep 17 00:00:00 2001
From: Jeremy L Thompson <jeremy@jeremylt.org>
Date: Tue, 14 Dec 2021 16:56:20 -0700
Subject: [PATCH] gpu - fix restr bug I added, need to sum into target

---
 backends/cuda/ceed-cuda-operator.c    | 4 +++-
 backends/cuda/ceed-cuda-restriction.c | 2 +-
 backends/cuda/ceed-cuda-vector.c      | 1 +
 backends/hip/ceed-hip-operator.c      | 3 +++
 backends/hip/ceed-hip-restriction.c   | 2 +-
 examples/petsc/area.c                 | 4 ++--
 6 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/backends/cuda/ceed-cuda-operator.c b/backends/cuda/ceed-cuda-operator.c
index f90a63c794..281f445253 100644
--- a/backends/cuda/ceed-cuda-operator.c
+++ b/backends/cuda/ceed-cuda-operator.c
@@ -6,7 +6,6 @@
 // libraries and APIs for efficient high-order finite element and spectral
 // element discretizations for exascale applications. For more information and
 // source code availability see http://github.com/ceed.
-//
 // The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
 // a collaborative effort of two U.S. Department of Energy organizations (Office
 // of Science and the National Nuclear Security Administration) responsible for
@@ -150,6 +149,9 @@ static int CeedOperatorSetupFields_Cuda(CeedQFunction qf, CeedOperator op,
         ierr = CeedElemRestrictionCreateVector(Erestrict, NULL,
                                                &evecs[i + starte]);
         CeedChkBackend(ierr);
+        // Allocate array
+        ierr = CeedVectorSetArray(evecs[i + starte], CEED_MEM_DEVICE, CEED_COPY_VALUES,
+                                  NULL); CeedChkBackend(ierr);
       }
     }
 
diff --git a/backends/cuda/ceed-cuda-restriction.c b/backends/cuda/ceed-cuda-restriction.c
index 452de42d99..7177510c79 100644
--- a/backends/cuda/ceed-cuda-restriction.c
+++ b/backends/cuda/ceed-cuda-restriction.c
@@ -145,7 +145,7 @@ static int CeedElemRestrictionApply_Cuda(CeedElemRestriction r,
   const CeedScalar *d_u;
   CeedScalar *d_v;
   ierr = CeedVectorGetArrayRead(u, CEED_MEM_DEVICE, &d_u); CeedChkBackend(ierr);
-  ierr = CeedVectorGetArrayWrite(v, CEED_MEM_DEVICE, &d_v); CeedChkBackend(ierr);
+  ierr = CeedVectorGetArray(v, CEED_MEM_DEVICE, &d_v); CeedChkBackend(ierr);
 
   // Restrict
   if (tmode == CEED_NOTRANSPOSE) {
diff --git a/backends/cuda/ceed-cuda-vector.c b/backends/cuda/ceed-cuda-vector.c
index 7264ecfa66..b09207dda3 100644
--- a/backends/cuda/ceed-cuda-vector.c
+++ b/backends/cuda/ceed-cuda-vector.c
@@ -469,6 +469,7 @@ static int CeedVectorGetArrayWrite_Cuda(const CeedVector vec,
 
   bool has_array_of_type = true;
   ierr = CeedVectorHasArrayOfType_Cuda(vec, mtype, &has_array_of_type);
+  CeedChkBackend(ierr);
   if (!has_array_of_type) {
     // Allocate if array is not yet allocated
     ierr = CeedVectorSetArray(vec, mtype, CEED_COPY_VALUES, NULL);
diff --git a/backends/hip/ceed-hip-operator.c b/backends/hip/ceed-hip-operator.c
index 50f2544471..c0e5b945ae 100644
--- a/backends/hip/ceed-hip-operator.c
+++ b/backends/hip/ceed-hip-operator.c
@@ -150,6 +150,9 @@ static int CeedOperatorSetupFields_Hip(CeedQFunction qf, CeedOperator op,
         ierr = CeedElemRestrictionCreateVector(Erestrict, NULL,
                                                &evecs[i + starte]);
         CeedChkBackend(ierr);
+        // Allocate array
+        ierr = CeedVectorSetArray(evecs[i + starte], CEED_MEM_DEVICE, CEED_COPY_VALUES,
+                                  NULL); CeedChkBackend(ierr);
       }
     }
 
diff --git a/backends/hip/ceed-hip-restriction.c b/backends/hip/ceed-hip-restriction.c
index b748645924..c3f35ecd42 100644
--- a/backends/hip/ceed-hip-restriction.c
+++ b/backends/hip/ceed-hip-restriction.c
@@ -144,7 +144,7 @@ static int CeedElemRestrictionApply_Hip(CeedElemRestriction r,
   const CeedScalar *d_u;
   CeedScalar *d_v;
   ierr = CeedVectorGetArrayRead(u, CEED_MEM_DEVICE, &d_u); CeedChkBackend(ierr);
-  ierr = CeedVectorGetArrayWrite(v, CEED_MEM_DEVICE, &d_v); CeedChkBackend(ierr);
+  ierr = CeedVectorGetArray(v, CEED_MEM_DEVICE, &d_v); CeedChkBackend(ierr);
 
   // Restrict
   if (tmode == CEED_NOTRANSPOSE) {
diff --git a/examples/petsc/area.c b/examples/petsc/area.c
index 442802a280..9c8456cd9d 100644
--- a/examples/petsc/area.c
+++ b/examples/petsc/area.c
@@ -223,8 +223,8 @@ int main(int argc, char **argv) {
   // Setup libCEED's objects and apply setup operator
   ierr = PetscMalloc1(1, &ceed_data); CHKERRQ(ierr);
   ierr = SetupLibceedByDegree(dm, ceed, degree, topo_dim, q_extra, num_comp_x,
-                              num_comp_u,
-                              g_size, xl_size, problem_options[problem_choice], ceed_data,
+                              num_comp_u, g_size, xl_size,
+                              problem_options[problem_choice], ceed_data,
                               false, (CeedVector)NULL, (CeedVector *)NULL);
   CHKERRQ(ierr);