mpi_fieldio.scmc

;paravec.scmc general_macros.scmc

(eval-scmc-global (begin (load "pscmc_config_runtime.ss") '()))
(include< "stdio.h")
(include< "stdint.h")
(include< "stdlib.h")
(include< "string.h")
(include< "assert.h")
;(include- "c_/c_pscmc_inc.h")
(input-scmc "paravec.scmc")
(input-scmc "general_macros.scmc")
(input-all-pscmc-struct)
(input-all-kernel-and-rt)
(include- "space_filling_curve.h")
(include< "cgapsio.h")
(defmacro gen-read-write-mpi-fields-particle (name is_read)
  `(defun-class-Field3D_MPI ,name int ,(append `((Gaps_IO_DataFile* gid_grid) (Gaps_IO_DataFile* gid_cu) (long* pgcache) (long* pcucache) (long numt)) (if is_read `((int reduce_dim) (double r_x_rat) (double r_y_rat) (double r_z_rat)) '()) '((int use_non_uni_cache)))
     ,(if is_read
	'(begin
	   (set! r_x_rat (/ 1.0 r_x_rat))
	   (set! r_y_rat (/ 1.0 r_y_rat))
	   (set! r_z_rat (/ 1.0 r_z_rat))
	   )
	`(begin
	   (define reduce_dim 0)
	   (define r_x_rat 1.0)
	   (define r_y_rat 1.0)
	   (define r_z_rat 1.0)
	   )
	)
     (define-int reduce_x (eq? reduce_dim 1))
     (define-int reduce_y (eq? reduce_dim 2))
     (define-int reduce_z (eq? reduce_dim 3))

     (declare-long (g_offset 0) (c_offset 0))
     (define-int64_t* pdimarr_cu (structp-ref gid_cu pdimarray))
     (define-int64_t* pdimarr_grid (structp-ref gid_grid pdimarray))
     (define-int64_t cu_one_cache_len (/ (- (/ (vrf pdimarr_cu 0) num_spec) 1) ,PARTICLE_LEN))
     (define-int64_t grid_one_cache_len (/ (- (/ (vrf pdimarr_grid 0) num_spec) 1) ,PARTICLE_LEN))
     (define-double r_r_r (case reduce_dim (1 r_x_rat) (2 r_y_rat) (3 r_z_rat) (else 0)))
     (for-from-to i 0 num_spec 
       (for-from-to j 0 num_runtime
	 (class-header-Field3D_Seq (+ data j))
	 (class-header-One_Particle_Collection (+ particles (+ j (* num_runtime i))))
	 ,(if is_read
	    '()
	    `(begin
	       (choose_the_runtime (pscmc_mem_sync_d2h cu_cache))
	       (choose_the_runtime (pscmc_mem_sync_d2h inoutput))
	       (choose_the_runtime (pscmc_mem_sync_d2h xyzw))
	       (choose_the_runtime (pscmc_mem_sync_d2h cu_xyzw))
	       )
	    )
	 (define-double* host_data (vector-ref (type-convert double** inoutput) 0))
	 (define-int* xyzw_data (vector-ref (type-convert int** xyzw) 0))
	 (define-double* cu_host_data (vector-ref (type-convert double** cu_cache) 0))
	 (define-int* cu_xyzw_data (vector-ref (type-convert int** cu_xyzw) 0))
	 (define-long alllenoff (* xlen ylen zlen))
	 (for-from-to cur_pcid 0 numvec 
	   (define ofx (vector-ref global_x_offset cur_pcid))
	   (define ofy (vector-ref global_y_offset cur_pcid))
	   (define ofz (vector-ref global_z_offset cur_pcid))
	   (define-double* grid_r_0 (+ host_data (* cur_pcid alllenoff ,PARTICLE_LEN grid_cache_len)))
	   (define-int* xyzw_0 (+ xyzw_data (* cur_pcid alllenoff 4)))
	   (block

	     (define-double* cu_r_0 (+ cu_host_data (* cur_pcid ,PARTICLE_LEN cu_cache_length)))
	     (define-int* cu_xyzw_0 (+ cu_xyzw_data (* cur_pcid 4)))
	     (define ox (/ ofx xlen))
	     (define oy (/ ofy ylen))
	     (define oz (/ ofz zlen))
	     ;(define-int reduce_x (b-and reduce_dim 1))
	     ;(define-int reduce_y (b-and (shift-r reduce_dim 1) 1))
	     ;(define-int reduce_z (b-and (shift-r reduce_dim 2) 1))
	     (define-long offset_len (+ (* (vrf pdimarr_cu 0) (+ (if reduce_x 0 ox) (* (if reduce_x 1 (vrf pdimarr_cu 1)) (+ (if reduce_y 0 oy) (* (if reduce_y 1 (vrf pdimarr_cu 2)) (if reduce_z 0 oz)))))) c_offset))
	     (define-double num_load (vrf cu_xyzw_0 0))
	     (GAPS_IO_DataSeek gid_cu numt offset_len)
	     ,(if is_read
		`(begin
		   (GAPS_IO_FRead gid_cu ("&" num_load) 1)
		   (set! (vrf cu_xyzw_0 0) (* num_load (if reduce_dim (vrf pdimarr_grid reduce_dim) 1)))
		   (assert (<= num_load cu_cache_length))
		   (GAPS_IO_FRead gid_cu cu_r_0 (* ,PARTICLE_LEN (vrf cu_xyzw_0 0)))
		   (for-from-to g 0 (if reduce_dim num_load 0)
		     (set! (vrf cu_r_0 (+ reduce_dim 2 (* g ,PARTICLE_LEN))) (* r_r_r (vrf cu_r_0 (+ reduce_dim 2 (* g ,PARTICLE_LEN)))))
		     )
		   (for-from-to rd 1 (if reduce_dim (vrf pdimarr_grid reduce_dim) 1)
		     (for-from-zero-to g num_load
		       (for-from-zero-to s ,PARTICLE_LEN
			 (set! (vrf cu_r_0 (+ s (* g ,PARTICLE_LEN) (* rd (type-convert int num_load) ,PARTICLE_LEN))) (vrf cu_r_0 (+ s (* g ,PARTICLE_LEN))))
			 )
		       (incf! (vrf cu_r_0 (+ (- reduce_dim 1) (* g ,PARTICLE_LEN) (* rd (type-convert int num_load) ,PARTICLE_LEN))) rd)
		       ))
		   )
		`(begin
		   (GAPS_IO_FWrite gid_cu ("&" num_load) 1)
		   (GAPS_IO_FWrite gid_cu cu_r_0 (* ,PARTICLE_LEN (vrf cu_xyzw_0 0)))
		   )
		)
	     )

	   (fffor-xyz-from-zero-to xlen ylen zlen
	     (define-double* grid_r (+ grid_r_0 (GET_IDXYZL xyzx xyzy xyzz 0 (* ,PARTICLE_LEN grid_cache_len) xlen ylen zlen)))
	     (define-int* grid_xyzw (+ xyzw_0 (GET_IDXYZL xyzx xyzy xyzz 0 4 xlen ylen zlen)))

	     (define ox (+ ofx xyzx))
	     (define oy (+ ofy xyzy))
	     (define oz (+ ofz xyzz))
	     ;(define-int64_t* pdimarr_grid (structp-ref gid_grid pdimarray))
	     (define-long offset_len (+ (* (vrf pdimarr_grid 0) (+ (if reduce_x 0 ox) (* (if reduce_x 1 (vrf pdimarr_grid 1)) (+ (if reduce_y 0 oy) (* (if reduce_y 1 (vrf pdimarr_grid 2)) (if reduce_z 0 oz)))))) g_offset))
	     ;(define-long offset_len (+ (* (vrf pdimarr_grid 0) (+ ox (* (vrf pdimarr_grid 1) (+ oy (* (vrf pdimarr_grid 2) oz))))) g_offset))

	     (define-int64_t offset_xyzlen (case reduce_dim (1 xyzx) (2 xyzy) (3 xyzz) (else 0)))
	     (GAPS_IO_DataSeek gid_grid numt offset_len)
	     (define-double num_load (vrf grid_xyzw 0))
	     ,(if is_read
		`(begin
		   (GAPS_IO_FRead gid_grid ("&" num_load) 1)
		   (set! (vrf grid_xyzw 0) num_load)
		   (assert (<= num_load grid_cache_len))
		   (GAPS_IO_FRead gid_grid grid_r (* ,PARTICLE_LEN (vrf grid_xyzw 0)))
		   (for-from-to g 0 (if reduce_dim num_load 0)
		     (set! (vrf grid_r (+ (* g ,PARTICLE_LEN) reduce_dim 2)) (* r_r_r (vrf grid_r (+ reduce_dim 2 (* g ,PARTICLE_LEN)))))
		     (incf! (vrf grid_r (+ (* g ,PARTICLE_LEN) reduce_dim -1)) offset_xyzlen)
		     )
		   )
		`(begin
		   (GAPS_IO_FWrite gid_grid ("&" num_load) 1)
		   (GAPS_IO_FWrite gid_grid grid_r (* ,PARTICLE_LEN (vrf grid_xyzw 0))))
		)
	     )

	   )
	 ,(if is_read
	    `(begin
	       (choose_the_runtime (pscmc_mem_sync_h2d cu_cache))
	       (choose_the_runtime (pscmc_mem_sync_h2d inoutput))
	       (choose_the_runtime (pscmc_mem_sync_h2d xyzw))
	       (choose_the_runtime (pscmc_mem_sync_h2d cu_xyzw))
	       )
	    '()
	    )
	 )
       (incf! g_offset (+ (* ,PARTICLE_LEN (if use_non_uni_cache (vector-ref pgcache i) grid_one_cache_len)) 1))
       (incf! c_offset (+ (* ,PARTICLE_LEN (if use_non_uni_cache (vector-ref pcucache i) cu_one_cache_len)) 1))
       )
     (return 0)
     )
  )
(eval-scmc-global
  `(begin
     (defun-class-Field3D_MPI init_parallel_file_particle_for_mpi_fields_V0 int ((Gaps_IO_DataFile* gid_grid) (Gaps_IO_DataFile* gid_cu) (long* pgcache) (long* pcucache) (char* pName_grid) (char* pName_cu) (long numt) (int ro))
       ;(if (eq? numt 0) (set! numt -1))
       (dec-array int64_t pdimarr_grid 4)
       (dec-array int64_t pdimarr_cu 4)
       (block
	 (class-header-Field3D_Seq data)
	 (block
	   (class-header-One_Particle_Collection particles)
	   (declare-long (gcl 0) (ccl 0))
	   (for-from-to g 0 num_spec
	     (incf! gcl (vrf pgcache g))
	     (incf! ccl (vrf pcucache g))
	     )
	   (vector-set! pdimarr_grid 0 (+ num_spec (* gcl ,PARTICLE_LEN)))
	   (vector-set! pdimarr_cu 0 (+ num_spec (* ccl ,PARTICLE_LEN)))
	   )
	 (vector-set! pdimarr_grid 1 (* xlen x_num_thread_block))
	 (vector-set! pdimarr_grid 2 (* ylen y_num_thread_block))
	 (vector-set! pdimarr_grid 3 (* zlen z_num_thread_block))
	 (vector-set! pdimarr_cu 1 x_num_thread_block)
	 (vector-set! pdimarr_cu 2 y_num_thread_block)
	 (vector-set! pdimarr_cu 3 z_num_thread_block)
	 )
       (GAPS_IO_InitDataInfo gid_grid GAPS_IO_FLOAT64 4 pdimarr_grid)
       (GAPS_IO_InitDataInfo gid_cu GAPS_IO_FLOAT64 4 pdimarr_cu)
       (if ro
	 (begin
	   (GAPS_IO_InitIFile gid_grid pName_grid)
	   (GAPS_IO_InitIFile gid_cu pName_cu)
	   )
	 (begin
	   (GAPS_IO_InitOFile_with_TimeStep gid_grid pName_grid -1)
	   (GAPS_IO_InitOFile_with_TimeStep gid_cu pName_cu -1)
	   (GAPS_IO_TruncateFile gid_grid (+ numt 1) -1)
	   (GAPS_IO_TruncateFile gid_cu (+ numt 1) -1)
	   (GAPS_IO_FileFlush gid_grid)
	   (GAPS_IO_FileFlush gid_cu)))
       (PS_MPI_Barrier comm)
       (return 0)
       )
     ))
(gen-read-write-mpi-fields-particle dump_particle_parallel_file_for_mpi_fields_V0 #f)
(gen-read-write-mpi-fields-particle read_particle_parallel_file_for_mpi_fields_V0 #t)
(defun-class-Field3D_MPI init_parallel_file_for_mpi_fields_V0 int ((Gaps_IO_DataFile* gid) (char* pName) (int64_t time_step))
  (dec-array int64_t pdimarr 4)
  (block
    (class-header-Field3D_Seq data)
    (vector-set! pdimarr 0 num_ele)
    (vector-set! pdimarr 1 (* xlen x_num_thread_block))
    (vector-set! pdimarr 2 (* ylen y_num_thread_block))
    (vector-set! pdimarr 3 (* zlen z_num_thread_block))
    )
  ;(LOG_RECORD_OUT "%ld %ld\n" GAPS_IO_FLOAT64 (vector-ref pdimarr 0))
  (GAPS_IO_InitDataInfo gid GAPS_IO_FLOAT64 4 pdimarr)
  ;(LOG_RECORD_OUT "bef ofile Done\n" GAPS_IO_FLOAT64 (vector-ref pdimarr 0))
  (GAPS_IO_InitOFile_with_TimeStep gid pName time_step) 
  (GAPS_IO_FileFlush gid)
  (PS_MPI_Barrier comm)
  (return 0)
  )
(defmacro reduce_switch (main-proc coop-proc . pgid)
  (if (null? pgid) (set! pgid 'gid) (set! pgid (car pgid)))
  `(if (and (eq? (structp-ref ,pgid version) 2) (structp-ref ,pgid num_reduce_proc))
     (begin
       (define-int cur_rank)
       (PS_MPI_Comm_rank PS_MPI_COMM_WORLD ("&" cur_rank))
       (if (neq? (structp-ref ,pgid cur_rank) (structp-ref ,pgid send_to_proc))
	 ,coop-proc
	 ,main-proc
	 )))
  )
(defun-class-Field3D_MPI init_parallel_file_for_mpi_fields int ((Gaps_IO_DataFile* gid) (char* pName) (int64_t time_step) (int version) (int num_reduce_proc))
  (dec-array int64_t pdimarr 4)
  (dec-array int64_t pblkarr 4)
  (cond
    ((== version 0)
    (block
      (class-header-Field3D_Seq data)
      (vector-set! pdimarr 0 num_ele)
      (vector-set! pdimarr 1 (* xlen x_num_thread_block))
      (vector-set! pdimarr 2 (* ylen y_num_thread_block))
      (vector-set! pdimarr 3 (* zlen z_num_thread_block))
      (GAPS_IO_InitDataInfo gid GAPS_IO_FLOAT64 4 pdimarr)
      ))
    (else
      (block
	(class-header-Field3D_Seq data)
	(vector-set! pdimarr 0 num_ele)
	(vector-set! pdimarr 1 xlen)
	(vector-set! pdimarr 2 ylen)
	(vector-set! pdimarr 3 zlen)

	(vector-set! pblkarr 0 1)
	(vector-set! pblkarr 1 x_num_thread_block)
	(vector-set! pblkarr 2 y_num_thread_block)
	(vector-set! pblkarr 3 z_num_thread_block)
	;(LOG_RECORD_INFO "xtb=%d\n" x_num_thread_block)
	;(exit 0)
	(if (== version 1)
	  (GAPS_IO_InitDataInfoV1 gid GAPS_IO_FLOAT64 4 pdimarr pblkarr)
	  (if (== version 2)
	    (begin
	      (define-long numvec_total 0)
	      (for-from-zero-to i num_runtime
		(class-header-Field3D_Seq (+ data i))
		(incf! numvec_total numvec)
		)
	      ;(assert (> numvec_total 0))
	      (define-int64_t* plocations (TYPE_MALLOC int64_t numvec_total))
	      (define-long numvec_offset 0)
	      (for-from-zero-to r num_runtime
		(class-header-Field3D_Seq (+ data r))
		(for-from-zero-to i numvec
		  (vector-set! plocations (+ i numvec_offset) (+ (/ (vrf global_x_offset i) xlen) (* x_num_thread_block (+ (/ (vrf global_y_offset i) ylen) (* y_num_thread_block (/ (vrf global_z_offset i) zlen))))))
		  )
		(incf! numvec_offset numvec)
		)
		(GAPS_IO_InitDataInfoV2 gid "" GAPS_IO_FLOAT64 4 pdimarr pblkarr plocations numvec_total 0)
		(if num_reduce_proc
		  (GAPS_IO_SetNumReduceProc gid num_reduce_proc)
		  )
	      ;(LOG_RECORD_INFO "vs_dbg=%d loc=0x%lx, bk[1]=%d\n" (structp-ref gid version) ("&" (structp-ref gid version)) (vector-ref pblkarr 1))
	      (free plocations)
	      ))
	  )
	)
      )
    )
  ;(LOG_RECORD_OUT "%ld %ld\n" GAPS_IO_FLOAT64 (vector-ref pdimarr 0))
  ;(LOG_RECORD_OUT "bef ofile Done %ld %ld %ld %ld\n" GAPS_IO_FLOAT64 (vector-ref pdimarr 0) version time_step)
  (if (>= version 2)
    (GAPS_IO_InitOFile_with_TimeStepV2 gid pName (structp-ref gid prefix) time_step)
    (GAPS_IO_InitOFile_with_TimeStep gid pName time_step) 
    )
  ;(LOG_RECORD_INFO "OK here %d\n" cur_rank)

  (GAPS_IO_FileFlush gid)
  (PS_MPI_Barrier comm)
  (return 0)
  )
(defun-class-Field3D_MPI read_parallel_file_for_mpi_fields int ((Gaps_IO_DataFile* gid) (int64_t timestep))
  (define i 0)
  (for i=0 (< i num_runtime) i++
    (class-header-Field3D_Seq (+ data i))
    (define-double* odata (vector-ref (type-convert double** main_data) 0))
    (define-size_t oneblocklen (* xblock yblock zblock num_ele))
    (define-size_t realoneblklen (* xlen ylen zlen num_ele))
    (define-double* write_buffer (malloc (* (sizeof double ) realoneblklen)))
    (define vid 0)
    (for vid=0 (< vid numvec) vid++
      (if (== (structp-ref gid version) 0)
	(fffor-xyz-from-zero-to 1 ylen zlen
	  (define-int64_t offsetlen (* num_ele (+ (vector-ref global_x_offset vid) 0 (* xlen x_num_thread_block (+ xyzy (vector-ref global_y_offset vid) (* ylen y_num_thread_block (+ xyzz (vector-ref global_z_offset vid))))))))
	  (GAPS_IO_DataSeek gid timestep offsetlen)
	  (GAPS_IO_FRead gid (+ odata (* oneblocklen vid) (* num_ele (+ ovlp (* xblock (+ xyzy ovlp (* yblock (+ ovlp xyzz))))))) (* num_ele xlen))
	  )
	(begin
	  (define-int64_t offsetlen 
	    (* realoneblklen 
	      (if (eq? (structp-ref gid version) 1) (+ (/ (vector-ref global_x_offset vid) xlen) (* x_num_thread_block (+ (/ (vector-ref global_y_offset vid) ylen) (* y_num_thread_block (/ (vector-ref global_z_offset vid) zlen))))) vid)))
	  (GAPS_IO_DataSeek gid timestep offsetlen)
	  (GAPS_IO_FRead gid write_buffer realoneblklen)
	  (fffor-xyz-from-zero-to xlen ylen zlen
	    (for-from-to l 0 num_ele
	    (vector-set! write_buffer (+ l (* num_ele (+ xyzx (* xlen (+ xyzy (* ylen xyzz)))))) 
	      (vector-ref odata (+ (* oneblocklen vid) (+ l (* num_ele (+ xyzx ovlp (* xblock (+ xyzy ovlp (* yblock (+ ovlp xyzz)))))))))))
	    )
	  )
	)
      )
    (free write_buffer)

    )
  (return 0)
  )
(defun-class-Field3D_MPI write_parallel_file_for_mpi_fields int ((Gaps_IO_DataFile* gid) (int64_t timestep))
  (define i 0)
  (if (eq? (structp-ref gid version) 2)
    (GAPS_IO_DataSeek gid timestep 0)
    )
  (define-double* write_buffer NULL)
  (for i=0 (< i num_runtime) i++
    (class-header-Field3D_Seq (+ data i))
    (define-double* odata (vector-ref (type-convert double** main_data) 0))
    (define-size_t oneblocklen (* xblock yblock zblock num_ele))
    (define-size_t realoneblklen (* xlen ylen zlen num_ele))
    (if (not write_buffer) (set! write_buffer (malloc (* (sizeof double ) realoneblklen))))
    (define vid 0)
    (if (eq? (structp-ref gid version) 2)
      (begin
	(if (and (neq? cur_rank (structp-ref gid send_to_proc)) (structp-ref gid num_reduce_proc))
	  (MPI_Send ("&" numvec) 1 PS_MPI_LONG (structp-ref gid send_to_proc) cur_rank PS_MPI_COMM_WORLD))
	)
      )
    (for vid=0 (< vid numvec) vid++
      (if (== (structp-ref gid version) 0)
	(fffor-xyz-from-zero-to 1 ylen zlen
	  (define-int64_t offsetlen (* num_ele (+ (vector-ref global_x_offset vid) 0 (* xlen x_num_thread_block (+ xyzy (vector-ref global_y_offset vid) (* ylen y_num_thread_block (+ xyzz (vector-ref global_z_offset vid))))))))
	  (GAPS_IO_DataSeek gid timestep offsetlen)
	  (GAPS_IO_FWrite gid (+ odata (* oneblocklen vid) (* num_ele (+ ovlp (* xblock (+ xyzy ovlp (* yblock (+ ovlp xyzz))))))) (* num_ele xlen))
	  )
	(begin
	  (define-int64_t offsetlen 
	    (if (eq? (structp-ref gid version) 1)
	      (* realoneblklen (+ (/ (vector-ref global_x_offset vid) xlen) (* x_num_thread_block (+ (/ (vector-ref global_y_offset vid) ylen) (* y_num_thread_block (/ (vector-ref global_z_offset vid) zlen))))))
	      (* realoneblklen vid)
	      ))
	  (if (eq? (structp-ref gid version) 1)
	    (GAPS_IO_DataSeek gid timestep offsetlen))
	  (fffor-xyz-from-zero-to xlen ylen zlen
	    (for-from-to l 0 num_ele
	      (vector-set! write_buffer (+ l (* num_ele (+ xyzx (* xlen (+ xyzy (* ylen xyzz)))))) 
		(vector-ref odata (+ (* oneblocklen vid) (+ l (* num_ele (+ xyzx ovlp (* xblock (+ xyzy ovlp (* yblock (+ ovlp xyzz)))))))))))
	    )
	  (GAPS_IO_FWrite gid write_buffer realoneblklen)
	  (reduce_switch 0
	    (begin
	      ;(LOG_RECORD_INFO "send err\n")
	    (PS_MPI_Send write_buffer realoneblklen PS_MPI_DOUBLE (structp-ref gid send_to_proc) cur_rank PS_MPI_COMM_WORLD))
	    )
	  )
	)
      ))
  (for i=0 (< i num_runtime) i++
    (class-header-Field3D_Seq (+ data i))
    (define-double* odata (vector-ref (type-convert double** main_data) 0))
    (define-size_t oneblocklen (* xblock yblock zblock num_ele))
    (define-size_t realoneblklen (* xlen ylen zlen num_ele))
    ;(LOG_RECORD_INFO "ok here %d\n" cur_rank)
    (reduce_switch
      (block
	;(class-header-Gaps_IO_DataFile gid)
	(define-int cur_rank (structp-ref gid cur_rank))
	(define-int num_reduce_proc (structp-ref gid num_reduce_proc))
	(define-int real_rank (structp-ref gid real_rank))
	;(LOG_RECORD_INFO "recv err\n")
	(for-from-to n (+ cur_rank 1) (* num_reduce_proc (+ real_rank 1))
	  (define-int num_proc)
	  (PS_MPI_Comm_size PS_MPI_COMM_WORLD ("&" num_proc))
	  (if (>= n num_proc) break)
	  (define-long remote_numvec)
	  (define-MPI_Status s1)
	  (PS_MPI_Recv ("&" remote_numvec) 1 PS_MPI_LONG n n PS_MPI_COMM_WORLD ("&" s1))
	  (for-from-zero-to g remote_numvec
	    (PS_MPI_Recv write_buffer realoneblklen PS_MPI_DOUBLE n n PS_MPI_COMM_WORLD ("&" s1))
	    (GAPS_IO_FWrite gid write_buffer realoneblklen))
	  )
	) 0)
    )
  (free write_buffer)
    
  (return 0)
  )
(defun-class-Field3D_MPI mpi_field_2_outfile int ((char* fname))
  (decl-var-and-pvar Gaps_IO_DataFile gid)
  (init_parallel_file_for_mpi_fields pthis pgid fname -1 0 0)
  (GAPS_IO_TruncateFile pgid 1 -1)
  (PS_MPI_Barrier comm)
  ;(LOG_RECORD_OUT "num_ele=%d\n" (structp-ref data num_ele))
  (write_parallel_file_for_mpi_fields pthis pgid 0)
  (GAPS_IO_DeleteDataInfo pgid)
  (return 0)
  )
(defun-class-Field3D_MPI mpi_field_write_to_file int ((Gaps_IO_DataFile* pgid) (long num_timestep))
  ;(define-int64_t num_timestep)
  (define-int64_t res)
  ;(PS_MPI_Barrier comm)
  ;(GAPS_IO_DataNumStepsAndResidue pgid ("&" num_timestep) ("&" res))
  (PS_MPI_Barrier comm)
  (GAPS_IO_TruncateFile pgid (+ num_timestep 1) -1)
  (PS_MPI_Barrier comm)
  (GAPS_IO_DataSeek pgid num_timestep 0)
  (write_parallel_file_for_mpi_fields pthis pgid num_timestep)
  (return 0)
  )