Add Beam Spin tracking #1071

AlexanderSinn · 2024-02-21T20:04:44Z

Comparison with FBPIC:

The non-zero sy and sz mean in FBPIC are caused by the random fluctuations of the Gaussian beam. In HiPACE++ this was suppressed with beam.do_symmetrize = true.

electron_beam.do_spin_tracking = true
electron_beam.initial_spin = 1 0 0
electron_beam.spin_anom = 0.00115965218128

Add spin tracking from fbpic/fbpic#672.

This PR fixes #1027 and is based on #1069, #1068, #1067 and #1066.

Register usage for the beam pusher in development, the high local memory usage is caused by the kernels that use the parser for external fields.

--- 176 registers, 0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads, function name:
void amrex::launch_global<(int)256, std::enable_if<amrex::MaybeDeviceRunnable<T3, void>::value, void>::type amrex::ParallelFor<(int)256, int, std::enable_if<std::is_integral<T2>::value||std::is_same<T2, amrex::Box>::value, bool>::type amrex::detail::ParallelFor_helper2<(int)256, int, AdvanceBeamParticlesSlice(BeamParticleContainer &, const Fields &, const amrex::Vector<amrex::Geometry, std::allocator<amrex::Geometry>> &, int, int)::[lambda(int, T1, T2) (instance 1)], std::integral_constant<int, (int)3>, std::integral_constant<int, (int)0>>(const T2 &, T3 &&, amrex::TypeList<T4...>, const std::array<int, sizeof...(T4)> &)::[lambda(int) (instance 1)], void>(const amrex::Gpu::KernelInfo &, T2, T3 &&)::[lambda() (instance 1)]>(T2)
--- 128 registers, 424 bytes stack frame, 256 bytes spill stores, 256 bytes spill loads, function name:
void amrex::launch_global<(int)256, std::enable_if<amrex::MaybeDeviceRunnable<T3, void>::value, void>::type amrex::ParallelFor<(int)256, int, std::enable_if<std::is_integral<T2>::value||std::is_same<T2, amrex::Box>::value, bool>::type amrex::detail::ParallelFor_helper2<(int)256, int, AdvanceBeamParticlesSlice(BeamParticleContainer &, const Fields &, const amrex::Vector<amrex::Geometry, std::allocator<amrex::Geometry>> &, int, int)::[lambda(int, T1, T2) (instance 1)], std::integral_constant<int, (int)3>, std::integral_constant<int, (int)1>>(const T2 &, T3 &&, amrex::TypeList<T4...>, const std::array<int, sizeof...(T4)> &)::[lambda(int) (instance 1)], void>(const amrex::Gpu::KernelInfo &, T2, T3 &&)::[lambda() (instance 1)]>(T2)
--- 128 registers, 240 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads, function name:
void amrex::launch_global<(int)256, std::enable_if<amrex::MaybeDeviceRunnable<T3, void>::value, void>::type amrex::ParallelFor<(int)256, int, std::enable_if<std::is_integral<T2>::value||std::is_same<T2, amrex::Box>::value, bool>::type amrex::detail::ParallelFor_helper2<(int)256, int, AdvanceBeamParticlesSlice(BeamParticleContainer &, const Fields &, const amrex::Vector<amrex::Geometry, std::allocator<amrex::Geometry>> &, int, int)::[lambda(int, T1, T2) (instance 1)], std::integral_constant<int, (int)2>, std::integral_constant<int, (int)1>>(const T2 &, T3 &&, amrex::TypeList<T4...>, const std::array<int, sizeof...(T4)> &)::[lambda(int) (instance 1)], void>(const amrex::Gpu::KernelInfo &, T2, T3 &&)::[lambda() (instance 1)]>(T2)
--- 128 registers, 0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads, function name:
void amrex::launch_global<(int)256, std::enable_if<amrex::MaybeDeviceRunnable<T3, void>::value, void>::type amrex::ParallelFor<(int)256, int, std::enable_if<std::is_integral<T2>::value||std::is_same<T2, amrex::Box>::value, bool>::type amrex::detail::ParallelFor_helper2<(int)256, int, AdvanceBeamParticlesSlice(BeamParticleContainer &, const Fields &, const amrex::Vector<amrex::Geometry, std::allocator<amrex::Geometry>> &, int, int)::[lambda(int, T1, T2) (instance 1)], std::integral_constant<int, (int)2>, std::integral_constant<int, (int)0>>(const T2 &, T3 &&, amrex::TypeList<T4...>, const std::array<int, sizeof...(T4)> &)::[lambda(int) (instance 1)], void>(const amrex::Gpu::KernelInfo &, T2, T3 &&)::[lambda() (instance 1)]>(T2)
--- 128 registers, 0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads, function name:
void amrex::launch_global<(int)256, std::enable_if<amrex::MaybeDeviceRunnable<T3, void>::value, void>::type amrex::ParallelFor<(int)256, int, std::enable_if<std::is_integral<T2>::value||std::is_same<T2, amrex::Box>::value, bool>::type amrex::detail::ParallelFor_helper2<(int)256, int, AdvanceBeamParticlesSlice(BeamParticleContainer &, const Fields &, const amrex::Vector<amrex::Geometry, std::allocator<amrex::Geometry>> &, int, int)::[lambda(int, T1, T2) (instance 1)], std::integral_constant<int, (int)1>, std::integral_constant<int, (int)0>>(const T2 &, T3 &&, amrex::TypeList<T4...>, const std::array<int, sizeof...(T4)> &)::[lambda(int) (instance 1)], void>(const amrex::Gpu::KernelInfo &, T2, T3 &&)::[lambda() (instance 1)]>(T2)
--- 126 registers, 240 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads, function name:
void amrex::launch_global<(int)256, std::enable_if<amrex::MaybeDeviceRunnable<T3, void>::value, void>::type amrex::ParallelFor<(int)256, int, std::enable_if<std::is_integral<T2>::value||std::is_same<T2, amrex::Box>::value, bool>::type amrex::detail::ParallelFor_helper2<(int)256, int, AdvanceBeamParticlesSlice(BeamParticleContainer &, const Fields &, const amrex::Vector<amrex::Geometry, std::allocator<amrex::Geometry>> &, int, int)::[lambda(int, T1, T2) (instance 1)], std::integral_constant<int, (int)1>, std::integral_constant<int, (int)1>>(const T2 &, T3 &&, amrex::TypeList<T4...>, const std::array<int, sizeof...(T4)> &)::[lambda(int) (instance 1)], void>(const amrex::Gpu::KernelInfo &, T2, T3 &&)::[lambda() (instance 1)]>(T2)
--- 109 registers, 240 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads, function name:
void amrex::launch_global<(int)256, std::enable_if<amrex::MaybeDeviceRunnable<T3, void>::value, void>::type amrex::ParallelFor<(int)256, int, std::enable_if<std::is_integral<T2>::value||std::is_same<T2, amrex::Box>::value, bool>::type amrex::detail::ParallelFor_helper2<(int)256, int, AdvanceBeamParticlesSlice(BeamParticleContainer &, const Fields &, const amrex::Vector<amrex::Geometry, std::allocator<amrex::Geometry>> &, int, int)::[lambda(int, T1, T2) (instance 1)], std::integral_constant<int, (int)0>, std::integral_constant<int, (int)1>>(const T2 &, T3 &&, amrex::TypeList<T4...>, const std::array<int, sizeof...(T4)> &)::[lambda(int) (instance 1)], void>(const amrex::Gpu::KernelInfo &, T2, T3 &&)::[lambda() (instance 1)]>(T2)
--- 102 registers, 0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads, function name:
void amrex::launch_global<(int)256, std::enable_if<amrex::MaybeDeviceRunnable<T3, void>::value, void>::type amrex::ParallelFor<(int)256, int, std::enable_if<std::is_integral<T2>::value||std::is_same<T2, amrex::Box>::value, bool>::type amrex::detail::ParallelFor_helper2<(int)256, int, AdvanceBeamParticlesSlice(BeamParticleContainer &, const Fields &, const amrex::Vector<amrex::Geometry, std::allocator<amrex::Geometry>> &, int, int)::[lambda(int, T1, T2) (instance 1)], std::integral_constant<int, (int)0>, std::integral_constant<int, (int)0>>(const T2 &, T3 &&, amrex::TypeList<T4...>, const std::array<int, sizeof...(T4)> &)::[lambda(int) (instance 1)], void>(const amrex::Gpu::KernelInfo &, T2, T3 &&)::[lambda() (instance 1)]>(T2)

PR:

--- 176 registers, 240 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads, function name:
void amrex::launch_global<(int)256, std::enable_if<amrex::MaybeDeviceRunnable<T3, void>::value, void>::type amrex::ParallelFor<(int)256, int, std::enable_if<std::is_integral<T2>::value||std::is_same<T2, amrex::Box>::value, bool>::type amrex::detail::ParallelFor_helper2<(int)256, int, AdvanceBeamParticlesSlice(BeamParticleContainer &, const Fields &, const amrex::Vector<amrex::Geometry, std::allocator<amrex::Geometry>> &, int, int)::[lambda(int, T1, T2) (instance 1)], std::integral_constant<int, (int)3>, std::integral_constant<int, (int)1>>(const T2 &, T3 &&, amrex::TypeList<T4...>, const std::array<int, sizeof...(T4)> &)::[lambda(int) (instance 1)], void>(const amrex::Gpu::KernelInfo &, T2, T3 &&)::[lambda() (instance 1)]>(T2)
--- 174 registers, 0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads, function name:
void amrex::launch_global<(int)256, std::enable_if<amrex::MaybeDeviceRunnable<T3, void>::value, void>::type amrex::ParallelFor<(int)256, int, std::enable_if<std::is_integral<T2>::value||std::is_same<T2, amrex::Box>::value, bool>::type amrex::detail::ParallelFor_helper2<(int)256, int, AdvanceBeamParticlesSlice(BeamParticleContainer &, const Fields &, const amrex::Vector<amrex::Geometry, std::allocator<amrex::Geometry>> &, int, int)::[lambda(int, T1, T2) (instance 1)], std::integral_constant<int, (int)3>, std::integral_constant<int, (int)0>>(const T2 &, T3 &&, amrex::TypeList<T4...>, const std::array<int, sizeof...(T4)> &)::[lambda(int) (instance 1)], void>(const amrex::Gpu::KernelInfo &, T2, T3 &&)::[lambda() (instance 1)]>(T2)
--- 128 registers, 0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads, function name:
void amrex::launch_global<(int)256, std::enable_if<amrex::MaybeDeviceRunnable<T3, void>::value, void>::type amrex::ParallelFor<(int)256, int, std::enable_if<std::is_integral<T2>::value||std::is_same<T2, amrex::Box>::value, bool>::type amrex::detail::ParallelFor_helper2<(int)256, int, AdvanceBeamParticlesSlice(BeamParticleContainer &, const Fields &, const amrex::Vector<amrex::Geometry, std::allocator<amrex::Geometry>> &, int, int)::[lambda(int, T1, T2) (instance 1)], std::integral_constant<int, (int)2>, std::integral_constant<int, (int)0>>(const T2 &, T3 &&, amrex::TypeList<T4...>, const std::array<int, sizeof...(T4)> &)::[lambda(int) (instance 1)], void>(const amrex::Gpu::KernelInfo &, T2, T3 &&)::[lambda() (instance 1)]>(T2)
--- 128 registers, 0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads, function name:
void amrex::launch_global<(int)256, std::enable_if<amrex::MaybeDeviceRunnable<T3, void>::value, void>::type amrex::ParallelFor<(int)256, int, std::enable_if<std::is_integral<T2>::value||std::is_same<T2, amrex::Box>::value, bool>::type amrex::detail::ParallelFor_helper2<(int)256, int, AdvanceBeamParticlesSlice(BeamParticleContainer &, const Fields &, const amrex::Vector<amrex::Geometry, std::allocator<amrex::Geometry>> &, int, int)::[lambda(int, T1, T2) (instance 1)], std::integral_constant<int, (int)1>, std::integral_constant<int, (int)0>>(const T2 &, T3 &&, amrex::TypeList<T4...>, const std::array<int, sizeof...(T4)> &)::[lambda(int) (instance 1)], void>(const amrex::Gpu::KernelInfo &, T2, T3 &&)::[lambda() (instance 1)]>(T2)
--- 127 registers, 240 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads, function name:
void amrex::launch_global<(int)256, std::enable_if<amrex::MaybeDeviceRunnable<T3, void>::value, void>::type amrex::ParallelFor<(int)256, int, std::enable_if<std::is_integral<T2>::value||std::is_same<T2, amrex::Box>::value, bool>::type amrex::detail::ParallelFor_helper2<(int)256, int, AdvanceBeamParticlesSlice(BeamParticleContainer &, const Fields &, const amrex::Vector<amrex::Geometry, std::allocator<amrex::Geometry>> &, int, int)::[lambda(int, T1, T2) (instance 1)], std::integral_constant<int, (int)2>, std::integral_constant<int, (int)1>>(const T2 &, T3 &&, amrex::TypeList<T4...>, const std::array<int, sizeof...(T4)> &)::[lambda(int) (instance 1)], void>(const amrex::Gpu::KernelInfo &, T2, T3 &&)::[lambda() (instance 1)]>(T2)
--- 127 registers, 240 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads, function name:
void amrex::launch_global<(int)256, std::enable_if<amrex::MaybeDeviceRunnable<T3, void>::value, void>::type amrex::ParallelFor<(int)256, int, std::enable_if<std::is_integral<T2>::value||std::is_same<T2, amrex::Box>::value, bool>::type amrex::detail::ParallelFor_helper2<(int)256, int, AdvanceBeamParticlesSlice(BeamParticleContainer &, const Fields &, const amrex::Vector<amrex::Geometry, std::allocator<amrex::Geometry>> &, int, int)::[lambda(int, T1, T2) (instance 1)], std::integral_constant<int, (int)1>, std::integral_constant<int, (int)1>>(const T2 &, T3 &&, amrex::TypeList<T4...>, const std::array<int, sizeof...(T4)> &)::[lambda(int) (instance 1)], void>(const amrex::Gpu::KernelInfo &, T2, T3 &&)::[lambda() (instance 1)]>(T2)
--- 127 registers, 240 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads, function name:
void amrex::launch_global<(int)256, std::enable_if<amrex::MaybeDeviceRunnable<T3, void>::value, void>::type amrex::ParallelFor<(int)256, int, std::enable_if<std::is_integral<T2>::value||std::is_same<T2, amrex::Box>::value, bool>::type amrex::detail::ParallelFor_helper2<(int)256, int, AdvanceBeamParticlesSlice(BeamParticleContainer &, const Fields &, const amrex::Vector<amrex::Geometry, std::allocator<amrex::Geometry>> &, int, int)::[lambda(int, T1, T2) (instance 1)], std::integral_constant<int, (int)0>, std::integral_constant<int, (int)1>>(const T2 &, T3 &&, amrex::TypeList<T4...>, const std::array<int, sizeof...(T4)> &)::[lambda(int) (instance 1)], void>(const amrex::Gpu::KernelInfo &, T2, T3 &&)::[lambda() (instance 1)]>(T2)
--- 112 registers, 0 bytes stack frame, 0 bytes spill stores, 0 bytes spill loads, function name:
void amrex::launch_global<(int)256, std::enable_if<amrex::MaybeDeviceRunnable<T3, void>::value, void>::type amrex::ParallelFor<(int)256, int, std::enable_if<std::is_integral<T2>::value||std::is_same<T2, amrex::Box>::value, bool>::type amrex::detail::ParallelFor_helper2<(int)256, int, AdvanceBeamParticlesSlice(BeamParticleContainer &, const Fields &, const amrex::Vector<amrex::Geometry, std::allocator<amrex::Geometry>> &, int, int)::[lambda(int, T1, T2) (instance 1)], std::integral_constant<int, (int)0>, std::integral_constant<int, (int)0>>(const T2 &, T3 &&, amrex::TypeList<T4...>, const std::array<int, sizeof...(T4)> &)::[lambda(int) (instance 1)], void>(const amrex::Gpu::KernelInfo &, T2, T3 &&)::[lambda() (instance 1)]>(T2)

Small enough (< few 100s of lines), otherwise it should probably be split into smaller PRs
Tested (describe the tests in the PR description)
Runs on GPU (basic: the code compiles and run well with the new module)
Contains an automated test (checksum and/or comparison with theory)
Documented: all elements (classes and their members, functions, namespaces, etc.) are documented
Constified (All that can be const is const)
Code is clean (no unwanted comments, )
Style and code conventions are respected at the bottom of https://github.com/Hi-PACE/hipace
Proper label and GitHub project, if applicable

…tracking

…n_for_runtime_components' into spin_tracking

…n_tracking

docs/source/run/parameters.rst

MaxThevenet

Thanks for this PR! Just left 1 comment in the code.

docs/source/run/parameters.rst

AlexanderSinn · 2024-03-13T16:28:44Z

I added the register usage for GPU to the PR description.

MaxThevenet

Great!

AlexanderSinn added 11 commits February 13, 2024 22:21

use id is_valid

21b43c3

use amrex removeInvalidParticles for the beam

3f04bad

prepare beam communication for runtime components

83bef52

Add support for spin to openpmd beam output

8edacf6

fix assert

216ca5e

merge dev

0bc4344

Merge branch 'Add_support_for_spin_to_openpmd_beam_output' into spin_…

782fec9

…tracking

Merge remote-tracking branch 'AlexanderSinn/prepare_beam_communicatio…

21a9050

…n_for_runtime_components' into spin_tracking

Merge branch 'use_amrex_removeInvalidParticles_for_the_beam' into spi…

da69354

…n_tracking

Merge remote-tracking branch 'AlexanderSinn/use_id_is_valid' into spi…

4a2d1ab

…n_tracking

add spin push

63f76f3

AlexanderSinn added the component: beam About the beam species label Feb 21, 2024

AlexanderSinn and others added 7 commits February 22, 2024 20:14

allocate runtime components

718659d

Merge branch 'development' into spin_tracking

2c0a66b

fix warning

ac3ec96

Merge branch 'development' into spin_tracking

541ba02

Merge branch 'development' into spin_tracking

27debd2

add input doc

29712b6

fix units

31a10e9

AlexanderSinn requested a review from MaxThevenet March 11, 2024 23:53

ax3l reviewed Mar 12, 2024

View reviewed changes

docs/source/run/parameters.rst Outdated Show resolved Hide resolved

ax3l mentioned this pull request Mar 12, 2024

Spin Tracking ECP-WarpX/WarpX#4766

Open

MaxThevenet reviewed Mar 13, 2024

View reviewed changes

docs/source/run/parameters.rst Outdated Show resolved Hide resolved

Add reference to the TBMT model

0db47c1

AlexanderSinn requested a review from MaxThevenet March 13, 2024 16:27

MaxThevenet approved these changes Mar 13, 2024

View reviewed changes

MaxThevenet merged commit b0339c1 into Hi-PACE:development Mar 13, 2024
10 checks passed

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Add Beam Spin tracking #1071

Add Beam Spin tracking #1071

AlexanderSinn commented Feb 21, 2024 •

edited

Loading

MaxThevenet left a comment

AlexanderSinn commented Mar 13, 2024

MaxThevenet left a comment

Add Beam Spin tracking #1071

Add Beam Spin tracking #1071

Conversation

AlexanderSinn commented Feb 21, 2024 • edited Loading

MaxThevenet left a comment

Choose a reason for hiding this comment

AlexanderSinn commented Mar 13, 2024

MaxThevenet left a comment

Choose a reason for hiding this comment

AlexanderSinn commented Feb 21, 2024 •

edited

Loading