diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2fa6c2f71..00b8923ee 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -20,7 +20,7 @@ jobs: strategy: matrix: os: [self-hosted, ubuntu-latest, macos-latest] - python-version: ['3.8', '3.10'] + python-version: ['3.8', '3.10', '3.12'] include: - os: ubuntu-20.04 python-version: '3.6' @@ -39,6 +39,7 @@ jobs: exuberant-ctags \ gfortran \ git \ + libhwloc-dev \ libopenblas-dev \ pkg-config \ software-properties-common @@ -50,8 +51,9 @@ jobs: ctags-exuberant \ gawk \ gnu-sed \ + hwloc \ pkg-config - - uses: actions/setup-python@v4.3.0 + - uses: actions/setup-python@v5.0.0 with: python-version: ${{ matrix.python-version }} - name: "Software Install - Python" @@ -110,8 +112,6 @@ jobs: coverage run --source=bifrost.ring,bifrost,bifrost.pipeline test_fft.py coverage run --source=bifrost.ring,bifrost,bifrost.pipeline your_first_block.py python download_breakthrough_listen_data.py -y - coverage run --source=bifrost.ring,bifrost,bifrost.pipeline test_guppi.py - coverage run --source=bifrost.ring,bifrost,bifrost.pipeline test_guppi_reader.py coverage run --source=bifrost.ring,bifrost,bifrost.pipeline test_fdmt.py ./testdata/pulsars/blc0_guppi_57407_61054_PSR_J1840%2B5640_0004.fil coverage xml - name: "Upload Coverage" diff --git a/Makefile.in b/Makefile.in index 852b2f951..29c5728b4 100644 --- a/Makefile.in +++ b/Makefile.in @@ -21,7 +21,7 @@ libbifrost: test: #$(MAKE) -C $(SRC_DIR) test ifeq ($(HAVE_PYTHON),1) - cd test && ./download_test_data.sh ; python -m unittest discover + cd test && ./download_test_data.sh ; python -m unittest discover -v endif .PHONY: test clean: diff --git a/config/cuda.m4 b/config/cuda.m4 index 987e69803..055418ea1 100644 --- a/config/cuda.m4 +++ b/config/cuda.m4 @@ -14,6 +14,11 @@ AC_DEFUN([AX_CHECK_CUDA], [enable_cuda=no], [enable_cuda=yes]) + NVCCLIBS="" + ac_compile_save="$ac_compile" + ac_link_save="$ac_link" + ac_run_save="$ac_run" + AC_SUBST([HAVE_CUDA], [0]) AC_SUBST([CUDA_VERSION], [0]) AC_SUBST([CUDA_HAVE_CXX20], [0]) @@ -38,11 +43,11 @@ AC_DEFUN([AX_CHECK_CUDA], CXXFLAGS_save="$CXXFLAGS" LDFLAGS_save="$LDFLAGS" - LIBS_save="$LIBS" + NVCCLIBS_save="$NVCCLIBS" ac_compile='$NVCC -c $NVCCFLAGS conftest.$ac_ext >&5' LDFLAGS="-L$CUDA_HOME/lib64 -L$CUDA_HOME/lib" - LIBS="$LIBS -lcuda -lcudart" + NVCCLIBS="$LIBS -lcuda -lcudart" ac_link='$NVCC -o conftest$ac_exeext $NVCCFLAGS $LDFLAGS $LIBS conftest.$ac_ext >&5' AC_LINK_IFELSE([ @@ -50,19 +55,31 @@ AC_DEFUN([AX_CHECK_CUDA], #include #include ]], [[cudaMalloc(0, 0);]])], - [CUDA_VERSION=$( ${NVCC} --version | ${GREP} -Po -e "release.*," | cut -d, -f1 | cut -d\ -f2 ) - CUDA_MAJOR=$( echo "${CUDA_VERSION}" | cut -d. -f1 ) - if test "${CUDA_MAJOR}" -ge 10; then - AC_MSG_RESULT(yes - v$CUDA_VERSION) - else - AC_MSG_RESULT(no - found v$CUDA_VERSION) - fi], - [AC_MSG_RESULT(no - build failure) - AC_SUBST([HAVE_CUDA], [0])]) + [], + [AC_SUBST([HAVE_CUDA], [0])]) + + if test "$HAVE_CUDA" = "1"; then + LDFLAGS="-L$CUDA_HOME/lib64 -L$CUDA_HOME/lib" + NVCCLIBS="$NVCCLIBS -lcuda -lcudart" + + ac_link='$NVCC -o conftest$ac_exeext $NVCCFLAGS $LDFLAGS $LIBS $NVCCLIBS conftest.$ac_ext >&5' + AC_LINK_IFELSE([ + AC_LANG_PROGRAM([[ + #include + #include ]], + [[cudaMalloc(0, 0);]])], + [CUDA_VERSION=$( ${NVCC} --version | ${GREP} -Po -e "release.*," | cut -d, -f1 | cut -d\ -f2 ) + AC_MSG_RESULT(yes - v$CUDA_VERSION)], + [AC_MSG_RESULT(no) + AC_SUBST([HAVE_CUDA], [0])]) + else + AC_MSG_RESULT(no) + AC_SUBST([HAVE_CUDA], [0]) + fi CXXFLAGS="$CXXFLAGS_save" LDFLAGS="$LDFLAGS_save" - LIBS="$LIBS_save" + NVCCLIBS="$NVCCLIBS_save" fi if test "$HAVE_CUDA" = "1"; then @@ -131,7 +148,7 @@ AC_DEFUN([AX_CHECK_CUDA], CXXFLAGS="$CXXFLAGS -DBF_CUDA_ENABLED=1" NVCCFLAGS="$NVCCFLAGS -DBF_CUDA_ENABLED=1" LDFLAGS="$LDFLAGS -L$CUDA_HOME/lib64 -L$CUDA_HOME/lib" - LIBS="$LIBS -lcuda -lcudart -lnvrtc -lcublas -lcudadevrt -L. -lcufft_static_pruned -lculibos -lnvToolsExt" + NVCCLIBS="$NVCCLIBS -lcuda -lcudart -lnvrtc -lcublas -lcudadevrt -L. -lcufft_static_pruned -lculibos -lnvToolsExt" fi AC_ARG_WITH([gpu_archs], @@ -150,11 +167,11 @@ AC_DEFUN([AX_CHECK_CUDA], CXXFLAGS_save="$CXXFLAGS" LDFLAGS_save="$LDFLAGS" - LIBS_save="$LIBS" + NVCCLIBS_save="$NVCCLIBS" LDFLAGS="-L$CUDA_HOME/lib64 -L$CUDA_HOME/lib" - LIBS="-lcuda -lcudart" - ac_run='$NVCC -o conftest$ac_ext $LDFLAGS $LIBS conftest.$ac_ext>&5' + NVCCLIBS="-lcuda -lcudart" + ac_run='$NVCC -o conftest$ac_ext $LDFLAGS $LIBS $NVCCLIBS conftest.$ac_ext>&5' AC_RUN_IFELSE([ AC_LANG_PROGRAM([[ #include @@ -204,7 +221,7 @@ AC_DEFUN([AX_CHECK_CUDA], CXXFLAGS="$CXXFLAGS_save" LDFLAGS="$LDFLAGS_save" - LIBS="$LIBS_save" + NVCCLIBS="$NVCCLIBS_save" else AC_SUBST([GPU_ARCHS], [$with_gpu_archs]) fi @@ -234,10 +251,10 @@ AC_DEFUN([AX_CHECK_CUDA], CXXFLAGS_save="$CXXFLAGS" LDFLAGS_save="$LDFLAGS" - LIBS_save="$LIBS" + NVCCLIBS_save="$NVCCLIBS" LDFLAGS="-L$CUDA_HOME/lib64 -L$CUDA_HOME/lib" - LIBS="-lcuda -lcudart" + NVCCLIBS="-lcuda -lcudart" ac_run='$NVCC -o conftest$ac_ext $LDFLAGS $LIBS conftest.$ac_ext>&5' AC_RUN_IFELSE([ AC_LANG_PROGRAM([[ @@ -275,7 +292,7 @@ AC_DEFUN([AX_CHECK_CUDA], CXXFLAGS="$CXXFLAGS_save" LDFLAGS="$LDFLAGS_save" - LIBS="$LIBS_save" + NVCCLIBS="$NVCCLIBS_save" else AC_SUBST([GPU_SHAREDMEM], [$with_shared_mem]) fi @@ -293,11 +310,11 @@ AC_DEFUN([AX_CHECK_CUDA], AC_MSG_CHECKING([for thrust pinned allocated support]) CXXFLAGS_save="$CXXFLAGS" LDFLAGS_save="$LDFLAGS" - LIBS_save="$LIBS" + NVCCLIBS_save="$NVCCLIBS" LDFLAGS="-L$CUDA_HOME/lib64 -L$CUDA_HOME/lib" - LIBS="-lcuda -lcudart" - ac_run='$NVCC -o conftest$ac_ext $LDFLAGS $LIBS conftest.$ac_ext>&5' + NVCCLIBS="-lcuda -lcudart" + ac_run='$NVCC -o conftest$ac_ext $LDFLAGS $LIBS $NVCCLIBS conftest.$ac_ext>&5' AC_RUN_IFELSE([ AC_LANG_PROGRAM([[ #include @@ -311,6 +328,13 @@ AC_DEFUN([AX_CHECK_CUDA], CXXFLAGS="$CXXFLAGS_save" LDFLAGS="$LDFLAGS_save" - LIBS="$LIBS_save" + NVCCLIBS="$NVCCLIBS_save" + else + AC_SUBST([GPU_PASCAL_MANAGEDMEM], [0]) + AC_SUBST([GPU_EXP_PINNED_ALLOC], [1]) fi + + ac_compile="$ac_compile_save" + ac_link="$ac_link_save" + ac_run="$ac_run_save" ]) diff --git a/config/intrinsics.m4 b/config/intrinsics.m4 new file mode 100644 index 000000000..2f99a93de --- /dev/null +++ b/config/intrinsics.m4 @@ -0,0 +1,121 @@ +# +# SSE +# + +AC_DEFUN([AX_CHECK_SSE], +[ + AC_PROVIDE([AX_CHECK_SSE]) + AC_ARG_ENABLE([sse], + [AS_HELP_STRING([--disable-sse], + [disable SSE support (default=no)])], + [enable_sse=no], + [enable_sse=yes]) + + AC_SUBST([HAVE_SSE], [0]) + + if test "$enable_sse" = "yes"; then + AC_MSG_CHECKING([for SSE support via '-msse']) + + CXXFLAGS_temp="$CXXFLAGS -msse" + + ac_run="$CXX -o conftest$ac_ext $CXXFLAGS_temp conftest.$ac_ext>&5" + AC_RUN_IFELSE([ + AC_LANG_PROGRAM([[ + #include ]], + [[ + __m128 x = _mm_set1_ps(1.0f); + x = _mm_add_ps(x, x); + return _mm_cvtss_f32(x) != 2.0f;]])], + [ + CXXFLAGS="$CXXFLAGS -msse" + AC_SUBST([HAVE_SSE], [1]) + AC_MSG_RESULT([yes]) + ], [ + AC_MSG_RESULT([no]) + ]) + fi +]) + + + +# +# AVX +# + +AC_DEFUN([AX_CHECK_AVX], +[ + AC_PROVIDE([AX_CHECK_AVX]) + AC_ARG_ENABLE([avx], + [AS_HELP_STRING([--disable-avx], + [disable AVX support (default=no)])], + [enable_avx=no], + [enable_avx=yes]) + + AC_SUBST([HAVE_AVX], [0]) + + if test "$enable_avx" = "yes"; then + AC_MSG_CHECKING([for AVX support via '-mavx']) + + CXXFLAGS_temp="$CXXFLAGS -mavx" + ac_run_save="$ac_run" + + ac_run="$CXX -o conftest$ac_ext $CXXFLAGS_temp conftest.$ac_ext>&5" + AC_RUN_IFELSE([ + AC_LANG_PROGRAM([[ + #include ]], + [[ + __m256d x = _mm256_set1_pd(1.0); + x = _mm256_add_pd(x, x); + return _mm256_cvtsd_f64(x) != 2.0;]])], + [ + CXXFLAGS="$CXXFLAGS -mavx" + AC_SUBST([HAVE_AVX], [1]) + AC_MSG_RESULT([yes]) + ], [ + AC_MSG_RESULT([no]) + ]) + + ac_run="$ac_run_save" + fi +]) + +# +# AVX512 +# + +AC_DEFUN([AX_CHECK_AVX512], +[ + AC_PROVIDE([AX_CHECK_AVX512]) + AC_ARG_ENABLE([avx512], + [AS_HELP_STRING([--disable-avx512], + [disable AVX512 support (default=no)])], + [enable_avx512=no], + [enable_avx512=yes]) + + AC_SUBST([HAVE_AVX512], [0]) + + if test "$enable_avx512" = "yes"; then + AC_MSG_CHECKING([for AVX-512 support via '-mavx512f']) + + CXXFLAGS_temp="$CXXFLAGS -mavx512f" + ac_run_save="$ac_run" + + ac_run="$CXX -o conftest$ac_ext $CXXFLAGS_temp conftest.$ac_ext>&5" + AC_RUN_IFELSE([ + AC_LANG_PROGRAM([[ + #include ]], + [[ + __m512d x = _mm512_set1_pd(1.0); + x = _mm512_add_pd(x, x); + return _mm512_cvtsd_f64(x) != 2.0;]])], + [ + CXXFLAGS="$CXXFLAGS -mavx512f" + AC_SUBST([HAVE_AVX512], [1]) + AC_MSG_RESULT([yes]) + ], [ + AC_MSG_RESULT([no]) + ]) + + ac_run="$ac_run_save" + fi +]) diff --git a/configure b/configure index b94108dcb..2fc51bfdd 100755 --- a/configure +++ b/configure @@ -658,6 +658,7 @@ ac_includes_default="\ ac_header_c_list= ac_subst_vars='OPTIONS LTLIBOBJS +NVCC_GENCODE STDCXX_IS_SET MAP_KERNEL_STDCXX PACKAGE_VERSION_MICRO @@ -712,6 +713,9 @@ HAVE_PYTHON HAVE_MAP_CACHE HAVE_CUDA_DEBUG enable_native_arch +HAVE_AVX512 +HAVE_AVX +HAVE_SSE HAVE_TRACE HAVE_DEBUG HAVE_TMPFS @@ -733,9 +737,14 @@ CUDA_HAVE_CXX20 CUDA_VERSION HAVE_CUDA CUDA_HOME +RDMA_MAXMEM +HAVE_RDMA +VERBS_SEND_PACING +VERBS_SEND_NPKTBUF +VERBS_NPKTBUF +HAVE_VERBS HAVE_VMA HAVE_HWLOC -HAVE_NUMA HAVE_FLOAT128 HAVE_OPENMP LIBOBJS @@ -847,9 +856,13 @@ with_gnu_ld with_sysroot enable_libtool_lock with_ctags -enable_numa enable_hwloc enable_vma +enable_verbs +with_rx_buffer_size +with_tx_buffer_size +enable_rdma +with_rdma_max_mem with_cuda_home enable_cuda with_nvcc_flags @@ -860,6 +873,9 @@ with_alignment with_logging_dir enable_debug enable_trace +enable_sse +enable_avx +enable_avx512 enable_native_arch enable_cuda_debug enable_map_cache @@ -1523,12 +1539,16 @@ Optional Features: --enable-fast-install[=PKGS] optimize for fast installation [default=yes] --disable-libtool-lock avoid locking (might break parallel builds) - --disable-numa disable numa support (default=no) --disable-hwloc disable hwloc support (default=no) --enable-vma enable vma support (default=no) + --disable-verbs disable Infiniband verbs support (default=no) + --disable-rdma disable RDMA support (default=no) --disable-cuda disable cuda support (default=no) --enable-debug enable debugging mode (default=no) --enable-trace enable tracing mode for nvprof/nvvp (default=no) + --disable-sse disable SSE support (default=no) + --disable-avx disable AVX support (default=no) + --disable-avx512 disable AVX512 support (default=no) --disable-native-arch disable native architecture compilation (default=no) --enable-cuda-debug enable CUDA debugging (nvcc -G; default=no) --disable-map-cache disable caching bifrost.map kernels (default=no) @@ -1557,6 +1577,12 @@ Optional Packages: --with-sysroot[=DIR] Search for dependent libraries within DIR (or the compiler's sysroot if not specified). --with-ctags=[PATH] absolute path to ctags executable + --with-rx-buffer-size=N default Infiniband verbs receive buffer size in + packets (default=8192) + --with-tx-buffer-size=N default Infiniband verbs send buffer size in packets + (default=512) + --with-rdma-max-mem=N maximum RDMA buffer size in bytes + (default=134217728) --with-cuda-home CUDA install path (default=/usr/local/cuda) --with-nvcc-flags flags to pass to NVCC (default='-O3 -Xcompiler "-Wall"') @@ -21006,65 +21032,6 @@ then : fi -# -# NUMA -# - -# Check whether --enable-numa was given. -if test ${enable_numa+y} -then : - enableval=$enable_numa; enable_numa=no -else $as_nop - enable_numa=yes -fi - -HAVE_NUMA=0 - -if test x$enable_numa != xno -then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for numa_node_of_cpu in -lnuma" >&5 -printf %s "checking for numa_node_of_cpu in -lnuma... " >&6; } -if test ${ac_cv_lib_numa_numa_node_of_cpu+y} -then : - printf %s "(cached) " >&6 -else $as_nop - ac_check_lib_save_LIBS=$LIBS -LIBS="-lnuma $LIBS" -cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ - -namespace conftest { - extern "C" int numa_node_of_cpu (); -} -int -main (void) -{ -return conftest::numa_node_of_cpu (); - ; - return 0; -} -_ACEOF -if ac_fn_cxx_try_link "$LINENO" -then : - ac_cv_lib_numa_numa_node_of_cpu=yes -else $as_nop - ac_cv_lib_numa_numa_node_of_cpu=no -fi -rm -f core conftest.err conftest.$ac_objext conftest.beam \ - conftest$ac_exeext conftest.$ac_ext -LIBS=$ac_check_lib_save_LIBS -fi -{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_numa_numa_node_of_cpu" >&5 -printf "%s\n" "$ac_cv_lib_numa_numa_node_of_cpu" >&6; } -if test "x$ac_cv_lib_numa_numa_node_of_cpu" = xyes -then : - HAVE_NUMA=1 - - LIBS="$LIBS -lnuma" -fi - -fi - # # HWLOC # @@ -21224,6 +21191,220 @@ fi fi +# +# Infiniband Verbs +# + +# Check whether --enable-verbs was given. +if test ${enable_verbs+y} +then : + enableval=$enable_verbs; enable_verbs=no +else $as_nop + enable_verbs=yes +fi + +HAVE_VERBS=0 + +if test x$enable_verbs != xno +then : + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for ibv_query_device in -libverbs" >&5 +printf %s "checking for ibv_query_device in -libverbs... " >&6; } +if test ${ac_cv_lib_ibverbs_ibv_query_device+y} +then : + printf %s "(cached) " >&6 +else $as_nop + ac_check_lib_save_LIBS=$LIBS +LIBS="-libverbs $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +namespace conftest { + extern "C" int ibv_query_device (); +} +int +main (void) +{ +return conftest::ibv_query_device (); + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_link "$LINENO" +then : + ac_cv_lib_ibverbs_ibv_query_device=yes +else $as_nop + ac_cv_lib_ibverbs_ibv_query_device=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_ibverbs_ibv_query_device" >&5 +printf "%s\n" "$ac_cv_lib_ibverbs_ibv_query_device" >&6; } +if test "x$ac_cv_lib_ibverbs_ibv_query_device" = xyes +then : + HAVE_VERBS=1 + + LIBS="$LIBS -libverbs" +fi + +fi + + +# Check whether --with-rx-buffer-size was given. +if test ${with_rx_buffer_size+y} +then : + withval=$with_rx_buffer_size; +else $as_nop + with_rx_buffer_size=8192 +fi + + VERBS_NPKTBUF=$with_rx_buffer_size + + if test x$HAVE_VERBS = x0 +then : + VERBS_NPKTBUF=0 + +fi + + +# Check whether --with-tx-buffer-size was given. +if test ${with_tx_buffer_size+y} +then : + withval=$with_tx_buffer_size; +else $as_nop + with_tx_buffer_size=512 +fi + + VERBS_SEND_NPKTBUF=$with_tx_buffer_size + + if test x$HAVE_VERBS = x0 +then : + VERBS_SEND_NPKTBUF=0 + +fi + + if test x$HAVE_VERBS != x0 +then : + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for Infiniband verbs packet pacing support" >&5 +printf %s "checking for Infiniband verbs packet pacing support... " >&6; } + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + + #include +int +main (void) +{ + + int ndev, d; + ibv_device** ibv_dev_list = NULL; + ibv_context* ibv_ctx = NULL; + ibv_device_attr_ex ibv_dev_attr; + + ibv_dev_list = ibv_get_device_list(&ndev); + for(d=0; d&5 +printf "%s\n" "yes" >&6; } +else $as_nop + VERBS_SEND_PACING=0 + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam conftest.$ac_ext + +fi + +# +# RDMA +# + +# Check whether --enable-rdma was given. +if test ${enable_rdma+y} +then : + enableval=$enable_rdma; enable_rdma=no +else $as_nop + enable_rdma=yes +fi + +HAVE_RDMA=0 + +if test x$enable_rdma != xno +then : + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for rdma_get_devices in -lrdmacm" >&5 +printf %s "checking for rdma_get_devices in -lrdmacm... " >&6; } +if test ${ac_cv_lib_rdmacm_rdma_get_devices+y} +then : + printf %s "(cached) " >&6 +else $as_nop + ac_check_lib_save_LIBS=$LIBS +LIBS="-lrdmacm $LIBS" +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +namespace conftest { + extern "C" int rdma_get_devices (); +} +int +main (void) +{ +return conftest::rdma_get_devices (); + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_link "$LINENO" +then : + ac_cv_lib_rdmacm_rdma_get_devices=yes +else $as_nop + ac_cv_lib_rdmacm_rdma_get_devices=no +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext conftest.$ac_ext +LIBS=$ac_check_lib_save_LIBS +fi +{ printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: $ac_cv_lib_rdmacm_rdma_get_devices" >&5 +printf "%s\n" "$ac_cv_lib_rdmacm_rdma_get_devices" >&6; } +if test "x$ac_cv_lib_rdmacm_rdma_get_devices" = xyes +then : + HAVE_RDMA=1 + + LIBS="$LIBS -lrdmacm" +fi + +fi + + +# Check whether --with-rdma_max_mem was given. +if test ${with_rdma_max_mem+y} +then : + withval=$with_rdma_max_mem; +else $as_nop + with_rdma_max_mem=134217728 +fi + +RDMA_MAXMEM=$with_rdma_max_mem + +if test x$HAVE_RDMA = x0 +then : + RDMA_MAXMEM=0 + +fi + # # CUDA # @@ -21259,6 +21440,11 @@ else $as_nop fi + NVCCLIBS="" + ac_compile_save="$ac_compile" + ac_link_save="$ac_link" + ac_run_save="$ac_run" + HAVE_CUDA=0 CUDA_VERSION=0 @@ -21434,11 +21620,11 @@ printf %s "checking for a working CUDA 10+ installation... " >&6; } CXXFLAGS_save="$CXXFLAGS" LDFLAGS_save="$LDFLAGS" - LIBS_save="$LIBS" + NVCCLIBS_save="$NVCCLIBS" ac_compile='$NVCC -c $NVCCFLAGS conftest.$ac_ext >&5' LDFLAGS="-L$CUDA_HOME/lib64 -L$CUDA_HOME/lib" - LIBS="$LIBS -lcuda -lcudart" + NVCCLIBS="$LIBS -lcuda -lcudart" ac_link='$NVCC -o conftest$ac_exeext $NVCCFLAGS $LDFLAGS $LIBS conftest.$ac_ext >&5' cat confdefs.h - <<_ACEOF >conftest.$ac_ext @@ -21456,28 +21642,57 @@ cudaMalloc(0, 0); } _ACEOF if ac_fn_cxx_try_link "$LINENO" +then : + +else $as_nop + HAVE_CUDA=0 + +fi +rm -f core conftest.err conftest.$ac_objext conftest.beam \ + conftest$ac_exeext conftest.$ac_ext + + if test "$HAVE_CUDA" = "1"; then + LDFLAGS="-L$CUDA_HOME/lib64 -L$CUDA_HOME/lib" + NVCCLIBS="$NVCCLIBS -lcuda -lcudart" + + ac_link='$NVCC -o conftest$ac_exeext $NVCCFLAGS $LDFLAGS $LIBS $NVCCLIBS conftest.$ac_ext >&5' + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + + #include + #include +int +main (void) +{ +cudaMalloc(0, 0); + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_link "$LINENO" then : CUDA_VERSION=$( ${NVCC} --version | ${GREP} -Po -e "release.*," | cut -d, -f1 | cut -d\ -f2 ) - CUDA_MAJOR=$( echo "${CUDA_VERSION}" | cut -d. -f1 ) - if test "${CUDA_MAJOR}" -ge 10; then { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes - v$CUDA_VERSION" >&5 printf "%s\n" "yes - v$CUDA_VERSION" >&6; } - else - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no - found v$CUDA_VERSION" >&5 -printf "%s\n" "no - found v$CUDA_VERSION" >&6; } - fi else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no - build failure" >&5 -printf "%s\n" "no - build failure" >&6; } - HAVE_CUDA=0 + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + HAVE_CUDA=0 fi rm -f core conftest.err conftest.$ac_objext conftest.beam \ conftest$ac_exeext conftest.$ac_ext + else + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + HAVE_CUDA=0 + + fi CXXFLAGS="$CXXFLAGS_save" LDFLAGS="$LDFLAGS_save" - LIBS="$LIBS_save" + NVCCLIBS="$NVCCLIBS_save" fi if test "$HAVE_CUDA" = "1"; then @@ -21568,7 +21783,7 @@ printf "%s\n" "no, only the 'legacy' stream model is supported" >&6; } CXXFLAGS="$CXXFLAGS -DBF_CUDA_ENABLED=1" NVCCFLAGS="$NVCCFLAGS -DBF_CUDA_ENABLED=1" LDFLAGS="$LDFLAGS -L$CUDA_HOME/lib64 -L$CUDA_HOME/lib" - LIBS="$LIBS -lcuda -lcudart -lnvrtc -lcublas -lcudadevrt -L. -lcufft_static_pruned -lculibos -lnvToolsExt" + NVCCLIBS="$NVCCLIBS -lcuda -lcudart -lnvrtc -lcublas -lcudadevrt -L. -lcufft_static_pruned -lculibos -lnvToolsExt" fi @@ -21594,11 +21809,11 @@ printf %s "checking which CUDA architectures to target... " >&6; } CXXFLAGS_save="$CXXFLAGS" LDFLAGS_save="$LDFLAGS" - LIBS_save="$LIBS" + NVCCLIBS_save="$NVCCLIBS" LDFLAGS="-L$CUDA_HOME/lib64 -L$CUDA_HOME/lib" - LIBS="-lcuda -lcudart" - ac_run='$NVCC -o conftest$ac_ext $LDFLAGS $LIBS conftest.$ac_ext>&5' + NVCCLIBS="-lcuda -lcudart" + ac_run='$NVCC -o conftest$ac_ext $LDFLAGS $LIBS $NVCCLIBS conftest.$ac_ext>&5' if test "$cross_compiling" = yes then : { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 @@ -21675,7 +21890,7 @@ fi CXXFLAGS="$CXXFLAGS_save" LDFLAGS="$LDFLAGS_save" - LIBS="$LIBS_save" + NVCCLIBS="$NVCCLIBS_save" else GPU_ARCHS=$with_gpu_archs @@ -21715,10 +21930,10 @@ printf %s "checking for minimum shared memory per block... " >&6; } CXXFLAGS_save="$CXXFLAGS" LDFLAGS_save="$LDFLAGS" - LIBS_save="$LIBS" + NVCCLIBS_save="$NVCCLIBS" LDFLAGS="-L$CUDA_HOME/lib64 -L$CUDA_HOME/lib" - LIBS="-lcuda -lcudart" + NVCCLIBS="-lcuda -lcudart" ac_run='$NVCC -o conftest$ac_ext $LDFLAGS $LIBS conftest.$ac_ext>&5' if test "$cross_compiling" = yes then : @@ -21782,7 +21997,7 @@ fi CXXFLAGS="$CXXFLAGS_save" LDFLAGS="$LDFLAGS_save" - LIBS="$LIBS_save" + NVCCLIBS="$NVCCLIBS_save" else GPU_SHAREDMEM=$with_shared_mem @@ -21807,11 +22022,11 @@ printf "%s\n" "no" >&6; } printf %s "checking for thrust pinned allocated support... " >&6; } CXXFLAGS_save="$CXXFLAGS" LDFLAGS_save="$LDFLAGS" - LIBS_save="$LIBS" + NVCCLIBS_save="$NVCCLIBS" LDFLAGS="-L$CUDA_HOME/lib64 -L$CUDA_HOME/lib" - LIBS="-lcuda -lcudart" - ac_run='$NVCC -o conftest$ac_ext $LDFLAGS $LIBS conftest.$ac_ext>&5' + NVCCLIBS="-lcuda -lcudart" + ac_run='$NVCC -o conftest$ac_ext $LDFLAGS $LIBS $NVCCLIBS conftest.$ac_ext>&5' if test "$cross_compiling" = yes then : { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 @@ -21853,9 +22068,18 @@ fi CXXFLAGS="$CXXFLAGS_save" LDFLAGS="$LDFLAGS_save" - LIBS="$LIBS_save" + NVCCLIBS="$NVCCLIBS_save" + else + GPU_PASCAL_MANAGEDMEM=0 + + GPU_EXP_PINNED_ALLOC=1 + fi + ac_compile="$ac_compile_save" + ac_link="$ac_link_save" + ac_run="$ac_run_save" + # # Bifrost memory alignment @@ -22012,6 +22236,207 @@ fi + # Check whether --enable-sse was given. +if test ${enable_sse+y} +then : + enableval=$enable_sse; enable_sse=no +else $as_nop + enable_sse=yes +fi + + + HAVE_SSE=0 + + + if test "$enable_sse" = "yes"; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for SSE support via '-msse'" >&5 +printf %s "checking for SSE support via '-msse'... " >&6; } + + CXXFLAGS_temp="$CXXFLAGS -msse" + + ac_run="$CXX -o conftest$ac_ext $CXXFLAGS_temp conftest.$ac_ext>&5" + if test "$cross_compiling" = yes +then : + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "cannot run test program while cross compiling +See \`config.log' for more details" "$LINENO" 5; } +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + + #include +int +main (void) +{ + + __m128 x = _mm_set1_ps(1.0f); + x = _mm_add_ps(x, x); + return _mm_cvtss_f32(x) != 2.0f; + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_run "$LINENO" +then : + + CXXFLAGS="$CXXFLAGS -msse" + HAVE_SSE=1 + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } + +else $as_nop + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + +fi +rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ + conftest.$ac_objext conftest.beam conftest.$ac_ext +fi + + fi + + + + # Check whether --enable-avx was given. +if test ${enable_avx+y} +then : + enableval=$enable_avx; enable_avx=no +else $as_nop + enable_avx=yes +fi + + + HAVE_AVX=0 + + + if test "$enable_avx" = "yes"; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for AVX support via '-mavx'" >&5 +printf %s "checking for AVX support via '-mavx'... " >&6; } + + CXXFLAGS_temp="$CXXFLAGS -mavx" + ac_run_save="$ac_run" + + ac_run="$CXX -o conftest$ac_ext $CXXFLAGS_temp conftest.$ac_ext>&5" + if test "$cross_compiling" = yes +then : + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "cannot run test program while cross compiling +See \`config.log' for more details" "$LINENO" 5; } +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + + #include +int +main (void) +{ + + __m256d x = _mm256_set1_pd(1.0); + x = _mm256_add_pd(x, x); + return _mm256_cvtsd_f64(x) != 2.0; + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_run "$LINENO" +then : + + CXXFLAGS="$CXXFLAGS -mavx" + HAVE_AVX=1 + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } + +else $as_nop + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + +fi +rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ + conftest.$ac_objext conftest.beam conftest.$ac_ext +fi + + + ac_run="$ac_run_save" + fi + + + + # Check whether --enable-avx512 was given. +if test ${enable_avx512+y} +then : + enableval=$enable_avx512; enable_avx512=no +else $as_nop + enable_avx512=yes +fi + + + HAVE_AVX512=0 + + + if test "$enable_avx512" = "yes"; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for AVX-512 support via '-mavx512f'" >&5 +printf %s "checking for AVX-512 support via '-mavx512f'... " >&6; } + + CXXFLAGS_temp="$CXXFLAGS -mavx512f" + ac_run_save="$ac_run" + + ac_run="$CXX -o conftest$ac_ext $CXXFLAGS_temp conftest.$ac_ext>&5" + if test "$cross_compiling" = yes +then : + { { printf "%s\n" "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +printf "%s\n" "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "cannot run test program while cross compiling +See \`config.log' for more details" "$LINENO" 5; } +else $as_nop + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + + #include +int +main (void) +{ + + __m512d x = _mm512_set1_pd(1.0); + x = _mm512_add_pd(x, x); + return _mm512_cvtsd_f64(x) != 2.0; + ; + return 0; +} +_ACEOF +if ac_fn_cxx_try_run "$LINENO" +then : + + CXXFLAGS="$CXXFLAGS -mavx512f" + HAVE_AVX512=1 + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +printf "%s\n" "yes" >&6; } + +else $as_nop + + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: result: no" >&5 +printf "%s\n" "no" >&6; } + +fi +rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ + conftest.$ac_objext conftest.beam conftest.$ac_ext +fi + + + ac_run="$ac_run_save" + fi + + + # Check whether --enable-native_arch was given. if test ${enable_native_arch+y} then : @@ -24740,6 +25165,20 @@ fi CXXFLAGS="$CXXFLAGS $lt_prog_compiler_pic_CXX" NVCCFLAGS="$NVCCFLAGS -Xcompiler \"$lt_prog_compiler_pic_CXX\"" +LIBS="$LIBS $NVCCLIBS" + +# +# Additional CUDA flags +# + +if test x$HAVE_CUDA != x0 +then : + NVCC_GENCODE=$(echo $GPU_ARCHS | ${SED} -e 's/\([0-9]\{2,3\}\)/-gencode arch=compute_\1,\\"code=sm_\1\\"/g;') + NVCC_GENCODE="$NVCC_GENCODE -gencode arch=compute_${GPU_MAX_ARCH},\\\"code=compute_${GPU_MAX_ARCH}\\\"" + NVCCFLAGS="$NVCCFLAGS ${NVCC_GENCODE}" + CPPFLAGS="$CPPFLAGS -I$CUDA_HOME/include" + +fi ac_config_files="$ac_config_files config.mk Makefile src/Makefile python/Makefile share/bifrost.pc src/bifrost/config.h" @@ -27035,15 +27474,6 @@ else $as_nop printf "%s\n" "$as_me: cuda: no" >&6;} fi -if test x$HAVE_NUMA = x1 -then : - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: numa: yes" >&5 -printf "%s\n" "$as_me: numa: yes" >&6;} -else $as_nop - { printf "%s\n" "$as_me:${as_lineno-$LINENO}: numa: no" >&5 -printf "%s\n" "$as_me: numa: no" >&6;} -fi - if test x$HAVE_HWLOC = x1 then : { printf "%s\n" "$as_me:${as_lineno-$LINENO}: hwloc: yes" >&5 @@ -27062,6 +27492,24 @@ else $as_nop printf "%s\n" "$as_me: libvma: no" >&6;} fi +if test x$HAVE_VERBS = x1 +then : + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: libverbs: yes" >&5 +printf "%s\n" "$as_me: libverbs: yes" >&6;} +else $as_nop + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: libverbs: no" >&5 +printf "%s\n" "$as_me: libverbs: no" >&6;} +fi + +if test x$HAVE_RDMA = x1 +then : + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: librdmacm: yes" >&5 +printf "%s\n" "$as_me: librdmacm: yes" >&6;} +else $as_nop + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: librdmacm: no" >&5 +printf "%s\n" "$as_me: librdmacm: no" >&6;} +fi + if test x$HAVE_PYTHON = x1 then : { printf "%s\n" "$as_me:${as_lineno-$LINENO}: python bindings: yes" >&5 @@ -27092,6 +27540,21 @@ if test x$enable_cuda_debug != xno then : OPTIONS="$OPTIONS cuda_debug" +fi +if test x$HAVE_SSE != x0 +then : + OPTIONS="$OPTIONS sse" + +fi +if test x$HAVE_AVX != x0 +then : + OPTIONS="$OPTIONS avx" + +fi +if test x$HAVE_AVX512 != x0 +then : + OPTIONS="$OPTIONS avx512" + fi if test x$enable_native_arch != xno then : diff --git a/configure.ac b/configure.ac index 062f52bca..2033cc92f 100644 --- a/configure.ac +++ b/configure.ac @@ -91,21 +91,6 @@ AC_SUBST([HAVE_FLOAT128], [0]) AS_IF([test x$HAVE_HAVE_LONG_DOUBLE_WIDER = x1], [AC_SUBST([HAVE_FLOAT128], [0])]) -# -# NUMA -# - -AC_ARG_ENABLE([numa], - AS_HELP_STRING([--disable-numa], - [disable numa support (default=no)]), - [enable_numa=no], - [enable_numa=yes]) -AC_SUBST([HAVE_NUMA], [0]) -AS_IF([test x$enable_numa != xno], - [AC_CHECK_LIB([numa], [numa_node_of_cpu], - [AC_SUBST([HAVE_NUMA], [1]) - LIBS="$LIBS -lnuma"])]) - # # HWLOC # @@ -139,6 +124,87 @@ AS_IF([test x$enable_vma != xno], [AC_SUBST([HAVE_VMA], [1]) LIBS="$LIBS -lvma"])]) +# +# Infiniband Verbs +# + +AC_ARG_ENABLE([verbs], + AS_HELP_STRING([--disable-verbs], + [disable Infiniband verbs support (default=no)]), + [enable_verbs=no], + [enable_verbs=yes]) +AC_SUBST([HAVE_VERBS], [0]) +AS_IF([test x$enable_verbs != xno], + [AC_CHECK_LIB([ibverbs], [ibv_query_device], + [AC_SUBST([HAVE_VERBS], [1]) + LIBS="$LIBS -libverbs"])]) + + AC_ARG_WITH([rx-buffer-size], + [AS_HELP_STRING([--with-rx-buffer-size=N], + [default Infiniband verbs receive buffer size in packets (default=8192)])], + [], + [with_rx_buffer_size=8192]) + AC_SUBST([VERBS_NPKTBUF], [$with_rx_buffer_size]) + AS_IF([test x$HAVE_VERBS = x0], + [AC_SUBST([VERBS_NPKTBUF], [0])]) + + AC_ARG_WITH([tx-buffer-size], + [AS_HELP_STRING([--with-tx-buffer-size=N], + [default Infiniband verbs send buffer size in packets (default=512)])], + [], + [with_tx_buffer_size=512]) + AC_SUBST([VERBS_SEND_NPKTBUF], [$with_tx_buffer_size]) + AS_IF([test x$HAVE_VERBS = x0], + [AC_SUBST([VERBS_SEND_NPKTBUF], [0])]) + + AS_IF([test x$HAVE_VERBS != x0], + [AC_MSG_CHECKING([for Infiniband verbs packet pacing support]) + AC_COMPILE_IFELSE([ + AC_LANG_PROGRAM([[ + #include ]], + [[ + int ndev, d; + ibv_device** ibv_dev_list = NULL; + ibv_context* ibv_ctx = NULL; + ibv_device_attr_ex ibv_dev_attr; + + ibv_dev_list = ibv_get_device_list(&ndev); + for(d=0; d int: def port(self) -> int: return _get(_bf.bfAddressGetPort, self.obj) @property + def is_multicast(self) -> bool: + return True if _get(_bf.bfAddressIsMulticast, self.obj) else False + @property def mtu(self) -> int: return _get(_bf.bfAddressGetMTU, self.obj) @property diff --git a/python/bifrost/packet_capture.py b/python/bifrost/packet_capture.py new file mode 100644 index 000000000..f470d6fa9 --- /dev/null +++ b/python/bifrost/packet_capture.py @@ -0,0 +1,176 @@ + +# Copyright (c) 2019-2024, The Bifrost Authors. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of The Bifrost Authors nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# **TODO: Write tests for this class + +from bifrost.libbifrost import _bf, _th, _check, _get, BifrostObject +from bifrost.udp_socket import UDPSocket +from bifrost.ring import Ring +from bifrost.ring2 import Ring as Ring2 + +import ctypes +from functools import reduce +from io import IOBase + +from typing import Optional, Union + +from bifrost import telemetry +telemetry.track_module() + +class PacketCaptureCallback(BifrostObject): + _ref_cache = {} + def __init__(self): + BifrostObject.__init__( + self, _bf.bfPacketCaptureCallbackCreate, _bf.bfPacketCaptureCallbackDestroy) + + def set_simple(self, fnc): + self._ref_cache['simple'] = _bf.BFpacketcapture_simple_sequence_callback(fnc) + _check(_bf.bfPacketCaptureCallbackSetSIMPLE( + self.obj, self._ref_cache['simple'])) + + def set_chips(self, fnc: _bf.BFpacketcapture_chips_sequence_callback): + self._ref_cache['chips'] = _bf.BFpacketcapture_chips_sequence_callback(fnc) + _check(_bf.bfPacketCaptureCallbackSetCHIPS( + self.obj, self._ref_cache['chips'])) + def set_snap2(self, fnc: _bf.BFpacketcapture_snap2_sequence_callback): + self._ref_cache['snap2'] = _bf.BFpacketcapture_snap2_sequence_callback(fnc) + _check(_bf.bfPacketCaptureCallbackSetSNAP2( + self.obj, self._ref_cache['snap2'])) + def set_ibeam(self, fnc: _bf.BFpacketcapture_ibeam_sequence_callback): + self._ref_cache['ibeam'] = _bf.BFpacketcapture_ibeam_sequence_callback(fnc) + _check(_bf.bfPacketCaptureCallbackSetIBeam( + self.obj, self._ref_cache['ibeam'])) + def set_pbeam(self, fnc: _bf.BFpacketcapture_pbeam_sequence_callback): + self._ref_cache['pbeam'] = _bf.BFpacketcapture_pbeam_sequence_callback(fnc) + _check(_bf.bfPacketCaptureCallbackSetPBeam( + self.obj, self._ref_cache['pbeam'])) + def set_cor(self, fnc: _bf.BFpacketcapture_cor_sequence_callback): + self._ref_cache['cor'] = _bf.BFpacketcapture_cor_sequence_callback(fnc) + _check(_bf.bfPacketCaptureCallbackSetCOR( + self.obj, self._ref_cache['cor'])) + def set_vdif(self, fnc: _bf.BFpacketcapture_vdif_sequence_callback): + self._ref_cache['vdif'] = _bf.BFpacketcapture_vdif_sequence_callback(fnc) + _check(_bf.bfPacketCaptureCallbackSetVDIF( + self.obj, self._ref_cache['vdif'])) + def set_tbn(self, fnc: _bf.BFpacketcapture_tbn_sequence_callback): + self._ref_cache['tbn'] = _bf.BFpacketcapture_tbn_sequence_callback(fnc) + _check(_bf.bfPacketCaptureCallbackSetTBN( + self.obj, self._ref_cache['tbn'])) + def set_drx(self, fnc: _bf.BFpacketcapture_drx_sequence_callback): + self._ref_cache['drx'] = _bf.BFpacketcapture_drx_sequence_callback(fnc) + _check(_bf.bfPacketCaptureCallbackSetDRX( + self.obj, self._ref_cache['drx'])) + def set_drx8(self, fnc: _bf.BFpacketcapture_drx8_sequence_callback): + self._ref_cache['drx8'] = _bf.BFpacketcapture_drx8_sequence_callback(fnc) + _check(_bf.bfPacketCaptureCallbackSetDRX8( + self.obj, self._ref_cache['drx8'])) + +class _CaptureBase(BifrostObject): + @staticmethod + def _flatten_value(value): + try: + value = reduce(lambda x,y: x*y, value, 1 if value else 0) + except TypeError: + pass + return value + def __enter__(self): + return self + def __exit__(self, type, value, tb): + self.end() + def recv(self) -> _th.BFcapture_enum: + status = _bf.BFpacketcapture_status() + _check(_bf.bfPacketCaptureRecv(self.obj, status)) + return status.value + def flush(self): + _check(_bf.bfPacketCaptureFlush(self.obj)) + def end(self): + _check(_bf.bfPacketCaptureEnd(self.obj)) + +class UDPCapture(_CaptureBase): + def __init__(self, fmt: str, sock: UDPSocket, ring: Union[Ring,Ring2], + nsrc: int, src0: int, max_payload_size: int, buffer_ntime: int, + slot_ntime: int, sequence_callback: PacketCaptureCallback, + core: Optional[int]=None): + nsrc = self._flatten_value(nsrc) + if core is None: + core = -1 + BifrostObject.__init__( + self, _bf.bfUdpCaptureCreate, _bf.bfPacketCaptureDestroy, + fmt.encode(), sock.fileno(), ring.obj, nsrc, src0, + max_payload_size, buffer_ntime, slot_ntime, + sequence_callback.obj, core) + +class UDPSniffer(_CaptureBase): + def __init__(self, fmt: str, sock: UDPSocket, ring: Union[Ring,Ring2], + nsrc: int, src0: int, max_payload_size: int, buffer_ntime: int, + slot_ntime: int, sequence_callback: PacketCaptureCallback, + core: Optional[int]=None): + nsrc = self._flatten_value(nsrc) + if core is None: + core = -1 + BifrostObject.__init__( + self, _bf.bfUdpSnifferCreate, _bf.bfPacketCaptureDestroy, + fmt.encode(), sock.fileno(), ring.obj, nsrc, src0, + max_payload_size, buffer_ntime, slot_ntime, + sequence_callback.obj, core) + +class UDPVerbsCapture(_CaptureBase): + def __init__(self, fmt: str, sock: UDPSocket, ring: Union[Ring,Ring2], + nsrc: int, src0: int, max_payload_size: int, buffer_ntime: int, + slot_ntime: int, sequence_callback: PacketCaptureCallback, + core: Optional[int]=None): + if core is None: + core = -1 + BifrostObject.__init__( + self, _bf.bfUdpVerbsCaptureCreate, _bf.bfPacketCaptureDestroy, + fmt.encode(), sock.fileno(), ring.obj, nsrc, src0, + max_payload_size, buffer_ntime, slot_ntime, + sequence_callback.obj, core) + +class DiskReader(_CaptureBase): + def __init__(self, fmt: str, fh: IOBase, ring: Union[Ring,Ring2], + nsrc: int, src0: int, buffer_nframe: int, slot_nframe: int, + sequence_callback: PacketCaptureCallback, + core: Optional[int]=None): + nsrc = self._flatten_value(nsrc) + if core is None: + core = -1 + BifrostObject.__init__( + self, _bf.bfDiskReaderCreate, _bf.bfPacketCaptureDestroy, + fmt.encode(), fh.fileno(), ring.obj, nsrc, src0, + buffer_nframe, slot_nframe, + sequence_callback.obj, core) + # Make sure we start in the same place in the file + self.seek(fh.tell(), _bf.BF_WHENCE_SET) + def seek(self, offset: int, whence: _th.BFwhence_enum=_bf.BF_WHENCE_CUR): + position = ctypes.c_ulong(0) + _check(_bf.bfPacketCaptureSeek(self.obj, offset, whence, position)) + return position.value + def tell(self) -> int: + position = ctypes.c_ulong(0) + _check(_bf.bfPacketCaptureTell(self.obj, position)) + return position.value diff --git a/python/bifrost/packet_writer.py b/python/bifrost/packet_writer.py new file mode 100644 index 000000000..0fbeae05b --- /dev/null +++ b/python/bifrost/packet_writer.py @@ -0,0 +1,103 @@ + +# Copyright (c) 2019-2024, The Bifrost Authors. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of The Bifrost Authors nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# **TODO: Write tests for this class + +from bifrost.libbifrost import _bf, _check, _get, BifrostObject +from bifrost.ndarray import asarray +from bifrost.ndarray import ndarray +from bifrost.udp_socket import UDPSocket + +from io import IOBase + +from typing import Optional + +from bifrost import telemetry +telemetry.track_module() + +class HeaderInfo(BifrostObject): + def __init__(self): + BifrostObject.__init__( + self, _bf.bfHeaderInfoCreate, _bf.bfHeaderInfoDestroy) + def set_nsrc(self, nsrc: int): + _check(_bf.bfHeaderInfoSetNSrc(self.obj, nsrc)) + def set_nchan(self, nchan: int): + _check(_bf.bfHeaderInfoSetNChan(self.obj, nchan)) + def set_chan0(self, chan0: int): + _check(_bf.bfHeaderInfoSetChan0(self.obj, chan0)) + def set_tuning(self, tuning: int): + _check(_bf.bfHeaderInfoSetTuning(self.obj, tuning)) + def set_gain(self, gain: int): + _check(_bf.bfHeaderInfoSetGain(self.obj, gain)) + def set_decimation(self, decimation: int): + _check(_bf.bfHeaderInfoSetDecimation(self.obj, decimation)) + +class _WriterBase(BifrostObject): + def __enter__(self): + return self + def __exit__(self, type, value, tb): + pass + def set_rate_limit(self, rate_limit_pps: int): + _check(_bf.bfPacketWriterSetRateLimit(self.obj, rate_limit_pps)) + def reset_counter(self): + _check(_bf.bfPacketWriterResetCounter(self.obj)) + def send(self, headerinfo: HeaderInfo, + seq: int, seq_increment: int, src: int, src_increment: int, + idata: ndarray): + _check(_bf.bfPacketWriterSend(self.obj, + headerinfo.obj, + seq, + seq_increment, + src, + src_increment, + asarray(idata).as_BFarray())) + +class UDPTransmit(_WriterBase): + def __init__(self, fmt: str, sock: UDPSocket, core: Optional[int]=None): + if core is None: + core = -1 + BifrostObject.__init__( + self, _bf.bfUdpTransmitCreate, _bf.bfPacketWriterDestroy, + fmt.encode(), sock.fileno(), core) + + +class UDPVerbsTransmit(_WriterBase): + def __init__(self, fmt: str, sock: UDPSocket, core: Optional[int]=None): + if core is None: + core = -1 + BifrostObject.__init__( + self, _bf.bfUdpVerbsTransmitCreate, _bf.bfPacketWriterDestroy, + fmt.encode(), sock.fileno(), core) + + +class DiskWriter(_WriterBase): + def __init__(self, fmt: str, fh: IOBase, core: Optional[int]=None): + if core is None: + core = -1 + BifrostObject.__init__( + self, _bf.bfDiskWriterCreate, _bf.bfPacketWriterDestroy, + fmt.encode(), fh.fileno(), core) diff --git a/python/bifrost/rdma.py b/python/bifrost/rdma.py new file mode 100644 index 000000000..aabc99274 --- /dev/null +++ b/python/bifrost/rdma.py @@ -0,0 +1,203 @@ + +# Copyright (c) 2022-2024, The Bifrost Authors. All rights reserved. +# Copyright (c) 2022-2024, The University of New Mexico. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of The Bifrost Authors nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +from bifrost.libbifrost import _bf, _th, _check, _get, BifrostObject +from bifrost.ndarray import ndarray, asarray +from bifrost.proclog import ProcLog +import bifrost.affinity as cpu_affinity +from bifrost.udp_socket import UDPSocket +from bifrost.ring import Ring +from bifrost.ring2 import Ring as Ring2 + +import time +import ctypes + +from typing import Optional, Tuple, Union + +from bifrost import telemetry +telemetry.track_module() + +class RdmaSender(BifrostObject): + def __init__(self, sock: UDPSocket, message_size: int): + BifrostObject.__init__( + self, _bf.bfRdmaCreate, _bf.bfRdmaDestroy, + sock.fileno(), message_size, 1) + def send_header(self, time_tag: int, header: str, offset_from_head: int): + header = header.encode() + header_buf = ctypes.create_string_buffer(header) + _check(_bf.bfRdmaSendHeader(self.obj, + time_tag, + len(header), + ctypes.cast(header_buf, ctypes.c_void_p), + offset_from_head)) + def send_span(self, span_data: ndarray): + _check(_bf.bfRdmaSendSpan(self.obj, + asarray(span_data).as_BFarray())) + +class RdmaReceiver(BifrostObject): + def __init__(self, sock: UDPSocket, message_size: int, buffer_factor: int=5): + BifrostObject.__init__( + self, _bf.bfRdmaCreate, _bf.bfRdmaDestroy, + sock.fileno(), message_size, 0) + self.message_size = message_size + self.buffer_factor = buffer_factor + + self.time_tag = ctypes.c_ulong(0) + self.header_size = ctypes.c_ulong(0) + self.offset_from_head = ctypes.c_ulong(0) + self.span_size = ctypes.c_ulong(0) + self.contents_bufs = [] + for i in range(self.buffer_factor): + contents_buf = ctypes.create_string_buffer(self.message_size) + self.contents_bufs.append(contents_buf) + self.index = 0 + def receive(self) -> Union[Tuple[int,int,str],ndarray]: + contents_buf = self.contents_bufs[self.index] + self.index += 1 + if self.index == self.buffer_factor: + self.index = 0 + + _check(_bf.bfRdmaReceive(self.obj, + ctypes.POINTER(ctypes.c_ulong)(self.time_tag), + ctypes.POINTER(ctypes.c_ulong)(self.header_size), + ctypes.POINTER(ctypes.c_ulong)(self.offset_from_head), + ctypes.POINTER(ctypes.c_ulong)(self.span_size), + ctypes.addressof(contents_buf))) + if self.header_size.value > 0: + contents = ctypes.cast(contents_buf, ctypes.c_char_p) + contents = contents.value + return self.time_tag.value, self.header_size.value, contents + else: + span_data = ndarray(shape=(self.span_size.value,), dtype='u8', buffer=ctypes.addressof(contents_buf)) + return span_data + +class RingSender(object): + def __init__(self, iring: Union[Ring,Ring2], sock: UDPSocket, gulp_size: int, + guarantee: bool=True, core: Optional[int]=None): + self.iring = iring + self.sock = sock + self.gulp_size = gulp_size + self.guarantee = guarantee + self.core = core + + self.bind_proclog = ProcLog(type(self).__name__+"/bind") + self.in_proclog = ProcLog(type(self).__name__+"/in") + self.sequence_proclog = ProcLog(type(self).__name__+"/sequence0") + self.perf_proclog = ProcLog(type(self).__name__+"/perf") + + self.in_proclog.update( {'nring':1, 'ring0':self.iring.name}) + + def main(self): + if self.core is not None: + cpu_affinity.set_core(self.core) + self.bind_proclog.update({'ncore': 1, + 'core0': cpu_affinity.get_core(),}) + + sender = RdmaSender(self.sock, self.gulp_size) + + for iseq in self.iring.read(guarantee=self.guarantee): + ihdr = json.loads(iseq.header.tostring()) + sender.send_header(iseq.time_tag, len(ihdr), ihdr, 0) + self.sequence_proclog.update(ihdr) + + prev_time = time.time() + iseq_spans = iseq.read(self.gulp_size) + for ispan in iseq_spans: + if ispan.size < self.gulp_size: + continue + curr_time = time.time() + acquire_time = curr_time - prev_time + prev_time = curr_time + + sender.send_span(ispan) + + curr_time = time.time() + process_time = curr_time - prev_time + prev_time = curr_time + self.perf_proclog.update({'acquire_time': acquire_time, + 'reserve_time': -1, + 'process_time': process_time,}) + + +class RingReceiver(object): + def __init__(self, oring: Union[Ring,Ring2], sock: UDPSocket, gulp_size: int, + guarantee: bool=True, core: Optional[int]=None): + self.oring = oring + self.sock = sock + self.gulp_size = gulp_size + self.guarantee = guarantee + self.core = core + + self.bind_proclog = ProcLog(type(self).__name__+"/bind") + self.out_proclog = ProcLog(type(self).__name__+"/out") + self.sequence_proclog = ProcLog(type(self).__name__+"/sequence0") + self.perf_proclog = ProcLog(type(self).__name__+"/perf") + + self.out_proclog.update( {'nring':1, 'ring0':self.oring.name}) + + def main(self): + if self.core is not None: + cpu_affinity.set_core(self.core) + self.bind_proclog.update({'ncore': 1, + 'core0': cpu_affinity.get_core(),}) + + receiver = RdmaReceiver(self.sock, self.gulp_size) + + with self.oring.begin_writing() as oring: + prev_time = time.time() + data = receiver.receive() + while True: + while not isinstance(data, tuple): + data = receiver.receive() + time_tag, _, ihdr = data + self.sequence_proclog.update(ihdr) + + ohdr = ihdr.copy() + + with oring.begin_sequence(time_tag=time_tag, header=ohdr) as oseq: + while True: + data = receiver.receive() + if isinstance(data, tuple): + break + curr_time = time.time() + acquire_time = curr_time - prev_time + prev_time = curr_time + + with oseq.reserve(self.gulp_size) as ospan: + curr_time = time.time() + reserve_time = curr_time - prev_time + prev_time = curr_time + + odata[...] = data + + curr_time = time.time() + process_time = curr_time - prev_time + prev_time = curr_time + self.perf_proclog.update({'acquire_time': acquire_time, + 'reserve_time': reserve_time, + 'process_time': process_time,}) diff --git a/python/bifrost/udp_capture.py b/python/bifrost/udp_capture.py deleted file mode 100644 index 7349b0330..000000000 --- a/python/bifrost/udp_capture.py +++ /dev/null @@ -1,56 +0,0 @@ - -# Copyright (c) 2016-2021, The Bifrost Authors. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of The Bifrost Authors nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# **TODO: Write tests for this class - -from bifrost.libbifrost import _bf, _check, BifrostObject - -from bifrost import telemetry -telemetry.track_module() - -class UDPCapture(BifrostObject): - def __init__(self, fmt, sock, ring, nsrc, src0, max_payload_size, - buffer_ntime, slot_ntime, sequence_callback, core=None): - if core is None: - core = -1 - BifrostObject.__init__( - self, _bf.bfUdpCaptureCreate, _bf.bfUdpCaptureDestroy, - fmt, sock.fileno(), ring.obj, nsrc, src0, - max_payload_size, buffer_ntime, slot_ntime, - sequence_callback, core) - def __enter__(self): - return self - def __exit__(self, type, value, tb): - self.end() - def recv(self): - status = _bf.BFudpcapture_status() - _check(_bf.bfUdpCaptureRecv(self.obj, status)) - return status - def flush(self): - _check(_bf.bfUdpCaptureFlush(self.obj)) - def end(self): - _check(_bf.bfUdpCaptureEnd(self.obj)) diff --git a/python/bifrost/udp_socket.py b/python/bifrost/udp_socket.py index 161e02d13..fede056e9 100644 --- a/python/bifrost/udp_socket.py +++ b/python/bifrost/udp_socket.py @@ -37,6 +37,8 @@ def __init__(self): BifrostObject.__init__(self, _bf.bfUdpSocketCreate, _bf.bfUdpSocketDestroy) def bind(self, local_addr): _check( _bf.bfUdpSocketBind(self.obj, local_addr.obj) ) + def sniff(self, local_addr): + _check( _bf.bfUdpSocketSniff(self.obj, local_addr.obj) ) def connect(self, remote_addr): _check( _bf.bfUdpSocketConnect(self.obj, remote_addr.obj) ) def shutdown(self): @@ -54,3 +56,9 @@ def timeout(self): @timeout.setter def timeout(self, secs): _check( _bf.bfUdpSocketSetTimeout(self.obj, secs) ) + @property + def promisc(self): + return _get(_bf.bfUdpSocketGetPromiscuous, self.obj) + @promisc.setter + def promisc(self, state): + _check( _bf.bfUdpSocketSetPromiscuous(self.obj, state) ) diff --git a/python/bifrost/udp_transmit.py b/python/bifrost/udp_transmit.py deleted file mode 100644 index f4f183c6f..000000000 --- a/python/bifrost/udp_transmit.py +++ /dev/null @@ -1,64 +0,0 @@ - -# Copyright (c) 2017-2023, The Bifrost Authors. All rights reserved. -# Copyright (c) 2017-2023, The University of New Mexico. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of The Bifrost Authors nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# **TODO: Write tests for this class - -from bifrost.libbifrost import _bf, _check, BifrostObject - -import ctypes - -from bifrost import telemetry -telemetry.track_module() - -def _packet2pointer(packet): - buf = ctypes.c_char_p(packet) - siz = ctypes.c_uint( len(packet) ) - return buf, siz - -def _packets2pointer(packets): - count = ctypes.c_uint( len(packets) ) - buf = ctypes.c_char_p("".join(packets)) - siz = ctypes.c_uint( len(packets[0]) ) - return buf, siz, count - -class UDPTransmit(BifrostObject): - def __init__(self, sock, core=-1): - BifrostObject.__init__( - self, _bf.bfUdpTransmitCreate, _bf.bfUdpTransmitDestroy, - sock.fileno(), core) - def __enter__(self): - return self - def __exit__(self, type, value, tb): - pass - def send(self, packet): - ptr, siz = _packet2pointer(packet) - _check(_bf.bfUdpTransmitSend(self.obj, ptr, siz)) - def sendmany(self, packets): - assert(type(packets) is list) - ptr, siz, count = _packets2pointer(packets) - _check(_bf.bfUdpTransmitSendMany(self.obj, ptr, siz, count)) diff --git a/python/bifrost/version/__main__.py b/python/bifrost/version/__main__.py index d6b37cac9..0976cc745 100644 --- a/python/bifrost/version/__main__.py +++ b/python/bifrost/version/__main__.py @@ -44,9 +44,10 @@ def _yes_no(value): print("\nConfiguration:") print(" Memory alignment: %i B" % BF_ALIGNMENT) print(" OpenMP support: %s" % _yes_no(BF_OPENMP_ENABLED)) - print(" NUMA support %s" % _yes_no(BF_NUMA_ENABLED)) print(" Hardware locality support: %s" % _yes_no(BF_HWLOC_ENABLED)) print(" Mellanox messaging accelerator (VMA) support: %s" % _yes_no(BF_VMA_ENABLED)) + print(" Infiniband verbs support: %s" % _yes_no(BF_VERBS_ENABLED)) + print(" RDMA ring transport support: %s" % _yes_no(BF_RDMA_ENABLED)) print(" Logging directory: %s" % BF_PROCLOG_DIR) print(" Debugging: %s" % _yes_no(BF_DEBUG_ENABLED)) print(" CUDA support: %s" % _yes_no(BF_CUDA_ENABLED)) diff --git a/python/typehinting.py b/python/typehinting.py index ae06c0b05..915515732 100644 --- a/python/typehinting.py +++ b/python/typehinting.py @@ -6,7 +6,8 @@ def build_typehinting(filename): 'space': {}, 'dtype': {}, 'capture': {}, - 'transmit': {}, + 'io': {}, + 'whence': {}, 'reduce': {}} with open(filename, 'r') as fh: @@ -23,6 +24,9 @@ def build_typehinting(filename): if tag == 'space': name = name.replace('BF_SPACE_', '') enums[tag][name.lower()] = value + elif tag == 'io': + name = name.replace('BF_IO_', '') + enums[tag][name.lower()] = value elif tag == 'reduce': name = name.replace('BF_REDUCE_', '') name = name.replace('POWER_', 'pwr') diff --git a/src/Makefile.in b/src/Makefile.in index df7edc97d..ca576c6fa 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -15,6 +15,7 @@ PYBUILDFLAGS ?= @PYBUILDFLAGS@ PYINSTALLFLAGS ?= @PYINSTALLFLAGS@ HAVE_RECVMSG ?= @HAVE_RECVMSG@ +HAVE_RDMA ?= @HAVE_RDMA@ HAVE_CUDA ?= @HAVE_CUDA@ @@ -25,10 +26,13 @@ CUDA_LIBDIR ?= $(CUDA_HOME)/lib CUDA_LIBDIR64 ?= $(CUDA_HOME)/lib64 CUDA_INCDIR ?= $(CUDA_HOME)/include +NVCC_GENCODE ?= @NVCC_GENCODE@ + LIBBIFROST_OBJS = \ common.o \ memory.o \ affinity.o \ + hw_locality.o \ cuda.o \ fileutils.o \ testsuite.o \ @@ -44,8 +48,12 @@ ifeq ($(HAVE_RECVMSG),1) LIBBIFROST_OBJS += \ address.o \ udp_socket.o \ - udp_capture.o \ - udp_transmit.o + packet_capture.o \ + packet_writer.o +endif +ifeq ($(HAVE_RDMA),1) + LIBBIFROST_OBJS += \ + rdma.o endif ifeq ($(HAVE_CUDA),1) # These files require the CUDA Toolkit to compile @@ -76,28 +84,12 @@ JIT_SOURCES ?= \ MAKEFILES = ../config.mk Makefile -ifeq ($(HAVE_CUDA),1) -# All CUDA archs supported by this version of nvcc -GPU_ARCHS_SUPPORTED := $(shell $(NVCC) -h | grep -Po "compute_[0-9]{2}" | cut -d_ -f2 | sort | uniq) -# Intersection of user-specified archs and supported archs -GPU_ARCHS_VALID := $(shell echo $(GPU_ARCHS) $(GPU_ARCHS_SUPPORTED) | xargs -n1 | sort | uniq -d | xargs) -# Latest valid arch -GPU_ARCH_LATEST := $(shell echo $(GPU_ARCHS_VALID) | rev | cut -d' ' -f1 | rev) - -# This creates SASS for all valid requested archs, and PTX for the latest one -NVCC_GENCODE ?= $(foreach arch, $(GPU_ARCHS_VALID), \ - -gencode arch=compute_$(arch),\"code=sm_$(arch)\") \ - -gencode arch=compute_$(GPU_ARCH_LATEST),\"code=compute_$(GPU_ARCH_LATEST)\" -endif - -NVCCFLAGS += $(NVCC_GENCODE) - #NVCCFLAGS += -Xcudafe "--diag_suppress=unrecognized_gcc_pragma" #NVCCFLAGS += --expt-relaxed-constexpr LIB_DIR = ../lib INC_DIR = . -CPPFLAGS += -I. -I$(INC_DIR) -I$(CUDA_INCDIR) +CPPFLAGS += -I. -I$(INC_DIR) LIBBIFROST_VERSION_FILE = $(LIBBIFROST_NAME).version LIBBIFROST_SO_STEM = $(LIB_DIR)/$(LIBBIFROST_NAME)$(SO_EXT) diff --git a/src/Socket.cpp b/src/Socket.cpp index 4b417e854..cec457c24 100644 --- a/src/Socket.cpp +++ b/src/Socket.cpp @@ -246,7 +246,7 @@ int Socket::discover_mtu(sockaddr_storage const& remote_address) { #endif } -void Socket::bind(sockaddr_storage const& local_address, +void Socket::bind(sockaddr_storage& local_address, int max_conn_queue) { if( _mode != Socket::MODE_CLOSED ) { throw Socket::Error("Socket is already open"); @@ -265,16 +265,106 @@ void Socket::bind(sockaddr_storage const& local_address, #warning "Kernel version does not support SO_REUSEPORT; multithreaded send/recv will not be possible" #endif - check_error(::bind(_fd, (struct sockaddr*)&local_address, sizeof(local_address)), - "bind socket"); - if( _type == SOCK_STREAM ) { - check_error(::listen(_fd, max_conn_queue), - "set socket to listen"); - _mode = Socket::MODE_LISTENING; - } - else { - _mode = Socket::MODE_BOUND; - } + // Determine multicast status... + int multicast = 0; + if( local_address.ss_family == AF_INET ) { + sockaddr_in *sa4 = reinterpret_cast(&local_address); + if( ((sa4->sin_addr.s_addr & 0xFF) >= 224) \ + && ((sa4->sin_addr.s_addr & 0xFF) < 240) ) { + multicast = 1; + } + } + + // ... and work accordingly + if( !multicast ) { + // Normal address + check_error(::bind(_fd, (struct sockaddr*)&local_address, sizeof(struct sockaddr)), + "bind socket"); + if( _type == SOCK_STREAM ) { + check_error(::listen(_fd, max_conn_queue), + "set socket to listen"); + _mode = Socket::MODE_LISTENING; + } + else { + _mode = Socket::MODE_BOUND; + } + } else { + // Multicast address + // Setup the INADDR_ANY socket base to bind to + struct sockaddr_in base_address; + memset(&base_address, 0, sizeof(sockaddr_in)); + base_address.sin_family = reinterpret_cast(&local_address)->sin_family; + base_address.sin_addr.s_addr = htonl(INADDR_ANY); + base_address.sin_port = htons(reinterpret_cast(&local_address)->sin_port); + check_error(::bind(_fd, (struct sockaddr*)&local_address, sizeof(struct sockaddr)), + "bind socket"); + + if( _type == SOCK_STREAM ) { + throw Socket::Error("SOCK_STREAM is not supported with multicast receive"); + } else { + // Deal with joining the multicast group + struct ip_mreq mreq; + memset(&mreq, 0, sizeof(ip_mreq)); + mreq.imr_multiaddr.s_addr = reinterpret_cast(&local_address)->sin_addr.s_addr; + mreq.imr_interface.s_addr = htonl(INADDR_ANY); + this->set_option(IP_ADD_MEMBERSHIP, mreq, IPPROTO_IP); + _mode = Socket::MODE_BOUND; + } + } +} + +void Socket::sniff(sockaddr_storage const& local_address, + int max_conn_queue) { + if( _mode != Socket::MODE_CLOSED ) { + throw Socket::Error("Socket is already open"); + } +#if defined __linux__ && __linux__ + this->open(AF_PACKET, htons(ETH_P_IP)); + + // Allow binding multiple sockets to one port + // See here for more info: https://lwn.net/Articles/542629/ + // TODO: This must be done before calling ::bind, which is slightly + // awkward with how this method is set up, as the user has + // no way to do it themselves. However, doing it by default + // is probably not a bad idea anyway. +#ifdef SO_REUSEPORT + this->set_option(SO_REUSEPORT, 1); +#else + #warning "Kernel version does not support SO_REUSEPORT; multithreaded send/recv will not be possible" +#endif + struct ifreq ethreq; + memset(ðreq, 0, sizeof(ethreq)); + + // Find out which interface this address corresponds to + this->interface_from_addr((struct sockaddr*)(&local_address), + ethreq.ifr_name, + local_address.ss_family); + check_error(::ioctl(_fd, SIOCGIFINDEX, ðreq), + "find sniff interface index"); + + // Bind to that interface + sockaddr_storage sas; + memset(&sas, 0, sizeof(sas)); + sockaddr_ll* sll = reinterpret_cast(&sas); + sll->sll_family = AF_PACKET; + sll->sll_ifindex = ethreq.ifr_ifindex; + sll->sll_protocol = htons(ETH_P_IP); + check_error(::bind(_fd, (struct sockaddr*)(&sas), sizeof(sas)), + "bind sniff socket"); + + // Final check to make sure we are golden + if( _type == SOCK_STREAM ) { + throw Socket::Error("SOCK_STREAM is not supported with packet sniffing"); + } else { + _mode = Socket::MODE_BOUND; + } + + // Make the socket promiscuous + this->set_promiscuous(true); +#else + #warning packet sniffing is not supported on this OS + check_error(-1, "unsupported on this OS"); +#endif } // TODO: Add timeout support? Bit of a pain to implement. @@ -522,12 +612,12 @@ size_t Socket::send_packet(void const* header_buf, packet_dest, timeout_secs); } -void Socket::open(sa_family_t family) { - this->close(); - _family = family; - check_error(_fd = ::socket(_family, _type, 0), - "create socket"); - this->set_default_options(); +void Socket::open(sa_family_t family, int protocol) { + this->close(); + _family = family; + check_error(_fd = ::socket(_family, _type, protocol), + "create socket"); + this->set_default_options(); } void Socket::set_default_options() { @@ -578,6 +668,10 @@ sockaddr_storage Socket::get_local_address() /*const*/ { void Socket::close() { if( _fd >= 0 ) { + try { + this->set_promiscuous(false); + } + catch( Socket::Error const& ) {} ::close(_fd); _fd = -1; _family = AF_UNSPEC; @@ -638,6 +732,133 @@ int Socket::addr_from_interface(const char* ifname, return found; } +int Socket::interface_from_addr(sockaddr* address, + char* ifname, + sa_family_t family) { + ifaddrs* ifaddr; + if( ::getifaddrs(&ifaddr) == -1 ) { + return 0; + } + bool found = false; + for( ifaddrs* ifa=ifaddr; ifa!=NULL; ifa=ifa->ifa_next ) { + if( ifa->ifa_name == NULL || + ifa->ifa_addr == NULL ) { + continue; + } + sa_family_t ifa_family = ifa->ifa_addr->sa_family; + if( (family == AF_UNSPEC && (ifa_family == AF_INET \ + || ifa_family == AF_INET6) \ + ) || ifa_family == family ) { + switch(ifa_family) { + case AF_INET: { + struct sockaddr_in* inaddr = (struct sockaddr_in*) ifa->ifa_addr; + struct sockaddr_in* inmask = (struct sockaddr_in*) ifa->ifa_netmask; + sockaddr_in* sa4 = reinterpret_cast(address); + + if( (inaddr->sin_addr.s_addr & inmask->sin_addr.s_addr) \ + == (sa4->sin_addr.s_addr & inmask->sin_addr.s_addr) ) { + found = true; + } + break; + } + case AF_INET6: { + struct sockaddr_in6* inaddr6 = (struct sockaddr_in6*) ifa->ifa_addr; + struct sockaddr_in6* inmask6 = (struct sockaddr_in6*) ifa->ifa_netmask; + sockaddr_in6* sa6 = reinterpret_cast(address); + + uint64_t lowerIA=0, upperIA=0, lowerIM=0, upperIM=0; + uint64_t lowerAA=0, upperAA=0; + for(int i=0; i<8; i++) { + lowerIA |= inaddr6->sin6_addr.s6_addr[i] << (8*i); + upperIA |= inaddr6->sin6_addr.s6_addr[8+i] << (8*i); + lowerIM |= inmask6->sin6_addr.s6_addr[i] << (8*i); + upperIM |= inmask6->sin6_addr.s6_addr[8+i] << (8*i); + + lowerAA |= sa6->sin6_addr.s6_addr[i] << (8*i); + upperAA |= sa6->sin6_addr.s6_addr[8+i] << (8*i); + } + + if( ((lowerIA & lowerIM) == (lowerAA & lowerIM)) \ + && ((upperIA & upperIM) == (upperAA & upperIM)) ) { + found = true; + } + break; + } + default: break; + } + if( found ) { + std::strcpy(ifname, ifa->ifa_name); + break; // Return first match + } + } + } + ::freeifaddrs(ifaddr); + return found; +} + +void Socket::set_promiscuous(int state) { +#if defined __linux__ && __linux__ + sockaddr_storage addr = this->get_local_address(); + + struct ifreq ethreq; + memset(ðreq, 0, sizeof(ethreq)); + if( ((sockaddr*)(&addr))->sa_family == AF_PACKET ) { + sockaddr_ll* sll = reinterpret_cast(&addr); + ethreq.ifr_ifindex = sll->sll_ifindex; + check_error(::ioctl(_fd, SIOCGIFNAME, ðreq), + "find interface name"); + } else { + this->interface_from_addr((sockaddr*)&addr, ethreq.ifr_name, _family); + } + + check_error(ioctl(_fd, SIOCGIFFLAGS, ðreq), + "read interface setup"); + if( state && (ethreq.ifr_flags & IFF_PROMISC) ) { + // Oh good, it's already done + } else if ( state && !(ethreq.ifr_flags & IFF_PROMISC) ) { + ethreq.ifr_flags |= IFF_PROMISC; + check_error(ioctl(_fd, SIOCSIFFLAGS, ðreq), + "change interface setup"); + } else if ( !state && (ethreq.ifr_flags & IFF_PROMISC) ){ + ethreq.ifr_flags ^= IFF_PROMISC; + check_error(ioctl(_fd, SIOCSIFFLAGS, ðreq), + "change interface setup"); + } +#else + #warning promiscuous network interfaces are not supported on this OS + check_error(-1, "unsupported on this OS"); +#endif +} + +int Socket::get_promiscuous() { +#if defined __linux__ && __linux__ + sockaddr_storage addr = this->get_local_address(); + + struct ifreq ethreq; + memset(ðreq, 0, sizeof(ethreq)); + if( ((sockaddr*)(&addr))->sa_family == AF_PACKET ) { + sockaddr_ll* sll = reinterpret_cast(&addr); + ethreq.ifr_ifindex = sll->sll_ifindex; + check_error(::ioctl(_fd, SIOCGIFNAME, ðreq), + "find interface name"); + } else { + this->interface_from_addr((sockaddr*)&addr, ethreq.ifr_name, _family); + } + + check_error(ioctl(_fd, SIOCGIFFLAGS, ðreq), + "read interface setup"); + if( ethreq.ifr_flags & IFF_PROMISC ) { + return true; + } else { + return false; + } +#else + #warning promiscuous network interfaces are not supported on this OS + check_error(-1, "unsupported on this OS"); + return false; +#endif +} + void Socket::replace(Socket& s) { _fd = s._fd; s._fd = -1; _type = std::move(s._type); diff --git a/src/Socket.hpp b/src/Socket.hpp index 11c2328dd..9f08051c8 100644 --- a/src/Socket.hpp +++ b/src/Socket.hpp @@ -125,9 +125,19 @@ client.send/recv_block/packet(...); #include #include +#if defined __linux__ && __linux__ + +//#include +#include +#include + +#endif + + #if defined __APPLE__ && __APPLE__ #include +#include #include #define SOCK_NONBLOCK O_NONBLOCK @@ -208,7 +218,9 @@ class Socket { static int discover_mtu(sockaddr_storage const& remote_address); // Server initialisation - void bind(sockaddr_storage const& local_address, + void bind(sockaddr_storage& local_address, + int max_conn_queue=DEFAULT_MAX_CONN_QUEUE); + void sniff(sockaddr_storage const& local_address, int max_conn_queue=DEFAULT_MAX_CONN_QUEUE); // Client initialisation void connect(sockaddr_storage const& remote_address); @@ -297,8 +309,11 @@ class Socket { timeval timeout = self->get_option(SO_RCVTIMEO); return timeout.tv_sec + timeout.tv_usec*1e-6; } + void set_promiscuous(int state); + int get_promiscuous(); + private: - void open(sa_family_t family); + void open(sa_family_t family, int protocol=0); void set_default_options(); void check_error(int retval, std::string what) { if( retval < 0 ) { @@ -326,6 +341,10 @@ class Socket { static int addr_from_interface(const char* ifname, sockaddr* address, sa_family_t family=AF_UNSPEC); + static int interface_from_addr(sockaddr* address, + char* ifname, + sa_family_t family=AF_UNSPEC); + int _fd; int _type; sa_family_t _family; diff --git a/src/address.cpp b/src/address.cpp index 7026ecdc1..52366623d 100644 --- a/src/address.cpp +++ b/src/address.cpp @@ -61,6 +61,24 @@ BFstatus bfAddressGetPort(BFaddress addr, int* port) { BF_TRY_RETURN_ELSE(*port = ntohs(((sockaddr_in*)addr)->sin_port), *port = 0); } +BFstatus bfAddressIsMulticast(BFaddress addr, int* multicast) { + BF_ASSERT(addr, BF_STATUS_INVALID_HANDLE); + BF_ASSERT(multicast, BF_STATUS_INVALID_POINTER); + + unsigned family; + bfAddressGetFamily(addr, &family); + + if( family == AF_INET ) { + sockaddr_in *sa4 = reinterpret_cast(addr); + if( ((sa4->sin_addr.s_addr & 0xFF) >= 224) \ + && ((sa4->sin_addr.s_addr & 0xFF) < 240) ) { + *multicast = 1; + } + } else { + *multicast = 0; + } + return BF_STATUS_SUCCESS; +} BFstatus bfAddressGetMTU(BFaddress addr, int* mtu) { BF_ASSERT(addr, BF_STATUS_INVALID_HANDLE); BF_ASSERT(mtu, BF_STATUS_INVALID_POINTER); diff --git a/src/bifrost/address.h b/src/bifrost/address.h index 83af21df8..a078f4ece 100644 --- a/src/bifrost/address.h +++ b/src/bifrost/address.h @@ -44,6 +44,7 @@ BFstatus bfAddressCreate(BFaddress* addr, BFstatus bfAddressDestroy(BFaddress addr); BFstatus bfAddressGetFamily(BFaddress addr, unsigned* family); BFstatus bfAddressGetPort(BFaddress addr, int* port); +BFstatus bfAddressIsMulticast(BFaddress addr, int* multicast); BFstatus bfAddressGetMTU(BFaddress addr, int* mtu); BFstatus bfAddressGetString(BFaddress addr, BFsize bufsize, // 128 should always be enough diff --git a/src/bifrost/config.h.in b/src/bifrost/config.h.in index 5e58f1c95..5a97d7f6f 100644 --- a/src/bifrost/config.h.in +++ b/src/bifrost/config.h.in @@ -55,11 +55,19 @@ extern "C" { #define BF_MAP_KERNEL_DISK_CACHE_VERSION (1000*@PACKAGE_VERSION_MAJOR@ + 10*@PACKAGE_VERSION_MINOR@) // Features +#define BF_SSE_ENABLED @HAVE_SSE@ +#define BF_AVX_ENABLED @HAVE_AVX@ +#define BF_AVX512_ENABLED @HAVE_AVX512@ #define BF_FLOAT128_ENABLED @HAVE_FLOAT128@ #define BF_OPENMP_ENABLED @HAVE_OPENMP@ -#define BF_NUMA_ENABLED @HAVE_NUMA@ #define BF_HWLOC_ENABLED @HAVE_HWLOC@ #define BF_VMA_ENABLED @HAVE_VMA@ +#define BF_VERBS_ENABLED @HAVE_VERBS@ +#define BF_VERBS_NPKTBUF @VERBS_NPKTBUF@ +#define BF_VERBS_SEND_NPKTBUF @VERBS_SEND_NPKTBUF@ +#define BF_VERBS_SEND_PACING @VERBS_SEND_PACING@ +#define BF_RDMA_ENABLED @HAVE_RDMA@ +#define BF_RDMA_MAXMEM @RDMA_MAXMEM@ // Debugging features #define BF_DEBUG_ENABLED @HAVE_DEBUG@ diff --git a/src/bifrost/udp_transmit.h b/src/bifrost/io.h similarity index 65% rename from src/bifrost/udp_transmit.h rename to src/bifrost/io.h index e3336ccad..a5ef3616e 100644 --- a/src/bifrost/udp_transmit.h +++ b/src/bifrost/io.h @@ -1,6 +1,5 @@ /* - * Copyright (c) 2017, The Bifrost Authors. All rights reserved. - * Copyright (c) 2017, The University of New Mexico. All rights reserved. + * Copyright (c) 2019, The Bifrost Authors. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -27,30 +26,31 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#ifndef BF_UDP_TRANSMIT_H_INCLUDE_GUARD_ -#define BF_UDP_TRANSMIT_H_INCLUDE_GUARD_ +#ifndef BF_IO_H_INCLUDE_GUARD_ +#define BF_IO_H_INCLUDE_GUARD_ + +#include #ifdef __cplusplus extern "C" { #endif -typedef struct BFudptransmit_impl* BFudptransmit; - -typedef enum BFudptransmit_status_ { - BF_TRANSMIT_CONTINUED, - BF_TRANSMIT_INTERRUPTED, - BF_TRANSMIT_ERROR -} BFudptransmit_status; +typedef enum BFiomethod_ { + BF_IO_GENERIC = 0, + BF_IO_DISK = 1, + BF_IO_UDP = 2, + BF_IO_SNIFFER = 3, + BF_IO_VERBS = 4 +} BFiomethod; -BFstatus bfUdpTransmitCreate(BFudptransmit* obj, - int fd, - int core); -BFstatus bfUdpTransmitDestroy(BFudptransmit obj); -BFstatus bfUdpTransmitSend(BFudptransmit obj, char* packet, unsigned int len); -BFstatus bfUdpTransmitSendMany(BFudptransmit obj, char* packets, unsigned int len, unsigned int npackets); +typedef enum BFiowhence_ { + BF_WHENCE_SET = SEEK_SET, + BF_WHENCE_CUR = SEEK_CUR, + BF_WHENCE_END = SEEK_END +} BFiowhence; #ifdef __cplusplus } // extern "C" #endif -#endif // BF_UDP_TRANSMIT_H_INCLUDE_GUARD_ +#endif // BF_IO_H_INCLUDE_GUARD_ diff --git a/src/bifrost/packet_capture.h b/src/bifrost/packet_capture.h new file mode 100644 index 000000000..c81485d2e --- /dev/null +++ b/src/bifrost/packet_capture.h @@ -0,0 +1,161 @@ +/* + * Copyright (c) 2019-2023, The Bifrost Authors. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of The Bifrost Authors nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef BF_PACKET_CAPTURE_H_INCLUDE_GUARD_ +#define BF_PACKET_CAPTURE_H_INCLUDE_GUARD_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include +#include + +// Callback setup + +typedef int (*BFpacketcapture_simple_sequence_callback)(BFoffset, int, int, int, + BFoffset*, void const**, size_t*); +typedef int (*BFpacketcapture_chips_sequence_callback)(BFoffset, int, int, int, + BFoffset*, void const**, size_t*); +typedef int (*BFpacketcapture_snap2_sequence_callback)(BFoffset, int, int, int, + BFoffset*, void const**, size_t*); +typedef int (*BFpacketcapture_ibeam_sequence_callback)(BFoffset, int, int, int, + BFoffset*, void const**, size_t*); +typedef int (*BFpacketcapture_pbeam_sequence_callback)(BFoffset, BFoffset, int, int, int, + int, void const**, size_t*); +typedef int (*BFpacketcapture_cor_sequence_callback)(BFoffset, BFoffset, int, int, + int, int, void const**, size_t*); +typedef int (*BFpacketcapture_vdif_sequence_callback)(BFoffset, BFoffset, int, int, int, + int, int, int, void const**, size_t*); +typedef int (*BFpacketcapture_tbn_sequence_callback)(BFoffset, BFoffset, int, int, + int, void const**, size_t*); +typedef int (*BFpacketcapture_drx_sequence_callback)(BFoffset, BFoffset, int, int, int, + int, void const**, size_t*); +typedef int (*BFpacketcapture_drx8_sequence_callback)(BFoffset, BFoffset, int, int, int, + int, void const**, size_t*); + +typedef struct BFpacketcapture_callback_impl* BFpacketcapture_callback; + +BFstatus bfPacketCaptureCallbackCreate(BFpacketcapture_callback* obj); +BFstatus bfPacketCaptureCallbackDestroy(BFpacketcapture_callback obj); +BFstatus bfPacketCaptureCallbackSetSIMPLE(BFpacketcapture_callback obj, + BFpacketcapture_simple_sequence_callback callback); +BFstatus bfPacketCaptureCallbackSetCHIPS(BFpacketcapture_callback obj, + BFpacketcapture_chips_sequence_callback callback); +BFstatus bfPacketCaptureCallbackSetSNAP2(BFpacketcapture_callback obj, + BFpacketcapture_snap2_sequence_callback callback); +BFstatus bfPacketCaptureCallbackSetIBeam(BFpacketcapture_callback obj, + BFpacketcapture_ibeam_sequence_callback callback); +BFstatus bfPacketCaptureCallbackSetPBeam(BFpacketcapture_callback obj, + BFpacketcapture_pbeam_sequence_callback callback); +BFstatus bfPacketCaptureCallbackSetCOR(BFpacketcapture_callback obj, + BFpacketcapture_cor_sequence_callback callback); +BFstatus bfPacketCaptureCallbackSetVDIF(BFpacketcapture_callback obj, + BFpacketcapture_vdif_sequence_callback callback); +BFstatus bfPacketCaptureCallbackSetTBN(BFpacketcapture_callback obj, + BFpacketcapture_tbn_sequence_callback callback); +BFstatus bfPacketCaptureCallbackSetDRX(BFpacketcapture_callback obj, + BFpacketcapture_drx_sequence_callback callback); +BFstatus bfPacketCaptureCallbackSetDRX8(BFpacketcapture_callback obj, + BFpacketcapture_drx8_sequence_callback callback); + +// Capture setup + +typedef struct BFpacketcapture_impl* BFpacketcapture; + +typedef enum BFpacketcapture_status_ { + BF_CAPTURE_STARTED, + BF_CAPTURE_ENDED, + BF_CAPTURE_CONTINUED, + BF_CAPTURE_CHANGED, + BF_CAPTURE_NO_DATA, + BF_CAPTURE_INTERRUPTED, + BF_CAPTURE_ERROR +} BFpacketcapture_status; + +BFstatus bfDiskReaderCreate(BFpacketcapture* obj, + const char* format, + int fd, + BFring ring, + BFsize nsrc, + BFsize src0, + BFsize buffer_ntime, + BFsize slot_ntime, + BFpacketcapture_callback sequence_callback, + int core); +BFstatus bfUdpCaptureCreate(BFpacketcapture* obj, + const char* format, + int fd, + BFring ring, + BFsize nsrc, + BFsize src0, + BFsize max_payload_size, + BFsize buffer_ntime, + BFsize slot_ntime, + BFpacketcapture_callback sequence_callback, + int core); +BFstatus bfUdpSnifferCreate(BFpacketcapture* obj, + const char* format, + int fd, + BFring ring, + BFsize nsrc, + BFsize src0, + BFsize max_payload_size, + BFsize buffer_ntime, + BFsize slot_ntime, + BFpacketcapture_callback sequence_callback, + int core); +BFstatus bfUdpVerbsCaptureCreate(BFpacketcapture* obj, + const char* format, + int fd, + BFring ring, + BFsize nsrc, + BFsize src0, + BFsize max_payload_size, + BFsize buffer_ntime, + BFsize slot_ntime, + BFpacketcapture_callback sequence_callback, + int core); +BFstatus bfPacketCaptureDestroy(BFpacketcapture obj); +BFstatus bfPacketCaptureRecv(BFpacketcapture obj, + BFpacketcapture_status* result); +BFstatus bfPacketCaptureFlush(BFpacketcapture obj); +BFstatus bfPacketCaptureSeek(BFpacketcapture obj, + BFoffset offset, + BFiowhence whence, + BFoffset* position); +BFstatus bfPacketCaptureTell(BFpacketcapture obj, + BFoffset* position); +BFstatus bfPacketCaptureEnd(BFpacketcapture obj); +// TODO: bfPacketCaptureGetXX + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // BF_PACKET_CAPTURE_H_INCLUDE_GUARD_ diff --git a/src/bifrost/packet_writer.h b/src/bifrost/packet_writer.h new file mode 100644 index 000000000..133a7ac4b --- /dev/null +++ b/src/bifrost/packet_writer.h @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2019-2023, The Bifrost Authors. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of The Bifrost Authors nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#ifndef BF_PACKET_WRITER_H_INCLUDE_GUARD_ +#define BF_PACKET_WRITER_H_INCLUDE_GUARD_ + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +typedef struct BFheaderinfo_impl* BFheaderinfo; + +BFstatus bfHeaderInfoCreate(BFheaderinfo* obj); +BFstatus bfHeaderInfoDestroy(BFheaderinfo obj); +BFstatus bfHeaderInfoSetNSrc(BFheaderinfo obj, + int nsrc); +BFstatus bfHeaderInfoSetNChan(BFheaderinfo obj, + int nchan); +BFstatus bfHeaderInfoSetChan0(BFheaderinfo obj, + int chan0); +BFstatus bfHeaderInfoSetTuning(BFheaderinfo obj, + int tuning); +BFstatus bfHeaderInfoSetGain(BFheaderinfo obj, + unsigned short int gain); +BFstatus bfHeaderInfoSetDecimation(BFheaderinfo obj, + unsigned int decimation); + +typedef struct BFpacketwriter_impl* BFpacketwriter; + +BFstatus bfDiskWriterCreate(BFpacketwriter* obj, + const char* format, + int fd, + int core); +BFstatus bfUdpTransmitCreate(BFpacketwriter* obj, + const char* format, + int fd, + int core); +BFstatus bfUdpVerbsTransmitCreate(BFpacketwriter* obj, + const char* format, + int fd, + int core); +BFstatus bfPacketWriterDestroy(BFpacketwriter obj); +BFstatus bfPacketWriterSetRateLimit(BFpacketwriter obj, + unsigned int rate_limit); +BFstatus bfPacketWriterResetRateLimitCounter(BFpacketwriter obj); +BFstatus bfPacketWriterResetCounter(BFpacketwriter obj); +BFstatus bfPacketWriterSend(BFpacketwriter obj, + BFheaderinfo info, + BFoffset seq, + BFoffset seq_increment, + BFoffset src, + BFoffset src_increment, + BFarray const* in); + +#ifdef __cplusplus +} // extern "C" +#endif + +#endif // BF_PACKET_WRITER_H_INCLUDE_GUARD_ diff --git a/src/bifrost/udp_capture.h b/src/bifrost/rdma.h similarity index 51% rename from src/bifrost/udp_capture.h rename to src/bifrost/rdma.h index 5b8fda741..93ccc9e6c 100644 --- a/src/bifrost/udp_capture.h +++ b/src/bifrost/rdma.h @@ -1,5 +1,6 @@ /* - * Copyright (c) 2016, The Bifrost Authors. All rights reserved. + * Copyright (c) 2022, The Bifrost Authors. All rights reserved. + * Copyright (c) 2022, The University of New Mexico. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -26,50 +27,38 @@ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -#ifndef BF_UDP_CAPTURE_H_INCLUDE_GUARD_ -#define BF_UDP_CAPTURE_H_INCLUDE_GUARD_ +#ifndef BF_RDMA_H_INCLUDE_GUARD_ +#define BF_RDMA_H_INCLUDE_GUARD_ #ifdef __cplusplus extern "C" { #endif -#include -#include +#include -typedef struct BFudpcapture_impl* BFudpcapture; +typedef struct BFrdma_impl* BFrdma; -typedef int (*BFudpcapture_sequence_callback)(BFoffset, int, int, int, - BFoffset*, void const**, size_t*); - -typedef enum BFudpcapture_status_ { - BF_CAPTURE_STARTED, - BF_CAPTURE_ENDED, - BF_CAPTURE_CONTINUED, - BF_CAPTURE_CHANGED, - BF_CAPTURE_NO_DATA, - BF_CAPTURE_INTERRUPTED, - BF_CAPTURE_ERROR -} BFudpcapture_status; - -BFstatus bfUdpCaptureCreate(BFudpcapture* obj, - const char* format, - int fd, - BFring ring, - BFsize nsrc, - BFsize src0, - BFsize max_payload_size, - BFsize buffer_ntime, - BFsize slot_ntime, - BFudpcapture_sequence_callback sequence_callback, - int core); -BFstatus bfUdpCaptureDestroy(BFudpcapture obj); -BFstatus bfUdpCaptureRecv(BFudpcapture obj, BFudpcapture_status* result); -BFstatus bfUdpCaptureFlush(BFudpcapture obj); -BFstatus bfUdpCaptureEnd(BFudpcapture obj); -// TODO: bfUdpCaptureGetXX +BFstatus bfRdmaCreate(BFrdma* obj, + int fd, + size_t message_size, + int is_server); +BFstatus bfRdmaDestroy(BFrdma obj); +BFstatus bfRdmaSendHeader(BFrdma obj, + BFoffset time_tag, + BFsize header_size, + const void* header, + BFoffset offset_from_head); +BFstatus bfRdmaSendSpan(BFrdma obj, + BFarray const* span); +BFstatus bfRdmaReceive(BFrdma obj, + BFoffset* time_tag, + BFsize* header_size, + BFoffset* offset_from_head, + BFsize* span_size, + void* contents); #ifdef __cplusplus } // extern "C" #endif -#endif // BF_UDP_CAPTURE_H_INCLUDE_GUARD_ +#endif // BF_RDMA_H_INCLUDE_GUARD_ diff --git a/src/bifrost/udp_socket.h b/src/bifrost/udp_socket.h index 80ff3d1d2..e773256cc 100644 --- a/src/bifrost/udp_socket.h +++ b/src/bifrost/udp_socket.h @@ -41,10 +41,13 @@ BFstatus bfUdpSocketCreate(BFudpsocket* obj); BFstatus bfUdpSocketDestroy(BFudpsocket obj); BFstatus bfUdpSocketConnect(BFudpsocket obj, BFaddress remote_addr); BFstatus bfUdpSocketBind( BFudpsocket obj, BFaddress local_addr); +BFstatus bfUdpSocketSniff( BFudpsocket obj, BFaddress local_addr); BFstatus bfUdpSocketShutdown(BFudpsocket obj); // Unblocks recv in another thread BFstatus bfUdpSocketClose(BFudpsocket obj); BFstatus bfUdpSocketSetTimeout(BFudpsocket obj, double secs); BFstatus bfUdpSocketGetTimeout(BFudpsocket obj, double* secs); +BFstatus bfUdpSocketSetPromiscuous(BFudpsocket obj, int promisc); +BFstatus bfUdpSocketGetPromiscuous(BFudpsocket obj, int* promisc); BFstatus bfUdpSocketGetMTU(BFudpsocket obj, int* mtu); BFstatus bfUdpSocketGetFD(BFudpsocket obj, int* fd); diff --git a/src/formats/base.hpp b/src/formats/base.hpp new file mode 100644 index 000000000..2cca0a789 --- /dev/null +++ b/src/formats/base.hpp @@ -0,0 +1,155 @@ +/* + * Copyright (c) 2019-2022, The Bifrost Authors. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of The Bifrost Authors nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#include // For posix_memalign +#include // For memcpy, memset +#include + +#include // For ntohs + +#if defined BF_SSE_ENABLED && BF_SSE_ENABLED + +#include + +#endif + +#if defined BF_AVX_ENABLED && BF_AVX_ENABLED + +#include + +#endif + +#if defined BF_AVX512_ENABLED && BF_AVX512_ENABLED + +#include + +#endif + +#if defined __APPLE__ && __APPLE__ + +#include +#define htobe16(x) OSSwapHostToBigInt16(x) +#define be16toh(x) OSSwapBigToHostInt16(x) +#define htobe32(x) OSSwapHostToBigInt32(x) +#define be32toh(x) OSSwapBigToHostInt32(x) +#define htobe64(x) OSSwapHostToBigInt64(x) +#define be64toh(x) OSSwapBigToHostInt64(x) + +#endif + +#define BF_UNPACK_FACTOR 1 + +#define JUMBO_FRAME_SIZE 9000 + +struct unaligned128_type { + uint8_t data[16]; +}; +struct __attribute__((aligned(16))) aligned128_type { + uint8_t data[16]; +}; +struct unaligned256_type { + uint8_t data[32]; +}; +struct __attribute__((aligned(32))) aligned256_type { + uint8_t data[32]; +}; +struct unaligned512_type { + uint8_t data[64]; +}; +struct __attribute__((aligned(64))) aligned512_type { + uint8_t data[64]; +}; + + +struct PacketDesc { + uint64_t seq; + int nsrc; + int src; + int nchan; + int chan0; + int nchan_tot; + int npol; + int pol0; + int npol_tot; + uint32_t sync; + uint64_t time_tag; + int tuning; + int tuning1 = 0; + uint8_t beam; + uint16_t gain; + uint32_t decimation; + uint8_t valid_mode; + int payload_size; + const uint8_t* payload_ptr; +}; + + +class PacketDecoder { +protected: + int _nsrc; + int _src0; +private: + virtual inline bool valid_packet(const PacketDesc* pkt) const { + return false; + } +public: + PacketDecoder(int nsrc, int src0) : _nsrc(nsrc), _src0(src0) {} + virtual inline bool operator()(const uint8_t* pkt_ptr, + int pkt_size, + PacketDesc* pkt) const { + return false; + } +}; + + +class PacketProcessor { +public: + virtual inline void operator()(const PacketDesc* pkt, + uint64_t seq0, + uint64_t nseq_per_obuf, + int nbuf, + uint8_t* obufs[], + size_t ngood_bytes[], + size_t* src_ngood_bytes[]) {} + virtual inline void blank_out_source(uint8_t* data, + int src, + int nsrc, + int nchan, + int nseq) {} +}; + + +class PacketHeaderFiller { +public: + virtual inline int get_size() { return 0; } + virtual inline void operator()(const PacketDesc* hdr_base, + BFoffset framecount, + char* hdr) {} +}; diff --git a/src/formats/chips.hpp b/src/formats/chips.hpp new file mode 100644 index 000000000..fd9a371bb --- /dev/null +++ b/src/formats/chips.hpp @@ -0,0 +1,183 @@ +/* + * Copyright (c) 2019, The Bifrost Authors. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of The Bifrost Authors nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#include "base.hpp" + +struct __attribute__((packed)) chips_hdr_type { + uint8_t roach; // Note: 1-based + uint8_t gbe; // (AKA tuning) + uint8_t nchan; // 109 + uint8_t nsubband; // 11 + uint8_t subband; // 0-11 + uint8_t nroach; // 16 + // Note: Big endian + uint16_t chan0; // First chan in packet + uint64_t seq; // Note: 1-based +}; + +class CHIPSDecoder : virtual public PacketDecoder { + inline bool valid_packet(const PacketDesc* pkt) const { + return (pkt->seq >= 0 && + pkt->src >= 0 && pkt->src < _nsrc && + pkt->chan0 >= 0); + } +public: + CHIPSDecoder(int nsrc, int src0) : PacketDecoder(nsrc, src0) {} + inline bool operator()(const uint8_t* pkt_ptr, + int pkt_size, + PacketDesc* pkt) const { + if( pkt_size < (int)sizeof(chips_hdr_type) ) { + return false; + } + const chips_hdr_type* pkt_hdr = (chips_hdr_type*)pkt_ptr; + const uint8_t* pkt_pld = pkt_ptr + sizeof(chips_hdr_type); + int pld_size = pkt_size - sizeof(chips_hdr_type); + pkt->seq = be64toh(pkt_hdr->seq) - 1; + //pkt->nsrc = pkt_hdr->nroach; + pkt->nsrc = _nsrc; + pkt->src = (pkt_hdr->roach - 1) - _src0; + pkt->nchan = pkt_hdr->nchan; + pkt->chan0 = ntohs(pkt_hdr->chan0); + pkt->payload_size = pld_size; + pkt->payload_ptr = pkt_pld; + return this->valid_packet(pkt); + } +}; + +class CHIPSProcessor : virtual public PacketProcessor { +public: + inline void operator()(const PacketDesc* pkt, + uint64_t seq0, + uint64_t nseq_per_obuf, + int nbuf, + uint8_t* obufs[], + size_t ngood_bytes[], + size_t* src_ngood_bytes[]) { + enum { + PKT_NINPUT = 32, + PKT_NBIT = 4 + }; + int obuf_idx = ((pkt->seq - seq0 >= 1*nseq_per_obuf) + + (pkt->seq - seq0 >= 2*nseq_per_obuf)); + size_t obuf_seq0 = seq0 + obuf_idx*nseq_per_obuf; + size_t nbyte = pkt->payload_size * BF_UNPACK_FACTOR; + ngood_bytes[obuf_idx] += nbyte; + src_ngood_bytes[obuf_idx][pkt->src] += nbyte; + // **CHANGED RECENTLY + int payload_size = pkt->payload_size;//pkt->nchan*(PKT_NINPUT*2*PKT_NBIT/8); + + size_t obuf_offset = (pkt->seq-obuf_seq0)*pkt->nsrc*payload_size; + typedef unaligned256_type itype; + typedef aligned256_type otype; + + obuf_offset *= BF_UNPACK_FACTOR; + + // Note: Using these SSE types allows the compiler to use SSE instructions + // However, they require aligned memory (otherwise segfault) + itype const* __restrict__ in = (itype const*)pkt->payload_ptr; + otype* __restrict__ out = (otype* )&obufs[obuf_idx][obuf_offset]; + + int chan = 0; + //cout << pkt->src << ", " << pkt->nsrc << endl; + //cout << pkt->nchan << endl; + for( ; channchan; ++chan ) { +#if defined BF_AVX_ENABLED && BF_AVX_ENABLED + const unaligned256_type* dsrc = (const unaligned256_type*) &in[chan]; + aligned256_type* ddst = (aligned256_type*) &out[pkt->src + pkt->nsrc*chan]; + + __m256 mtemp = _mm256_loadu_si256(reinterpret_cast(dsrc)); + _mm256_stream_si256(reinterpret_cast<__m256i*>(ddst), mtemp); +#else +#if defined BF_SSE_ENABLED && BF_SSE_ENABLED + const unaligned128_type* dsrc = (const unaligned128_type*) &in[chan]; + aligned128_type* ddst = (aligned128_type*) &out[pkt->src + pkt->nsrc*chan]; + + __m128i mtemp = _mm_loadu_si128(reinterpret_cast(dsrc)); + _mm_stream_si128(reinterpret_cast<__m128i*>(ddst), mtemp); + mtemp = _mm_loadu_si128(reinterpret_cast(dsrc+1)); + _mm_stream_si128(reinterpret_cast<__m128i*>(ddst+1), mtemp); +#else + ::memcpy(&out[pkt->src + pkt->nsrc*chan], + &in[chan], sizeof(otype)); +#endif +#endif + } + } + + inline void blank_out_source(uint8_t* data, + int src, + int nsrc, + int nchan, + int nseq) { + typedef aligned256_type otype; + otype* __restrict__ aligned_data = (otype*)data; + for( int t=0; t(ddst), mtemp); +#else +#if defined BF_SSE_ENABLED && BF_SSE_ENABLED + aligned128_type* ddst = (aligned128_type*) &aligned_data[src + nsrc*(c + nchan*t)]; + + __m128i mtemp = _mm_setzero_si128(); + _mm_stream_si128(reinterpret_cast<__m128i*>(ddst), mtemp); + _mm_stream_si128(reinterpret_cast<__m128i*>(ddst+1), mtemp); +#else + ::memset(&aligned_data[src + nsrc*(c + nchan*t)], + 0, sizeof(otype)); +#endif +#endif + } + } + } +}; + +class CHIPSHeaderFiller : virtual public PacketHeaderFiller { +public: + inline int get_size() { return sizeof(chips_hdr_type); } + inline void operator()(const PacketDesc* hdr_base, + BFoffset framecount, + char* hdr) { + chips_hdr_type* header = reinterpret_cast(hdr); + memset(header, 0, sizeof(chips_hdr_type)); + + header->roach = hdr_base->src + 1; + header->gbe = hdr_base->tuning; + header->nchan = hdr_base->nchan; + header->nsubband = 1; // Should be changable? + header->subband = 0; // Should be changable? + header->nroach = hdr_base->nsrc; + header->chan0 = htons(hdr_base->chan0); + header->seq = htobe64(hdr_base->seq); + } +}; diff --git a/src/formats/cor.hpp b/src/formats/cor.hpp new file mode 100644 index 000000000..bf1660ded --- /dev/null +++ b/src/formats/cor.hpp @@ -0,0 +1,184 @@ +/* + * Copyright (c) 2019-2021, The Bifrost Authors. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of The Bifrost Authors nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#include "base.hpp" +#include + +struct __attribute__((packed)) cor_hdr_type { + uint32_t sync_word; + uint32_t frame_count_word; + uint32_t second_count; + uint16_t first_chan; + uint16_t gain; + uint64_t time_tag; + uint32_t navg; + uint16_t stand0; + uint16_t stand1; +}; + +class CORDecoder : virtual public PacketDecoder { + inline bool valid_packet(const PacketDesc* pkt) const { + return (pkt->sync == 0x5CDEC0DE && + pkt->src >= 0 && + pkt->src < pkt->nsrc && + pkt->time_tag >= 0 && + pkt->chan0 >= 0); + } +public: + CORDecoder(int nsrc, int src0) : PacketDecoder(nsrc, src0) {} + inline bool operator()(const uint8_t* pkt_ptr, + int pkt_size, + PacketDesc* pkt) const { + if( pkt_size < (int)sizeof(cor_hdr_type) ) { + return false; + } + const cor_hdr_type* pkt_hdr = (cor_hdr_type*)pkt_ptr; + const uint8_t* pkt_pld = pkt_ptr + sizeof(cor_hdr_type); + int pld_size = pkt_size - sizeof(cor_hdr_type); + uint8_t nchan_decim = (be32toh(pkt_hdr->frame_count_word) >> 16) & 0xFF; + uint8_t nserver = (be32toh(pkt_hdr->frame_count_word) >> 8) & 0xFF; + uint8_t server = be32toh(pkt_hdr->frame_count_word) & 0xFF; + uint16_t nchan_pkt = (pld_size/(8*4)); + uint16_t stand0 = be16toh(pkt_hdr->stand0) - 1; + uint16_t stand1 = be16toh(pkt_hdr->stand1) - 1; + uint16_t nstand = (sqrt(8*_nsrc/nserver+1)-1)/2; + pkt->sync = pkt_hdr->sync_word; + pkt->time_tag = be64toh(pkt_hdr->time_tag); + pkt->decimation = be32toh(pkt_hdr->navg); + pkt->seq = pkt->time_tag / 196000000 / (pkt->decimation / 100); + pkt->nsrc = _nsrc; + pkt->src = (stand0*(2*(nstand-1)+1-stand0)/2 + stand1 + 1 - _src0)*nserver \ + + (server - 1); + pkt->chan0 = be16toh(pkt_hdr->first_chan) \ + - nchan_decim*nchan_pkt * (server - 1); + pkt->nchan = nchan_pkt; + pkt->tuning = (nserver << 8) | (server - 1); // Stores the number of servers and + // the server that sent this packet + pkt->gain = be16toh(pkt_hdr->gain); + pkt->payload_size = pld_size; + pkt->payload_ptr = pkt_pld; + /* + if( stand0 == 0 || (stand0 == 1 && stand1 < 2) ) { + std::cout << "nsrc: " << pkt->nsrc << std::endl; + std::cout << "stand0: " << stand0 << std::endl; + std::cout << "stand1: " << stand1 << std::endl; + std::cout << "server: " << server << std::endl; + std::cout << "src: " << pkt->src << std::endl; + std::cout << "chan0: " << pkt->chan0 << std::endl; + std::cout << "nchan: " << pld_size/32 << std::endl; + std::cout << "navg: " << pkt->decimation << std::endl; + std::cout << "tuning: " << pkt->tuning << std::endl; + std::cout << "valid: " << this->valid_packet(pkt) << std::endl; + } + */ + return this->valid_packet(pkt); + } +}; + +class CORProcessor : virtual public PacketProcessor { +public: + inline void operator()(const PacketDesc* pkt, + uint64_t seq0, + uint64_t nseq_per_obuf, + int nbuf, + uint8_t* obufs[], + size_t ngood_bytes[], + size_t* src_ngood_bytes[]) { + int obuf_idx = ((pkt->seq - seq0 >= 1*nseq_per_obuf) + + (pkt->seq - seq0 >= 2*nseq_per_obuf)); + size_t obuf_seq0 = seq0 + obuf_idx*nseq_per_obuf; + size_t nbyte = pkt->payload_size; + ngood_bytes[obuf_idx] += nbyte; + src_ngood_bytes[obuf_idx][pkt->src] += nbyte; + int payload_size = pkt->payload_size; + + size_t obuf_offset = (pkt->seq-obuf_seq0)*pkt->nsrc*payload_size; + typedef unaligned256_type itype; // 32+32 * 4 pol. products + typedef aligned256_type otype; + + // Note: Using these SSE types allows the compiler to use SSE instructions + // However, they require aligned memory (otherwise segfault) + itype const* __restrict__ in = (itype const*)pkt->payload_ptr; + otype* __restrict__ out = (otype* )&obufs[obuf_idx][obuf_offset]; + + // Convenience + int bl_server = pkt->src; + int nchan = pkt->nchan; + + int chan = 0; + for( ; chan(hdr); + memset(header, 0, sizeof(cor_hdr_type)); + + // Find the number of antennas needed to reach this number of baselines + int N = (sqrt(8*hdr_base->nsrc+1) - 1) / 2; + + // Find the indices of the two stands that form this baseline + int b = 2 + 2*(N-1)+1; + int stand0 = (b - sqrt(b*b-8*hdr_base->src)) / 2; + int stand1 = hdr_base->src - stand0*(2*(N-1)+1-stand0)/2; + + header->sync_word = 0x5CDEC0DE; + // Bits 9-32 are the server identifier; bits 1-8 are the COR packet flag + header->frame_count_word = htobe32((hdr_base->tuning & 0xFFFFFF) \ + | ((uint32_t) 0x02 << 24)); + header->first_chan = htons(hdr_base->chan0); + header->gain = htons(hdr_base->gain); + header->time_tag = htobe64(hdr_base->seq); + header->navg = htobe32(hdr_base->decimation); + header->stand0 = htons(stand0 + 1); + header->stand1 = htons(stand1 + 1); + } +}; diff --git a/src/formats/drx.hpp b/src/formats/drx.hpp new file mode 100644 index 000000000..55ad4406e --- /dev/null +++ b/src/formats/drx.hpp @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2019-2023, The Bifrost Authors. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of The Bifrost Authors nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#include "base.hpp" + +#define DRX_FRAME_SIZE 4128 + +struct __attribute__((packed)) drx_hdr_type { + uint32_t sync_word; + uint32_t frame_count_word; + uint32_t seconds_count; + uint16_t decimation; + uint16_t time_offset; + uint64_t time_tag; + uint32_t tuning_word; + uint32_t flags; +}; + +class DRXDecoder : virtual public PacketDecoder { + inline bool valid_packet(const PacketDesc* pkt) const { + return (pkt->sync == 0x5CDEC0DE && + pkt->src >= 0 && + pkt->src < _nsrc && + pkt->time_tag >= 0 && + (((_nsrc == 4) && + (pkt->tuning > 0 && + pkt->tuning1 > 0)) || + (_nsrc == 2 && + (pkt->tuning > 0 || + pkt->tuning1 > 0))) && + pkt->valid_mode == 0); + } +public: + DRXDecoder(int nsrc, int src0) : PacketDecoder(nsrc, src0) {} + inline bool operator()(const uint8_t* pkt_ptr, + int pkt_size, + PacketDesc* pkt) const { + if( pkt_size != DRX_FRAME_SIZE ) { + return false; + } + const drx_hdr_type* pkt_hdr = (drx_hdr_type*)pkt_ptr; + const uint8_t* pkt_pld = pkt_ptr + sizeof(drx_hdr_type); + int pld_size = pkt_size - sizeof(drx_hdr_type); + int pkt_id = pkt_hdr->frame_count_word & 0xFF; + pkt->beam = (pkt_id & 0x7) - 1; + int pkt_tune = ((pkt_id >> 3) & 0x7) - 1; + int pkt_pol = ((pkt_id >> 7) & 0x1); + pkt_id = (pkt_tune << 1) | pkt_pol; + pkt->sync = pkt_hdr->sync_word; + pkt->time_tag = be64toh(pkt_hdr->time_tag) - be16toh(pkt_hdr->time_offset); + pkt->seq = pkt->time_tag / be16toh(pkt_hdr->decimation) / 4096; + pkt->nsrc = _nsrc; + pkt->src = pkt_id - _src0; + if( pkt->src / 2 == 0 ) { + pkt->tuning = be32toh(pkt_hdr->tuning_word); + } else { + pkt->tuning1 = be32toh(pkt_hdr->tuning_word); + } + pkt->decimation = be16toh(pkt_hdr->decimation); + pkt->valid_mode = ((pkt_id >> 6) & 0x1); + pkt->payload_size = pld_size; + pkt->payload_ptr = pkt_pld; + return this->valid_packet(pkt); + } +}; + + +class DRXProcessor : virtual public PacketProcessor { +public: + inline void operator()(const PacketDesc* pkt, + uint64_t seq0, + uint64_t nseq_per_obuf, + int nbuf, + uint8_t* obufs[], + size_t ngood_bytes[], + size_t* src_ngood_bytes[]) { + int obuf_idx = ((pkt->seq - seq0 >= 1*nseq_per_obuf) + + (pkt->seq - seq0 >= 2*nseq_per_obuf)); + size_t obuf_seq0 = seq0 + obuf_idx*nseq_per_obuf; + size_t nbyte = pkt->payload_size; + ngood_bytes[obuf_idx] += nbyte; + src_ngood_bytes[obuf_idx][pkt->src] += nbyte; + int payload_size = pkt->payload_size; + + size_t obuf_offset = (pkt->seq-obuf_seq0)*pkt->nsrc*payload_size; + + // Note: Using these SSE types allows the compiler to use SSE instructions + // However, they require aligned memory (otherwise segfault) + uint8_t const* __restrict__ in = (uint8_t const*)pkt->payload_ptr; + uint8_t* __restrict__ out = (uint8_t* )&obufs[obuf_idx][obuf_offset]; + + for( int samp=0; samp<4096; ++samp ) { // HACK TESTING + *(out + pkt->src) = *in; + in++; + out += pkt->nsrc; + } + } + + inline void blank_out_source(uint8_t* data, + int src, + int nsrc, + int nchan, + int nseq) { + uint8_t* __restrict__ aligned_data = (uint8_t*) data; + for( int t=0; t(hdr); + memset(header, 0, sizeof(drx_hdr_type)); + + header->sync_word = 0x5CDEC0DE; + // ID is stored in the lowest 8 bits; bit 2 is reserved + header->frame_count_word = htobe32((uint32_t) (hdr_base->src & 0xBF) << 24); + header->decimation = htobe16((uint16_t) hdr_base->decimation); + header->time_offset = 0; + header->time_tag = htobe64(hdr_base->seq); + header->tuning_word = htobe32(hdr_base->tuning); + } +}; diff --git a/src/formats/drx8.hpp b/src/formats/drx8.hpp new file mode 100644 index 000000000..cf1f38289 --- /dev/null +++ b/src/formats/drx8.hpp @@ -0,0 +1,159 @@ +/* + * Copyright (c) 2023, The Bifrost Authors. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of The Bifrost Authors nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#include "base.hpp" + +#define DRX8_FRAME_SIZE 8224 + +struct __attribute__((packed)) drx8_hdr_type { + uint32_t sync_word; + uint32_t frame_count_word; + uint32_t seconds_count; + uint16_t decimation; + uint16_t time_offset; + uint64_t time_tag; + uint32_t tuning_word; + uint32_t flags; +}; + +class DRX8Decoder : virtual public PacketDecoder { + inline bool valid_packet(const PacketDesc* pkt) const { + return (pkt->sync == 0x5CDEC0DE && + pkt->src >= 0 && + pkt->src < _nsrc && + pkt->time_tag >= 0 && + (((_nsrc == 4) && + (pkt->tuning > 0 && + pkt->tuning1 > 0)) || + (_nsrc == 2 && + (pkt->tuning > 0 || + pkt->tuning1 > 0))) && + pkt->valid_mode == 0); + } +public: + DRX8Decoder(int nsrc, int src0) : PacketDecoder(nsrc, src0) {} + inline bool operator()(const uint8_t* pkt_ptr, + int pkt_size, + PacketDesc* pkt) const { + if( pkt_size != DRX_FRAME_SIZE ) { + return false; + } + const drx8_hdr_type* pkt_hdr = (drx8_hdr_type*)pkt_ptr; + const uint8_t* pkt_pld = pkt_ptr + sizeof(drx8_hdr_type); + int pld_size = pkt_size - sizeof(drx8_hdr_type); + int pkt_id = pkt_hdr->frame_count_word & 0xFF; + pkt->beam = (pkt_id & 0x7) - 1; + int pkt_tune = ((pkt_id >> 3) & 0x7) - 1; + int pkt_pol = ((pkt_id >> 7) & 0x1); + pkt_id = (pkt_tune << 1) | pkt_pol; + pkt->sync = pkt_hdr->sync_word; + pkt->time_tag = be64toh(pkt_hdr->time_tag) - be16toh(pkt_hdr->time_offset); + pkt->seq = pkt->time_tag / be16toh(pkt_hdr->decimation) / 4096; + pkt->nsrc = _nsrc; + pkt->src = pkt_id - _src0; + if( pkt->src / 2 == 0 ) { + pkt->tuning = be32toh(pkt_hdr->tuning_word); + } else { + pkt->tuning1 = be32toh(pkt_hdr->tuning_word); + } + pkt->decimation = be16toh(pkt_hdr->decimation); + pkt->valid_mode = ((pkt_id >> 6) & 0x1); + pkt->payload_size = pld_size; + pkt->payload_ptr = pkt_pld; + return this->valid_packet(pkt); + } +}; + + +class DRX8Processor : virtual public PacketProcessor { +public: + inline void operator()(const PacketDesc* pkt, + uint64_t seq0, + uint64_t nseq_per_obuf, + int nbuf, + uint8_t* obufs[], + size_t ngood_bytes[], + size_t* src_ngood_bytes[]) { + int obuf_idx = ((pkt->seq - seq0 >= 1*nseq_per_obuf) + + (pkt->seq - seq0 >= 2*nseq_per_obuf)); + size_t obuf_seq0 = seq0 + obuf_idx*nseq_per_obuf; + size_t nbyte = pkt->payload_size; + ngood_bytes[obuf_idx] += nbyte; + src_ngood_bytes[obuf_idx][pkt->src] += nbyte; + int payload_size = pkt->payload_size; + + size_t obuf_offset = (pkt->seq-obuf_seq0)*pkt->nsrc*payload_size; + + // Note: Using these SSE types allows the compiler to use SSE instructions + // However, they require aligned memory (otherwise segfault) + uint16_t const* __restrict__ in = (uint16_t const*)pkt->payload_ptr; + uint16_t* __restrict__ out = (uint16_t* )&obufs[obuf_idx][obuf_offset]; + + for( int samp=0; samp<4096; ++samp ) { // HACK TESTING + *(out + pkt->src) = *in; + in++; + out += pkt->nsrc; + } + } + + inline void blank_out_source(uint8_t* data, + int src, + int nsrc, + int nchan, + int nseq) { + uint16_t* __restrict__ aligned_data = (uint16_t*) data; + for( int t=0; t(hdr); + memset(header, 0, sizeof(drx8_hdr_type)); + + header->sync_word = 0x5CDEC0DE; + // ID is stored in the lowest 8 bits; bit 2 is reserved for DRX8 vs DRX + header->frame_count_word = htobe32((uint32_t) ((hdr_base->src & 0xBF) | 0x40) << 24); + header->decimation = htobe16((uint16_t) hdr_base->decimation); + header->time_offset = 0; + header->time_tag = htobe64(hdr_base->seq); + header->tuning_word = htobe32(hdr_base->tuning); + } +}; diff --git a/src/formats/formats.hpp b/src/formats/formats.hpp new file mode 100644 index 000000000..8f30468e5 --- /dev/null +++ b/src/formats/formats.hpp @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2019-2024, The Bifrost Authors. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of The Bifrost Authors nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#include "chips.hpp" +#include "cor.hpp" +#include "vdif.hpp" +#include "drx.hpp" +#include "drx8.hpp" +#include "tbn.hpp" +#include "tbf.hpp" +#include "ibeam.hpp" +#include "snap2.hpp" +#include "pbeam.hpp" +#include "simple.hpp" +#include "vbeam.hpp" diff --git a/src/formats/ibeam.hpp b/src/formats/ibeam.hpp new file mode 100644 index 000000000..02c5fd9cb --- /dev/null +++ b/src/formats/ibeam.hpp @@ -0,0 +1,175 @@ +/* + * Copyright (c) 2019, The Bifrost Authors. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of The Bifrost Authors nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#include "base.hpp" + +//#include // SSE + +struct __attribute__((packed)) ibeam_hdr_type { + uint8_t server; // Note: 1-based + uint8_t gbe; // (AKA tuning) + uint8_t nchan; // 109 + uint8_t nbeam; // 2 + uint8_t nserver; // 6 + // Note: Big endian + uint16_t chan0; // First chan in packet + uint64_t seq; // Note: 1-based +}; + +template +class IBeamDecoder: virtual public PacketDecoder { + uint8_t _nbeam = B; + + inline bool valid_packet(const PacketDesc* pkt) const { + return (pkt->seq >= 0 && + pkt->src >= 0 && pkt->src < _nsrc && + pkt->beam == _nbeam && + pkt->chan0 >= 0); + } +public: + IBeamDecoder(int nsrc, int src0) : PacketDecoder(nsrc, src0) {} + inline bool operator()(const uint8_t* pkt_ptr, + int pkt_size, + PacketDesc* pkt) const { + if( pkt_size < (int)sizeof(ibeam_hdr_type) ) { + return false; + } + const ibeam_hdr_type* pkt_hdr = (ibeam_hdr_type*)pkt_ptr; + const uint8_t* pkt_pld = pkt_ptr + sizeof(ibeam_hdr_type); + int pld_size = pkt_size - sizeof(ibeam_hdr_type); + pkt->seq = be64toh(pkt_hdr->seq) - 1; + //pkt->nsrc = pkt_hdr->nserver; + pkt->nsrc = _nsrc; + pkt->src = (pkt_hdr->server - 1) - _src0; + pkt->beam = pkt_hdr->nbeam; + pkt->nchan = pkt_hdr->nchan; + pkt->chan0 = ntohs(pkt_hdr->chan0) - pkt->nchan * pkt->src; + pkt->payload_size = pld_size; + pkt->payload_ptr = pkt_pld; + return this->valid_packet(pkt); + } +}; + +template +class IBeamProcessor : virtual public PacketProcessor { + uint8_t _nbeam = B; + +public: + inline void operator()(const PacketDesc* pkt, + uint64_t seq0, + uint64_t nseq_per_obuf, + int nbuf, + uint8_t* obufs[], + size_t ngood_bytes[], + size_t* src_ngood_bytes[]) { + int obuf_idx = ((pkt->seq - seq0 >= 1*nseq_per_obuf) + + (pkt->seq - seq0 >= 2*nseq_per_obuf)); + size_t obuf_seq0 = seq0 + obuf_idx*nseq_per_obuf; + size_t nbyte = pkt->payload_size * BF_UNPACK_FACTOR; + ngood_bytes[obuf_idx] += nbyte; + src_ngood_bytes[obuf_idx][pkt->src] += nbyte; + // **CHANGED RECENTLY + int payload_size = pkt->payload_size; + + size_t obuf_offset = (pkt->seq-obuf_seq0)*pkt->nsrc*payload_size; + typedef unaligned128_type itype; // cf32 * 1 beam * 2 pol + typedef aligned128_type otype; + + obuf_offset *= BF_UNPACK_FACTOR; + + // Note: Using these SSE types allows the compiler to use SSE instructions + // However, they require aligned memory (otherwise segfault) + itype const* __restrict__ in = (itype const*)pkt->payload_ptr; + otype* __restrict__ out = (otype* )&obufs[obuf_idx][obuf_offset]; + + int chan, beam; + for(chan=0; channchan; ++chan ) { + for(beam=0; beam<_nbeam; ++beam) { +#if defined BF_SSE_ENABLED && BF_SSE_ENABLED + const unaligned128_type* dsrc = (const unaligned128_type*) &in[chan*_nbeam + beam]; + aligned128_type* ddst = (aligned128_type*) &out[pkt->src*pkt->nchan*_nbeam + chan*_nbeam + beam]; + + __m128i mtemp = _mm_loadu_si128(reinterpret_cast(dsrc)); + _mm_stream_si128(reinterpret_cast<__m128i*>(ddst), mtemp); +#else + ::memcpy(&out[pkt->src*pkt->nchan*_nbeam + chan*_nbeam + beam], + &in[chan*_nbeam + beam], sizeof(otype)); + //out[pkt->src*pkt->nchan*_nbeam + chan*_nbeam + beam] = in[chan*_nbeam + beam]; +#endif + } + } + } + + inline void blank_out_source(uint8_t* data, + int src, + int nsrc, + int nchan, + int nseq) { + typedef aligned128_type otype; + otype* __restrict__ aligned_data = (otype*)data; + for( int t=0; t(ddst), mtemp); +#else + ::memset(&aligned_data[t*nsrc*nchan*_nbeam + src*nchan*_nbeam + c*_nbeam + b], + 0, sizeof(otype)); +#endif + } + } + } + } +}; + +template +class IBeamHeaderFiller : virtual public PacketHeaderFiller { + uint8_t _nbeam = B; + +public: + inline int get_size() { return sizeof(ibeam_hdr_type); } + inline void operator()(const PacketDesc* hdr_base, + BFoffset framecount, + char* hdr) { + ibeam_hdr_type* header = reinterpret_cast(hdr); + memset(header, 0, sizeof(ibeam_hdr_type)); + + header->server = hdr_base->src + 1; + header->gbe = hdr_base->tuning; + header->nchan = hdr_base->nchan; + header->nbeam = _nbeam; + header->nserver = hdr_base->nsrc; + header->chan0 = htons(hdr_base->chan0); + header->seq = htobe64(hdr_base->seq); + } +}; diff --git a/src/formats/pbeam.hpp b/src/formats/pbeam.hpp new file mode 100644 index 000000000..91d644b59 --- /dev/null +++ b/src/formats/pbeam.hpp @@ -0,0 +1,172 @@ +/* + * Copyright (c) 2020, The Bifrost Authors. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of The Bifrost Authors nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#include "base.hpp" + +//#include // SSE + +struct __attribute__((packed)) pbeam_hdr_type { + uint8_t server; // Note: 1-based + uint8_t beam; // Note: 1-based + uint8_t gbe; // (AKA tuning) + uint8_t nchan; // 109 + uint8_t nbeam; // 2 + uint8_t nserver; // 6 + // Note: Big endian + uint16_t navg; // Number of raw spectra averaged + uint16_t chan0; // First chan in packet + uint64_t seq; // Note: 1-based +}; + +class PBeamDecoder: virtual public PacketDecoder { + inline bool valid_packet(const PacketDesc* pkt) const { + return (pkt->seq >= 0 && + pkt->src >= 0 && pkt->src < _nsrc && + pkt->chan0 >= 0); + } +public: + PBeamDecoder(int nsrc, int src0) : PacketDecoder(nsrc, src0) {} + inline bool operator()(const uint8_t* pkt_ptr, + int pkt_size, + PacketDesc* pkt) const { + if( pkt_size < (int)sizeof(pbeam_hdr_type) ) { + return false; + } + const pbeam_hdr_type* pkt_hdr = (pbeam_hdr_type*)pkt_ptr; + const uint8_t* pkt_pld = pkt_ptr + sizeof(pbeam_hdr_type); + int pld_size = pkt_size - sizeof(pbeam_hdr_type); + pkt->decimation = be16toh(pkt_hdr->navg); + pkt->time_tag = be64toh(pkt_hdr->seq); + pkt->seq = pkt->time_tag / pkt->decimation; + //pkt->nsrc = pkt_hdr->nserver; + pkt->nsrc = _nsrc; + pkt->src = (pkt_hdr->beam - _src0) * pkt_hdr->nserver + (pkt_hdr->server - 1); + pkt->beam = pkt_hdr->nbeam; + pkt->nchan = pkt_hdr->nchan; + pkt->chan0 = ntohs(pkt_hdr->chan0) - pkt->nchan * pkt->src; + pkt->payload_size = pld_size; + pkt->payload_ptr = pkt_pld; + return this->valid_packet(pkt); + } +}; + +class PBeamProcessor : virtual public PacketProcessor { +public: + inline void operator()(const PacketDesc* pkt, + uint64_t seq0, + uint64_t nseq_per_obuf, + int nbuf, + uint8_t* obufs[], + size_t ngood_bytes[], + size_t* src_ngood_bytes[]) { + int obuf_idx = ((pkt->seq - seq0 >= 1*nseq_per_obuf) + + (pkt->seq - seq0 >= 2*nseq_per_obuf)); + size_t obuf_seq0 = seq0 + obuf_idx*nseq_per_obuf; + size_t nbyte = pkt->payload_size * BF_UNPACK_FACTOR; + ngood_bytes[obuf_idx] += nbyte; + src_ngood_bytes[obuf_idx][pkt->src] += nbyte; + // **CHANGED RECENTLY + int payload_size = pkt->payload_size; + + size_t obuf_offset = (pkt->seq-obuf_seq0)*pkt->nsrc*payload_size; + typedef unaligned128_type itype; // f32 * 1 beam * 4 pol (XX, YY, R(XY), I(XY)) + typedef aligned128_type otype; + + obuf_offset *= BF_UNPACK_FACTOR; + + // Note: Using these SSE types allows the compiler to use SSE instructions + // However, they require aligned memory (otherwise segfault) + itype const* __restrict__ in = (itype const*)pkt->payload_ptr; + otype* __restrict__ out = (otype* )&obufs[obuf_idx][obuf_offset]; + + // Convenience + int beam_server = pkt->src; + int nchan = pkt->nchan; + + int chan; + for(chan=0; chan(dsrc)); + _mm_stream_si128(reinterpret_cast<__m128i*>(ddst), mtemp); +#else + ::memcpy(&out[beam_server*nchan + chan], &in[chan], sizeof(otype)); +#endif + } + } + + inline void blank_out_source(uint8_t* data, + int src, // beam_server + int nsrc, // nbeam_server + int nchan, + int nseq) { + typedef aligned128_type otype; + otype* __restrict__ aligned_data = (otype*)data; + for( int t=0; t(ddst), mtemp); +#else + ::memset(&aligned_data[t*nsrc*nchan + src*nchan], + 0, nchan*sizeof(otype)); +#endif + } + } +}; + +template +class PBeamHeaderFiller : virtual public PacketHeaderFiller { + uint8_t _nbeam = B; + +public: + inline int get_size() { return sizeof(pbeam_hdr_type); } + inline void operator()(const PacketDesc* hdr_base, + BFoffset framecount, + char* hdr) { + pbeam_hdr_type* header = reinterpret_cast(hdr); + memset(header, 0, sizeof(pbeam_hdr_type)); + + int nserver = hdr_base->nsrc / _nbeam; + + header->server = (hdr_base->src % nserver) + 1; // Modulo? + header->beam = (hdr_base->src / nserver) + 1; + header->gbe = hdr_base->tuning; + header->nchan = hdr_base->nchan; + header->nbeam = _nbeam; + header->nserver = nserver; + header->navg = htons((uint16_t) hdr_base->decimation); + header->chan0 = htons((uint16_t) hdr_base->chan0); + header->seq = htobe64(hdr_base->seq); + } +}; diff --git a/src/formats/simple.hpp b/src/formats/simple.hpp new file mode 100644 index 000000000..98f4f244d --- /dev/null +++ b/src/formats/simple.hpp @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2019, The Bifrost Authors. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of The Bifrost Authors nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#include "base.hpp" + +struct __attribute__((packed)) simple_hdr_type { + uint64_t seq; +}; + +class SIMPLEDecoder : virtual public PacketDecoder { + inline bool valid_packet(const PacketDesc* pkt) const { + return (pkt->seq >= 0 ); + } +public: + SIMPLEDecoder(int nsrc, int src0) : PacketDecoder(nsrc, src0) {} + inline bool operator()(const uint8_t* pkt_ptr, + int pkt_size, + PacketDesc* pkt) const { + if( pkt_size < (int)sizeof(simple_hdr_type) ) { + return false; + } + const simple_hdr_type* pkt_hdr = (simple_hdr_type*)pkt_ptr; + const uint8_t* pkt_pld = pkt_ptr + sizeof(simple_hdr_type); + int pld_size = pkt_size - sizeof(simple_hdr_type); + pkt->seq = be64toh(pkt_hdr->seq); + pkt->nsrc = 1; + pkt->src = 0; + pkt->payload_size = pld_size; + pkt->payload_ptr = pkt_pld; + pkt->nchan = 1; + pkt->chan0 = 0; + return this->valid_packet(pkt); + } +}; + +class SIMPLEProcessor : virtual public PacketProcessor { +public: + inline void operator()(const PacketDesc* pkt, + uint64_t seq0, + uint64_t nseq_per_obuf, + int nbuf, + uint8_t* obufs[], + size_t ngood_bytes[], + size_t* src_ngood_bytes[]) { + int obuf_idx = ((pkt->seq - seq0 >= 1*nseq_per_obuf) + + (pkt->seq - seq0 >= 2*nseq_per_obuf)); + size_t obuf_seq0 = seq0 + obuf_idx*nseq_per_obuf; + size_t nbyte = pkt->payload_size * BF_UNPACK_FACTOR; + ngood_bytes[obuf_idx] += nbyte; + src_ngood_bytes[obuf_idx][0] += nbyte; + // **CHANGED RECENTLY + int payload_size = pkt->payload_size;//pkt->nchan*(PKT_NINPUT*2*PKT_NBIT/8); + + size_t obuf_offset = (pkt->seq-obuf_seq0)*pkt->nsrc*payload_size; + typedef unaligned256_type itype; + typedef aligned256_type otype; + + obuf_offset *= BF_UNPACK_FACTOR; + + itype const* __restrict__ in = (itype const*)pkt->payload_ptr; + otype* __restrict__ out = (otype* )&obufs[obuf_idx][obuf_offset]; + + int chan = 0; + int nelem = 256; + for( ; chan(hdr); + memset(header, 0, sizeof(simple_hdr_type)); + + header->seq = htobe64(hdr_base->seq); + } +}; diff --git a/src/formats/snap2.hpp b/src/formats/snap2.hpp new file mode 100644 index 000000000..a1150727a --- /dev/null +++ b/src/formats/snap2.hpp @@ -0,0 +1,216 @@ +/* + * Copyright (c) 2019-2023, The Bifrost Authors. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of The Bifrost Authors nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#include "base.hpp" + +// All entries are network (i.e. big) endian +struct __attribute__((packed)) snap2_hdr_type { + uint64_t seq; // Spectra counter == packet counter + uint32_t sync_time; // UNIX sync time + uint16_t npol; // Number of pols in this packet + uint16_t npol_tot; // Number of pols total + uint16_t nchan; // Number of channels in this packet + uint16_t nchan_tot; // Number of channels total (for this pipeline) + uint32_t chan_block_id; // ID of this block of chans + uint32_t chan0; // First channel in this packet + uint32_t pol0; // First pol in this packet +}; + +/* + * The PacketDecoder's job is to unpack + * a packet into a standard PacketDesc + * format, and verify that a packet + * is valid. + */ + +#define BF_SNAP2_DEBUG 0 + +class SNAP2Decoder : virtual public PacketDecoder { +protected: + inline bool valid_packet(const PacketDesc* pkt) const { +//#if BF_SNAP2_DEBUG +// cout << "seq: "<< pkt->seq << endl; +// cout << "src: "<< pkt->src << endl; +// cout << "nsrc: "<< pkt->nsrc << endl; +// cout << "nchan: "<< pkt->nchan << endl; +// cout << "chan0: "<< pkt->chan0 << endl; +//#endif + return ( + pkt->seq >= 0 + && pkt->src >= 0 + && pkt->src < _nsrc + && pkt->nsrc == _nsrc + && pkt->chan0 >= 0 + ); + } +public: + SNAP2Decoder(int nsrc, int src0) : PacketDecoder(nsrc, src0) {} + inline bool operator()(const uint8_t* pkt_ptr, + int pkt_size, + PacketDesc* pkt) const { + if( pkt_size < (int)sizeof(snap2_hdr_type) ) { + return false; + } + const snap2_hdr_type* pkt_hdr = (snap2_hdr_type*)pkt_ptr; + const uint8_t* pkt_pld = pkt_ptr + sizeof(snap2_hdr_type); + int pld_size = pkt_size - sizeof(snap2_hdr_type); + pkt->seq = be64toh(pkt_hdr->seq); + pkt->time_tag = be32toh(pkt_hdr->sync_time); +#if BF_SNAP2_DEBUG + fprintf(stderr, "seq: %lu\t", pkt->seq); + fprintf(stderr, "sync_time: %lu\t", pkt->time_tag); + fprintf(stderr, "nchan: %lu\t", be16toh(pkt_hdr->nchan)); + fprintf(stderr, "npol: %lu\t", be16toh(pkt_hdr->npol)); +#endif + int npol_blocks = (be16toh(pkt_hdr->npol_tot) / be16toh(pkt_hdr->npol)); + int nchan_blocks = (be16toh(pkt_hdr->nchan_tot) / be16toh(pkt_hdr->nchan)); + + pkt->tuning = be32toh(pkt_hdr->chan0); // Abuse this so we can use chan0 to reference channel within pipeline + pkt->nsrc = npol_blocks * nchan_blocks;// _nsrc; + pkt->nchan = be16toh(pkt_hdr->nchan); + pkt->chan0 = be32toh(pkt_hdr->chan_block_id) * be16toh(pkt_hdr->nchan); + pkt->nchan_tot = be16toh(pkt_hdr->nchan_tot); + pkt->npol = be16toh(pkt_hdr->npol); + pkt->npol_tot = be16toh(pkt_hdr->npol_tot); + pkt->pol0 = be32toh(pkt_hdr->pol0); + pkt->src = (pkt->pol0 / pkt->npol) + be32toh(pkt_hdr->chan_block_id) * npol_blocks; + pkt->payload_size = pld_size; + pkt->payload_ptr = pkt_pld; +#if BF_SNAP2_DEBUG + fprintf(stderr, "nsrc: %lu\t", pkt->nsrc); + fprintf(stderr, "src: %lu\t", pkt->src); + fprintf(stderr, "chan0: %lu\t", pkt->chan0); + fprintf(stderr, "chan_block_id: %lu\t", be32toh(pkt_hdr->chan_block_id)); + fprintf(stderr, "nchan_tot: %lu\t", pkt->nchan_tot); + fprintf(stderr, "npol_tot: %lu\t", pkt->npol_tot); + fprintf(stderr, "pol0: %lu\n", pkt->pol0); +#endif + return this->valid_packet(pkt); + } +}; + +class SNAP2Processor : virtual public PacketProcessor { +public: + inline void operator()(const PacketDesc* pkt, + uint64_t seq0, + uint64_t nseq_per_obuf, + int nbuf, + uint8_t* obufs[], + size_t ngood_bytes[], + size_t* src_ngood_bytes[]) { + int obuf_idx = ((pkt->seq - seq0 >= 1*nseq_per_obuf) + + (pkt->seq - seq0 >= 2*nseq_per_obuf)); + size_t obuf_seq0 = seq0 + obuf_idx*nseq_per_obuf; + size_t nbyte = pkt->payload_size * BF_UNPACK_FACTOR; + ngood_bytes[obuf_idx] += nbyte; + src_ngood_bytes[obuf_idx][pkt->src] += nbyte; + int payload_size = pkt->payload_size; + + size_t obuf_offset = (pkt->seq-obuf_seq0)*pkt->nsrc*payload_size; + typedef unaligned256_type itype; //256 bits = 32 pols / word + typedef aligned256_type otype; + + obuf_offset *= BF_UNPACK_FACTOR; + + // Note: Using these SSE types allows the compiler to use SSE instructions + // However, they require aligned memory (otherwise segfault) + itype const* __restrict__ in = (itype const*)pkt->payload_ptr; + otype* __restrict__ out = (otype* )&obufs[obuf_idx][obuf_offset]; + + int words_per_chan_out = pkt->npol_tot >> 5; + int pol_offset_out = pkt->pol0 >> 5; + int pkt_chan = pkt->chan0; // The first channel in this packet + + // Copy packet payload one channel at a time. + // Packets have payload format nchans x npols x complexity. + // Output buffer order is chans * npol_total * complexity + // Spacing with which channel chunks are copied depends + // on the total number of channels/pols in the system + int c=0; +#if defined BF_AVX_ENABLED && BF_AVX_ENABLED + __m256i *dest_p; + __m256i vecbuf[2]; + uint64_t *in64 = (uint64_t *)in; + dest_p = (__m256i *)(out + (words_per_chan_out * (pkt_chan)) + pol_offset_out); +#endif + //if((pol_offset_out == 0) && (pkt_chan==0) && ((pkt->seq % 120)==0) ){ + // fprintf(stderr, "nsrc: %d seq: %d, dest_p: %p obuf idx %d, obuf offset %lu, nseq_per_obuf %d, seq0 %d, nbuf: %d\n", pkt->nsrc, pkt->seq, dest_p, obuf_idx, obuf_offset, nseq_per_obuf, seq0, nbuf); + //} + for(c=0; cnchan; c++) { +#if defined BF_AVX_ENABLED && BF_AVX_ENABLED + vecbuf[0] = _mm256_set_epi64x(in64[3], in64[2], in64[1], in64[0]); + vecbuf[1] = _mm256_set_epi64x(in64[7], in64[6], in64[5], in64[4]); + _mm256_stream_si256(dest_p, vecbuf[0]); + _mm256_stream_si256(dest_p+1, vecbuf[1]); + in64 += 8; + dest_p += words_per_chan_out; +#else + ::memcpy(&out[pkt->src + pkt->nsrc*c], + &in[c], sizeof(otype)); +#endif + } + } + + inline void blank_out_source(uint8_t* data, + int src, + int nsrc, + int nchan, + int nseq) { + typedef aligned256_type otype; + otype* __restrict__ aligned_data = (otype*)data; + for( int t=0; t(hdr); + memset(header, 0, sizeof(snap2_hdr_type)); + + header->seq = htobe64(hdr_base->seq); + header->npol = 2; + header->npol_tot = 2; + header->nchan = hdr_base->nchan; + header->nchan_tot = hdr_base->nchan * hdr_base->nsrc; + header->chan_block_id = hdr_base->src; + header->chan0 = htons(hdr_base->chan0); + header->pol0 = 0; + + } +}; diff --git a/src/formats/tbf.hpp b/src/formats/tbf.hpp new file mode 100644 index 000000000..af90066ce --- /dev/null +++ b/src/formats/tbf.hpp @@ -0,0 +1,59 @@ +/* + * Copyright (c) 2019-2023, The Bifrost Authors. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of The Bifrost Authors nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#include "base.hpp" + +struct __attribute__((packed)) tbf_hdr_type { + uint32_t sync_word; + uint32_t frame_count_word; + uint32_t seconds_count; + uint16_t first_chan; + uint16_t nstand; + uint64_t time_tag; +}; + +class TBFHeaderFiller : virtual public PacketHeaderFiller { +public: + inline int get_size() { return sizeof(tbf_hdr_type); } + inline void operator()(const PacketDesc* hdr_base, + BFoffset framecount, + char* hdr) { + tbf_hdr_type* header = reinterpret_cast(hdr); + memset(header, 0, sizeof(tbf_hdr_type)); + + header->sync_word = 0x5CDEC0DE; + // Bits 9-32 are the frame count; bits 1-8 are the TBF packet flag + header->frame_count_word = htobe32((framecount & 0xFFFFFF) \ + | ((uint32_t) 0x01 << 24)); + header->first_chan = htons(hdr_base->src); + header->nstand = htobe16(hdr_base->nsrc); + header->time_tag = htobe64(hdr_base->seq); + } +}; diff --git a/src/formats/tbn.hpp b/src/formats/tbn.hpp new file mode 100644 index 000000000..2a805c41f --- /dev/null +++ b/src/formats/tbn.hpp @@ -0,0 +1,173 @@ +/* + * Copyright (c) 2019, The Bifrost Authors. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of The Bifrost Authors nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#include "base.hpp" + +#define TBN_FRAME_SIZE 1048 + +struct __attribute__((packed)) tbn_hdr_type { + uint32_t sync_word; + uint32_t frame_count_word; + uint32_t tuning_word; + uint16_t tbn_id; + uint16_t gain; + uint64_t time_tag; +}; + +class TBNCache { + uint64_t _timestamp_last = 0; + int _pid_last = -1000; + uint16_t _decimation = 1; + bool _active = false; +public: + TBNCache() {} + inline uint16_t get_decimation() { return _decimation; } + inline bool update(int packet_id, + uint64_t timestamp) { + if( !_active ) { + if( packet_id == _pid_last ) { + _decimation = ((timestamp - _timestamp_last) / 512); + _active = true; + } else if( _pid_last == -1000 ) { + _pid_last = packet_id; + _timestamp_last = timestamp; + } + } + return _active; + } +}; + +class TBNDecoder : virtual public PacketDecoder { + TBNCache* _cache; + + inline bool valid_packet(const PacketDesc* pkt) const { + return (pkt->sync == 0x5CDEC0DE && + pkt->src >= 0 && + pkt->src < _nsrc && + pkt->time_tag >= 0 && + pkt->tuning >= 0 && + pkt->valid_mode == 0); + } +public: + TBNDecoder(int nsrc, int src0) + : PacketDecoder(nsrc, src0) { + _cache = new TBNCache(); + } + inline bool operator()(const uint8_t* pkt_ptr, + int pkt_size, + PacketDesc* pkt) const { + if( pkt_size != TBN_FRAME_SIZE ) { + return false; + } + const tbn_hdr_type* pkt_hdr = (tbn_hdr_type*)pkt_ptr; + const uint8_t* pkt_pld = pkt_ptr + sizeof(tbn_hdr_type); + int pld_size = pkt_size - sizeof(tbn_hdr_type); + pkt->sync = pkt_hdr->sync_word; + pkt->time_tag = be64toh(pkt_hdr->time_tag); + pkt->nsrc = _nsrc; + pkt->src = (be16toh(pkt_hdr->tbn_id) & 1023) - 1 - _src0; + pkt->tuning = be32toh(pkt_hdr->tuning_word); + pkt->valid_mode = (be16toh(pkt_hdr->tbn_id) >> 15) & 1; + pkt->gain = (be16toh(pkt_hdr->gain)); + pkt->payload_size = pld_size; + pkt->payload_ptr = pkt_pld; + bool valid = this->valid_packet(pkt); + if( valid ) { + valid &= _cache->update(pkt->src, pkt->time_tag); + pkt->decimation = _cache->get_decimation(); + pkt->seq = pkt->time_tag / pkt->decimation / 512; + } + return valid; + } +}; + +class TBNProcessor : virtual public PacketProcessor { +public: + inline void operator()(const PacketDesc* pkt, + uint64_t seq0, + uint64_t nseq_per_obuf, + int nbuf, + uint8_t* obufs[], + size_t ngood_bytes[], + size_t* src_ngood_bytes[]) { + int obuf_idx = ((pkt->seq - seq0 >= 1*nseq_per_obuf) + + (pkt->seq - seq0 >= 2*nseq_per_obuf)); + size_t obuf_seq0 = seq0 + obuf_idx*nseq_per_obuf; + size_t nbyte = pkt->payload_size; + ngood_bytes[obuf_idx] += nbyte; + src_ngood_bytes[obuf_idx][pkt->src] += nbyte; + int payload_size = pkt->payload_size; + + size_t obuf_offset = (pkt->seq-obuf_seq0)*pkt->nsrc*payload_size; + + // Note: Using these SSE types allows the compiler to use SSE instructions + // However, they require aligned memory (otherwise segfault) + uint16_t const* __restrict__ in = (uint16_t const*)pkt->payload_ptr; + uint16_t* __restrict__ out = (uint16_t* )&obufs[obuf_idx][obuf_offset]; + + int samp = 0; + for( ; samp<512; ++samp ) { // HACK TESTING + out[samp*pkt->nsrc + pkt->src] = in[samp]; + } + } + + inline void blank_out_source(uint8_t* data, + int src, + int nsrc, + int nchan, + int nseq) { + uint16_t* __restrict__ aligned_data = (uint16_t*)data; + for( int t=0; t(hdr); + memset(header, 0, sizeof(tbn_hdr_type)); + + header->sync_word = 0x5CDEC0DE; + // Bits 9-32 are the frame count; bits 1-8 are zero + header->frame_count_word = htobe32((framecount & 0xFFFFFF)); + header->tuning_word = htobe32(hdr_base->tuning); + // ID is the upper 14 bits; bit 2 is reserved; bit 1 is the TBW flag + header->tbn_id = htons((hdr_base->src + 1) & 0x3FFF); + header->gain = htons(hdr_base->gain); + header->time_tag = htobe64(hdr_base->seq); + } +}; diff --git a/src/formats/vbeam.hpp b/src/formats/vbeam.hpp new file mode 100644 index 000000000..29f98d749 --- /dev/null +++ b/src/formats/vbeam.hpp @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2019, The Bifrost Authors. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of The Bifrost Authors nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#include "base.hpp" + +struct __attribute__((packed)) vbeam_hdr_type { + uint64_t sync_word; + uint64_t sync_time; + uint64_t time_tag; + double bw_hz; + double sfreq; + uint32_t nchan; + uint32_t chan0; + uint32_t npol; +}; + +class VBeamHeaderFiller : virtual public PacketHeaderFiller { +public: + inline int get_size() { return sizeof(vbeam_hdr_type); } + inline void operator()(const PacketDesc* hdr_base, + BFoffset framecount, + char* hdr) { + vbeam_hdr_type* header = reinterpret_cast(hdr); + memset(header, 0, sizeof(vbeam_hdr_type)); + + header->sync_word = 0xAABBCCDD00000000L; + header->time_tag = htobe64(hdr_base->seq); + } +}; diff --git a/src/formats/vdif.hpp b/src/formats/vdif.hpp new file mode 100644 index 000000000..ef0bd8f16 --- /dev/null +++ b/src/formats/vdif.hpp @@ -0,0 +1,203 @@ +/* + * Copyright (c) 2019, The Bifrost Authors. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of The Bifrost Authors nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#include "base.hpp" + +struct __attribute__((packed)) vdif_hdr_type { + struct __attribute__((packed)) word_1_ { + uint32_t seconds_from_epoch:30; + uint8_t is_legacy:1; + uint8_t is_invalid:1; + } word_1; + struct __attribute__((packed)) word_2_ { + uint32_t frame_in_second:24; + uint16_t ref_epoch:6; + uint8_t unassigned:2; + } word_2; + struct __attribute__((packed)) word_3_ { + uint32_t frame_length:24; + uint32_t log2_nchan:5; + uint8_t version:3; + } word_3; + struct __attribute__((packed)) word_4_ { + uint16_t station_id:16; + uint16_t thread_id:10; + uint8_t bits_per_sample_minus_one:5; + uint8_t is_complex:1; + } word_4; +}; + +struct __attribute__((packed)) vdif_ext_type { + uint32_t extended_word_1; + uint32_t extended_word_2; + uint32_t extended_word_3; + uint32_t extended_word_4; +}; + +class VDIFCache { + BFoffset _frame_last = 0; + uint32_t _frames_per_second = 0; + uint32_t _sample_rate = 0; + bool _active = false; +public: + VDIFCache() {} + inline uint32_t get_frames_per_second() { return _frames_per_second; } + inline uint32_t get_sample_rate() { return _sample_rate; } + inline bool update(uint32_t seconds_from_epoch, + uint32_t frame_in_second, + uint32_t samples_per_frame) { + if( !_active ) { + if( frame_in_second < _frame_last ) { + _frames_per_second = _frame_last + 1; + _sample_rate = _frames_per_second * samples_per_frame; + _active = true; + } + _frame_last = frame_in_second; + } + return _active; + } +}; + +class VDIFDecoder : virtual public PacketDecoder { + VDIFCache* _cache; + + inline bool valid_packet(const PacketDesc* pkt) const { + return (pkt->src >= 0 && + pkt->src < _nsrc && + pkt->valid_mode == 0); + } +public: + VDIFDecoder(int nsrc, int src0) + : PacketDecoder(nsrc, src0) { + _cache = new VDIFCache(); + } + inline bool operator()(const uint8_t* pkt_ptr, + int pkt_size, + PacketDesc* pkt) const { + if( pkt_size < (int)sizeof(vdif_hdr_type) ) { + return false; + } + const vdif_hdr_type* pkt_hdr = (vdif_hdr_type*)pkt_ptr; + const uint8_t* pkt_pld = pkt_ptr + sizeof(vdif_hdr_type); + int pld_size = pkt_size - sizeof(vdif_hdr_type); + if( pkt_hdr->word_1.is_legacy == 0 ) { + // Do not try to decode the extended header entries + pkt_pld += sizeof(vdif_ext_type); + pld_size -= sizeof(vdif_ext_type); + } + + uint32_t nsamples = pld_size * 8 \ + / (pkt_hdr->word_4.bits_per_sample_minus_one + 1) \ + / ((int) 1 << pkt_hdr->word_3.log2_nchan) \ + / (1 + pkt_hdr->word_4.is_complex); + bool is_active = _cache->update(pkt_hdr->word_1.seconds_from_epoch, + pkt_hdr->word_2.frame_in_second, + nsamples); + + pkt->seq = (BFoffset) pkt_hdr->word_1.seconds_from_epoch*_cache->get_frames_per_second() \ + + (BFoffset) pkt_hdr->word_2.frame_in_second; + pkt->time_tag = pkt->seq*_cache->get_sample_rate(); + pkt->src = pkt_hdr->word_4.thread_id - _src0; + pkt->nsrc = _nsrc; + pkt->chan0 = (int) 1 << pkt_hdr->word_3.log2_nchan; + pkt->nchan = pld_size / 8; + pkt->sync = _cache->get_sample_rate(); + pkt->tuning = (((int) pkt_hdr->word_2.ref_epoch) << 16) \ + | (((int) pkt_hdr->word_4.bits_per_sample_minus_one + 1) << 8) \ + | pkt_hdr->word_4.is_complex; + pkt->valid_mode = pkt_hdr->word_1.is_invalid || (not is_active); + pkt->payload_size = pld_size; + pkt->payload_ptr = pkt_pld; + /* + if( this->valid_packet(pkt) && (pkt_hdr->word_2.frame_in_second == 1) ) { + std::cout << "pld_size: " << pld_size << std::endl; + std::cout << " : " << pkt_hdr->word_3.frame_length*8 - 32 + 16*pkt_hdr->word_1.is_legacy<< std::endl; + std::cout << "seconds_from_epoch: " << pkt_hdr->word_1.seconds_from_epoch << std::endl; + std::cout << "frame_in_second: " << pkt_hdr->word_2.frame_in_second << std::endl; + std::cout << "frames_per_second: " << _cache->get_frames_per_second() << std::endl; + std::cout << "sample_rate: " << _cache->get_sample_rate() << std::endl; + std::cout << "src: " << pkt->src << std::endl; + std::cout << "is_legacy: " << pkt_hdr->word_1.is_legacy << std::endl; + std::cout << "valid: " << this->valid_packet(pkt) << std::endl; + } + */ + return this->valid_packet(pkt); + } +}; + +class VDIFProcessor : virtual public PacketProcessor { +public: + inline void operator()(const PacketDesc* pkt, + uint64_t seq0, + uint64_t nseq_per_obuf, + int nbuf, + uint8_t* obufs[], + size_t ngood_bytes[], + size_t* src_ngood_bytes[]) { + int obuf_idx = ((pkt->seq - seq0 >= 1*nseq_per_obuf) + + (pkt->seq - seq0 >= 2*nseq_per_obuf)); + size_t obuf_seq0 = seq0 + obuf_idx*nseq_per_obuf; + size_t nbyte = pkt->payload_size; + ngood_bytes[obuf_idx] += nbyte; + src_ngood_bytes[obuf_idx][pkt->src] += nbyte; + int payload_size = pkt->payload_size; + + size_t obuf_offset = (pkt->seq-obuf_seq0)*pkt->nsrc*payload_size; + typedef uint64_t itype; // 8-byte chunks in the file + typedef uint64_t otype; + + // Note: Using these SSE types allows the compiler to use SSE instructions + // However, they require aligned memory (otherwise segfault) + itype const* __restrict__ in = (itype const*)pkt->payload_ptr; + otype* __restrict__ out = (otype* )&obufs[obuf_idx][obuf_offset]; + + // Convenience + int src = pkt->src; + int nchan = pkt->nchan; + + int chan = 0; + for( ; chan + +#if BF_HWLOC_ENABLED +int HardwareLocality::get_numa_node_of_core(int core) { + int core_depth = hwloc_get_type_or_below_depth(_topo, HWLOC_OBJ_PU); + int ncore = hwloc_get_nbobjs_by_depth(_topo, core_depth); + int ret = -1; + if( 0 <= core && core < ncore ) { + // Find correct processing unit (PU) for the core (which is an os_index) + hwloc_obj_t obj = NULL; + hwloc_obj_t tmp = NULL; + while( (tmp = hwloc_get_next_obj_by_type(_topo, HWLOC_OBJ_PU, tmp)) != NULL ) { + if( tmp->os_index == core ) { + obj = tmp; + break; + } + } + // Find the correct NUMA node for the PU + tmp = NULL; + while( obj != NULL && (tmp = hwloc_get_next_obj_by_type(_topo, HWLOC_OBJ_NUMANODE, tmp)) != NULL ) { + if( hwloc_bitmap_intersects(obj->cpuset, tmp->cpuset) ) { + ret = tmp->os_index; + break; + } + } + } + return ret; +} + +int HardwareLocality::bind_thread_memory_to_core(int core) { + int core_depth = hwloc_get_type_or_below_depth(_topo, HWLOC_OBJ_PU); + int ncore = hwloc_get_nbobjs_by_depth(_topo, core_depth); + int ret = 0; + if( 0 <= core && core < ncore ) { + // Find correct processing unit (PU) for the core (with is a os_index) + hwloc_obj_t obj = NULL; + hwloc_obj_t tmp = NULL; + while( (tmp = hwloc_get_next_obj_by_type(_topo, HWLOC_OBJ_PU, tmp)) != NULL ) { + if( tmp->os_index == core ) { + obj = tmp; + break; + } + } + if( obj != NULL ) { +#if HWLOC_API_VERSION >= 0x00020000 + hwloc_cpuset_t cpuset = hwloc_bitmap_dup(obj->cpuset); + if( !hwloc_bitmap_intersects(cpuset, hwloc_topology_get_allowed_cpuset(_topo)) ) { + throw std::runtime_error("requested core is not in the list of allowed cores"); + } +#else + hwloc_cpuset_t cpuset = hwloc_bitmap_dup(obj->allowed_cpuset); +#endif + hwloc_bitmap_singlify(cpuset); // Avoid hyper-threads + hwloc_membind_policy_t policy = HWLOC_MEMBIND_BIND; + hwloc_membind_flags_t flags = HWLOC_MEMBIND_THREAD; + ret = hwloc_set_membind(_topo, cpuset, policy, flags); + hwloc_bitmap_free(cpuset); + } + } + return ret; +} + +int HardwareLocality::bind_memory_area_to_numa_node(const void* addr, size_t size, int node) { + int nnode = hwloc_get_nbobjs_by_type(_topo, HWLOC_OBJ_NUMANODE); + int ret = -1; + if( 0 <= node && node < nnode ) { + // Find correct NUMA node for the node (which is an os_index) + hwloc_obj_t obj = NULL; + hwloc_obj_t tmp = NULL; + while( (tmp = hwloc_get_next_obj_by_type(_topo, HWLOC_OBJ_NUMANODE, tmp)) != NULL ) { + if( tmp->os_index == node ) { + obj = tmp; + break; + } + } + if( obj != NULL ) { + #if HWLOC_API_VERSION >= 0x00020000 + hwloc_cpuset_t cpuset = hwloc_bitmap_dup(obj->nodeset); + #else + hwloc_cpuset_t cpuset = hwloc_bitmap_dup(obj->allowed_cpuset); + #endif + hwloc_bitmap_singlify(cpuset); // Avoid hyper-threads + hwloc_membind_policy_t policy = HWLOC_MEMBIND_BIND; + hwloc_membind_flags_t flags = HWLOC_MEMBIND_THREAD; + ret = hwloc_set_area_membind(_topo, addr, size, cpuset, policy, flags); + hwloc_bitmap_free(cpuset); + } + } + return ret; +} +#endif // BF_HWLOC_ENABLED diff --git a/src/hw_locality.hpp b/src/hw_locality.hpp new file mode 100644 index 000000000..3e80be59d --- /dev/null +++ b/src/hw_locality.hpp @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2019-2022, The Bifrost Authors. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of The Bifrost Authors nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#include +#include + +#if BF_HWLOC_ENABLED +#include +class HardwareLocality { + hwloc_topology_t _topo; + HardwareLocality(HardwareLocality const&); + HardwareLocality& operator=(HardwareLocality const&); +public: + HardwareLocality() { + hwloc_topology_init(&_topo); + hwloc_topology_load(_topo); + } + ~HardwareLocality() { + hwloc_topology_destroy(_topo); + } + int get_numa_node_of_core(int core); + int bind_thread_memory_to_core(int core); + int bind_memory_area_to_numa_node(const void* addr, size_t size, int node); +}; +#endif // BF_HWLOC_ENABLED + +class BoundThread { +#if BF_HWLOC_ENABLED + HardwareLocality _hwloc; +#endif +public: + BoundThread(int core) { + bfAffinitySetCore(core); +#if BF_HWLOC_ENABLED + assert(_hwloc.bind_thread_memory_to_core(core) == 0); +#endif + } +}; diff --git a/src/ib_verbs.hpp b/src/ib_verbs.hpp new file mode 100644 index 000000000..187f7ce0a --- /dev/null +++ b/src/ib_verbs.hpp @@ -0,0 +1,729 @@ +/* + * Copyright (c) 2020-2023, The Bifrost Authors. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of The Bifrost Authors nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#ifndef BF_VERBS_NQP +#define BF_VERBS_NQP 1 +#endif + +#ifndef BF_VERBS_NPKTBUF +#define BF_VERBS_NPKTBUF 8192 +#endif + +#ifndef BF_VERBS_WCBATCH +#define BF_VERBS_WCBATCH 16 +#endif + +#define BF_VERBS_PAYLOAD_OFFSET 42 + +struct bf_ibv_recv_pkt{ + ibv_recv_wr wr; + ibv_sge sg; + uint32_t length; +}; + +struct bf_ibv_flow { + ibv_flow_attr attr; + ibv_flow_spec_eth eth; + ibv_flow_spec_ipv4 ipv4; + ibv_flow_spec_tcp_udp udp; +} __attribute__((packed)); + +struct bf_ibv { + ibv_context* ctx; + uint8_t port_num; + ibv_pd* pd; + ibv_comp_channel* cc; + ibv_cq** cq; + ibv_qp** qp; + ibv_flow** flows; + + uint8_t* mr_buf; + size_t mr_size; + ibv_mr* mr; + + bf_ibv_recv_pkt* pkt_buf; + bf_ibv_recv_pkt* pkt; + bf_ibv_recv_pkt* pkt_batch; +}; + + +class Verbs { + int _fd; + size_t _pkt_size_max; + int _timeout; + bf_ibv _verbs; + + void get_interface_name(char* name) { + sockaddr_in sin; + char ip[INET_ADDRSTRLEN]; + socklen_t len = sizeof(sin); + check_error(::getsockname(_fd, (sockaddr *)&sin, &len), + "query socket name"); + inet_ntop(AF_INET, &(sin.sin_addr), ip, INET_ADDRSTRLEN); + + // TODO: Is there a better way to find this? + char cmd[256] = {'\0'}; + char line[256] = {'\0'}; + int is_lo = 0; + sprintf(cmd, "ip route get to %s | grep dev | awk '{print $4}'", ip); + FILE* fp = popen(cmd, "r"); + if( fgets(line, sizeof(line), fp) != NULL) { + if( line[strlen(line)-1] == '\n' ) { + line[strlen(line)-1] = '\0'; + } + if( strncmp(&(line[0]), "lo", 2) == 0 ) { + is_lo = 1; + } + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wstringop-truncation" + strncpy(name, &(line[0]), IFNAMSIZ); + #pragma GCC diagnostic pop + } + pclose(fp); + + if( is_lo ) { + // TODO: Is there a way to avoid having to do this? + sprintf(cmd, "ip route show | grep %s | grep -v default | awk '{print $3}'", ip); + fp = popen(cmd, "r"); + if( fgets(line, sizeof(line), fp) != NULL) { + if( line[strlen(line)-1] == '\n' ) { + line[strlen(line)-1] = '\0'; + } + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wstringop-truncation" + strncpy(name, &(line[0]), IFNAMSIZ); + #pragma GCC diagnostic pop + } + pclose(fp); + } + } + void get_mac_address(uint8_t* mac) { + ifreq ethreq; + this->get_interface_name(&(ethreq.ifr_name[0])); + check_error(::ioctl(_fd, SIOCGIFHWADDR, ðreq), + "query interface hardware address"); + + ::memcpy(mac, (uint8_t*) ethreq.ifr_hwaddr.sa_data, 6); + } + void get_ip_address(char* ip) { + sockaddr_in sin; + socklen_t len = sizeof(sin); + check_error(::getsockname(_fd, (sockaddr *)&sin, &len), + "query socket name"); + inet_ntop(AF_INET, &(sin.sin_addr), ip, INET_ADDRSTRLEN); + } + uint16_t get_port() { + sockaddr_in sin; + socklen_t len = sizeof(sin); + check_error(::getsockname(_fd, (sockaddr *)&sin, &len), + "query socket name"); + return ntohs(sin.sin_port); + } + int get_timeout_ms() { + timeval value; + socklen_t size = sizeof(value); + check_error(::getsockopt(_fd, SOL_SOCKET, SO_RCVTIMEO, &value, &size), + "query socket timeout"); + return int(value.tv_sec*1000) + int(value.tv_usec/1000); + } + uint64_t get_interface_gid() { + uint64_t id; + uint8_t mac[6] = {0}; + uint8_t buf[8] = {0}; + this->get_mac_address(&(mac[0])); + + ::memcpy(buf, (unsigned char*) &(mac[0]), 3); + buf[0] ^= 2; // Toggle G/L bit per modified EUI-64 spec + buf[3] = 0xff; + buf[4] = 0xfe; + ::memcpy(buf+5, (unsigned char*) &(mac[3]), 3); + ::memcpy(&id, buf, 8); + return id; + } + void create_context() { + int d, p, g; + int ndev, found; + ibv_device** ibv_dev_list = NULL; + ibv_context* ibv_ctx = NULL; + ibv_device_attr_ex ibv_dev_attr; + ibv_port_attr ibv_port_attr; + union ibv_gid ibv_gid; + + // Get the interface MAC address and GID + found = 0; + uint8_t mac[6] = {0}; + this->get_mac_address(&(mac[0])); + uint64_t gid = this->get_interface_gid(); + + // Find the right device + /* Query all devices */ + ibv_dev_list = ibv_get_device_list(&ndev); + check_null(ibv_dev_list, + "ibv_get_device_list"); + + /* Interogate */ + for(d=0; dfd, F_GETFL); + check_error(::fcntl(_verbs.cc->fd, F_SETFL, flags | O_NONBLOCK), + "set receive completion channel to non-blocking"); + flags = ::fcntl(_verbs.cc->fd, F_GETFD); + check_error(::fcntl(_verbs.cc->fd, F_SETFD, flags | O_CLOEXEC), + "set receive completion channel to non-blocking"); + ::madvise(_verbs.cc, sizeof(ibv_pd), MADV_DONTFORK); + + // Setup the completion queues + _verbs.cq = (ibv_cq**) ::malloc(BF_VERBS_NQP * sizeof(ibv_cq*)); + check_null(_verbs.cq, + "allocate receive completion queues"); + ::memset(_verbs.cq, 0, BF_VERBS_NQP * sizeof(ibv_cq*)); + for(i=0; ilkey; + } + } + + // Link the work requests to receive queue + for(i=0; iget_port()); + flow.udp.mask.dst_port = 0xffff; + + // Filter on IP address in the IPv4 header + uint32_t ip; + char ip_str[INET_ADDRSTRLEN]; + this->get_ip_address(&(ip_str[0])); + inet_pton(AF_INET, &(ip_str[0]), &ip); + ::memcpy(&(flow.ipv4.val.dst_ip), &ip, 4); + ::memset(&(flow.ipv4.mask.dst_ip), 0xff, 4); + + // Filter on the destination MAC address (actual or multicast) + uint8_t mac[6] = {0}; + this->get_mac_address(&(mac[0])); + if( ((ip & 0xFF) >= 224) && ((ip & 0xFF) < 240) ) { + ETHER_MAP_IP_MULTICAST(&ip, mac); + } + ::memcpy(&flow.eth.val.dst_mac, &mac, 6); + ::memset(&flow.eth.mask.dst_mac, 0xff, 6); + + // Create the flows + _verbs.flows = (ibv_flow**) ::malloc(BF_VERBS_NQP * sizeof(ibv_flow*)); + check_null(_verbs.flows, + "allocate flows"); + ::memset(_verbs.flows, 0, BF_VERBS_NQP * sizeof(ibv_flow*)); + for(i=0; iwr.wr_id / BF_VERBS_NPKTBUF; + return ibv_post_recv(_verbs.qp[i], &recv_pkt->wr, &recv_wr_bad); + } + inline bf_ibv_recv_pkt* receive() { + int i; + int num_wce; + uint64_t wr_id; + pollfd pfd; + ibv_qp_attr qp_attr; + ibv_cq *ev_cq; + intptr_t ev_cq_ctx; + ibv_wc wc[BF_VERBS_WCBATCH]; + bf_ibv_recv_pkt *recv_head = NULL; + ibv_recv_wr *recv_tail = NULL; + + // Ensure the queue pairs are in a state suitable for receiving + for(i=0; istate) { + case IBV_QPS_RESET: // Unexpected, but maybe user reset it + qp_attr.qp_state = IBV_QPS_INIT; + qp_attr.port_num = _verbs.port_num; + if( ibv_modify_qp(_verbs.qp[i], &qp_attr, IBV_QP_STATE|IBV_QP_PORT) ) { + return NULL; + } + case IBV_QPS_INIT: + qp_attr.qp_state = IBV_QPS_RTR; + qp_attr.port_num = _verbs.port_num; + if( ibv_modify_qp(_verbs.qp[i], &qp_attr, IBV_QP_STATE) ) { + return NULL; + } + break; + case IBV_QPS_RTR: + case IBV_QPS_RTS: + break; + default: + return NULL; + } + } + + // Setup for poll + pfd.fd = _verbs.cc->fd; + pfd.events = POLLIN; + pfd.revents = 0; + + // poll completion channel's fd with given timeout + int rc = ::poll(&pfd, 1, _timeout); + if( rc == 0) { + // Timeout + errno = EAGAIN; + throw Verbs::Error("timeout"); + } else if( rc < 0) { + // Error + throw Verbs::Error("error"); + } + + // Get the completion event + if( ibv_get_cq_event(_verbs.cc, &ev_cq, (void **)&ev_cq_ctx) ) { + return NULL; + } + + // Ack the event + ibv_ack_cq_events(ev_cq, 1); + + // Request notification upon the next completion event + // Do NOT restrict to solicited-only completions + if( ibv_req_notify_cq(ev_cq, 0) ) { + return NULL; + } + + // Empty the CQ: poll all of the completions from the CQ (if any exist) + do { + num_wce = ibv_poll_cq(ev_cq, BF_VERBS_WCBATCH, &wc[0]); + if( num_wce < 0 ) { + return NULL; + } + + // Loop through all work completions + for(i=0; iwr; + } else { + recv_tail->next = &(_verbs.pkt_buf[wr_id].wr); + recv_tail = recv_tail->next; + } + } // for each work completion + } while(num_wce); + + // Ensure list is NULL terminated (if we have a list) + if(recv_tail) { + recv_tail->next = NULL; + } + + return recv_head; + } + inline void check_error(int retval, std::string what) { + if( retval < 0 ) { + destroy_flows(); + destroy_queues(); + destroy_buffers(); + destroy_context(); + + std::stringstream ss; + ss << "Failed to " << what << ": (" << errno << ") " + << strerror(errno); + throw Verbs::Error(ss.str()); + } + } + inline void check_null(void* ptr, std::string what) { + if( ptr == NULL ) { + destroy_flows(); + destroy_queues(); + destroy_buffers(); + destroy_context(); + + std::stringstream ss; + ss << "Failed to " << what << ": (" << errno << ") " + << strerror(errno); + throw Verbs::Error(ss.str()); + } + } + inline void check_null_qp(void* ptr, std::string what) { + if( ptr == NULL ) { + destroy_flows(); + destroy_queues(); + destroy_buffers(); + destroy_context(); + + std::stringstream ss; + ss << "Failed to " << what << ": (" << errno << ") " + << strerror(errno); + if( errno == EPERM ) { + ss << " Do you need to set 'options ibverbs disable_raw_qp_enforcement=1' " + << "or add the CAP_NET_RAW capability?"; + } + throw Verbs::Error(ss.str()); + } + } +public: + class Error : public std::runtime_error { + typedef std::runtime_error super_t; + protected: + virtual const char* what() const throw() { + return super_t::what(); + } + public: + Error(const std::string& what_arg) + : super_t(what_arg) {} + }; + + Verbs(int fd, size_t pkt_size_max) + : _fd(fd), _pkt_size_max(pkt_size_max), _timeout(1) { + _timeout = get_timeout_ms(); + + ::memset(&_verbs, 0, sizeof(_verbs)); + check_error(ibv_fork_init(), + "make verbs fork safe"); + create_context(); + create_buffers(); + create_queues(); + link_work_requests(); + create_flows(); + } + ~Verbs() { + destroy_flows(); + destroy_queues(); + destroy_buffers(); + destroy_context(); + } + inline int recv_packet(uint8_t** pkt_ptr, int flags=0) { + // If we don't have a work-request queue on the go, + // get some new packets. + if( _verbs.pkt != NULL) { + _verbs.pkt = (bf_ibv_recv_pkt *) _verbs.pkt->wr.next; + if( _verbs.pkt == NULL ) { + this->release(_verbs.pkt_batch); + _verbs.pkt_batch = NULL; + } + } + + while( _verbs.pkt_batch == NULL ) { + try { + _verbs.pkt_batch = this->receive(); + } catch(Verbs::Error const&) { + _verbs.pkt = NULL; + errno = EAGAIN; + return -1; + } + _verbs.pkt = _verbs.pkt_batch; + } + + // IBV returns Eth/UDP/IP headers. Strip them off here + *pkt_ptr = (uint8_t *) _verbs.pkt->wr.sg_list->addr + BF_VERBS_PAYLOAD_OFFSET; + return _verbs.pkt->length - BF_VERBS_PAYLOAD_OFFSET; + } +}; diff --git a/src/ib_verbs_send.hpp b/src/ib_verbs_send.hpp new file mode 100644 index 000000000..40d2e7093 --- /dev/null +++ b/src/ib_verbs_send.hpp @@ -0,0 +1,839 @@ +/* + * Copyright (c) 2020-2023, The Bifrost Authors. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of The Bifrost Authors nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +// Catch for older InfiniBand verbs installs that do not have the +//#define BF_ENABLE_VERBS_OFFLOAD 0 + +#ifndef BF_VERBS_SEND_NQP +#define BF_VERBS_SEND_NQP 1 +#endif + +#ifndef BF_VERBS_SEND_NPKTBUF +#define BF_VERBS_SEND_NPKTBUF 512 +#endif + +#ifndef BF_VERBS_SEND_WCBATCH +#define BF_VERBS_SEND_WCBATCH 16 +#endif + +#ifndef BF_VERBS_SEND_NPKTBURST +#define BF_VERBS_SEND_NPKTBURST 16 +#endif + +#ifndef BF_VERBS_SEND_PACING +#define BF_VERBS_SEND_PACING 0 +#endif + +#define BF_VERBS_SEND_PAYLOAD_OFFSET 42 + +struct bf_ibv_send_pkt{ + ibv_send_wr wr; + ibv_sge sg; +}; + +struct bf_ibv_send { + ibv_context* ctx; + uint8_t port_num; + ibv_pd* pd; + ibv_comp_channel* cc; + ibv_cq** cq; + ibv_qp** qp; + + uint8_t* mr_buf; + size_t mr_size; + ibv_mr* mr; + + bf_ibv_send_pkt* pkt_buf; + bf_ibv_send_pkt* pkt_head; + + int nqueued; + + uint8_t offload_csum; + uint32_t hardware_pacing[2]; +}; + +struct __attribute__((packed)) bf_ethernet_hdr { + uint8_t dst_mac[6]; + uint8_t src_mac[6]; + uint16_t type; +}; + +struct __attribute__((packed)) bf_ipv4_hdr { + uint8_t version_ihl; + uint8_t tos; + uint16_t length; + uint16_t id; + uint16_t flags_frag; + uint8_t ttl; + uint8_t proto; + uint16_t checksum; + uint32_t src_addr; + uint32_t dst_addr; +}; + +inline void bf_ipv4_update_checksum(bf_ipv4_hdr* hdr) { + hdr->checksum = 0; + uint16_t *block = reinterpret_cast(hdr); + + uint32_t checksum = 0; + for(uint32_t i=0; i 0xFFFF ) { + checksum = (checksum & 0xFFFF) + (checksum >> 16); + } + hdr->checksum = ~htons((uint16_t) checksum); +} + +struct __attribute__((packed)) bf_udp_hdr { + uint16_t src_port; + uint16_t dst_port; + uint16_t length; + uint16_t checksum; +}; + +struct __attribute__((packed)) bf_comb_udp_hdr { + bf_ethernet_hdr ethernet; + bf_ipv4_hdr ipv4; + bf_udp_hdr udp; +}; + + +class VerbsSend { + int _fd; + size_t _pkt_size_max; + int _timeout; + uint32_t _rate_limit; + bf_ibv_send _verbs; + + void get_interface_name(char* name) { + sockaddr_in sin; + socklen_t len = sizeof(sin); + check_error(::getsockname(_fd, (sockaddr *)&sin, &len), + "query socket name"); + + ifaddrs *ifaddr; + check_error(::getifaddrs(&ifaddr), + "query interfaces"); + for(ifaddrs *ifa=ifaddr; ifa != NULL; ifa=ifa->ifa_next) { + if( ifa->ifa_addr != NULL) { + if( reinterpret_cast(ifa->ifa_addr)->sin_addr.s_addr == sin.sin_addr.s_addr ) { + #pragma GCC diagnostic push + #pragma GCC diagnostic ignored "-Wstringop-truncation" + ::strncpy(name, ifa->ifa_name, IFNAMSIZ); + #pragma GCC diagnostic pop + break; + } + } + } + ::freeifaddrs(ifaddr); + } + void get_mac_address(uint8_t* mac) { + ifreq ethreq; + this->get_interface_name(&(ethreq.ifr_name[0])); + check_error(::ioctl(_fd, SIOCGIFHWADDR, ðreq), + "query interface hardware address"); + + ::memcpy(mac, (uint8_t*) ethreq.ifr_hwaddr.sa_data, 6); + } + void get_remote_mac_address(uint8_t* mac) { + uint32_t ip; + char ip_str[INET_ADDRSTRLEN]; + this->get_remote_ip_address(&(ip_str[0])); + ::inet_pton(AF_INET, &(ip_str[0]), &ip); + + if( ((ip & 0xFF) >= 224) && ((ip & 0xFF) < 240) ) { + ETHER_MAP_IP_MULTICAST(&ip, mac); + } else { + int ret = -1; + char cmd[256] = {'\0'}; + char line[256] = {'\0'}; + char ip_entry[INET_ADDRSTRLEN]; + unsigned int hw_type, flags; + char mac_str[17]; + char mask[24]; + char dev[IFNAMSIZ]; + char* end; + + sprintf(cmd, "ping -c 1 %s", ip_str); + FILE* fp = popen(cmd, "r"); + + fp = fopen("/proc/net/arp", "r"); + while( ::fgets(line, sizeof(line), fp) != NULL) { + ::sscanf(line, "%s 0x%x 0x%x %s %s %s\n", + ip_entry, &hw_type, &flags, mac_str, mask, dev); + + if( (::strcmp(ip_str, ip_entry) == 0) && (flags & ATF_COM) ) { + ret = 0; + mac[0] = (uint8_t) strtol(&mac_str[0], &end, 16); + mac[1] = (uint8_t) strtol(&mac_str[3], &end, 16); + mac[2] = (uint8_t) strtol(&mac_str[6], &end, 16); + mac[3] = (uint8_t) strtol(&mac_str[9], &end, 16); + mac[4] = (uint8_t) strtol(&mac_str[12], &end, 16); + mac[5] = (uint8_t) strtol(&mac_str[15], &end, 16); + break; + } + } + fclose(fp); + check_error(ret, "determine remote hardware address"); + } + } + void get_ip_address(char* ip) { + sockaddr_in sin; + socklen_t len = sizeof(sin); + check_error(::getsockname(_fd, (sockaddr *)&sin, &len), + "query socket name"); + inet_ntop(AF_INET, &(sin.sin_addr), ip, INET_ADDRSTRLEN); + } + void get_remote_ip_address(char* ip) { + sockaddr_in sin; + socklen_t len = sizeof(sin); + check_error(::getpeername(_fd, (sockaddr *)&sin, &len), + "query peer name"); + inet_ntop(AF_INET, &(sin.sin_addr), ip, INET_ADDRSTRLEN); + } + uint16_t get_port() { + sockaddr_in sin; + socklen_t len = sizeof(sin); + check_error(::getsockname(_fd, (sockaddr *)&sin, &len), + "query socket name"); + return ntohs(sin.sin_port); + } + uint16_t get_remote_port() { + sockaddr_in sin; + socklen_t len = sizeof(sin); + check_error(::getpeername(_fd, (sockaddr *)&sin, &len), + "query peer name"); + return ntohs(sin.sin_port); + } + uint8_t get_ttl() { + uint8_t ttl; + socklen_t len = sizeof(ttl); + check_error(::getsockopt(_fd, IPPROTO_IP, IP_TTL, &ttl, &len), + "determine TTL"); + return ttl; + } + int get_timeout_ms() { + timeval value; + socklen_t size = sizeof(value); + check_error(::getsockopt(_fd, SOL_SOCKET, SO_RCVTIMEO, &value, &size), + "query socket timeout"); + return int(value.tv_sec*1000) + int(value.tv_usec/1000); + } + uint64_t get_interface_gid() { + uint64_t id; + uint8_t mac[6] = {0}; + uint8_t buf[8] = {0}; + this->get_mac_address(&(mac[0])); + + ::memcpy(buf, (unsigned char*) &(mac[0]), 3); + buf[0] ^= 2; // Toggle G/L bit per modified EUI-64 spec + buf[3] = 0xff; + buf[4] = 0xfe; + ::memcpy(buf+5, (unsigned char*) &(mac[3]), 3); + ::memcpy(&id, buf, 8); + return id; + } + void create_context() { + int d, p, g; + int ndev, found; + ibv_device** ibv_dev_list = NULL; + ibv_context* ibv_ctx = NULL; + ibv_device_attr_ex ibv_dev_attr; + ibv_port_attr ibv_port_attr; + union ibv_gid ibv_gid; + + // Get the interface MAC address and GID + found = 0; + uint8_t mac[6] = {0}; + this->get_mac_address(&(mac[0])); + uint64_t gid = this->get_interface_gid(); + + // Find the right device + /* Query all devices */ + ibv_dev_list = ibv_get_device_list(&ndev); + check_null(ibv_dev_list, + "ibv_get_device_list"); + + /* Interogate */ + for(d=0; dfd, F_GETFL); + check_error(::fcntl(_verbs.cc->fd, F_SETFL, flags | O_NONBLOCK), + "set send completion channel to non-blocking"); + flags = ::fcntl(_verbs.cc->fd, F_GETFD); + check_error(::fcntl(_verbs.cc->fd, F_SETFD, flags | O_CLOEXEC), + "set send completion channel to non-blocking"); + ::madvise(_verbs.cc, sizeof(ibv_pd), MADV_DONTFORK); + + // Setup the completion queues + _verbs.cq = (ibv_cq**) ::malloc(BF_VERBS_SEND_NQP * sizeof(ibv_cq*)); + check_null(_verbs.cq, + "allocate send completion queues"); + ::memset(_verbs.cq, 0, BF_VERBS_SEND_NQP * sizeof(ibv_cq*)); + for(i=0; ilkey; + } + } + + // Link the work requests to send queue + uint32_t send_flags = IBV_SEND_SIGNALED; + #if defined BF_ENABLE_VERBS_OFFLOAD && BF_ENABLE_VERBS_OFFLOAD + if( _verbs.offload_csum ) { + send_flags |= IBV_SEND_IP_CSUM; + } + #endif + + for(i=0; istate) { + case IBV_QPS_RESET: // Unexpected, but maybe user reset it + qp_attr.qp_state = IBV_QPS_INIT; + qp_attr.port_num = _verbs.port_num; + if( ibv_modify_qp(_verbs.qp[i], &qp_attr, IBV_QP_STATE|IBV_QP_PORT) ) { + return NULL; + } + case IBV_QPS_INIT: + qp_attr.qp_state = IBV_QPS_RTR; + qp_attr.port_num = _verbs.port_num; + if( ibv_modify_qp(_verbs.qp[i], &qp_attr, IBV_QP_STATE) ) { + return NULL; + } + case IBV_QPS_RTR: + qp_attr.qp_state = IBV_QPS_RTS; + if(ibv_modify_qp(_verbs.qp[i], &qp_attr, IBV_QP_STATE)) { + return NULL; + } + break; + case IBV_QPS_RTS: + break; + default: + return NULL; + } + } + + // Get the completion event(s) + while( ibv_get_cq_event(_verbs.cc, &ev_cq, (void **)&ev_cq_ctx) == 0 ) { + // Ack the event + ibv_ack_cq_events(ev_cq, 1); + } + + for(i=0; i 0 ) { + for(i=0; iwr.next = &(_verbs.pkt_head->wr); + _verbs.pkt_head = send_pkt; + } // for each work completion + + // Decrement the number of packet buffers in use + _verbs.nqueued -= num_wce; + } while(num_wce); + } + } + + if( npackets == 0 || !_verbs.pkt_head ) { + return NULL; + } + + send_head = _verbs.pkt_head; + send_tail = _verbs.pkt_head; + for(i=0; iwr.next; i++) { + send_tail = (bf_ibv_send_pkt*) send_tail->wr.next; + } + + _verbs.pkt_head = (bf_ibv_send_pkt*) send_tail->wr.next; + send_tail->wr.next = NULL; + + _verbs.nqueued += npackets; + + return send_head; + } + inline void check_error(int retval, std::string what) { + if( retval < 0 ) { + destroy_queues(); + destroy_buffers(); + destroy_context(); + + std::stringstream ss; + ss << "Failed to " << what << ": (" << errno << ") " + << strerror(errno); + throw VerbsSend::Error(ss.str()); + } + } + inline void check_null(void* ptr, std::string what) { + if( ptr == NULL ) { + destroy_queues(); + destroy_buffers(); + destroy_context(); + + std::stringstream ss; + ss << "Failed to " << what << ": (" << errno << ") " + << strerror(errno); + throw VerbsSend::Error(ss.str()); + } + } + inline void check_null_qp(void* ptr, std::string what) { + if( ptr == NULL ) { + destroy_queues(); + destroy_buffers(); + destroy_context(); + + std::stringstream ss; + ss << "Failed to " << what << ": (" << errno << ") " + << strerror(errno); + if( errno == EPERM ) { + ss << " Do you need to set 'options ibverbs disable_raw_qp_enforcement=1' " + << "or add the CAP_NET_RAW capability?"; + } + throw VerbsSend::Error(ss.str()); + } + } +public: + class Error : public std::runtime_error { + typedef std::runtime_error super_t; + protected: + virtual const char* what() const throw() { + return super_t::what(); + } + public: + Error(const std::string& what_arg) + : super_t(what_arg) {} + }; + + VerbsSend(int fd, size_t pkt_size_max) + : _fd(fd), _pkt_size_max(pkt_size_max), _timeout(1), _rate_limit(0) { + _timeout = get_timeout_ms(); + + ::memset(&_verbs, 0, sizeof(_verbs)); + check_error(ibv_fork_init(), + "make verbs fork safe"); + create_context(); + create_buffers(); + create_queues(); + link_work_requests(); + } + ~VerbsSend() { + destroy_queues(); + destroy_buffers(); + destroy_context(); + } + inline void set_rate_limit(uint32_t rate_limit, size_t udp_length, size_t max_burst_size=BF_VERBS_SEND_NPKTBURST) { + int i; + + // Converts to B/s to kb/s assuming a packet size + size_t pkt_size = udp_length + BF_VERBS_SEND_PAYLOAD_OFFSET; + rate_limit = ((float) rate_limit) / udp_length * pkt_size * 8 / 1000; + + // Verify that this rate limit is valid + if( rate_limit == 0 ) { + rate_limit = _verbs.hardware_pacing[1]; + } + if( rate_limit < _verbs.hardware_pacing[0] || rate_limit > _verbs.hardware_pacing[1] ) { + throw VerbsSend::Error("Failed to set rate limit, specified rate limit is out of range"); + } + + // Apply the rate limit + #if defined BF_VERBS_SEND_PACING && BF_VERBS_SEND_PACING + ibv_qp_rate_limit_attr rl_attr; + ::memset(&rl_attr, 0, sizeof(ibv_qp_rate_limit_attr)); + rl_attr.rate_limit = rate_limit; + rl_attr.typical_pkt_sz = pkt_size; + rl_attr.max_burst_sz = max_burst_size*pkt_size; + for(i=0; iget_mac_address(&(src_mac[0])); + this->get_remote_mac_address(&(dst_mac[0])); + + ::memset(hdr, 0, sizeof(bf_ethernet_hdr)); + ::memcpy(hdr->dst_mac, dst_mac, 6); + ::memcpy(hdr->src_mac, src_mac, 6); + hdr->type = htons(0x0800); // IPv4 + } + inline void get_ipv4_header(bf_ipv4_hdr* hdr, size_t udp_length=0) { + uint8_t ttl = this->get_ttl(); + uint32_t src_ip, dst_ip; + char src_ip_str[INET_ADDRSTRLEN], dst_ip_str[INET_ADDRSTRLEN]; + this->get_ip_address(&(src_ip_str[0])); + inet_pton(AF_INET, &(src_ip_str[0]), &src_ip); + this->get_remote_ip_address(&(dst_ip_str[0])); + inet_pton(AF_INET, &(dst_ip_str[0]), &dst_ip); + + ::memset(hdr, 0, sizeof(bf_ipv4_hdr)); + hdr->version_ihl = htons(0x4500); // v4 + 20-byte header + hdr->length = htons((uint16_t) (20 + 8 + udp_length)); + hdr->flags_frag = htons(1<<14); // don't fragment + hdr->ttl = ttl; + hdr->proto = 0x11; // UDP + hdr->src_addr = src_ip; + hdr->dst_addr = dst_ip; + if( !_verbs.offload_csum ) { + bf_ipv4_update_checksum(hdr); + } + } + inline void get_udp_header(bf_udp_hdr* hdr, size_t udp_length=0) { + uint16_t src_port, dst_port; + src_port = this->get_port(); + dst_port = this->get_remote_port(); + + ::memset(hdr, 0, sizeof(bf_udp_hdr)); + hdr->src_port = htons(src_port); + hdr->dst_port = htons(dst_port); + hdr->length = htons((uint16_t) (8 + udp_length)); + } + inline int sendmmsg(mmsghdr *mmsg, int npackets, int flags=0) { + int ret; + bf_ibv_send_pkt* head; + ibv_send_wr *s; + + if( npackets > BF_VERBS_SEND_NPKTBUF ) { + throw VerbsSend::Error(std::string("Too many packets for the current buffer size")); + } + + // Reclaim a set of buffers to use + head = this->get_packet_buffers(npackets); + + // Load in the new data + int i; + uint64_t offset; + for(i=0; iwr), &s); + if( ret ) { + ret = -1; + } else { + ret = npackets; + } + return ret; + } +}; diff --git a/src/memory.cpp b/src/memory.cpp index bd8819395..e76567cfd 100644 --- a/src/memory.cpp +++ b/src/memory.cpp @@ -61,18 +61,18 @@ BFstatus bfGetSpace(const void* ptr, BFspace* space) { // WAR to avoid the ignored failure showing up later cudaGetLastError(); #if defined(CUDA_VERSION) && CUDA_VERSION >= 10000 -} else { - switch( ptr_attrs.type ) { - case cudaMemoryTypeUnregistered: *space = BF_SPACE_SYSTEM; break; - case cudaMemoryTypeHost: *space = BF_SPACE_CUDA_HOST; break; - case cudaMemoryTypeDevice: *space = BF_SPACE_CUDA; break; - case cudaMemoryTypeManaged: *space = BF_SPACE_CUDA_MANAGED; break; - default: { - // This should never be reached - BF_FAIL("Valid memoryType", BF_STATUS_INTERNAL_ERROR); - } - } - } + } else { + switch( ptr_attrs.type ) { + case cudaMemoryTypeUnregistered: *space = BF_SPACE_SYSTEM; break; + case cudaMemoryTypeHost: *space = BF_SPACE_CUDA_HOST; break; + case cudaMemoryTypeDevice: *space = BF_SPACE_CUDA; break; + case cudaMemoryTypeManaged: *space = BF_SPACE_CUDA_MANAGED; break; + default: { + // This should never be reached + BF_FAIL("Valid memoryType", BF_STATUS_INTERNAL_ERROR); + } + } + } #else } else if( ptr_attrs.isManaged ) { *space = BF_SPACE_CUDA_MANAGED; diff --git a/src/packet_capture.cpp b/src/packet_capture.cpp new file mode 100644 index 000000000..ad99f1ea5 --- /dev/null +++ b/src/packet_capture.cpp @@ -0,0 +1,430 @@ +/* + * Copyright (c) 2019-2023, The Bifrost Authors. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of The Bifrost Authors nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "packet_capture.hpp" + +#define BF_JAYCE_DEBUG 0 + +#if BF_JAYCE_DEBUG +#define BF_PRINTD(stmt) \ + std::cout << stmt << std::endl +#else // not BF_JAYCE_DEBUG +#define BF_PRINTD(stmt) +#endif + +// Reads, decodes and unpacks frames into the provided buffers +// Note: Read continues until the first frame that belongs +// beyond the end of the provided buffers. This frame is +// saved, accessible via get_last_frame(), and will be +// processed on the next call to run() if possible. +template +int PacketCaptureThread::run(uint64_t seq_beg, + uint64_t nseq_per_obuf, + int nbuf, + uint8_t* obufs[], + size_t* ngood_bytes[], + size_t* src_ngood_bytes[], + PDC* decode, + PPC* process) { + uint64_t seq_end = seq_beg + nbuf*nseq_per_obuf; + size_t local_ngood_bytes[2] = {0, 0}; + int ret; + while( true ) { + if( !_have_pkt ) { + uint8_t* pkt_ptr; + int pkt_size = _method->recv_packet(&pkt_ptr); + if( pkt_size <= 0 ) { + if( errno == EAGAIN || errno == EWOULDBLOCK ) { + ret = CAPTURE_TIMEOUT; // Timed out + } else if( errno == EINTR ) { + ret = CAPTURE_INTERRUPTED; // Interrupted by signal + } else if( pkt_size == 0 ) { + ret = CAPTURE_NO_DATA; + } else { + ret = CAPTURE_ERROR; // Socket error + } + break; + } + BF_PRINTD("HERE"); + if( !(*decode)(pkt_ptr, pkt_size, &_pkt) ) { + BF_PRINTD("INVALID " << std::hex << _pkt.sync << " " << std::dec << _pkt.src << " " << _pkt.src << " " << _pkt.time_tag << " " << _pkt.tuning << " " << _pkt.valid_mode); + ++_stats.ninvalid; + _stats.ninvalid_bytes += pkt_size; + continue; + } + BF_PRINTD("VALID " << std::hex << _pkt.sync << " " << std::dec << _pkt.src << " " << _pkt.src << " " << _pkt.time_tag << " " << _pkt.tuning << " " << _pkt.valid_mode); + _have_pkt = true; + } + BF_PRINTD("NOW" << " " << _pkt.seq << " >= " << seq_end); + if( greater_equal(_pkt.seq, seq_end) ) { + // Reached the end of this processing gulp, so leave this + // packet unprocessed and return. + ret = CAPTURE_SUCCESS; + BF_PRINTD("BREAK NOW"); + break; + } + BF_PRINTD("HERE" << " " << _pkt.seq << " < " << seq_beg); + _have_pkt = false; + if( less_than(_pkt.seq, seq_beg) ) { + ++_stats.nlate; + _stats.nlate_bytes += _pkt.payload_size; + ++_src_stats[_pkt.src].nlate; + _src_stats[_pkt.src].nlate_bytes += _pkt.payload_size; + BF_PRINTD("CONTINUE HERE"); + continue; + } + BF_PRINTD("FINALLY"); + ++_stats.nvalid; + _stats.nvalid_bytes += _pkt.payload_size; + ++_src_stats[_pkt.src].nvalid; + _src_stats[_pkt.src].nvalid_bytes += _pkt.payload_size; + // HACK TODO: src_ngood_bytes should be accumulated locally and + // then atomically updated, like ngood_bytes. The + // current way is not thread-safe. + (*process)(&_pkt, seq_beg, nseq_per_obuf, nbuf, obufs, + local_ngood_bytes, /*local_*/src_ngood_bytes); + } + if( nbuf > 0 ) { atomic_add_and_fetch(ngood_bytes[0], local_ngood_bytes[0]); } + if( nbuf > 1 ) { atomic_add_and_fetch(ngood_bytes[1], local_ngood_bytes[1]); } + return ret; +} + +BFstatus bfPacketCaptureCallbackCreate(BFpacketcapture_callback* obj) { + BF_TRY_RETURN_ELSE(*obj = new BFpacketcapture_callback_impl(), + *obj = 0); +} + +BFstatus bfPacketCaptureCallbackDestroy(BFpacketcapture_callback obj) { + BF_ASSERT(obj, BF_STATUS_INVALID_HANDLE); + delete obj; + return BF_STATUS_SUCCESS; +} + +BFstatus bfPacketCaptureCallbackSetSIMPLE(BFpacketcapture_callback obj, + BFpacketcapture_simple_sequence_callback callback) { + BF_ASSERT(obj, BF_STATUS_INVALID_HANDLE); + obj->set_simple(callback); + return BF_STATUS_SUCCESS; +} + +BFstatus bfPacketCaptureCallbackSetCHIPS(BFpacketcapture_callback obj, + BFpacketcapture_chips_sequence_callback callback) { + BF_ASSERT(obj, BF_STATUS_INVALID_HANDLE); + obj->set_chips(callback); + return BF_STATUS_SUCCESS; +} + +BFstatus bfPacketCaptureCallbackSetSNAP2(BFpacketcapture_callback obj, + BFpacketcapture_snap2_sequence_callback callback) { + BF_ASSERT(obj, BF_STATUS_INVALID_HANDLE); + obj->set_snap2(callback); + return BF_STATUS_SUCCESS; +} + +BFstatus bfPacketCaptureCallbackSetIBeam(BFpacketcapture_callback obj, + BFpacketcapture_ibeam_sequence_callback callback) { + BF_ASSERT(obj, BF_STATUS_INVALID_HANDLE); + obj->set_ibeam(callback); + return BF_STATUS_SUCCESS; +} + +BFstatus bfPacketCaptureCallbackSetPBeam(BFpacketcapture_callback obj, + BFpacketcapture_pbeam_sequence_callback callback) { + BF_ASSERT(obj, BF_STATUS_INVALID_HANDLE); + obj->set_pbeam(callback); + return BF_STATUS_SUCCESS; +} + +BFstatus bfPacketCaptureCallbackSetCOR(BFpacketcapture_callback obj, + BFpacketcapture_cor_sequence_callback callback) { + BF_ASSERT(obj, BF_STATUS_INVALID_HANDLE); + obj->set_cor(callback); + return BF_STATUS_SUCCESS; +} + +BFstatus bfPacketCaptureCallbackSetVDIF(BFpacketcapture_callback obj, + BFpacketcapture_vdif_sequence_callback callback) { + BF_ASSERT(obj, BF_STATUS_INVALID_HANDLE); + obj->set_vdif(callback); + return BF_STATUS_SUCCESS; +} + +BFstatus bfPacketCaptureCallbackSetTBN(BFpacketcapture_callback obj, + BFpacketcapture_tbn_sequence_callback callback) { + BF_ASSERT(obj, BF_STATUS_INVALID_HANDLE); + obj->set_tbn(callback); + return BF_STATUS_SUCCESS; +} + +BFstatus bfPacketCaptureCallbackSetDRX(BFpacketcapture_callback obj, + BFpacketcapture_drx_sequence_callback callback) { + BF_ASSERT(obj, BF_STATUS_INVALID_HANDLE); + obj->set_drx(callback); + return BF_STATUS_SUCCESS; +} + +BFstatus bfPacketCaptureCallbackSetDRX8(BFpacketcapture_callback obj, + BFpacketcapture_drx8_sequence_callback callback) { + BF_ASSERT(obj, BF_STATUS_INVALID_HANDLE); + obj->set_drx8(callback); + return BF_STATUS_SUCCESS; +} + +BFpacketcapture_status BFpacketcapture_impl::recv() { + _t0 = std::chrono::high_resolution_clock::now(); + + uint8_t* buf_ptrs[2]; + // Minor HACK to access the buffers in a 2-element queue + buf_ptrs[0] = _bufs.size() > 0 ? (uint8_t*)_bufs.front()->data() : NULL; + buf_ptrs[1] = _bufs.size() > 1 ? (uint8_t*)_bufs.back()->data() : NULL; + + size_t* ngood_bytes_ptrs[2]; + ngood_bytes_ptrs[0] = _buf_ngood_bytes.size() > 0 ? &_buf_ngood_bytes.front() : NULL; + ngood_bytes_ptrs[1] = _buf_ngood_bytes.size() > 1 ? &_buf_ngood_bytes.back() : NULL; + + size_t* src_ngood_bytes_ptrs[2]; + src_ngood_bytes_ptrs[0] = _buf_src_ngood_bytes.size() > 0 ? &_buf_src_ngood_bytes.front()[0] : NULL; + src_ngood_bytes_ptrs[1] = _buf_src_ngood_bytes.size() > 1 ? &_buf_src_ngood_bytes.back()[0] : NULL; + + int state = _capture->run(_seq, + _nseq_per_buf, + _bufs.size(), + buf_ptrs, + ngood_bytes_ptrs, + src_ngood_bytes_ptrs, + _decoder, + _processor); + BF_PRINTD("OUTSIDE"); + if( state & PacketCaptureThread::CAPTURE_ERROR ) { + return BF_CAPTURE_ERROR; + } else if( state & PacketCaptureThread::CAPTURE_INTERRUPTED ) { + return BF_CAPTURE_INTERRUPTED; + } else if( state & PacketCaptureThread::CAPTURE_INTERRUPTED ) { + if( _active ) { + return BF_CAPTURE_ENDED; + } else { + return BF_CAPTURE_NO_DATA; + } + } + const PacketStats* stats = _capture->get_stats(); + _stat_log.update() << "ngood_bytes : " << _ngood_bytes << "\n" + << "nmissing_bytes : " << _nmissing_bytes << "\n" + << "ninvalid : " << stats->ninvalid << "\n" + << "ninvalid_bytes : " << stats->ninvalid_bytes << "\n" + << "nlate : " << stats->nlate << "\n" + << "nlate_bytes : " << stats->nlate_bytes << "\n" + << "nvalid : " << stats->nvalid << "\n" + << "nvalid_bytes : " << stats->nvalid_bytes << "\n"; + + _t1 = std::chrono::high_resolution_clock::now(); + + BFoffset seq0, time_tag=0; + const void* hdr=NULL; + size_t hdr_size=0; + + BFpacketcapture_status ret; + bool was_active = _active; + _active = state & PacketCaptureThread::CAPTURE_SUCCESS; + BF_PRINTD("ACTIVE: " << _active << " WAS ACTIVE: " << was_active); + if( _active ) { + BF_PRINTD("START"); + const PacketDesc* pkt = _capture->get_last_packet(); + BF_PRINTD("PRE-CALL"); + this->on_sequence_active(pkt); + BF_PRINTD("POST-CALL"); + if( !was_active ) { + BF_PRINTD("Beginning of sequence, first pkt seq = " << pkt->seq); + this->on_sequence_start(pkt, &seq0, &time_tag, &hdr, &hdr_size); + this->begin_sequence(seq0, time_tag, hdr, hdr_size); + ret = BF_CAPTURE_STARTED; + } else { + //cout << "Continuing data, seq = " << seq << endl; + if( this->has_sequence_changed(pkt) ) { + this->on_sequence_changed(pkt, &seq0, &time_tag, &hdr, &hdr_size); + this->end_sequence(); + this->begin_sequence(seq0, time_tag, hdr, hdr_size); + ret = BF_CAPTURE_CHANGED; + } else { + ret = BF_CAPTURE_CONTINUED; + } + } + if( _bufs.size() == 2 ) { + this->commit_buf(); + } + this->reserve_buf(); + } else { + + if( was_active ) { + this->flush(); + ret = BF_CAPTURE_ENDED; + } else { + ret = BF_CAPTURE_NO_DATA; + } + } + + _t2 = std::chrono::high_resolution_clock::now(); + _process_time = std::chrono::duration_cast>(_t1-_t0); + _reserve_time = std::chrono::duration_cast>(_t2-_t1); + _perf_log.update() << "acquire_time : " << -1.0 << "\n" + << "process_time : " << _process_time.count() << "\n" + << "reserve_time : " << _reserve_time.count() << "\n"; + + return ret; +} + +BFstatus bfDiskReaderCreate(BFpacketcapture* obj, + const char* format, + int fd, + BFring ring, + BFsize nsrc, + BFsize src0, + BFsize buffer_ntime, + BFsize slot_ntime, + BFpacketcapture_callback sequence_callback, + int core) { + return BFpacketcapture_create(obj, + format, + fd, + ring, + nsrc, + src0, + 9000, + buffer_ntime, + slot_ntime, + sequence_callback, + core, + BF_IO_DISK); +} + +BFstatus bfUdpCaptureCreate(BFpacketcapture* obj, + const char* format, + int fd, + BFring ring, + BFsize nsrc, + BFsize src0, + BFsize max_payload_size, + BFsize buffer_ntime, + BFsize slot_ntime, + BFpacketcapture_callback sequence_callback, + int core) { + return BFpacketcapture_create(obj, + format, + fd, + ring, + nsrc, + src0, + max_payload_size, + buffer_ntime, + slot_ntime, + sequence_callback, + core, + BF_IO_UDP); +} + +BFstatus bfUdpSnifferCreate(BFpacketcapture* obj, + const char* format, + int fd, + BFring ring, + BFsize nsrc, + BFsize src0, + BFsize max_payload_size, + BFsize buffer_ntime, + BFsize slot_ntime, + BFpacketcapture_callback sequence_callback, + int core) { + return BFpacketcapture_create(obj, + format, + fd, + ring, + nsrc, + src0, + max_payload_size, + buffer_ntime, + slot_ntime, + sequence_callback, + core, + BF_IO_SNIFFER); +} + +BFstatus bfUdpVerbsCaptureCreate(BFpacketcapture* obj, + const char* format, + int fd, + BFring ring, + BFsize nsrc, + BFsize src0, + BFsize max_payload_size, + BFsize buffer_ntime, + BFsize slot_ntime, + BFpacketcapture_callback sequence_callback, + int core) { + return BFpacketcapture_create(obj, + format, + fd, + ring, + nsrc, + src0, + max_payload_size, + buffer_ntime, + slot_ntime, + sequence_callback, + core, + BF_IO_VERBS); +} + +BFstatus bfPacketCaptureDestroy(BFpacketcapture obj) { + BF_ASSERT(obj, BF_STATUS_INVALID_HANDLE); + delete obj; + return BF_STATUS_SUCCESS; +} + +BFstatus bfPacketCaptureRecv(BFpacketcapture obj, + BFpacketcapture_status* result) { + BF_ASSERT(obj, BF_STATUS_INVALID_HANDLE); + BF_TRY_RETURN_ELSE(*result = obj->recv(), + *result = BF_CAPTURE_ERROR); +} + +BFstatus bfPacketCaptureFlush(BFpacketcapture obj) { + BF_ASSERT(obj, BF_STATUS_INVALID_HANDLE); + BF_TRY_RETURN(obj->flush()); +} + +BFstatus bfPacketCaptureSeek(BFpacketcapture obj, BFoffset offset, BFiowhence whence, BFoffset* position) { + BF_ASSERT(obj, BF_STATUS_INVALID_HANDLE); + BF_TRY_RETURN(*position = obj->seek(offset, whence)); +} + +BFstatus bfPacketCaptureTell(BFpacketcapture obj, BFoffset* position) { + BF_ASSERT(obj, BF_STATUS_INVALID_HANDLE); + BF_TRY_RETURN(*position = obj->tell()); +} + +BFstatus bfPacketCaptureEnd(BFpacketcapture obj) { + BF_ASSERT(obj, BF_STATUS_INVALID_HANDLE); + BF_TRY_RETURN(obj->end_writing()); +} diff --git a/src/packet_capture.hpp b/src/packet_capture.hpp new file mode 100644 index 000000000..d077485c4 --- /dev/null +++ b/src/packet_capture.hpp @@ -0,0 +1,1522 @@ +/* + * Copyright (c) 2019-2024, The Bifrost Authors. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of The Bifrost Authors nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#pragma once + +#include "assert.hpp" +#include +#include +#include +using bifrost::ring::RingWrapper; +using bifrost::ring::RingWriter; +using bifrost::ring::WriteSpan; +using bifrost::ring::WriteSequence; +#include "proclog.hpp" +#include "Socket.hpp" +#include "formats/formats.hpp" +#include "hw_locality.hpp" + +#include // For ntohs +#include // For recvfrom + +#include +#include +#include +#include // For posix_memalign +#include // For memcpy, memset +#include + +#include +#include +#include +#include + +#if defined __APPLE__ && __APPLE__ + +#define lseek64 lseek + +#endif + +#ifndef BF_HWLOC_ENABLED +#define BF_HWLOC_ENABLED 0 +//#define BF_HWLOC_ENABLED 1 +#endif + +template +inline T atomic_add_and_fetch(T* dst, T val) { + return __sync_add_and_fetch(dst, val); // GCC builtin +} +template +inline T atomic_fetch_and_add(T* dst, T val) { + return __sync_fetch_and_add(dst, val); // GCC builtin +} + +// Wrap-safe comparisons +inline bool greater_equal(uint64_t a, uint64_t b) { return int64_t(a-b) >= 0; } +inline bool less_than( uint64_t a, uint64_t b) { return int64_t(a-b) < 0; } + +template +class AlignedBuffer { + enum { DEFAULT_ALIGNMENT = 4096 }; + T* _buf; + size_t _size; + size_t _alignment; + void alloc() { + int err = ::posix_memalign((void**)&_buf, _alignment, _size*sizeof(T)); + if( err ) { + throw std::runtime_error("Allocation failed"); + } + } + void copy(T const* srcbuf, size_t n) { + ::memcpy(_buf, srcbuf, n*sizeof(T)); + } + void free() { + if( _buf ) { + ::free(_buf); + _buf = 0; + _size = 0; + } + } +public: + //explicit AlignedBuffer(size_t alignment=DEFAULT_ALIGNMENT) + // : _buf(0), _size(0), _alignment(alignment) {} + AlignedBuffer(size_t size=0, size_t alignment=DEFAULT_ALIGNMENT) + : _buf(0), _size(size), _alignment(alignment) { + this->alloc(); + } + AlignedBuffer(AlignedBuffer const& other) + : _buf(0), _size(other.size), _alignment(other.alignment) { + this->alloc(); + this->copy(other._buf, other._size); + } + AlignedBuffer& operator=(AlignedBuffer const& other) { + if( &other != this ) { + this->free(); + _size = other.size; + _alignment = other.alignment; + this->alloc(); + this->copy(other._buf, other._size); + } + return *this; + } + ~AlignedBuffer() { + this->free(); + } + inline void swap(AlignedBuffer & other) { + std::swap(_buf, other._buf); + std::swap(_size, other._size); + std::swap(_alignment, other._alignment); + } + inline void resize(size_t n) { + if( n <= _size ) { + _size = n; + } else { + AlignedBuffer tmp(n, _alignment); + tmp.copy(&_buf[0], _size); + tmp.swap(*this); + } + } + inline size_t size() const { return _size; } + inline size_t alignment() const { return _alignment; } + inline T & operator[](size_t i) { return _buf[i]; } + inline T const& operator[](size_t i) const { return _buf[i]; } +}; + +class PacketCaptureMethod { +protected: + int _fd; + size_t _pkt_size_max; + AlignedBuffer _buf; + BFiomethod _io_method; +public: + PacketCaptureMethod(int fd, size_t pkt_size_max=9000, BFiomethod io_method=BF_IO_GENERIC) + : _fd(fd), _pkt_size_max(pkt_size_max), _buf(pkt_size_max), _io_method(io_method) {} + virtual int recv_packet(uint8_t** pkt_ptr, int flags=0) { + return 0; + } + virtual const char* get_name() { return "generic_capture"; } + inline const size_t get_max_size() { return _pkt_size_max; } + inline const BFiomethod get_io_method() { return _io_method; } + virtual inline BFoffset seek(BFoffset offset, BFiowhence whence=BF_WHENCE_CUR) { return 0; } + inline BFoffset tell() { return this->seek(0, BF_WHENCE_CUR); } +}; + +class DiskPacketReader : public PacketCaptureMethod { +public: + DiskPacketReader(int fd, size_t pkt_size_max=9000) + : PacketCaptureMethod(fd, pkt_size_max, BF_IO_DISK) {} + int recv_packet(uint8_t** pkt_ptr, int flags=0) { + *pkt_ptr = &_buf[0]; + return ::read(_fd, &_buf[0], _buf.size()); + } + inline const char* get_name() { return "disk_reader"; } + inline BFoffset seek(BFoffset offset, BFiowhence whence=BF_WHENCE_CUR) { + return ::lseek64(_fd, offset, whence); + } +}; + +// TODO: The VMA API is returning unaligned buffers, which prevents use of SSE +#ifndef BF_VMA_ENABLED +#define BF_VMA_ENABLED 0 +#endif + +#if BF_VMA_ENABLED +#include +class VMAReceiver { + int _fd; + vma_api_t* _api; + vma_packet_t* _pkt; + inline void clean_cache() { + if( _pkt ) { + _api->free_packets(_fd, _pkt, 1); + _pkt = 0; + } + } +public: + VMAReceiver(int fd) + : _fd(fd), _api(vma_get_api()), _pkt(0) {} + VMAReceiver(VMAReceiver const& other) + : _fd(other._fd), _api(other._api), _pkt(0) {} + VMAReceiver& operator=(VMAReceiver const& other) { + if( &other != this ) { + this->clean_cache(); + _fd = other._fd; + _api = other._api; + } + return *this; + } + ~VMAReceiver() { this->clean_cache(); } + inline int recv_packet(uint8_t* buf, size_t bufsize, uint8_t** pkt_ptr, int flags=0) { + this->clean_cache(); + int ret = _api->recvfrom_zcopy(_fd, buf, bufsize, &flags, 0, 0); + if( ret < 0 ) { + return ret; + } + if( flags & MSG_VMA_ZCOPY ) { + _pkt = &((vma_packets_t*)buf)->pkts[0]; + *pkt_ptr = (uint8_t*)_pkt->iov[0].iov_base; + } else { + *pkt_ptr = buf; + } + return ret; + } + inline operator bool() const { return _api != NULL; } +}; +#endif // BF_VMA_ENABLED + +class UDPPacketReceiver : public PacketCaptureMethod { +#if BF_VMA_ENABLED + VMAReceiver _vma; +#endif +public: + UDPPacketReceiver(int fd, size_t pkt_size_max=JUMBO_FRAME_SIZE) + : PacketCaptureMethod(fd, pkt_size_max, BF_IO_UDP) +#if BF_VMA_ENABLED + , _vma(fd) +#endif + {} + inline int recv_packet(uint8_t** pkt_ptr, int flags=0) { + +#if BF_VMA_ENABLED + if( _vma ) { + *pkt_ptr = 0; + return _vma.recv_packet(&_buf[0], _buf.size(), pkt_ptr, flags); + } else { +#endif + *pkt_ptr = &_buf[0]; + return ::recvfrom(_fd, &_buf[0], _buf.size(), flags, 0, 0); +#if BF_VMA_ENABLED + } +#endif + } + inline const char* get_name() { return "udp_capture"; } +}; + +class UDPPacketSniffer : public PacketCaptureMethod { +#if BF_VMA_ENABLED + VMAReceiver _vma; +#endif +public: + UDPPacketSniffer(int fd, size_t pkt_size_max=JUMBO_FRAME_SIZE) + : PacketCaptureMethod(fd, pkt_size_max, BF_IO_SNIFFER) +#if BF_VMA_ENABLED + , _vma(fd) +#endif + {} + inline int recv_packet(uint8_t** pkt_ptr, int flags=0) { +#if BF_VMA_ENABLED + if( _vma ) { + *pkt_ptr = 0; + int rc = _vma.recv_packet(&_buf[0], _buf.size(), pkt_ptr, flags) - 28; + *pkt_ptr = *(pkt_ptr + 28); + return rc; + } else { +#endif + *pkt_ptr = &_buf[28]; // Offset for the IP+UDP headers + return ::recvfrom(_fd, &_buf[0], _buf.size(), flags, 0, 0) - 28; +#if BF_VMA_ENABLED + } +#endif + } + inline const char* get_name() { return "udp_sniffer"; } +}; + +#if defined BF_VERBS_ENABLED && BF_VERBS_ENABLED +#include "ib_verbs.hpp" + +class UDPVerbsReceiver : public PacketCaptureMethod { + Verbs _ibv; +public: + UDPVerbsReceiver(int fd, size_t pkt_size_max=JUMBO_FRAME_SIZE) + : PacketCaptureMethod(fd, pkt_size_max, BF_IO_VERBS), _ibv(fd, pkt_size_max) {} + inline int recv_packet(uint8_t** pkt_ptr, int flags=0) { + *pkt_ptr = 0; + return _ibv.recv_packet(pkt_ptr, flags); + } + inline const char* get_name() { return "udp_verbs_capture"; } +}; +#endif // BF_VERBS_ENABLED + +struct PacketStats { + size_t ninvalid; + size_t ninvalid_bytes; + size_t nlate; + size_t nlate_bytes; + size_t nvalid; + size_t nvalid_bytes; +}; + +class PacketCaptureThread : public BoundThread { +private: + PacketCaptureMethod* _method; + PacketStats _stats; + std::vector _src_stats; + bool _have_pkt; + PacketDesc _pkt; + int _core; + +public: + enum { + CAPTURE_SUCCESS = 1 << 0, + CAPTURE_TIMEOUT = 1 << 1, + CAPTURE_INTERRUPTED = 1 << 2, + CAPTURE_NO_DATA = 1 << 3, + CAPTURE_ERROR = 1 << 4 + }; + PacketCaptureThread(PacketCaptureMethod* method, int nsrc, int core=0) + : BoundThread(core), _method(method), _src_stats(nsrc), + _have_pkt(false), _core(core) { + this->reset_stats(); + } + template + int run(uint64_t seq_beg, + uint64_t nseq_per_obuf, + int nbuf, + uint8_t* obufs[], + size_t* ngood_bytes[], + size_t* src_ngood_bytes[], + PDC* decode, + PPC* process); + inline const char* get_name() { return _method->get_name(); } + inline const size_t get_max_size() { return _method->get_max_size(); } + inline const BFiomethod get_io_method() { return _method->get_io_method(); } + inline const int get_core() { return _core; } + inline BFoffset seek(BFoffset offset, BFiowhence whence=BF_WHENCE_CUR) { return _method->seek(offset, whence); } + inline BFoffset tell() { return _method->tell(); } + inline const PacketDesc* get_last_packet() const { + return _have_pkt ? &_pkt : NULL; + } + inline void reset_last_packet() { + _have_pkt = false; + } + inline const PacketStats* get_stats() const { return &_stats; } + inline const PacketStats* get_stats(int src) const { return &_src_stats[src]; } + inline void reset_stats() { + ::memset(&_stats, 0, sizeof(_stats)); + ::memset(&_src_stats[0], 0, _src_stats.size()*sizeof(PacketStats)); + } +}; + +inline uint64_t round_up(uint64_t val, uint64_t mult) { + return (val == 0 ? + 0 : + ((val-1)/mult+1)*mult); +} +inline uint64_t round_nearest(uint64_t val, uint64_t mult) { + return (2*val/mult+1)/2*mult; +} + +class BFpacketcapture_callback_impl { + BFpacketcapture_simple_sequence_callback _simple_callback; + BFpacketcapture_chips_sequence_callback _chips_callback; + BFpacketcapture_snap2_sequence_callback _snap2_callback; + BFpacketcapture_ibeam_sequence_callback _ibeam_callback; + BFpacketcapture_pbeam_sequence_callback _pbeam_callback; + BFpacketcapture_cor_sequence_callback _cor_callback; + BFpacketcapture_vdif_sequence_callback _vdif_callback; + BFpacketcapture_tbn_sequence_callback _tbn_callback; + BFpacketcapture_drx_sequence_callback _drx_callback; + BFpacketcapture_drx8_sequence_callback _drx8_callback; +public: + BFpacketcapture_callback_impl() + : _simple_callback(NULL), _chips_callback(NULL), _ibeam_callback(NULL), + _pbeam_callback(NULL), _cor_callback(NULL), _vdif_callback(NULL), + _tbn_callback(NULL), _drx_callback(NULL), _drx8_callback(NULL), + _snap2_callback(NULL) {} + inline void set_simple(BFpacketcapture_simple_sequence_callback callback) { + _simple_callback = callback; + } + inline BFpacketcapture_simple_sequence_callback get_simple() { + return _simple_callback; + } + inline void set_chips(BFpacketcapture_chips_sequence_callback callback) { + _chips_callback = callback; + } + inline BFpacketcapture_chips_sequence_callback get_chips() { + return _chips_callback; + } + inline void set_snap2(BFpacketcapture_snap2_sequence_callback callback) { + _snap2_callback = callback; + } + inline BFpacketcapture_snap2_sequence_callback get_snap2() { + return _snap2_callback; + } + inline void set_ibeam(BFpacketcapture_ibeam_sequence_callback callback) { + _ibeam_callback = callback; + } + inline BFpacketcapture_ibeam_sequence_callback get_ibeam() { + return _ibeam_callback; + } + inline void set_pbeam(BFpacketcapture_pbeam_sequence_callback callback) { + _pbeam_callback = callback; + } + inline BFpacketcapture_pbeam_sequence_callback get_pbeam() { + return _pbeam_callback; + } + inline void set_cor(BFpacketcapture_cor_sequence_callback callback) { + _cor_callback = callback; + } + inline BFpacketcapture_cor_sequence_callback get_cor() { + return _cor_callback; + } + inline void set_vdif(BFpacketcapture_vdif_sequence_callback callback) { + _vdif_callback = callback; + } + inline BFpacketcapture_vdif_sequence_callback get_vdif() { + return _vdif_callback; + } + inline void set_tbn(BFpacketcapture_tbn_sequence_callback callback) { + _tbn_callback = callback; + } + inline BFpacketcapture_tbn_sequence_callback get_tbn() { + return _tbn_callback; + } + inline void set_drx(BFpacketcapture_drx_sequence_callback callback) { + _drx_callback = callback; + } + inline BFpacketcapture_drx_sequence_callback get_drx() { + return _drx_callback; + } + inline void set_drx8(BFpacketcapture_drx8_sequence_callback callback) { + _drx8_callback = callback; + } + inline BFpacketcapture_drx8_sequence_callback get_drx8() { + return _drx8_callback; + } +}; + +class BFpacketcapture_impl { +protected: + std::string _name; + PacketCaptureThread* _capture; + PacketDecoder* _decoder; + PacketProcessor* _processor; + ProcLog _bind_log; + ProcLog _out_log; + ProcLog _size_log; + ProcLog _stat_log; + ProcLog _perf_log; + pid_t _pid; + + int _nsrc; + int _nseq_per_buf; + int _slot_ntime; + BFoffset _seq; + int _chan0; + int _nchan; + int _payload_size; + bool _active; + +private: + std::chrono::high_resolution_clock::time_point _t0; + std::chrono::high_resolution_clock::time_point _t1; + std::chrono::high_resolution_clock::time_point _t2; + std::chrono::duration _process_time; + std::chrono::duration _reserve_time; + + RingWrapper _ring; + RingWriter _oring; + std::queue > _bufs; + std::queue _buf_ngood_bytes; + std::queue > _buf_src_ngood_bytes; + std::shared_ptr _sequence; + size_t _ngood_bytes; + size_t _nmissing_bytes; + + inline size_t bufsize(int payload_size=-1) { + if( payload_size == -1 ) { + payload_size = _payload_size; + } + return _nseq_per_buf * _nsrc * payload_size * BF_UNPACK_FACTOR; + } + inline void reserve_buf() { + _buf_ngood_bytes.push(0); + _buf_src_ngood_bytes.push(std::vector(_nsrc, 0)); + size_t size = this->bufsize(); + // TODO: Can make this simpler? + _bufs.push(std::shared_ptr(new bifrost::ring::WriteSpan(_oring, size))); + } + inline void commit_buf() { + size_t expected_bytes = _bufs.front()->size(); + + for( int src=0; src<_nsrc; ++src ) { + // TODO: This assumes all sources contribute equally; should really + // allow non-uniform partitioning. + size_t src_expected_bytes = expected_bytes / _nsrc; + size_t src_ngood_bytes = _buf_src_ngood_bytes.front()[src]; + size_t src_nmissing_bytes = src_expected_bytes - src_ngood_bytes; + // Detect >50% missing data from this source + if( src_nmissing_bytes > src_ngood_bytes ) { + // Zero-out this source's contribution to the buffer + uint8_t* data = (uint8_t*)_bufs.front()->data(); + _processor->blank_out_source(data, src, _nsrc, + _nchan, _nseq_per_buf); + } + } + _buf_src_ngood_bytes.pop(); + + _ngood_bytes += _buf_ngood_bytes.front(); + //_nmissing_bytes += _bufs.front()->size() - _buf_ngood_bytes.front(); + //// HACK TESTING 15/16 correction for missing roach11 + //_nmissing_bytes += _bufs.front()->size()*15/16 - _buf_ngood_bytes.front(); + _nmissing_bytes += expected_bytes - _buf_ngood_bytes.front(); + _buf_ngood_bytes.pop(); + + _bufs.front()->commit(); + _bufs.pop(); + _seq += _nseq_per_buf; + } + inline void begin_sequence(BFoffset seq0, BFoffset time_tag, const void* hdr, size_t hdr_size) { + const char* name = ""; + int nringlet = 1; + _sequence.reset(new WriteSequence(_oring, name, time_tag, + hdr_size, hdr, nringlet)); + } + inline void end_sequence() { + _sequence.reset(); // Note: This is releasing the shared_ptr + } + virtual void on_sequence_start(const PacketDesc* pkt, BFoffset* seq0, BFoffset* time_tag, const void** hdr, size_t* hdr_size) {} + virtual void on_sequence_active(const PacketDesc* pkt) {} + virtual inline bool has_sequence_changed(const PacketDesc* pkt) { return false; } + virtual void on_sequence_changed(const PacketDesc* pkt, BFoffset* seq0, BFoffset* time_tag, const void** hdr, size_t* hdr_size) {} +public: + inline BFpacketcapture_impl(PacketCaptureThread* capture, + PacketDecoder* decoder, + PacketProcessor* processor, + BFring ring, + int nsrc, + int buffer_ntime, + int slot_ntime) + : _name(capture->get_name()), _capture(capture), _decoder(decoder), _processor(processor), + _bind_log(_name+"/bind"), + _out_log(_name+"/out"), + _size_log(_name+"/sizes"), + _stat_log(_name+"/stats"), + _perf_log(_name+"/perf"), + _nsrc(nsrc), _nseq_per_buf(buffer_ntime), _slot_ntime(slot_ntime), + _seq(), _chan0(), _nchan(), _active(false), + _ring(ring), _oring(_ring), + // TODO: Add reset method for stats + _ngood_bytes(0), _nmissing_bytes(0) { + size_t contig_span = this->bufsize(_capture->get_max_size()); + // Note: 2 write bufs may be open for writing at one time + size_t total_span = contig_span * 4; + size_t nringlet_max = 1; + _ring.resize(contig_span, total_span, nringlet_max); + _bind_log.update("ncore : %i\n" + "core0 : %i\n", + 1, _capture->get_core()); + _out_log.update("nring : %i\n" + "ring0 : %s\n", + 1, _ring.name()); + _size_log.update("nsrc : %i\n" + "nseq_per_buf : %i\n" + "slot_ntime : %i\n", + _nsrc, _nseq_per_buf, _slot_ntime); + } + virtual ~BFpacketcapture_impl() {} + inline void flush() { + while( _bufs.size() ) { + this->commit_buf(); + } + if( _sequence ) { + this->end_sequence(); + } + } + inline BFoffset seek(BFoffset offset, BFiowhence whence=BF_WHENCE_CUR) { + BF_ASSERT(_capture->get_io_method() == BF_IO_DISK, BF_STATUS_UNSUPPORTED); + BFoffset moved = _capture->seek(offset, whence); + this->flush(); + return moved; + } + inline BFoffset tell() { + BF_ASSERT(_capture->get_io_method() == BF_IO_DISK, BF_STATUS_UNSUPPORTED); + return _capture->tell(); + } + inline void end_writing() { + this->flush(); + _oring.close(); + } + BFpacketcapture_status recv(); +}; + +class BFpacketcapture_simple_impl : public BFpacketcapture_impl { + ProcLog _type_log; + ProcLog _chan_log; + + BFpacketcapture_simple_sequence_callback _sequence_callback; + + void on_sequence_start(const PacketDesc* pkt, BFoffset* seq0, BFoffset* time_tag, const void** hdr, size_t* hdr_size ) { + // TODO: Might be safer to round to nearest here, but the current firmware + // always starts things ~3 seq's before the 1sec boundary anyway. + //seq = round_up(pkt->seq, _slot_ntime); + //*_seq = round_nearest(pkt->seq, _slot_ntime); + _seq = round_up(pkt->seq, _slot_ntime); + this->on_sequence_changed(pkt, seq0, time_tag, hdr, hdr_size); + } + void on_sequence_active(const PacketDesc* pkt) { + } + inline bool has_sequence_changed(const PacketDesc* pkt) { + return 0; + } + void on_sequence_changed(const PacketDesc* pkt, BFoffset* seq0, BFoffset* time_tag, const void** hdr, size_t* hdr_size) { + *seq0 = _seq;// + _nseq_per_buf*_bufs.size(); + _payload_size = pkt->payload_size; + + if( _sequence_callback ) { + int status = (*_sequence_callback)(*seq0, + _chan0, + _nchan, + _nsrc, + time_tag, + hdr, + hdr_size); + if( status != 0 ) { + // TODO: What to do here? Needed? + throw std::runtime_error("BAD HEADER CALLBACK STATUS"); + } + } else { + // Simple default for easy testing + *time_tag = *seq0; + *hdr = NULL; + *hdr_size = 0; + } + + } +public: + inline BFpacketcapture_simple_impl(PacketCaptureThread* capture, + BFring ring, + int nsrc, + int src0, + int buffer_ntime, + int slot_ntime, + BFpacketcapture_callback sequence_callback) + : BFpacketcapture_impl(capture, nullptr, nullptr, ring, nsrc, buffer_ntime, slot_ntime), + _type_log((std::string(capture->get_name())+"/type").c_str()), + _chan_log((std::string(capture->get_name())+"/chans").c_str()), + _sequence_callback(sequence_callback->get_simple()) { + _decoder = new SIMPLEDecoder(nsrc, src0); + _processor = new SIMPLEProcessor(); + _type_log.update("type : %s\n", "simple"); + } +}; +class BFpacketcapture_chips_impl : public BFpacketcapture_impl { + ProcLog _type_log; + ProcLog _chan_log; + + BFpacketcapture_chips_sequence_callback _sequence_callback; + + void on_sequence_start(const PacketDesc* pkt, BFoffset* seq0, BFoffset* time_tag, const void** hdr, size_t* hdr_size ) { + // TODO: Might be safer to round to nearest here, but the current firmware + // always starts things ~3 seq's before the 1sec boundary anyway. + //seq = round_up(pkt->seq, _slot_ntime); + //*_seq = round_nearest(pkt->seq, _slot_ntime); + _seq = round_up(pkt->seq, _slot_ntime); + this->on_sequence_changed(pkt, seq0, time_tag, hdr, hdr_size); + } + void on_sequence_active(const PacketDesc* pkt) { + if( pkt ) { + //cout << "Latest nchan, chan0 = " << pkt->nchan << ", " << pkt->chan0 << endl; + } + else { + //cout << "No latest packet" << endl; + } + } + inline bool has_sequence_changed(const PacketDesc* pkt) { + return (pkt->chan0 != _chan0) \ + || (pkt->nchan != _nchan); + } + void on_sequence_changed(const PacketDesc* pkt, BFoffset* seq0, BFoffset* time_tag, const void** hdr, size_t* hdr_size) { + *seq0 = _seq;// + _nseq_per_buf*_bufs.size(); + _chan0 = pkt->chan0; + _nchan = pkt->nchan; + _payload_size = pkt->payload_size; + + if( _sequence_callback ) { + int status = (*_sequence_callback)(*seq0, + _chan0, + _nchan, + _nsrc, + time_tag, + hdr, + hdr_size); + if( status != 0 ) { + // TODO: What to do here? Needed? + throw std::runtime_error("BAD HEADER CALLBACK STATUS"); + } + } else { + // Simple default for easy testing + *time_tag = *seq0; + *hdr = NULL; + *hdr_size = 0; + } + + _chan_log.update() << "chan0 : " << _chan0 << "\n" + << "nchan : " << _nchan << "\n" + << "payload_size : " << _payload_size << "\n"; + } +public: + inline BFpacketcapture_chips_impl(PacketCaptureThread* capture, + BFring ring, + int nsrc, + int src0, + int buffer_ntime, + int slot_ntime, + BFpacketcapture_callback sequence_callback) + : BFpacketcapture_impl(capture, nullptr, nullptr, ring, nsrc, buffer_ntime, slot_ntime), + _type_log((std::string(capture->get_name())+"/type").c_str()), + _chan_log((std::string(capture->get_name())+"/chans").c_str()), + _sequence_callback(sequence_callback->get_chips()) { + _decoder = new CHIPSDecoder(nsrc, src0); + _processor = new CHIPSProcessor(); + _type_log.update("type : %s\n", "chips"); + } +}; + +class BFpacketcapture_snap2_impl : public BFpacketcapture_impl { + ProcLog _type_log; + ProcLog _chan_log; + BFoffset _last_seq; + BFoffset _last_time_tag; + + BFpacketcapture_snap2_sequence_callback _sequence_callback; + + void on_sequence_start(const PacketDesc* pkt, BFoffset* seq0, BFoffset* time_tag, const void** hdr, size_t* hdr_size ) { + this->on_sequence_changed(pkt, seq0, time_tag, hdr, hdr_size); + } + void on_sequence_active(const PacketDesc* pkt) { + if( pkt ) { + //cout << "Latest nchan, chan0 = " << pkt->nchan << ", " << pkt->chan0 << endl; + } + else { + //cout << "No latest packet" << endl; + } + } + // Has the configuration changed? I.e., different channels being sent. + inline bool has_sequence_changed(const PacketDesc* pkt) { + // TODO: Decide what a sequence actually is! + // Currently a new sequence starts whenever a block finishes and the next + // packet isn't from the next block + // TODO. Is this actually reasonable? Does it recover from upstream resyncs? + bool is_new_seq = false; + if ( (_last_time_tag != pkt->time_tag) || (pkt->seq != _last_seq + _nseq_per_buf) ) { + // We could have a packet sequence number which isn't what we expect + // but is only wrong because of missing packets. Set an upper bound on + // two slots of loss + if (pkt->seq > _last_seq + 2*_slot_ntime) { + is_new_seq = true; + this->flush(); + } + } + _last_seq = pkt->seq; + return is_new_seq; + } + void on_sequence_changed(const PacketDesc* pkt, BFoffset* seq0, BFoffset* time_tag, const void** hdr, size_t* hdr_size) { + _seq = round_up(pkt->seq, _slot_ntime); + *time_tag = (BFoffset) pkt->time_tag; + _last_time_tag = pkt->time_tag; + _last_seq = _seq; + *seq0 = _seq;// + _nseq_per_buf*_bufs.size(); + _chan0 = pkt->chan0; + _nchan = pkt->nchan; + _payload_size = pkt->payload_size; + + if( _sequence_callback ) { + int status = (*_sequence_callback)(*seq0, + pkt->tuning, // Hacked to contain chan0 + _nchan, + _nsrc, + time_tag, + hdr, + hdr_size); + if( status != 0 ) { + // TODO: What to do here? Needed? + throw std::runtime_error("BAD HEADER CALLBACK STATUS"); + } + } else { + // Simple default for easy testing + *time_tag = *seq0; + *hdr = NULL; + *hdr_size = 0; + } + + _chan_log.update() << "chan0 : " << _chan0 << "\n" + << "nchan : " << _nchan << "\n" + << "payload_size : " << _payload_size << "\n"; + } +public: + inline BFpacketcapture_snap2_impl(PacketCaptureThread* capture, + BFring ring, + int nsrc, + int src0, + int buffer_ntime, + int slot_ntime, + BFpacketcapture_callback sequence_callback) + : BFpacketcapture_impl(capture, nullptr, nullptr, ring, nsrc, buffer_ntime, slot_ntime), + _type_log((std::string(capture->get_name())+"/type").c_str()), + _chan_log((std::string(capture->get_name())+"/chans").c_str()), + _sequence_callback(sequence_callback->get_snap2()) { + _decoder = new SNAP2Decoder(nsrc, src0); + _processor = new SNAP2Processor(); + _type_log.update("type : %s\n", "snap2"); + } +}; + +template +class BFpacketcapture_ibeam_impl : public BFpacketcapture_impl { + uint8_t _nbeam = B; + ProcLog _type_log; + ProcLog _chan_log; + + BFpacketcapture_ibeam_sequence_callback _sequence_callback; + + void on_sequence_start(const PacketDesc* pkt, BFoffset* seq0, BFoffset* time_tag, const void** hdr, size_t* hdr_size ) { + // TODO: Might be safer to round to nearest here, but the current firmware + // always starts things ~3 seq's before the 1sec boundary anyway. + //seq = round_up(pkt->seq, _slot_ntime); + //*_seq = round_nearest(pkt->seq, _slot_ntime); + _seq = round_up(pkt->seq, _slot_ntime); + this->on_sequence_changed(pkt, seq0, time_tag, hdr, hdr_size); + } + void on_sequence_active(const PacketDesc* pkt) { + if( pkt ) { + //cout << "Latest nchan, chan0 = " << pkt->nchan << ", " << pkt->chan0 << endl; + } + else { + //cout << "No latest packet" << endl; + } + } + inline bool has_sequence_changed(const PacketDesc* pkt) { + return (pkt->chan0 != _chan0) \ + || (pkt->nchan != _nchan); + } + void on_sequence_changed(const PacketDesc* pkt, BFoffset* seq0, BFoffset* time_tag, const void** hdr, size_t* hdr_size) { + *seq0 = _seq;// + _nseq_per_buf*_bufs.size(); + _chan0 = pkt->chan0; + _nchan = pkt->nchan; + _payload_size = pkt->payload_size; + + if( _sequence_callback ) { + int status = (*_sequence_callback)(*seq0, + _chan0, + _nchan*_nsrc, + _nbeam, + time_tag, + hdr, + hdr_size); + if( status != 0 ) { + // TODO: What to do here? Needed? + throw std::runtime_error("BAD HEADER CALLBACK STATUS"); + } + } else { + // Simple default for easy testing + *time_tag = *seq0; + *hdr = NULL; + *hdr_size = 0; + } + + _chan_log.update() << "chan0 : " << _chan0 << "\n" + << "nchan : " << _nchan << "\n" + << "payload_size : " << _payload_size << "\n"; + } +public: + inline BFpacketcapture_ibeam_impl(PacketCaptureThread* capture, + BFring ring, + int nsrc, + int src0, + int buffer_ntime, + int slot_ntime, + BFpacketcapture_callback sequence_callback) + : BFpacketcapture_impl(capture, nullptr, nullptr, ring, nsrc, buffer_ntime, slot_ntime), + _type_log((std::string(capture->get_name())+"/type").c_str()), + _chan_log((std::string(capture->get_name())+"/chans").c_str()), + _sequence_callback(sequence_callback->get_ibeam()) { + _decoder = new IBeamDecoder(nsrc, src0); + _processor = new IBeamProcessor(); + _type_log.update("type : %s%i\n", "ibeam", _nbeam); + } +}; + +class BFpacketcapture_pbeam_impl : public BFpacketcapture_impl { + uint8_t _nbeam; + uint8_t _navg; + ProcLog _type_log; + ProcLog _chan_log; + + BFpacketcapture_pbeam_sequence_callback _sequence_callback; + + void on_sequence_start(const PacketDesc* pkt, BFoffset* seq0, BFoffset* time_tag, const void** hdr, size_t* hdr_size ) { + // TODO: Might be safer to round to nearest here, but the current firmware + // always starts things ~3 seq's before the 1sec boundary anyway. + //seq = round_up(pkt->seq, _slot_ntime); + _seq = round_nearest(pkt->seq, _slot_ntime); + this->on_sequence_changed(pkt, seq0, time_tag, hdr, hdr_size); + } + void on_sequence_active(const PacketDesc* pkt) { + if( pkt ) { + //cout << "Latest nchan, chan0 = " << pkt->nchan << ", " << pkt->chan0 << endl; + } + else { + //cout << "No latest packet" << endl; + } + } + inline bool has_sequence_changed(const PacketDesc* pkt) { + return (pkt->chan0 != _chan0) \ + || (pkt->nchan != _nchan) \ + || (pkt->decimation != _navg); + } + void on_sequence_changed(const PacketDesc* pkt, BFoffset* seq0, BFoffset* time_tag, const void** hdr, size_t* hdr_size) { + *seq0 = _seq;// + _nseq_per_buf*_bufs.size(); + *time_tag = pkt->time_tag; + _chan0 = pkt->chan0; + _nchan = pkt->nchan; + _nbeam = pkt->beam; + _navg = pkt->decimation; + _payload_size = pkt->payload_size; + + if( _sequence_callback ) { + int status = (*_sequence_callback)(*seq0, + *time_tag, + _navg, + _chan0, + _nchan*_nsrc/_nbeam, + _nbeam, + hdr, + hdr_size); + if( status != 0 ) { + // TODO: What to do here? Needed? + throw std::runtime_error("BAD HEADER CALLBACK STATUS"); + } + } else { + // Simple default for easy testing + *time_tag = *seq0; + *hdr = NULL; + *hdr_size = 0; + } + + _chan_log.update() << "chan0 : " << _chan0 << "\n" + << "nchan : " << _nchan << "\n" + << "nbeam : " << _nbeam << "\n" + << "navg : " << _navg << "\n" + << "payload_size : " << _payload_size << "\n"; + } +public: + inline BFpacketcapture_pbeam_impl(PacketCaptureThread* capture, + BFring ring, + int nsrc, + int src0, + int buffer_ntime, + int slot_ntime, + BFpacketcapture_callback sequence_callback) + : BFpacketcapture_impl(capture, nullptr, nullptr, ring, nsrc, buffer_ntime, slot_ntime), + _type_log((std::string(capture->get_name())+"/type").c_str()), + _chan_log((std::string(capture->get_name())+"/chans").c_str()), + _sequence_callback(sequence_callback->get_pbeam()) { + _decoder = new PBeamDecoder(nsrc, src0); + _processor = new PBeamProcessor(); + _type_log.update("type : %s\n", "pbeam"); + } +}; + +class BFpacketcapture_cor_impl : public BFpacketcapture_impl { + ProcLog _type_log; + ProcLog _chan_log; + + BFpacketcapture_cor_sequence_callback _sequence_callback; + + BFoffset _time_tag; + int _navg; + + void on_sequence_start(const PacketDesc* pkt, BFoffset* seq0, BFoffset* time_tag, const void** hdr, size_t* hdr_size ) { + _seq = round_nearest(pkt->seq, _nseq_per_buf); + this->on_sequence_changed(pkt, seq0, time_tag, hdr, hdr_size); + } + void on_sequence_active(const PacketDesc* pkt) { + if( pkt ) { + //cout << "Latest time_tag, tuning = " << pkt->time_tag << ", " << pkt->tuning << endl; + } + else { + //cout << "No latest packet" << endl; + } + } + inline bool has_sequence_changed(const PacketDesc* pkt) { + return ((pkt->chan0 != _chan0) \ + || (pkt->nchan != _nchan) \ + || (pkt->decimation != _navg)); + } + void on_sequence_changed(const PacketDesc* pkt, BFoffset* seq0, BFoffset* time_tag, const void** hdr, size_t* hdr_size) { + *seq0 = _seq;// + _nseq_per_buf*_bufs.size(); + *time_tag = pkt->time_tag; + _chan0 = pkt->chan0; + _nchan = pkt->nchan; + _navg = pkt->decimation; + _payload_size = pkt->payload_size; + + if( _sequence_callback ) { + int status = (*_sequence_callback)(*seq0, + *time_tag, + _chan0, + _nchan*((pkt->tuning >> 8) & 0xFF), + _navg, + _nsrc/((pkt->tuning >> 8) & 0xFF), + hdr, + hdr_size); + if( status != 0 ) { + // TODO: What to do here? Needed? + throw std::runtime_error("BAD HEADER CALLBACK STATUS"); + } + } else { + // Simple default for easy testing + *time_tag = *seq0; + *hdr = NULL; + *hdr_size = 0; + } + + _chan_log.update() << "chan0 : " << _chan0 << "\n" + << "nchan : " << _nchan*((pkt->tuning >> 8) & 0xFF) << "\n" + << "payload_size : " << _payload_size << "\n"; + } +public: + inline BFpacketcapture_cor_impl(PacketCaptureThread* capture, + BFring ring, + int nsrc, + int src0, + int buffer_ntime, + int slot_ntime, + BFpacketcapture_callback sequence_callback) + : BFpacketcapture_impl(capture, nullptr, nullptr, ring, nsrc, buffer_ntime, slot_ntime), + _type_log((std::string(capture->get_name())+"/type").c_str()), + _chan_log((std::string(capture->get_name())+"/chans").c_str()), + _sequence_callback(sequence_callback->get_cor()) { + _decoder = new CORDecoder(nsrc, src0); + _processor = new CORProcessor(); + _type_log.update("type : %s\n", "cor"); + } +}; + +class BFpacketcapture_vdif_impl : public BFpacketcapture_impl { + ProcLog _type_log; + ProcLog _chan_log; + + BFpacketcapture_vdif_sequence_callback _sequence_callback; + + BFoffset _time_tag; + int _tuning; + int _sample_rate; + + void on_sequence_start(const PacketDesc* pkt, BFoffset* seq0, BFoffset* time_tag, const void** hdr, size_t* hdr_size ) { + _seq = round_nearest(pkt->seq, _nseq_per_buf); + this->on_sequence_changed(pkt, seq0, time_tag, hdr, hdr_size); + } + void on_sequence_active(const PacketDesc* pkt) { + if( pkt ) { + //cout << "Latest time_tag, tuning = " << pkt->time_tag << ", " << pkt->tuning << endl; + } + else { + //cout << "No latest packet" << endl; + } + } + inline bool has_sequence_changed(const PacketDesc* pkt) { + return ((pkt->chan0 != _chan0) \ + || (pkt->nchan != _nchan) \ + || (pkt->tuning != _tuning)); + } + void on_sequence_changed(const PacketDesc* pkt, BFoffset* seq0, BFoffset* time_tag, const void** hdr, size_t* hdr_size) { + *seq0 = _seq;// + _nseq_per_buf*_bufs.size(); + *time_tag = pkt->time_tag; + _chan0 = pkt->chan0; + _nchan = pkt->nchan; + _tuning = pkt->tuning; + _sample_rate = pkt->sync; + _payload_size = pkt->payload_size; + + int ref_epoch = (_tuning >> 16) & 0xFF; + int bit_depth = (_tuning >> 8) & 0xFF; + int is_complex = (_tuning & 1); + + if( _sequence_callback ) { + int status = (*_sequence_callback)(*seq0, + *time_tag, + ref_epoch, + _sample_rate, + _chan0, + bit_depth, + is_complex, + _nsrc, + hdr, + hdr_size); + if( status != 0 ) { + // TODO: What to do here? Needed? + throw std::runtime_error("BAD HEADER CALLBACK STATUS"); + } + } else { + // Simple default for easy testing + *time_tag = *seq0; + *hdr = NULL; + *hdr_size = 0; + } + + _chan_log.update() << "nchan : " << _chan0 << "\n" + << "bitdepth : " << bit_depth << "\n" + << "complex : " << is_complex << "\n" + << "payload_size : " << (_payload_size*8) << "\n"; + } +public: + inline BFpacketcapture_vdif_impl(PacketCaptureThread* capture, + BFring ring, + int nsrc, + int src0, + int buffer_ntime, + int slot_ntime, + BFpacketcapture_callback sequence_callback) + : BFpacketcapture_impl(capture, nullptr, nullptr, ring, nsrc, buffer_ntime, slot_ntime), + _type_log((std::string(capture->get_name())+"/type").c_str()), + _chan_log((std::string(capture->get_name())+"/chans").c_str()), + _sequence_callback(sequence_callback->get_vdif()) { + _decoder = new VDIFDecoder(nsrc, src0); + _processor = new VDIFProcessor(); + _type_log.update("type : %s\n", "vdif"); + } +}; + +class BFpacketcapture_tbn_impl : public BFpacketcapture_impl { + ProcLog _type_log; + ProcLog _chan_log; + + BFpacketcapture_tbn_sequence_callback _sequence_callback; + + BFoffset _time_tag; + uint16_t _decim; + + void on_sequence_start(const PacketDesc* pkt, BFoffset* seq0, BFoffset* time_tag, const void** hdr, size_t* hdr_size ) { + _seq = round_nearest(pkt->seq, _nseq_per_buf); + this->on_sequence_changed(pkt, seq0, time_tag, hdr, hdr_size); + } + void on_sequence_active(const PacketDesc* pkt) { + if( pkt ) { + //cout << "Latest time_tag, tuning = " << pkt->time_tag << ", " << pkt->tuning << endl; + } + else { + //cout << "No latest packet" << endl; + } + } + inline bool has_sequence_changed(const PacketDesc* pkt) { + return ( (pkt->tuning != _chan0 ) + || ( pkt->decimation != _decim) ); + } + void on_sequence_changed(const PacketDesc* pkt, BFoffset* seq0, BFoffset* time_tag, const void** hdr, size_t* hdr_size) { + //cout << "Sequence changed" << endl; + *seq0 = _seq;// + _nseq_per_buf*_bufs.size(); + *time_tag = pkt->time_tag; + _time_tag = pkt->time_tag; + _decim = pkt->decimation; + _chan0 = pkt->tuning; + _payload_size = pkt->payload_size; + + if( _sequence_callback ) { + int status = (*_sequence_callback)(*seq0, + *time_tag, + _decim, + pkt->tuning, + _nsrc, + hdr, + hdr_size); + if( status != 0 ) { + // TODO: What to do here? Needed? + throw std::runtime_error("BAD HEADER CALLBACK STATUS"); + } + } else { + // Simple default for easy testing + *time_tag = *seq0; + *hdr = NULL; + *hdr_size = 0; + } + + _chan_log.update() << "chan0 : " << _chan0 << "\n" + << "payload_size : " << _payload_size << "\n"; + } +public: + inline BFpacketcapture_tbn_impl(PacketCaptureThread* capture, + BFring ring, + int nsrc, + int src0, + int buffer_ntime, + int slot_ntime, + BFpacketcapture_callback sequence_callback) + : BFpacketcapture_impl(capture, nullptr, nullptr, ring, nsrc, buffer_ntime, slot_ntime), + _type_log((std::string(capture->get_name())+"/type").c_str()), + _chan_log((std::string(capture->get_name())+"/chans").c_str()), + _sequence_callback(sequence_callback->get_tbn()), + _decim(0) { + _decoder = new TBNDecoder(nsrc, src0); + _processor = new TBNProcessor(); + _type_log.update("type : %s\n", "tbn"); + } +}; + +class BFpacketcapture_drx_impl : public BFpacketcapture_impl { + ProcLog _type_log; + ProcLog _chan_log; + + BFpacketcapture_drx_sequence_callback _sequence_callback; + + BFoffset _time_tag; + uint16_t _decim; + int _chan1; + + void on_sequence_start(const PacketDesc* pkt, BFoffset* seq0, BFoffset* time_tag, const void** hdr, size_t* hdr_size ) { + _seq = round_nearest(pkt->seq, _nseq_per_buf); + this->on_sequence_changed(pkt, seq0, time_tag, hdr, hdr_size); + } + void on_sequence_active(const PacketDesc* pkt) { + if( pkt ) { + //cout << "Latest nchan, chan0 = " << pkt->nchan << ", " << pkt->chan0 << endl; + } + else { + //cout << "No latest packet" << endl; + } + } + inline bool has_sequence_changed(const PacketDesc* pkt) { + return ( (pkt->tuning != _chan0) + || (pkt->tuning1 != _chan1) + || (pkt->decimation != _decim) ); + } + void on_sequence_changed(const PacketDesc* pkt, BFoffset* seq0, BFoffset* time_tag, const void** hdr, size_t* hdr_size) { + *seq0 = _seq;// + _nseq_per_buf*_bufs.size(); + *time_tag = pkt->time_tag; + _time_tag = pkt->time_tag; + _chan0 = pkt->tuning; + _chan1 = pkt->tuning1; + if( _nsrc == 2 ) { + _chan0 = std::max(_chan0, _chan1); + _chan1 = 0; + } + _decim = pkt->decimation; + _payload_size = pkt->payload_size; + + if( _sequence_callback ) { + int status = (*_sequence_callback)(*seq0, + *time_tag, + _decim, + _chan0, + _chan1, + _nsrc, + hdr, + hdr_size); + if( status != 0 ) { + // TODO: What to do here? Needed? + throw std::runtime_error("BAD HEADER CALLBACK STATUS"); + } + } else { + // Simple default for easy testing + *time_tag = *seq0; + *hdr = NULL; + *hdr_size = 0; + } + + _chan_log.update() << "chan0 : " << _chan0 << "\n" + << "chan1 : " << _chan1 << "\n" + << "payload_size : " << _payload_size << "\n"; + } +public: + inline BFpacketcapture_drx_impl(PacketCaptureThread* capture, + BFring ring, + int nsrc, + int src0, + int buffer_ntime, + int slot_ntime, + BFpacketcapture_callback sequence_callback) + : BFpacketcapture_impl(capture, nullptr, nullptr, ring, nsrc, buffer_ntime, slot_ntime), + _type_log((std::string(capture->get_name())+"/type").c_str()), + _chan_log((std::string(capture->get_name())+"/chans").c_str()), + _sequence_callback(sequence_callback->get_drx()), + _decim(0), _chan1(0) { + _decoder = new DRXDecoder(nsrc, src0); + _processor = new DRXProcessor(); + _type_log.update("type : %s\n", "drx"); + } +}; + +class BFpacketcapture_drx8_impl : public BFpacketcapture_impl { + ProcLog _type_log; + ProcLog _chan_log; + + BFpacketcapture_drx8_sequence_callback _sequence_callback; + + BFoffset _time_tag; + uint16_t _decim; + int _chan1; + + void on_sequence_start(const PacketDesc* pkt, BFoffset* seq0, BFoffset* time_tag, const void** hdr, size_t* hdr_size ) { + _seq = round_nearest(pkt->seq, _nseq_per_buf); + this->on_sequence_changed(pkt, seq0, time_tag, hdr, hdr_size); + } + void on_sequence_active(const PacketDesc* pkt) { + if( pkt ) { + //cout << "Latest nchan, chan0 = " << pkt->nchan << ", " << pkt->chan0 << endl; + } + else { + //cout << "No latest packet" << endl; + } + } + inline bool has_sequence_changed(const PacketDesc* pkt) { + return ( (pkt->tuning != _chan0) + || (pkt->tuning1 != _chan1) + || (pkt->decimation != _decim) ); + } + void on_sequence_changed(const PacketDesc* pkt, BFoffset* seq0, BFoffset* time_tag, const void** hdr, size_t* hdr_size) { + *seq0 = _seq;// + _nseq_per_buf*_bufs.size(); + *time_tag = pkt->time_tag; + _time_tag = pkt->time_tag; + _chan0 = pkt->tuning; + _chan1 = pkt->tuning1; + if( _nsrc == 2 ) { + _chan0 = std::max(_chan0, _chan1); + _chan1 = 0; + } + _decim = pkt->decimation; + _payload_size = pkt->payload_size; + + if( _sequence_callback ) { + int status = (*_sequence_callback)(*seq0, + *time_tag, + _decim, + _chan0, + _chan1, + _nsrc, + hdr, + hdr_size); + if( status != 0 ) { + // TODO: What to do here? Needed? + throw std::runtime_error("BAD HEADER CALLBACK STATUS"); + } + } else { + // Simple default for easy testing + *time_tag = *seq0; + *hdr = NULL; + *hdr_size = 0; + } + + _chan_log.update() << "chan0 : " << _chan0 << "\n" + << "chan1 : " << _chan1 << "\n" + << "payload_size : " << _payload_size << "\n"; + } +public: + inline BFpacketcapture_drx8_impl(PacketCaptureThread* capture, + BFring ring, + int nsrc, + int src0, + int buffer_ntime, + int slot_ntime, + BFpacketcapture_callback sequence_callback) + : BFpacketcapture_impl(capture, nullptr, nullptr, ring, nsrc, buffer_ntime, slot_ntime), + _type_log((std::string(capture->get_name())+"/type").c_str()), + _chan_log((std::string(capture->get_name())+"/chans").c_str()), + _sequence_callback(sequence_callback->get_drx8()), + _decim(0), _chan1(0) { + _decoder = new DRX8Decoder(nsrc, src0); + _processor = new DRX8Processor(); + _type_log.update("type : %s\n", "drx8"); + } +}; + +BFstatus BFpacketcapture_create(BFpacketcapture* obj, + const char* format, + int fd, + BFring ring, + BFsize nsrc, + BFsize src0, + BFsize max_payload_size, + BFsize buffer_ntime, + BFsize slot_ntime, + BFpacketcapture_callback sequence_callback, + int core, + BFiomethod backend) { + BF_ASSERT(obj, BF_STATUS_INVALID_POINTER); + + if( std::string(format).substr(0, 6) == std::string("simple") ) { + if( backend == BF_IO_DISK ) { + // Need to know how much to read at a time + int nchan = 1; + max_payload_size = 8200; + } + } else if( std::string(format).substr(0, 5) == std::string("chips") ) { + if( backend == BF_IO_DISK ) { + // Need to know how much to read at a time + int nchan = std::atoi((std::string(format).substr(6, std::string(format).length())).c_str()); + max_payload_size = sizeof(chips_hdr_type) + (4*nchan); + } + } else if( std::string(format).substr(0, 5) == std::string("ibeam") ) { + if( backend == BF_IO_DISK ) { + // Need to know how much to read at a time + int nbeam = std::atoi((std::string(format).substr(5, 1).c_str())); + int nchan = std::atoi((std::string(format).substr(7, std::string(format).length())).c_str()); + max_payload_size = sizeof(ibeam_hdr_type) + (8*2*nbeam*nchan); + } + } else if( std::string(format).substr(0, 5) == std::string("pbeam") ) { + if( backend == BF_IO_DISK ) { + // Need to know how much to read at a time + int nchan = std::atoi((std::string(format).substr(6, std::string(format).length())).c_str()); + max_payload_size = sizeof(pbeam_hdr_type) + (4*4*nchan); + } + } else if(std::string(format).substr(0, 3) == std::string("cor") ) { + if( backend == BF_IO_DISK ) { + // Need to know how much to read at a time + int nchan = std::atoi((std::string(format).substr(4, std::string(format).length())).c_str()); + max_payload_size = sizeof(cor_hdr_type) + (8*4*nchan); + } + } else if(std::string(format).substr(0, 4) == std::string("vdif") ) { + if( backend == BF_IO_DISK ) { + // Need to know how much to read at a time + int nchan = std::atoi((std::string(format).substr(5, std::string(format).length())).c_str()); + max_payload_size = nchan; + } + } else if( format == std::string("tbn") ) { + max_payload_size = TBN_FRAME_SIZE; + } else if( format == std::string("drx") ) { + max_payload_size = DRX_FRAME_SIZE; + } else if( format == std::string("drx8") ) { + max_payload_size = DRX8_FRAME_SIZE; + } + + PacketCaptureMethod* method; + if( backend == BF_IO_DISK ) { + method = new DiskPacketReader(fd, max_payload_size); + } else if( backend == BF_IO_UDP ) { + method = new UDPPacketReceiver(fd, max_payload_size); + } else if( backend == BF_IO_SNIFFER ) { + method = new UDPPacketSniffer(fd, max_payload_size); +#if defined BF_VERBS_ENABLED && BF_VERBS_ENABLED + } else if( backend == BF_IO_VERBS ) { + method = new UDPVerbsReceiver(fd, max_payload_size); +#endif + } else { + return BF_STATUS_UNSUPPORTED; + } + PacketCaptureThread* capture = new PacketCaptureThread(method, nsrc, core); + if (std::string(format).substr(0, 6) == std::string("simple") ) { + BF_TRY_RETURN_ELSE(*obj = new BFpacketcapture_simple_impl(capture, ring, nsrc, src0, + buffer_ntime, slot_ntime, + sequence_callback), + *obj = 0); + + } else if( std::string(format).substr(0, 5) == std::string("chips") ) { + BF_TRY_RETURN_ELSE(*obj = new BFpacketcapture_chips_impl(capture, ring, nsrc, src0, + buffer_ntime, slot_ntime, + sequence_callback), + *obj = 0); + } else if( std::string(format).substr(0, 5) == std::string("snap2") ) { + BF_TRY_RETURN_ELSE(*obj = new BFpacketcapture_snap2_impl(capture, ring, nsrc, src0, + buffer_ntime, slot_ntime, + sequence_callback), + *obj = 0); +#define MATCH_IBEAM_MODE(NBEAM) \ + } else if( std::string(format).substr(0, 6) == std::string("ibeam"#NBEAM) ) { \ + BF_TRY_RETURN_ELSE(*obj = new BFpacketcapture_ibeam_impl(capture, ring, nsrc, src0, \ + buffer_ntime, slot_ntime, \ + sequence_callback), \ + *obj = 0); + MATCH_IBEAM_MODE(1) + MATCH_IBEAM_MODE(2) + MATCH_IBEAM_MODE(3) + MATCH_IBEAM_MODE(4) +#undef MATCH_IBEAM_MODE + } else if( std::string(format).substr(0, 5) == std::string("pbeam") ) { + BF_TRY_RETURN_ELSE(*obj = new BFpacketcapture_pbeam_impl(capture, ring, nsrc, src0, + buffer_ntime, slot_ntime, + sequence_callback), + *obj = 0); + } else if( std::string(format).substr(0, 3) == std::string("cor") ) { + BF_TRY_RETURN_ELSE(*obj = new BFpacketcapture_cor_impl(capture, ring, nsrc, src0, + buffer_ntime, slot_ntime, + sequence_callback), + *obj = 0); + } else if( std::string(format).substr(0, 4) == std::string("vdif") ) { + BF_TRY_RETURN_ELSE(*obj = new BFpacketcapture_vdif_impl(capture, ring, nsrc, src0, + buffer_ntime, slot_ntime, + sequence_callback), + *obj = 0); + } else if( format == std::string("tbn") ) { + BF_TRY_RETURN_ELSE(*obj = new BFpacketcapture_tbn_impl(capture, ring, nsrc, src0, + buffer_ntime, slot_ntime, + sequence_callback), + *obj = 0); + } else if( format == std::string("drx") ) { + BF_TRY_RETURN_ELSE(*obj = new BFpacketcapture_drx_impl(capture, ring, nsrc, src0, + buffer_ntime, slot_ntime, + sequence_callback), + *obj = 0); + } else if( format == std::string("drx8") ) { + BF_TRY_RETURN_ELSE(*obj = new BFpacketcapture_drx8_impl(capture, ring, nsrc, src0, + buffer_ntime, slot_ntime, + sequence_callback), + *obj = 0); + } else { + return BF_STATUS_UNSUPPORTED; + } +} diff --git a/src/packet_writer.cpp b/src/packet_writer.cpp new file mode 100644 index 000000000..6acf4fbce --- /dev/null +++ b/src/packet_writer.cpp @@ -0,0 +1,195 @@ +/* + * Copyright (c) 2019-2023, The Bifrost Authors. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of The Bifrost Authors nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "packet_writer.hpp" + +#define BF_JAYCE_DEBUG 0 + +#if BF_JAYCE_DEBUG +#define BF_PRINTD(stmt) \ + std::cout << stmt << std::endl +#else // not BF_JAYCE_DEBUG +#define BF_PRINTD(stmt) +#endif + +BFstatus BFpacketwriter_impl::send(BFheaderinfo info, + BFoffset seq, + BFoffset seq_increment, + BFoffset src, + BFoffset src_increment, + BFarray const* in) { + BF_ASSERT(info, BF_STATUS_INVALID_HANDLE); + BF_ASSERT(in, BF_STATUS_INVALID_POINTER); + BF_ASSERT(in->dtype == _dtype, BF_STATUS_INVALID_DTYPE); + BF_ASSERT(in->ndim == 3, BF_STATUS_INVALID_SHAPE); + BF_ASSERT(in->shape[2] == _nsamples, BF_STATUS_INVALID_SHAPE); + BF_ASSERT(is_contiguous(in), BF_STATUS_UNSUPPORTED_STRIDE); + BF_ASSERT(space_accessible_from(in->space, BF_SPACE_SYSTEM), + BF_STATUS_UNSUPPORTED_SPACE); + + PacketDesc* hdr_base = info->get_description(); + + int i, j; + int hdr_size = _filler->get_size(); + int data_size = (BF_DTYPE_NBIT(in->dtype)/8) * _nsamples; + int npackets = in->shape[0]*in->shape[1]; + + if( hdr_size != _last_size || npackets > _last_count ) { + if( _pkt_hdrs ) { + ::munlock(_pkt_hdrs, _last_count*_last_size*sizeof(char)); + free(_pkt_hdrs); + } + + _last_size = hdr_size; + _last_count = npackets; + _pkt_hdrs = (char*) malloc(npackets*hdr_size*sizeof(char)); + ::mlock(_pkt_hdrs, npackets*hdr_size*sizeof(char)); + } + + for(i=0; ishape[0]; i++) { + hdr_base->seq = seq + i*seq_increment; + for(j=0; jshape[1]; j++) { + hdr_base->src = src + j*src_increment; + (*_filler)(hdr_base, _framecount, _pkt_hdrs+hdr_size*(i*in->shape[1]+j)); + } + _framecount++; + } + + _writer->send(_pkt_hdrs, hdr_size, (char*) in->data, data_size, npackets); + this->update_stats_log(); + + return BF_STATUS_SUCCESS; +} + +BFstatus bfHeaderInfoCreate(BFheaderinfo* obj) { + BF_ASSERT(obj, BF_STATUS_INVALID_HANDLE); + BF_TRY_RETURN_ELSE(*obj = new BFheaderinfo_impl(), + *obj = 0); +} + +BFstatus bfHeaderInfoDestroy(BFheaderinfo obj) { + BF_ASSERT(obj, BF_STATUS_INVALID_HANDLE); + delete obj; + return BF_STATUS_SUCCESS; +} + +BFstatus bfHeaderInfoSetNSrc(BFheaderinfo obj, + int nsrc) { + BF_ASSERT(obj, BF_STATUS_INVALID_HANDLE); + obj->set_nsrc(nsrc); + return BF_STATUS_SUCCESS; +} + +BFstatus bfHeaderInfoSetNChan(BFheaderinfo obj, + int nchan) { + BF_ASSERT(obj, BF_STATUS_INVALID_HANDLE); + obj->set_nchan(nchan); + return BF_STATUS_SUCCESS; +} + +BFstatus bfHeaderInfoSetChan0(BFheaderinfo obj, + int chan0) { + BF_ASSERT(obj, BF_STATUS_INVALID_HANDLE); + obj->set_chan0(chan0); + return BF_STATUS_SUCCESS; +} + +BFstatus bfHeaderInfoSetTuning(BFheaderinfo obj, + int tuning) { + BF_ASSERT(obj, BF_STATUS_INVALID_HANDLE); + obj->set_tuning(tuning); + return BF_STATUS_SUCCESS; +} + +BFstatus bfHeaderInfoSetGain(BFheaderinfo obj, + unsigned short int gain) { + BF_ASSERT(obj, BF_STATUS_INVALID_HANDLE); + obj->set_gain(gain); + return BF_STATUS_SUCCESS; +} + +BFstatus bfHeaderInfoSetDecimation(BFheaderinfo obj, + unsigned int decimation) { + BF_ASSERT(obj, BF_STATUS_INVALID_HANDLE); + obj->set_decimation(decimation); + return BF_STATUS_SUCCESS; +} + +BFstatus bfDiskWriterCreate(BFpacketwriter* obj, + const char* format, + int fd, + int core) { + return BFpacketwriter_create(obj, format, fd, core, BF_IO_DISK); +} + +BFstatus bfUdpTransmitCreate(BFpacketwriter* obj, + const char* format, + int fd, + int core) { + return BFpacketwriter_create(obj, format, fd, core, BF_IO_UDP); +} + +BFstatus bfUdpVerbsTransmitCreate(BFpacketwriter* obj, + const char* format, + int fd, + int core) { + return BFpacketwriter_create(obj, format, fd, core, BF_IO_VERBS); +} + +BFstatus bfPacketWriterDestroy(BFpacketwriter obj) { + BF_ASSERT(obj, BF_STATUS_INVALID_HANDLE); + delete obj; + return BF_STATUS_SUCCESS; +} + +BFstatus bfPacketWriterSetRateLimit(BFpacketwriter obj, + uint32_t rate_limit) { + BF_ASSERT(obj, BF_STATUS_INVALID_HANDLE); + BF_TRY_RETURN(obj->set_rate_limit(rate_limit)); +} + +BFstatus bfPacketWriterResetRateLimitCounter(BFpacketwriter obj) { + BF_ASSERT(obj, BF_STATUS_INVALID_HANDLE); + BF_TRY_RETURN(obj->reset_rate_limit_counter()); +} + +BFstatus bfPacketWriterResetCounter(BFpacketwriter obj) { + BF_ASSERT(obj, BF_STATUS_INVALID_HANDLE); + BF_TRY_RETURN(obj->reset_counter()); +} + +BFstatus bfPacketWriterSend(BFpacketwriter obj, + BFheaderinfo info, + BFoffset seq, + BFoffset seq_increment, + BFoffset src, + BFoffset src_increment, + BFarray const* in) { + BF_ASSERT(obj, BF_STATUS_INVALID_HANDLE); + return obj->send(info, seq, seq_increment, src, src_increment, in); +} diff --git a/src/packet_writer.hpp b/src/packet_writer.hpp new file mode 100644 index 000000000..03bd6d704 --- /dev/null +++ b/src/packet_writer.hpp @@ -0,0 +1,674 @@ +/* + * Copyright (c) 2019-2023, The Bifrost Authors. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of The Bifrost Authors nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "assert.hpp" +#include +#include +#include "proclog.hpp" +#include "Socket.hpp" +#include "formats/formats.hpp" +#include "utils.hpp" +#include "hw_locality.hpp" + +#include // For ntohs +#include // For recvfrom + +#include +#include +#include +#include // For posix_memalign +#include // For memcpy, memset +#include + +#include +#include +#include +#include +#include +#include + +#ifndef BF_SEND_NPKTBURST +#define BF_SEND_NPKTBURST 16 +#endif + +class RateLimiter { + uint32_t _rate; + uint64_t _counter; + bool _first; + std::chrono::high_resolution_clock::time_point _start; + std::chrono::high_resolution_clock::time_point _stop; +public: + RateLimiter(uint32_t rate_limit=0) + : _rate(rate_limit), _counter(0), _first(true) {} + inline void set_rate(uint32_t rate_limit) { _rate = rate_limit; } + inline uint32_t get_rate() { return _rate; } + inline void reset_counter() { _first = true; _counter = 0; } + inline void begin() { + if( _first ) { + _start = std::chrono::high_resolution_clock::now(); + _first = false; + } + } + inline void end_and_wait(size_t npackets) { + if( _rate > 0 ) { + _stop = std::chrono::high_resolution_clock::now(); + _counter += std::max((size_t) 0, npackets); + double elapsed_needed = (double) _counter / _rate; + std::chrono::duration elapsed_actual = std::chrono::duration_cast>(_stop-_start); + + double sleep_needed = elapsed_needed - elapsed_actual.count(); + if( sleep_needed > 0.001 ) { + timespec sleep; + sleep.tv_sec = (int) sleep_needed; + sleep.tv_nsec = (int) ((sleep_needed - sleep.tv_sec)*1e9); + nanosleep(&sleep, NULL); + } + } + } +}; + +class PacketWriterMethod { +protected: + int _fd; + size_t _max_burst_size; + RateLimiter _limiter; +public: + PacketWriterMethod(int fd, size_t max_burst_size=BF_SEND_NPKTBURST) + : _fd(fd), _max_burst_size(max_burst_size), _limiter(0) {} + virtual ssize_t send_packets(char* hdrs, + int hdr_size, + char* data, + int data_size, + int npackets, + int flags=0) { + return 0; + } + virtual const char* get_name() { return "generic_writer"; } + inline void set_rate(uint32_t rate_limit) { _limiter.set_rate(rate_limit); } + inline uint32_t get_rate() { return _limiter.get_rate(); } + inline void reset_counter() { _limiter.reset_counter(); } +}; + +class DiskPacketWriter : public PacketWriterMethod { +public: + DiskPacketWriter(int fd, size_t max_burst_size=BF_SEND_NPKTBURST) + : PacketWriterMethod(fd, max_burst_size) {} + ssize_t send_packets(char* hdrs, + int hdr_size, + char* data, + int data_size, + int npackets, + int flags=0) { + int i = 0; + ssize_t status, nsend, nsent = 0; + while(npackets > 0) { + _limiter.begin(); + if( _max_burst_size > 0 ) { + nsend = std::min(_max_burst_size, (size_t) npackets); + } else { + nsend = npackets; + } + for(int j=0; j _last_count ) { + if( _mmsg ) { + ::munlock(_mmsg, sizeof(struct mmsghdr)*_last_count); + free(_mmsg); + } + if( _iovs ) { + ::munlock(_iovs, sizeof(struct iovec)*2*_last_count); + free(_iovs); + } + + _last_count = npackets; + _mmsg = (struct mmsghdr *) malloc(sizeof(struct mmsghdr)*npackets); + _iovs = (struct iovec *) malloc(sizeof(struct iovec)*2*npackets); + ::mlock(_mmsg, sizeof(struct mmsghdr)*npackets); + ::mlock(_iovs, sizeof(struct iovec)*2*npackets); + + ::memset(_mmsg, 0, sizeof(struct mmsghdr)*npackets); + + for(int i=0; i 0) { + _limiter.begin(); + if( _max_burst_size > 0 ) { + nsend = std::min(_max_burst_size, (size_t) npackets); + } else { + nsend = npackets; + } + nsent_batch = ::sendmmsg(_fd, _mmsg+i, nsend, flags); + if( nsent_batch > 0 ) { + nsent += nsent_batch; + } + /* + if( nsent_batch == -1 ) { + std::cout << "sendmmsg failed: " << std::strerror(errno) << " with " << hdr_size << " and " << data_size << std::endl; + } + */ + i += nsend; + npackets -= nsend; + _limiter.end_and_wait(nsend); + } + + return nsent; + } + inline const char* get_name() { return "udp_transmit"; } +}; + +#if defined BF_VERBS_ENABLED && BF_VERBS_ENABLED +#include "ib_verbs_send.hpp" + +class UDPVerbsSender : public PacketWriterMethod { + VerbsSend _ibv; + bf_comb_udp_hdr _udp_hdr; + int _last_size; + int _last_count; + mmsghdr* _mmsg; + iovec* _iovs; +public: + UDPVerbsSender(int fd, size_t max_burst_size=BF_VERBS_SEND_NPKTBURST) + : PacketWriterMethod(fd, max_burst_size), _ibv(fd, JUMBO_FRAME_SIZE), _last_size(0), + _last_count(0), _mmsg(NULL), _iovs(NULL) {} + ~UDPVerbsSender() { + if( _mmsg ) { + free(_mmsg); + } + if( _iovs ) { + free(_iovs); + } + } + ssize_t send_packets(char* hdrs, + int hdr_size, + char* data, + int data_size, + int npackets, + int flags=0) { + if( npackets > _last_count ) { + if( _mmsg ) { + ::munlock(_mmsg, sizeof(struct mmsghdr)*_last_count); + free(_mmsg); + } + if( _iovs ) { + ::munlock(_iovs, sizeof(struct iovec)*3*_last_count); + free(_iovs); + } + + _last_count = npackets; + _mmsg = (struct mmsghdr *) malloc(sizeof(struct mmsghdr)*npackets); + _iovs = (struct iovec *) malloc(sizeof(struct iovec)*3*npackets); + ::mlock(_mmsg, sizeof(struct mmsghdr)*npackets); + ::mlock(_iovs, sizeof(struct iovec)*3*npackets); + + ::memset(_mmsg, 0, sizeof(struct mmsghdr)*npackets); + + for(int i=0; i 0 ) { + _ibv.set_rate_limit(_limiter.get_rate()*_last_size, _last_size, _max_burst_size); + } + } + + for(int i=0; ireset_stats(); + } + inline void set_rate_limit(uint32_t rate_limit) { _method->set_rate(rate_limit); } + inline uint32_t get_rate_limit() { return _method->get_rate(); } + inline void reset_rate_limit_counter() { _method->reset_counter(); } + inline ssize_t send(char* hdrs, + int hdr_size, + char* datas, + int data_size, + int npackets) { + ssize_t nsent = _method->send_packets(hdrs, hdr_size, datas, data_size, npackets); + if( nsent == -1 ) { + _stats.ninvalid += npackets; + _stats.ninvalid_bytes += npackets * (hdr_size + data_size); + } else { + _stats.nvalid += nsent; + _stats.nvalid_bytes += nsent * (hdr_size + data_size); + _stats.ninvalid += (npackets - nsent); + _stats.ninvalid_bytes += (npackets - nsent) * (hdr_size + data_size); + } + return nsent; + } + inline const char* get_name() { return _method->get_name(); } + inline const int get_core() { return _core; } + inline const PacketStats* get_stats() const { return &_stats; } + inline void reset_stats() { + ::memset(&_stats, 0, sizeof(_stats)); + } +}; + +class BFheaderinfo_impl { + PacketDesc _desc; +public: + inline BFheaderinfo_impl() { + ::memset(&_desc, 0, sizeof(PacketDesc)); + } + inline PacketDesc* get_description() { return &_desc; } + inline void set_nsrc(int nsrc) { _desc.nsrc = nsrc; } + inline void set_nchan(int nchan) { _desc.nchan = nchan; } + inline void set_chan0(int chan0) { _desc.chan0 = chan0; } + inline void set_tuning(int tuning) { _desc.tuning = tuning; } + inline void set_gain(uint16_t gain) { _desc.gain = gain; } + inline void set_decimation(uint32_t decimation) { _desc.decimation = decimation; } +}; + +class BFpacketwriter_impl { +protected: + std::string _name; + PacketWriterThread* _writer; + PacketHeaderFiller* _filler; + int _nsamples; + BFdtype _dtype; + + ProcLog _bind_log; + ProcLog _stat_log; + pid_t _pid; + + char* _pkt_hdrs; + int _last_size; + int _last_count; + BFoffset _framecount; +private: + void update_stats_log() { + const PacketStats* stats = _writer->get_stats(); + _stat_log.update() << "ngood_bytes : " << stats->nvalid_bytes << "\n" + << "nmissing_bytes : " << stats->ninvalid_bytes << "\n" + << "ninvalid : " << stats->ninvalid << "\n" + << "ninvalid_bytes : " << stats->ninvalid_bytes << "\n" + << "nlate : " << stats->nlate << "\n" + << "nlate_bytes : " << stats->nlate_bytes << "\n" + << "nvalid : " << stats->nvalid << "\n" + << "nvalid_bytes : " << stats->nvalid_bytes << "\n"; + } +public: + inline BFpacketwriter_impl(PacketWriterThread* writer, + PacketHeaderFiller* filler, + int nsamples, + BFdtype dtype) + : _name(writer->get_name()), _writer(writer), _filler(filler), + _nsamples(nsamples), _dtype(dtype), + _bind_log(_name+"/bind"), + _stat_log(_name+"/stats"), + _pkt_hdrs(NULL), _last_size(0), _last_count(0), + _framecount(0) { + _bind_log.update() << "ncore : " << 1 << "\n" + << "core0 : " << _writer->get_core() << "\n"; + } + inline ~BFpacketwriter_impl() { + if(_pkt_hdrs) { + free(_pkt_hdrs); + } + } + inline void set_rate_limit(uint32_t rate_limit) { _writer->set_rate_limit(rate_limit); } + inline void reset_rate_limit_counter() { _writer->reset_rate_limit_counter(); } + inline void reset_counter() { _framecount = 0; } + BFstatus send(BFheaderinfo info, + BFoffset seq, + BFoffset seq_increment, + BFoffset src, + BFoffset src_increment, + BFarray const* in); +}; + +class BFpacketwriter_generic_impl : public BFpacketwriter_impl { + ProcLog _type_log; +public: + inline BFpacketwriter_generic_impl(PacketWriterThread* writer, + int nsamples) + : BFpacketwriter_impl(writer, nullptr, nsamples, BF_DTYPE_U8), + _type_log((std::string(writer->get_name())+"/type").c_str()) { + _filler = new PacketHeaderFiller(); + _type_log.update("type : %s\n", "generic"); + } +}; + +class BFpacketwriter_simple_impl : public BFpacketwriter_impl { + ProcLog _type_log; +public: + inline BFpacketwriter_simple_impl(PacketWriterThread* writer, + int nsamples) + : BFpacketwriter_impl(writer, nullptr, nsamples, BF_DTYPE_CI16), + _type_log((std::string(writer->get_name())+"/type").c_str()) { + _filler = new SIMPLEHeaderFiller(); + _type_log.update("type : %s\n", "simple"); + } +}; + +class BFpacketwriter_chips_impl : public BFpacketwriter_impl { + ProcLog _type_log; +public: + inline BFpacketwriter_chips_impl(PacketWriterThread* writer, + int nsamples) + : BFpacketwriter_impl(writer, nullptr, nsamples, BF_DTYPE_CI4), + _type_log((std::string(writer->get_name())+"/type").c_str()) { + _filler = new CHIPSHeaderFiller(); + _type_log.update("type : %s\n", "chips"); + } +}; + +template +class BFpacketwriter_ibeam_impl : public BFpacketwriter_impl { + uint8_t _nbeam = B; + ProcLog _type_log; +public: + inline BFpacketwriter_ibeam_impl(PacketWriterThread* writer, + int nsamples) + : BFpacketwriter_impl(writer, nullptr, nsamples, BF_DTYPE_CF32), + _type_log((std::string(writer->get_name())+"/type").c_str()) { + _filler = new IBeamHeaderFiller(); + _type_log.update("type : %s%i\n", "ibeam", _nbeam); + } +}; + +template +class BFpacketwriter_pbeam_impl : public BFpacketwriter_impl { + uint8_t _nbeam = B; + ProcLog _type_log; +public: + inline BFpacketwriter_pbeam_impl(PacketWriterThread* writer, + int nsamples) + : BFpacketwriter_impl(writer, nullptr, nsamples, BF_DTYPE_F32), + _type_log((std::string(writer->get_name())+"/type").c_str()) { + _filler = new PBeamHeaderFiller(); + _type_log.update("type : %s%i\n", "pbeam", _nbeam); + } +}; + +class BFpacketwriter_cor_impl : public BFpacketwriter_impl { + ProcLog _type_log; +public: + inline BFpacketwriter_cor_impl(PacketWriterThread* writer, + int nsamples) + : BFpacketwriter_impl(writer, nullptr, nsamples, BF_DTYPE_CF32), + _type_log((std::string(writer->get_name())+"/type").c_str()) { + _filler = new CORHeaderFiller(); + _type_log.update("type : %s\n", "cor"); + } +}; + +class BFpacketwriter_tbn_impl : public BFpacketwriter_impl { + ProcLog _type_log; +public: + inline BFpacketwriter_tbn_impl(PacketWriterThread* writer, + int nsamples) + : BFpacketwriter_impl(writer, nullptr, nsamples, BF_DTYPE_CI8), + _type_log((std::string(writer->get_name())+"/type").c_str()) { + _filler = new TBNHeaderFiller(); + _type_log.update("type : %s\n", "tbn"); + } +}; + +class BFpacketwriter_drx_impl : public BFpacketwriter_impl { + ProcLog _type_log; +public: + inline BFpacketwriter_drx_impl(PacketWriterThread* writer, + int nsamples) + : BFpacketwriter_impl(writer, nullptr, nsamples, BF_DTYPE_CI4), + _type_log((std::string(writer->get_name())+"/type").c_str()) { + _filler = new DRXHeaderFiller(); + _type_log.update("type : %s\n", "drx"); + } +}; + +class BFpacketwriter_drx8_impl : public BFpacketwriter_impl { + ProcLog _type_log; +public: + inline BFpacketwriter_drx8_impl(PacketWriterThread* writer, + int nsamples) + : BFpacketwriter_impl(writer, nullptr, nsamples, BF_DTYPE_CI8), + _type_log((std::string(writer->get_name())+"/type").c_str()) { + _filler = new DRX8HeaderFiller(); + _type_log.update("type : %s\n", "drx8"); + } +}; + +class BFpacketwriter_tbf_impl : public BFpacketwriter_impl { + int16_t _nstand; + ProcLog _type_log; +public: + inline BFpacketwriter_tbf_impl(PacketWriterThread* writer, + int nsamples) + : BFpacketwriter_impl(writer, nullptr, nsamples, BF_DTYPE_CI4), + _nstand(0), _type_log((std::string(writer->get_name())+"/type").c_str()) { + _filler = new TBFHeaderFiller(); + _nstand = nsamples / 2 / 12; + _type_log.update("type : %s%i\n", "tbf", _nstand); + } +}; + +class BFpacketwriter_vbeam_impl : public BFpacketwriter_impl { + ProcLog _type_log; +public: + inline BFpacketwriter_vbeam_impl(PacketWriterThread* writer, + int nsamples) + : BFpacketwriter_impl(writer, nullptr, nsamples, BF_DTYPE_CF32), + _type_log((std::string(writer->get_name())+"/type").c_str()) { + _filler = new VBeamHeaderFiller(); + _type_log.update("type : %s\n", "tbf"); + } +}; + +BFstatus BFpacketwriter_create(BFpacketwriter* obj, + const char* format, + int fd, + int core, + BFiomethod backend) { + BF_ASSERT(obj, BF_STATUS_INVALID_POINTER); + + int nsamples = 0; + if(std::string(format).substr(0, 8) == std::string("generic_") ) { + nsamples = std::atoi((std::string(format).substr(8, std::string(format).length())).c_str()); + } else if( std::string(format).substr(0, 6) == std::string("simple") ) { + nsamples = 2048; + } else if( std::string(format).substr(0, 6) == std::string("chips_") ) { + int nchan = std::atoi((std::string(format).substr(6, std::string(format).length())).c_str()); + nsamples = 32*nchan; + } else if( std::string(format).substr(0, 5) == std::string("ibeam") ) { + int nbeam = std::stoi(std::string(format).substr(5, 1)); + int nchan = std::atoi((std::string(format).substr(7, std::string(format).length())).c_str()); + nsamples = 2*nbeam*nchan; + } else if( std::string(format).substr(0, 5) == std::string("pbeam") ) { + int nchan = std::atoi((std::string(format).substr(7, std::string(format).length())).c_str()); + nsamples = 4*nchan; + } else if( std::string(format).substr(0, 4) == std::string("cor_") ) { + int nchan = std::atoi((std::string(format).substr(4, std::string(format).length())).c_str()); + nsamples = 4*nchan; + } else if( format == std::string("tbn") ) { + nsamples = 512; + } else if( format == std::string("drx") ) { + nsamples = 4096; + } else if( format == std::string("drx8") ) { + nsamples = 4096; + } else if( format == std::string("tbf") ) { + nsamples = 6144; + } else if( std::string(format).substr(0, 6) == std::string("vbeam_") ) { + // e.g. "vbeam_184" is a 184-channel voltage beam" + int nchan = std::atoi((std::string(format).substr(13, std::string(format).length())).c_str()); + nsamples = 2*nchan; // 2 polarizations. Natively 32-bit floating complex (see implementation class) + } + + PacketWriterMethod* method; + if( backend == BF_IO_DISK ) { + method = new DiskPacketWriter(fd); + } else if( backend == BF_IO_UDP ) { + method = new UDPPacketSender(fd); +#if defined BF_VERBS_ENABLED && BF_VERBS_ENABLED + } else if( backend == BF_IO_VERBS ) { + method = new UDPVerbsSender(fd); +#endif + } else { + return BF_STATUS_UNSUPPORTED; + } + PacketWriterThread* writer = new PacketWriterThread(method, core); + + if( std::string(format).substr(0, 8) == std::string("generic_") ) { + BF_TRY_RETURN_ELSE(*obj = new BFpacketwriter_generic_impl(writer, nsamples), + *obj = 0); + } else if( std::string(format).substr(0, 6) == std::string("simple") ) { + BF_TRY_RETURN_ELSE(*obj = new BFpacketwriter_simple_impl(writer, nsamples), + *obj = 0); + } else if( std::string(format).substr(0, 6) == std::string("chips_") ) { + BF_TRY_RETURN_ELSE(*obj = new BFpacketwriter_chips_impl(writer, nsamples), + *obj = 0); +#define MATCH_IBEAM_MODE(NBEAM) \ + } else if( std::string(format).substr(0, 7) == std::string("ibeam"#NBEAM"_") ) { \ + BF_TRY_RETURN_ELSE(*obj = new BFpacketwriter_ibeam_impl(writer, nsamples), \ + *obj = 0); + MATCH_IBEAM_MODE(1) + MATCH_IBEAM_MODE(2) + MATCH_IBEAM_MODE(3) + MATCH_IBEAM_MODE(4) +#undef MATCH_IBEAM_MODE +#define MATCH_PBEAM_MODE(NBEAM) \ + } else if( std::string(format).substr(0, 7) == std::string("pbeam"#NBEAM"_") ) { \ + BF_TRY_RETURN_ELSE(*obj = new BFpacketwriter_pbeam_impl(writer, nsamples), \ + *obj = 0); + MATCH_PBEAM_MODE(1) +#undef MATCH_PBEAM_MODE + } else if( std::string(format).substr(0, 4) == std::string("cor_") ) { + BF_TRY_RETURN_ELSE(*obj = new BFpacketwriter_cor_impl(writer, nsamples), + *obj = 0); + } else if( format == std::string("tbn") ) { + BF_TRY_RETURN_ELSE(*obj = new BFpacketwriter_tbn_impl(writer, nsamples), + *obj = 0); + } else if( format == std::string("drx") ) { + BF_TRY_RETURN_ELSE(*obj = new BFpacketwriter_drx_impl(writer, nsamples), + *obj = 0); + } else if( format == std::string("drx8") ) { + BF_TRY_RETURN_ELSE(*obj = new BFpacketwriter_drx8_impl(writer, nsamples), + *obj = 0); + } else if( std::string(format).substr(0, 3) == std::string("tbf") ) { + BF_TRY_RETURN_ELSE(*obj = new BFpacketwriter_tbf_impl(writer, nsamples), + *obj = 0); + } else if( std::string(format).substr(0, 6) == std::string("vbeam_") ) { + BF_TRY_RETURN_ELSE(*obj = new BFpacketwriter_vbeam_impl(writer, nsamples), + *obj = 0); + } else { + return BF_STATUS_UNSUPPORTED; + } +} diff --git a/src/rdma.cpp b/src/rdma.cpp new file mode 100644 index 000000000..102c693ae --- /dev/null +++ b/src/rdma.cpp @@ -0,0 +1,144 @@ +/* + * Copyright (c) 2022, The Bifrost Authors. All rights reserved. + * Copyright (c) 2022, The University of New Mexico. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of The Bifrost Authors nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "rdma.hpp" +#include "utils.hpp" + +ssize_t BFrdma_impl::send_header(BFoffset time_tag, + BFsize header_size, + const void* header, + BFoffset offset_from_head) { + BF_ASSERT(header_size <= this->max_message_size(), BF_STATUS_INVALID_SHAPE); + + bf_message msg; + msg.type = bf_message::MSG_HEADER; + msg.time_tag = time_tag; + msg.length = header_size; + msg.offset = offset_from_head; + + return _rdma->send(&msg, header); +} + +ssize_t BFrdma_impl::send_span(BFarray const* span) { + size_t span_size = BF_DTYPE_NBYTE(span->dtype); + for(int i=0; indim; i++) { + span_size *= span->shape[i]; + } + BF_ASSERT(span_size <= this->max_message_size(), BF_STATUS_INVALID_SHAPE); + + bf_message msg; + msg.type = bf_message::MSG_SPAN; + msg.time_tag = 0; + msg.length = span_size; + msg.offset = 0; + + return _rdma->send(&msg, span->data); +} + +ssize_t BFrdma_impl::receive(BFoffset* time_tag, + BFsize* header_size, + BFoffset* offset_from_head, + BFsize* span_size, + void* contents) { + bf_message msg; + ::memset(&msg, 0, sizeof(bf_message)); + ssize_t nrecv; + nrecv = _rdma->receive(&msg, contents); + if( msg.type == bf_message::MSG_HEADER) { + *time_tag = msg.time_tag; + *header_size = msg.length; + *offset_from_head = msg.offset; + *span_size = 0; + } else { + *time_tag = 0; + *header_size = 0; + *offset_from_head = 0; + *span_size = msg.length; + + } + + return nrecv; +} + +BFstatus bfRdmaCreate(BFrdma* obj, + int fd, + size_t message_size, + int is_server) { + BF_ASSERT(message_size <= BF_RDMA_MAXMEM, BF_STATUS_INSUFFICIENT_STORAGE); + BF_TRY_RETURN_ELSE(*obj = new BFrdma_impl(fd, message_size, is_server), + *obj = 0); +} + +BFstatus bfRdmaDestroy(BFrdma obj) { + BF_ASSERT(obj, BF_STATUS_INVALID_HANDLE); + delete obj; + return BF_STATUS_SUCCESS; +} + +BFstatus bfRdmaSendHeader(BFrdma obj, + BFoffset time_tag, + BFsize header_size, + const void* header, + BFoffset offset_from_head) { + BF_ASSERT(obj, BF_STATUS_INVALID_HANDLE); + BF_ASSERT(header, BF_STATUS_INVALID_POINTER); + ssize_t nsent; + nsent = obj->send_header(time_tag, header_size, header, offset_from_head); + if( nsent < 0 ) { + return BF_STATUS_INTERNAL_ERROR; + } + return BF_STATUS_SUCCESS; +} + +BFstatus bfRdmaSendSpan(BFrdma obj, + BFarray const* span) { + BF_ASSERT(obj, BF_STATUS_INVALID_HANDLE); + BF_ASSERT(span, BF_STATUS_INVALID_POINTER); + ssize_t nsent; + nsent = obj->send_span(span); + if( nsent < 0 ) { + return BF_STATUS_INTERNAL_ERROR; + } + return BF_STATUS_SUCCESS; +} + +BFstatus bfRdmaReceive(BFrdma obj, + BFoffset* time_tag, + BFsize* header_size, + BFoffset* offset_from_head, + BFsize* span_size, + void* contents) { + BF_ASSERT(obj, BF_STATUS_INVALID_HANDLE); + ssize_t nrecv; + nrecv = obj->receive(time_tag, header_size, offset_from_head, span_size, contents); + if( nrecv < 0 ) { + return BF_STATUS_INTERNAL_ERROR; + } + return BF_STATUS_SUCCESS; +} diff --git a/src/rdma.hpp b/src/rdma.hpp new file mode 100644 index 000000000..9c2819ca3 --- /dev/null +++ b/src/rdma.hpp @@ -0,0 +1,310 @@ +/* + * Copyright (c) 2022, The Bifrost Authors. All rights reserved. + * Copyright (c) 2022, The University of New Mexico. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * * Neither the name of The Bifrost Authors nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY + * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY + * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include "assert.hpp" +#include +#include + +#include // For ntohs +#include // For mlock + +#include +#include +#include +#include +#include // For posix_memalign +#include // For memcpy, memset + +#include +#include + +struct __attribute__((aligned(8))) bf_message { + enum { + MSG_HEADER = 0, + MSG_SPAN = 1, + MSG_REPLY = 2, + } type; + uint64_t time_tag; + uint64_t offset; + uint64_t length; +}; + +struct bf_rdma { + struct rdma_cm_id* listen_id; + struct rdma_cm_id* id; + + struct rdma_addrinfo* res; + struct ibv_wc wc; + + size_t buf_size; + uint8_t* buf; + uint8_t* send_buf; + struct ibv_mr* mr; + struct ibv_mr* send_mr; +}; + +class Rdma { + int _fd; + size_t _message_size; + bool _is_server; + bf_rdma _rdma; + + void get_ip_address(char* ip) { + sockaddr_in sin; + socklen_t len = sizeof(sin); + if( _is_server ) { + check_error(::getsockname(_fd, (sockaddr *)&sin, &len), + "query socket name"); + } else { + check_error(::getpeername(_fd, (sockaddr *)&sin, &len), + "query peer name"); + } + inet_ntop(AF_INET, &(sin.sin_addr), ip, INET_ADDRSTRLEN); + } + uint16_t get_port() { + sockaddr_in sin; + socklen_t len = sizeof(sin); + if( _is_server ) { + check_error(::getsockname(_fd, (sockaddr *)&sin, &len), + "query socket name"); + } else { + check_error(::getpeername(_fd, (sockaddr *)&sin, &len), + "query peer name"); + } + return ntohs(sin.sin_port); + } + void create_connection(size_t message_size) { + struct rdma_addrinfo hints; + struct ibv_qp_init_attr init_attr; + struct ibv_qp_attr qp_attr; + + char server[INET_ADDRSTRLEN]; + char port[7]; + this->get_ip_address(&(server[0])); + snprintf(&(port[0]), 7, "%d", this->get_port()); + + ::memset(&hints, 0, sizeof(hints)); + if( _is_server ) { + hints.ai_flags = RAI_PASSIVE; + } + hints.ai_port_space = RDMA_PS_TCP; + check_error(rdma_getaddrinfo(&(server[0]), &(port[0]), &hints, &(_rdma.res)), + "query RDMA address information"); + + ::memset(&init_attr, 0, sizeof(init_attr)); + init_attr.cap.max_send_wr = init_attr.cap.max_recv_wr = 1; + init_attr.cap.max_send_sge = init_attr.cap.max_recv_sge = 1; + init_attr.cap.max_inline_data = 0; + if( !_is_server ) { + init_attr.qp_context = _rdma.id; + } + init_attr.sq_sig_all = 1; + + if( _is_server ) { + check_error(rdma_create_ep(&(_rdma.listen_id), _rdma.res, NULL, &init_attr), + "create the RDMA identifier"); + + check_error(rdma_listen(_rdma.listen_id, 0), + "listen for incoming connections"); + + check_error(rdma_get_request(_rdma.listen_id, &(_rdma.id)), + "get connection request"); + + ::memset(&qp_attr, 0, sizeof(qp_attr)); + ::memset(&init_attr, 0, sizeof(init_attr)); + check_error(ibv_query_qp(_rdma.id->qp, &qp_attr, IBV_QP_CAP, &init_attr), + "query the queue pair"); + } else { + check_error(rdma_create_ep(&(_rdma.id), _rdma.res, NULL, &init_attr), + "create the RDMA identifier"); + } + + _rdma.buf_size = message_size + sizeof(bf_message); + check_error(::posix_memalign((void**)&(_rdma.buf), 32, _rdma.buf_size), + "allocate buffer"); + check_error(::mlock(_rdma.buf, _rdma.buf_size), + "lock buffer"); + _rdma.mr = rdma_reg_msgs(_rdma.id, _rdma.buf, _rdma.buf_size); + check_null(_rdma.mr, "create memory region"); + + check_error(::posix_memalign((void**)&(_rdma.send_buf), 32, _rdma.buf_size), + "allocate send buffer"); + check_error(::mlock(_rdma.send_buf, _rdma.buf_size), + "lock send buffer"); + _rdma.send_mr = rdma_reg_msgs(_rdma.id, _rdma.send_buf, _rdma.buf_size); + check_null(_rdma.send_mr, "create send memory region"); + + if( _is_server ) { + check_error(rdma_accept(_rdma.id, NULL), + "accept incomining connections"); + } else { + check_error(rdma_connect(_rdma.id, NULL), + "connect"); + + check_error(rdma_post_recv(_rdma.id, NULL, _rdma.buf, _rdma.buf_size, _rdma.mr), + "set RDMA post receive"); + } + } + inline void close_connection() { + if( _rdma.id ) { + rdma_disconnect(_rdma.id); + } + + if( _rdma.mr ) { + rdma_dereg_mr(_rdma.mr); + } + + if( _rdma.send_mr ) { + rdma_dereg_mr(_rdma.send_mr); + } + + if( _rdma.buf ) { + ::munlock(_rdma.buf, _rdma.buf_size); + ::free(_rdma.buf); + } + + if( _rdma.send_buf ) { + ::munlock(_rdma.send_buf, _rdma.buf_size); + ::free(_rdma.send_buf); + } + + if( _rdma.id ) { + rdma_destroy_ep(_rdma.id); + } + + if( _rdma.listen_id ) { + rdma_destroy_ep(_rdma.listen_id); + } + + if( _rdma.res ) { + rdma_freeaddrinfo(_rdma.res); + } + } + inline void check_error(int retval, std::string what) { + if( retval < 0 ) { + close_connection(); + + std::stringstream ss; + ss << "Failed to " << what << ": (" << errno << ") " + << strerror(errno); + throw Rdma::Error(ss.str()); + } + } + inline void check_null(void* ptr, std::string what) { + if( ptr == NULL ) { + close_connection(); + + std::stringstream ss; + ss << "Failed to " << what << ": (" << errno << ") " + << strerror(errno); + throw Rdma::Error(ss.str()); + } + } +public: + class Error : public std::runtime_error { + typedef std::runtime_error super_t; + protected: + virtual const char* what() const throw() { + return super_t::what(); + } + public: + Error(const std::string& what_arg) + : super_t(what_arg) {} + }; + + Rdma(int fd, size_t message_size, bool is_server) + : _fd(fd), _message_size(message_size), _is_server(is_server) { + ::memset(&_rdma, 0, sizeof(_rdma)); + create_connection(message_size); + } + ~Rdma() { + close_connection(); + } + inline size_t max_message_size() { return _message_size; } + inline ssize_t send(const bf_message* msg, const void* buf) { + ::memcpy(_rdma.send_buf, msg, sizeof(bf_message)); + ::memcpy(_rdma.send_buf+sizeof(bf_message), buf, msg->length); + + check_error(rdma_post_send(_rdma.id, NULL, _rdma.send_buf, _rdma.buf_size, _rdma.send_mr, 0), + "queue send request"); + check_error(rdma_get_send_comp(_rdma.id, &(_rdma.wc)), + "get send completion"); + + bf_message *reply; + check_error(rdma_post_recv(_rdma.id, NULL, _rdma.buf, sizeof(bf_message), _rdma.mr), + "queue receive request"); + check_error(rdma_get_recv_comp(_rdma.id, &(_rdma.wc)), + "get receive completion"); + + reply = reinterpret_cast(_rdma.buf); + if( reply->type != bf_message::MSG_REPLY ) { + return -1; + } + return msg->length; + } + inline ssize_t receive(bf_message* msg, void* buf) { + check_error(rdma_get_recv_comp(_rdma.id, &(_rdma.wc)), + "get receive completion"); + + ::memcpy(msg, _rdma.buf, sizeof(bf_message)); + ::memcpy(buf, _rdma.buf+sizeof(bf_message), msg->length); + + bf_message* reply; + reply = reinterpret_cast(_rdma.send_buf); + reply->type = bf_message::MSG_REPLY; + check_error(rdma_post_send(_rdma.id, NULL, _rdma.send_buf, sizeof(bf_message), _rdma.send_mr, 0), + "queue send request"); + check_error(rdma_get_send_comp(_rdma.id, &(_rdma.wc)), + "get send completion"); + check_error(rdma_post_recv(_rdma.id, NULL, _rdma.buf, _rdma.buf_size, _rdma.mr), + "queue receive request"); + + return msg->length; + } +}; + +class BFrdma_impl { + Rdma* _rdma; + +public: + inline BFrdma_impl(int fd, size_t message_size, int is_server) { + _rdma = new Rdma(fd, message_size, is_server); + } + inline size_t max_message_size() { return _rdma->max_message_size(); } + ssize_t send_header(BFoffset time_tag, + BFsize header_size, + const void* header, + BFoffset offset_from_head); + ssize_t send_span(BFarray const* span); + ssize_t receive(BFoffset* time_tag, + BFsize* header_size, + BFoffset* offset_from_head, + BFsize* span_size, + void* contents); +}; diff --git a/src/ring.cpp b/src/ring.cpp index cf96eb372..0cf0f3d79 100644 --- a/src/ring.cpp +++ b/src/ring.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, The Bifrost Authors. All rights reserved. + * Copyright (c) 2016-2022, The Bifrost Authors. All rights reserved. * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -69,7 +69,7 @@ BFstatus bfRingGetSpace(BFring ring, BFspace* space) { BFstatus bfRingSetAffinity(BFring ring, int core) { BF_ASSERT(ring, BF_STATUS_INVALID_HANDLE); BF_ASSERT(core >= -1, BF_STATUS_INVALID_ARGUMENT); - BF_ASSERT(BF_NUMA_ENABLED, BF_STATUS_UNSUPPORTED); + BF_ASSERT(BF_HWLOC_ENABLED, BF_STATUS_UNSUPPORTED); BF_TRY_RETURN(ring->set_core(core)); } BFstatus bfRingGetAffinity(BFring ring, int* core) { diff --git a/src/ring_impl.cpp b/src/ring_impl.cpp index e0471b7ab..cc452ac24 100644 --- a/src/ring_impl.cpp +++ b/src/ring_impl.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, The Bifrost Authors. All rights reserved. + * Copyright (c) 2016-2022, The Bifrost Authors. All rights reserved. * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -45,6 +45,7 @@ // Work out whether/how to support independent specification of // buffer_factor. +#include "hw_locality.hpp" #include "ring_impl.hpp" #include "utils.hpp" #include "assert.hpp" @@ -54,10 +55,6 @@ #include #include "cuda.hpp" -#if BF_NUMA_ENABLED -#include -#endif - // This implements a lock with the condition that no reads or writes // can be open while it is held. class RingReallocLock { @@ -151,23 +148,22 @@ void BFring_impl::resize(BFsize contiguous_span, BFsize new_stride = new_span + new_ghost_span; BFsize new_nbyte = new_stride*new_nringlet; //pointer new_buf = (pointer)bfMalloc(new_nbyte, _space); - //std::cout << "new_buf = " << (void*)new_buf << std::endl; // HACK TESTING + //std::std::cout << "new_buf = " << (void*)new_buf << std::endl; // HACK TESTING pointer new_buf = nullptr; - //std::cout << "contig_span: " << contiguous_span << std::endl; - //std::cout << "total_span: " << total_span << std::endl; - //std::cout << "new_span: " << new_span << std::endl; - //std::cout << "new_ghost_span: " << new_ghost_span << std::endl; - //std::cout << "new_nringlet: " << new_nringlet << std::endl; - //std::cout << "new_stride: " << new_stride << std::endl; - //std::cout << "Allocating " << new_nbyte << std::endl; + //std::std::cout << "contig_span: " << contiguous_span << std::endl; + //std::std::cout << "total_span: " << total_span << std::endl; + //std::std::cout << "new_span: " << new_span << std::endl; + //std::std::cout << "new_ghost_span: " << new_ghost_span << std::endl; + //std::std::cout << "new_nringlet: " << new_nringlet << std::endl; + //std::std::cout << "new_stride: " << new_stride << std::endl; + //std::std::cout << "Allocating " << new_nbyte << std::endl; BF_ASSERT_EXCEPTION(bfMalloc((void**)&new_buf, new_nbyte, _space) == BF_STATUS_SUCCESS, BF_STATUS_MEM_ALLOC_FAILED); -#if BF_NUMA_ENABLED +#if BF_HWLOC_ENABLED if( _core != -1 ) { - BF_ASSERT_EXCEPTION(numa_available() != -1, BF_STATUS_UNSUPPORTED); - int node = numa_node_of_cpu(_core); + int node = _hwloc.get_numa_node_of_core(_core); BF_ASSERT_EXCEPTION(node != -1, BF_STATUS_INVALID_ARGUMENT); - numa_tonode_memory(new_buf, new_nbyte, node); + _hwloc.bind_memory_area_to_numa_node(new_buf, new_nbyte, node); } #endif if( _buf ) { @@ -479,7 +475,7 @@ void BFring_impl::finish_sequence(BFsequence_sptr sequence, void BFring_impl::_write_proclog_entry() { char cinfo[32]=""; - #if BF_NUMA_ENABLED + #if BF_HWLOC_ENABLED snprintf(cinfo, 31, "binding : %i\n", _core); #endif _size_log.update("space : %s\n" @@ -591,7 +587,7 @@ void BFring_impl::commit_span(BFoffset begin, BFsize reserve_size, BFsize commit // in which case they will block here until they are // in order (i.e., they will automatically synchronise). // This is useful for multithreading with OpenMP - //std::cout << "(1) begin, head, rhead: " << begin << ", " << _head << ", " << _reserve_head << std::endl; + //std::std::cout << "(1) begin, head, rhead: " << begin << ", " << _head << ", " << _reserve_head << std::endl; _write_close_condition.wait(lock, [&]() { return (begin == _head); }); @@ -606,7 +602,7 @@ void BFring_impl::commit_span(BFoffset begin, BFsize reserve_size, BFsize commit // There are reservations in front of this one, so we // are not allowed to commit less than size. // TODO: How to deal with error here? - //std::cout << "BFRING ERROR: Must commit whole wspan when other spans are reserved" << std::endl; + //std::std::cout << "BFRING ERROR: Must commit whole wspan when other spans are reserved" << std::endl; //return; BF_ASSERT_EXCEPTION(false, BF_STATUS_INVALID_STATE); } diff --git a/src/ring_impl.hpp b/src/ring_impl.hpp index 5824fd923..94c9e274a 100644 --- a/src/ring_impl.hpp +++ b/src/ring_impl.hpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016, The Bifrost Authors. All rights reserved. + * Copyright (c) 2016-2022, The Bifrost Authors. All rights reserved. * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -32,6 +32,7 @@ #include #include "assert.hpp" #include "proclog.hpp" +#include "hw_locality.hpp" #include #include @@ -96,8 +97,11 @@ class BFring_impl { BFsize _nwrite_open; BFsize _nrealloc_pending; - int _core; - ProcLog _size_log; +#if BF_HWLOC_ENABLED + HardwareLocality _hwloc; +#endif + int _core; + ProcLog _size_log; std::queue _sequence_queue; std::map _sequence_map; diff --git a/src/udp_capture.cpp b/src/udp_capture.cpp deleted file mode 100644 index 86058c978..000000000 --- a/src/udp_capture.cpp +++ /dev/null @@ -1,844 +0,0 @@ -/* - * Copyright (c) 2016-2021, The Bifrost Authors. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of The Bifrost Authors nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY - * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include "assert.hpp" -#include -#include -#include -#include -using bifrost::ring::RingWrapper; -using bifrost::ring::RingWriter; -using bifrost::ring::WriteSpan; -using bifrost::ring::WriteSequence; -#include "proclog.hpp" - -#include // For ntohs -#include // For recvfrom - -#include -#include -#include -#include // For posix_memalign -#include // For memcpy, memset -#include - -#include -#include -#include -#include - -//#include // SSE - -// TODO: The VMA API is returning unaligned buffers, which prevents use of SSE - -#define BF_UNPACK_FACTOR 1 - -enum { - JUMBO_FRAME_SIZE = 9000 -}; - -#if defined __APPLE__ && __APPLE__ - -#include - -#define be64toh(x) OSSwapBigToHostInt64(x) - -#endif - -template -inline T atomic_add_and_fetch(T* dst, T val) { - return __sync_add_and_fetch(dst, val); // GCC builtin -} -template -inline T atomic_fetch_and_add(T* dst, T val) { - return __sync_fetch_and_add(dst, val); // GCC builtin -} - -// Wrap-safe comparisons -inline bool greater_equal(uint64_t a, uint64_t b) { return int64_t(a-b) >= 0; } -inline bool less_than( uint64_t a, uint64_t b) { return int64_t(a-b) < 0; } - -#if BF_VMA_ENABLED -#include -class VMAReceiver { - int _fd; - vma_api_t* _api; - vma_packet_t* _pkt; - inline void clean_cache() { - if( _pkt ) { - _api->free_packets(_fd, _pkt, 1); - _pkt = 0; - } - } -public: - VMAReceiver(int fd) - : _fd(fd), _api(vma_get_api()), _pkt(0) {} - VMAReceiver(VMAReceiver const& other) - : _fd(other._fd), _api(other._api), _pkt(0) {} - VMAReceiver& operator=(VMAReceiver const& other) { - if( &other != this ) { - this->clean_cache(); - _fd = other._fd; - _api = other._api; - } - return *this; - } - ~VMAReceiver() { this->clean_cache(); } - inline int recv_packet(uint8_t* buf, size_t bufsize, uint8_t** pkt_ptr, int flags=0) { - this->clean_cache(); - int ret = _api->recvfrom_zcopy(_fd, buf, bufsize, &flags, 0, 0); - if( ret < 0 ) { - return ret; - } - if( flags & MSG_VMA_ZCOPY ) { - _pkt = &((vma_packets_t*)buf)->pkts[0]; - *pkt_ptr = (uint8_t*)_pkt->iov[0].iov_base; - } else { - *pkt_ptr = buf; - } - return ret; - } - inline operator bool() const { return _api != NULL; } -}; -#endif // BF_VMA_ENABLED - -template -class AlignedBuffer { - enum { DEFAULT_ALIGNMENT = 4096 }; - T* _buf; - size_t _size; - size_t _alignment; - void alloc() { - int err = ::posix_memalign((void**)&_buf, _alignment, _size*sizeof(T)); - if( err ) { - throw std::runtime_error("Allocation failed"); - } - } - void copy(T const* srcbuf, size_t n) { - ::memcpy(_buf, srcbuf, n*sizeof(T)); - } - void free() { - if( _buf ) { - ::free(_buf); - _buf = 0; - _size = 0; - } - } -public: - //explicit AlignedBuffer(size_t alignment=DEFAULT_ALIGNMENT) - // : _buf(0), _size(0), _alignment(alignment) {} - AlignedBuffer(size_t size=0, size_t alignment=DEFAULT_ALIGNMENT) - : _buf(0), _size(size), _alignment(alignment) { - this->alloc(); - } - AlignedBuffer(AlignedBuffer const& other) - : _buf(0), _size(other.size), _alignment(other.alignment) { - this->alloc(); - this->copy(other._buf, other._size); - } - AlignedBuffer& operator=(AlignedBuffer const& other) { - if( &other != this ) { - this->free(); - _size = other.size; - _alignment = other.alignment; - this->alloc(); - this->copy(other._buf, other._size); - } - return *this; - } - ~AlignedBuffer() { - this->free(); - } - inline void swap(AlignedBuffer & other) { - std::swap(_buf, other._buf); - std::swap(_size, other._size); - std::swap(_alignment, other._alignment); - } - inline void resize(size_t n) { - if( n <= _size ) { - _size = n; - } else { - AlignedBuffer tmp(n, _alignment); - tmp.copy(&_buf[0], _size); - tmp.swap(*this); - } - } - inline size_t size() const { return _size; } - inline T & operator[](size_t i) { return _buf[i]; } - inline T const& operator[](size_t i) const { return _buf[i]; } -}; - -#if BF_HWLOC_ENABLED -#include -class HardwareLocality { - hwloc_topology_t _topo; - HardwareLocality(HardwareLocality const&); - HardwareLocality& operator=(HardwareLocality const&); -public: - HardwareLocality() { - hwloc_topology_init(&_topo); - hwloc_topology_load(_topo); - } - ~HardwareLocality() { - hwloc_topology_destroy(_topo); - } - int bind_memory_to_core(int core) { - int core_depth = hwloc_get_type_or_below_depth(_topo, HWLOC_OBJ_CORE); - int ncore = hwloc_get_nbobjs_by_depth(_topo, core_depth); - int ret = 0; - if( 0 <= core && core < ncore ) { - hwloc_obj_t obj = hwloc_get_obj_by_depth(_topo, core_depth, core); - hwloc_cpuset_t cpuset = hwloc_bitmap_dup(obj->cpuset); - hwloc_bitmap_singlify(cpuset); // Avoid hyper-threads - hwloc_membind_policy_t policy = HWLOC_MEMBIND_BIND; - hwloc_membind_flags_t flags = HWLOC_MEMBIND_THREAD; - ret = hwloc_set_membind(_topo, cpuset, policy, flags); - hwloc_bitmap_free(cpuset); - } - return ret; - } -}; -#endif // BF_HWLOC_ENABLED - -class BoundThread { -#if BF_HWLOC_ENABLED - HardwareLocality _hwloc; -#endif -public: - BoundThread(int core) { - bfAffinitySetCore(core); -#if BF_HWLOC_ENABLED - assert(_hwloc.bind_memory_to_core(core) == 0); -#endif - } -}; - -class UDPPacketReceiver { - int _fd; - AlignedBuffer _buf; -#if BF_VMA_ENABLED - VMAReceiver _vma; -#endif -public: - UDPPacketReceiver(int fd, size_t pkt_size_max=JUMBO_FRAME_SIZE) - : _fd(fd), _buf(pkt_size_max) -#if BF_VMA_ENABLED - , _vma(fd) -#endif - {} - inline int recv_packet(uint8_t** pkt_ptr, int flags=0) { -#if BF_VMA_ENABLED - if( _vma ) { - *pkt_ptr = 0; - return _vma.recv_packet(&_buf[0], _buf.size(), pkt_ptr, flags); - } else { -#endif - *pkt_ptr = &_buf[0]; - return ::recvfrom(_fd, &_buf[0], _buf.size(), flags, 0, 0); -#if BF_VMA_ENABLED - } -#endif - } -}; - -struct PacketDesc { - uint64_t seq; - int nsrc; - int src; - int nchan; - int chan0; - int payload_size; - const uint8_t* payload_ptr; -}; -struct PacketStats { - size_t ninvalid; - size_t ninvalid_bytes; - size_t nlate; - size_t nlate_bytes; - size_t nvalid; - size_t nvalid_bytes; -}; - -class UDPCaptureThread : public BoundThread { - UDPPacketReceiver _udp; - PacketStats _stats; - std::vector _src_stats; - bool _have_pkt; - PacketDesc _pkt; -public: - enum { - CAPTURE_SUCCESS = 1 << 0, - CAPTURE_TIMEOUT = 1 << 1, - CAPTURE_INTERRUPTED = 1 << 2, - CAPTURE_ERROR = 1 << 3 - }; - UDPCaptureThread(int fd, int nsrc, int core=0, size_t pkt_size_max=9000) - : BoundThread(core), _udp(fd, pkt_size_max), _src_stats(nsrc), - _have_pkt(false) { - this->reset_stats(); - } - // Captures, decodes and unpacks packets into the provided buffers - // Note: Capture continues until the first packet that belongs - // beyond the end of the provided buffers. This packet is - // saved, accessible via get_last_packet(), and will be - // processed on the next call to run() if possible. - template - int run(uint64_t seq_beg, - uint64_t nseq_per_obuf, - int nbuf, - uint8_t* obufs[], - size_t* ngood_bytes[], - size_t* src_ngood_bytes[], - PacketDecoder* decode, - PacketProcessor* process) { - uint64_t seq_end = seq_beg + nbuf*nseq_per_obuf; - size_t local_ngood_bytes[2] = {0, 0}; - int ret; - while( true ) { - if( !_have_pkt ) { - uint8_t* pkt_ptr; - int pkt_size = _udp.recv_packet(&pkt_ptr); - if( pkt_size <= 0 ) { - if( errno == EAGAIN || errno == EWOULDBLOCK ) { - ret = CAPTURE_TIMEOUT; // Timed out - } else if( errno == EINTR ) { - ret = CAPTURE_INTERRUPTED; // Interrupted by signal - } else { - ret = CAPTURE_ERROR; // Socket error - } - break; - } - if( !(*decode)(pkt_ptr, pkt_size, &_pkt) ) { - ++_stats.ninvalid; - _stats.ninvalid_bytes += pkt_size; - continue; - } - _have_pkt = true; - } - if( greater_equal(_pkt.seq, seq_end) ) { - // Reached the end of this processing gulp, so leave this - // packet unprocessed and return. - ret = CAPTURE_SUCCESS; - break; - } - _have_pkt = false; - if( less_than(_pkt.seq, seq_beg) ) { - ++_stats.nlate; - _stats.nlate_bytes += _pkt.payload_size; - ++_src_stats[_pkt.src].nlate; - _src_stats[_pkt.src].nlate_bytes += _pkt.payload_size; - continue; - } - ++_stats.nvalid; - _stats.nvalid_bytes += _pkt.payload_size; - ++_src_stats[_pkt.src].nvalid; - _src_stats[_pkt.src].nvalid_bytes += _pkt.payload_size; - // HACK TODO: src_ngood_bytes should be accumulated locally and - // then atomically updated, like ngood_bytes. The - // current way is not thread-safe. - (*process)(&_pkt, seq_beg, nseq_per_obuf, nbuf, obufs, - local_ngood_bytes, /*local_*/src_ngood_bytes); - } - if( nbuf > 0 ) { atomic_add_and_fetch(ngood_bytes[0], local_ngood_bytes[0]); } - if( nbuf > 1 ) { atomic_add_and_fetch(ngood_bytes[1], local_ngood_bytes[1]); } - return ret; - } - inline const PacketDesc* get_last_packet() const { - return _have_pkt ? &_pkt : NULL; - } - inline const PacketStats* get_stats() const { return &_stats; } - inline const PacketStats* get_stats(int src) const { return &_src_stats[src]; } - inline void reset_stats() { - ::memset(&_stats, 0, sizeof(_stats)); - ::memset(&_src_stats[0], 0, _src_stats.size()*sizeof(PacketStats)); - } -}; - -#pragma pack(1) -struct chips_hdr_type { - uint8_t roach; // Note: 1-based - uint8_t gbe; // (AKA tuning) - uint8_t nchan; // 109 - uint8_t nsubband; // 11 - uint8_t subband; // 0-11 - uint8_t nroach; // 16 - // Note: Big endian - uint16_t chan0; // First chan in packet - uint64_t seq; // Note: 1-based -}; - -class CHIPSDecoder { - // TODO: See if can remove these once firmware supports per-GbE nroach - int _nsrc; - int _src0; - inline bool valid_packet(const PacketDesc* pkt) const { - return (pkt->seq >= 0 && - pkt->src >= 0 && pkt->src < _nsrc && - pkt->chan0 >= 0); - } -public: - CHIPSDecoder(int nsrc, int src0) : _nsrc(nsrc), _src0(src0) {} - inline bool operator()(const uint8_t* pkt_ptr, - int pkt_size, - PacketDesc* pkt) const { - if( pkt_size < (int)sizeof(chips_hdr_type) ) { - return false; - } - const chips_hdr_type* pkt_hdr = (chips_hdr_type*)pkt_ptr; - const uint8_t* pkt_pld = pkt_ptr + sizeof(chips_hdr_type); - int pld_size = pkt_size - sizeof(chips_hdr_type); - pkt->seq = be64toh(pkt_hdr->seq) - 1; - //pkt->nsrc = pkt_hdr->nroach; - pkt->nsrc = _nsrc; - pkt->src = (pkt_hdr->roach - 1) - _src0; - pkt->nchan = pkt_hdr->nchan; - pkt->chan0 = ntohs(pkt_hdr->chan0); - pkt->payload_size = pld_size; - pkt->payload_ptr = pkt_pld; - return this->valid_packet(pkt); - } -}; - -struct __attribute__((aligned(32))) aligned256_type { - uint8_t data[32]; -}; -struct __attribute__((aligned(64))) aligned512_type { - uint8_t data[64]; -}; - -class CHIPSProcessor8bit { -public: - inline void operator()(const PacketDesc* pkt, - uint64_t seq0, - uint64_t nseq_per_obuf, - int nbuf, - uint8_t* obufs[], - size_t ngood_bytes[], - size_t* src_ngood_bytes[]) { - enum { - PKT_NINPUT = 32, - PKT_NBIT = 4 - }; - int obuf_idx = ((pkt->seq - seq0 >= 1*nseq_per_obuf) + - (pkt->seq - seq0 >= 2*nseq_per_obuf)); - size_t obuf_seq0 = seq0 + obuf_idx*nseq_per_obuf; - size_t nbyte = pkt->payload_size * BF_UNPACK_FACTOR; - ngood_bytes[obuf_idx] += nbyte; - src_ngood_bytes[obuf_idx][pkt->src] += nbyte; - // **CHANGED RECENTLY - int payload_size = pkt->payload_size;//pkt->nchan*(PKT_NINPUT*2*PKT_NBIT/8); - - size_t obuf_offset = (pkt->seq-obuf_seq0)*pkt->nsrc*payload_size; - typedef aligned256_type itype; - typedef aligned256_type otype; - - obuf_offset *= BF_UNPACK_FACTOR; - - // Note: Using these SSE types allows the compiler to use SSE instructions - // However, they require aligned memory (otherwise segfault) - itype const* __restrict__ in = (itype const*)pkt->payload_ptr; - otype* __restrict__ out = (otype* )&obufs[obuf_idx][obuf_offset]; - - int chan = 0; - //cout << pkt->src << ", " << pkt->nsrc << endl; - //cout << pkt->nchan << endl; - /* - // HACK TESTING disabled - for( ; channchan/4*4; chan+=4 ) { - __m128i tmp0 = ((__m128i*)&in[chan])[0]; - __m128i tmp1 = ((__m128i*)&in[chan])[1]; - __m128i tmp2 = ((__m128i*)&in[chan+1])[0]; - __m128i tmp3 = ((__m128i*)&in[chan+1])[1]; - __m128i tmp4 = ((__m128i*)&in[chan+2])[0]; - __m128i tmp5 = ((__m128i*)&in[chan+2])[1]; - __m128i tmp6 = ((__m128i*)&in[chan+3])[0]; - __m128i tmp7 = ((__m128i*)&in[chan+3])[1]; - _mm_store_si128(&((__m128i*)&out[pkt->src + pkt->nsrc*chan])[0], tmp0); - _mm_store_si128(&((__m128i*)&out[pkt->src + pkt->nsrc*chan])[1], tmp1); - _mm_store_si128(&((__m128i*)&out[pkt->src + pkt->nsrc*(chan+1)])[0], tmp2); - _mm_store_si128(&((__m128i*)&out[pkt->src + pkt->nsrc*(chan+1)])[1], tmp3); - _mm_store_si128(&((__m128i*)&out[pkt->src + pkt->nsrc*(chan+2)])[0], tmp4); - _mm_store_si128(&((__m128i*)&out[pkt->src + pkt->nsrc*(chan+2)])[1], tmp5); - _mm_store_si128(&((__m128i*)&out[pkt->src + pkt->nsrc*(chan+3)])[0], tmp6); - _mm_store_si128(&((__m128i*)&out[pkt->src + pkt->nsrc*(chan+3)])[1], tmp7); - } - */ - //for( ; channchan; ++chan ) { - /* - for( ; chan<10; ++chan ) { // HACK TESTING - __m128i tmp0 = ((__m128i*)&in[chan])[0]; - __m128i tmp1 = ((__m128i*)&in[chan])[1]; - _mm_store_si128(&((__m128i*)&out[pkt->src + pkt->nsrc*chan])[0], tmp0); - _mm_store_si128(&((__m128i*)&out[pkt->src + pkt->nsrc*chan])[1], tmp1); - } - */ - //if( pkt->src < 8 ) { // HACK TESTING - //for( ; chan<32; ++chan ) { // HACK TESTING - for( ; channchan; ++chan ) { // HACK TESTING - out[pkt->src + pkt->nsrc*chan] = in[chan]; - //::memset( - } - //} - } - inline void blank_out_source(uint8_t* data, - int src, - int nsrc, - int nchan, - int nseq) { - typedef aligned256_type otype; - otype* __restrict__ aligned_data = (otype*)data; - for( int t=0; t _process_time; - std::chrono::duration _reserve_time; - - int _nsrc; - int _nseq_per_buf; - int _slot_ntime; - BFoffset _seq; - int _chan0; - int _nchan; - int _payload_size; - bool _active; - BFudpcapture_sequence_callback _sequence_callback; - - RingWrapper _ring; - RingWriter _oring; - std::queue > _bufs; - std::queue _buf_ngood_bytes; - std::queue > _buf_src_ngood_bytes; - std::shared_ptr _sequence; - size_t _ngood_bytes; - size_t _nmissing_bytes; - - inline size_t bufsize(int payload_size=-1) { - if( payload_size == -1 ) { - payload_size = _payload_size; - } - return (size_t) _nseq_per_buf * _nsrc * payload_size * BF_UNPACK_FACTOR; - } - inline void reserve_buf() { - _buf_ngood_bytes.push(0); - _buf_src_ngood_bytes.push(std::vector(_nsrc, 0)); - size_t size = this->bufsize(); - // TODO: Can make this simpler? - _bufs.push(std::shared_ptr(new bifrost::ring::WriteSpan(_oring, size))); - } - inline void commit_buf() { - size_t expected_bytes = _bufs.front()->size(); - - for( int src=0; src<_nsrc; ++src ) { - // TODO: This assumes all sources contribute equally; should really - // allow non-uniform partitioning. - size_t src_expected_bytes = expected_bytes / _nsrc; - size_t src_ngood_bytes = _buf_src_ngood_bytes.front()[src]; - size_t src_nmissing_bytes = src_expected_bytes - src_ngood_bytes; - // Detect >50% missing data from this source - if( src_nmissing_bytes > src_ngood_bytes ) { - // Zero-out this source's contribution to the buffer - uint8_t* data = (uint8_t*)_bufs.front()->data(); - _processor.blank_out_source(data, src, _nsrc, - _nchan, _nseq_per_buf); - } - } - _buf_src_ngood_bytes.pop(); - - _ngood_bytes += _buf_ngood_bytes.front(); - //_nmissing_bytes += _bufs.front()->size() - _buf_ngood_bytes.front(); - //// HACK TESTING 15/16 correction for missing roach11 - //_nmissing_bytes += _bufs.front()->size()*15/16 - _buf_ngood_bytes.front(); - _nmissing_bytes += expected_bytes - _buf_ngood_bytes.front(); - _buf_ngood_bytes.pop(); - - _bufs.front()->commit(); - _bufs.pop(); - _seq += _nseq_per_buf; - } - inline void begin_sequence() { - BFoffset seq0 = _seq;// + _nseq_per_buf*_bufs.size(); - const void* hdr; - size_t hdr_size; - BFoffset time_tag; - if( _sequence_callback ) { - int status = (*_sequence_callback)(seq0, - _chan0, - _nchan, - _nsrc, - &time_tag, - &hdr, - &hdr_size); - if( status != 0 ) { - // TODO: What to do here? Needed? - throw std::runtime_error("BAD HEADER CALLBACK STATUS"); - } - } else { - // Simple default for easy testing - time_tag = seq0; - hdr = NULL; - hdr_size = 0; - } - const char* name = ""; - int nringlet = 1; - _sequence.reset(new WriteSequence(_oring, name, time_tag, - hdr_size, hdr, nringlet)); - } - inline void end_sequence() { - _sequence.reset(); // Note: This is releasing the shared_ptr - } -public: - inline BFudpcapture_impl(int fd, - BFring ring, - int nsrc, - int src0, - int max_payload_size, - int buffer_ntime, - int slot_ntime, - BFudpcapture_sequence_callback sequence_callback, - int core) - : _capture(fd, nsrc, core), _decoder(nsrc, src0), _processor(), - _type_log("udp_capture/type"), - _bind_log("udp_capture/bind"), - _out_log("udp_capture/out"), - _size_log("udp_capture/sizes"), - _chan_log("udp_capture/chans"), - _stat_log("udp_capture/stats"), - _perf_log("udp_capture/perf"), - _nsrc(nsrc), _nseq_per_buf(buffer_ntime), _slot_ntime(slot_ntime), - _seq(), _chan0(), _nchan(), _active(false), - _sequence_callback(sequence_callback), - _ring(ring), _oring(_ring), - // TODO: Add reset method for stats - _ngood_bytes(0), _nmissing_bytes(0) { - size_t contig_span = this->bufsize(max_payload_size); - // Note: 2 write bufs may be open for writing at one time - size_t total_span = contig_span * 4; - size_t nringlet_max = 1; - _ring.resize(contig_span, total_span, nringlet_max); - _type_log.update("type : %s", "chips"); - _bind_log.update("ncore : %i\n" - "core0 : %i\n", - 1, core); - _out_log.update("nring : %i\n" - "ring0 : %s\n", - 1, _ring.name()); - _size_log.update("nsrc : %i\n" - "nseq_per_buf : %i\n" - "slot_ntime : %i\n", - _nsrc, _nseq_per_buf, _slot_ntime); - } - inline void flush() { - while( _bufs.size() ) { - this->commit_buf(); - } - if( _sequence ) { - this->end_sequence(); - } - } - inline void end_writing() { - this->flush(); - _oring.close(); - } - BFudpcapture_status recv() { - _t0 = std::chrono::high_resolution_clock::now(); - - uint8_t* buf_ptrs[2]; - // Minor HACK to access the buffers in a 2-element queue - buf_ptrs[0] = _bufs.size() > 0 ? (uint8_t*)_bufs.front()->data() : NULL; - buf_ptrs[1] = _bufs.size() > 1 ? (uint8_t*)_bufs.back()->data() : NULL; - - size_t* ngood_bytes_ptrs[2]; - ngood_bytes_ptrs[0] = _buf_ngood_bytes.size() > 0 ? &_buf_ngood_bytes.front() : NULL; - ngood_bytes_ptrs[1] = _buf_ngood_bytes.size() > 1 ? &_buf_ngood_bytes.back() : NULL; - - size_t* src_ngood_bytes_ptrs[2]; - src_ngood_bytes_ptrs[0] = _buf_src_ngood_bytes.size() > 0 ? &_buf_src_ngood_bytes.front()[0] : NULL; - src_ngood_bytes_ptrs[1] = _buf_src_ngood_bytes.size() > 1 ? &_buf_src_ngood_bytes.back()[0] : NULL; - - int state = _capture.run(_seq, - _nseq_per_buf, - _bufs.size(), - buf_ptrs, - ngood_bytes_ptrs, - src_ngood_bytes_ptrs, - &_decoder, - &_processor); - if( state & UDPCaptureThread::CAPTURE_ERROR ) { - return BF_CAPTURE_ERROR; - } else if( state & UDPCaptureThread::CAPTURE_INTERRUPTED ) { - return BF_CAPTURE_INTERRUPTED; - } - const PacketStats* stats = _capture.get_stats(); - _stat_log.update() << "ngood_bytes : " << _ngood_bytes << "\n" - << "nmissing_bytes : " << _nmissing_bytes << "\n" - << "ninvalid : " << stats->ninvalid << "\n" - << "ninvalid_bytes : " << stats->ninvalid_bytes << "\n" - << "nlate : " << stats->nlate << "\n" - << "nlate_bytes : " << stats->nlate_bytes << "\n" - << "nvalid : " << stats->nvalid << "\n" - << "nvalid_bytes : " << stats->nvalid_bytes << "\n"; - - _t1 = std::chrono::high_resolution_clock::now(); - - BFudpcapture_status ret; - bool was_active = _active; - _active = state & UDPCaptureThread::CAPTURE_SUCCESS; - if( _active ) { - const PacketDesc* pkt = _capture.get_last_packet(); - if( pkt ) { - //cout << "Latest nchan, chan0 = " << pkt->nchan << ", " << pkt->chan0 << endl; - } - else { - //cout << "No latest packet" << endl; - } - if( !was_active ) { - //cout << "Beginning of sequence, first pkt seq = " << pkt->seq << endl; - // TODO: Might be safer to round to nearest here, but the current firmware - // always starts things ~3 seq's before the 1sec boundary anyway. - //seq = round_up(pkt->seq, _slot_ntime); - //*_seq = round_nearest(pkt->seq, _slot_ntime); - _seq = round_up(pkt->seq, _slot_ntime); - _chan0 = pkt->chan0; - _nchan = pkt->nchan; - _payload_size = pkt->payload_size; - _chan_log.update() << "chan0 : " << _chan0 << "\n" - << "nchan : " << _nchan << "\n" - << "payload_size : " << _payload_size << "\n"; - this->begin_sequence(); - ret = BF_CAPTURE_STARTED; - } else { - //cout << "Continuing data, seq = " << seq << endl; - if( pkt->chan0 != _chan0 || - pkt->nchan != _nchan ) { - _chan0 = pkt->chan0; - _nchan = pkt->nchan; - _payload_size = pkt->payload_size; - _chan_log.update() << "chan0 : " << _chan0 << "\n" - << "nchan : " << _nchan << "\n" - << "payload_size : " << _payload_size << "\n"; - this->end_sequence(); - this->begin_sequence(); - ret = BF_CAPTURE_CHANGED; - } else { - ret = BF_CAPTURE_CONTINUED; - } - } - if( _bufs.size() == 2 ) { - this->commit_buf(); - } - this->reserve_buf(); - } else { - - if( was_active ) { - this->flush(); - ret = BF_CAPTURE_ENDED; - } else { - ret = BF_CAPTURE_NO_DATA; - } - } - - _t2 = std::chrono::high_resolution_clock::now(); - _process_time = std::chrono::duration_cast>(_t1-_t0); - _reserve_time = std::chrono::duration_cast>(_t2-_t1); - _perf_log.update() << "acquire_time : " << -1.0 << "\n" - << "process_time : " << _process_time.count() << "\n" - << "reserve_time : " << _reserve_time.count() << "\n"; - - return ret; - } -}; - -BFstatus bfUdpCaptureCreate(BFudpcapture* obj, - const char* format, - int fd, - BFring ring, - BFsize nsrc, - BFsize src0, - BFsize max_payload_size, - BFsize buffer_ntime, - BFsize slot_ntime, - BFudpcapture_sequence_callback sequence_callback, - int core) { - BF_ASSERT(obj, BF_STATUS_INVALID_POINTER); - if( format == std::string("chips") ) { - BF_TRY_RETURN_ELSE(*obj = new BFudpcapture_impl(fd, ring, nsrc, src0, max_payload_size, - buffer_ntime, slot_ntime, - sequence_callback, core), - *obj = 0); - } else { - return BF_STATUS_UNSUPPORTED; - } -} -BFstatus bfUdpCaptureDestroy(BFudpcapture obj) { - BF_ASSERT(obj, BF_STATUS_INVALID_HANDLE); - delete obj; - return BF_STATUS_SUCCESS; -} -BFstatus bfUdpCaptureRecv(BFudpcapture obj, BFudpcapture_status* result) { - BF_ASSERT(obj, BF_STATUS_INVALID_HANDLE); - BF_TRY_RETURN_ELSE(*result = obj->recv(), - *result = BF_CAPTURE_ERROR); -} -BFstatus bfUdpCaptureFlush(BFudpcapture obj) { - BF_ASSERT(obj, BF_STATUS_INVALID_HANDLE); - BF_TRY_RETURN(obj->flush()); -} -BFstatus bfUdpCaptureEnd(BFudpcapture obj) { - BF_ASSERT(obj, BF_STATUS_INVALID_HANDLE); - BF_TRY_RETURN(obj->end_writing()); -} diff --git a/src/udp_socket.cpp b/src/udp_socket.cpp index 32d83125d..bd2cd10b4 100644 --- a/src/udp_socket.cpp +++ b/src/udp_socket.cpp @@ -1,5 +1,5 @@ /* - * Copyright (c) 2016-2021, The Bifrost Authors. All rights reserved. + * Copyright (c) 2016-2023, The Bifrost Authors. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -55,6 +55,11 @@ BFstatus bfUdpSocketBind( BFudpsocket obj, BFaddress local_addr) { BF_ASSERT(local_addr, BF_STATUS_INVALID_ARGUMENT); BF_TRY_RETURN(obj->bind(*(sockaddr_storage*)local_addr)); } +BFstatus bfUdpSocketSniff( BFudpsocket obj, BFaddress local_addr) { + BF_ASSERT(obj, BF_STATUS_INVALID_HANDLE); + BF_ASSERT(local_addr, BF_STATUS_INVALID_ARGUMENT); + BF_TRY_RETURN(obj->sniff(*(sockaddr_storage*)local_addr)); +} BFstatus bfUdpSocketShutdown(BFudpsocket obj) { BF_ASSERT(obj, BF_STATUS_INVALID_HANDLE); BF_TRY_RETURN(obj->shutdown()); @@ -85,6 +90,27 @@ BFstatus bfUdpSocketGetTimeout(BFudpsocket obj, double* secs) { //BF_TRY(*secs = obj->get_timeout(), // *secs = 0); } +BFstatus bfUdpSocketSetPromiscuous(BFudpsocket obj, int state) { + BF_ASSERT(obj, BF_STATUS_INVALID_HANDLE); + BF_TRY_RETURN(obj->set_promiscuous(state)); +} +BFstatus bfUdpSocketGetPromiscuous(BFudpsocket obj, int* promisc) { + BF_ASSERT(obj, BF_STATUS_INVALID_HANDLE); + BF_ASSERT(promisc, BF_STATUS_INVALID_POINTER); + // WAR for old Socket implem returning Socket::Error not BFexception + try { + *promisc = obj->get_promiscuous(); + } + catch( Socket::Error& ) { + *promisc = 0; + return BF_STATUS_INVALID_STATE; + } + catch(...) { + *promisc = 0; + return BF_STATUS_INTERNAL_ERROR; + } + return BF_STATUS_SUCCESS; +} BFstatus bfUdpSocketGetMTU(BFudpsocket obj, int* mtu) { BF_ASSERT(obj, BF_STATUS_INVALID_HANDLE); BF_ASSERT(mtu, BF_STATUS_INVALID_POINTER); diff --git a/src/udp_transmit.cpp b/src/udp_transmit.cpp deleted file mode 100644 index 0557dedef..000000000 --- a/src/udp_transmit.cpp +++ /dev/null @@ -1,235 +0,0 @@ -/* - * Copyright (c) 2017-2021, The Bifrost Authors. All rights reserved. - * Copyright (c) 2017-2021, The University of New Mexico. All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions - * are met: - * * Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. - * * Redistributions in binary form must reproduce the above copyright - * notice, this list of conditions and the following disclaimer in the - * documentation and/or other materials provided with the distribution. - * * Neither the name of The Bifrost Authors nor the names of its - * contributors may be used to endorse or promote products derived - * from this software without specific prior written permission. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY - * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR - * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY - * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#include "assert.hpp" -#include -#include -#include -#include "proclog.hpp" -#include "Socket.hpp" - -#include // For ntohs -#include // For recvfrom - -#include -#include -#include -#include // For posix_memalign -#include // For memcpy, memset -#include - -#include -#include -#include - -#if BF_HWLOC_ENABLED -#include -class HardwareLocality { - hwloc_topology_t _topo; - HardwareLocality(HardwareLocality const&); - HardwareLocality& operator=(HardwareLocality const&); -public: - HardwareLocality() { - hwloc_topology_init(&_topo); - hwloc_topology_load(_topo); - } - ~HardwareLocality() { - hwloc_topology_destroy(_topo); - } - int bind_memory_to_core(int core) { - int core_depth = hwloc_get_type_or_below_depth(_topo, HWLOC_OBJ_CORE); - int ncore = hwloc_get_nbobjs_by_depth(_topo, core_depth); - int ret = 0; - if( 0 <= core && core < ncore ) { - hwloc_obj_t obj = hwloc_get_obj_by_depth(_topo, core_depth, core); - hwloc_cpuset_t cpuset = hwloc_bitmap_dup(obj->cpuset); - hwloc_bitmap_singlify(cpuset); // Avoid hyper-threads - hwloc_membind_policy_t policy = HWLOC_MEMBIND_BIND; - hwloc_membind_flags_t flags = HWLOC_MEMBIND_THREAD; - ret = hwloc_set_membind(_topo, cpuset, policy, flags); - hwloc_bitmap_free(cpuset); - } - return ret; - } -}; -#endif // BF_HWLOC_ENABLED - -class BoundThread { -#if BF_HWLOC_ENABLED - HardwareLocality _hwloc; -#endif -public: - BoundThread(int core) { - bfAffinitySetCore(core); -#if BF_HWLOC_ENABLED - assert(_hwloc.bind_memory_to_core(core) == 0); -#endif - } -}; - -struct PacketStats { - size_t ninvalid; - size_t ninvalid_bytes; - size_t nlate; - size_t nlate_bytes; - size_t nvalid; - size_t nvalid_bytes; -}; - -class UDPTransmitThread : public BoundThread { - PacketStats _stats; - - int _fd; - -public: - UDPTransmitThread(int fd, int core=0) - : BoundThread(core), _fd(fd) { - this->reset_stats(); - } - inline ssize_t send(msghdr* packet) { - ssize_t nsent = sendmsg(_fd, packet, 0); - if( nsent == -1 ) { - ++_stats.ninvalid; - _stats.ninvalid_bytes += packet->msg_iovlen; - } else { - ++_stats.nvalid; - _stats.nvalid_bytes += nsent; - } - return nsent; - } - inline ssize_t sendmany(mmsghdr *packets, unsigned int npackets) { - ssize_t nsent = sendmmsg(_fd, packets, npackets, 0); - if( nsent == -1 ) { - _stats.ninvalid += npackets; - _stats.ninvalid_bytes += (size_t) npackets * packets->msg_len; - } else { - _stats.nvalid += npackets; - _stats.nvalid_bytes += (size_t) npackets * packets->msg_len; - } - return nsent; - } - inline const PacketStats* get_stats() const { return &_stats; } - inline void reset_stats() { - ::memset(&_stats, 0, sizeof(_stats)); - } -}; - -class BFudptransmit_impl { - UDPTransmitThread _transmit; - ProcLog _type_log; - ProcLog _bind_log; - ProcLog _stat_log; - - void update_stats_log() { - const PacketStats* stats = _transmit.get_stats(); - _stat_log.update() << "ngood_bytes : " << stats->nvalid_bytes << "\n" - << "nmissing_bytes : " << stats->ninvalid_bytes << "\n" - << "ninvalid : " << stats->ninvalid << "\n" - << "ninvalid_bytes : " << stats->ninvalid_bytes << "\n" - << "nlate : " << stats->nlate << "\n" - << "nlate_bytes : " << stats->nlate_bytes << "\n" - << "nvalid : " << stats->nvalid << "\n" - << "nvalid_bytes : " << stats->nvalid_bytes << "\n"; - } -public: - inline BFudptransmit_impl(int fd, - int core) - : _transmit(fd, core), - _type_log("udp_transmit/type"), - _bind_log("udp_transmit/bind"), - _stat_log("udp_transmit/stats") { - _type_log.update() << "type : " << "generic"; - _bind_log.update() << "ncore : " << 1 << "\n" - << "core0 : " << core << "\n"; - } - BFudptransmit_status send(char *packet, unsigned int len) { - ssize_t state; - struct msghdr msg; - struct iovec iov[1]; - - memset(&msg, 0, sizeof(msg)); - msg.msg_iov = iov; - msg.msg_iovlen = 1; - iov[0].iov_base = packet; - iov[0].iov_len = len; - - state = _transmit.send( &msg ); - if( state == -1 ) { - return BF_TRANSMIT_ERROR; - } - this->update_stats_log(); - return BF_TRANSMIT_CONTINUED; - } - BFudptransmit_status sendmany(char *packets, unsigned int len, unsigned int npackets) { - ssize_t state; - unsigned int i; - struct mmsghdr *mmsg = NULL; - struct iovec *iovs = NULL; - - mmsg = (struct mmsghdr *) malloc(sizeof(struct mmsghdr)*npackets); - iovs = (struct iovec *) malloc(sizeof(struct iovec)*npackets); - memset(mmsg, 0, sizeof(struct mmsghdr)*npackets); - for(i=0; iupdate_stats_log(); - return BF_TRANSMIT_CONTINUED; - } -}; - -BFstatus bfUdpTransmitCreate(BFudptransmit* obj, - int fd, - int core) { - BF_ASSERT(obj, BF_STATUS_INVALID_POINTER); - BF_TRY_RETURN_ELSE(*obj = new BFudptransmit_impl(fd, core), - *obj = 0); - -} -BFstatus bfUdpTransmitDestroy(BFudptransmit obj) { - BF_ASSERT(obj, BF_STATUS_INVALID_HANDLE); - delete obj; - return BF_STATUS_SUCCESS; -} -BFstatus bfUdpTransmitSend(BFudptransmit obj, char* packet, unsigned int len) { - BF_TRY_RETURN(obj->send(packet, len)); -} -BFstatus bfUdpTransmitSendMany(BFudptransmit obj, char* packets, unsigned int len, unsigned int npackets) { - BF_ASSERT(obj, BF_STATUS_INVALID_HANDLE); - BF_TRY_RETURN(obj->sendmany(packets, len, npackets)); -} diff --git a/test/download_test_data.sh b/test/download_test_data.sh index 9ae9fbbc3..3424d9d23 100755 --- a/test/download_test_data.sh +++ b/test/download_test_data.sh @@ -3,4 +3,3 @@ curl -L -O https://fornax.phys.unm.edu/lwa/data/bf_test_files.tar.gz tar xzf bf_test_files.tar.gz mv for_test_suite data rm bf_test_files.tar.gz - diff --git a/test/test_binary_io.py b/test/test_binary_io.py index ddc117627..128217ee1 100644 --- a/test/test_binary_io.py +++ b/test/test_binary_io.py @@ -25,6 +25,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import unittest +import os import bifrost as bf import numpy as np from bifrost import blocks @@ -97,3 +98,10 @@ def data_callback(ispan, ospan): callback = CallbackBlock(b_read, seq_callback, data_callback) pipeline.run() + def tearDown(self): + for filename in self.filenames: + os.unlink(filename) + try: + os.unlink(filename+".out") + except OSError: + pass \ No newline at end of file diff --git a/test/test_disk_io.py b/test/test_disk_io.py new file mode 100644 index 000000000..2c1ca6aed --- /dev/null +++ b/test/test_disk_io.py @@ -0,0 +1,615 @@ +# Copyright (c) 2019-2024, The Bifrost Authors. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of The Bifrost Authors nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest +import os +import json +import ctypes +import threading +import bifrost as bf +from bifrost.ring import Ring +from bifrost.packet_writer import HeaderInfo, DiskWriter +from bifrost.packet_capture import PacketCaptureCallback, DiskReader +from bifrost.quantize import quantize +from bifrost.pipeline import SourceBlock, SinkBlock +import datetime +import numpy as np + +class AccumulateOp(object): + def __init__(self, ring, output_timetags, output_data, size, dtype=np.uint8): + self.ring = ring + self.output_timetags = output_timetags + self.output_data = output_data + self.size = size*(dtype().nbytes) + self.dtype = dtype + + def main(self): + for iseq in self.ring.read(guarantee=True): + self.output_timetags.append(iseq.time_tag) + self.output_data.append([]) + + iseq_spans = iseq.read(self.size) + while not self.ring.writing_ended(): + for ispan in iseq_spans: + idata = ispan.data_view(self.dtype) + self.output_data[-1].append(idata.copy()) + +class BaseDiskIOTest(object): + class BaseDiskIOTestCase(unittest.TestCase): + def setUp(self): + """Generate some dummy data to read""" + # Generate test vector and save to file + t = np.arange(256*4096*2) + w = 0.2 + self.s0 = 5*np.cos(w * t, dtype='float32') \ + + 3j*np.sin(w * t, dtype='float32') + # Filename cache so we can cleanup later + self._cache = [] + def _open(self, filename, mode): + fh = open(filename, mode) + if filename not in self._cache: + self._cache.append(filename) + return fh + def tearDown(self): + for filename in self._cache: + os.unlink(filename) + + +class TBNReader(object): + def __init__(self, sock, ring, nsrc=32): + self.sock = sock + self.ring = ring + self.nsrc = nsrc + def callback(self, seq0, time_tag, decim, chan0, nsrc, hdr_ptr, hdr_size_ptr): + hdr = {'time_tag': time_tag, + 'seq0': seq0, + 'chan0': chan0, + 'cfreq': 196e6 * chan0/2.**32, + 'bw': 196e6/decim, + 'nstand': nsrc/2, + 'npol': 2, + 'complex': True, + 'nbit': 8} + try: + hdr_str = json.dumps(hdr).encode() + except AttributeError: + # Python2 catch + pass + # TODO: Can't pad with NULL because returned as C-string + #hdr_str = json.dumps(hdr).ljust(4096, '\0') + #hdr_str = json.dumps(hdr).ljust(4096, ' ') + header_buf = ctypes.create_string_buffer(hdr_str) + hdr_ptr[0] = ctypes.cast(header_buf, ctypes.c_void_p) + hdr_size_ptr[0] = len(hdr_str) + return 0 + def main(self): + seq_callback = PacketCaptureCallback() + seq_callback.set_tbn(self.callback) + with DiskReader("tbn", self.sock, self.ring, self.nsrc, 0, 16, 128, + sequence_callback=seq_callback) as capture: + while True: + status = capture.recv() + if status in (1,4,5,6): + break + del capture + +class TBNDiskIOTest(BaseDiskIOTest.BaseDiskIOTestCase): + """Test simple IO for the disk-based TBN packet reader and writing""" + def _get_data(self): + # Setup the packet HeaderInfo + hdr_desc = HeaderInfo() + hdr_desc.set_tuning(int(round(74e6 / 196e6 * 2**32))) + hdr_desc.set_gain(20) + + # Reorder as packets, stands, time + data = self.s0.reshape(512,32,-1) + data = data.transpose(2,1,0).copy() + # Convert to ci8 for TBN + data_q = bf.ndarray(shape=data.shape, dtype='ci8') + quantize(data, data_q, scale=10) + + # Update the number of data sources and return + hdr_desc.set_nsrc(data_q.shape[1]) + return 1, hdr_desc, data_q + def test_write(self): + fh = self._open('test_tbn.dat', 'wb') + oop = DiskWriter('tbn', fh) + + # Get TBN data + timetag0, hdr_desc, data = self._get_data() + + # Go! + oop.send(hdr_desc, timetag0, 1960*512, 0, 1, data) + fh.close() + + self.assertEqual(os.path.getsize('test_tbn.dat'), \ + 1048*data.shape[0]*data.shape[1]) + def test_read(self): + # Write + fh = self._open('test_tbn.dat', 'wb') + oop = DiskWriter('tbn', fh) + + # Get TBN data + timetag0, hdr_desc, data = self._get_data() + + # Go! + oop.send(hdr_desc, timetag0, 1960*512, 0, 1, data) + fh.close() + + # Read + fh = self._open('test_tbn.dat', 'rb') + ring = Ring(name="capture_tbn") + iop = TBNReader(fh, ring, nsrc=32) + ## Data accumulation + times = [] + final = [] + aop = AccumulateOp(ring, times, final, 32*512*2) + + # Start the reader and accumlator threads + reader = threading.Thread(target=iop.main) + accumu = threading.Thread(target=aop.main) + reader.start() + accumu.start() + + # Get TBN data + reader.join() + accumu.join() + + # Compare + for seq_timetag,seq_data in zip(times, final): + ## Loop over sequences + seq_data = np.array(seq_data, dtype=np.uint8) + seq_data = seq_data.reshape(-1,512,32,2) + seq_data = seq_data.transpose(0,2,1,3).copy() + ## Drop the last axis (complexity) since we are going to ci8 + seq_data = bf.ndarray(shape=seq_data.shape[:-1], dtype='ci8', buffer=seq_data.ctypes.data) + + ## Ignore the first set of packets + np.testing.assert_equal(seq_data[1:,...], data[1:,...]) + + # Clean up + del oop + fh.close() + + +class DRXReader(object): + def __init__(self, sock, ring, nsrc=4): + self.sock = sock + self.ring = ring + self.nsrc = nsrc + def callback(self, seq0, time_tag, decim, chan0, chan1, nsrc, hdr_ptr, hdr_size_ptr): + #print "++++++++++++++++ seq0 =", seq0 + #print " time_tag =", time_tag + hdr = {'time_tag': time_tag, + 'seq0': seq0, + 'chan0': chan0, + 'chan1': chan1, + 'cfreq0': 196e6 * chan0/2.**32, + 'cfreq1': 196e6 * chan1/2.**32, + 'bw': 196e6/decim, + 'nstand': nsrc/2, + 'npol': 2, + 'complex': True, + 'nbit': 4} + #print "******** CFREQ:", hdr['cfreq'] + try: + hdr_str = json.dumps(hdr).encode() + except AttributeError: + # Python2 catch + pass + # TODO: Can't pad with NULL because returned as C-string + #hdr_str = json.dumps(hdr).ljust(4096, '\0') + #hdr_str = json.dumps(hdr).ljust(4096, ' ') + header_buf = ctypes.create_string_buffer(hdr_str) + hdr_ptr[0] = ctypes.cast(header_buf, ctypes.c_void_p) + hdr_size_ptr[0] = len(hdr_str) + return 0 + def main(self): + seq_callback = PacketCaptureCallback() + seq_callback.set_drx(self.callback) + with DiskReader("drx", self.sock, self.ring, self.nsrc, 0, 16, 128, + sequence_callback=seq_callback) as capture: + while True: + status = capture.recv() + if status in (1,4,5,6): + break + del capture + +class DRXDiskIOTest(BaseDiskIOTest.BaseDiskIOTestCase): + """Test simple IO for the disk-based DRX packet reader and writing""" + def _get_data(self): + # Setup the packet HeaderInfo + hdr_desc = HeaderInfo() + hdr_desc.set_decimation(10) + hdr_desc.set_tuning(int(round(74e6 / 196e6 * 2**32))) + + # Reorder as packets, beams, time + data = self.s0.reshape(4096,4,-1) + data = data.transpose(2,1,0).copy() + # Convert to ci4 for DRX + data_q = bf.ndarray(shape=data.shape, dtype='ci4') + quantize(data, data_q) + + # Update the number of data sources and return + hdr_desc.set_nsrc(data_q.shape[1]) + return 1, hdr_desc, data_q + def test_write(self): + fh = self._open('test_drx.dat', 'wb') + oop = DiskWriter('drx', fh) + + # Get DRX data + timetag0, hdr_desc, data = self._get_data() + + # Go! + oop.send(hdr_desc, timetag0, 10*4096, (1<<3), 128, data) + fh.close() + + self.assertEqual(os.path.getsize('test_drx.dat'), \ + 4128*data.shape[0]*data.shape[1]) + def test_read(self): + # Write + fh = self._open('test_drx.dat', 'wb') + oop = DiskWriter('drx', fh) + + # Get DRX data and write it out + timetag0, hdr_desc, data = self._get_data() + for p in range(data.shape[0]): + oop.send(hdr_desc, timetag0+p*10*4096, 10*4096, (1<<3), 128, data[p,[0,1],...].reshape(1,2,4096)) + oop.send(hdr_desc, timetag0+p*10*4096, 10*4096, (2<<3), 128, data[p,[2,3],...].reshape(1,2,4096)) + fh.close() + + # Read + fh = self._open('test_drx.dat', 'rb') + ring = Ring(name="capture_drx") + iop = DRXReader(fh, ring, nsrc=4) + ## Data accumulation + times = [] + final = [] + aop = AccumulateOp(ring, times, final, 4*4096*1) + + # Start the reader + reader = threading.Thread(target=iop.main) + accumu = threading.Thread(target=aop.main) + reader.start() + accumu.start() + + # Get DRX data + reader.join() + accumu.join() + + # Compare + for seq_timetag,seq_data in zip(times, final): + ## Reorder to match what we sent out + seq_data = np.array(seq_data, dtype=np.uint8) + seq_data = seq_data.reshape(-1,4096,4) + seq_data = seq_data.transpose(0,2,1).copy() + seq_data = bf.ndarray(shape=seq_data.shape, dtype='ci4', buffer=seq_data.ctypes.data) + + np.testing.assert_equal(seq_data[1:,...], data[1:,...]) + + # Clean up + del oop + fh.close() + def test_write_single(self): + fh = self._open('test_drx_single.dat', 'wb') + oop = DiskWriter('drx', fh) + + # Get DRX data + timetag0, hdr_desc, data = self._get_data() + hdr_desc.set_nsrc(2) + data = data[:,[0,1],:].copy() + + # Go! + oop.send(hdr_desc, timetag0, 10*4096, (1<<3), 128, data) + fh.close() + + self.assertEqual(os.path.getsize('test_drx_single.dat'), \ + 4128*data.shape[0]*data.shape[1]) + def test_read_single(self): + # Write + fh = self._open('test_drx_single.dat', 'wb') + oop = DiskWriter('drx', fh) + + # Get DRX data and write it out + timetag0, hdr_desc, data = self._get_data() + hdr_desc.set_nsrc(2) + data = data[:,[0,1],:].copy() + oop.send(hdr_desc, timetag0, 10*4096, (1<<3), 128, data) + fh.close() + + # Read + fh = self._open('test_drx_single.dat', 'rb') + ring = Ring(name="capture_drx_single") + iop = DRXReader(fh, ring, nsrc=2) + ## Data accumulation + times = [] + final = [] + aop = AccumulateOp(ring, times, final, 2*4096*1) + + # Start the reader + reader = threading.Thread(target=iop.main) + accumu = threading.Thread(target=aop.main) + reader.start() + accumu.start() + + # Get DRX data + reader.join() + accumu.join() + + # Compare + for seq_timetag,seq_data in zip(times, final): + ## Reorder to match what we sent out + seq_data = np.array(seq_data, dtype=np.uint8) + seq_data = seq_data.reshape(-1,4096,2) + seq_data = seq_data.transpose(0,2,1).copy() + seq_data = bf.ndarray(shape=seq_data.shape, dtype='ci4', buffer=seq_data.ctypes.data) + + np.testing.assert_equal(seq_data[1:,...], data[1:,...]) + + # Clean up + del oop + fh.close() + + +class PBeamReader(object): + def __init__(self, sock, ring, nchan, nsrc=1): + self.sock = sock + self.ring = ring + self.nchan = nchan + self.nsrc = nsrc + def callback(self, seq0, time_tag, navg, chan0, nchan, nbeam, hdr_ptr, hdr_size_ptr): + #print "++++++++++++++++ seq0 =", seq0 + #print " time_tag =", time_tag + hdr = {'time_tag': time_tag, + 'seq0': seq0, + 'chan0': chan0, + 'cfreq0': chan0*(196e6/8192), + 'bw': nchan*(196e6/8192), + 'navg': navg, + 'nbeam': nbeam, + 'npol': 4, + 'complex': False, + 'nbit': 32} + #print("******** HDR:", hdr) + try: + hdr_str = json.dumps(hdr).encode() + except AttributeError: + # Python2 catch + pass + # TODO: Can't pad with NULL because returned as C-string + #hdr_str = json.dumps(hdr).ljust(4096, '\0') + #hdr_str = json.dumps(hdr).ljust(4096, ' ') + header_buf = ctypes.create_string_buffer(hdr_str) + hdr_ptr[0] = ctypes.cast(header_buf, ctypes.c_void_p) + hdr_size_ptr[0] = len(hdr_str) + return 0 + def main(self): + seq_callback = PacketCaptureCallback() + seq_callback.set_pbeam(self.callback) + with DiskReader("pbeam_%i" % self.nchan, self.sock, self.ring, self.nsrc, 1, 16, 128, + sequence_callback=seq_callback) as capture: + while True: + status = capture.recv() + if status in (1,4,5,6): + break + del capture + +class PBeamDiskIOTest(BaseDiskIOTest.BaseDiskIOTestCase): + """Test simple IO for the disk-based PBeam packet reader and writing""" + def _get_data(self): + # Setup the packet HeaderInfo + hdr_desc = HeaderInfo() + hdr_desc.set_tuning(2) + hdr_desc.set_chan0(1034) + hdr_desc.set_nchan(128) + hdr_desc.set_decimation(24) + + # Reorder as packets, beam, chan/pol + data = self.s0.reshape(128*4,1,-1) + data = data.transpose(2,1,0) + data = data.real[:1024,...].copy() + + # Update the number of data sources and return + hdr_desc.set_nsrc(data.shape[1]) + return 1, hdr_desc, data + def test_write(self): + fh = self._open('test_pbeam.dat', 'wb') + oop = DiskWriter('pbeam1_128', fh) + + # Get PBeam data + timetag0, hdr_desc, data = self._get_data() + + # Go! + oop.send(hdr_desc, timetag0, 24, 0, 1, data) + fh.close() + + self.assertEqual(os.path.getsize('test_pbeam.dat'), \ + (18+128*4*4)*data.shape[0]*data.shape[1]) + def test_read(self): + # Write + fh = self._open('test_pbeam.dat', 'wb') + oop = DiskWriter('pbeam1_128', fh) + + # Get PBeam data + timetag0, hdr_desc, data = self._get_data() + + # Go! + oop.send(hdr_desc, timetag0, 24, 0, 1, data) + fh.close() + + # Read + fh = self._open('test_pbeam.dat', 'rb') + ring = Ring(name="capture_pbeam") + iop = PBeamReader(fh, ring, 128, nsrc=1) + ## Data accumulation + times = [] + final = [] + aop = AccumulateOp(ring, times, final, 1*128*4, dtype=np.float32) + + # Start the reader and accumlator threads + reader = threading.Thread(target=iop.main) + accumu = threading.Thread(target=aop.main) + reader.start() + accumu.start() + + # Get PBeam data + reader.join() + accumu.join() + + # Compare + for seq_timetag,seq_data in zip(times, final): + ## Reorder to match what we sent out + seq_data = np.array(seq_data, dtype=np.float32) + seq_data = seq_data.reshape(-1,128*4,1) + seq_data = seq_data.transpose(0,2,1).copy() + + np.testing.assert_equal(seq_data[1:,...], data[1:,...]) + + # Clean up + del oop + fh.close() + + +start_pipeline = datetime.datetime.now() + +class SIMPLEReader(object): + def __init__(self, sock, ring): + self.sock = sock + self.ring = ring + self.nsrc = 1 + def seq_callback(self, seq0, chan0, nchan, nsrc, + time_tag_ptr, hdr_ptr, hdr_size_ptr): + FS = 196.0e6 + CHAN_BW = 1e3 + # timestamp0 = (self.utc_start - ADP_EPOCH).total_seconds() + # time_tag0 = timestamp0 * int(FS) + time_tag = int((datetime.datetime.now() - start_pipeline).total_seconds()*1e6) + time_tag_ptr[0] = time_tag + cfreq = 55e6 + hdr = {'time_tag': time_tag, + 'seq0': seq0, + 'chan0': chan0, + 'nchan': nchan, + 'cfreq': cfreq, + 'bw': CHAN_BW, + 'nstand': 1, + #'stand0': src0*16, # TODO: Pass src0 to the callback too(?) + 'npol': 1, + 'complex': True, + 'nbit': 16} + hdr_str = json.dumps(hdr).encode() + # TODO: Can't pad with NULL because returned as C-string + #hdr_str = json.dumps(hdr).ljust(4096, '\0') + #hdr_str = json.dumps(hdr).ljust(4096, ' ') + self.header_buf = ctypes.create_string_buffer(hdr_str) + hdr_ptr[0] = ctypes.cast(self.header_buf, ctypes.c_void_p) + hdr_size_ptr[0] = len(hdr_str) + return 0 + def main(self): + seq_callback = PacketCaptureCallback() + seq_callback.set_simple(self.seq_callback) + with DiskReader("simple" , self.sock, self.ring, self.nsrc, 0, 16, 128, + sequence_callback=seq_callback) as capture: + while True: + status = capture.recv() + if status in (1,4,5,6): + break + del capture + +class SimpleDiskIOTest(BaseDiskIOTest.BaseDiskIOTestCase): + """Test simple IO for the disk-based Simple packet reader and writing""" + def _get_simple_data(self): + hdr_desc = HeaderInfo() + + # Reorder as packets, stands, time + data = self.s0.reshape(2048,1,-1) + data = data.transpose(2,1,0).copy() + # Convert to ci16 for simple + data_q = bf.ndarray(shape=data.shape, dtype='ci16') + quantize(data, data_q, scale=10) + + return 128, hdr_desc, data_q + + def test_write_simple(self): + fh = self._open('test_simple.dat','wb') + oop = DiskWriter('simple', fh) + timetag0, hdr_desc, data = self._get_simple_data() + oop.send(hdr_desc, timetag0, 1, 0, 1, data) + fh.close() + + nelements = 2048 + byteperelem = 4 + bytehdrperelem = 8 + npackets = 1024 + expectedsize = (2048*4 + 8)*1024 + self.assertEqual(os.path.getsize('test_simple.dat'), \ + expectedsize) + + def test_read_simple(self): + # Write + fh = self._open('test_simple.dat', 'wb') + oop = DiskWriter('simple', fh) + + # Get data + timetag0, hdr_desc, data = self._get_simple_data() + + # Go! + oop.send(hdr_desc, timetag0, 1, 0, 1, data) + fh.close() + + # Read + + fh = self._open('test_simple.dat', 'rb') + ring = Ring(name="capture_simple") + iop = SIMPLEReader(fh, ring) + ## Data accumulation + times = [] + final = [] + expectedsize = 1*2048*4 + aop = AccumulateOp(ring, times, final, expectedsize, dtype=np.uint16) + + # Start the reader and accumlator threads + reader = threading.Thread(target=iop.main) + accumu = threading.Thread(target=aop.main) + reader.start() + accumu.start() + + # Get simple data + reader.join() + accumu.join() + + # Compare + for seq_timetag,seq_data in zip(times, final): + seq_data = np.array(seq_data, dtype=np.uint16) + seq_data = seq_data.reshape(-1,2048,1,2) + seq_data = seq_data.transpose(0,2,1,3).copy() + ## Drop the last axis (complexity) since we are going to ci16 + seq_data = bf.ndarray(shape=seq_data.shape[:-1], dtype='ci16', buffer=seq_data.ctypes.data) + + np.testing.assert_equal(seq_data[1:,...], data[1:,...]) + + del oop + fh.close() diff --git a/test/test_header_standard.py b/test/test_header_standard.py index b32598b68..90e015155 100644 --- a/test/test_header_standard.py +++ b/test/test_header_standard.py @@ -1,4 +1,4 @@ -# Copyright (c) 2016, The Bifrost Authors. All rights reserved. +# Copyright (c) 2016-2022, The Bifrost Authors. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions diff --git a/test/test_scripts.py b/test/test_scripts.py index 03a061932..baef30311 100644 --- a/test/test_scripts.py +++ b/test/test_scripts.py @@ -29,7 +29,6 @@ import unittest import os import re -import imp import sys import glob @@ -41,8 +40,6 @@ TEST_DIR = os.path.dirname(__file__) TOOLS_DIR = os.path.join(TEST_DIR, '..', 'tools') TESTBENCH_DIR = os.path.join(TEST_DIR, '..', 'testbench') -modInfoBuild = imp.find_module('bifrost', [os.path.join(TEST_DIR, '..', 'python')]) -BIFROST_DIR = os.path.abspath(modInfoBuild[1]) run_scripts_tests = False try: @@ -53,7 +50,7 @@ pass run_scripts_tests &= (sys.version_info[0] >= 3) -_LINT_RE = re.compile('(?P.*?)\:(?P\d+)\: \[(?P.*?)\] (?P.*)') +_LINT_RE = re.compile('(?P.*?):(?P[0-9]+): \[(?P.*?)\] (?P.*)') @unittest.skipUnless(run_scripts_tests, "requires the 'pylint' module") class ScriptTest(unittest.TestCase): diff --git a/test/test_udp_io.py b/test/test_udp_io.py new file mode 100644 index 000000000..0e0897377 --- /dev/null +++ b/test/test_udp_io.py @@ -0,0 +1,671 @@ +# Copyright (c) 2019-2024, The Bifrost Authors. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of The Bifrost Authors nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import unittest +import os +import json +import time +import ctypes +import threading +import bifrost as bf +import datetime +from contextlib import closing +from bifrost.ring import Ring +from bifrost.address import Address +from bifrost.udp_socket import UDPSocket +from bifrost.packet_writer import HeaderInfo, UDPTransmit +from bifrost.packet_capture import PacketCaptureCallback, UDPCapture +from bifrost.quantize import quantize +import numpy as np + +class AccumulateOp(object): + def __init__(self, ring, output_timetags, output_data, size, dtype=np.uint8): + self.ring = ring + self.output_timetags = output_timetags + self.output_data = output_data + self.size = size*(dtype().nbytes) + self.dtype = dtype + + def main(self): + for iseq in self.ring.read(guarantee=True): + self.output_timetags.append(iseq.time_tag) + self.output_data.append([]) + + iseq_spans = iseq.read(self.size) + while not self.ring.writing_ended(): + for ispan in iseq_spans: + idata = ispan.data_view(self.dtype) + self.output_data[-1].append(idata.copy()) + +class BaseUDPIOTest(object): + class BaseUDPIOTestCase(unittest.TestCase): + def setUp(self): + """Generate some dummy data to read""" + # Generate test vector and save to file + t = np.arange(256*4096*2) + w = 0.2 + self.s0 = 5*np.cos(w * t, dtype='float32') \ + + 3j*np.sin(w * t, dtype='float32') + + +class TBNReader(object): + def __init__(self, sock, ring, nsrc=32): + self.sock = sock + self.ring = ring + self.nsrc = nsrc + def callback(self, seq0, time_tag, decim, chan0, nsrc, hdr_ptr, hdr_size_ptr): + #print "++++++++++++++++ seq0 =", seq0 + #print " time_tag =", time_tag + hdr = {'time_tag': time_tag, + 'seq0': seq0, + 'chan0': chan0, + 'cfreq': 196e6 * chan0/2.**32, + 'bw': 196e6/decim, + 'nstand': nsrc/2, + 'npol': 2, + 'complex': True, + 'nbit': 8} + #print "******** CFREQ:", hdr['cfreq'] + try: + hdr_str = json.dumps(hdr).encode() + except AttributeError: + # Python2 catch + pass + # TODO: Can't pad with NULL because returned as C-string + #hdr_str = json.dumps(hdr).ljust(4096, '\0') + #hdr_str = json.dumps(hdr).ljust(4096, ' ') + header_buf = ctypes.create_string_buffer(hdr_str) + hdr_ptr[0] = ctypes.cast(header_buf, ctypes.c_void_p) + hdr_size_ptr[0] = len(hdr_str) + return 0 + def main(self): + seq_callback = PacketCaptureCallback() + seq_callback.set_tbn(self.callback) + with UDPCapture("tbn", self.sock, self.ring, self.nsrc, 0, 9000, 16, 128, + sequence_callback=seq_callback) as capture: + while True: + status = capture.recv() + if status in (1,4,5,6): + break + del capture + +class TBNUDPIOTest(BaseUDPIOTest.BaseUDPIOTestCase): + """Test simple IO for the UDP-based TBN packet reader and writing""" + def _get_data(self): + # Setup the packet HeaderInfo + hdr_desc = HeaderInfo() + hdr_desc.set_tuning(int(round(74e6 / 196e6 * 2**32))) + hdr_desc.set_gain(20) + + # Reorder as packets, stands, time + data = self.s0.reshape(512,32,-1) + data = data.transpose(2,1,0).copy() + # Convert to ci8 for TBN + data_q = bf.ndarray(shape=data.shape, dtype='ci8') + quantize(data, data_q, scale=10) + + # Update the number of data sources and return + hdr_desc.set_nsrc(data_q.shape[1]) + return 1, hdr_desc, data_q + def test_write(self): + addr = Address('127.0.0.1', 7147) + with closing(UDPSocket()) as sock: + sock.connect(addr) + op = UDPTransmit('tbn', sock) + + # Get TBN data + timetag0, hdr_desc, data = self._get_data() + + # Go! + op.send(hdr_desc, timetag0, 1960*512, 0, 1, data) + def test_read(self): + # Setup the ring + ring = Ring(name="capture_tbn") + + # Setup the blocks + addr = Address('127.0.0.1', 7147) + ## Output via UDPTransmit + with closing(UDPSocket()) as osock: + osock.connect(addr) + oop = UDPTransmit('tbn', osock) + ## Input via UDPCapture + with closing(UDPSocket()) as isock: + isock.bind(addr) + isock.timeout = 0.1 + iop = TBNReader(isock, ring, nsrc=32) + ## Data accumulation + times = [] + final = [] + aop = AccumulateOp(ring, times, final, 32*512*2) + + # Start the reader and accumlator threads + reader = threading.Thread(target=iop.main) + accumu = threading.Thread(target=aop.main) + reader.start() + accumu.start() + + # Get TBN data and send it off + timetag0, hdr_desc, data = self._get_data() + for p in range(data.shape[0]): + oop.send(hdr_desc, timetag0+p*1960*512, 1960*512, 0, 1, data[[p],...]) + time.sleep(0.001) + reader.join() + accumu.join() + + # Compare + for seq_timetag,seq_data in zip(times, final): + ## Loop over sequences + seq_data = np.array(seq_data, dtype=np.uint8) + seq_data = seq_data.reshape(-1,512,32,2) + seq_data = seq_data.transpose(0,2,1,3).copy() + ## Drop the last axis (complexity) since we are going to ci8 + seq_data = bf.ndarray(shape=seq_data.shape[:-1], dtype='ci8', buffer=seq_data.ctypes.data) + + ## Ignore the first set of packets + np.testing.assert_equal(seq_data[1:,...], data[1:,...]) + + # Clean up + del oop + def test_write_multicast(self): + addr = Address('224.0.0.251', 7147) + with closing(UDPSocket()) as sock: + sock.connect(addr) + op = UDPTransmit('tbn', sock) + + # Get TBN data + timetag0, hdr_desc, data = self._get_data() + + # Go! + op.send(hdr_desc, timetag0, 1960*512, 0, 1, data) + def test_read_multicast(self): + # Setup the ring + ring = Ring(name="capture_multi") + + # Setup the blocks + addr = Address('224.0.0.251', 7147) + ## Output via UDPTransmit + with closing(UDPSocket()) as osock: + osock.connect(addr) + oop = UDPTransmit('tbn', osock) + ## Input via UDPCapture + with closing(UDPSocket()) as isock: + isock.bind(addr) + isock.timeout = 0.1 + iop = TBNReader(isock, ring, nsrc=32) + # Data accumulation + times = [] + final = [] + aop = AccumulateOp(ring, times, final, 32*512*2) + + # Start the reader and accumlator threads + reader = threading.Thread(target=iop.main) + accumu = threading.Thread(target=aop.main) + reader.start() + accumu.start() + + # Get TBN data and send it off + timetag0, hdr_desc, data = self._get_data() + for p in range(data.shape[0]): + oop.send(hdr_desc, timetag0+p*1960*512, 1960*512, 0, 1, data[[p],...]) + time.sleep(0.001) + reader.join() + accumu.join() + + # Compare + for seq_timetag,seq_data in zip(times, final): + ## Loop over sequences + seq_data = np.array(seq_data, dtype=np.uint8) + seq_data = seq_data.reshape(-1,512,32,2) + seq_data = seq_data.transpose(0,2,1,3).copy() + ## Drop the last axis (complexity) since we are going to ci8 + seq_data = bf.ndarray(shape=seq_data.shape[:-1], dtype='ci8', buffer=seq_data.ctypes.data) + + ## Ignore the first set of packets + np.testing.assert_equal(seq_data[1:,...], data[1:,...]) + + # Clean up + del oop + + +class DRXReader(object): + def __init__(self, sock, ring, nsrc=4): + self.sock = sock + self.ring = ring + self.nsrc = nsrc + def callback(self, seq0, time_tag, decim, chan0, chan1, nsrc, hdr_ptr, hdr_size_ptr): + #print "++++++++++++++++ seq0 =", seq0 + #print " time_tag =", time_tag + hdr = {'time_tag': time_tag, + 'seq0': seq0, + 'chan0': chan0, + 'chan1': chan1, + 'cfreq0': 196e6 * chan0/2.**32, + 'cfreq1': 196e6 * chan1/2.**32, + 'bw': 196e6/decim, + 'nstand': nsrc/2, + 'npol': 2, + 'complex': True, + 'nbit': 4} + #print "******** CFREQ:", hdr['cfreq'] + try: + hdr_str = json.dumps(hdr).encode() + except AttributeError: + # Python2 catch + pass + # TODO: Can't pad with NULL because returned as C-string + #hdr_str = json.dumps(hdr).ljust(4096, '\0') + #hdr_str = json.dumps(hdr).ljust(4096, ' ') + header_buf = ctypes.create_string_buffer(hdr_str) + hdr_ptr[0] = ctypes.cast(header_buf, ctypes.c_void_p) + hdr_size_ptr[0] = len(hdr_str) + return 0 + def main(self): + seq_callback = PacketCaptureCallback() + seq_callback.set_drx(self.callback) + with UDPCapture("drx", self.sock, self.ring, self.nsrc, 0, 9000, 16, 128, + sequence_callback=seq_callback) as capture: + while True: + status = capture.recv() + if status in (1,4,5,6): + break + del capture + +class DRXUDPIOTest(BaseUDPIOTest.BaseUDPIOTestCase): + """Test simple IO for the UDP-based DRX packet reader and writing""" + def _get_data(self): + # Setup the packet HeaderInfo + hdr_desc = HeaderInfo() + hdr_desc.set_decimation(10) + hdr_desc.set_tuning(int(round(74e6 / 196e6 * 2**32))) + + # Reorder as packets, beams, time + data = self.s0.reshape(4096,4,-1) + data = data.transpose(2,1,0).copy() + # Convert to ci4 for DRX + data_q = bf.ndarray(shape=data.shape, dtype='ci4') + quantize(data, data_q) + + # Update the number of data sources and return + hdr_desc.set_nsrc(data_q.shape[1]) + return 1, hdr_desc, data_q + def test_write(self): + addr = Address('127.0.0.1', 7147) + with closing(UDPSocket()) as sock: + sock.connect(addr) + op = UDPTransmit('drx', sock) + + # Get TBN data + timetag0, hdr_desc, data = self._get_data() + + # Go! + op.send(hdr_desc, timetag0, 10*4096, (1<<3), 128, data) + def test_read(self): + # Setup the ring + ring = Ring(name="capture_drx") + + # Setup the blocks + addr = Address('127.0.0.1', 7147) + ## Output via UDPTransmit + with closing(UDPSocket()) as osock: + osock.connect(addr) + oop = UDPTransmit('drx', osock) + ## Input via UDPCapture + with closing(UDPSocket()) as isock: + isock.bind(addr) + isock.timeout = 0.1 + iop = DRXReader(isock, ring, nsrc=4) + ## Data accumulation + times = [] + final = [] + aop = AccumulateOp(ring, times, final, 4*4096*1) + + # Start the reader + reader = threading.Thread(target=iop.main) + accumu = threading.Thread(target=aop.main) + reader.start() + accumu.start() + + # Get DRX data and send it off + timetag0, hdr_desc, data = self._get_data() + for p in range(data.shape[0]): + oop.send(hdr_desc, timetag0+p*10*4096, 10*4096, (1<<3), 128, data[p,[0,1],...].reshape(1,2,4096)) + oop.send(hdr_desc, timetag0+p*10*4096, 10*4096, (2<<3), 128, data[p,[2,3],...].reshape(1,2,4096)) + time.sleep(0.001) + reader.join() + accumu.join() + + # Compare + for seq_timetag,seq_data in zip(times, final): + ## Reorder to match what we sent out + seq_data = np.array(seq_data, dtype=np.uint8) + seq_data = seq_data.reshape(-1,4096,4) + seq_data = seq_data.transpose(0,2,1).copy() + seq_data = bf.ndarray(shape=seq_data.shape, dtype='ci4', buffer=seq_data.ctypes.data) + + np.testing.assert_equal(seq_data[1:,...], data[1:,...]) + + # Clean up + del oop + def test_write_single(self): + addr = Address('127.0.0.1', 7147) + with closing(UDPSocket()) as sock: + sock.connect(addr) + op = UDPTransmit('drx', sock) + + # Get DRX data + timetag0, hdr_desc, data = self._get_data() + hdr_desc.set_nsrc(2) + data = data[:,[0,1],:].copy() + + # Go! + op.send(hdr_desc, timetag0, 10*4096, (1<<3), 128, data) + def test_read_single(self): + # Setup the ring + ring = Ring(name="capture_drx_single") + + # Setup the blocks + addr = Address('127.0.0.1', 7147) + ## Output via UDPTransmit + with closing(UDPSocket()) as osock: + osock.connect(addr) + oop = UDPTransmit('drx', osock) + ## Input via UDPCapture + with closing(UDPSocket()) as isock: + isock.bind(addr) + isock.timeout = 0.1 + iop = DRXReader(isock, ring, nsrc=2) + ## Data accumulation + times = [] + final = [] + aop = AccumulateOp(ring, times, final, 2*4096*1) + + # Start the reader + reader = threading.Thread(target=iop.main) + accumu = threading.Thread(target=aop.main) + reader.start() + accumu.start() + + # Get DRX data and send it off + timetag0, hdr_desc, data = self._get_data() + data = data[:,[0,1],:].copy() + for p in range(data.shape[0]): + oop.send(hdr_desc, timetag0+p*10*4096, 10*4096, (1<<3), 128, data[[p],...]) + time.sleep(0.001) + reader.join() + accumu.join() + + # Compare + for seq_timetag,seq_data in zip(times, final): + ## Reorder to match what we sent out + seq_data = np.array(seq_data, dtype=np.uint8) + seq_data = seq_data.reshape(-1,4096,2) + seq_data = seq_data.transpose(0,2,1).copy() + seq_data = bf.ndarray(shape=seq_data.shape, dtype='ci4', buffer=seq_data.ctypes.data) + + np.testing.assert_equal(seq_data[1:,...], data[1:,...]) + + # Clean up + del oop + + +class PBeamReader(object): + def __init__(self, sock, ring, nsrc=1): + self.sock = sock + self.ring = ring + self.nsrc = nsrc + def callback(self, seq0, time_tag, navg, chan0, nchan, nbeam, hdr_ptr, hdr_size_ptr): + #print "++++++++++++++++ seq0 =", seq0 + #print " time_tag =", time_tag + hdr = {'time_tag': time_tag, + 'seq0': seq0, + 'chan0': chan0, + 'cfreq0': chan0*(196e6/8192), + 'bw': nchan*(196e6/8192), + 'navg': navg, + 'nbeam': nbeam, + 'npol': 4, + 'complex': False, + 'nbit': 32} + #print("******** HDR:", hdr) + try: + hdr_str = json.dumps(hdr).encode() + except AttributeError: + # Python2 catch + pass + # TODO: Can't pad with NULL because returned as C-string + #hdr_str = json.dumps(hdr).ljust(4096, '\0') + #hdr_str = json.dumps(hdr).ljust(4096, ' ') + header_buf = ctypes.create_string_buffer(hdr_str) + hdr_ptr[0] = ctypes.cast(header_buf, ctypes.c_void_p) + hdr_size_ptr[0] = len(hdr_str) + return 0 + def main(self): + seq_callback = PacketCaptureCallback() + seq_callback.set_pbeam(self.callback) + with UDPCapture("pbeam", self.sock, self.ring, self.nsrc, 1, 9000, 16, 128, + sequence_callback=seq_callback) as capture: + while True: + status = capture.recv() + if status in (1,4,5,6): + break + del capture + +class PBeamUDPIOTest(BaseUDPIOTest.BaseUDPIOTestCase): + """Test simple IO for the UDP-based PBeam packet reader and writing""" + def _get_data(self): + # Setup the packet HeaderInfo + hdr_desc = HeaderInfo() + hdr_desc.set_tuning(1) + hdr_desc.set_chan0(345) + hdr_desc.set_nchan(128) + hdr_desc.set_decimation(24) + + # Reorder as packets, beam, chan/pol + data = self.s0.reshape(128*4,1,-1) + data = data.transpose(2,1,0) + data = data.real[:1024,...].copy() + + # Update the number of data sources and return + hdr_desc.set_nsrc(data.shape[1]) + return 1, hdr_desc, data + def test_write(self): + addr = Address('127.0.0.1', 7147) + with closing(UDPSocket()) as sock: + sock.connect(addr) + op = UDPTransmit('pbeam1_128', sock) + + # Get PBeam data + timetag0, hdr_desc, data = self._get_data() + + # Go! + op.send(hdr_desc, timetag0, 24, 0, 1, data) + def test_read(self): + # Setup the ring + ring = Ring(name="capture_pbeam") + + # Setup the blocks + addr = Address('127.0.0.1', 7147) + ## Output via UDPTransmit + with closing(UDPSocket()) as osock: + osock.connect(addr) + oop = UDPTransmit('pbeam1_128', osock) + ## Input via UDPCapture + with closing(UDPSocket()) as isock: + isock.bind(addr) + isock.timeout = 0.1 + iop = PBeamReader(isock, ring, nsrc=1) + ## Data accumulation + times = [] + final = [] + aop = AccumulateOp(ring, times, final, 1*128*4, dtype=np.float32) + + # Start the reader and accumlator threads + reader = threading.Thread(target=iop.main) + accumu = threading.Thread(target=aop.main) + reader.start() + accumu.start() + + # Get PBeam data and send it off + timetag0, hdr_desc, data = self._get_data() + for p in range(data.shape[0]): + oop.send(hdr_desc, timetag0+p*24, 24, 0, 1, data[[p],...]) + time.sleep(0.001) + reader.join() + accumu.join() + + # Compare + for seq_timetag,seq_data in zip(times, final): + ## Reorder to match what we sent out + seq_data = np.array(seq_data, dtype=np.float32) + seq_data = seq_data.reshape(-1,128*4,1) + seq_data = seq_data.transpose(0,2,1).copy() + + np.testing.assert_equal(seq_data[1:,...], data[1:,...]) + + # Clean up + del oop + + +start_pipeline = datetime.datetime.now() + +class SIMPLEReader(object): + def __init__(self, sock, ring): + self.sock = sock + self.ring = ring + self.nsrc = 1 + def seq_callback(self, seq0, chan0, nchan, nsrc, + time_tag_ptr, hdr_ptr, hdr_size_ptr): + FS = 196.0e6 + CHAN_BW = 1e3 + # timestamp0 = (self.utc_start - ADP_EPOCH).total_seconds() + # time_tag0 = timestamp0 * int(FS) + time_tag = int((datetime.datetime.now() - start_pipeline).total_seconds()*1e6) + time_tag_ptr[0] = time_tag + cfreq = 55e6 + hdr = {'time_tag': time_tag, + 'seq0': seq0, + 'chan0': chan0, + 'nchan': nchan, + 'cfreq': cfreq, + 'bw': CHAN_BW, + 'nstand': 2, + #'stand0': src0*16, # TODO: Pass src0 to the callback too(?) + 'npol': 2, + 'complex': True, + 'nbit': 16} + hdr_str = json.dumps(hdr).encode() + # TODO: Can't pad with NULL because returned as C-string + #hdr_str = json.dumps(hdr).ljust(4096, '\0') + #hdr_str = json.dumps(hdr).ljust(4096, ' ') + self.header_buf = ctypes.create_string_buffer(hdr_str) + hdr_ptr[0] = ctypes.cast(self.header_buf, ctypes.c_void_p) + hdr_size_ptr[0] = len(hdr_str) + return 0 + def main(self): + seq_callback = PacketCaptureCallback() + seq_callback.set_simple(self.seq_callback) + with UDPCapture("simple" , self.sock, self.ring, self.nsrc, 0, 9000, 16, 128, + sequence_callback=seq_callback) as capture: + while True: + status = capture.recv() + if status in (1,4,5,6): + break + del capture + + +class SimpleUDPIOTest(BaseUDPIOTest.BaseUDPIOTestCase): + """Test simple IO for the UDP-based Simple packet reader and writing""" + def _get_data(self): + hdr_desc = HeaderInfo() + + # Reorder as packets, stands, time + data = self.s0.reshape(2048,1,-1) + data = data.transpose(2,1,0).copy() + # Convert to ci16 for simple + data_q = bf.ndarray(shape=data.shape, dtype='ci16') + quantize(data, data_q, scale=10) + + return 128, hdr_desc, data_q + + def test_write(self): + addr = Address('127.0.0.1', 7147) + with closing(UDPSocket()) as sock: + sock.connect(addr) + # Get simple data + op = UDPTransmit('simple', sock) + + timetag0, hdr_desc, data = self._get_data() + # Go! + op.send(hdr_desc, timetag0, 1, 0, 1, data) + + def test_read(self): + # Setup the ring + ring = Ring(name="capture_simple") + + # Setup the blocks + addr = Address('127.0.0.1', 7147) + ## Output via UDPTransmit + with closing(UDPSocket()) as osock: + osock.connect(addr) + oop = UDPTransmit('simple', osock) + ## Input via UDPCapture + with closing(UDPSocket()) as isock: + isock.bind(addr) + isock.timeout = 1.0 + iop = SIMPLEReader(isock, ring) + ## Data accumulation + times = [] + final = [] + expectedsize = 1*2048*4 + aop = AccumulateOp(ring, times, final, expectedsize, dtype=np.int16) + + + # Start the reader and accumlator threads + reader = threading.Thread(target=iop.main) + accumu = threading.Thread(target=aop.main) + reader.start() + accumu.start() + + # Get simple data and send it off + timetag0, hdr_desc, data = self._get_data() + for p in range(data.shape[0]): + oop.send(hdr_desc, timetag0+p*1, 1, 0, 1, data[[p],...]) + time.sleep(0.001) + reader.join() + accumu.join() + + # Compare + for seq_timetag,seq_data in zip(times, final): + seq_data = np.array(seq_data, dtype=np.uint16) + seq_data = seq_data.reshape(-1,2048,1,2) + seq_data = seq_data.transpose(0,2,1,3).copy() + ## Drop the last axis (complexity) since we are going to ci16 + seq_data = bf.ndarray(shape=seq_data.shape[:-1], dtype='ci16', buffer=seq_data.ctypes.data) + + np.testing.assert_equal(seq_data[1:,...], data[1:,...]) + + # Clean up + del oop diff --git a/test/test_version.py b/test/test_version.py new file mode 100644 index 000000000..4ab08501e --- /dev/null +++ b/test/test_version.py @@ -0,0 +1,10 @@ +import unittest +import subprocess +import sys + +class TestVersion(unittest.TestCase): + def test_plain_version(self): + subprocess.check_output([sys.executable, '-m', 'bifrost.version']) + + def test_version_config(self): + subprocess.check_output([sys.executable, '-m', 'bifrost.version', '--config']) diff --git a/tools/like_bmon.py b/tools/like_bmon.py index 93e6eadbb..63406f9e6 100755 --- a/tools/like_bmon.py +++ b/tools/like_bmon.py @@ -114,7 +114,9 @@ def get_statistics(blockList, prevList): # Loop over the blocks to find udp_capture and udp_transmit blocks output = {'updated': datetime.now()} for block in blockList: - if block.find('udp_capture') != -1: + if block.find('udp_capture') != -1 \ + or block.find('udp_sniffer') != -1 \ + or block.find('udp_verbs_capture') != -1: ## udp_capture is RX good = True type = 'rx' diff --git a/tools/like_pmap.py b/tools/like_pmap.py index f5a2e2ee4..31b6a18f6 100755 --- a/tools/like_pmap.py +++ b/tools/like_pmap.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 -# Copyright (c) 2017-2023, The Bifrost Authors. All rights reserved. -# Copyright (c) 2017-2023, The University of New Mexico. All rights reserved. +# Copyright (c) 2017-2024, The Bifrost Authors. All rights reserved. +# Copyright (c) 2017-2024, The University of New Mexico. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -92,7 +92,7 @@ def main(args): raise RuntimeError("Cannot find NUMA memory info for PID: %i" % args.pid) # Parse out the anonymous entries in this file - _numaRE = re.compile('(?P[0-9a-f]+).*[(anon)|(mapped)]=(?P\d+).*(swapcache=(?P\d+))?.*N(?P\d+)=(?P\d+)') + _numaRE = re.compile('(?P[0-9a-f]+).*[(anon)|(mapped)]=(?P[0-9]+).*(swapcache=(?P[0-9]+))?.*N(?P[0-9]+)=(?P[0-9]+)') areas = {} files = {} for line in numaInfo.split('\n'):