From dc7faf65ad5e5e94551aa10135d58eab7bc9e9fb Mon Sep 17 00:00:00 2001 From: Sean Hefty Date: Wed, 20 Sep 2023 13:42:57 -0700 Subject: [PATCH] prov/netdir: Remove provider NetworkDirect support is supported by verbs provider. Signed-off-by: Sean Hefty --- .appveyor.ps1 | 2 +- README.md | 9 +- libfabric.vcxproj | 54 +- libfabric.vcxproj.filters | 93 +- prov/netdir/NetDirect/README.NetworkDirect | 12 - prov/netdir/src/netdir.h | 197 --- prov/netdir/src/netdir_buf.h | 255 ---- prov/netdir/src/netdir_cntr.c | 208 ---- prov/netdir/src/netdir_cq.c | 1067 ----------------- prov/netdir/src/netdir_cq.h | 210 ---- prov/netdir/src/netdir_domain.c | 545 --------- prov/netdir/src/netdir_ep.c | 880 -------------- prov/netdir/src/netdir_ep_msg.c | 666 ---------- prov/netdir/src/netdir_ep_rma.c | 710 ----------- prov/netdir/src/netdir_ep_srx.c | 337 ------ prov/netdir/src/netdir_eq.c | 411 ------- prov/netdir/src/netdir_fabric.c | 121 -- prov/netdir/src/netdir_iface.h | 273 ----- prov/netdir/src/netdir_init.c | 205 ---- prov/netdir/src/netdir_log.h | 195 --- prov/netdir/src/netdir_mr.c | 355 ------ prov/netdir/src/netdir_ndinit.c | 694 ----------- prov/netdir/src/netdir_ov.c | 159 --- prov/netdir/src/netdir_ov.h | 249 ---- prov/netdir/src/netdir_pep.c | 481 -------- prov/netdir/src/netdir_queue.h | 187 --- prov/netdir/src/netdir_unexp.c | 532 -------- prov/netdir/src/netdir_unexp.h | 84 -- prov/netdir/src/netdir_util.h | 176 --- prov/verbs/include/windows/verbs_nd.h | 6 + .../src/windows/verbs_nd_addr.c} | 6 - .../src/windows/verbs_nd_fs.c} | 7 +- prov/verbs/src/windows/verbs_nd_init.c | 5 - 33 files changed, 35 insertions(+), 9356 deletions(-) delete mode 100644 prov/netdir/NetDirect/README.NetworkDirect delete mode 100644 prov/netdir/src/netdir.h delete mode 100644 prov/netdir/src/netdir_buf.h delete mode 100644 prov/netdir/src/netdir_cntr.c delete mode 100644 prov/netdir/src/netdir_cq.c delete mode 100644 prov/netdir/src/netdir_cq.h delete mode 100644 prov/netdir/src/netdir_domain.c delete mode 100644 prov/netdir/src/netdir_ep.c delete mode 100644 prov/netdir/src/netdir_ep_msg.c delete mode 100644 prov/netdir/src/netdir_ep_rma.c delete mode 100644 prov/netdir/src/netdir_ep_srx.c delete mode 100644 prov/netdir/src/netdir_eq.c delete mode 100644 prov/netdir/src/netdir_fabric.c delete mode 100644 prov/netdir/src/netdir_iface.h delete mode 100644 prov/netdir/src/netdir_init.c delete mode 100644 prov/netdir/src/netdir_log.h delete mode 100644 prov/netdir/src/netdir_mr.c delete mode 100644 prov/netdir/src/netdir_ndinit.c delete mode 100644 prov/netdir/src/netdir_ov.c delete mode 100644 prov/netdir/src/netdir_ov.h delete mode 100644 prov/netdir/src/netdir_pep.c delete mode 100644 prov/netdir/src/netdir_queue.h delete mode 100644 prov/netdir/src/netdir_unexp.c delete mode 100644 prov/netdir/src/netdir_unexp.h delete mode 100644 prov/netdir/src/netdir_util.h rename prov/{netdir/src/netdir_addr.c => verbs/src/windows/verbs_nd_addr.c} (97%) rename prov/{netdir/src/netdir_fs.c => verbs/src/windows/verbs_nd_fs.c} (98%) diff --git a/.appveyor.ps1 b/.appveyor.ps1 index 77282cdac99..a44a887c9fd 100644 --- a/.appveyor.ps1 +++ b/.appveyor.ps1 @@ -12,7 +12,7 @@ $wd=$PWD.Path; & { Add-Type -A "System.IO.Compression.FileSystem"; [IO.Compressi Write-Verbose "done" Write-Verbose "moving NetworkDirect headers.." -move NetDirect\include\* prov\netdir\NetDirect +move NetDirect\include\* include\windows Write-Verbose "done" $efaWinVersion="1.0.0" diff --git a/README.md b/README.md index c4ec349fb31..829ae944df6 100644 --- a/README.md +++ b/README.md @@ -282,9 +282,8 @@ See the `fi_netdir(7)` man page for more details. - The Network Direct provider requires Network Direct SPI. If you are compiling libfabric from source and want to enable Network Direct support, you will also need the matching header files for the Network Direct SPI. - If the libraries and header files are not in default paths (the default path is - root of provier directory, i.e. \prov\netdir\NetDirect, where NetDirect contains - the header files), specify them in the configuration properties of the VS project. + If the libraries and header files are not in default paths, specify them in the + configuration properties of the VS project. ### shm @@ -326,8 +325,8 @@ It is possible to compile and link libfabric with windows applications. on page press Download button and select NetworkDirect_DDK.zip. Extract header files from downloaded - NetworkDirect_DDK.zip:`\NetDirect\include\` file into `\prov\netdir\NetDirect\`, - or add path to NetDirect headers into VS include paths + NetworkDirect_DDK.zip:`\NetDirect\include\` into `include\windows`, or + add the path to NetDirect headers into VS include paths - 2. compiling: libfabric has 6 Visual Studio solution configurations: diff --git a/libfabric.vcxproj b/libfabric.vcxproj index 7b7b2592104..cffccc1383f 100644 --- a/libfabric.vcxproj +++ b/libfabric.vcxproj @@ -192,7 +192,7 @@ NotUsing Level4 - $(ProjectDir)include;$(ProjectDir)include\windows;$(ProjectDir)prov\netdir\NetDirect;$(ProjectDir)prov\verbs\include;$(ProjectDir)prov\hook\src;$(ProjectDir)prov\hook\include;$(ProjectDir)prov\hook\perf\include;$(ProjectDir)packages\NetworkDirect.2.0.1\include;$(ProjectDir)prov\efa\src;$(ProjectDir)prov\efa\include;$(ProjectDir)prov\efa\src\rxr;$(ProjectDir)prov\efa\src\efa_verbs;$(ProjectDir)prov\efa\src\efa_verbs\plat + $(ProjectDir)include;$(ProjectDir)include\windows;$(ProjectDir)prov\verbs\include;$(ProjectDir)prov\hook\src;$(ProjectDir)prov\hook\include;$(ProjectDir)prov\hook\perf\include;$(ProjectDir)packages\NetworkDirect.2.0.1\include;$(ProjectDir)prov\efa\src;$(ProjectDir)prov\efa\include;$(ProjectDir)prov\efa\src\rxr;$(ProjectDir)prov\efa\src\efa_verbs;$(ProjectDir)prov\efa\src\efa_verbs\plat true CompileAsC false @@ -209,7 +209,7 @@ Disabled WIN32;_WINSOCKAPI_=;_CRT_SECURE_NO_WARNINGS;_WINSOCK_DEPRECATED_NO_WARNINGS;_WINDOWS;_USRDLL;LIBFABRIC_EXPORTS;HAVE_CONFIG_H;ENABLE_DEBUG;%(PreprocessorDefinitions) 4127;4200;4204;4221;4115;4201;4100 - $(ProjectDir)include;$(ProjectDir)include\windows;$(ProjectDir)prov\netdir\NetDirect;$(ProjectDir)prov\verbs\include;$(ProjectDir)prov\hook\src;$(ProjectDir)prov\hook\include;$(ProjectDir)prov\hook\perf\include;$(ProjectDir)packages\NetworkDirect.2.0.1\include;$(ProjectDir)prov\efa\src;$(ProjectDir)prov\efa\include;$(ProjectDir)prov\efa\src\rxr;$(ProjectDir)prov\efa\src\efa_verbs;$(ProjectDir)prov\efa\src\efa_verbs\plat + $(ProjectDir)include;$(ProjectDir)include\windows;$(ProjectDir)prov\verbs\include;$(ProjectDir)prov\hook\src;$(ProjectDir)prov\hook\include;$(ProjectDir)prov\hook\perf\include;$(ProjectDir)packages\NetworkDirect.2.0.1\include;$(ProjectDir)prov\efa\src;$(ProjectDir)prov\efa\include;$(ProjectDir)prov\efa\src\rxr;$(ProjectDir)prov\efa\src\efa_verbs;$(ProjectDir)prov\efa\src\efa_verbs\plat true MultiThreadedDebug false @@ -223,7 +223,7 @@ Disabled WIN32;_WINSOCKAPI_=;_CRT_SECURE_NO_WARNINGS;_WINSOCK_DEPRECATED_NO_WARNINGS;_WINDOWS;_USRDLL;LIBFABRIC_EXPORTS;HAVE_CONFIG_H;ENABLE_DEBUG;%(PreprocessorDefinitions) true - $(ProjectDir)include;$(ProjectDir)include\windows;$(ProjectDir)prov\netdir\NetDirect;$(ProjectDir)prov\verbs\include;$(ProjectDir)prov\hook\src;$(ProjectDir)prov\hook\include;$(ProjectDir)prov\hook\perf\include;$(ProjectDir)packages\NetworkDirect.2.0.1\include;$(ProjectDir)prov\efa\src;$(ProjectDir)prov\efa\include;$(ProjectDir)prov\efa\src\rxr;$(ProjectDir)prov\efa\src\efa_verbs;$(ProjectDir)prov\efa\src\efa_verbs\plat + $(ProjectDir)include;$(ProjectDir)include\windows;$(ProjectDir)prov\verbs\include;$(ProjectDir)prov\hook\src;$(ProjectDir)prov\hook\include;$(ProjectDir)prov\hook\perf\include;$(ProjectDir)packages\NetworkDirect.2.0.1\include;$(ProjectDir)prov\efa\src;$(ProjectDir)prov\efa\include;$(ProjectDir)prov\efa\src\rxr;$(ProjectDir)prov\efa\src\efa_verbs;$(ProjectDir)prov\efa\src\efa_verbs\plat CompileAsC 4127;4200;4204;4221;4115;4201;4100 true @@ -238,7 +238,7 @@ Disabled WIN32;_WINSOCKAPI_=;_CRT_SECURE_NO_WARNINGS;_WINSOCK_DEPRECATED_NO_WARNINGS;_WINDOWS;_USRDLL;LIBFABRIC_EXPORTS;HAVE_CONFIG_H;ENABLE_DEBUG;%(PreprocessorDefinitions) 4127;4200;4204;4221;4115;4201;4100 - $(ProjectDir)include;$(ProjectDir)include\windows;$(ProjectDir)prov\netdir\NetDirect;$(ProjectDir)prov\verbs\include;$(ProjectDir)prov\hook\src;$(ProjectDir)prov\hook\include;$(ProjectDir)prov\hook\perf\include;$(ProjectDir)packages\NetworkDirect.2.0.1\include;$(ProjectDir)prov\efa\src;$(ProjectDir)prov\efa\include;$(ProjectDir)prov\efa\src\rxr;$(ProjectDir)prov\efa\src\efa_verbs;$(ProjectDir)prov\efa\src\efa_verbs\plat + $(ProjectDir)include;$(ProjectDir)include\windows;$(ProjectDir)prov\verbs\include;$(ProjectDir)prov\hook\src;$(ProjectDir)prov\hook\include;$(ProjectDir)prov\hook\perf\include;$(ProjectDir)packages\NetworkDirect.2.0.1\include;$(ProjectDir)prov\efa\src;$(ProjectDir)prov\efa\include;$(ProjectDir)prov\efa\src\rxr;$(ProjectDir)prov\efa\src\efa_verbs;$(ProjectDir)prov\efa\src\efa_verbs\plat true MultiThreadedDebug false @@ -250,7 +250,7 @@ Disabled WIN32;_WINSOCKAPI_=;_CRT_SECURE_NO_WARNINGS;_WINSOCK_DEPRECATED_NO_WARNINGS;_WINDOWS;_USRDLL;LIBFABRIC_EXPORTS;HAVE_CONFIG_H;ENABLE_DEBUG;%(PreprocessorDefinitions) 4127;4200;4204;4221;4115;4201;4100 - $(ProjectDir)include;$(ProjectDir)include\windows;$(ProjectDir)prov\sockets\include;$(ProjectDir)prov\netdir\NetDirect;$(ProjectDir)prov\hook\src;$(ProjectDir)prov\hook\include;$(ProjectDir)prov\hook\perf\include;$(ProjectDir)prov\efa\src;$(ProjectDir)prov\efa\src\rxr;$(ProjectDir)prov\efa\src\windows; + $(ProjectDir)include;$(ProjectDir)include\windows;$(ProjectDir)prov\sockets\include;$(ProjectDir)prov\hook\src;$(ProjectDir)prov\hook\include;$(ProjectDir)prov\hook\perf\include;$(ProjectDir)prov\efa\src;$(ProjectDir)prov\efa\src\rxr;$(ProjectDir)prov\efa\src\windows; true MultiThreadedDebug false @@ -265,7 +265,7 @@ Disabled WIN32;_WINSOCKAPI_=;_CRT_SECURE_NO_WARNINGS;_WINSOCK_DEPRECATED_NO_WARNINGS;_WINDOWS;_USRDLL;LIBFABRIC_EXPORTS;HAVE_CONFIG_H;ENABLE_DEBUG;%(PreprocessorDefinitions) true - $(ProjectDir)include;$(ProjectDir)include\windows;$(ProjectDir)prov\netdir\NetDirect;$(ProjectDir)prov\verbs\include;$(ProjectDir)prov\hook\src;$(ProjectDir)prov\hook\include;$(ProjectDir)prov\hook\perf\include;$(ProjectDir)packages\NetworkDirect.2.0.1\include;$(ProjectDir)prov\efa\src;$(ProjectDir)prov\efa\include;$(ProjectDir)prov\efa\src\rxr;$(ProjectDir)prov\efa\src\efa_verbs;$(ProjectDir)prov\efa\src\efa_verbs\plat + $(ProjectDir)include;$(ProjectDir)include\windows;$(ProjectDir)prov\verbs\include;$(ProjectDir)prov\hook\src;$(ProjectDir)prov\hook\include;$(ProjectDir)prov\hook\perf\include;$(ProjectDir)packages\NetworkDirect.2.0.1\include;$(ProjectDir)prov\efa\src;$(ProjectDir)prov\efa\include;$(ProjectDir)prov\efa\src\rxr;$(ProjectDir)prov\efa\src\efa_verbs;$(ProjectDir)prov\efa\src\efa_verbs\plat CompileAsC 4127;4200;94;4204;4221;869 true @@ -289,7 +289,7 @@ true WIN32;_WINSOCKAPI_=;_CRT_SECURE_NO_WARNINGS;_WINSOCK_DEPRECATED_NO_WARNINGS;_WINDOWS;_USRDLL;LIBFABRIC_EXPORTS;HAVE_CONFIG_H;%(PreprocessorDefinitions) true - $(ProjectDir)include;$(ProjectDir)include\windows;$(ProjectDir)prov\netdir\NetDirect;$(ProjectDir)prov\verbs\include;$(ProjectDir)prov\hook\src;$(ProjectDir)prov\hook\include;$(ProjectDir)prov\hook\perf\include;$(ProjectDir)packages\NetworkDirect.2.0.1\include;$(ProjectDir)prov\efa\src;$(ProjectDir)prov\efa\include;$(ProjectDir)prov\efa\src\rxr;$(ProjectDir)prov\efa\src\efa_verbs;$(ProjectDir)prov\efa\src\efa_verbs\plat + $(ProjectDir)include;$(ProjectDir)include\windows;$(ProjectDir)prov\verbs\include;$(ProjectDir)prov\hook\src;$(ProjectDir)prov\hook\include;$(ProjectDir)prov\hook\perf\include;$(ProjectDir)packages\NetworkDirect.2.0.1\include;$(ProjectDir)prov\efa\src;$(ProjectDir)prov\efa\include;$(ProjectDir)prov\efa\src\rxr;$(ProjectDir)prov\efa\src\efa_verbs;$(ProjectDir)prov\efa\src\efa_verbs\plat 4127;4200;4204;4221;4115;4201;4100 true false @@ -315,7 +315,7 @@ true WIN32;_WINSOCKAPI_=;_CRT_SECURE_NO_WARNINGS;_WINSOCK_DEPRECATED_NO_WARNINGS;_WINDOWS;_USRDLL;LIBFABRIC_EXPORTS;HAVE_CONFIG_H;%(PreprocessorDefinitions) true - $(ProjectDir)include;$(ProjectDir)include\windows;$(ProjectDir)prov\netdir\NetDirect;$(ProjectDir)prov\verbs\include;$(ProjectDir)prov\hook\src;$(ProjectDir)prov\hook\include;$(ProjectDir)prov\hook\perf\include;$(ProjectDir)packages\NetworkDirect.2.0.1\include;$(ProjectDir)prov\efa\src;$(ProjectDir)prov\efa\include;$(ProjectDir)prov\efa\src\rxr;$(ProjectDir)prov\efa\src\efa_verbs;$(ProjectDir)prov\efa\src\efa_verbs\plat + $(ProjectDir)include;$(ProjectDir)include\windows;$(ProjectDir)prov\verbs\include;$(ProjectDir)prov\hook\src;$(ProjectDir)prov\hook\include;$(ProjectDir)prov\hook\perf\include;$(ProjectDir)packages\NetworkDirect.2.0.1\include;$(ProjectDir)prov\efa\src;$(ProjectDir)prov\efa\include;$(ProjectDir)prov\efa\src\rxr;$(ProjectDir)prov\efa\src\efa_verbs;$(ProjectDir)prov\efa\src\efa_verbs\plat 4127;4200;4204;4221;4115;4201;4100 true false @@ -335,7 +335,7 @@ true true WIN32;_WINSOCKAPI_=;_CRT_SECURE_NO_WARNINGS;_WINSOCK_DEPRECATED_NO_WARNINGS;_WINDOWS;_USRDLL;LIBFABRIC_EXPORTS;HAVE_CONFIG_H;%(PreprocessorDefinitions) - $(ProjectDir)include;$(ProjectDir)include\windows;$(ProjectDir)prov\netdir\NetDirect;$(ProjectDir)prov\verbs\include;$(ProjectDir)prov\hook\src;$(ProjectDir)prov\hook\include;$(ProjectDir)prov\hook\perf\include;$(ProjectDir)packages\NetworkDirect.2.0.1\include;$(ProjectDir)prov\efa\src;$(ProjectDir)prov\efa\include;$(ProjectDir)prov\efa\src\rxr;$(ProjectDir)prov\efa\src\efa_verbs;$(ProjectDir)prov\efa\src\efa_verbs\plat + $(ProjectDir)include;$(ProjectDir)include\windows;$(ProjectDir)prov\verbs\include;$(ProjectDir)prov\hook\src;$(ProjectDir)prov\hook\include;$(ProjectDir)prov\hook\perf\include;$(ProjectDir)packages\NetworkDirect.2.0.1\include;$(ProjectDir)prov\efa\src;$(ProjectDir)prov\efa\include;$(ProjectDir)prov\efa\src\rxr;$(ProjectDir)prov\efa\src\efa_verbs;$(ProjectDir)prov\efa\src\efa_verbs\plat 4127;4200;4204;4221;4115;4201;4100 true true @@ -356,7 +356,7 @@ true true WIN32;_WINSOCKAPI_=;_CRT_SECURE_NO_WARNINGS;_WINSOCK_DEPRECATED_NO_WARNINGS;_WINDOWS;_USRDLL;LIBFABRIC_EXPORTS;HAVE_CONFIG_H;%(PreprocessorDefinitions) - $(ProjectDir)include;$(ProjectDir)include\windows;$(ProjectDir)prov\sockets\include;$(ProjectDir)prov\netdir\NetDirect;$(ProjectDir)prov\hook\src;$(ProjectDir)prov\hook\include;$(ProjectDir)prov\hook\perf\include;$(ProjectDir)prov\efa\src;$(ProjectDir)prov\efa\src\rxr;$(ProjectDir)prov\efa\src\windows; + $(ProjectDir)include;$(ProjectDir)include\windows;$(ProjectDir)prov\sockets\include;$(ProjectDir)prov\hook\src;$(ProjectDir)prov\hook\include;$(ProjectDir)prov\hook\perf\include;$(ProjectDir)prov\efa\src;$(ProjectDir)prov\efa\src\rxr;$(ProjectDir)prov\efa\src\windows; 4127;4200;4204;4221;4115;4201;4100 true true @@ -378,7 +378,7 @@ true WIN32;_WINSOCKAPI_=;_CRT_SECURE_NO_WARNINGS;_WINSOCK_DEPRECATED_NO_WARNINGS;_WINDOWS;_USRDLL;LIBFABRIC_EXPORTS;HAVE_CONFIG_H;%(PreprocessorDefinitions) true - $(ProjectDir)include;$(ProjectDir)include\windows;$(ProjectDir)prov\netdir\NetDirect;$(ProjectDir)prov\verbs\include;$(ProjectDir)prov\hook\src;$(ProjectDir)prov\hook\include;$(ProjectDir)prov\hook\perf\include;$(ProjectDir)packages\NetworkDirect.2.0.1\include;$(ProjectDir)prov\efa\src;$(ProjectDir)prov\efa\include;$(ProjectDir)prov\efa\src\rxr;$(ProjectDir)prov\efa\src\efa_verbs;$(ProjectDir)prov\efa\src\efa_verbs\plat + $(ProjectDir)include;$(ProjectDir)include\windows;$(ProjectDir)prov\verbs\include;$(ProjectDir)prov\hook\src;$(ProjectDir)prov\hook\include;$(ProjectDir)prov\hook\perf\include;$(ProjectDir)packages\NetworkDirect.2.0.1\include;$(ProjectDir)prov\efa\src;$(ProjectDir)prov\efa\include;$(ProjectDir)prov\efa\src\rxr;$(ProjectDir)prov\efa\src\efa_verbs;$(ProjectDir)prov\efa\src\efa_verbs\plat 4127;4200;94;4204;4221;869 true false @@ -406,12 +406,6 @@ - - - - - - @@ -471,17 +465,6 @@ - - - - - - - - - - - $(ProjectDir)prov\sockets\include;%(AdditionalIncludeDirectories) $(ProjectDir)prov\sockets\include;%(AdditionalIncludeDirectories) @@ -858,6 +841,12 @@ $(ProjectDir)prov\verbs\src;$(ProjectDir)prov\verbs\include;$(ProjectDir)prov\verbs\include\windows;%(AdditionalIncludeDirectories) + + $(ProjectDir)prov\verbs\src;$(ProjectDir)prov\verbs\include;$(ProjectDir)prov\verbs\include\windows;%(AdditionalIncludeDirectories) + + + $(ProjectDir)prov\verbs\src;$(ProjectDir)prov\verbs\include;$(ProjectDir)prov\verbs\include\windows;%(AdditionalIncludeDirectories) + $(ProjectDir)prov\verbs\src;$(ProjectDir)prov\verbs\include;$(ProjectDir)prov\verbs\include\windows;%(AdditionalIncludeDirectories) @@ -981,18 +970,9 @@ - - - - - - - - - diff --git a/libfabric.vcxproj.filters b/libfabric.vcxproj.filters index 72962eb9394..5d7bd424e2f 100644 --- a/libfabric.vcxproj.filters +++ b/libfabric.vcxproj.filters @@ -79,15 +79,6 @@ {e89f94da-f0f0-4e62-aad4-d738963d7dd9} - - {bbe0be7c-1187-45ab-bd2e-5c1f61095a9b} - - - {9ba21998-278b-4f63-9ff2-8fb533eeee84} - - - {b3471da7-1f83-4d3f-b31c-89b197aa6327} - {5547753c-42f5-4cb6-a311-07911724f5cf} @@ -432,57 +423,6 @@ Source Files\src - - Source Files\prov\netdir\src - - - Source Files\prov\netdir\src - - - Source Files\prov\netdir\src - - - Source Files\prov\netdir\src - - - Source Files\prov\netdir\src - - - Source Files\prov\netdir\src - - - Source Files\prov\netdir\src - - - Source Files\prov\netdir\src - - - Source Files\prov\netdir\src - - - Source Files\prov\netdir\src - - - Source Files\prov\netdir\src - - - Source Files\prov\netdir\src - - - Source Files\prov\netdir\src - - - Source Files\prov\netdir\src - - - Source Files\prov\netdir\src - - - Source Files\prov\netdir\src - - - Source Files\prov\netdir\src - Source Files\prov\util @@ -645,6 +585,12 @@ Source Files\prov\verbs\src\windows + + Source Files\prov\verbs\src\windows + + + Source Files\prov\verbs\src\windows + @@ -833,33 +779,6 @@ Source Files\prov\rxm\include - - Source Files\prov\netdir\include - - - Source Files\prov\netdir\include - - - Source Files\prov\netdir\include - - - Source Files\prov\netdir\include - - - Source Files\prov\netdir\include - - - Source Files\prov\netdir\include - - - Source Files\prov\netdir\include - - - Source Files\prov\netdir\include - - - Source Files\prov\netdir\include - Header Files\windows\sys diff --git a/prov/netdir/NetDirect/README.NetworkDirect b/prov/netdir/NetDirect/README.NetworkDirect deleted file mode 100644 index a6d113a247c..00000000000 --- a/prov/netdir/NetDirect/README.NetworkDirect +++ /dev/null @@ -1,12 +0,0 @@ -Network Direct SDK/DDK may be obtained as a nuget package (preferred) from: - -https://www.nuget.org/packages/NetworkDirect - -or downloaded from: - -https://www.microsoft.com/en-us/download/details.aspx?id=36043 -on page press Download button and select NetworkDirect_DDK.zip. - -Extract header files from downloaded -NetworkDirect_DDK.zip:\NetDirect\include\ file into this directory, -or add path to NetDirect headers into VS include paths diff --git a/prov/netdir/src/netdir.h b/prov/netdir/src/netdir.h deleted file mode 100644 index 7fc361b6f79..00000000000 --- a/prov/netdir/src/netdir.h +++ /dev/null @@ -1,197 +0,0 @@ -/* -* Copyright (c) 2015-2016 Intel Corporation, Inc. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -*/ - -#ifndef _FI_NETDIR_H_ -#define _FI_NETDIR_H_ - -#if HAVE_CONFIG_H -# include -#endif /* HAVE_CONFIG_H */ - -#include -#include -#include - -#include -#include "ofi_osd.h" - -#ifdef __cplusplus -extern "C" { -#endif /* __cplusplus */ - - -#define ND_MSG_IOV_LIMIT (256) -#define ND_MSG_INTERNAL_IOV_LIMIT (512) -#define OFI_ND_MAX_MR_CNT (1 << 16) - -#define OFI_ND_DOMAIN_CAPS (FI_LOCAL_COMM | FI_REMOTE_COMM) - -#define OFI_ND_EP_CAPS (FI_MSG | FI_RMA | \ - FI_SEND | FI_RECV | \ - FI_READ | FI_WRITE | \ - FI_REMOTE_READ | FI_REMOTE_WRITE) - -#define OFI_ND_TX_OP_FLAGS (FI_INJECT | FI_COMPLETION | FI_TRANSMIT_COMPLETE | \ - FI_INJECT_COMPLETE | FI_DELIVERY_COMPLETE | \ - FI_SELECTIVE_COMPLETION) - -#define OFI_ND_MSG_ORDER (OFI_ORDER_RAR_SET | OFI_ORDER_RAW_SET | FI_ORDER_RAS | \ - OFI_ORDER_WAW_SET | FI_ORDER_WAS | FI_ORDER_SAW | FI_ORDER_SAS ) - -extern struct gl_data { - int inline_thr; - int prepost_cnt; - int prepost_buf_cnt; - int flow_control_cnt; - int total_avail; -} gl_data; - -extern struct fi_provider ofi_nd_prov; -extern struct util_prov ofi_nd_util_prov; -extern struct fi_info ofi_nd_info; - -extern const char ofi_nd_prov_name[]; - -int ofi_nd_cq_open(struct fid_domain *domain, struct fi_cq_attr *attr, - struct fid_cq **cq_fid, void *context); -int ofi_nd_eq_open(struct fid_fabric *fabric, struct fi_eq_attr *attr, - struct fid_eq **peq, void *context); -int ofi_nd_endpoint(struct fid_domain *domain, struct fi_info *info, - struct fid_ep **ep_fid, void *context); -int ofi_nd_passive_endpoint(struct fid_fabric *fabric, struct fi_info *info, - struct fid_pep **pep, void *context); -int ofi_nd_domain_open(struct fid_fabric *fabric, struct fi_info *info, - struct fid_domain **domain, void *context); -int ofi_nd_srx_ctx(struct fid_domain *domain, - struct fi_rx_attr *attr, struct fid_ep **rx_ep, - void *context); -int ofi_nd_fabric(struct fi_fabric_attr *attr, struct fid_fabric **fabric, - void *context); - -int ofi_nd_getinfo(uint32_t version, const char *node, const char *service, - uint64_t flags, const struct fi_info *hints, struct fi_info **info); -void ofi_nd_fini(void); - -int ofi_nd_mr_reg(struct fid *fid, const void *buf, size_t len, - uint64_t access, uint64_t offset, uint64_t requested_key, - uint64_t flags, struct fid_mr **mr, void *context); -int ofi_nd_mr_regv(struct fid *fid, const struct iovec *iov, - size_t count, uint64_t access, - uint64_t offset, uint64_t requested_key, - uint64_t flags, struct fid_mr **mr, void *context); -int ofi_nd_mr_regattr(struct fid *fid, const struct fi_mr_attr *attr, - uint64_t flags, struct fid_mr **mr); -int ofi_nd_cntr_open(struct fid_domain *pdomain, struct fi_cntr_attr *attr, - struct fid_cntr **pcntr, void *context); - -typedef int(*ofi_nd_adapter_cb_t)(const ND2_ADAPTER_INFO* info, const char *name); - -void ofi_nd_send_event(ND2_RESULT *result); -void ofi_nd_read_event(ND2_RESULT *result); -void ofi_nd_write_event(ND2_RESULT *result); - -HRESULT ofi_nd_startup(ofi_nd_adapter_cb_t cb); -HRESULT ofi_nd_shutdown(); - -int ofi_nd_lookup_adapter(const char *name, IND2Adapter **adapter, struct sockaddr** addr); - -int ofi_nd_is_valid_addr(const SOCKADDR *addr); -int ofi_nd_addr_cmp(const void* vaddr1, const void* vaddr2); - -int ofi_nd_is_same_file(const wchar_t* path1, const wchar_t* path2); -int ofi_nd_file_exists(const wchar_t* path); -int ofi_nd_is_directory(const wchar_t* path); -const wchar_t *ofi_nd_filename(const wchar_t *path); - -static inline size_t unique(void *base, size_t num, size_t width, - int(*cmp)(const void *, const void *)) -{ - char *dst = (char*)base; - char *src = (char*)base + width; - - size_t i; - size_t n = 1; - - if (num < 2) - return num; - - for (i = 1; i < num; i++) { - if (cmp(dst, src)) { - dst += width; - if (dst != src) - memcpy(dst, src, width); - n++; - } - src += width; - } - - return n; -} - -#define H2F(x) ofi_nd_hresult_2_fierror(x) - -static inline int ofi_nd_hresult_2_fierror(HRESULT hr) -{ - switch (hr) { - case S_OK: - case ND_PENDING: - return FI_SUCCESS; - case ND_BUFFER_OVERFLOW: - return -FI_EOVERFLOW; - case ND_CONNECTION_REFUSED: - return -FI_ECONNREFUSED; - case ND_TIMEOUT: - return -FI_ETIMEDOUT; - default: - return -FI_EOTHER; - } -} - -#define OFI_ND_TIMEOUT_INIT(timeout) \ - uint64_t sfinish = ((timeout) >= 0) ? \ - (ofi_gettime_ms() + (timeout) * 10000) : -1; - -#define OFI_ND_TIMEDOUT() ((sfinish > 0) ? ofi_gettime_ms() >= sfinish : 0) - -#ifdef ENABLE_DEBUG -# define NODEFAULT assert(0) -#else -# define NODEFAULT __assume(0) -#endif - - -#ifdef __cplusplus -} -#endif /* __cplusplus */ - -#endif /* _FI_NETDIR_H_ */ - diff --git a/prov/netdir/src/netdir_buf.h b/prov/netdir/src/netdir_buf.h deleted file mode 100644 index d399c20c3c4..00000000000 --- a/prov/netdir/src/netdir_buf.h +++ /dev/null @@ -1,255 +0,0 @@ -/* -* Copyright (c) 2015-2016 Intel Corporation, Inc. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -*/ - -#ifndef _FI_NETDIR_BUF_H_ -#define _FI_NETDIR_BUF_H_ - -#include - -#include "rdma/fabric.h" - -#include "ofi.h" -#include "ofi_osd.h" - -#include "netdir.h" - -#ifdef __cplusplus -extern "C" { -#endif /* __cplusplus */ - -/* Implementation of lock-free thread safe allocator of fixed size data */ -/* - * Lock free allocator is implemented using atomics add/compare-and-swap/sub. - * There is three levels of data: - * footer - high level structure, holds links to first chunk, first non-used - * | element, number of allocated elelements and number of used elements. - * | - * +-chunk - one-directional linked list where stored items - * | - * +-item - one directional linked list where stored user's data - * - * chunk elements are linked to be freed only, nothing else used linked nature of - * chunk. - * item elemens may have two states: used/not-used. All not-used (free) elements - * are linked into one-directional linked list, pointer to first non-used - * element is located at footer. - * - * when user allocates new element function buf_alloc looks for number of used - * elements and compares it with allocated number of elements: used atomic - * API ofi_atomic_inc32 to increment counter by 1 and check if there - * is non-used items. In case if no more available elements exist - * (used > counter) - allocate new chunk. - * - * All pointers are updated using atomic API compare-and-swap which guarantee - * thread safe of access to allocator. Only init/fini calls are not thread safe. - * - * Call of ofi_nd_buf_init_*** API is not mandatory - default initialization - * by NULL is enough - * - * Note: underscored (__ofi_nd_buf_****) functions are used to implement footer - * inside of another objects - */ - -#define OFI_ND_BUF_DEFCOUNT 1024 -#define ICEP InterlockedCompareExchangePointer -#define countof(x) _countof(x) /*(sizeof(x) / sizeof(*(x)))*/ - -#define OFI_ND_NB_BUF_IMP(name) struct nd_buf_footer_##name nd_footer_##name = {0}; - -#define OFI_ND_NB_BUF_IMP_ALLOC(name, a, f) \ - struct nd_buf_footer_##name nd_footer_##name = {.alloc_chunk = a, .free_chunk = f}; - -#define OFI_ND_NB_BUF(name) OFI_ND_NB_BUF_TYPED(name, name) - -#define OFI_ND_NB_BUF_TYPED(name, type) \ - struct nd_buf_item_##name { \ - struct nd_buf_item_##name *next; /* next free element */ \ - type data; /* user's data */ \ - }; \ - \ - struct nd_buf_chunk_##name { /* cache of elements for fast alloc */ \ - struct nd_buf_chunk_##name *next; \ - struct nd_buf_item_##name item[OFI_ND_BUF_DEFCOUNT]; \ - }; \ - \ - struct nd_buf_footer_##name { \ - LONG count; /* number of elements */ \ - LONG used; /* count of used elements */ \ - struct nd_buf_chunk_##name*(*alloc_chunk)( \ - struct nd_buf_footer_##name *footer, size_t *count); \ - void(*free_chunk)(struct nd_buf_chunk_##name *chunk); \ - volatile struct nd_buf_chunk_##name *chunk_head; \ - volatile struct nd_buf_item_##name *item_free; \ - }; \ - \ - extern struct nd_buf_footer_##name nd_footer_##name; \ - \ - static inline int __ofi_nd_buf_init_##name(struct nd_buf_footer_##name *footer) \ - { \ - assert(footer); \ - memset(footer, 0, sizeof(*footer)); \ - return FI_SUCCESS; \ - } \ - \ - static inline int ofi_nd_buf_init_##name(void) \ - { \ - return __ofi_nd_buf_init_##name(&nd_footer_##name); \ - } \ - \ - static inline void __ofi_nd_buf_fini_##name(struct nd_buf_footer_##name *footer) \ - { \ - typedef struct nd_buf_chunk_##name nd_buf_chunk_##name; \ - assert(footer); \ - assert(!footer->used); \ - \ - nd_buf_chunk_##name *next = (nd_buf_chunk_##name*)footer->chunk_head; \ - while (next) { \ - nd_buf_chunk_##name *chunk = next; \ - next = chunk->next; \ - if(!footer->free_chunk) \ - free(chunk); \ - else \ - footer->free_chunk(chunk); \ - } \ - memset(footer, 0, sizeof(*footer)); \ - } \ - \ - static inline void ofi_nd_buf_fini_##name(void) \ - { \ - __ofi_nd_buf_fini_##name(&nd_footer_##name); \ - } \ - \ - /* Do NOT clean item after allocation because there may be user's data */ \ - static inline type *__ofi_nd_buf_alloc_##name(struct nd_buf_footer_##name *footer) \ - { \ - typedef struct nd_buf_chunk_##name nd_buf_chunk_##name; \ - typedef struct nd_buf_item_##name nd_buf_item_##name; \ - \ - assert(footer); \ - \ - if (InterlockedIncrement(&footer->used) > footer->count) { \ - /* allocate new chunk of data */ \ - size_t i; \ - nd_buf_chunk_##name *data; \ - nd_buf_chunk_##name *next; \ - nd_buf_item_##name *next_free; \ - size_t count; \ - \ - if(!footer->alloc_chunk) { \ - data = (nd_buf_chunk_##name*)malloc(sizeof(*data)); \ - memset(data, 0, sizeof(*data)); \ - count = countof(data->item); \ - } \ - else { \ - data = footer->alloc_chunk(footer, &count); \ - } \ - if (!data || !count) \ - return 0; \ - for (i = 0; i < count - 1; i++) { \ - data->item[i].next = &data->item[i + 1]; \ - } \ - /* insert new created free elements into linked list */ \ - /* data->item[0] will be returned to user as used element */ \ - /* items data->item[1..N] are linked to non-used list elements */ \ - do { \ - next_free = (nd_buf_item_##name*)footer->item_free; \ - data->item[count - 1].next = next_free; \ - } while (ICEP((volatile PVOID*)&footer->item_free, \ - (void*)&data->item[1], (void*)next_free) != next_free); \ - \ - InterlockedAdd(&footer->count, (LONG)count); \ - \ - /* add new chunk into footer */ \ - do { \ - next = (nd_buf_chunk_##name*)footer->chunk_head; \ - data->next = next; \ - } while (ICEP((volatile PVOID*)&footer->chunk_head, \ - (void*)data, (void*)next) != next); \ - return &data->item[0].data; \ - } \ - \ - nd_buf_item_##name *top_free; \ - nd_buf_item_##name *next_free; \ - do { \ - assert(footer->item_free); \ - top_free = (nd_buf_item_##name *)footer->item_free; \ - next_free = (nd_buf_item_##name *)footer->item_free->next; \ - } while (ICEP((volatile PVOID*)&footer->item_free, \ - (void*)next_free, (void*)top_free) != top_free); \ - \ - return &top_free->data; \ - } \ - \ - static inline type *ofi_nd_buf_alloc_##name(void) \ - { \ - return __ofi_nd_buf_alloc_##name(&nd_footer_##name); \ - } \ - \ - static inline \ - void __ofi_nd_buf_free_##name(type *data, struct nd_buf_footer_##name *footer) \ - { \ - typedef struct nd_buf_item_##name nd_buf_item_##name; \ - \ - assert(footer); \ - assert(data); \ - \ - nd_buf_item_##name *item = container_of(data, struct nd_buf_item_##name, data); \ - \ - do { \ - item->next = (nd_buf_item_##name*)footer->item_free; \ - } while (ICEP((volatile PVOID*)&footer->item_free, \ - (void*)item, (void*)item->next) != item->next); \ - \ - LONG dec = InterlockedDecrement(&footer->used); \ - assert(dec >= 0); \ - OFI_UNUSED(dec); \ - } \ - \ - static inline void ofi_nd_buf_free_##name(type *data) \ - { \ - __ofi_nd_buf_free_##name(data, &nd_footer_##name); \ - } - -#define ND_BUF_ALLOC(name) ofi_nd_buf_alloc_##name() -#define ND_BUF_FREEPTR(name) ofi_nd_buf_free_##name -#define ND_BUF_FREE(name, ptr) ND_BUF_FREEPTR(name)(ptr) -#define ND_BUF_CHUNK(name) struct nd_buf_chunk_##name -#define ND_BUF_FOOTER(name) struct nd_buf_footer_##name -#define ND_BUF_FINIPTR(name) ofi_nd_buf_fini_##name -#define ND_BUF_FINI(name) ND_BUF_FINIPTR(name)() - - -#ifdef __cplusplus -} -#endif /* __cplusplus */ - -#endif /* _FI_NETDIR_BUF_H_ */ diff --git a/prov/netdir/src/netdir_cntr.c b/prov/netdir/src/netdir_cntr.c deleted file mode 100644 index cd3a1978192..00000000000 --- a/prov/netdir/src/netdir_cntr.c +++ /dev/null @@ -1,208 +0,0 @@ -/* -* Copyright (c) 2015-2016 Intel Corporation, Inc. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -*/ - -#ifdef _WIN32 - -#include "netdir.h" -#include "netdir_ov.h" -#include "netdir_iface.h" - -#include "rdma/fabric.h" -#include "ofi_util.h" - -static int ofi_nd_cntr_close(struct fid *fid); -static uint64_t ofi_nd_cntr_read(struct fid_cntr *cntr); -static uint64_t ofi_nd_cntr_readerr(struct fid_cntr *cntr); -static int ofi_nd_cntr_add(struct fid_cntr *cntr, uint64_t value); -static int ofi_nd_cntr_set(struct fid_cntr *cntr, uint64_t value); -static int ofi_nd_cntr_wait(struct fid_cntr *cntr, - uint64_t threshold, int timeout); - -static struct fi_ops ofi_nd_fi_ops = { - .size = sizeof(ofi_nd_fi_ops), - .close = ofi_nd_cntr_close, - .bind = fi_no_bind, - .control = fi_no_control, - .ops_open = fi_no_ops_open, -}; - -static struct fid ofi_nd_fid = { - .fclass = FI_CLASS_CNTR, - .context = NULL, - .ops = &ofi_nd_fi_ops -}; - -static struct fi_ops_cntr ofi_nd_cntr_ops = { - .size = sizeof(ofi_nd_cntr_ops), - .read = ofi_nd_cntr_read, - .readerr = ofi_nd_cntr_readerr, - .add = ofi_nd_cntr_add, - .set = ofi_nd_cntr_set, - .wait = ofi_nd_cntr_wait -}; - -static int ofi_nd_cntr_close(struct fid *fid) -{ - assert(fid->fclass == FI_CLASS_CNTR); - if (fid->fclass != FI_CLASS_CQ) - return -FI_EINVAL; - - struct nd_cntr *cntr = container_of(fid, struct nd_cntr, fid.fid); - - free(cntr); - - return FI_SUCCESS; -} - -int ofi_nd_cntr_open(struct fid_domain *pdomain, struct fi_cntr_attr *attr, - struct fid_cntr **pcntr, void *context) -{ - OFI_UNUSED(context); - - assert(pdomain); - assert(pdomain->fid.fclass == FI_CLASS_DOMAIN); - - if (attr) { - if (attr->wait_obj != FI_WAIT_NONE && - attr->wait_obj != FI_WAIT_UNSPEC) - return -FI_EBADFLAGS; - } - - struct nd_cntr *cntr = (struct nd_cntr*)calloc(1, sizeof(*cntr)); - if (!cntr) - return -FI_ENOMEM; - - struct nd_cntr def = { - .fid = { - .fid = ofi_nd_fid, - .ops = &ofi_nd_cntr_ops - }, - }; - - *cntr = def; - - *pcntr = &cntr->fid; - - return FI_SUCCESS; -} - -static uint64_t ofi_nd_cntr_read(struct fid_cntr *pcntr) -{ - assert(pcntr); - assert(pcntr->fid.fclass == FI_CLASS_CNTR); - - struct nd_cntr *cntr = container_of(pcntr, struct nd_cntr, fid); - return cntr->counter; -} - -static uint64_t ofi_nd_cntr_readerr(struct fid_cntr *pcntr) -{ - assert(pcntr); - assert(pcntr->fid.fclass == FI_CLASS_CNTR); - - struct nd_cntr *cntr = container_of(pcntr, struct nd_cntr, fid); - return cntr->err; -} - -static int ofi_nd_cntr_add(struct fid_cntr *pcntr, uint64_t value) -{ - assert(pcntr); - assert(pcntr->fid.fclass == FI_CLASS_CNTR); - - if (pcntr->fid.fclass != FI_CLASS_CNTR) - return -FI_EINVAL; - - struct nd_cntr *cntr = container_of(pcntr, struct nd_cntr, fid); - - cntr->counter += value; - WakeByAddressAll((void*)&cntr->counter); - - return FI_SUCCESS; -} - -static int ofi_nd_cntr_set(struct fid_cntr *pcntr, uint64_t value) -{ - assert(pcntr); - assert(pcntr->fid.fclass == FI_CLASS_CNTR); - - if (pcntr->fid.fclass != FI_CLASS_CNTR) - return -FI_EINVAL; - - struct nd_cntr *cntr = container_of(pcntr, struct nd_cntr, fid); - - cntr->counter = value; - WakeByAddressAll((void*)&cntr->counter); - - return FI_SUCCESS; -} - -static int ofi_nd_cntr_wait(struct fid_cntr *pcntr, - uint64_t threshold, int timeout) -{ - assert(pcntr); - assert(pcntr->fid.fclass == FI_CLASS_CNTR); - - if (pcntr->fid.fclass != FI_CLASS_CNTR) - return -FI_EINVAL; - - struct nd_cntr *cntr = container_of(pcntr, struct nd_cntr, fid); - - /* process corner timeouts separately to optimize */ - if (!timeout) { /* no wait */ - return (cntr->counter >= (LONGLONG)threshold) ? - FI_SUCCESS : -FI_ETIMEDOUT; - } - else if (timeout < 0) { /* infinite wait */ - while (cntr->counter < (LONG64)threshold) { - LONG64 val = cntr->counter; - WaitOnAddress(&cntr->counter, &val, - sizeof(val), INFINITE); - } - return FI_SUCCESS; - } - else { /* timeout wait */ - OFI_ND_TIMEOUT_INIT(timeout); - - do { - if (cntr->counter >= (LONG64)threshold) - return FI_SUCCESS; - LONG64 val = cntr->counter; - WaitOnAddress(&cntr->counter, &val, - sizeof(val), timeout); - } while (!OFI_ND_TIMEDOUT()); - } - - return FI_SUCCESS; -} - -#endif /* _WIN32 */ - diff --git a/prov/netdir/src/netdir_cq.c b/prov/netdir/src/netdir_cq.c deleted file mode 100644 index b21328a03af..00000000000 --- a/prov/netdir/src/netdir_cq.c +++ /dev/null @@ -1,1067 +0,0 @@ -/* -* Copyright (c) 2015-2016 Intel Corporation, Inc. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -*/ - -#ifdef _WIN32 - -#include "netdir.h" -#include "netdir_cq.h" -#include "netdir_iface.h" -#include "netdir_unexp.h" - -#include "rdma/fabric.h" -#include "ofi_util.h" - -static int ofi_nd_cq_close(struct fid *fid); -static ssize_t ofi_nd_cq_read(struct fid_cq *cq, void *buf, size_t count); -static ssize_t ofi_nd_cq_readfrom(struct fid_cq *cq, void *buf, size_t count, - fi_addr_t *src_addr); -static ssize_t ofi_nd_cq_readerr(struct fid_cq *cq, struct fi_cq_err_entry *buf, - uint64_t flags); -static ssize_t ofi_nd_cq_sread(struct fid_cq *cq, void *buf, size_t count, - const void *cond, int timeout); -static ssize_t ofi_nd_cq_sreadfrom(struct fid_cq *cq, void *buf, size_t count, - fi_addr_t *src_addr, const void *cond, - int timeout); -static int ofi_nd_cq_signal(struct fid_cq* fid_cq); -static const char *ofi_nd_cq_strerror(struct fid_cq *cq, int prov_errno, - const void *err_data, char *buf, - size_t len); -ssize_t ofi_nd_ep_injectdata(struct fid_ep *ep, const void *buf, size_t len, - uint64_t data, fi_addr_t dest_addr); - -static struct fi_ops ofi_nd_fi_ops = { - .size = sizeof(ofi_nd_fi_ops), - .close = ofi_nd_cq_close, - .bind = fi_no_bind, - .control = fi_no_control, - .ops_open = fi_no_ops_open, -}; - -static struct fid ofi_nd_fid = { - .fclass = FI_CLASS_CQ, - .context = NULL, - .ops = &ofi_nd_fi_ops -}; - -static struct fi_ops_cq ofi_nd_cq_ops = { - .size = sizeof(ofi_nd_cq_ops), - .read = ofi_nd_cq_read, - .readfrom = ofi_nd_cq_readfrom, - .readerr = ofi_nd_cq_readerr, - .sread = ofi_nd_cq_sread, - .sreadfrom = ofi_nd_cq_sreadfrom, - .signal = ofi_nd_cq_signal, - .strerror = ofi_nd_cq_strerror -}; - -typedef struct nd_cq_action { - nd_event_base base; - struct nd_cq *cq; -} nd_cq_action; - -OFI_ND_NB_BUF(nd_cq_action); - -OFI_ND_NB_BUF_IMP(nd_cq_action); -OFI_ND_NB_BUF_IMP(nd_cq_entry); - -OFI_ND_NB_BUF_IMP(nd_send_entry); -OFI_ND_NB_BUF_IMP(nd_sge); - -/* State-Event matrix callbacks. Must have the following signature - * "void (nd_cq_entry*, void*)" */ -static inline void ofi_nd_handle_unknown(nd_cq_entry *entry, void *misc); -static inline void ofi_nd_unexp_2_read(nd_cq_entry *entry, void *unexpected); -static inline void ofi_nd_read_2_send_ack(nd_cq_entry *entry, void *res); -static inline void ofi_nd_event_2_cq(nd_cq_entry *entry, void *misc); -static inline void ofi_nd_unexp_ack_2_cq(nd_cq_entry *entry, void *unexpected); - -/* Auxillary functions to ensure workability of callbacks of S-E matrix - * and are used implicitly inside these callbakcs */ -static void ofi_nd_unexp_2_cq(nd_cq_entry *entry, nd_unexpected_entry *unexp); -static void ofi_nd_read_2_cq(nd_cq_entry *entry, ND2_RESULT *result); - -#define UNKNWN ofi_nd_handle_unknown -#define UNEXP_2_READ ofi_nd_unexp_2_read -#define READ_2_SENDACK ofi_nd_read_2_send_ack -#define EVENT_2_CQ ofi_nd_event_2_cq -#define UNEXP_ACK_2_CQ ofi_nd_unexp_ack_2_cq - -typedef void(*cq_matrix_cb)(struct nd_cq_entry *entry, void *misc); - -static cq_matrix_cb cq_matrix[MAX_STATE][MAX_EVENT] = { - { EVENT_2_CQ, UNEXP_2_READ, UNKNWN }, - { UNKNWN, READ_2_SENDACK, UNKNWN }, - { UNKNWN, UNKNWN, UNEXP_ACK_2_CQ } -}; - -void ofi_nd_dispatch_cq_event(ofi_nd_cq_event event, nd_cq_entry *entry, void *misc) -{ - ofi_nd_cq_state state = entry->state; - cq_matrix_cb cb = cq_matrix[state][event]; - ND_LOG_DEBUG(FI_LOG_EP_DATA, "Dispatch Event - %d:%d\n", - state, event); - cb(entry, misc); -} - -static int ofi_nd_cq_close(struct fid *fid) -{ - assert(fid->fclass == FI_CLASS_CQ); - if (fid->fclass != FI_CLASS_CQ) - return -FI_EINVAL; - - struct nd_cq *cq = container_of(fid, struct nd_cq, fid.fid); - - if (cq->iocp && cq->iocp != INVALID_HANDLE_VALUE) - CloseHandle(cq->iocp); - if (cq->err && cq->err != INVALID_HANDLE_VALUE) - CloseHandle(cq->err); - - free(cq); - - return FI_SUCCESS; -} - -int ofi_nd_cq_open(struct fid_domain *pdomain, struct fi_cq_attr *attr, - struct fid_cq **pcq_fid, void *context) -{ - OFI_UNUSED(context); - - assert(pdomain); - assert(pdomain->fid.fclass == FI_CLASS_DOMAIN); - - if (pdomain->fid.fclass != FI_CLASS_DOMAIN) - return -FI_EINVAL; - - ND_REGISTER_FINI(ND_BUF_FINIPTR(nd_cq_action)); - ND_REGISTER_FINI(ND_BUF_FINIPTR(nd_cq_entry)); - - if (pdomain->fid.fclass != FI_CLASS_DOMAIN) - return -FI_EINVAL; - - HRESULT hr; - - if (attr) { - if (attr->wait_obj != FI_WAIT_NONE && - attr->wait_obj != FI_WAIT_UNSPEC) - return -FI_EBADFLAGS; - } - - struct nd_cq *cq = (struct nd_cq*)calloc(1, sizeof(*cq)); - if (!cq) - return -FI_ENOMEM; - - struct nd_cq def = { - .fid = { - .fid = ofi_nd_fid, - .ops = &ofi_nd_cq_ops - }, - .format = attr ? attr->format : FI_CQ_FORMAT_CONTEXT - }; - - *cq = def; - - if (cq->format == FI_CQ_FORMAT_UNSPEC) { - cq->format = FI_CQ_FORMAT_CONTEXT; - if (attr) - attr->format = cq->format; - } - - struct nd_domain *domain = container_of(pdomain, struct nd_domain, fid); - assert(domain->adapter); - assert(domain->adapter_file); - OFI_UNUSED(domain); - - cq->iocp = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0); - if (!cq->iocp || cq->iocp == INVALID_HANDLE_VALUE) { - hr = -FI_EINVAL; - goto hr_fail; - } - cq->err = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0); - if (!cq->err || cq->err == INVALID_HANDLE_VALUE) { - hr = -FI_EINVAL; - goto hr_fail; - } - - *pcq_fid = &cq->fid; - - return FI_SUCCESS; - -hr_fail: - ofi_nd_cq_close(&cq->fid.fid); - ND_LOG_WARN(FI_LOG_CQ, ofi_nd_strerror((DWORD)hr, NULL)); - return H2F(hr); -} - -static uint64_t ofi_nd_cq_sanitize_flags(uint64_t flags) -{ - return (flags & (FI_SEND | FI_RECV | FI_RMA | FI_ATOMIC | - FI_MSG | FI_TAGGED | - FI_READ | FI_WRITE | - FI_REMOTE_READ | FI_REMOTE_WRITE | - FI_REMOTE_CQ_DATA | FI_MULTI_RECV)); -} - -static void ofi_nd_cq_ov2buf(struct nd_cq *cq, OVERLAPPED_ENTRY *ov, - void* buf, ULONG count) -{ - ULONG i; - struct nd_msgprefix *prefix; - - switch (cq->format) { - case FI_CQ_FORMAT_CONTEXT: - { - struct fi_cq_entry *entry = (struct fi_cq_entry*)buf; - for (i = 0; i < count; i++) { - struct nd_cq_entry *cqen = container_of(ov[i].lpOverlapped, struct nd_cq_entry, base.ov); - entry[i].op_context = cqen->context; - ofi_nd_free_cq_entry(cqen); - } - } - break; - case FI_CQ_FORMAT_MSG: - { - struct fi_cq_msg_entry *entry = (struct fi_cq_msg_entry*)buf; - for (i = 0; i < count; i++) { - struct nd_cq_entry *cqen = container_of(ov[i].lpOverlapped, struct nd_cq_entry, base.ov); - entry[i].op_context = cqen->context; - entry[i].flags = ofi_nd_cq_sanitize_flags(cqen->flags); - /* for send/receive operations there message header used, - and common size of transferred message is bit - bigger, in this case decrement transferred message - size by header size */ - size_t header_len = (cqen->result.RequestType == Nd2RequestTypeSend || - cqen->result.RequestType == Nd2RequestTypeReceive) ? - sizeof(prefix->header) : 0; - entry[i].len = cqen->result.BytesTransferred - header_len; - ofi_nd_free_cq_entry(cqen); - } - } - break; - case FI_CQ_FORMAT_DATA: - { - struct fi_cq_data_entry *entry = (struct fi_cq_data_entry*)buf; - for (i = 0; i < count; i++) { - struct nd_cq_entry *cqen = container_of(ov[i].lpOverlapped, struct nd_cq_entry, base.ov); - entry[i].op_context = cqen->context; - entry[i].flags = ofi_nd_cq_sanitize_flags(cqen->flags); - size_t header_len = (cqen->result.RequestType == Nd2RequestTypeSend || - cqen->result.RequestType == Nd2RequestTypeReceive) ? - sizeof(prefix->header) : 0; - entry[i].len = cqen->result.BytesTransferred - header_len; - entry[i].buf = cqen->buf; - ofi_nd_free_cq_entry(cqen); - } - } - break; - case FI_CQ_FORMAT_TAGGED: - { - struct fi_cq_tagged_entry *entry = (struct fi_cq_tagged_entry*)buf; - for (i = 0; i < count; i++) { - struct nd_cq_entry *cqen = container_of(ov[i].lpOverlapped, struct nd_cq_entry, base.ov); - entry[i].op_context = cqen->context; - entry[i].flags = ofi_nd_cq_sanitize_flags(cqen->flags); - size_t header_len = (cqen->result.RequestType == Nd2RequestTypeSend || - cqen->result.RequestType == Nd2RequestTypeReceive) ? - sizeof(prefix->header) : 0; - entry[i].len = cqen->result.BytesTransferred - header_len; - entry[i].buf = cqen->buf; - entry[i].tag = 0; - ofi_nd_free_cq_entry(cqen); - } - } - break; - default: - ND_LOG_WARN(FI_LOG_CQ, "incorrect CQ format: %d\n", cq->format); - break; - } -} - -static ssize_t ofi_nd_cq_read(struct fid_cq *pcq, void *buf, size_t count) -{ - assert(pcq); - assert(pcq->fid.fclass == FI_CLASS_CQ); - - if (pcq->fid.fclass != FI_CLASS_CQ) - return -FI_EINVAL; - - struct nd_cq *cq = container_of(pcq, struct nd_cq, fid); - - ULONG cnt = (ULONG)count; - ULONG dequeue = 0; - ssize_t res = 0; - OVERLAPPED_ENTRY _ov[256]; - - if (!cq->count) - return -FI_EAGAIN; - - OVERLAPPED_ENTRY *ov = (cnt <= countof(_ov)) ? - _ov : malloc(cnt * sizeof(*ov)); - - if (!ov) { - ND_LOG_WARN(FI_LOG_CQ, "failed to allocate OV\n"); - return -FI_ENOMEM; - } - - assert(cq->iocp && cq->iocp != INVALID_HANDLE_VALUE); - if (!GetQueuedCompletionStatusEx(cq->iocp, ov, cnt, &dequeue, 0, FALSE) || - !dequeue) { - res = cq->count ? -FI_EAVAIL : -FI_EAGAIN; - goto fn_complete; - } - - ofi_nd_cq_ov2buf(cq, ov, buf, dequeue); - res = (ssize_t)dequeue; - InterlockedAdd(&cq->count, -(LONG)dequeue); - assert(cq->count >= 0); - -fn_complete: - if (ov != _ov) - free(ov); - return res; -} - -static ssize_t ofi_nd_cq_readfrom(struct fid_cq *cq, void *buf, size_t count, - fi_addr_t *src_addr) -{ - size_t i; - for(i = 0; i < count; i++) - src_addr[i] = FI_ADDR_NOTAVAIL; - return ofi_nd_cq_read(cq, buf, count); -} - -static ssize_t ofi_nd_cq_readerr(struct fid_cq *pcq, struct fi_cq_err_entry *buf, - uint64_t flags) -{ - assert(pcq); - assert(pcq->fid.fclass == FI_CLASS_CQ); - assert(buf); - - OFI_UNUSED(flags); - - if (pcq->fid.fclass != FI_CLASS_CQ) - return -FI_EINVAL; - - struct nd_cq *cq = container_of(pcq, struct nd_cq, fid); - - ULONG_PTR key = 0; - DWORD bytes; - OVERLAPPED *ov; - - if (!cq->count) - return -FI_EAGAIN; - - assert(cq->err && cq->err != INVALID_HANDLE_VALUE); - if (!GetQueuedCompletionStatus(cq->err, &bytes, &key, &ov, 0)) - return -FI_EAGAIN; - - struct nd_cq_entry *entry = container_of(ov, struct nd_cq_entry, base.ov); - - buf->op_context = entry->result.RequestContext; - buf->flags = entry->flags; - buf->len = entry->len; - buf->buf = entry->buf; - buf->data = entry->data; - buf->tag = 0; /* while tagged send/recv isn't added */ - buf->olen = 0; - buf->err = -H2F(entry->result.Status); - buf->prov_errno = entry->result.Status; - buf->err_data_size = 0; - - InterlockedDecrement(&cq->count); - assert(cq->count >= 0); - - return 0; -} - -static ssize_t ofi_nd_cq_sread(struct fid_cq *pcq, void *buf, size_t count, - const void *cond, int timeout) -{ - assert(pcq); - assert(pcq->fid.fclass == FI_CLASS_CQ); - - OFI_UNUSED(cond); - - if (pcq->fid.fclass != FI_CLASS_CQ) - return -FI_EINVAL; - - struct nd_cq *cq = container_of(pcq, struct nd_cq, fid); - - ULONG cnt = (ULONG)count; - ULONG dequeue = 0; - ssize_t res = 0; - OVERLAPPED_ENTRY _ov[256]; - - OVERLAPPED_ENTRY *ov = (cnt <= countof(_ov)) ? - _ov : malloc(cnt * sizeof(*ov)); - - if (!ov) { - ND_LOG_WARN(FI_LOG_CQ, "failed to allocate OV\n"); - return -FI_ENOMEM; - } - - LONG zero = 0; - OFI_ND_TIMEOUT_INIT(timeout); - - do { - do { - if (!WaitOnAddress( - &cq->count, &zero, sizeof(cq->count), - (DWORD)timeout) && timeout >= 0) { - res = -FI_EAGAIN; - goto fn_complete; - } - } while (!cq->count && !OFI_ND_TIMEDOUT()); - - if (cq->count <= 0) { - res = -FI_EAGAIN; - goto fn_complete; - } - - - - assert(cq->iocp && cq->iocp != INVALID_HANDLE_VALUE); - if (!GetQueuedCompletionStatusEx(cq->iocp, ov, cnt, &dequeue, 0, FALSE) || - !dequeue) { - if (cq->count) { - res = -FI_EAVAIL; - goto fn_complete; - } - else { - continue; - } - } - - ofi_nd_cq_ov2buf(cq, ov, buf, dequeue); - res = (ssize_t)dequeue; - InterlockedAdd(&cq->count, -(LONG)dequeue); - assert(cq->count >= 0); - goto fn_complete; - } while (!OFI_ND_TIMEDOUT()); - -fn_complete: - if (ov != _ov) - free(ov); - return res; -} - -static ssize_t ofi_nd_cq_sreadfrom(struct fid_cq *cq, void *buf, size_t count, - fi_addr_t *src_addr, const void *cond, - int timeout) -{ - size_t i; - for (i = 0; i < count; i++) - src_addr[i] = FI_ADDR_NOTAVAIL; - return ofi_nd_cq_sread(cq, buf, count, cond, timeout); -} - -static int ofi_nd_cq_signal(struct fid_cq* fid_cq) -{ - assert(fid_cq); - assert(fid_cq->fid.fclass == FI_CLASS_CQ); - - if (fid_cq->fid.fclass != FI_CLASS_CQ) - return -FI_EINVAL; - - struct nd_cq* cq = container_of(fid_cq, struct nd_cq, fid); - - InterlockedDecrement(&cq->count); - WakeByAddressAll(&cq->count); - - return FI_SUCCESS; -} - -static const char *ofi_nd_cq_strerror(struct fid_cq *cq, int prov_errno, - const void *err_data, char *buf, - size_t len) -{ - OFI_UNUSED(cq); - OFI_UNUSED(err_data); - - if (buf && len) - return strncpy(buf, fi_strerror(-prov_errno), len); - return fi_strerror(-prov_errno); -} - -static void ofi_nd_cq_event_free(nd_event_base *base) -{ - nd_cq_action *action = container_of(base, nd_cq_action, base); - ND_BUF_FREE(nd_cq_action, action); -} - -void ofi_nd_handle_unknown(nd_cq_entry *entry, void *unexp) -{ - OFI_UNUSED(entry); - OFI_UNUSED(unexp); - - ND_LOG_DEBUG(FI_LOG_CQ, "Unknown event-state, " - "the event can't be handled\n"); - - /* Shouldn't go here */ - assert(0); - - return; -} - -void ofi_nd_event_2_cq(nd_cq_entry *entry, void *misc) -{ - /* Memory region is set in CQ entry only in case of RMA operation. - * Use this fact to realize what kind of operation is completed */ - if (!entry->mr_count) - ofi_nd_unexp_2_cq(entry, (nd_unexpected_entry *)misc); - else - ofi_nd_read_2_cq(entry, (ND2_RESULT *)misc); -} - -/* do NOT release unexpected here, becuase it just an allocated on stack entry */ -void ofi_nd_unexp_ack_2_cq(nd_cq_entry *entry, void *unexpected) -{ - nd_unexpected_entry *unexp = (nd_unexpected_entry *)unexpected; - struct nd_ep *ep = unexp->ep; - ND2_RESULT *result = &unexp->result; - - struct nd_cq_entry *parent_entry = entry->aux_entry; - - if (ep->cntr_send) { - if (result->Status != S_OK) { - InterlockedIncrement64(&ep->cntr_send->err); - } - InterlockedIncrement64(&ep->cntr_send->counter); - WakeByAddressAll((void*)&ep->cntr_send->counter); - } - - int notify = ofi_nd_util_completion_blackmagic( - ep->info->tx_attr->op_flags, ep->send_flags, parent_entry->flags) || - result->Status != S_OK; - - if (notify) { - PostQueuedCompletionStatus( - parent_entry->result.Status == S_OK ? ep->cq_send->iocp : ep->cq_send->err, - 0, 0, &parent_entry->base.ov); - InterlockedIncrement(&ep->cq_send->count); - WakeByAddressAll((void*)&ep->cq_send->count); - } - else { /* if notification is not requested - just free entry */ - ofi_nd_free_cq_entry(parent_entry); - } - - /* This CQ is no longer needed for us, let's release it. - * Set address of parent entry (that's used for initial - * send opertation) to NULL, it's just to not release it - * during free parent CQ entry below */ - entry->aux_entry = NULL; - ofi_nd_free_cq_entry(entry); -} - -void ofi_nd_unexp_2_cq(nd_cq_entry *entry, nd_unexpected_entry *unexp) -{ - assert(entry); - assert(unexp); - assert(unexp->ep); - assert(unexp->ep->fid.fid.fclass == FI_CLASS_EP || - unexp->ep->fid.fid.fclass == FI_CLASS_SRX_CTX); - - /* copy data to user's buffer */ - size_t i; - char *buf = unexp->buf->received_buf.data; - - size_t len = unexp->result.BytesTransferred - sizeof(unexp->buf->header); - for (i = 0; i < entry->iov_cnt && len; i++) { - size_t cp = min(entry->iov[i].iov_len, len); - memcpy(entry->iov[i].iov_base, buf, cp); - len -= cp; - buf += cp; - } - - entry->data = unexp->buf->header.data; /* copy send data */ - - int status = (unexp->result.Status == S_OK && - (unexp->result.BytesTransferred - sizeof(unexp->buf->header)) <= entry->len) ? - S_OK : (unexp->result.Status != S_OK ? - H2F(unexp->result.Status) : FI_ETRUNC); - - struct nd_ep *ep = unexp->ep; - - ofi_nd_release_unexp_entry(unexp); - - if (ep->cntr_recv) { - if (status != S_OK) { - InterlockedIncrement64(&ep->cntr_recv->err); - } - InterlockedIncrement64(&ep->cntr_recv->counter); - WakeByAddressAll((void*)&ep->cntr_recv->counter); - } - - int notify = ofi_nd_util_completion_blackmagic( - ep->info->rx_attr->op_flags, - ep->recv_flags, entry->flags); - - if (status == S_OK) { - if (notify && ep->cq_recv) { - PostQueuedCompletionStatus( - ep->cq_recv->iocp, 0, 0, &entry->base.ov); - InterlockedIncrement(&ep->cq_recv->count); - WakeByAddressAll((void*)&ep->cq_recv->count); - } - else { /* if notification is not requested - just free entry */ - ofi_nd_free_cq_entry(entry); - } - } - else { - if (ep->cq_recv) { - PostQueuedCompletionStatus( - ep->cq_recv->err, 0, 0, &entry->base.ov); - InterlockedIncrement(&ep->cq_recv->count); - WakeByAddressAll((void*)&ep->cq_recv->count); - } - else { /* TODO add warning here */ - ofi_nd_free_cq_entry(entry); - } - } -} - -void ofi_nd_unexp_2_read(nd_cq_entry *entry, void *unexpected) -{ - nd_unexpected_entry *unexp = (nd_unexpected_entry *)unexpected; - - assert(entry); - assert(unexp); - assert(unexp->ep); - assert(unexp->ep->fid.fid.fclass == FI_CLASS_EP || - unexp->ep->fid.fid.fclass == FI_CLASS_SRX_CTX); - - size_t location_cnt = unexp->buf->header.location_cnt; - struct nd_msg_location *locations = unexp->buf->received_buf.locations; - struct nd_ep *ep = unexp->ep; - HRESULT hr = 0; - size_t i; - - ofi_nd_release_unexp_entry(unexp); - - struct nd_cq_entry *rma_entries[ND_MSG_IOV_LIMIT]; - - struct iovec from_iovecs[ND_MSG_IOV_LIMIT]; - for (i = 0; i < location_cnt; i++) { - from_iovecs[i].iov_base = (void *)locations[i].addr; - from_iovecs[i].iov_len = locations[i].len; - } - - struct iovec new_iovecs[ND_MSG_INTERNAL_IOV_LIMIT]; - size_t new_iovecs_count = 0; - size_t from_split_map[ND_MSG_INTERNAL_IOV_LIMIT]; - size_t to_split_map[ND_MSG_INTERNAL_IOV_LIMIT]; - uint64_t remote_addr[ND_MSG_INTERNAL_IOV_LIMIT]; - - ofi_nd_repack_iovecs(from_iovecs, location_cnt, - entry->iov, entry->iov_cnt, - new_iovecs, &new_iovecs_count, - from_split_map, to_split_map, remote_addr); - assert(new_iovecs_count <= ND_MSG_INTERNAL_IOV_LIMIT); - - entry->wait_completion.comp_count = 0; - entry->wait_completion.total_count = new_iovecs_count; - - InitializeCriticalSection(&entry->wait_completion.comp_lock); - - for (i = 0; i < new_iovecs_count; i++) { - rma_entries[i] = ofi_nd_buf_alloc_nd_cq_entry(); - if (!rma_entries[i]) - goto fn_fail_alloc; - memset(rma_entries[i], 0, sizeof(*rma_entries[i])); - - rma_entries[i]->len = entry->len; - rma_entries[i]->data = entry->data; - rma_entries[i]->flags = entry->flags; - rma_entries[i]->domain = entry->domain; - rma_entries[i]->context = entry->context; - rma_entries[i]->iov[0].iov_base = new_iovecs[i].iov_base; - rma_entries[i]->iov[0].iov_len = new_iovecs[i].iov_len; - rma_entries[i]->iov_cnt = 1; - rma_entries[i]->seq = entry->seq; - /* Store native CQ entry not to be forgotten for free */ - rma_entries[i]->aux_entry = entry; - rma_entries[i]->rma_location.count = location_cnt; - rma_entries[i]->rma_location.locations = locations; - - void *tobuf = rma_entries[i]->iov[0].iov_base; - ULONG tobuf_len = (ULONG)rma_entries[i]->iov[0].iov_len; - - hr = ep->domain->adapter->lpVtbl->CreateMemoryRegion( - ep->domain->adapter, &IID_IND2MemoryRegion, - ep->domain->adapter_file, (void**)&rma_entries[i]->mr[0]); - if (FAILED(hr)) - goto fn_fail; - rma_entries[i]->mr_count = 1; - - hr = ofi_nd_util_register_mr( - rma_entries[i]->mr[0], tobuf, tobuf_len, - ND_MR_FLAG_ALLOW_LOCAL_WRITE | - ND_MR_FLAG_ALLOW_REMOTE_READ | - ND_MR_FLAG_ALLOW_REMOTE_WRITE); - if (FAILED(hr)) - goto fn_fail; - - ND2_SGE sge = { - .Buffer = tobuf, - .BufferLength = tobuf_len, - .MemoryRegionToken = rma_entries[i]->mr[0]->lpVtbl->GetLocalToken(rma_entries[i]->mr[0]) - }; - - rma_entries[i]->state = LARGE_MSG_RECV_REQ; - - hr = ep->qp->lpVtbl->Read(ep->qp, rma_entries[i], &sge, 1, - remote_addr[i], locations[from_split_map[i]].remote_mr_token, 0); - if (FAILED(hr)) - goto fn_fail; - } - return; - -fn_fail_alloc: - while (i--) - ofi_nd_free_cq_entry(rma_entries[i]); - /* TODO: generate cq_err if RMA read fails */ - ND_LOG_WARN(FI_LOG_EP_DATA, ofi_nd_strerror((DWORD)hr, NULL)); - return; -fn_fail: - while (i) - ofi_nd_free_cq_entry(rma_entries[i--]); - /* TODO: generate cq_err if RMA read fails */ - ND_LOG_WARN(FI_LOG_EP_DATA, ofi_nd_strerror((DWORD)hr, NULL)); -} - -void ofi_nd_read_2_cq(nd_cq_entry *entry, ND2_RESULT *result) -{ - struct nd_ep *ep = (struct nd_ep*)result->QueuePairContext; - - assert(ep); - assert(ep->fid.fid.fclass == FI_CLASS_EP); - assert(entry); - - if (entry->inline_buf) { - assert(result->BytesTransferred <= (ULONG)gl_data.inline_thr); - assert(!entry->mr); - memcpy(entry->iov[0].iov_base, entry->inline_buf->buffer, - result->BytesTransferred); - } - - if (ep->cntr_read) { - if (result->Status != S_OK) { - InterlockedIncrement64(&ep->cntr_read->err); - } - InterlockedIncrement64(&ep->cntr_read->counter); - WakeByAddressAll((void*)&ep->cntr_read->counter); - } - - int notify = ofi_nd_util_completion_blackmagic( - ep->info->rx_attr->op_flags, ep->recv_flags, entry->flags) || - result->Status != S_OK; - - if (notify) { - PostQueuedCompletionStatus( - entry->result.Status == S_OK ? ep->cq_recv->iocp : ep->cq_recv->err, - 0, 0, &entry->base.ov); - InterlockedIncrement(&ep->cq_recv->count); - WakeByAddressAll((void*)&ep->cq_recv->count); - } - else { /* if notification is not requested - just free entry */ - ofi_nd_free_cq_entry(entry); - } -} - -void ofi_nd_read_2_send_ack(nd_cq_entry *entry, void *res) -{ - ND2_RESULT *result = res; - struct nd_ep *ep = (struct nd_ep*)result->QueuePairContext; - size_t i; - - assert(ep); - assert(ep->fid.fid.fclass == FI_CLASS_EP); - assert(entry); - - HRESULT hr; - struct nd_cq_entry *ack_entry = ofi_nd_buf_alloc_nd_cq_entry(); - if (!ack_entry) { - ND_LOG_WARN(FI_LOG_EP_DATA, "Unable to allocate buffer for CQ entry"); - return; - } - memset(ack_entry, 0, sizeof(*ack_entry)); - - ack_entry->data = entry->data; - ack_entry->flags = entry->flags; - ack_entry->domain = entry->domain; - ack_entry->context = entry->context; - ack_entry->iov_cnt = entry->iov_cnt; - ack_entry->seq = entry->seq; - ack_entry->state = NORMAL_STATE; - - ack_entry->prefix = __ofi_nd_buf_alloc_nd_msgprefix( - &ep->domain->msgfooter); - if (!ack_entry->prefix) { - hr = ND_NO_MEMORY; - goto fn_fail; - } - - nd_flow_cntrl_flags flow_control_flags = { - .req_ack = 0, - .ack = 0, - .empty = 0 - }; - - struct nd_msgheader header_def = { - .data = entry->data, - .event = LARGE_MSG_ACK, - .flags = flow_control_flags, - .location_cnt = entry->rma_location.count - }; - ack_entry->prefix->header = header_def; - ack_entry->event = LARGE_MSG_ACK; - ack_entry->flow_cntrl_flags = flow_control_flags; - - ack_entry->notify_buf = __ofi_nd_buf_alloc_nd_notifybuf( - &ep->domain->notifybuf); - if (!ack_entry->notify_buf) { - hr = ND_NO_MEMORY; - goto fn_fail; - } - - /* Fill the header by values of received header from - * originator of large message transmission */ - for (i = 0; i < entry->rma_location.count; i++) { - struct nd_msg_location *parent_location = - &entry->rma_location.locations[i]; - struct nd_msg_location location_def = { - .addr = parent_location->addr, - .len = parent_location->len, - .remote_mr_token = parent_location->remote_mr_token, - }; - - ack_entry->notify_buf->location[i] = location_def; - } - - /* Generate CQ to notify that data successfuly read - * and can be obtained by user. Use intial CQ for that */ - ofi_nd_read_2_cq(entry->aux_entry, result); - - /* Set intial CQ entry to NULL just to avoid releasing of - * CQ entry for which CQ event haven't generated yet to - * requestor of large message RECV operation */ - entry->aux_entry = NULL; - ofi_nd_free_cq_entry(entry); - - /* Gracefully complete receiving large message - - * ACK to peer should be sent */ - ND2_SGE sge[2] = { - { - .Buffer = &ack_entry->prefix->header, - .BufferLength = (ULONG)sizeof(ack_entry->prefix->header), - .MemoryRegionToken = ack_entry->prefix->token - }, - { - .Buffer = &ack_entry->notify_buf->location, - .BufferLength = (ULONG)(sizeof(*ack_entry->notify_buf->location) * entry->rma_location.count), - .MemoryRegionToken = ack_entry->notify_buf->token - } - }; - - nd_sge *sge_entry = ofi_nd_buf_alloc_nd_sge(); - if (!sge_entry) { - ND_LOG_WARN(FI_LOG_EP_DATA, "SGE entry buffer can't be allocated"); - hr = ND_NO_MEMORY; - goto fn_fail; - } - memset(sge_entry, 0, sizeof(*sge_entry)); - - sge_entry->count = 2; - for (i = 0; i < sge_entry->count; i++) - sge_entry->entries[i] = sge[i]; - - nd_send_entry *send_entry = ofi_nd_buf_alloc_nd_send_entry(); - if (!send_entry) { - ND_LOG_WARN(FI_LOG_EP_DATA, "Send entry buffer can't be allocated"); - hr = ND_NO_MEMORY; - goto fn_fail; - } - memset(send_entry, 0, sizeof(*send_entry)); - - send_entry->cq_entry = ack_entry; - send_entry->sge = sge_entry; - send_entry->ep = ep; - - /* Push the transmission of ACK into - * the Send Queue for furhter handling */ - ack_entry->send_entry = send_entry; - ofi_nd_queue_push(&ep->send_queue, &send_entry->queue_item); - - /* Let's progress Send Queue for current EP if possible */ - ofi_nd_ep_progress(ep); - - return; - -fn_fail: - ofi_nd_free_cq_entry(ack_entry); - ND_LOG_WARN(FI_LOG_EP_DATA, ofi_nd_strerror((DWORD)hr, NULL)); -} - -void ofi_nd_send_ack(nd_cq_entry *entry, struct nd_ep *ep) -{ - assert(ep); - assert(ep->fid.fid.fclass == FI_CLASS_EP); - assert(entry); - - HRESULT hr; - struct nd_cq_entry *ack_entry = NULL; - struct nd_queue_item *qentry = NULL; - nd_send_entry *send_entry = NULL; - - EnterCriticalSection(&ep->send_op.send_lock); - if (ofi_nd_queue_peek(&ep->send_queue, &qentry)) { - send_entry = container_of(qentry, nd_send_entry, queue_item); - struct nd_msgheader *header = (struct nd_msgheader *) - send_entry->sge->entries[0].Buffer; - header->flags.ack = 1; - } - else { - ack_entry = ofi_nd_buf_alloc_nd_cq_entry(); - if (!ack_entry) { - ND_LOG_WARN(FI_LOG_EP_DATA, "Unable to allocate buffer " - "for CQ entry"); - LeaveCriticalSection(&ep->send_op.send_lock); - return; - } - memset(ack_entry, 0, sizeof(*ack_entry)); - - ack_entry->data = entry->data; - ack_entry->flags = entry->flags; - ack_entry->domain = entry->domain; - ack_entry->context = entry->context; - ack_entry->iov_cnt = entry->iov_cnt; - ack_entry->seq = entry->seq; - ack_entry->state = NORMAL_STATE; - - ack_entry->prefix = __ofi_nd_buf_alloc_nd_msgprefix( - &ep->domain->msgfooter); - if (!ack_entry->prefix) { - hr = ND_NO_MEMORY; - goto fn_fail; - } - - nd_flow_cntrl_flags flow_control_flags = { - .req_ack = 0, - .ack = 1, - .empty = 1 - }; - - struct nd_msgheader header_def = { - .data = entry->data, - .event = NORMAL_EVENT, - .flags = flow_control_flags, - .location_cnt = 0 - }; - ack_entry->prefix->header = header_def; - ack_entry->event = NORMAL_EVENT; - ack_entry->flow_cntrl_flags = flow_control_flags; - - ND2_SGE sge = { - .Buffer = &ack_entry->prefix->header, - .BufferLength = (ULONG)sizeof(ack_entry->prefix->header), - .MemoryRegionToken = ack_entry->prefix->token - }; - - hr = ep->qp->lpVtbl->Send(ep->qp, ack_entry, &sge, 1, 0); - if (FAILED(hr)) - ND_LOG_WARN(FI_LOG_CQ, "Send failed from Send Queue\n"); - } - LeaveCriticalSection(&ep->send_op.send_lock); - - /* Let's progress Send Queue for current EP if possible */ - ofi_nd_ep_progress(ep); - - return; -fn_fail: - ofi_nd_free_cq_entry(ack_entry); -} - -void ofi_nd_repack_iovecs(const struct iovec *from_iovecs, const size_t from_count, - const struct iovec *to_iovecs, const size_t to_count, - struct iovec new_iovecs[ND_MSG_INTERNAL_IOV_LIMIT], - size_t *new_count, - size_t from_split_map[ND_MSG_INTERNAL_IOV_LIMIT], - size_t to_split_map[ND_MSG_INTERNAL_IOV_LIMIT], - uint64_t remote_addr[ND_MSG_INTERNAL_IOV_LIMIT]) -{ - size_t from_iter = 0; - size_t to_iter = 0; - size_t new_iter = 0; - size_t to_offset = 0; - size_t from_offset = 0; - - for(;;) { - new_iovecs[new_iter].iov_base = (char *)to_iovecs[to_iter].iov_base + to_offset; - remote_addr[new_iter] = - (uint64_t)((char *)from_iovecs[from_iter].iov_base + from_offset); - from_split_map[new_iter] = from_iter; - to_split_map[new_iter] = to_iter; - ND_LOG_DEBUG(FI_LOG_EP_DATA, "\nFL = %lu, FO = %lu, FI = %lu " - "\nTL = %lu, TO = %lu, TI = %lu\n", - from_iovecs[from_iter].iov_len, from_offset, from_iter, - to_iovecs[to_iter].iov_len, to_offset, to_iter); - - if (from_iovecs[from_iter].iov_len - from_offset < to_iovecs[to_iter].iov_len - to_offset) { - new_iovecs[new_iter].iov_len = from_iovecs[from_iter].iov_len - from_offset; - to_offset += from_iovecs[from_iter].iov_len - from_offset; - from_iter++; - from_offset = 0; - } - else if (to_iovecs[to_iter].iov_len - to_offset < from_iovecs[from_iter].iov_len - from_offset) { - new_iovecs[new_iter].iov_len = to_iovecs[to_iter].iov_len - to_offset; - from_offset += to_iovecs[to_iter].iov_len - to_offset; - to_iter++; - to_offset = 0; - } - else { - new_iovecs[new_iter].iov_len = to_iovecs[to_iter].iov_len; - from_iter++; - to_iter++; - to_offset = 0; - from_offset = 0; - } - - new_iter++; - /* Check that whether some iovecs was emptied */ - if ((from_iter == from_count) && (!from_offset) || - ((to_iter == to_count) && (!to_offset))) - break; - } - *new_count = new_iter; -} - -#endif /* _WIN32 */ - diff --git a/prov/netdir/src/netdir_cq.h b/prov/netdir/src/netdir_cq.h deleted file mode 100644 index 98d6959850e..00000000000 --- a/prov/netdir/src/netdir_cq.h +++ /dev/null @@ -1,210 +0,0 @@ -/* -* Copyright (c) 2015-2016 Intel Corporation, Inc. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -*/ - -#ifndef _FI_NETDIR_CQ_H_ -#define _FI_NETDIR_CQ_H_ - -#include -#include -#include - -#include "ndspi.h" - -#include "rdma/fabric.h" -#include "ofi_mem.h" - -#include "netdir.h" -#include "netdir_buf.h" -#include "netdir_log.h" -#include "netdir_util.h" -#include "netdir_iface.h" -#include "netdir_queue.h" - -#include "rdma/fi_eq.h" - -#ifdef __cplusplus -extern "C" { -#endif /* __cplusplus */ - -static inline void ofi_nd_free_cq_entry(struct nd_cq_entry *entry) -{ - assert(entry); - - if (entry->prefix) - __ofi_nd_buf_free_nd_msgprefix(entry->prefix, - &entry->domain->msgfooter); - - if (entry->inline_buf) - __ofi_nd_buf_free_nd_inlinebuf(entry->inline_buf, - &entry->domain->inlinebuf); - - while (entry->mr_count) { - entry->mr_count--; - entry->mr[entry->mr_count]->lpVtbl->Release(entry->mr[entry->mr_count]); - } - - /* Means that waiting of completion are used. The completion - * critical section must be released */ - if (entry->wait_completion.total_count != 0) - DeleteCriticalSection(&entry->wait_completion.comp_lock); - - /* Release nested entry */ - if (entry->aux_entry) - ofi_nd_free_cq_entry(entry->aux_entry); - - ND_BUF_FREE(nd_cq_entry, entry); -} - -static inline ssize_t ofi_nd_cq_cancel(fid_t fid, void *context) -{ - assert(context); - - ssize_t ret = -ENOENT; - struct nd_cq_entry *entry = (struct nd_cq_entry *)ND_FI_CONTEXT(context); - CRITICAL_SECTION *prepost_lock; - struct nd_queue_queue *prepost; - struct nd_srx *srx; - struct nd_ep *ep; - - switch (fid->fclass) { - case FI_CLASS_SRX_CTX: - srx = container_of(fid, struct nd_srx, fid.fid); - prepost_lock = &srx->prepost_lock; - prepost = &srx->prepost; - break; - case FI_CLASS_EP: - ep = container_of(fid, struct nd_ep, fid.fid); - prepost_lock = &ep->prepost_lock; - prepost = &ep->prepost; - break; - default: - ND_LOG_WARN(FI_LOG_EP_DATA, "Invalid endpoint type \n"); - return -FI_EINVAL; - } - - if (entry) { - struct nd_queue_item *item = &entry->queue_item; - - EnterCriticalSection(prepost_lock); - - ofi_nd_queue_pop(prepost, &item); - ofi_nd_free_cq_entry(entry); - ND_FI_CONTEXT(context) = 0; - LeaveCriticalSection(prepost_lock); - - ret = 0; - } - - return ret; -} - -/* do NOT forget to add progress of Send Queue in all places - * where entry is enqueued for specific EP. Just to don't - * rely on ND's asynchronous invocation of providers callback - * about completion of an operation */ -static inline void ofi_nd_ep_progress(struct nd_ep *ep) -{ - HRESULT hr; - struct nd_queue_item *qentry = NULL; - nd_send_entry *send_entry = NULL; - - EnterCriticalSection(&ep->send_op.send_lock); - while (ofi_nd_queue_peek(&ep->send_queue, &qentry) && - !(ep->send_op.flags.is_send_blocked)) { - ep->send_op.used_counter++; - send_entry = container_of(qentry, nd_send_entry, queue_item); - ofi_nd_queue_pop(&ep->send_queue, &qentry); - - if (!(ep->send_op.used_counter % gl_data.prepost_cnt)) { - ep->send_op.flags.is_send_blocked = 1; - ep->send_op.used_counter = 0; - struct nd_msgheader *header = (struct nd_msgheader *) - send_entry->sge->entries[0].Buffer; - header->flags.req_ack = 1; - } - - /* If there is prepost entry (it means that this SEND event - * expects an answer). In this case, push CQ entry to prepost - * queue to receive event(answer) */ - if (send_entry->prepost_entry) { - ND_LOG_DEBUG(FI_LOG_EP_DATA, "Posted entry(state = %d) that " - "expects an answer from peer to which the send " - "event is belong\n", send_entry->prepost_entry->state); - ofi_nd_queue_push(&ep->internal_prepost, - &send_entry->prepost_entry->queue_item); - } - - hr = send_entry->ep->qp->lpVtbl->Send(send_entry->ep->qp, - send_entry->cq_entry, - send_entry->sge->entries, - send_entry->sge->count, 0); - if (FAILED(hr)) - ND_LOG_WARN(FI_LOG_CQ, "Send failed from Send Queue\n"); - } - LeaveCriticalSection(&ep->send_op.send_lock); -} - -#if 0 -static inline void ofi_nd_progress(void *arg) -{ - struct nd_domain *domain = arg; - struct dlist_entry *item; - struct nd_ep *ep; - - while (domain->do_progress) { - dlist_foreach(&domain->ep_list, item) { - ep = container_of(item, struct nd_ep, entry); - ofi_nd_ep_progress(ep); - } - } -} -#endif - -void ofi_nd_repack_iovecs(const struct iovec *from_iovecs, const size_t from_count, - const struct iovec *to_iovecs, const size_t to_count, - struct iovec new_iovecs[ND_MSG_INTERNAL_IOV_LIMIT], - size_t *new_count, - size_t from_split_map[ND_MSG_INTERNAL_IOV_LIMIT], - size_t to_split_map[ND_MSG_INTERNAL_IOV_LIMIT], - uint64_t remote_addr[ND_MSG_INTERNAL_IOV_LIMIT]); - - -void ofi_nd_dispatch_cq_event(ofi_nd_cq_event event, nd_cq_entry *entry, - void *misc); -void ofi_nd_send_ack(nd_cq_entry *entry, struct nd_ep *ep); - -#ifdef __cplusplus -} -#endif /* __cplusplus */ - -#endif /* _FI_NETDIR_CQ_H_ */ - diff --git a/prov/netdir/src/netdir_domain.c b/prov/netdir/src/netdir_domain.c deleted file mode 100644 index 590e1aad41b..00000000000 --- a/prov/netdir/src/netdir_domain.c +++ /dev/null @@ -1,545 +0,0 @@ -/* -* Copyright (c) 2015-2016 Intel Corporation, Inc. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -*/ - -#ifdef _WIN32 - -#include -#define WIN32_NO_STATUS - -#include "netdir.h" -#include "netdir_ov.h" -#include "netdir_log.h" -#include "netdir_util.h" -#include "netdir_iface.h" -#include "netdir_unexp.h" -#include "netdir_cq.h" - -#include "ofi.h" -#include "ofi_util.h" -#include "ofi_enosys.h" -#include "rdma/fabric.h" -#include "rdma/fi_domain.h" - -static int ofi_nd_domain_close(fid_t fid); -static int ofi_nd_domain_bind(struct fid *fid, struct fid *bfid, - uint64_t flags); - -struct nd_msgchunk { - IND2MemoryRegion *mr; - ND_BUF_CHUNK(nd_msgprefix) chunk; -}; - -struct nd_inlinechunk { - IND2MemoryRegion *mr; - char* base; - ND_BUF_CHUNK(nd_inlinebuf) chunk; -}; - -struct nd_notifychunk { - IND2MemoryRegion *mr; - ND_BUF_CHUNK(nd_notifybuf) chunk; -}; - -static ND_BUF_CHUNK(nd_msgprefix) -*ofi_nd_alloc_msgprefix_chunk(ND_BUF_FOOTER(nd_msgprefix) *footer, - size_t *count); -static void ofi_nd_free_msgprefix_chunk(ND_BUF_CHUNK(nd_msgprefix) *chunk); - -static ND_BUF_CHUNK(nd_inlinebuf) -*ofi_nd_alloc_inlinebuf_chunk(ND_BUF_FOOTER(nd_inlinebuf) *footer, size_t *count); -static void ofi_nd_free_inlinebuf_chunk(ND_BUF_CHUNK(nd_inlinebuf) *pchunk); - -static ND_BUF_CHUNK(nd_notifybuf) -*ofi_nd_alloc_notifybuf_chunk(ND_BUF_FOOTER(nd_notifybuf) *footer, size_t *count); -static void ofi_nd_free_notifybuf_chunk(ND_BUF_CHUNK(nd_notifybuf) *pchunk); - -static HRESULT ofi_nd_domain_notify(struct nd_domain *domain); -static void ofi_nd_domain_event(struct nd_event_base* base, DWORD bytes); -static void ofi_nd_domain_err(struct nd_event_base* base, DWORD bytes, DWORD err); - -static struct fi_ops_domain ofi_nd_domain_ops = { - .size = sizeof(ofi_nd_domain_ops), - .av_open = fi_no_av_open, - .cq_open = ofi_nd_cq_open, - .endpoint = ofi_nd_endpoint, - .scalable_ep = fi_no_scalable_ep, - .cntr_open = ofi_nd_cntr_open, - .poll_open = fi_no_poll_open, - .stx_ctx = fi_no_stx_context, - .srx_ctx = ofi_nd_srx_ctx -}; - -static struct fi_ops_mr ofi_nd_mr_ops = { - .size = sizeof(ofi_nd_mr_ops), - .reg = ofi_nd_mr_reg, - .regv = ofi_nd_mr_regv, - .regattr = ofi_nd_mr_regattr -}; - -static struct fi_ops ofi_nd_fi_ops = { - .size = sizeof(ofi_nd_fi_ops), - .close = ofi_nd_domain_close, - .bind = ofi_nd_domain_bind, - .control = fi_no_control, - .ops_open = fi_no_ops_open -}; - -static struct fid ofi_nd_fid = { - .fclass = FI_CLASS_DOMAIN, - .context = NULL, - .ops = &ofi_nd_fi_ops -}; - -static int ofi_nd_domain_close(fid_t fid) -{ - assert(fid->fclass == FI_CLASS_DOMAIN); - - struct nd_domain *domain = container_of(fid, struct nd_domain, fid.fid); - - DWORD ref = 0; -#if 0 - domain->do_progress = 0; - pthread_join(domain->progress_thread, NULL); -#endif - - if (domain->cq) { - domain->cq->lpVtbl->CancelOverlappedRequests(domain->cq); - while (!domain->cq_canceled || nd_async_progress) - SwitchToThread(); - domain->cq->lpVtbl->Release(domain->cq); - } - if (domain->info) - fi_freeinfo(domain->info); - if (domain->adapter_file && domain->adapter_file != INVALID_HANDLE_VALUE) - CloseHandle(domain->adapter_file); - if (domain->adapter) { - ref = domain->adapter->lpVtbl->Release(domain->adapter); - ND_LOG_DEBUG(FI_LOG_EP_CTRL, "domain->adapter ref count: %d\n", ref); - } - - __ofi_nd_buf_fini_nd_msgprefix(&domain->msgfooter); - __ofi_nd_buf_fini_nd_inlinebuf(&domain->inlinebuf); - - free(domain); - - return 0; -} - -int ofi_nd_domain_open(struct fid_fabric *fabric, struct fi_info *info, - struct fid_domain **pdomain, void *context) -{ - OFI_UNUSED(context); - - assert(fabric); - assert(fabric->fid.fclass == FI_CLASS_FABRIC); - assert(info); - assert(info->domain_attr); - assert(info->domain_attr->name); - - if (!info || !info->domain_attr || !info->domain_attr->name) - return -FI_EINVAL; - - HRESULT hr; - int res; - struct sockaddr* addr; - - struct nd_domain *domain = (struct nd_domain*)calloc(1, sizeof(*domain)); - if (!domain) - return -FI_ENOMEM; - - struct nd_domain def = { - .fid = { - .fid = ofi_nd_fid, - .ops = &ofi_nd_domain_ops, - .mr = &ofi_nd_mr_ops - }, - .info = fi_dupinfo(info), - .msgfooter = { - .alloc_chunk = ofi_nd_alloc_msgprefix_chunk, - .free_chunk = ofi_nd_free_msgprefix_chunk - }, - .inlinebuf = { - .alloc_chunk = ofi_nd_alloc_inlinebuf_chunk, - .free_chunk = ofi_nd_free_inlinebuf_chunk - }, - .notifybuf = { - .alloc_chunk = ofi_nd_alloc_notifybuf_chunk, - .free_chunk = ofi_nd_free_notifybuf_chunk - } - }; - - *domain = def; - - dlist_init(&domain->ep_list); -#if 0 - domain->do_progress = 1; - if (pthread_create(&domain->progress_thread, NULL, - ofi_nd_progress, domain)) { - ofi_nd_domain_close(&domain->fid.fid); - return -FI_ENOMEM;; - } -#endif - - res = ofi_nd_lookup_adapter(info->domain_attr->name, &domain->adapter, &addr); - if (res || !domain->adapter) { - ofi_nd_domain_close(&domain->fid.fid); - return res; - } - - memcpy(&domain->addr, addr, ofi_sizeofaddr(addr)); - - hr = domain->adapter->lpVtbl->CreateOverlappedFile(domain->adapter, - &domain->adapter_file); - - if (FAILED(hr)) - goto hr_failed; - - if (!BindIoCompletionCallback(domain->adapter_file, domain_io_cb, 0)) { - hr = HRESULT_FROM_WIN32(GetLastError()); - goto hr_failed; - } - - domain->ainfo.InfoVersion = ND_VERSION_2; - ULONG len = sizeof(domain->ainfo); - hr = domain->adapter->lpVtbl->Query(domain->adapter, &domain->ainfo, - &len); - if (FAILED(hr)) - goto hr_failed; - - hr = domain->adapter->lpVtbl->CreateCompletionQueue( - domain->adapter, &IID_IND2CompletionQueue, domain->adapter_file, - domain->ainfo.MaxCompletionQueueDepth, 0, 0, - (void**)&domain->cq); - if (FAILED(hr)) - goto hr_failed; - - *pdomain = &domain->fid; - - ND_LOG_DEBUG(FI_LOG_DOMAIN, "domain notification OV: %p\n", &domain->ov.ov); - hr = ofi_nd_domain_notify(domain); - if (FAILED(hr)) - goto hr_failed; - - return FI_SUCCESS; - -hr_failed: - ofi_nd_domain_close(&domain->fid.fid); - return H2F(hr); -} - -static int ofi_nd_domain_bind(struct fid *fid, struct fid *bfid, - uint64_t flags) -{ - assert(fid->fclass == FI_CLASS_DOMAIN); - - struct nd_domain *domain = container_of(fid, struct nd_domain, fid.fid); - - switch (bfid->fclass) { - case FI_CLASS_EQ: - domain->eq = container_of(bfid, struct nd_eq, fid.fid); - domain->eq_flags = flags; - break; - default: - ND_LOG_WARN(FI_LOG_DOMAIN, - "ofi_nd_domain_bind: incorrect bind object class: %d", - bfid->fclass); - return -FI_EINVAL; - } - - return FI_SUCCESS; -} - -static ND_BUF_CHUNK(nd_msgprefix) -*ofi_nd_alloc_msgprefix_chunk(ND_BUF_FOOTER(nd_msgprefix) *footer, size_t *count) -{ - struct nd_domain *dom = container_of(footer, struct nd_domain, msgfooter); - assert(dom->fid.fid.fclass == FI_CLASS_DOMAIN); - - HRESULT hr; - size_t i; - UINT32 token; - - struct nd_msgchunk *chunk = malloc(sizeof(*chunk)); - if (!chunk) - return 0; - memset(chunk, 0, sizeof(*chunk)); - assert(count); - *count = countof(chunk->chunk.item); - - assert(dom->adapter); - - hr = dom->adapter->lpVtbl->CreateMemoryRegion( - dom->adapter, &IID_IND2MemoryRegion, dom->adapter_file, (void**)&chunk->mr); - if (FAILED(hr)) - goto fn_fail; - - hr = ofi_nd_util_register_mr( - chunk->mr, &chunk->chunk, sizeof(chunk->chunk), - ND_MR_FLAG_ALLOW_LOCAL_WRITE); - if (FAILED(hr)) - goto fn_fail_mr; - - token = chunk->mr->lpVtbl->GetLocalToken(chunk->mr); - - for (i = 0; i < countof(chunk->chunk.item); i++) - chunk->chunk.item[i].data.token = token; - - return &chunk->chunk; - -fn_fail_mr: - chunk->mr->lpVtbl->Release(chunk->mr); -fn_fail: - free(chunk); - return 0; -} - -static void ofi_nd_free_msgprefix_chunk(ND_BUF_CHUNK(nd_msgprefix) *pchunk) -{ - assert(pchunk); - - struct nd_msgchunk *chunk = container_of(pchunk, struct nd_msgchunk, chunk); - if (chunk->mr) { - ofi_nd_util_unregister_mr(chunk->mr); - chunk->mr->lpVtbl->Release(chunk->mr); - } - free(chunk); -} - -static ND_BUF_CHUNK(nd_inlinebuf) -*ofi_nd_alloc_inlinebuf_chunk(ND_BUF_FOOTER(nd_inlinebuf) *footer, size_t *count) -{ - struct nd_domain *dom = container_of(footer, struct nd_domain, inlinebuf); - assert(dom->fid.fid.fclass == FI_CLASS_DOMAIN); - - HRESULT hr; - size_t i; - UINT32 token; - - struct nd_inlinechunk *chunk = malloc(sizeof(*chunk)); - if (!chunk) - return 0; - memset(chunk, 0, sizeof(*chunk)); - assert(count); - *count = countof(chunk->chunk.item); - - size_t len = gl_data.inline_thr * countof(chunk->chunk.item); - chunk->base = malloc(len); - if (!chunk->base) - goto fn_fail; - - assert(dom->adapter); - - hr = dom->adapter->lpVtbl->CreateMemoryRegion( - dom->adapter, &IID_IND2MemoryRegion, dom->adapter_file, (void**)&chunk->mr); - if (FAILED(hr)) - goto fn_fail_base; - - hr = ofi_nd_util_register_mr( - chunk->mr, chunk->base, len, - ND_MR_FLAG_ALLOW_LOCAL_WRITE | - ND_MR_FLAG_ALLOW_REMOTE_READ | - ND_MR_FLAG_ALLOW_REMOTE_WRITE); - if (FAILED(hr)) - goto fn_fail_mr; - - token = chunk->mr->lpVtbl->GetLocalToken(chunk->mr); - - for (i = 0; i < countof(chunk->chunk.item); i++) { - chunk->chunk.item[i].data.token = token; - chunk->chunk.item[i].data.buffer = chunk->base + (i * gl_data.inline_thr); - } - - return &chunk->chunk; - -fn_fail_mr: - chunk->mr->lpVtbl->Release(chunk->mr); -fn_fail_base: - free(chunk->base); -fn_fail: - free(chunk); - return 0; -} - -static void ofi_nd_free_inlinebuf_chunk(ND_BUF_CHUNK(nd_inlinebuf) *pchunk) -{ - assert(pchunk); - - struct nd_inlinechunk *chunk = container_of(pchunk, struct nd_inlinechunk, chunk); - if (chunk->mr) { - ofi_nd_util_unregister_mr(chunk->mr); - chunk->mr->lpVtbl->Release(chunk->mr); - } - if (chunk->base) - free(chunk->base); - free(chunk); -} - -static ND_BUF_CHUNK(nd_notifybuf) -*ofi_nd_alloc_notifybuf_chunk(ND_BUF_FOOTER(nd_notifybuf) *footer, size_t *count) -{ - struct nd_domain *dom = container_of(footer, struct nd_domain, notifybuf); - assert(dom->fid.fid.fclass == FI_CLASS_DOMAIN); - - HRESULT hr; - size_t i; - UINT32 token; - - struct nd_notifychunk *chunk = malloc(sizeof(*chunk)); - if (!chunk) - return 0; - memset(chunk, 0, sizeof(*chunk)); - assert(count); - *count = countof(chunk->chunk.item); - - assert(dom->adapter); - - hr = dom->adapter->lpVtbl->CreateMemoryRegion( - dom->adapter, &IID_IND2MemoryRegion, dom->adapter_file, (void**)&chunk->mr); - if (FAILED(hr)) - goto fn_fail; - - hr = ofi_nd_util_register_mr( - chunk->mr, &chunk->chunk, sizeof(chunk->chunk), - ND_MR_FLAG_ALLOW_LOCAL_WRITE); - if (FAILED(hr)) - goto fn_fail_mr; - - token = chunk->mr->lpVtbl->GetLocalToken(chunk->mr); - - for (i = 0; i < countof(chunk->chunk.item); i++) - chunk->chunk.item[i].data.token = token; - - return &chunk->chunk; - -fn_fail_mr: - chunk->mr->lpVtbl->Release(chunk->mr); - fn_fail: - free(chunk); - return 0; -} - -static void ofi_nd_free_notifybuf_chunk(ND_BUF_CHUNK(nd_notifybuf) *pchunk) -{ - assert(pchunk); - - struct nd_notifychunk *chunk = container_of(pchunk, struct nd_notifychunk, chunk); - if (chunk->mr) { - ofi_nd_util_unregister_mr(chunk->mr); - chunk->mr->lpVtbl->Release(chunk->mr); - } - free(chunk); -} - -static HRESULT ofi_nd_domain_notify(struct nd_domain *domain) -{ - assert(domain); - assert(domain->fid.fid.fclass == FI_CLASS_DOMAIN); - assert(domain->cq); - - nd_event_base ov = { - .event_cb = ofi_nd_domain_event, - .err_cb = ofi_nd_domain_err - }; - - domain->ov = ov; - return domain->cq->lpVtbl->Notify(domain->cq, ND_CQ_NOTIFY_ANY, &domain->ov.ov); -} - -static void ofi_nd_domain_event(struct nd_event_base* base, DWORD bytes) -{ - OFI_UNUSED(bytes); - - assert(base); - struct nd_domain *domain = container_of(base, struct nd_domain, ov); - - assert(domain->fid.fid.fclass == FI_CLASS_DOMAIN); - assert(domain->cq); - - ND2_RESULT result[256]; - DWORD count; - nd_unexpected_ctx *ctx; - do { - count = domain->cq->lpVtbl->GetResults(domain->cq, result, countof(result)); - size_t i; - for (i = 0; i < count; i++) { - ND_LOG_DEBUG(FI_LOG_EP_DATA, "Domain event is %d with status %s\n", - result[i].RequestType, - ofi_nd_error_str(result[i].Status)); - switch (result[i].RequestType) { - case Nd2RequestTypeReceive: - ctx = (nd_unexpected_ctx *)result[i].RequestContext; - if (!OFI_ND_IS_SERVICE_EVENT(ctx->entry->header.event)) - ofi_nd_unexp_event(&result[i]); - else - ofi_nd_unexp_service_event(&result[i]); - break; - case Nd2RequestTypeSend: - ofi_nd_send_event(&result[i]); - break; - case Nd2RequestTypeRead: - ofi_nd_read_event(&result[i]); - break; - case Nd2RequestTypeWrite: - ofi_nd_write_event(&result[i]); - break; - default: - /* shouldn't go here */ - NODEFAULT; - } - - /* Let's walk through sending queue to send data - * that are ready to be transmitted */ - struct nd_ep *ep = (struct nd_ep*)result[i].QueuePairContext; - ofi_nd_ep_progress(ep); - } - } while (count == countof(result)); - - ofi_nd_domain_notify(domain); -} - -static void ofi_nd_domain_err(struct nd_event_base* base, DWORD bytes, DWORD err) -{ - OFI_UNUSED(err); - if (err == STATUS_CANCELLED) { - struct nd_domain *domain = container_of(base, struct nd_domain, ov); - - assert(domain->fid.fid.fclass == FI_CLASS_DOMAIN); - assert(domain->cq); - domain->cq_canceled = 1; - return; - } - - ofi_nd_domain_event(base, bytes); -} - -#endif /* _WIN32 */ - diff --git a/prov/netdir/src/netdir_ep.c b/prov/netdir/src/netdir_ep.c deleted file mode 100644 index c956fbbcef8..00000000000 --- a/prov/netdir/src/netdir_ep.c +++ /dev/null @@ -1,880 +0,0 @@ -/* -* Copyright (c) 2015-2016 Intel Corporation, Inc. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -*/ - -#ifdef _WIN32 - -#include -#define WIN32_NO_STATUS - -#include "netdir.h" -#include "netdir_ov.h" -#include "netdir_log.h" -#include "netdir_util.h" -#include "netdir_iface.h" -#include "netdir_unexp.h" -#include "netdir_cq.h" - -#include "rdma/fabric.h" -#include "rdma/fi_endpoint.h" - -#include "ofi.h" -#include "ofi_util.h" - -static int ofi_nd_ep_control(struct fid *fid, int command, void *arg); -static int ofi_nd_ep_close(struct fid *fid); -static int ofi_nd_ep_bind(fid_t ep, fid_t cq, uint64_t flags); -static int ofi_nd_ep_getname(fid_t fid, void *addr, size_t *addrlen); -static int ofi_nd_ep_getpeer(struct fid_ep *fid, void *addr, size_t *addrlen); -static int ofi_nd_ep_connect(struct fid_ep *ep, const void *addr, - const void *param, size_t paramlen); -static int ofi_nd_ep_accept(struct fid_ep *ep, const void *param, - size_t paramlen); -static int ofi_nd_ep_shutdown(struct fid_ep *ep, uint64_t flags); - -static ND_BUF_CHUNK(nd_ep_msgprefix) - *ofi_nd_ep_alloc_chunk(ND_BUF_FOOTER(nd_ep_msgprefix) *footer, - size_t *count); -static void ofi_nd_ep_free_chunk(ND_BUF_CHUNK(nd_ep_msgprefix) *chunk); - -static void ofi_nd_ep_disconnected_free(struct nd_event_base* base); -static void ofi_nd_ep_disconnected(struct nd_event_base* base, DWORD bytes); -static void ofi_nd_ep_disconnected_err(struct nd_event_base* base, DWORD bytes, DWORD err); -static ssize_t ofi_nd_ep_cancel(fid_t fid, void *context); -int ofi_nd_ep_getopt(struct fid* ep, int level, int optname, - void* optval, size_t* optlen); - -static struct fi_ops ofi_nd_fi_ops = { - .size = sizeof(ofi_nd_fi_ops), - .close = ofi_nd_ep_close, - .bind = ofi_nd_ep_bind, - .control = ofi_nd_ep_control, - .ops_open = fi_no_ops_open, -}; - -static struct fi_ops_cm ofi_nd_cm_ops = { - .size = sizeof(ofi_nd_cm_ops), - .setname = fi_no_setname, - .getname = ofi_nd_ep_getname, - .getpeer = ofi_nd_ep_getpeer, - .connect = fi_no_connect, - .listen = fi_no_listen, - .accept = fi_no_accept, - .reject = fi_no_reject, - .shutdown = ofi_nd_ep_shutdown, - .join = fi_no_join, -}; - -extern struct fi_ops_msg ofi_nd_ep_msg; -extern struct fi_ops_rma ofi_nd_ep_rma; - -static struct fi_ops_ep ofi_nd_ep_ops = { - .size = sizeof(ofi_nd_ep_ops), - .cancel = ofi_nd_ep_cancel, - .getopt = ofi_nd_ep_getopt, - .setopt = fi_no_setopt, - .tx_ctx = fi_no_tx_ctx, - .rx_ctx = fi_no_rx_ctx, - .rx_size_left = fi_no_rx_size_left, - .tx_size_left = fi_no_tx_size_left, -}; - -typedef struct nd_ep_connect_data { - struct { - struct nd_msg_location send_res; - } flow_control; - - struct { - void *param; - size_t paramlen; - } user_data; - - struct { - void *data; - size_t size; - } total_conn_data; -} nd_ep_connect_data; - -typedef struct nd_ep_connect { - nd_event_base base; - struct nd_ep *ep; - struct nd_eq *eq; - IND2Connector *connector; - int active; -} nd_ep_connect; - -typedef struct nd_ep_completed { - nd_event_base base; - struct nd_ep *ep; - struct nd_eq *eq; - IND2Connector *connector; -} nd_ep_completed; - -OFI_ND_NB_BUF(nd_ep_connect); -OFI_ND_NB_BUF(nd_ep_completed); - -OFI_ND_NB_BUF_IMP(nd_ep_connect); -OFI_ND_NB_BUF_IMP(nd_ep_completed); - -OFI_ND_NB_BUF_TYPED(nd_connreq, struct nd_connreq); - -int ofi_nd_endpoint(struct fid_domain *pdomain, struct fi_info *info, - struct fid_ep **ep_fid, void *context) -{ - assert(info); - assert(pdomain); - assert(pdomain->fid.fclass == FI_CLASS_DOMAIN); - - ND_REGISTER_FINI(ND_BUF_FINIPTR(nd_ep_connect)); - ND_REGISTER_FINI(ND_BUF_FINIPTR(nd_ep_completed)); - ND_REGISTER_FINI(ND_BUF_FINIPTR(ofi_nd_util_ov)); - ND_REGISTER_FINI(ND_BUF_FINIPTR(nd_send_entry)); - ND_REGISTER_FINI(ND_BUF_FINIPTR(nd_sge)); - - HRESULT hr; - - struct nd_domain *domain = container_of(pdomain, struct nd_domain, fid); - struct nd_connreq *connreq = 0; - struct nd_ep *ep = (struct nd_ep*) calloc(1, sizeof(*ep)); - if (!ep) - return -FI_ENOMEM; - - struct nd_ep def = { - .fid = { - .fid = { - .fclass = FI_CLASS_EP, - .context = context, - .ops = &ofi_nd_fi_ops - }, - .ops = &ofi_nd_ep_ops, - .cm = &ofi_nd_cm_ops, - .msg = &ofi_nd_ep_msg, - .rma = &ofi_nd_ep_rma - }, - .info = fi_dupinfo(info), - .domain = domain, - .eq = domain->eq, - .disconnect_ov = { - .free = ofi_nd_ep_disconnected_free, - .event_cb = ofi_nd_ep_disconnected, - .err_cb = ofi_nd_ep_disconnected_err - } - }; - - *ep = def; - - /* Initialzie flow control counter */ - ep->send_op.used_counter = 0; - InitializeCriticalSection(&ep->send_op.send_lock); - - if (info->handle) { - assert(info->handle->fclass == FI_CLASS_CONNREQ); - if (info->handle->fclass != FI_CLASS_CONNREQ) { - hr = E_HANDLE; - goto fn_fail; - } - connreq = container_of(info->handle, struct nd_connreq, - handle); - } - - InitializeCriticalSection(&ep->prepost_lock); - - assert(domain->adapter); - - if (connreq) { - assert(connreq->connector); - ep->connector = connreq->connector; - ND_BUF_FREE(nd_connreq, connreq); - ep->fid.cm->accept = ofi_nd_ep_accept; - } - else { - hr = domain->adapter->lpVtbl->CreateConnector(domain->adapter, - &IID_IND2Connector, - domain->adapter_file, - (void**)&ep->connector); - if (FAILED(hr)) - goto fn_fail; - - hr = ep->connector->lpVtbl->Bind(ep->connector, - &domain->addr.addr, - (ULONG)ofi_sizeofaddr(&domain->addr.addr)); - if (FAILED(hr)) - goto fn_fail; - - ep->fid.cm->connect = ofi_nd_ep_connect; - } - - dlist_insert_tail(&ep->entry, &domain->ep_list); - - /* do NOT create real ND endpoint here: we could not know - how CQ will be attached here */ - - *ep_fid = &ep->fid; - hr = ofi_nd_unexp_init(ep); - - return 0; - -fn_fail: - ofi_nd_ep_close(&ep->fid.fid); - ND_LOG_WARN(FI_LOG_EP_CTRL, ofi_nd_strerror((DWORD)hr, NULL)); - return H2F(hr); -} - -static int ofi_nd_ep_control(struct fid *fid, int command, void *arg) -{ - OFI_UNUSED(arg); - - assert(fid->fclass == FI_CLASS_EP); - - HRESULT hr; - - if (command != FI_ENABLE) - return -FI_EINVAL; - - struct nd_ep *ep = container_of(fid, struct nd_ep, fid.fid); - - if (ep->qp) - return FI_SUCCESS; /* already enabled */ - - hr = ep->domain->adapter->lpVtbl->CreateQueuePair( - ep->domain->adapter, &IID_IND2QueuePair, - (IUnknown*)ep->domain->cq, - (IUnknown*)ep->domain->cq, - ep, - (ULONG) ep->info->rx_attr->size, - (ULONG) ep->info->tx_attr->size, - (ULONG) ep->info->rx_attr->iov_limit, - (ULONG) ep->info->tx_attr->iov_limit, - 0, (void**)&ep->qp); - if (FAILED(hr)) - return H2F(hr); - - /* Initialzie unexpected functionality */ - InitializeCriticalSection(&ep->unexpected.unexp_lock); - ofi_nd_unexp_run(ep); - - return FI_SUCCESS; -} - -static int ofi_nd_ep_close(struct fid *fid) -{ - ND_LOG_DEBUG(FI_LOG_EP_CTRL, "closing ep\n"); - - assert(fid->fclass == FI_CLASS_EP); - - struct nd_ep *ep = container_of(fid, struct nd_ep, fid.fid); - - ofi_nd_ep_shutdown(&ep->fid, 0); - - int res; - if (ep->connector) { - res = (int)ep->connector->lpVtbl->Release(ep->connector); - ND_LOG_DEBUG(FI_LOG_EP_CTRL, "ep->connector ref count: %d\n", res); - } - if (ep->qp) { - res = (int)ep->qp->lpVtbl->Release(ep->qp); - ND_LOG_DEBUG(FI_LOG_EP_CTRL, "ep->qp ref count: %d\n", res); - } - if (ep->info) - fi_freeinfo(ep->info); - - DeleteCriticalSection(&ep->prepost_lock); - /* Release Critical Section for unexpected events */ - DeleteCriticalSection(&ep->unexpected.unexp_lock); - - /* Retrieve this endpoint from domain EP list */ - dlist_remove(&ep->entry); - DeleteCriticalSection(&ep->send_op.send_lock); - free(ep); - ep = NULL; - - return 0; -} - -static void ofi_nd_ep_completed_free(nd_event_base *base) -{ - assert(base); - - nd_ep_completed *compl = container_of(base, nd_ep_completed, base); - assert(compl->connector); - compl->connector->lpVtbl->Release(compl->connector); - ND_BUF_FREE(nd_ep_completed, compl); -} - -static void ofi_nd_ep_completed(nd_event_base *base, DWORD bytes) -{ - OFI_UNUSED(bytes); - assert(base); - assert(base->free); - - nd_ep_completed *compl = container_of(base, nd_ep_completed, base); - assert(compl->connector); - OFI_UNUSED(compl); - - base->free(base); -} - -static void ofi_nd_ep_completed_err(nd_event_base *base, DWORD bytes, - DWORD error) -{ - OFI_UNUSED(bytes); - assert(base); - assert(base->free); - - nd_ep_completed *compl = container_of(base, nd_ep_completed, base); - - struct nd_eq_event *err = ND_BUF_ALLOC(nd_eq_event); - if (!err) { - ND_LOG_WARN(FI_LOG_EP_CTRL, - "failed to allocate error event\n"); - goto fn_completed; - } - - memset(err, 0, sizeof(*err)); - err->error.err = -H2F(error); - err->error.prov_errno = (int)error; - err->error.fid = &compl->ep->fid.fid; - ofi_nd_eq_push_err(compl->eq, err); - -fn_completed: - base->free(base); -} - -static void ofi_nd_ep_accepted_free(nd_event_base *base) -{ - assert(base); - - nd_ep_connect *connect = container_of(base, nd_ep_connect, base); - if (connect->connector) - connect->connector->lpVtbl->Release(connect->connector); - ND_BUF_FREE(nd_ep_connect, connect); -} - -static void ofi_nd_ep_accepted(nd_event_base *base, DWORD bytes) -{ - assert(base); - OFI_UNUSED(bytes); - - HRESULT hr; - ULONG len = 0; - nd_ep_connect *connect = container_of(base, nd_ep_connect, base); - struct nd_eq_event *err; - nd_ep_completed *compl = NULL; - - assert(connect->connector); - assert(connect->ep); - assert(connect->eq); - - struct nd_eq_event *ev = ND_BUF_ALLOC(nd_eq_event); - if (!ev) { - hr = ND_NO_MEMORY; - goto fn_fail_ev; - } - memset(ev, 0, sizeof(*ev)); - ev->eq_event = FI_CONNECTED; - - hr = connect->connector->lpVtbl->GetPrivateData( - connect->connector, NULL, &len); - - if (connect->active) { - hr = connect->connector->lpVtbl->GetPrivateData( - connect->connector, NULL, &len); - - if (FAILED(hr) && hr != ND_BUFFER_OVERFLOW) { - ND_LOG_WARN(FI_LOG_EP_CTRL, - "failed to get connection data\n"); - goto fn_fail_data; - } - - if (len) { - ev->data = malloc(len); - if (!ev->data) { - ND_LOG_WARN(FI_LOG_EP_CTRL, - "failed to allocate connection data\n"); - hr = ND_NO_MEMORY; - ev->len = 0; - goto fn_fail_data; - } - - hr = connect->connector->lpVtbl->GetPrivateData( - connect->connector, ev->data, &len); - if (FAILED(hr)) { - ND_LOG_WARN(FI_LOG_EP_CTRL, - "failed to copy connection data\n"); - free(ev->data); - ev->len = 0; - goto fn_fail_data; - } - } - ev->len = (size_t)len; - - compl = ND_BUF_ALLOC(nd_ep_completed); - if (!compl) { - ND_LOG_WARN(FI_LOG_EP_CTRL, - "failed to allocate connection-complete event\n"); - goto fn_fail_data; - } - memset(compl, 0 , sizeof(*compl)); - compl->base.event_cb = ofi_nd_ep_completed; - compl->base.err_cb = ofi_nd_ep_completed_err; - compl->base.free = ofi_nd_ep_completed_free; - compl->ep = connect->ep; - compl->eq = connect->eq; - compl->connector = connect->connector; - connect->connector->lpVtbl->AddRef(connect->connector); - - hr = connect->connector->lpVtbl->CompleteConnect(connect->connector, - &compl->base.ov); - if (FAILED(hr)) { - ND_LOG_WARN(FI_LOG_EP_CTRL, - "failed to complete connection\n"); - ND_BUF_FREE(nd_ep_completed, compl); - goto fn_fail_compl; - } - } - - ND_LOG_DEBUG(FI_LOG_EP_CTRL, "register disconnect notification: %p\n", - &connect->ep->disconnect_ov.ov); - hr = connect->connector->lpVtbl->NotifyDisconnect( - connect->connector, &connect->ep->disconnect_ov.ov); - if (FAILED(hr)) { - ND_LOG_WARN(FI_LOG_EP_CTRL, - "failed to notify disconnect\n"); - ND_BUF_FREE(nd_ep_completed, compl); - goto fn_fail_compl; - } - - struct fi_eq_cm_entry *cm = (struct fi_eq_cm_entry*)&ev->operation; - cm->fid = &connect->ep->fid.fid; - ofi_nd_eq_push(connect->eq, ev); - ofi_nd_ep_accepted_free(&connect->base); - connect->ep->connected = 1; - return; - -fn_fail_compl: - if (len) { - free(ev->data); - ev->len = 0; - } - connect->connector->lpVtbl->Release(connect->connector); - -fn_fail_data: - ofi_nd_buf_free_nd_eq_event(ev); - -fn_fail_ev: - err = ofi_nd_buf_alloc_nd_eq_event(); - if (!err) { - ND_LOG_WARN(FI_LOG_EP_CTRL, - "failed to allocate error event\n"); - ofi_nd_ep_accepted_free(&connect->base); - return; - } - memset(err, 0, sizeof(*err)); - err->error.err = -H2F(hr); - err->error.prov_errno = (int)hr; - err->error.fid = &connect->ep->fid.fid; - ofi_nd_eq_push_err(connect->eq, err); - ofi_nd_ep_accepted_free(&connect->base); -} - -static void ofi_nd_ep_rejected(nd_event_base *base, DWORD bytes, DWORD error) -{ - assert(base); - OFI_UNUSED(bytes); - - nd_ep_connect *connect = container_of(base, nd_ep_connect, base); - - assert(connect->connector); - assert(connect->ep); - assert(connect->eq); - - HRESULT hr = S_OK; - - struct nd_eq_event *err = ofi_nd_buf_alloc_nd_eq_event(); - if (!err) { - ND_LOG_WARN(FI_LOG_EP_CTRL, - "failed to allocate error event\n"); - ofi_nd_ep_accepted_free(&connect->base); - return; - } - memset(err, 0, sizeof(*err)); - err->error.err = -H2F(error); - err->error.prov_errno = (int)error; - err->error.fid = &connect->ep->fid.fid; - ofi_nd_eq_push_err(connect->eq, err); - - if (error == ND_CONNECTION_REFUSED) { - ULONG len = 0; - hr = connect->connector->lpVtbl->GetPrivateData( - connect->connector, NULL, &len); - - if (FAILED(hr) && hr != ND_BUFFER_OVERFLOW) { - ND_LOG_WARN(FI_LOG_EP_CTRL, - "failed to get connection data\n"); - goto fn_complete; - } - - if (len) { - err->error.err_data = malloc((size_t)len); - if (!err->error.err_data) { - ND_LOG_WARN(FI_LOG_EP_CTRL, - "failed to allocate connection data\n"); - hr = ND_NO_MEMORY; - goto fn_complete; - } - hr = connect->connector->lpVtbl->GetPrivateData( - connect->connector, err->error.err_data, &len); - - if (FAILED(hr)) { - ND_LOG_WARN(FI_LOG_EP_CTRL, - "failed to copy connection data\n"); - goto fn_complete; - } - err->error.err_data_size = (size_t)len; - } - } - -fn_complete: - ofi_nd_ep_accepted_free(&connect->base); -} - -static int ofi_nd_ep_connect(struct fid_ep *pep, const void *addr, - const void *param, size_t paramlen) -{ - assert(pep->fid.fclass == FI_CLASS_EP); - - struct nd_ep *ep = container_of(pep, struct nd_ep, fid); - - if (!addr) - return -FI_EINVAL; - - int res = fi_enable(&ep->fid); - if (res) - return res; - - assert(ep->connector); - assert(ep->qp); - - HRESULT hr; - - struct nd_ep_connect *wait = ofi_nd_buf_alloc_nd_ep_connect(); - if (!wait) - return -FI_ENOMEM; - - memset(wait, 0, sizeof(*wait)); - wait->ep = ep; - wait->eq = ep->eq; - wait->connector = ep->connector; - wait->base.event_cb = ofi_nd_ep_accepted; - wait->base.err_cb = ofi_nd_ep_rejected; - wait->base.free = ofi_nd_ep_accepted_free; - wait->active = 1; - ep->connector->lpVtbl->AddRef(ep->connector); - - hr = ep->connector->lpVtbl->Connect( - ep->connector, (IUnknown*)ep->qp, - (struct sockaddr*)addr, (ULONG)ofi_sizeofaddr((struct sockaddr*)addr), - ep->domain->ainfo.MaxInboundReadLimit, - ep->domain->ainfo.MaxOutboundReadLimit, - param, (ULONG)paramlen, &wait->base.ov); - return H2F(hr); -} - -static int ofi_nd_ep_accept(struct fid_ep *pep, const void *param, size_t paramlen) -{ - assert(pep->fid.fclass == FI_CLASS_EP); - - struct nd_ep *ep = container_of(pep, struct nd_ep, fid); - - int res = fi_enable(&ep->fid); - if (res) - return res; - - assert(ep->connector); - assert(ep->qp); - - HRESULT hr; - - struct nd_ep_connect *accept = ofi_nd_buf_alloc_nd_ep_connect(); - if (!accept) - return -FI_ENOMEM; - - memset(accept, 0, sizeof(*accept)); - accept->ep = ep; - accept->eq = ep->eq; - accept->connector = ep->connector; - accept->base.event_cb = ofi_nd_ep_accepted; - accept->base.err_cb = ofi_nd_ep_rejected; - accept->base.free = ofi_nd_ep_accepted_free; - accept->connector->lpVtbl->AddRef(accept->connector); - - ND_LOG_DEBUG(FI_LOG_EP_CTRL, "sending accept message\n"); - - hr = ep->connector->lpVtbl->Accept( - ep->connector, (IUnknown*)ep->qp, - ep->domain->ainfo.MaxInboundReadLimit, - ep->domain->ainfo.MaxOutboundReadLimit, - param, (ULONG)paramlen, &accept->base.ov); - if (FAILED(hr)) - ND_LOG_WARN(FI_LOG_EP_CTRL, "failed to send accept message: %x\n", - hr); - - return H2F(hr); -} - -static int ofi_nd_ep_getname(fid_t fid, void *addr, size_t *addrlen) -{ - assert(fid && fid->fclass == FI_CLASS_EP); - - if (fid->fclass != FI_CLASS_EP) - return -FI_EINVAL; - - HRESULT hr; - ULONG len = (ULONG)*addrlen; - struct nd_ep *ep = container_of(fid, struct nd_ep, fid.fid); - - if (!ep->connector) - return -FI_EOPBADSTATE; - - hr = ep->connector->lpVtbl->GetLocalAddress(ep->connector, - (struct sockaddr *)addr, - &len); - if (*addrlen < len) { - ND_LOG_INFO(FI_LOG_EP_CTRL, - "Provided buffer (size = %"PRIu64") is too small, required = %"PRIu64, - addrlen, len); - *addrlen = (size_t)len; - return -FI_ETOOSMALL; - } - *addrlen = (size_t)len; - - return H2F(hr); -} - -static int ofi_nd_ep_getpeer(struct fid_ep *pep, void *addr, size_t *addrlen) -{ - assert(pep); - assert(pep->fid.fclass == FI_CLASS_EP); - - if (pep->fid.fclass != FI_CLASS_EP) - return -FI_EINVAL; - - HRESULT hr; - ULONG len = (ULONG)*addrlen; - struct nd_ep *ep = container_of(pep, struct nd_ep, fid.fid); - - if (!ep->connector) - return -FI_EOPBADSTATE; - - hr = ep->connector->lpVtbl->GetPeerAddress(ep->connector, - (struct sockaddr*)addr, &len); - - *addrlen = (size_t)len; - - return H2F(hr); -} - -static int ofi_nd_ep_bind(fid_t pep, fid_t bfid, uint64_t flags) -{ - assert(pep->fclass == FI_CLASS_EP); - - if (pep->fclass != FI_CLASS_EP) - return -FI_EINVAL; - - struct nd_ep *ep = container_of(pep, struct nd_ep, fid.fid); - - switch (bfid->fclass) { - case FI_CLASS_EQ: - ep->eq = container_of(bfid, struct nd_eq, fid.fid); - return FI_SUCCESS; - case FI_CLASS_CQ: - if (flags & FI_TRANSMIT) { - ep->cq_send = container_of(bfid, struct nd_cq, fid.fid); - ep->send_flags = flags; - } - if (flags & FI_RECV) { - ep->cq_recv = container_of(bfid, struct nd_cq, fid.fid); - ep->recv_flags = flags; - } - if (flags & FI_REMOTE_READ || flags & FI_REMOTE_WRITE) - return -FI_EBADFLAGS; - return FI_SUCCESS; - case FI_CLASS_CNTR: - if (flags & FI_SEND) - ep->cntr_send = container_of(bfid, struct nd_cntr, fid.fid); - if (flags & FI_RECV) - ep->cntr_recv = container_of(bfid, struct nd_cntr, fid.fid); - if (flags & FI_READ) - ep->cntr_read = container_of(bfid, struct nd_cntr, fid.fid); - if (flags & FI_WRITE) - ep->cntr_write = container_of(bfid, struct nd_cntr, fid.fid); - if (flags & FI_REMOTE_READ || flags & FI_REMOTE_WRITE) - return -FI_EBADFLAGS; - return FI_SUCCESS; - case FI_CLASS_SRX_CTX: - ep->srx = container_of(bfid, struct nd_srx, fid.fid); - return FI_SUCCESS; - default: - ND_LOG_WARN(FI_LOG_EP_CTRL, - "ofi_nd_ep_bind: unknown bind class: %d", - (int)bfid->fclass); - return -FI_EINVAL; - } -} - -static int ofi_nd_ep_shutdown(struct fid_ep *pep, uint64_t flags) -{ - assert(pep); - assert(pep->fid.fclass == FI_CLASS_EP); - - OFI_UNUSED(flags); - - if (pep->fid.fclass != FI_CLASS_EP) - return -FI_EINVAL; - - struct nd_ep *ep = container_of(pep, struct nd_ep, fid.fid); - - if (!ep->qp) - return FI_SUCCESS; - - ofi_nd_unexp_fini(ep); - - HRESULT hr = S_OK; - ofi_nd_util_ov *ov = NULL; - if (ep->connected) { - ep->connected = 0; - - ov = ND_BUF_ALLOC(ofi_nd_util_ov); - if (!ov) - return -FI_ENOMEM; - - hr = ep->connector->lpVtbl->Disconnect(ep->connector, &ov->base.ov); - if (FAILED(hr)) - goto fn_fail; - - hr = ofi_nd_util_ov_wait(ep->connector, ov); - } - - return H2F(hr); - -fn_fail: - if (ov) - ND_BUF_FREE(ofi_nd_util_ov, ov); - return H2F(hr); -} - -static void ofi_nd_ep_disconnected_free(struct nd_event_base* base) -{ - OFI_UNUSED(base); -} - -static void ofi_nd_ep_disconnected(struct nd_event_base* base, DWORD bytes) -{ - OFI_UNUSED(bytes); - - struct nd_ep *ep = container_of(base, struct nd_ep, disconnect_ov); - assert(ep->fid.fid.fclass == FI_CLASS_EP); - - ep->connected = 0; - - struct nd_eq_event *ev = ND_BUF_ALLOC(nd_eq_event); - if (!ev) { - return; - } - memset(ev, 0, sizeof(*ev)); - struct fi_eq_cm_entry *cm = (struct fi_eq_cm_entry*)&ev->operation; - ev->eq_event = FI_SHUTDOWN; - cm->fid = &ep->fid.fid; - ofi_nd_eq_push(ep->eq, ev); - - //ofi_nd_ep_shutdown(&ep->fid, 0); -} - -static void ofi_nd_ep_disconnected_err(struct nd_event_base* base, DWORD bytes, - DWORD err) -{ - if (err == STATUS_CONNECTION_DISCONNECTED) { - ofi_nd_ep_disconnected(base, bytes); - } - else { - struct nd_ep *ep = container_of(base, struct nd_ep, disconnect_ov); - - struct nd_eq_event *ev = ND_BUF_ALLOC(nd_eq_event); - if (!ev) { - return; - } - memset(ev, 0, sizeof(*ev)); - ev->eq_event = FI_SHUTDOWN; - ev->error.err = H2F(err); - ev->error.prov_errno = err; - ev->error.fid = &ep->fid.fid; - ofi_nd_eq_push_err(ep->eq, ev); - } -} - -static ssize_t ofi_nd_ep_cancel(fid_t fid, void *context) -{ - assert(fid); - assert(fid->fclass == FI_CLASS_EP); - assert(context); - - if (!context) { - ND_LOG_WARN(FI_LOG_EP_DATA, "Context is NULL \n"); - return -FI_EINVAL; - } - - return ofi_nd_cq_cancel(fid, context); -} - -int ofi_nd_ep_getopt(struct fid* fid, int level, int optname, - void* optval, size_t* optlen) -{ - assert(fid->fclass == FI_CLASS_EP); - struct nd_ep* ep = container_of(fid, struct nd_ep, fid.fid); - - assert(optval); - assert(optlen); - - if (level != FI_OPT_ENDPOINT || optname != FI_OPT_CM_DATA_SIZE) - return -FI_ENOPROTOOPT; - - if (*optlen < sizeof(size_t)) { - *optlen = sizeof(size_t); - return -FI_ETOOSMALL; - } - - *((size_t*)optval) = ep->domain->ainfo.MaxCallerData; - *optlen = sizeof(size_t); - - return 0; -} - -#endif /* _WIN32 */ diff --git a/prov/netdir/src/netdir_ep_msg.c b/prov/netdir/src/netdir_ep_msg.c deleted file mode 100644 index 422474774d9..00000000000 --- a/prov/netdir/src/netdir_ep_msg.c +++ /dev/null @@ -1,666 +0,0 @@ -/* -* Copyright (c) 2015-2016 Intel Corporation, Inc. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -*/ - -#ifdef _WIN32 - -#include -#define WIN32_NO_STATUS - -#include "netdir.h" -#include "netdir_ov.h" -#include "netdir_cq.h" -#include "netdir_log.h" -#include "netdir_iface.h" -#include "netdir_unexp.h" - -#include "rdma/fabric.h" -#include "rdma/fi_endpoint.h" - -#include "ofi.h" -#include "ofi_util.h" - -static ssize_t ofi_nd_ep_recv(struct fid_ep *ep, void *buf, size_t len, - void *desc, fi_addr_t src_addr, void *context); -static ssize_t ofi_nd_ep_send(struct fid_ep *ep, const void *buf, size_t len, - void *desc, fi_addr_t src_addr, void *context); -static ssize_t ofi_nd_ep_recvmsg(struct fid_ep *ep_fid, const struct fi_msg *msg, - uint64_t flags); -static ssize_t ofi_nd_ep_recvv(struct fid_ep *ep_fid, const struct iovec *iov, - void **desc, size_t count, fi_addr_t src_addr, - void *context); -static ssize_t ofi_nd_ep_sendmsg(struct fid_ep *ep_fid, const struct fi_msg *msg, - uint64_t flags); -static ssize_t ofi_nd_ep_sendv(struct fid_ep *ep_fid, const struct iovec *iov, - void **desc, size_t count, fi_addr_t dest_addr, - void *context); -static ssize_t ofi_nd_ep_inject(struct fid_ep *ep_fid, const void *buf, size_t len, - fi_addr_t dest_addr); -static ssize_t ofi_nd_ep_senddata(struct fid_ep *ep, const void *buf, size_t len, - void *desc, uint64_t data, fi_addr_t dest_addr, - void *context); -ssize_t ofi_nd_ep_injectdata(struct fid_ep *ep, const void *buf, size_t len, - uint64_t data, fi_addr_t dest_addr); - -struct fi_ops_msg ofi_nd_ep_msg = { - .size = sizeof(ofi_nd_ep_msg), - .recv = ofi_nd_ep_recv, - .recvv = ofi_nd_ep_recvv, - .recvmsg = ofi_nd_ep_recvmsg, - .send = ofi_nd_ep_send, - .sendv = ofi_nd_ep_sendv, - .sendmsg = ofi_nd_ep_sendmsg, - .inject = ofi_nd_ep_inject, - .senddata = ofi_nd_ep_senddata, - .injectdata = ofi_nd_ep_injectdata -}; - -static int ofi_nd_ep_sendmsg_inline(struct nd_ep *ep, - struct nd_cq_entry *entry, - const struct fi_msg *msg, - size_t len) -{ - int res; - size_t i; - - nd_flow_cntrl_flags flow_control_flags = { - .req_ack = 0, - .ack = 0, - .empty = 0 - }; - - struct nd_msgheader header_def = { - .data = entry->data, - .event = NORMAL_EVENT, - .flags = flow_control_flags, - .location_cnt = 0 - }; - entry->prefix->header = header_def; - entry->event = NORMAL_EVENT; - entry->flow_cntrl_flags = flow_control_flags; - - - nd_sge *sge_entry = ofi_nd_buf_alloc_nd_sge(); - if (!sge_entry) { - ND_LOG_WARN(FI_LOG_EP_DATA, "SGE entry buffer can't be allocated"); - res = -FI_ENOMEM; - goto fn_fail_1; - } - memset(sge_entry, 0, sizeof(*sge_entry)); - - if (entry->flags & FI_INJECT) { - if (len) { - entry->inline_buf = __ofi_nd_buf_alloc_nd_inlinebuf(&ep->domain->inlinebuf); - if (!entry->inline_buf) { - res = -FI_ENOMEM; - goto fn_fail_2; - } - - char *buf = (char*)entry->inline_buf->buffer; - for (i = 0; i < msg->iov_count; i++) { - memcpy(buf, msg->msg_iov[i].iov_base, msg->msg_iov[i].iov_len); - buf += msg->msg_iov[i].iov_len; - } - } - - ND2_SGE sge[2] = { - { - .Buffer = &entry->prefix->header, - .BufferLength = (ULONG)sizeof(entry->prefix->header), - .MemoryRegionToken = entry->prefix->token - }, - { - .Buffer = len ? entry->inline_buf->buffer : 0, - .BufferLength = (ULONG)len, - .MemoryRegionToken = len ? entry->inline_buf->token : 0 - } - }; - - sge_entry->count = 2; - for (i = 0; i < sge_entry->count; i++) - sge_entry->entries[i] = sge[i]; - } - else { - ND2_SGE sge = { - .Buffer = &entry->prefix->header, - .BufferLength = (ULONG)sizeof(entry->prefix->header), - .MemoryRegionToken = entry->prefix->token - }; - sge_entry->entries[0] = sge; - - for (i = 0; i < msg->iov_count; i++) { - ND2_SGE sge_def = { - .Buffer = msg->msg_iov[i].iov_base, - .BufferLength = (ULONG)msg->msg_iov[i].iov_len, - .MemoryRegionToken = (UINT32)(uintptr_t)msg->desc[i] - }; - sge_entry->entries[i + 1] = sge_def; - } - - sge_entry->count = (ULONG)msg->iov_count + 1; - } - - nd_send_entry *send_entry = ofi_nd_buf_alloc_nd_send_entry(); - if (!send_entry) { - ND_LOG_WARN(FI_LOG_EP_DATA, "Send entry buffer can't be allocated"); - res = -FI_ENOMEM; - goto fn_fail_3; - } - memset(send_entry, 0, sizeof(*send_entry)); - - send_entry->cq_entry = entry; - send_entry->sge = sge_entry; - send_entry->ep = ep; - - /* Push the user's transmission request into - * the Send Queue for furhter handling */ - entry->send_entry = send_entry; - ofi_nd_queue_push(&ep->send_queue, &send_entry->queue_item); - - return FI_SUCCESS; -fn_fail_3: - if (entry->inline_buf) - __ofi_nd_buf_free_nd_inlinebuf(entry->inline_buf, - &ep->domain->inlinebuf); -fn_fail_2: - ofi_nd_buf_free_nd_sge(sge_entry); -fn_fail_1: - ND_LOG_WARN(FI_LOG_EP_DATA, "The error happened during handling Send"); - return res; -} - -static int ofi_nd_ep_prepare_sendmsg_large(struct nd_ep *ep, - struct nd_cq_entry *entry, - struct nd_cq_entry *wait_ack_entry, - const struct fi_msg *msg) -{ - size_t i; - HRESULT hr; - - for (i = 0; i < msg->iov_count; i++) { - uint64_t addr = (uint64_t)msg->msg_iov[i].iov_base; - size_t len = msg->msg_iov[i].iov_len; - - /* Register MR to share data via RMA, store MR descriptor - * in allocated CQ entry for receiving ACK */ - hr = ep->domain->adapter->lpVtbl->CreateMemoryRegion( - ep->domain->adapter, &IID_IND2MemoryRegion, - ep->domain->adapter_file, (void**)&wait_ack_entry->mr[i]); - if (FAILED(hr)) { - /* TODO: we leak previously created MRs */ - ND_LOG_WARN(FI_LOG_EP_DATA, ofi_nd_strerror((DWORD)hr, NULL)); - return H2F(hr); - } - wait_ack_entry->mr_count++; - - hr = ofi_nd_util_register_mr( - wait_ack_entry->mr[i], (void *)addr, len, - ND_MR_FLAG_ALLOW_LOCAL_WRITE | - ND_MR_FLAG_ALLOW_REMOTE_READ | - ND_MR_FLAG_ALLOW_REMOTE_WRITE); - if (FAILED(hr)) { - /* TODO: we leak previously created MRs */ - return H2F(hr); - } - - struct nd_msg_location location_def = { - .addr = addr, - .len = len, - .remote_mr_token = wait_ack_entry->mr[i]->lpVtbl->GetRemoteToken( - wait_ack_entry->mr[i]) - }; - - entry->notify_buf->location[i] = location_def; - } - - return FI_SUCCESS; -} - - - -static int ofi_nd_ep_sendmsg_large(struct nd_ep *ep, - struct nd_cq_entry *entry, - const struct fi_msg *msg) -{ - int res; - size_t i; - struct nd_cq_entry *wait_ack_entry; - - nd_flow_cntrl_flags flow_control_flags = { - .req_ack = 0, - .ack = 0, - .empty = 0 - }; - - struct nd_msgheader header_def = { - .data = entry->data, - .event = LARGE_MSG_REQ, - .flags = flow_control_flags, - .location_cnt = msg->iov_count - }; - entry->prefix->header = header_def; - entry->event = LARGE_MSG_REQ; - entry->flow_cntrl_flags = flow_control_flags; - - entry->notify_buf = __ofi_nd_buf_alloc_nd_notifybuf( - &ep->domain->notifybuf); - if (!entry->notify_buf) { - res = -FI_ENOMEM; - goto fn_fail_1; - } - - /* The CQ entry to wait ACK of read completion from peer */ - wait_ack_entry = ofi_nd_buf_alloc_nd_cq_entry(); - if (!wait_ack_entry) { - res = -FI_ENOMEM; - goto fn_fail_2; - } - memset(wait_ack_entry, 0, sizeof(*wait_ack_entry)); - wait_ack_entry->notify_buf = __ofi_nd_buf_alloc_nd_notifybuf( - &ep->domain->notifybuf); - if (!wait_ack_entry->notify_buf) { - res = -FI_ENOMEM; - goto fn_fail_3; - } - wait_ack_entry->buf = wait_ack_entry->notify_buf; - wait_ack_entry->len = sizeof(struct nd_notifybuf); - wait_ack_entry->data = msg->data; - wait_ack_entry->flags = FI_MSG | FI_RECV; - wait_ack_entry->domain = ep->domain; - wait_ack_entry->context = msg->context; - wait_ack_entry->seq = entry->seq; - wait_ack_entry->state = LARGE_MSG_WAIT_ACK; - wait_ack_entry->aux_entry = entry; - - res = ofi_nd_ep_prepare_sendmsg_large(ep, entry, wait_ack_entry, msg); - if (res) - goto fn_fail_4; - - entry->state = LARGE_MSG_WAIT_ACK; - ND2_SGE sge[2] = { - { - .Buffer = &entry->prefix->header, - .BufferLength = (ULONG)sizeof(entry->prefix->header), - .MemoryRegionToken = entry->prefix->token - }, - { - .Buffer = entry->notify_buf->location, - .BufferLength = (ULONG)(sizeof(*entry->notify_buf->location) * msg->iov_count), - .MemoryRegionToken = entry->notify_buf->token - } - }; - - nd_sge *sge_entry = ofi_nd_buf_alloc_nd_sge(); - if (!sge_entry) { - ND_LOG_WARN(FI_LOG_EP_DATA, "SGE entry buffer can't be allocated"); - res = -FI_ENOMEM; - goto fn_fail_4; - } - memset(sge_entry, 0, sizeof(*sge_entry)); - - sge_entry->count = 2; - for (i = 0; i < sge_entry->count; i++) - sge_entry->entries[i] = sge[i]; - - nd_send_entry *send_entry = ofi_nd_buf_alloc_nd_send_entry(); - if (!send_entry) { - ND_LOG_WARN(FI_LOG_EP_DATA, "Send entry buffer can't be allocated"); - res = -FI_ENOMEM; - goto fn_fail_5; - } - memset(send_entry, 0, sizeof(*send_entry)); - - send_entry->cq_entry = entry; - send_entry->sge = sge_entry; - send_entry->ep = ep; - send_entry->prepost_entry = wait_ack_entry; - - /* Push the user's transmission request into - * the Send Queue for furhter handling */ - entry->send_entry = send_entry; - ofi_nd_queue_push(&ep->send_queue, &send_entry->queue_item); - - return FI_SUCCESS; -fn_fail_5: - ofi_nd_buf_free_nd_sge(sge_entry); -fn_fail_4: - __ofi_nd_buf_free_nd_notifybuf(wait_ack_entry->notify_buf, - &ep->domain->notifybuf); -fn_fail_3: - ofi_nd_free_cq_entry(wait_ack_entry); -fn_fail_2: - __ofi_nd_buf_free_nd_notifybuf(entry->notify_buf, - &ep->domain->notifybuf); -fn_fail_1: - ND_LOG_WARN(FI_LOG_EP_DATA, "The error happened during handling Send"); - return res; -} - -static ssize_t -ofi_nd_ep_sendmsg(struct fid_ep *pep, const struct fi_msg *msg, uint64_t flags) -{ - assert(pep->fid.fclass == FI_CLASS_EP); - assert(msg); - - if (pep->fid.fclass != FI_CLASS_EP) - return -FI_EINVAL; - - size_t i; - size_t len = 0; - ssize_t res = FI_SUCCESS; - struct nd_ep *ep = container_of(pep, struct nd_ep, fid); - - if (!ep->qp) - return -FI_EOPBADSTATE; - - for (i = 0; i < msg->iov_count; i++) { - if (msg->msg_iov[i].iov_len && !msg->msg_iov[i].iov_base) - return -FI_EINVAL; - len += msg->msg_iov[i].iov_len; - } - - if ((msg->iov_count > min(ep->domain->ainfo.MaxReceiveSge, ND_MSG_IOV_LIMIT) - 1) || - (len > ep->domain->info->ep_attr->max_msg_size)) - return -FI_EINVAL; - - struct nd_cq_entry *entry = ofi_nd_buf_alloc_nd_cq_entry(); - if (!entry) - return -FI_ENOMEM; - memset(entry, 0, sizeof(*entry)); - - entry->buf = (msg->iov_count == 1) ? msg->msg_iov[0].iov_base : 0; - entry->len = len; - entry->data = msg->data; - entry->flags = flags | FI_MSG | FI_SEND; - entry->domain = ep->domain; - entry->context = msg->context; - entry->seq = InterlockedAdd64(&ep->domain->msg_cnt, 1); - - /* since send operation can't be canceled, set NULL into - * the 1st pointer of internal data of context */ - if (msg->context) - ND_FI_CONTEXT(msg->context) = 0; - - entry->prefix = __ofi_nd_buf_alloc_nd_msgprefix( - &ep->domain->msgfooter); - if (!entry->prefix) { - res = -FI_ENOMEM; - goto fn_fail_1; - } - - if (entry->len <= gl_data.inline_thr) - res = ofi_nd_ep_sendmsg_inline(ep, entry, msg, len); - else - res = ofi_nd_ep_sendmsg_large(ep, entry, msg); - if (res) - goto fn_fail_2; - /* Let's progress Send Queue for current EP if possible */ - ofi_nd_ep_progress(ep); - - return FI_SUCCESS; -fn_fail_2: - __ofi_nd_buf_free_nd_msgprefix(entry->prefix, &ep->domain->msgfooter); -fn_fail_1: - ofi_nd_buf_free_nd_cq_entry(entry); - return res; -} - -static ssize_t ofi_nd_ep_inject(struct fid_ep *pep, const void *buf, size_t len, - fi_addr_t dest_addr) -{ - return ofi_nd_ep_injectdata(pep, buf, len, 0, dest_addr); -} - -ssize_t -ofi_nd_ep_injectdata(struct fid_ep *pep, const void *buf, size_t len, - uint64_t data, fi_addr_t dest_addr) -{ - struct iovec iov = { - .iov_base = (void*)buf, - .iov_len = len - }; - - struct fi_msg msg = { - .msg_iov = &iov, - .desc = 0, - .iov_count = 1, - .addr = dest_addr, - .context = 0, - .data = data - }; - - return ofi_nd_ep_sendmsg(pep, &msg, FI_INJECT); -} - -static ssize_t ofi_nd_ep_senddata(struct fid_ep *pep, const void *buf, size_t len, void *desc, - uint64_t data, fi_addr_t dest_addr, void *context) -{ - struct iovec iov = { - .iov_base = (void*)buf, - .iov_len = len - }; - - struct fi_msg msg = { - .msg_iov = &iov, - .desc = &desc, - .iov_count = 1, - .addr = dest_addr, - .context = context, - .data = data - }; - - assert(pep->fid.fclass == FI_CLASS_EP); - - if (pep->fid.fclass != FI_CLASS_EP) - return -FI_EINVAL; - - struct nd_ep *ep = container_of(pep, struct nd_ep, fid); - - return ofi_nd_ep_sendmsg(pep, &msg, ep->info->tx_attr->op_flags); -} - -static ssize_t ofi_nd_ep_send(struct fid_ep *pep, const void *buf, size_t len, - void *desc, fi_addr_t dest_addr, void *context) -{ - return ofi_nd_ep_senddata(pep, buf, len, desc, 0, dest_addr, context); -} - -static ssize_t ofi_nd_ep_sendv(struct fid_ep *pep, const struct iovec *iov, - void **desc, size_t count, fi_addr_t dest_addr, - void *context) -{ - struct fi_msg msg = { - .msg_iov = iov, - .desc = desc, - .iov_count = count, - .addr = dest_addr, - .context = context, - .data = 0 - }; - - assert(pep->fid.fclass == FI_CLASS_EP); - - if (pep->fid.fclass != FI_CLASS_EP) - return -FI_EINVAL; - - struct nd_ep *ep = container_of(pep, struct nd_ep, fid); - - return ofi_nd_ep_sendmsg(pep, &msg, ep->info->tx_attr->op_flags); -} - -static ssize_t ofi_nd_ep_recvmsg(struct fid_ep *pep, const struct fi_msg *msg, - uint64_t flags) -{ - assert(pep->fid.fclass == FI_CLASS_EP); - assert(msg); - - if (pep->fid.fclass != FI_CLASS_EP) - return -FI_EINVAL; - - size_t i; - size_t len = 0; - - struct nd_ep *ep = container_of(pep, struct nd_ep, fid); - - if (!ep->qp) - return -FI_EOPBADSTATE; - - for (i = 0; i < msg->iov_count; i++) { - if (msg->msg_iov[i].iov_len && !msg->msg_iov[i].iov_base) - return -FI_EINVAL; - len += msg->msg_iov[i].iov_len; - } - - if ((msg->iov_count > min(ep->domain->ainfo.MaxReceiveSge, ND_MSG_IOV_LIMIT) - 1) || - (len > ep->domain->info->ep_attr->max_msg_size)) - return -FI_EINVAL; - - struct nd_cq_entry *entry = ofi_nd_buf_alloc_nd_cq_entry(); - if (!entry) - return -FI_ENOMEM; - memset(entry, 0, sizeof(*entry)); - - entry->buf = (msg->iov_count == 1) ? msg->msg_iov[0].iov_base : NULL; - entry->len = len; - entry->data = msg->data; - entry->flags = flags | FI_MSG | FI_RECV; - entry->domain = ep->domain; - entry->context = msg->context; - entry->iov_cnt = msg->iov_count; - entry->seq = InterlockedAdd64(&ep->domain->msg_cnt, 1); - - for (i = 0; i < msg->iov_count; i++) - entry->iov[i] = msg->msg_iov[i]; - - /* store allocated entry in 1st pointer of internal data of context */ - if (msg->context) - ND_FI_CONTEXT(msg->context) = entry; - - ofi_nd_queue_push(&ep->prepost, &entry->queue_item); - - ofi_nd_unexp_match(ep); - - return FI_SUCCESS; -} - -static ssize_t ofi_nd_ep_recvv(struct fid_ep *pep, const struct iovec *iov, - void **desc, - size_t count, fi_addr_t src_addr, void *context) -{ - struct fi_msg msg = { - .msg_iov = iov, - .desc = desc, - .iov_count = count, - .addr = src_addr, - .context = context, - .data = 0 - }; - - assert(pep->fid.fclass == FI_CLASS_EP); - - if (pep->fid.fclass != FI_CLASS_EP) - return -FI_EINVAL; - - struct nd_ep *ep = container_of(pep, struct nd_ep, fid); - - return ofi_nd_ep_recvmsg(pep, &msg, ep->info->rx_attr->op_flags); -} - -static ssize_t ofi_nd_ep_recv(struct fid_ep *pep, void *buf, size_t len, - void *desc, fi_addr_t src_addr, void *context) -{ - struct iovec iov = { - .iov_base = buf, - .iov_len = len - }; - - return ofi_nd_ep_recvv(pep, &iov, &desc, 1, src_addr, context); -} - -void ofi_nd_send_event(ND2_RESULT *result) -{ - assert(result); - assert(result->RequestType == Nd2RequestTypeSend); - - nd_cq_entry *entry = (nd_cq_entry*)result->RequestContext; - assert(entry); - - struct nd_ep *ep = (struct nd_ep*)result->QueuePairContext; - assert(ep); - assert(ep->fid.fid.fclass == FI_CLASS_EP); - - ND_LOG_EVENT_INFO(entry); - - /* Send entry is no more needed */ - if (entry->send_entry) - ofi_nd_free_send_entry(entry->send_entry); - - if (entry->state == LARGE_MSG_WAIT_ACK) { - /* If send operation isn't able to transmit large message, don't - * notify user as long as we didn't received ACK of completion - * read of shared buffer. This CQ entry will be released - * when auxillary CQ for ACK will be received */ - return; - } - else if (entry->event == LARGE_MSG_ACK || - (entry->flow_cntrl_flags.ack && entry->flow_cntrl_flags.empty)) { - /* Silently release this CQ entry. From now we can consider - * that read of large message is completed successfuly */ - ofi_nd_free_cq_entry(entry); - return; - } - - if (ep->cntr_send) { - if (result->Status != S_OK) { - InterlockedIncrement64(&ep->cntr_send->err); - } - InterlockedIncrement64(&ep->cntr_send->counter); - WakeByAddressAll((void*)&ep->cntr_send->counter); - } - - int notify = ofi_nd_util_completion_blackmagic( - ep->info->tx_attr->op_flags, ep->send_flags, entry->flags) || - result->Status != S_OK; - - if (notify) { - PostQueuedCompletionStatus( - entry->result.Status == S_OK ? ep->cq_send->iocp : ep->cq_send->err, - 0, 0, &entry->base.ov); - InterlockedIncrement(&ep->cq_send->count); - WakeByAddressAll((void*)&ep->cq_send->count); - } - else { /* if notification is not requested - just free entry */ - ofi_nd_free_cq_entry(entry); - } -} - -#endif /* _WIN32 */ - diff --git a/prov/netdir/src/netdir_ep_rma.c b/prov/netdir/src/netdir_ep_rma.c deleted file mode 100644 index 2132720249d..00000000000 --- a/prov/netdir/src/netdir_ep_rma.c +++ /dev/null @@ -1,710 +0,0 @@ -/* -* Copyright (c) 2015-2016 Intel Corporation, Inc. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -*/ - -#ifdef _WIN32 - -#include "netdir.h" -#include "netdir_ov.h" -#include "netdir_cq.h" -#include "netdir_log.h" -#include "netdir_iface.h" - -#include "rdma/fabric.h" -#include "rdma/fi_endpoint.h" - -#include "ofi.h" -#include "ofi_util.h" - -static ssize_t -ofi_nd_ep_read(struct fid_ep *ep, void *buf, size_t len, void *desc, - fi_addr_t src_addr, uint64_t addr, uint64_t key, void *context); -static ssize_t -ofi_nd_ep_readv(struct fid_ep *ep, const struct iovec *iov, void **desc, - size_t count, fi_addr_t src_addr, uint64_t addr, uint64_t key, - void *context); -static ssize_t -ofi_nd_ep_readmsg(struct fid_ep *ep, const struct fi_msg_rma *msg, - uint64_t flags); -static ssize_t -ofi_nd_ep_write(struct fid_ep *ep, const void *buf, size_t len, void *desc, - fi_addr_t dest_addr, uint64_t addr, uint64_t key, void *context); -static ssize_t -ofi_nd_ep_writev(struct fid_ep *ep, const struct iovec *iov, void **desc, - size_t count, fi_addr_t dest_addr, uint64_t addr, uint64_t key, - void *context); -static ssize_t -ofi_nd_ep_writemsg(struct fid_ep *ep, const struct fi_msg_rma *msg, - uint64_t flags); -static ssize_t -ofi_nd_ep_inject(struct fid_ep *ep, const void *buf, size_t len, - fi_addr_t dest_addr, uint64_t addr, uint64_t key); -static ssize_t -ofi_nd_ep_writedata(struct fid_ep *ep, const void *buf, size_t len, void *desc, - uint64_t data, fi_addr_t dest_addr, uint64_t addr, uint64_t key, - void *context); -static ssize_t -ofi_nd_ep_writeinjectdata(struct fid_ep *ep, const void *buf, size_t len, - uint64_t data, fi_addr_t dest_addr, uint64_t addr, - uint64_t key); -ssize_t ofi_nd_ep_injectdata(struct fid_ep *ep, const void *buf, size_t len, - uint64_t data, fi_addr_t dest_addr); -static void -ofi_nd_split_msg_iov_2_rma_iov(const struct fi_rma_iov *rma_iovecs, const size_t rma_count, - const struct iovec *msg_iovecs, const size_t msg_count, - struct fi_rma_iov res_iovecs[ND_MSG_INTERNAL_IOV_LIMIT], size_t *res_count, - size_t from_split_map[ND_MSG_INTERNAL_IOV_LIMIT], - size_t to_split_map[ND_MSG_INTERNAL_IOV_LIMIT], - uint64_t remote_addr[ND_MSG_INTERNAL_IOV_LIMIT]); - -struct fi_ops_rma ofi_nd_ep_rma = { - .size = sizeof(ofi_nd_ep_rma), - .read = ofi_nd_ep_read, - .readv = ofi_nd_ep_readv, - .readmsg = ofi_nd_ep_readmsg, - .write = ofi_nd_ep_write, - .writev = ofi_nd_ep_writev, - .writemsg = ofi_nd_ep_writemsg, - .inject = ofi_nd_ep_inject, - .writedata = ofi_nd_ep_writedata, - .injectdata = ofi_nd_ep_writeinjectdata -}; - - -static ssize_t -ofi_nd_ep_read(struct fid_ep *ep, void *buf, size_t len, void *desc, - fi_addr_t src_addr, uint64_t addr, uint64_t key, void *context) -{ - struct iovec iov = { - .iov_base = buf, - .iov_len = len - }; - return ofi_nd_ep_readv(ep, &iov, &desc, 1, src_addr, addr, key, context); -} - -static ssize_t -ofi_nd_ep_readv(struct fid_ep *pep, const struct iovec *iov, void **desc, - size_t count, fi_addr_t src_addr, uint64_t addr, uint64_t key, - void *context) -{ - struct fi_rma_iov rma_iov = { - .addr = addr, - .len = iov[0].iov_len, - .key = key - }; - - struct fi_msg_rma msg = { - .msg_iov = iov, - .desc = desc, - .iov_count = count, - .addr = src_addr, - .rma_iov = &rma_iov, - .rma_iov_count = 1, - .context = context, - .data = 0 - }; - - assert(pep->fid.fclass == FI_CLASS_EP); - - if (pep->fid.fclass != FI_CLASS_EP) - return -FI_EINVAL; - - struct nd_ep *ep = container_of(pep, struct nd_ep, fid); - - return ofi_nd_ep_readmsg(pep, &msg, ep->info->rx_attr->op_flags); -} - -static ssize_t -ofi_nd_ep_readmsg(struct fid_ep *pep, const struct fi_msg_rma *msg, - uint64_t flags) -{ - assert(pep->fid.fclass == FI_CLASS_EP); - assert(msg); - - if (pep->fid.fclass != FI_CLASS_EP) - return -FI_EINVAL; - - size_t msg_len = 0, rma_len = 0, i; - HRESULT hr = 0; - - struct nd_ep *ep = container_of(pep, struct nd_ep, fid); - - if (!ep->qp) - return -FI_EOPBADSTATE; - - for (i = 0; i < msg->iov_count; i++) { - if (msg->msg_iov[i].iov_len && !msg->msg_iov[i].iov_base) - return -FI_EINVAL; - msg_len += msg->msg_iov[i].iov_len; - } - - for (i = 0; i < msg->rma_iov_count; i++) { - if (msg->rma_iov[i].len && !msg->rma_iov[i].addr) - return -FI_EINVAL; - rma_len += msg->rma_iov[i].len; - } - - /* Check the following: */ - if ((msg_len != rma_len) || /* - msg and rma len are correlated */ - /* - iov counts are less or equal than supported */ - (msg->iov_count > ND_MSG_IOV_LIMIT || - msg->rma_iov_count > ND_MSG_IOV_LIMIT) || - /* - transmitted length is less or equal than max possible */ - (msg_len > ep->domain->info->ep_attr->max_msg_size)) - return -FI_EINVAL; - - struct nd_cq_entry *main_entry = ofi_nd_buf_alloc_nd_cq_entry(); - if (!main_entry) - return -FI_ENOMEM; - memset(main_entry, 0, sizeof(*main_entry)); - main_entry->data = msg->data; - main_entry->flags = flags; - main_entry->domain = ep->domain; - main_entry->context = msg->context; - main_entry->seq = InterlockedAdd64(&ep->domain->msg_cnt, 1); - - /* since write operation can't be canceled, set NULL into - * the 1st pointer of internal data of context */ - if (msg->context) - ND_FI_CONTEXT(msg->context) = 0; - - struct fi_rma_iov rma_iovecs[ND_MSG_INTERNAL_IOV_LIMIT]; - size_t rma_count = 0; - size_t from_split_map[ND_MSG_INTERNAL_IOV_LIMIT]; - size_t to_split_map[ND_MSG_INTERNAL_IOV_LIMIT]; - uint64_t remote_addr[ND_MSG_INTERNAL_IOV_LIMIT]; - - ofi_nd_split_msg_iov_2_rma_iov(msg->rma_iov, msg->rma_iov_count, - msg->msg_iov, msg->iov_count, - rma_iovecs, &rma_count, - from_split_map, to_split_map, remote_addr); - - assert(rma_count <= ND_MSG_INTERNAL_IOV_LIMIT); - - main_entry->wait_completion.comp_count = 0; - main_entry->wait_completion.total_count = rma_count; - - InitializeCriticalSection(&main_entry->wait_completion.comp_lock); - - struct nd_cq_entry *entries[ND_MSG_IOV_LIMIT]; - - for (i = 0; i < rma_count; i++) { - entries[i] = ofi_nd_buf_alloc_nd_cq_entry(); - if (!entries[i]) - goto fn_fail; - memset(entries[i], 0, sizeof(*entries[i])); - - entries[i]->data = msg->data; - entries[i]->flags = flags; - entries[i]->domain = ep->domain; - entries[i]->context = msg->context; - entries[i]->seq = main_entry->seq; - entries[i]->aux_entry = main_entry; - - hr = ep->domain->adapter->lpVtbl->CreateMemoryRegion( - ep->domain->adapter, &IID_IND2MemoryRegion, - ep->domain->adapter_file, (void**)&entries[i]->mr[0]); - if (FAILED(hr)) - goto fn_fail; - entries[i]->mr_count = 1; - - hr = ofi_nd_util_register_mr( - entries[i]->mr[0], - (const void *)remote_addr[i], - rma_iovecs[i].len, - ND_MR_FLAG_ALLOW_LOCAL_WRITE | - ND_MR_FLAG_ALLOW_REMOTE_READ | - ND_MR_FLAG_ALLOW_REMOTE_WRITE); - if (FAILED(hr)) - goto fn_fail; - - ND2_SGE sge = { - .Buffer = (void *)remote_addr[i], - .BufferLength = (ULONG)rma_iovecs[i].len, - .MemoryRegionToken = (UINT32)(uintptr_t)msg->desc[to_split_map[i]] - }; - - hr = ep->qp->lpVtbl->Read(ep->qp, entries[i], &sge, 1, - (UINT64)rma_iovecs[i].addr, (UINT32)rma_iovecs[i].key, 0); - if (FAILED(hr)) - goto fn_fail; - } - - return FI_SUCCESS; - -fn_fail: - while (i-- > 0) - ofi_nd_free_cq_entry(entries[i]); - ND_LOG_WARN(FI_LOG_EP_DATA, ofi_nd_strerror((DWORD)hr, NULL)); - return H2F(hr); -} - -static ssize_t -ofi_nd_ep_write(struct fid_ep *ep, const void *buf, size_t len, void *desc, - fi_addr_t dest_addr, uint64_t addr, uint64_t key, void *context) -{ - struct iovec iov = { - .iov_base = (void*)buf, - .iov_len = len - }; - return ofi_nd_ep_writev(ep, &iov, &desc, 1, dest_addr, addr, key, context); -} - -static ssize_t -ofi_nd_ep_writev(struct fid_ep *pep, const struct iovec *iov, void **desc, - size_t count, fi_addr_t dest_addr, uint64_t addr, uint64_t key, - void *context) -{ - struct fi_rma_iov rma_iov = { - .addr = addr, - .len = iov[0].iov_len, - .key = key - }; - - struct fi_msg_rma msg = { - .msg_iov = iov, - .desc = desc, - .iov_count = count, - .addr = dest_addr, - .rma_iov = &rma_iov, - .rma_iov_count = 1, - .context = context, - .data = 0 - }; - - assert(pep->fid.fclass == FI_CLASS_EP); - - if (pep->fid.fclass != FI_CLASS_EP) - return -FI_EINVAL; - - struct nd_ep *ep = container_of(pep, struct nd_ep, fid); - - return ofi_nd_ep_writemsg(pep, &msg, ep->info->tx_attr->op_flags); -} - -static ssize_t -ofi_nd_ep_writemsg(struct fid_ep *pep, const struct fi_msg_rma *msg, - uint64_t flags) -{ - assert(pep->fid.fclass == FI_CLASS_EP); - assert(msg); - - if (pep->fid.fclass != FI_CLASS_EP) - return -FI_EINVAL; - - size_t msg_len = 0, rma_len = 0, i; - HRESULT hr = 0; - - struct nd_cq_entry *entries[ND_MSG_IOV_LIMIT]; - struct nd_ep *ep = container_of(pep, struct nd_ep, fid); - - if (!ep->qp) - return -FI_EOPBADSTATE; - - for (i = 0; i < msg->iov_count; i++) { - if (msg->msg_iov[i].iov_len && !msg->msg_iov[i].iov_base) - return -FI_EINVAL; - msg_len += msg->msg_iov[i].iov_len; - } - - if ((msg_len > ep->domain->info->ep_attr->max_msg_size) && - (flags & FI_INJECT)) - return -FI_EINVAL; - - for (i = 0; i < msg->rma_iov_count; i++) { - if (msg->rma_iov[i].len && !msg->rma_iov[i].addr) - return -FI_EINVAL; - rma_len += msg->rma_iov[i].len; - } - - /* Check the following: */ - if ((msg_len != rma_len) || /* - msg and rma len are correlated */ - /* - iov counts are less or equal than supported */ - ((msg->iov_count > ND_MSG_IOV_LIMIT || - msg->rma_iov_count > ND_MSG_IOV_LIMIT)) || - /* - transmitted length is less or equal than max possible */ - (msg_len > ep->domain->info->ep_attr->max_msg_size) || - /* - if INJECT, data should be inlined */ - ((flags & FI_INJECT) && - (msg_len > ep->domain->info->tx_attr->inject_size))) - return -FI_EINVAL; - - struct nd_cq_entry *main_entry = ofi_nd_buf_alloc_nd_cq_entry(); - if (!main_entry) - return -FI_ENOMEM; - memset(main_entry, 0, sizeof(*main_entry)); - main_entry->data = msg->data; - main_entry->flags = flags; - main_entry->domain = ep->domain; - main_entry->context = msg->context; - main_entry->seq = InterlockedAdd64(&ep->domain->msg_cnt, 1); - - /* since write operation can't be canceled, set NULL into - * the 1st pointer of internal data of context */ - if (msg->context) - ND_FI_CONTEXT(msg->context) = 0; - - /* TODO */ - if (msg_len > (size_t)gl_data.inline_thr) { - struct fi_rma_iov rma_iovecs[ND_MSG_INTERNAL_IOV_LIMIT]; - size_t rma_count = 0; - size_t from_split_map[ND_MSG_INTERNAL_IOV_LIMIT]; - size_t to_split_map[ND_MSG_INTERNAL_IOV_LIMIT]; - uint64_t remote_addr[ND_MSG_INTERNAL_IOV_LIMIT]; - - ofi_nd_split_msg_iov_2_rma_iov(msg->rma_iov, msg->rma_iov_count, - msg->msg_iov, msg->iov_count, - rma_iovecs, &rma_count, - from_split_map, to_split_map, remote_addr); - - assert(rma_count <= ND_MSG_INTERNAL_IOV_LIMIT); - - main_entry->wait_completion.comp_count = 0; - main_entry->wait_completion.total_count = rma_count; - - InitializeCriticalSection(&main_entry->wait_completion.comp_lock); - - for (i = 0; i < rma_count; i++) { - entries[i] = ofi_nd_buf_alloc_nd_cq_entry(); - if (!entries[i]) - goto fn_fail; - memset(entries[i], 0, sizeof(*entries[i])); - - entries[i]->data = msg->data; - entries[i]->flags = flags; - entries[i]->domain = ep->domain; - entries[i]->context = msg->context; - entries[i]->seq = main_entry->seq; - entries[i]->aux_entry = main_entry; - - ND2_SGE sge = { - .Buffer = (void *)remote_addr[i], - .BufferLength = (ULONG)rma_iovecs[i].len, - .MemoryRegionToken = (UINT32)(uintptr_t)msg->desc[to_split_map[i]] - }; - - hr = ep->qp->lpVtbl->Write(ep->qp, entries[i], &sge, 1, - (UINT64)rma_iovecs[i].addr, (UINT32)rma_iovecs[i].key, 0); - if (FAILED(hr)) - goto fn_fail; - } - - return FI_SUCCESS; - } - else { - if (msg_len) { - main_entry->inline_buf = __ofi_nd_buf_alloc_nd_inlinebuf(&ep->domain->inlinebuf); - if (!main_entry->inline_buf) - return -FI_ENOMEM; - - char *buf = (char*)main_entry->inline_buf->buffer; - for (i = 0; i < msg->iov_count; i++) { - memcpy(buf, msg->msg_iov[i].iov_base, msg->msg_iov[i].iov_len); - buf += msg->msg_iov[i].iov_len; - } - } - - for (i = 0; i < msg->rma_iov_count; i++) { - char *buf = (char *)main_entry->inline_buf->buffer; - - entries[i] = ofi_nd_buf_alloc_nd_cq_entry(); - if (!entries[i]) - goto fn_fail; - memset(entries[i], 0, sizeof(*entries[i])); - - entries[i]->data = msg->data; - entries[i]->flags = flags; - entries[i]->domain = ep->domain; - entries[i]->context = msg->context; - entries[i]->seq = main_entry->seq; - entries[i]->aux_entry = main_entry; - - ND2_SGE sge = { - .Buffer = (void *)(buf + msg->rma_iov[i].len), - .BufferLength = (ULONG)msg->rma_iov[i].len, - .MemoryRegionToken = main_entry->inline_buf->token - }; - - hr = ep->qp->lpVtbl->Write(ep->qp, entries[i], &sge, 1, - (UINT64)msg->rma_iov[i].addr, (UINT32)msg->rma_iov[i].key, 0); - if (FAILED(hr)) - goto fn_fail; - } - - return FI_SUCCESS; - } -fn_fail: - while (i-- > 0) - ofi_nd_free_cq_entry(entries[i]); - ND_LOG_WARN(FI_LOG_EP_DATA, ofi_nd_strerror((DWORD)hr, NULL)); - return H2F(hr); -} - -static ssize_t -ofi_nd_ep_inject(struct fid_ep *pep, const void *buf, size_t len, - fi_addr_t dest_addr, uint64_t addr, uint64_t key) -{ - struct iovec iov = { - .iov_base = (void*)buf, - .iov_len = len - }; - - struct fi_rma_iov rma_iov = { - .addr = addr, - .len = len, - .key = key - }; - - struct fi_msg_rma msg = { - .msg_iov = &iov, - .desc = 0, - .iov_count = 1, - .addr = dest_addr, - .rma_iov = &rma_iov, - .rma_iov_count = 1, - .context = 0, - .data = 0 - }; - - return ofi_nd_ep_writemsg(pep, &msg, FI_INJECT); -} - -static ssize_t -ofi_nd_ep_writedata(struct fid_ep *pep, const void *buf, size_t len, void *desc, - uint64_t data, fi_addr_t dest_addr, uint64_t addr, uint64_t key, - void *context) -{ - struct iovec iov = { - .iov_base = (void*)buf, - .iov_len = len - }; - - struct fi_rma_iov rma_iov = { - .addr = addr, - .len = len, - .key = key - }; - - struct fi_msg_rma msg = { - .msg_iov = &iov, - .desc = &desc, - .iov_count = 1, - .addr = dest_addr, - .rma_iov = &rma_iov, - .rma_iov_count = 1, - .context = context, - .data = data - }; - - assert(pep->fid.fclass == FI_CLASS_EP); - - if (pep->fid.fclass != FI_CLASS_EP) - return -FI_EINVAL; - - struct nd_ep *ep = container_of(pep, struct nd_ep, fid); - - return ofi_nd_ep_writemsg(pep, &msg, ep->info->tx_attr->op_flags | FI_REMOTE_CQ_DATA); -} - -static ssize_t -ofi_nd_ep_writeinjectdata(struct fid_ep *ep, const void *buf, size_t len, - uint64_t data, fi_addr_t dest_addr, uint64_t addr, - uint64_t key) -{ - struct iovec iov = { - .iov_base = (void*)buf, - .iov_len = len - }; - - struct fi_rma_iov rma_iov = { - .addr = addr, - .len = len, - .key = key - }; - - struct fi_msg_rma msg = { - .msg_iov = &iov, - .desc = 0, - .iov_count = 1, - .addr = dest_addr, - .rma_iov = &rma_iov, - .rma_iov_count = 1, - .context = 0, - .data = data - }; - - return ofi_nd_ep_writemsg(ep, &msg, FI_INJECT | FI_REMOTE_CQ_DATA); -} - -void ofi_nd_read_event(ND2_RESULT *result) -{ - assert(result); - assert(result->RequestType == Nd2RequestTypeRead); - - nd_cq_entry *entry = (nd_cq_entry*)result->RequestContext; - assert(entry); - - ND_LOG_EVENT_INFO(entry); - - /* Check whether the operation is complex, i.e. read operation - * may consists from several subtasks of read */ - if (entry->aux_entry) { - EnterCriticalSection(&entry->aux_entry->wait_completion.comp_lock); - entry->aux_entry->wait_completion.comp_count++; - ND_LOG_DEBUG(FI_LOG_EP_DATA, "READ Event comp_count = %d, total_count = %d\n", - entry->aux_entry->wait_completion.comp_count, - entry->aux_entry->wait_completion.total_count); - if (entry->aux_entry->wait_completion.comp_count < entry->aux_entry->wait_completion.total_count) { - /* Should wait some remaining completion events about read operation */ - LeaveCriticalSection(&entry->aux_entry->wait_completion.comp_lock); - entry->aux_entry = NULL; - ofi_nd_free_cq_entry(entry); - return; - } - LeaveCriticalSection(&entry->aux_entry->wait_completion.comp_lock); - } - - /*TODO: Handle erroneous case "result->Status != S_OK" */ - ofi_nd_dispatch_cq_event(entry->state == LARGE_MSG_RECV_REQ ? - LARGE_MSG_REQ : NORMAL_EVENT, entry, result); -} - -void ofi_nd_write_event(ND2_RESULT *result) -{ - assert(result); - assert(result->RequestType == Nd2RequestTypeWrite); - - nd_cq_entry *entry = (nd_cq_entry*)result->RequestContext; - assert(entry); - - struct nd_ep *ep = (struct nd_ep*)result->QueuePairContext; - assert(ep); - assert(ep->fid.fid.fclass == FI_CLASS_EP); - - ND_LOG_EVENT_INFO(entry); - - /* Check whether the operation is complex, i.e. write operation - * may consist from several subtasks of write */ - if (entry->aux_entry) { - EnterCriticalSection(&entry->aux_entry->wait_completion.comp_lock); - entry->aux_entry->wait_completion.comp_count++; - - if (entry->aux_entry->wait_completion.comp_count < entry->aux_entry->wait_completion.total_count) { - /* Should wait some remaining completion events about write operation */ - LeaveCriticalSection(&entry->aux_entry->wait_completion.comp_lock); - entry->aux_entry = NULL; - ofi_nd_free_cq_entry(entry); - return; - } - LeaveCriticalSection(&entry->aux_entry->wait_completion.comp_lock); - } - - if (!entry->context) { - /* This means that this write was an internal event, - * just release it */ - ofi_nd_free_cq_entry(entry); - return; - } - - if (entry->flags & FI_REMOTE_CQ_DATA) { - if (ofi_nd_ep_injectdata( - &ep->fid, 0, 0, entry->data, - FI_ADDR_UNSPEC) != FI_SUCCESS) - ND_LOG_WARN(FI_LOG_CQ, "failed to write-inject"); - } - - if (ep->cntr_write) { - if (result->Status != S_OK) { - InterlockedIncrement64(&ep->cntr_write->err); - } - InterlockedIncrement64(&ep->cntr_write->counter); - WakeByAddressAll((void*)&ep->cntr_write->counter); - } - - int notify = ofi_nd_util_completion_blackmagic( - ep->info->tx_attr->op_flags, ep->send_flags, entry->flags) || - result->Status != S_OK; - - if (notify) { - PostQueuedCompletionStatus( - entry->result.Status == S_OK ? ep->cq_send->iocp : ep->cq_send->err, - 0, 0, &entry->base.ov); - InterlockedIncrement(&ep->cq_send->count); - WakeByAddressAll((void*)&ep->cq_send->count); - } - else { /* if notification is not requested - just free entry */ - ofi_nd_free_cq_entry(entry); - } -} - -void ofi_nd_split_msg_iov_2_rma_iov(const struct fi_rma_iov *rma_iovecs, const size_t rma_count, - const struct iovec *msg_iovecs, const size_t msg_count, - struct fi_rma_iov res_iovecs[ND_MSG_INTERNAL_IOV_LIMIT], - size_t *res_count, - size_t from_split_map[ND_MSG_INTERNAL_IOV_LIMIT], - size_t to_split_map[ND_MSG_INTERNAL_IOV_LIMIT], - uint64_t remote_addr[ND_MSG_INTERNAL_IOV_LIMIT]) -{ - size_t i; - - struct iovec from_rma_iovecs[ND_MSG_IOV_LIMIT]; - size_t from_rma_count = rma_count; - - struct iovec res_msg_iovecs[ND_MSG_IOV_LIMIT]; - size_t res_msg_count = 0; - - - /* Convert RMA iovecs to MSG iovecs to be able to reuse - * them in @ofi_nd_repack_iovecs */ - for (i = 0; i < rma_count; i++) { - from_rma_iovecs[i].iov_base = (void *)rma_iovecs[i].addr; - from_rma_iovecs[i].iov_len = rma_iovecs[i].len; - } - - ofi_nd_repack_iovecs(from_rma_iovecs, from_rma_count, - msg_iovecs, msg_count, - res_msg_iovecs, &res_msg_count, - from_split_map, to_split_map, remote_addr); - - /* Extract MSG iov to RMA iovecs and returns them */ - for (i = 0; i < res_msg_count; i++) { - res_iovecs[i].addr = remote_addr[i]; - res_iovecs[i].len = res_msg_iovecs[i].iov_len; - res_iovecs[i].key = rma_iovecs[from_split_map[i]].key; - - remote_addr[i] = (uint64_t)res_msg_iovecs[i].iov_base; - } - - *res_count = res_msg_count; -} - -#endif /* _WIN32 */ - diff --git a/prov/netdir/src/netdir_ep_srx.c b/prov/netdir/src/netdir_ep_srx.c deleted file mode 100644 index e42dc05044c..00000000000 --- a/prov/netdir/src/netdir_ep_srx.c +++ /dev/null @@ -1,337 +0,0 @@ -/* -* Copyright (c) 2015-2016 Intel Corporation, Inc. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -*/ - -#ifdef _WIN32 - -#include "netdir.h" -#include "netdir_ov.h" -#include "netdir_cq.h" -#include "netdir_log.h" -#include "netdir_iface.h" -#include "netdir_unexp.h" - -#include "rdma/fabric.h" -#include "rdma/fi_endpoint.h" - -#include "ofi.h" -#include "ofi_util.h" - -static ssize_t ofi_nd_srx_recv(struct fid_ep *ep, void *buf, size_t len, - void *desc, fi_addr_t src_addr, void *context); -static ssize_t ofi_nd_srx_recvmsg(struct fid_ep *ep_fid, const struct fi_msg *msg, - uint64_t flags); -static ssize_t ofi_nd_srx_recvv(struct fid_ep *ep_fid, const struct iovec *iov, void **desc, - size_t count, fi_addr_t src_addr, void *context); -static ssize_t ofi_nd_no_send(struct fid_ep *ep, const void *buf, size_t len, - void *desc, fi_addr_t src_addr, void *context); -static ssize_t ofi_nd_no_sendmsg(struct fid_ep *ep_fid, const struct fi_msg *msg, - uint64_t flags); -static ssize_t ofi_nd_no_sendv(struct fid_ep *ep_fid, const struct iovec *iov, void **desc, - size_t count, fi_addr_t dest_addr, void *context); -static ssize_t ofi_nd_no_inject(struct fid_ep *ep_fid, const void *buf, size_t len, - fi_addr_t dest_addr); -static ssize_t ofi_nd_no_senddata(struct fid_ep *ep, const void *buf, size_t len, void *desc, - uint64_t data, fi_addr_t dest_addr, void *context); -static ssize_t ofi_nd_no_injectdata(struct fid_ep *ep, const void *buf, size_t len, - uint64_t data, fi_addr_t dest_addr); -static int ofi_nd_srx_close(struct fid *fid); -static ssize_t ofi_nd_srx_cancel(fid_t fid, void *context); -extern int ofi_nd_ep_getopt(struct fid* ep, int level, int optname, - void* optval, size_t* optlen); - -struct fi_ops_msg ofi_nd_srx_msg = { - .size = sizeof(ofi_nd_srx_msg), - .recv = ofi_nd_srx_recv, - .recvv = ofi_nd_srx_recvv, - .recvmsg = ofi_nd_srx_recvmsg, - .send = ofi_nd_no_send, - .sendv = ofi_nd_no_sendv, - .sendmsg = ofi_nd_no_sendmsg, - .inject = ofi_nd_no_inject, - .senddata = ofi_nd_no_senddata, - .injectdata = ofi_nd_no_injectdata -}; - -static struct fi_ops ofi_nd_fi_ops = { - .size = sizeof(ofi_nd_fi_ops), - .close = ofi_nd_srx_close, - .bind = fi_no_bind, - .control = fi_no_control, - .ops_open = fi_no_ops_open -}; - -static struct fid ofi_nd_fid = { - .fclass = FI_CLASS_SRX_CTX, - .context = NULL, - .ops = &ofi_nd_fi_ops -}; - -static struct fi_ops_ep ofi_nd_ep_ops = { - .size = sizeof(ofi_nd_ep_ops), - .cancel = ofi_nd_srx_cancel, - .getopt = ofi_nd_ep_getopt, - .setopt = fi_no_setopt, - .tx_ctx = fi_no_tx_ctx, - .rx_ctx = fi_no_rx_ctx, - .rx_size_left = fi_no_rx_size_left, - .tx_size_left = fi_no_tx_size_left -}; - -int ofi_nd_srx_ctx(struct fid_domain *pdomain, - struct fi_rx_attr *attr, struct fid_ep **rx_ep, - void *context) -{ - OFI_UNUSED(attr); - struct nd_domain *domain = container_of(pdomain, struct nd_domain, fid); - struct nd_srx *srx = (struct nd_srx*) calloc(1, sizeof(*srx)); - if (!srx) - return -FI_ENOMEM; - - struct nd_srx def = { - .fid = { - .fid = { - .fclass = FI_CLASS_SRX_CTX, - .context = context, - .ops = &ofi_nd_fi_ops - }, - .ops = &ofi_nd_ep_ops, - .msg = &ofi_nd_srx_msg - }, - .domain = domain, - .attr = { - .caps = FI_MSG | FI_RECV, - .mode = 0, - .op_flags = 0, - .comp_order = FI_ORDER_STRICT, - .total_buffered_recv = 0, - .size = (size_t)gl_data.inline_thr, - .iov_limit = (size_t)min(domain->ainfo.MaxReceiveSge, ND_MSG_IOV_LIMIT) - 1 - } - }; - - *srx = def; - /* TODO */ - dlist_init(&srx->received); - - *rx_ep = &srx->fid; - - InitializeCriticalSection(&srx->prepost_lock); - - return FI_SUCCESS; -} - -static ssize_t ofi_nd_srx_recvmsg(struct fid_ep *pep, const struct fi_msg *msg, - uint64_t flags) -{ - assert(pep->fid.fclass == FI_CLASS_SRX_CTX); - assert(msg); - - if (pep->fid.fclass != FI_CLASS_SRX_CTX) - return -FI_EINVAL; - - size_t i; - size_t len = 0; - - struct nd_srx *srx = container_of(pep, struct nd_srx, fid); - - if (msg->iov_count > min(srx->domain->ainfo.MaxReceiveSge, ND_MSG_IOV_LIMIT) - 1) - return -FI_EINVAL; - - for (i = 0; i < msg->iov_count; i++) { - if (msg->msg_iov[i].iov_len && !msg->msg_iov[i].iov_base) - return -FI_EINVAL; - len += msg->msg_iov[i].iov_len; - } - - struct nd_cq_entry *entry = ofi_nd_buf_alloc_nd_cq_entry(); - if (!entry) - return -FI_ENOMEM; - memset(entry, 0, sizeof(*entry)); - - entry->buf = (msg->iov_count == 1) ? msg->msg_iov[0].iov_base : 0; - entry->len = len; - entry->data = msg->data; - entry->flags = flags | FI_MSG | FI_RECV; - entry->domain = srx->domain; - entry->context = msg->context; - entry->iov_cnt = msg->iov_count; - entry->seq = InterlockedAdd64(&srx->domain->msg_cnt, 1); - - for (i = 0; i < msg->iov_count; i++) { - entry->iov[i] = msg->msg_iov[i]; - } - - /* store allocated entry in 1st pointer of internal data of context */ - if (msg->context) - ND_FI_CONTEXT(msg->context) = entry; - - ofi_nd_queue_push(&srx->prepost, &entry->queue_item); - - ofi_nd_srx_match(srx); - - return FI_SUCCESS; -} - -static ssize_t ofi_nd_srx_recvv(struct fid_ep *pep, const struct iovec *iov, - void **desc, - size_t count, fi_addr_t src_addr, void *context) -{ - struct fi_msg msg = { - .msg_iov = iov, - .desc = desc, - .iov_count = count, - .addr = src_addr, - .context = context, - .data = 0 - }; - - assert(pep->fid.fclass == FI_CLASS_SRX_CTX); - - if (pep->fid.fclass != FI_CLASS_SRX_CTX) - return -FI_EINVAL; - - struct nd_srx *rx_ctx = container_of(pep, struct nd_srx, fid); - - return ofi_nd_srx_recvmsg(pep, &msg, rx_ctx->attr.op_flags); -} - -static ssize_t ofi_nd_srx_recv(struct fid_ep *pep, void *buf, size_t len, - void *desc, fi_addr_t src_addr, void *context) -{ - struct iovec iov = { - .iov_base = buf, - .iov_len = len - }; - - return ofi_nd_srx_recvv(pep, &iov, &desc, 1, src_addr, context); -} - -static int ofi_nd_srx_close(struct fid *fid) -{ - assert(fid); - assert(fid->fclass == FI_CLASS_SRX_CTX); - - if (fid->fclass != FI_CLASS_SRX_CTX) - return -FI_EINVAL; - - struct nd_srx *srx = container_of(fid, struct nd_srx, fid.fid); - - DeleteCriticalSection(&srx->prepost_lock); - if (srx->srx) - srx->srx->lpVtbl->Release(srx->srx); - free(srx); - - return FI_SUCCESS; -} - -static ssize_t ofi_nd_no_send(struct fid_ep *ep, const void *buf, size_t len, - void *desc, fi_addr_t src_addr, void *context) -{ - OFI_UNUSED(ep); - OFI_UNUSED(buf); - OFI_UNUSED(len); - OFI_UNUSED(desc); - OFI_UNUSED(src_addr); - OFI_UNUSED(context); - return -FI_ENOSYS; -} - -static ssize_t ofi_nd_no_sendmsg(struct fid_ep *ep_fid, const struct fi_msg *msg, - uint64_t flags) -{ - OFI_UNUSED(ep_fid); - OFI_UNUSED(msg); - OFI_UNUSED(flags); - return -FI_ENOSYS; -} - -static ssize_t ofi_nd_no_sendv(struct fid_ep *ep_fid, const struct iovec *iov, void **desc, - size_t count, fi_addr_t dest_addr, void *context) -{ - OFI_UNUSED(ep_fid); - OFI_UNUSED(iov); - OFI_UNUSED(desc); - OFI_UNUSED(count); - OFI_UNUSED(dest_addr); - OFI_UNUSED(context); - return -FI_ENOSYS; -} -static ssize_t ofi_nd_no_inject(struct fid_ep *ep_fid, const void *buf, size_t len, - fi_addr_t dest_addr) -{ - OFI_UNUSED(ep_fid); - OFI_UNUSED(buf); - OFI_UNUSED(len); - OFI_UNUSED(dest_addr); - return -FI_ENOSYS; -} - -static ssize_t ofi_nd_no_senddata(struct fid_ep *ep, const void *buf, size_t len, void *desc, - uint64_t data, fi_addr_t dest_addr, void *context) -{ - OFI_UNUSED(ep); - OFI_UNUSED(buf); - OFI_UNUSED(len); - OFI_UNUSED(desc); - OFI_UNUSED(data); - OFI_UNUSED(dest_addr); - OFI_UNUSED(context); - return -FI_ENOSYS; -} - -static ssize_t ofi_nd_no_injectdata(struct fid_ep *ep, const void *buf, size_t len, - uint64_t data, fi_addr_t dest_addr) -{ - OFI_UNUSED(ep); - OFI_UNUSED(buf); - OFI_UNUSED(len); - OFI_UNUSED(data); - OFI_UNUSED(dest_addr); - return -FI_ENOSYS; -} - -static ssize_t ofi_nd_srx_cancel(fid_t fid, void *context) -{ - assert(fid); - assert(fid->fclass == FI_CLASS_SRX_CTX); - assert(context); - - if (!context) { - ND_LOG_WARN(FI_LOG_EP_DATA, "Context is NULL \n"); - return -FI_EINVAL; - } - - return ofi_nd_cq_cancel(fid, context); -} - -#endif /* _WIN32 */ - diff --git a/prov/netdir/src/netdir_eq.c b/prov/netdir/src/netdir_eq.c deleted file mode 100644 index 9deeef24aa7..00000000000 --- a/prov/netdir/src/netdir_eq.c +++ /dev/null @@ -1,411 +0,0 @@ -/* -* Copyright (c) 2015-2016 Intel Corporation, Inc. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -*/ - -#ifdef _WIN32 - -#include "netdir.h" -#include "netdir_ov.h" -#include "netdir_log.h" -#include "netdir_iface.h" - -#include "ofi_util.h" - -static int ofi_nd_eq_close(struct fid *fid); -static ssize_t ofi_nd_eq_read(struct fid_eq *eq, uint32_t *event, - void *buf, size_t len, uint64_t flags); -static ssize_t ofi_nd_eq_write(struct fid_eq *eq, uint32_t ev, - const void *buf, size_t len, uint64_t flags); -static ssize_t ofi_nd_eq_readerr(struct fid_eq *eq, struct fi_eq_err_entry *buf, - uint64_t flags); -static ssize_t ofi_nd_eq_sread(struct fid_eq *eq, uint32_t *event, - void *buf, size_t len, int timeout, - uint64_t flags); -static const char *ofi_nd_eq_strerror(struct fid_eq *eq, int prov_errno, - const void *err_data, char *buf, - size_t len); - -static struct fi_ops ofi_nd_fi_ops = { - .size = sizeof(ofi_nd_fi_ops), - .close = ofi_nd_eq_close, - .bind = fi_no_bind, - .control = fi_no_control, - .ops_open = fi_no_ops_open, -}; - -static struct fi_ops_eq ofi_nd_eq_ops = { - .size = sizeof(ofi_nd_eq_ops), - .read = ofi_nd_eq_read, - .readerr = ofi_nd_eq_readerr, - .write = ofi_nd_eq_write, - .sread = ofi_nd_eq_sread, - .strerror = ofi_nd_eq_strerror -}; - -int ofi_nd_eq_open(struct fid_fabric *fabric, struct fi_eq_attr *attr, - struct fid_eq **peq, void *context) -{ - assert(fabric); - assert(fabric->fid.fclass == FI_CLASS_FABRIC); - - if (attr) { - if (attr->wait_obj != FI_WAIT_NONE && attr->wait_obj != FI_WAIT_UNSPEC) - return -FI_EBADFLAGS; - } - - struct nd_eq *eq = (struct nd_eq*)calloc(1, sizeof(*eq)); - if (!eq) - return -FI_ENOMEM; - - struct nd_eq def = { - .fid = { - .fid = { - .fclass = FI_CLASS_EQ, - .context = context, - .ops = &ofi_nd_fi_ops - }, - .ops = &ofi_nd_eq_ops - } - }; - - *eq = def; - - InitializeCriticalSection(&eq->lock); - - eq->iocp = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0); - if (!eq->iocp || eq->iocp == INVALID_HANDLE_VALUE) { - ofi_nd_eq_close(&eq->fid.fid); - return H2F(HRESULT_FROM_WIN32(GetLastError())); - } - - eq->err = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0); - if (!eq->err || eq->err == INVALID_HANDLE_VALUE) { - ofi_nd_eq_close(&eq->fid.fid); - return H2F(HRESULT_FROM_WIN32(GetLastError())); - } - - *peq = &eq->fid; - - return FI_SUCCESS; -} - -static int ofi_nd_eq_close(struct fid *fid) -{ - assert(fid->fclass == FI_CLASS_EQ); - - struct nd_eq *eq = container_of(fid, struct nd_eq, fid.fid); - - if (eq->iocp && eq->iocp != INVALID_HANDLE_VALUE) - CloseHandle(eq->iocp); - if (eq->err && eq->err != INVALID_HANDLE_VALUE) - CloseHandle(eq->err); - - DeleteCriticalSection(&eq->lock); - - free(eq); - return FI_SUCCESS; -} - -static inline ssize_t ofi_nd_eq_ev2buf(struct nd_eq_event *ev, - void *buf, size_t len) -{ - assert(ev); - - size_t copylen = 0; - char* dst = (char *)buf; - - if (!ev->is_custom) { - switch (ev->eq_event) { - case FI_CONNREQ: - case FI_CONNECTED: - case FI_SHUTDOWN: - copylen = min(sizeof(struct fi_eq_cm_entry), len); - break; - case FI_AV_COMPLETE: - case FI_MR_COMPLETE: - copylen = min(sizeof(struct fi_eq_entry), len); - break; - default: - ND_LOG_WARN(FI_LOG_EQ, "unknown event type: %d\n", - ev->eq_event); - copylen = min(sizeof(struct fi_eq_entry), len); - break; - } - } - - if (copylen) - memcpy(dst, &ev->operation, copylen); - - if (ev->len) { - assert(ev->data); - if (len > copylen) { - dst += copylen; - memcpy(dst, ev->data, min(len - copylen, ev->len)); - copylen += min(len - copylen, ev->len); - } - } - return (ssize_t)copylen; -} - -static ssize_t ofi_nd_eq_read(struct fid_eq *peq, uint32_t *pev, - void *buf, size_t len, uint64_t flags) -{ - assert(peq); - assert(pev); - assert(peq->fid.fclass == FI_CLASS_EQ); - - struct nd_eq *eq = container_of(peq, struct nd_eq, fid); - - DWORD bytes; - ULONG_PTR key; - OVERLAPPED *ov; - ssize_t res = 0; - - struct nd_eq_event *ev = 0; - - if (!eq->count) - return -FI_EAGAIN; - - /* we have to use critical section here because concurrent thread - may read event with FI_PEEK flag */ - EnterCriticalSection(&eq->lock); - - /* check again because it may be changed on critical section barrier */ - if (!eq->count) { - res = -FI_EAGAIN; - goto fn_complete; - } - - /* if there is peeked item - use it, else - try to read from queue */ - if (eq->peek) { - ev = eq->peek; - } - else { - assert(eq->iocp); - if (GetQueuedCompletionStatus(eq->iocp, &bytes, &key, &ov, 0)) { - ev = container_of(ov, struct nd_eq_event, ov); - } - } - - /* in case if no event available, but counter is non-zero - error available */ - if (!ev && eq->count) { - res = -FI_EAVAIL; - goto fn_complete; - } - - res = ofi_nd_eq_ev2buf(ev, buf, len); - *pev = ev->eq_event; - - if (flags & FI_PEEK) { - eq->peek = ev; - /* we updated peek ptr, notify other waiters about this */ - WakeByAddressAll((void*)&eq->count); - } - else { - eq->peek = NULL; - InterlockedDecrement(&eq->count); - assert(eq->count >= 0); - } - -fn_complete: - LeaveCriticalSection(&eq->lock); - return res; -} - -static ssize_t ofi_nd_eq_readerr(struct fid_eq *peq, - struct fi_eq_err_entry *buf, uint64_t flags) -{ - assert(peq); - assert(peq->fid.fclass == FI_CLASS_EQ); - assert(buf); - - OFI_UNUSED(flags); - - struct nd_eq *eq = container_of(peq, struct nd_eq, fid); - - DWORD bytes; - ULONG_PTR key; - OVERLAPPED *ov; - - struct nd_eq_event *ev = NULL; - - if (!eq->errdata) { - free(eq->errdata); - eq->errdata = NULL; - } - - assert(eq->err); - if (!GetQueuedCompletionStatus(eq->err, &bytes, &key, &ov, 0)) - return -FI_EAGAIN; - - InterlockedDecrement(&eq->count); - assert(eq->count >= 0); - ev = container_of(ov, struct nd_eq_event, ov); - - if (buf->err_data && buf->err_data_size) { - memcpy(buf, &ev->error, offsetof(struct fi_eq_err_entry, err_data)); - buf->err_data_size = MIN(buf->err_data_size, ev->error.err_data_size); - memcpy(buf->err_data, ev->error.err_data, buf->err_data_size); - /* to be sure that the errdata in EQ is NULL */ - eq->errdata = NULL; - } else { - /* for compatibility purposes (release < 1.5 or passed err_data_size is 0) */ - memcpy(buf, &ev->error, sizeof(ev->error)); - eq->errdata = ev->error.err_data; - } - - return 0; -} - -static ssize_t ofi_nd_eq_sread(struct fid_eq *peq, uint32_t *pev, - void *buf, size_t len, int timeout, - uint64_t flags) -{ - assert(peq); - assert(pev); - assert(peq->fid.fclass == FI_CLASS_EQ); - - struct nd_eq *eq = container_of(peq, struct nd_eq, fid); - - DWORD bytes; - ULONG_PTR key; - OVERLAPPED *ov; - ssize_t res = 0; - - struct nd_eq_event *ev = 0; - - LONG zero = 0; - - for (;;) { - do { - if (!WaitOnAddress( - &eq->count, &zero, sizeof(eq->count), - (DWORD)timeout) && timeout >= 0) - return -FI_EAGAIN; - } while (!eq->count); - - /* we have to use critical section here because concurrent thread - may read event with FI_PEEK flag */ - EnterCriticalSection(&eq->lock); - - if (!eq->count) { - LeaveCriticalSection(&eq->lock); - if (timeout >= 0) - return -FI_EAGAIN; - else - continue; - } - - /* if there is peeked item - use it, else - try to read from queue */ - if (eq->peek) { - ev = eq->peek; - } - else { - assert(eq->iocp); - if (GetQueuedCompletionStatus( - eq->iocp, &bytes, &key, &ov, 0)) { - ev = container_of(ov, struct nd_eq_event, ov); - } - } - - /* in case if no event available, but counter is non-zero - error available */ - if (!ev && eq->count) { - res = -FI_EAVAIL; - goto fn_complete; - } - - res = ofi_nd_eq_ev2buf(ev, buf, len); - *pev = ev->eq_event; - - if (flags & FI_PEEK) { - eq->peek = ev; - /* we updated peek ptr, notify other waiters about this */ - WakeByAddressAll((void*)&eq->count); - } - else { - eq->peek = NULL; - InterlockedDecrement(&eq->count); - assert(eq->count >= 0); - } - -fn_complete: - LeaveCriticalSection(&eq->lock); - return res; - } -} - -static const char *ofi_nd_eq_strerror(struct fid_eq *eq, int prov_errno, - const void *err_data, char *buf, size_t len) -{ - OFI_UNUSED(eq); - OFI_UNUSED(err_data); - - if (buf && len) - return strncpy(buf, fi_strerror(-prov_errno), len); - return fi_strerror(-prov_errno); -} - -static ssize_t ofi_nd_eq_write(struct fid_eq *peq, uint32_t ev, - const void *buf, size_t len, uint64_t flags) -{ - OFI_UNUSED(flags); - - assert(peq); - assert(peq->fid.fclass == FI_CLASS_EQ); - - struct nd_eq *eq = container_of(peq, struct nd_eq, fid); - - nd_eq_event *custom = ofi_nd_buf_alloc_nd_eq_event(); - if (!custom) - return -FI_ENOMEM; - memset(custom, 0, sizeof(*custom)); - - custom->is_custom = 1; - custom->eq_event = ev; - if (len) { - assert(buf); - custom->data = malloc(len); - if (!custom->data) { - ofi_nd_eq_free_event(custom); - return -FI_ENOMEM; - } - custom->len = len; - memcpy(custom->data, buf, len); - } - - ofi_nd_eq_push(eq, custom); - - return len; -} - - -#endif /* _WIN32 */ - diff --git a/prov/netdir/src/netdir_fabric.c b/prov/netdir/src/netdir_fabric.c deleted file mode 100644 index 7c98f30615d..00000000000 --- a/prov/netdir/src/netdir_fabric.c +++ /dev/null @@ -1,121 +0,0 @@ -/* -* Copyright (c) 2015-2016 Intel Corporation, Inc. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -*/ - -#ifdef _WIN32 - -#include -#include - -#include "netdir.h" -#include "ofi_util.h" -#include "ofi_enosys.h" -#include "rdma/fabric.h" - -#include "netdir_ov.h" -#include "netdir_iface.h" - -static int ofi_nd_fabric_close(fid_t fid); - -static struct fi_ops ofi_nd_fi_ops = { - .size = sizeof(ofi_nd_fi_ops), - .close = ofi_nd_fabric_close, - .bind = fi_no_bind, - .control = fi_no_control, - .ops_open = fi_no_ops_open, -}; - -static struct fid ofi_nd_fid = { - .fclass = FI_CLASS_FABRIC, - .context = NULL, - .ops = &ofi_nd_fi_ops -}; - -static struct fi_ops_fabric ofi_nd_fabric_ops = { - .size = sizeof(ofi_nd_fabric_ops), - .domain = ofi_nd_domain_open, - .passive_ep = ofi_nd_passive_endpoint, - .eq_open = ofi_nd_eq_open, - .wait_open = fi_no_wait_open, - .trywait = fi_no_trywait -}; - -static int ofi_nd_fabric_close(fid_t fid) -{ - struct nd_fabric *fabric; - fabric = container_of(fid, struct nd_fabric, fid.fid); - free(fabric); - /* due to issues in cleanup NetworkDirect on library - unload make clening here, on fabric close */ - ofi_nd_shutdown(); - return FI_SUCCESS; -} - -int ofi_nd_fabric(struct fi_fabric_attr *attr, struct fid_fabric **fab, - void *context) -{ - OFI_UNUSED(context); - - if (attr) { - if (attr->name && strcmp(attr->name, ofi_nd_prov.name)) - return -FI_EINVAL; - if (attr->prov_name && strcmp(attr->prov_name, ofi_nd_prov.name)) - return -FI_EINVAL; - if (attr->prov_version && attr->prov_version != ofi_nd_prov.version) - return -FI_EINVAL; - } - - struct nd_fabric *fabric = (struct nd_fabric*)calloc(1, sizeof(*fabric)); - if (!fabric) - return -FI_ENOMEM; - - struct nd_fabric def = { - .fid = { - .fid = ofi_nd_fid, - .ops = &ofi_nd_fabric_ops - } - }; - - *fabric = def; - - *fab = &fabric->fid; - - fi_param_get_int(&ofi_nd_prov, "inlinethr", &gl_data.inline_thr); - fi_param_get_int(&ofi_nd_prov, "prepostcnt", &gl_data.prepost_cnt); - fi_param_get_int(&ofi_nd_prov, "prepostbufcnt", &gl_data.prepost_buf_cnt); - - gl_data.total_avail = gl_data.prepost_cnt * gl_data.prepost_buf_cnt; - - return FI_SUCCESS; -} - -#endif /* _WIN32 */ - diff --git a/prov/netdir/src/netdir_iface.h b/prov/netdir/src/netdir_iface.h deleted file mode 100644 index 0b5829713ec..00000000000 --- a/prov/netdir/src/netdir_iface.h +++ /dev/null @@ -1,273 +0,0 @@ -/* -* Copyright (c) 2015-2016 Intel Corporation, Inc. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -*/ - -#ifndef _FI_NETDIR_IFACE_H_ -#define _FI_NETDIR_IFACE_H_ - -#include -#include -#include - -#include - -#include "rdma/fabric.h" - -#include "ofi_mem.h" -#include "ofi_list.h" -#include "rdma/fi_eq.h" -#include "rdma/fi_domain.h" -#include "rdma/fi_endpoint.h" - -#include "netdir_buf.h" -#include "netdir_queue.h" -#include "netdir_ov.h" - -#ifdef __cplusplus -extern "C" { -#endif /* __cplusplus */ - -typedef void(*nd_free_event_t)(struct nd_event_base* base); -typedef void(*nd_event_t)(struct nd_event_base* base, DWORD bytes); -typedef void(*nd_err_t)(struct nd_event_base* base, DWORD bytes, DWORD err); - -typedef struct nd_event_base { - OVERLAPPED ov; - - nd_free_event_t free; - nd_event_t event_cb; - nd_err_t err_cb; -} nd_event_base; - -struct nd_fabric { - struct fid_fabric fid; -}; - -typedef struct nd_flow_cntrl_flags { - unsigned req_ack : 1; - unsigned ack : 1; - unsigned empty : 1; -} nd_flow_cntrl_flags; - -struct nd_msgheader { - uint64_t data; - enum ofi_nd_cq_event event; - nd_flow_cntrl_flags flags; - size_t location_cnt; -}; - -struct nd_msgprefix { - UINT32 token; - struct nd_msgheader header; -}; - -struct nd_inlinebuf { - UINT32 token; - void* buffer; -}; - -struct nd_msg_location { - uint64_t addr; - size_t len; - uint32_t remote_mr_token; -}; - -struct nd_notifybuf { - UINT32 token; - struct nd_msg_location location[ND_MSG_IOV_LIMIT]; -}; - -OFI_ND_NB_BUF_TYPED(nd_msgprefix, struct nd_msgprefix); -OFI_ND_NB_BUF_TYPED(nd_inlinebuf, struct nd_inlinebuf); -OFI_ND_NB_BUF_TYPED(nd_notifybuf, struct nd_notifybuf); - -struct nd_domain { - struct fid_domain fid; - struct nd_eq *eq; - struct fi_info *info; - - uint64_t eq_flags; - - IND2Adapter *adapter; - IND2CompletionQueue *cq; - - nd_event_base ov; - - HANDLE adapter_file; - ND2_ADAPTER_INFO ainfo; - - LONG64 msg_cnt; - - LONG cq_canceled; - - ND_BUF_FOOTER(nd_msgprefix) msgfooter; - ND_BUF_FOOTER(nd_inlinebuf) inlinebuf; - ND_BUF_FOOTER(nd_notifybuf) notifybuf; - - union { - struct sockaddr addr; - struct sockaddr_in addr4; - struct sockaddr_in6 addr6; - } addr; -#if 0 - pthread_t progress_thread; - int do_progress; -#endif - struct dlist_entry ep_list; -}; - -struct nd_pep { - struct fid_pep fid; - struct fi_info *info; - - struct nd_eq *eq; - - IND2Adapter *adapter; - IND2Listener *listener; - - HANDLE adapter_file; -}; - -struct nd_eq { - struct fid_eq fid; - size_t cnum; - HANDLE iocp; - HANDLE err; - volatile LONG count; /* total number of available events, - including peek, queued & errors */ - struct nd_eq_event *peek; - - CRITICAL_SECTION lock; - void* errdata; -}; - - -struct nd_cq { - struct fid_cq fid; - enum fi_cq_format format; - - HANDLE iocp; - HANDLE err; - volatile LONG count; /* total number of available events, - including queued & errors */ -}; - -struct nd_cntr { - struct fid_cntr fid; - volatile LONG64 counter; - volatile LONG64 err; -}; - -struct nd_connreq { - struct fid handle; - IND2Connector *connector; -}; - -struct nd_unexpected { - IND2MemoryRegion *mr; - UINT32 token; - struct nd_unexpected_buf **unexpected; -#if 0 - size_t used_counter; -#endif - CRITICAL_SECTION unexp_lock; - struct dlist_entry received; - LONG active; -}; - -typedef struct nd_flow_block_flags { - unsigned is_send_blocked : 1; -} nd_flow_block_flags; - -struct nd_ep { - struct fid_ep fid; - struct fi_info *info; - - struct nd_domain *domain; - struct nd_eq *eq; - struct nd_srx *srx; - - struct nd_cq *cq_send; - struct nd_cq *cq_recv; - - uint64_t send_flags; - uint64_t recv_flags; - - struct nd_cntr *cntr_send; - struct nd_cntr *cntr_recv; - struct nd_cntr *cntr_read; - struct nd_cntr *cntr_write; - - IND2Connector *connector; - IND2QueuePair *qp; - - struct nd_unexpected unexpected; - struct nd_queue_queue prepost; - struct nd_queue_queue internal_prepost; - - nd_event_base disconnect_ov; - - CRITICAL_SECTION prepost_lock; - LONG shutdown; - LONG connected; - - struct dlist_entry entry; - struct { - nd_flow_block_flags flags; - size_t used_counter; - CRITICAL_SECTION send_lock; - } send_op; - struct nd_queue_queue send_queue; -}; - -struct nd_srx { - struct fid_ep fid; - struct fi_rx_attr attr; - IND2SharedReceiveQueue *srx; - struct nd_domain *domain; - struct dlist_entry received; - CRITICAL_SECTION prepost_lock; - struct nd_queue_queue prepost; -}; - -struct nd_mr { - struct fid_mr fid; - - IND2MemoryRegion *mr; - IND2MemoryWindow *wnd; -}; - -#ifdef __cplusplus -} -#endif /* __cplusplus */ - -#endif /* _FI_NETDIR_IFACE_H_ */ - diff --git a/prov/netdir/src/netdir_init.c b/prov/netdir/src/netdir_init.c deleted file mode 100644 index 3c8b0cc9336..00000000000 --- a/prov/netdir/src/netdir_init.c +++ /dev/null @@ -1,205 +0,0 @@ -/* -* Copyright (c) 2015-2016 Intel Corporation, Inc. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -*/ - -#ifdef _WIN32 - -#include "netdir.h" -#include "netdir_buf.h" -#include "ofi_prov.h" -#include "ofi_util.h" -#include "ofi_mem.h" - -#include "netdir_ov.h" -#include "netdir_iface.h" - -#include "netdir_queue.h" - -const char ofi_nd_prov_name[] = "netdir"; - -struct fi_provider ofi_nd_prov = { - .name = ofi_nd_prov_name, - .version = OFI_VERSION_DEF_PROV, - .fi_version = OFI_VERSION_LATEST, - .getinfo = ofi_nd_getinfo, - .fabric = ofi_nd_fabric, - .cleanup = ofi_nd_fini -}; - -static void ofi_nd_alter_defaults(uint32_t version, const struct fi_info *hints, - const struct fi_info *base_info, - struct fi_info *dest_info); - -struct util_prov ofi_nd_util_prov = { - .prov = &ofi_nd_prov, - .info = 0, - .alter_defaults = &ofi_nd_alter_defaults, - .flags = UTIL_RX_SHARED_CTX, -}; - -struct gl_data gl_data = { - /* 8 KByte */ - .inline_thr = 8192, - .prepost_cnt = 8, - .prepost_buf_cnt = 1, - .flow_control_cnt = 1, - .total_avail = 64 -}; - -static size_t nd_default_tx_iov_limit = 8; -static size_t nd_default_tx_size = 384; -static size_t nd_default_rx_iov_limit = 8; -static size_t nd_default_rx_size = 384; - -static void ofi_nd_alter_defaults(uint32_t version, const struct fi_info *hints, - const struct fi_info *base_info, - struct fi_info *dest_info) -{ - dest_info->tx_attr->iov_limit = min(base_info->tx_attr->iov_limit, - nd_default_tx_iov_limit); - dest_info->tx_attr->size = min(base_info->tx_attr->size, - nd_default_tx_size); - dest_info->rx_attr->iov_limit = min(base_info->rx_attr->iov_limit, - nd_default_rx_iov_limit); - dest_info->rx_attr->size = min(base_info->rx_attr->size, - nd_default_rx_size); -} - -int ofi_nd_getinfo(uint32_t version, const char *node, const char *service, - uint64_t flags, const struct fi_info *hints, - struct fi_info **info) -{ - if (ofi_nd_util_prov.info) { - return util_getinfo(&ofi_nd_util_prov, version, node, service, flags, - hints, info); - } else { - *info = NULL; - return -FI_EINVAL; - } -} - -void ofi_nd_fini(void) -{ - if (ofi_nd_util_prov.info) { - fi_freeinfo((void*)ofi_nd_util_prov.info); - ofi_nd_util_prov.info = 0; - } - ofi_nd_shutdown(); - nd_buf_fini_apply(); -} - -extern struct fi_provider ofi_nd_prov; - -static int ofi_nd_adapter_cb(const ND2_ADAPTER_INFO *adapter, const char *name) -{ - struct fi_info *info = fi_allocinfo(); - if (!info) - return -FI_ENOMEM; - - info->tx_attr->caps = FI_MSG | FI_SEND; - info->tx_attr->mode = FI_CONTEXT; - info->tx_attr->comp_order = FI_ORDER_STRICT; - info->tx_attr->inject_size = (size_t)gl_data.inline_thr; - info->tx_attr->size = (size_t)adapter->MaxInitiatorQueueDepth; - /* TODO: if optimization will be needed, we can use adapter->MaxInitiatorSge, - * and use ND SGE to send/write iovecs */ - info->tx_attr->iov_limit = (size_t)min(adapter->MaxInitiatorSge, ND_MSG_IOV_LIMIT); - info->tx_attr->rma_iov_limit = 1; - info->tx_attr->op_flags = OFI_ND_TX_OP_FLAGS; - info->tx_attr->msg_order = OFI_ND_MSG_ORDER; - - info->rx_attr->caps = FI_MSG | FI_RECV; - info->rx_attr->mode = FI_CONTEXT; - info->rx_attr->comp_order = FI_ORDER_STRICT; - info->rx_attr->total_buffered_recv = 0; - info->rx_attr->size = (size_t)adapter->MaxReceiveQueueDepth; - /* TODO: if optimization will be needed, we can use adapter->MaxInitiatorSge, - * and use ND SGE to recv iovecs */ - info->rx_attr->iov_limit = (size_t)min(adapter->MaxReceiveSge, ND_MSG_IOV_LIMIT); - info->rx_attr->msg_order = OFI_ND_MSG_ORDER; - - info->ep_attr->type = FI_EP_MSG; - info->ep_attr->protocol = FI_PROTO_NETWORKDIRECT; - info->ep_attr->protocol_version = 0; - info->ep_attr->max_msg_size = (size_t)adapter->MaxTransferLength; - - info->domain_attr->caps = OFI_ND_DOMAIN_CAPS; - info->domain_attr->name = strdup(name); - info->domain_attr->threading = FI_THREAD_SAFE; - info->domain_attr->control_progress = FI_PROGRESS_AUTO; - info->domain_attr->data_progress = FI_PROGRESS_AUTO; - info->domain_attr->resource_mgmt = FI_RM_DISABLED; - info->domain_attr->av_type = FI_AV_UNSPEC; - info->domain_attr->mr_mode = FI_MR_BASIC | OFI_MR_BASIC_MAP | FI_MR_LOCAL; - info->domain_attr->cq_cnt = (size_t)adapter->MaxCompletionQueueDepth; - info->domain_attr->mr_iov_limit = ND_MSG_IOV_LIMIT; - info->domain_attr->mr_cnt = OFI_ND_MAX_MR_CNT; - - info->fabric_attr->name = strdup(ofi_nd_prov_name); - info->fabric_attr->prov_version = OFI_VERSION_DEF_PROV; - - info->caps = OFI_ND_EP_CAPS | OFI_ND_DOMAIN_CAPS; - info->mode = FI_CONTEXT; - info->addr_format = FI_SOCKADDR; - - if (!ofi_nd_util_prov.info) { - ofi_nd_util_prov.info = info; - } else { - struct fi_info *finfo = (struct fi_info *) ofi_nd_util_prov.info; - - while (finfo->next) - finfo = finfo->next; - finfo->next = info; - } - - return FI_SUCCESS; -} - -NETDIR_INI -{ - fi_param_define(&ofi_nd_prov, "inlinethr", FI_PARAM_INT, - "Inline threshold: size of buffer to be send using pre-allocated buffer"); - fi_param_define(&ofi_nd_prov, "largemsgthr", FI_PARAM_INT, - "Large msg threshold: size of user data that is considered as large message"); - fi_param_define(&ofi_nd_prov, "prepostcnt", FI_PARAM_INT, - "Prepost Buffer Count: number of buffers to be preposted per EP and " - "not required internal ACK"); - fi_param_define(&ofi_nd_prov, "prepostbufcnt", FI_PARAM_INT, - "Count of Entries in Array of Preposted Buffers: number of set of buffer " - "in each entry array of buffers to be preposted per EP"); - - ofi_nd_startup(ofi_nd_adapter_cb); - return &ofi_nd_prov; -} - - -#endif /* _WIN32 */ - diff --git a/prov/netdir/src/netdir_log.h b/prov/netdir/src/netdir_log.h deleted file mode 100644 index dd716d0f959..00000000000 --- a/prov/netdir/src/netdir_log.h +++ /dev/null @@ -1,195 +0,0 @@ -/* -* Copyright (c) 2015-2016 Intel Corporation, Inc. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -*/ - -#ifndef _FI_NETDIR_LOG_H_ -#define _FI_NETDIR_LOG_H_ - -#include "config.h" - -#include - -#include "rdma/providers/fi_log.h" - -#ifdef __cplusplus -extern "C" { -#endif /* __cplusplus */ - -extern struct fi_provider ofi_nd_prov; - -#define ND_LOG_WARN(subsystem, ...) FI_WARN(&ofi_nd_prov, subsystem, __VA_ARGS__) -#define ND_LOG_INFO(subsystem, ...) FI_INFO(&ofi_nd_prov, subsystem, __VA_ARGS__) -#define ND_LOG_DEBUG(subsystem, ...) FI_DBG(&ofi_nd_prov, subsystem, __VA_ARGS__) - -#if ENABLE_DEBUG -# define ND_LOG_EVENT_INFO(entry) \ - ND_LOG_DEBUG(FI_LOG_EP_DATA, "\nflags: req_ack - %d, ack - %d, empty - %d\n" \ - "common: state - %d, event - %d\n", \ - (entry)->flow_cntrl_flags.req_ack, \ - (entry)->flow_cntrl_flags.ack, \ - (entry)->flow_cntrl_flags.empty, \ - (entry)->state, \ - (entry)->event) -#else -# define ND_LOG_EVENT_INFO(entry) -#endif - -#define FI_ND_GUID_FORMAT "%08lX-%04hX-%04hX-%02hhX%02hhX-%02hhX%02hhX%02hhX%02hhX%02hhX%02hhX" -#define FI_ND_GUID_ARG(guid) \ - (guid).Data1, (guid).Data2, (guid).Data3, \ - (guid).Data4[0], (guid).Data4[1], (guid).Data4[2], \ - (guid).Data4[3], (guid).Data4[4], (guid).Data4[5], \ - (guid).Data4[6], (guid).Data4[7] - -/* ofi_nd_strerror generates string message based on err value (GetLastError) - returned string is valid till next call of ofi_nd_strerror -*/ -static inline char *ofi_nd_strerror(DWORD err, HMODULE module) -{ - static char *message = NULL; - size_t size; - - /* if message is allocated - free it */ - if (message) - LocalFree(message); - - size = FormatMessageA(FORMAT_MESSAGE_ALLOCATE_BUFFER | - FORMAT_MESSAGE_FROM_SYSTEM | - FORMAT_MESSAGE_IGNORE_INSERTS | - (module ? FORMAT_MESSAGE_FROM_HMODULE : 0), - module, err, MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), - (LPSTR)&message, 0, NULL); - - return size ? message : (char*)""; -} -static inline char * -ofi_nd_get_last_error_str(HRESULT hr, char *errmsg, SIZE_T max_msg_len) -{ - LPVOID lpMsgBuf; - DWORD dw = (DWORD)hr; - errno_t rc; - - FormatMessage( - FORMAT_MESSAGE_ALLOCATE_BUFFER | - FORMAT_MESSAGE_FROM_SYSTEM | - FORMAT_MESSAGE_IGNORE_INSERTS, - NULL, - dw, - MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT), - (LPTSTR)&lpMsgBuf, - 0, NULL); - - strcpy_s(errmsg, max_msg_len, "NTStatus: "); - rc = strncat_s(errmsg, max_msg_len - strlen(errmsg), - lpMsgBuf, _TRUNCATE); - - LocalFree(lpMsgBuf); - - return errmsg; -} - -#define OFI_NDERR(err, str) \ - case err: \ - str = #err ; \ - break - -#define ND_FLUSHED 0x10000L /* undocumented ND error code */ -#define ND_DISCONNECTED 0xc000020C - -static char *ofi_nd_error_str(HRESULT hr) -{ - static char lerr[128]; - char *err_str = NULL; - - switch (hr) { - OFI_NDERR(ND_SUCCESS, err_str); - OFI_NDERR(ND_FLUSHED, err_str); - OFI_NDERR(ND_TIMEOUT, err_str); - OFI_NDERR(ND_PENDING, err_str); - OFI_NDERR(ND_BUFFER_OVERFLOW, err_str); - OFI_NDERR(ND_DEVICE_BUSY, err_str); - OFI_NDERR(ND_NO_MORE_ENTRIES, err_str); - OFI_NDERR(ND_UNSUCCESSFUL, err_str); - OFI_NDERR(ND_ACCESS_VIOLATION, err_str); - OFI_NDERR(ND_INVALID_HANDLE, err_str); - OFI_NDERR(ND_INVALID_DEVICE_REQUEST, err_str); - OFI_NDERR(ND_INVALID_PARAMETER, err_str); - OFI_NDERR(ND_NO_MEMORY, err_str); - OFI_NDERR(ND_INVALID_PARAMETER_MIX, err_str); - OFI_NDERR(ND_DATA_OVERRUN, err_str); - OFI_NDERR(ND_SHARING_VIOLATION, err_str); - OFI_NDERR(ND_INSUFFICIENT_RESOURCES, err_str); - OFI_NDERR(ND_DEVICE_NOT_READY, err_str); - OFI_NDERR(ND_IO_TIMEOUT, err_str); - OFI_NDERR(ND_NOT_SUPPORTED, err_str); - OFI_NDERR(ND_INTERNAL_ERROR, err_str); - OFI_NDERR(ND_INVALID_PARAMETER_1, err_str); - OFI_NDERR(ND_INVALID_PARAMETER_2, err_str); - OFI_NDERR(ND_INVALID_PARAMETER_3, err_str); - OFI_NDERR(ND_INVALID_PARAMETER_4, err_str); - OFI_NDERR(ND_INVALID_PARAMETER_5, err_str); - OFI_NDERR(ND_INVALID_PARAMETER_6, err_str); - OFI_NDERR(ND_INVALID_PARAMETER_7, err_str); - OFI_NDERR(ND_INVALID_PARAMETER_8, err_str); - OFI_NDERR(ND_INVALID_PARAMETER_9, err_str); - OFI_NDERR(ND_INVALID_PARAMETER_10, err_str); - OFI_NDERR(ND_CANCELED, err_str); - OFI_NDERR(ND_REMOTE_ERROR, err_str); - OFI_NDERR(ND_INVALID_ADDRESS, err_str); - OFI_NDERR(ND_INVALID_DEVICE_STATE, err_str); - OFI_NDERR(ND_INVALID_BUFFER_SIZE, err_str); - OFI_NDERR(ND_TOO_MANY_ADDRESSES, err_str); - OFI_NDERR(ND_ADDRESS_ALREADY_EXISTS, err_str); - OFI_NDERR(ND_CONNECTION_REFUSED, err_str); - OFI_NDERR(ND_CONNECTION_INVALID, err_str); - OFI_NDERR(ND_CONNECTION_ACTIVE, err_str); - OFI_NDERR(ND_HOST_UNREACHABLE, err_str); - OFI_NDERR(ND_CONNECTION_ABORTED, err_str); - OFI_NDERR(ND_DEVICE_REMOVED, err_str); - OFI_NDERR(ND_DISCONNECTED, err_str); - default: - err_str = ofi_nd_get_last_error_str(hr, lerr, sizeof(lerr)); - if (err_str == NULL) { - _snprintf(lerr, sizeof(lerr), "Unknown ND error %#08ld", hr); - err_str = lerr; - } - break; - } - return err_str; -} -#undef NDERR - -#ifdef __cplusplus -} -#endif /* __cplusplus */ - -#endif /* _FI_NETDIR_LOG_H_ */ - diff --git a/prov/netdir/src/netdir_mr.c b/prov/netdir/src/netdir_mr.c deleted file mode 100644 index 02dbf98442a..00000000000 --- a/prov/netdir/src/netdir_mr.c +++ /dev/null @@ -1,355 +0,0 @@ -/* -* Copyright (c) 2015-2016 Intel Corporation, Inc. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -*/ - -#ifdef _WIN32 - -#include "netdir.h" - -#include "netdir_ov.h" -#include "netdir_iface.h" - -#include "ofi.h" -#include "ofi_util.h" - -static int ofi_nd_mr_close(struct fid *fid); - -static struct fi_ops ofi_nd_fi_ops = { - .size = sizeof(ofi_nd_fi_ops), - .close = ofi_nd_mr_close, - .bind = fi_no_bind, - .control = fi_no_control, - .ops_open = fi_no_ops_open, -}; - -static struct fid ofi_nd_fid = { - .fclass = FI_CLASS_MR, - .context = NULL, - .ops = &ofi_nd_fi_ops -}; - -typedef struct ofi_nd_mr_ov { - nd_event_base base; - struct nd_eq *eq; - fid_t fid; - void *context; - LONG cnt; -} ofi_nd_mr_ov; - -static void ofi_nd_mr_ov_free(struct nd_event_base* base); -static void ofi_nd_mr_ov_event(struct nd_event_base* base, DWORD bytes); -static void ofi_nd_mr_ov_err(struct nd_event_base* base, DWORD bytes, DWORD err); - -OFI_ND_NB_BUF_TYPED(nd_mr, struct nd_mr); -OFI_ND_NB_BUF_IMP(nd_mr); - -OFI_ND_NB_BUF_TYPED(mr_ov, ofi_nd_mr_ov); -OFI_ND_NB_BUF_IMP(mr_ov); - -static inline void ofi_nd_mr_fini_handler() -{ - ND_REGISTER_FINI(ND_BUF_FINIPTR(nd_mr)); - ND_REGISTER_FINI(ND_BUF_FINIPTR(mr_ov)); -} - -int ofi_nd_mr_reg(struct fid *fid, const void *buf, size_t len, - uint64_t access, uint64_t offset, uint64_t requested_key, - uint64_t flags, struct fid_mr **pmr, void *context) -{ - OFI_UNUSED(requested_key); - - assert(fid->fclass == FI_CLASS_DOMAIN); - assert(!offset); - - HRESULT hr; - - ofi_nd_mr_fini_handler(); - - if (fid->fclass != FI_CLASS_DOMAIN) - return -FI_EINVAL; - if (offset) - return -FI_EINVAL; - if (flags) - return -FI_EINVAL; - - struct nd_domain *domain = container_of(fid, struct nd_domain, fid.fid); - - assert(domain->adapter); - assert(domain->adapter_file); - - struct nd_mr *mr = ofi_nd_buf_alloc_nd_mr(); - if (!mr) - return -FI_ENOMEM; - - struct nd_mr def = { - .fid = { - .fid = ofi_nd_fid - } - }; - - *mr = def; - - hr = domain->adapter->lpVtbl->CreateMemoryRegion( - domain->adapter, &IID_IND2MemoryRegion, domain->adapter_file, - (void**)&mr->mr); - - if (FAILED(hr)) - goto fn_fail; - - ULONG ind2flag = 0; - - if (access & FI_REMOTE_READ) - ind2flag |= ND_MR_FLAG_ALLOW_REMOTE_READ; - if (access & FI_REMOTE_WRITE) - ind2flag |= ND_MR_FLAG_ALLOW_REMOTE_WRITE; - if ((access & FI_WRITE) || (access & FI_RECV)) - ind2flag |= ND_MR_FLAG_ALLOW_LOCAL_WRITE; - - /* there is bug in mlx4 module: it always generates - IO completion (even for cases when hEvent value - of OVERLAPPED structure is initialized). To - workaround this we have to use dynamically allocated - ov */ - ofi_nd_mr_ov *ov = ND_BUF_ALLOC(mr_ov); - if (!ov) { - hr = ND_NO_MEMORY; - goto fn_fail; - } - memset(ov, 0, sizeof(*ov)); - - ofi_nd_mr_ov ovdef = { - .base = { - .free = ofi_nd_mr_ov_free, - .event_cb = ofi_nd_mr_ov_event, - .err_cb = ofi_nd_mr_ov_err - }, - .eq = domain->eq, - .fid = &mr->fid.fid, - .context = context - }; - - *ov = ovdef; - if (!(domain->eq_flags & FI_MR_COMPLETE)) { - ov->cnt = 2; - ov->base.ov.hEvent = CreateEvent(0, TRUE, FALSE, NULL); - } - - hr = mr->mr->lpVtbl->Register(mr->mr, buf, len, ind2flag, &ov->base.ov); - if (FAILED(hr)) { - ofi_nd_mr_ov_free(&ov->base); - goto fn_fail; - } - - if (!(domain->eq_flags & FI_MR_COMPLETE)) { - /* sync memory registration */ - hr = mr->mr->lpVtbl->GetOverlappedResult(mr->mr, &ov->base.ov, TRUE); - if (!InterlockedDecrement(&ov->cnt)) - ofi_nd_mr_ov_free(&ov->base); - if (FAILED(hr)) - goto fn_fail; - mr->fid.key = mr->mr->lpVtbl->GetRemoteToken(mr->mr); - mr->fid.mem_desc = (void *)(uintptr_t)mr->mr->lpVtbl->GetLocalToken(mr->mr); - } - else { - /* async memory registration */ - hr = mr->mr->lpVtbl->Register( - mr->mr, buf, len, ind2flag, &ov->base.ov); - if (FAILED(hr)) { - ofi_nd_mr_ov_free(&ov->base); - goto fn_fail; - } - } - - *pmr = &mr->fid; - - return FI_SUCCESS; - -fn_fail: - ofi_nd_mr_close(&mr->fid.fid); - return H2F(hr); -} - -int ofi_nd_mr_regv(struct fid *fid, const struct iovec *iov, - size_t count, uint64_t access, - uint64_t offset, uint64_t requested_key, - uint64_t flags, struct fid_mr **mr, void *context) -{ - OFI_UNUSED(fid); - OFI_UNUSED(iov); - OFI_UNUSED(count); - OFI_UNUSED(access); - OFI_UNUSED(offset); - OFI_UNUSED(requested_key); - OFI_UNUSED(flags); - OFI_UNUSED(fid); - OFI_UNUSED(mr); - OFI_UNUSED(context); - - /* This functionality wasn't implemented due to impossibility - * to do it by means of ND services. To avoid problems in future, - * just to not implement it until no support from ND */ - - ofi_nd_mr_fini_handler(); - assert(0); - return FI_SUCCESS; -} - -int ofi_nd_mr_regattr(struct fid *fid, const struct fi_mr_attr *attr, - uint64_t flags, struct fid_mr **mr) -{ - OFI_UNUSED(fid); - OFI_UNUSED(attr); - OFI_UNUSED(flags); - OFI_UNUSED(mr); - - ofi_nd_mr_fini_handler(); - assert(0); - return FI_SUCCESS; -} - -static int ofi_nd_mr_close(struct fid *fid) -{ - ND_LOG_DEBUG(FI_LOG_MR, "closing mr\n"); - assert(fid->fclass == FI_CLASS_MR); - if (fid->fclass != FI_CLASS_MR) - return -FI_EINVAL; - - struct nd_mr *mr = container_of(fid, struct nd_mr, fid.fid); - - if (mr->mr) - mr->mr->lpVtbl->Release(mr->mr); - if (mr->wnd) - mr->wnd->lpVtbl->Release(mr->wnd); - - ofi_nd_buf_free_nd_mr(mr); - - return FI_SUCCESS; -} - -static void ofi_nd_mr_ov_free(struct nd_event_base* base) -{ - ofi_nd_mr_ov *ov = container_of(base, ofi_nd_mr_ov, base); - if (ov->base.ov.hEvent && ov->base.ov.hEvent != INVALID_HANDLE_VALUE) - CloseHandle(ov->base.ov.hEvent); - - ofi_nd_buf_free_mr_ov(ov); -} - -static void ofi_nd_mr_ov_event(struct nd_event_base* base, DWORD bytes) -{ - OFI_UNUSED(bytes); - - HRESULT hr; - - ofi_nd_mr_ov *ov = container_of(base, ofi_nd_mr_ov, base); - - if (ov->cnt) { /* this is sync mr reg operation */ - if (!InterlockedDecrement(&ov->cnt)) - ofi_nd_mr_ov_free(&ov->base); - return; - } - - assert(ov->eq); - assert(ov->fid); - assert(ov->fid->fclass == FI_CLASS_MR); - - struct nd_mr *mr = container_of(ov->fid, struct nd_mr, fid.fid); - assert(mr->mr); - mr->fid.key = mr->mr->lpVtbl->GetRemoteToken(mr->mr); - mr->fid.mem_desc = (void *)(uintptr_t)mr->mr->lpVtbl->GetLocalToken(mr->mr); - - struct fi_eq_entry entry = {.fid = ov->fid, .context = ov->context}; - ofi_nd_mr_ov_free(base); - - struct nd_eq_event *err; - struct nd_eq_event *ev = ofi_nd_buf_alloc_nd_eq_event(); - if (!ev) { - hr = ND_NO_MEMORY; - goto fn_fail; - } - memset(ev, 0, sizeof(*ev)); - ev->eq_event = FI_MR_COMPLETE; - ev->operation = entry; - ofi_nd_eq_push(ov->eq, ev); - return; - -fn_fail: - err = ofi_nd_buf_alloc_nd_eq_event(); - if (!err) { - ND_LOG_WARN(FI_LOG_EP_CTRL, - "failed to allocate error event\n"); - return; - } - memset(err, 0, sizeof(*err)); - err->error.err = -H2F(hr); - err->error.prov_errno = (int)hr; - err->error.fid = ov->fid; - ofi_nd_eq_push_err(ov->eq, err); -} - -static void ofi_nd_mr_ov_err(struct nd_event_base* base, DWORD bytes, - DWORD error) -{ - OFI_UNUSED(bytes); - - ofi_nd_mr_ov *ov = container_of(base, ofi_nd_mr_ov, base); - - assert(ov->eq); - assert(ov->fid); - assert(ov->fid->fclass == FI_CLASS_MR); - - struct nd_mr *mr = container_of(ov->fid, struct nd_mr, fid.fid); - assert(mr->mr); - OFI_UNUSED(mr); - - struct fi_eq_err_entry entry = { - .fid = ov->fid, - .context = ov->context, - .err = H2F(error), - .prov_errno = error - }; - - ofi_nd_mr_ov_free(base); - - struct nd_eq_event *err = ofi_nd_buf_alloc_nd_eq_event(); - if (!err) { - ND_LOG_WARN(FI_LOG_EP_CTRL, - "failed to allocate error event\n"); - return; - } - memset(err, 0, sizeof(*err)); - err->error = entry; - ofi_nd_eq_push_err(ov->eq, err); -} - - -#endif /* _WIN32 */ - diff --git a/prov/netdir/src/netdir_ndinit.c b/prov/netdir/src/netdir_ndinit.c deleted file mode 100644 index 945a50d735e..00000000000 --- a/prov/netdir/src/netdir_ndinit.c +++ /dev/null @@ -1,694 +0,0 @@ -/* -* Copyright (c) 2015-2016 Intel Corporation, Inc. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -*/ - -#ifdef _WIN32 - -#include -#include - -#include -#include -#include "ndspi.h" - -#include "netdir.h" -#include "netdir_log.h" - -#ifndef ofi_sizeofaddr -#define ofi_sizeofaddr(address) \ - (address)->sa_family == AF_INET ? \ - sizeof(struct sockaddr_in) : \ - sizeof(struct sockaddr_in6) -#endif - -#define FI_ND_PROTO_FLAG (XP1_GUARANTEED_DELIVERY | XP1_GUARANTEED_ORDER | \ - XP1_MESSAGE_ORIENTED | XP1_CONNECT_DATA) - -static int ofi_nd_startup_done = 0; - -typedef HRESULT(*can_unload_now_t)(void); -typedef HRESULT(*get_class_object_t)(REFCLSID rclsid, REFIID rrid, LPVOID* ppv); - -struct module_t { - const wchar_t *path; - HMODULE module; - can_unload_now_t can_unload_now; - get_class_object_t get_class_object; -}; - -struct factory_t { - WSAPROTOCOL_INFOW protocol; - IClassFactory *class_factory; - IND2Provider *provider; - struct module_t *module; - SOCKET_ADDRESS_LIST *addr_list; -}; - -struct adapter_t { - union { - struct sockaddr addr; - struct sockaddr_in addr4; - struct sockaddr_in6 addr6; - } address; - ND2_ADAPTER_INFO info; - IND2Adapter *adapter; - struct factory_t *factory; - const char *name; -}; - -static struct ofi_nd_infra_t { - struct modules_t { - struct module_t *modules; - size_t count; - } providers; - - struct class_factory_t { - struct factory_t *factory; - size_t count; - } class_factories; - - struct adapters_t { - struct adapter_t *adapter; - size_t count; - } adapters; -} ofi_nd_infra = {0}; - -/* release all objects, do not free strings or arrays */ -static inline void ofi_nd_release_infra() -{ - size_t i; - - if (ofi_nd_infra.adapters.count) { - assert(ofi_nd_infra.adapters.adapter); - for (i = 0; i < ofi_nd_infra.adapters.count; i++) { - struct adapter_t *adapter = &ofi_nd_infra.adapters.adapter[i]; - if (adapter->adapter) { - adapter->adapter->lpVtbl->Release(adapter->adapter); - adapter->adapter = 0; - } - } - } - - if (ofi_nd_infra.class_factories.count) { - assert(ofi_nd_infra.class_factories.factory); - for (i = 0; i < ofi_nd_infra.class_factories.count; i++) { - struct factory_t *factory = &ofi_nd_infra.class_factories.factory[i]; - if (factory->provider) { - factory->provider->lpVtbl->Release(factory->provider); - factory->provider = 0; - } - if (factory->class_factory) { - factory->class_factory->lpVtbl->Release(factory->class_factory); - factory->class_factory = 0; - } - } - } -} - -static inline void ofi_nd_free_infra() -{ - size_t i; - - ofi_nd_release_infra(); - - if (ofi_nd_infra.adapters.count) { - assert(ofi_nd_infra.adapters.adapter); - for (i = 0; i < ofi_nd_infra.adapters.count; i++) { - struct adapter_t *adapter = &ofi_nd_infra.adapters.adapter[i]; - if (adapter->name) { - free((void*)adapter->name); - adapter->name = 0; - } - } - free(ofi_nd_infra.adapters.adapter); - ofi_nd_infra.adapters.adapter = 0; - ofi_nd_infra.adapters.count = 0; - } - - if (ofi_nd_infra.class_factories.count) { - assert(ofi_nd_infra.class_factories.factory); - for (i = 0; i < ofi_nd_infra.class_factories.count; i++) { - struct factory_t *factory = &ofi_nd_infra.class_factories.factory[i]; - assert(factory->module); - if (factory->addr_list) { - free(factory->addr_list); - factory->addr_list = 0; - } - } - free(ofi_nd_infra.class_factories.factory); - ofi_nd_infra.class_factories.factory = 0; - ofi_nd_infra.class_factories.count = 0; - } - - if (ofi_nd_infra.providers.count) { - assert(ofi_nd_infra.providers.modules); - for (i = 0; i < ofi_nd_infra.providers.count; i++) { - struct module_t *module = &ofi_nd_infra.providers.modules[i]; - assert(module->path); - free((void*)module->path); - } - free(ofi_nd_infra.providers.modules); - ofi_nd_infra.providers.modules = 0; - ofi_nd_infra.providers.count = 0; - } -} - -static inline HRESULT ofi_nd_alloc_infra(size_t cnt) -{ - memset(&ofi_nd_infra, 0, sizeof(*(&ofi_nd_infra))); - - ofi_nd_infra.providers.modules = (struct module_t*)malloc(cnt * sizeof(*ofi_nd_infra.providers.modules)); - if (!ofi_nd_infra.providers.modules) - return ND_NO_MEMORY; - - ofi_nd_infra.class_factories.factory = (struct factory_t*)malloc(cnt * sizeof(*ofi_nd_infra.class_factories.factory)); - if (!ofi_nd_infra.class_factories.factory) { - ofi_nd_free_infra(); - return ND_NO_MEMORY; - } - - return S_OK; -} - -static inline wchar_t *ofi_nd_get_provider_path(const WSAPROTOCOL_INFOW *proto) -{ - assert(proto); - - int len = 0, lenex, err, res; - wchar_t *prov, *provex; - - res = WSCGetProviderPath((GUID*)&proto->ProviderId, NULL, &len, &err); - if (err != WSAEFAULT || !len) - return NULL; - - prov = (wchar_t*)malloc(len * sizeof(*prov)); - if (!prov) - return NULL; - - res = WSCGetProviderPath((GUID*)&proto->ProviderId, prov, &len, &err); - if (res) - goto fn1; - - lenex = ExpandEnvironmentStringsW(prov, NULL, 0); - if (!lenex) - goto fn1; - - provex = (wchar_t*)malloc(lenex * sizeof(*provex)); - if (!provex) - goto fn1; - - lenex = ExpandEnvironmentStringsW(prov, provex, lenex); - if (!lenex) - goto fn2; - - free(prov); - return provex; - -fn2: - free(provex); -fn1: - free(prov); - return NULL; -} - -static inline int ofi_nd_is_valid_proto(const WSAPROTOCOL_INFOW *proto) -{ - assert(proto); - - if ((proto->dwServiceFlags1 & FI_ND_PROTO_FLAG) != FI_ND_PROTO_FLAG) - return 0; - - if (!(proto->iAddressFamily == AF_INET || - proto->iAddressFamily == AF_INET6)) - return 0; - - if (proto->iSocketType != -1) - return 0; - - if (proto->iProtocol || proto->iProtocolMaxOffset) - return 0; - return 1; -} - -static inline struct module_t *ofi_nd_search_module(const wchar_t* path) -{ - size_t i; - size_t j; - - for (i = 0; i < ofi_nd_infra.providers.count; i++) { - if (path && ofi_nd_file_exists(path) && - !ofi_nd_is_directory(path)) { - for (j = 0; j < ofi_nd_infra.providers.count; j++) { - if (ofi_nd_is_same_file(path, ofi_nd_infra.providers.modules[j].path)) { - return &ofi_nd_infra.providers.modules[j]; - } - } - } - } - return NULL; -} - -static inline struct module_t *ofi_nd_create_module(const wchar_t* path) -{ - struct module_t *module; - HMODULE hmodule; - can_unload_now_t unload; - get_class_object_t getclass; - - assert(ofi_nd_infra.providers.modules); - - module = ofi_nd_search_module(path); - if (module) - return module; - - /* ok, this is not duplicate. try to - load it and get class factory*/ - hmodule = LoadLibraryW(path); - if (!hmodule) { - ND_LOG_WARN(FI_LOG_CORE, - "ofi_nd_create_module: provider : %S, failed to load: %s\n", - path, ofi_nd_strerror(GetLastError(), 0)); - return NULL; - } - - unload = (can_unload_now_t)GetProcAddress(hmodule, "DllCanUnloadNow"); - getclass = (get_class_object_t)GetProcAddress(hmodule, "DllGetClassObject"); - if (!unload || !getclass) { - ND_LOG_WARN(FI_LOG_CORE, - "ofi_nd_create_module: provider: %S, failed to import interface\n", - path); - goto fn_noiface; - } - - module = &ofi_nd_infra.providers.modules[ofi_nd_infra.providers.count]; - ofi_nd_infra.providers.count++; - - module->path = _wcsdup(path); - module->module = hmodule; - module->can_unload_now = unload; - module->get_class_object = getclass; - - return module; - -fn_noiface: - FreeLibrary(hmodule); - return NULL; -} - -static inline HRESULT ofi_nd_create_factory(const WSAPROTOCOL_INFOW* proto) -{ - wchar_t *path; - struct module_t *module; - IClassFactory* factory; - HRESULT hr; - struct factory_t *ftr; - - assert(proto); - assert(ofi_nd_is_valid_proto(proto)); - assert(ofi_nd_infra.class_factories.factory); - - path = ofi_nd_get_provider_path(proto); - if (path) - ND_LOG_INFO(FI_LOG_CORE, - "ofi_nd_create_factory: provider " FI_ND_GUID_FORMAT " path: %S \n", - FI_ND_GUID_ARG(proto->ProviderId), path); - else /* can't get provider path. just return */ - return S_OK; - - module = ofi_nd_create_module(path); - free(path); - if (!module) - return S_OK; - - assert(module->get_class_object); - hr = module->get_class_object(&proto->ProviderId, &IID_IClassFactory, - (void**)&factory); - if (FAILED(hr)) - return hr; - - ftr = &ofi_nd_infra.class_factories.factory[ofi_nd_infra.class_factories.count]; - ofi_nd_infra.class_factories.count++; - ftr->class_factory = factory; - ftr->module = module; - ftr->protocol = *proto; - - return S_OK; -} - -static int ofi_nd_adapter_cmp(const void *adapter1, const void *adapter2) -{ - return ofi_nd_addr_cmp(&((struct adapter_t*)adapter1)->address, - &((struct adapter_t*)adapter2)->address); -} - -static HRESULT ofi_nd_create_adapter(void) -{ - size_t addr_count = 0; - HRESULT hr; - - for (size_t i = 0; i < ofi_nd_infra.class_factories.count; i++) { - struct factory_t *factory = &ofi_nd_infra.class_factories.factory[i]; - ULONG listsize = 0; - - assert(factory->class_factory); - - hr = factory->class_factory->lpVtbl->CreateInstance(factory->class_factory, - NULL, &IID_IND2Provider, (void**)&factory->provider); - if (FAILED(hr)) - return hr; - hr = factory->provider->lpVtbl->QueryAddressList(factory->provider, NULL, &listsize); - if (hr != ND_BUFFER_OVERFLOW) - return hr; - if (!listsize) { - continue; - } - - factory->addr_list = (SOCKET_ADDRESS_LIST*)malloc(listsize); - if (!factory->addr_list) - return ND_NO_MEMORY; - - hr = factory->provider->lpVtbl->QueryAddressList(factory->provider, - factory->addr_list, &listsize); - if (FAILED(hr)) - return hr; - - for (INT j = 0; j < factory->addr_list->iAddressCount; j++) { - if (ofi_nd_is_valid_addr(factory->addr_list->Address[j].lpSockaddr)) - addr_count++; - } - } - - if (!addr_count) - return E_NOINTERFACE; - - ofi_nd_infra.adapters.adapter = (struct adapter_t*)malloc(addr_count * sizeof(*ofi_nd_infra.adapters.adapter)); - if (!ofi_nd_infra.adapters.adapter) - return ND_NO_MEMORY; - - /* put all available valid addresses into common array */ - for (size_t i = 0; i < ofi_nd_infra.class_factories.count; i++) { - struct factory_t *factory = &ofi_nd_infra.class_factories.factory[i]; - for (INT j = 0; j < factory->addr_list->iAddressCount; j++) { - if (ofi_nd_is_valid_addr(factory->addr_list->Address[j].lpSockaddr)) { - struct adapter_t *adapter = &ofi_nd_infra.adapters.adapter[ofi_nd_infra.adapters.count]; - assert((int)sizeof(adapter->address) >= factory->addr_list->Address[j].iSockaddrLength); - memcpy(&adapter->address, - factory->addr_list->Address[j].lpSockaddr, - factory->addr_list->Address[j].iSockaddrLength); - adapter->factory = factory; - ofi_nd_infra.adapters.count++; - } - } - } - - if (!ofi_nd_infra.adapters.count) - return E_NOINTERFACE; - - /* sort adapters by addresses to set IP4 addresses first. then remove - duplicates */ - qsort(ofi_nd_infra.adapters.adapter, ofi_nd_infra.adapters.count, - sizeof(struct adapter_t), ofi_nd_adapter_cmp); - ofi_nd_infra.adapters.count = unique(ofi_nd_infra.adapters.adapter, - ofi_nd_infra.adapters.count, - sizeof(struct adapter_t), ofi_nd_adapter_cmp); - - for (size_t i = 0; i < ofi_nd_infra.adapters.count; i++) { - struct adapter_t *adapter = &ofi_nd_infra.adapters.adapter[i]; - struct factory_t *factory = adapter->factory; - wchar_t *saddr; - DWORD addrlen = 0; - UINT64 id; - int res; - - assert(factory); - assert(factory->provider); - - assert(adapter->address.addr.sa_family == AF_INET || - adapter->address.addr.sa_family == AF_INET6); - - hr = factory->provider->lpVtbl->ResolveAddress(factory->provider, - &adapter->address.addr, - ofi_sizeofaddr(&adapter->address.addr), &id); - - if (FAILED(hr)) - return hr; - - hr = factory->provider->lpVtbl->OpenAdapter(factory->provider, - &IID_IND2Adapter, id, (void**)&adapter->adapter); - if (FAILED(hr)) - return hr; - - ULONG linfo = sizeof(adapter->info); - adapter->info.InfoVersion = ND_VERSION_2; - hr = adapter->adapter->lpVtbl->Query(adapter->adapter, &adapter->info, &linfo); - if (FAILED(hr) && hr == ND_BUFFER_OVERFLOW) { - ND2_ADAPTER_INFO *info = (ND2_ADAPTER_INFO*)malloc(linfo); - if (!info) - return ND_NO_MEMORY; - info->InfoVersion = ND_VERSION_2; - hr = adapter->adapter->lpVtbl->Query(adapter->adapter, info, &linfo); - if (FAILED(hr)) - return hr; - adapter->info = *info; - free(info); - } else if (FAILED(hr)) { - return hr; - } - - /* generate adapter's name */ - res = WSAAddressToStringW(&adapter->address.addr, - ofi_sizeofaddr(&adapter->address.addr), - NULL, NULL, &addrlen); - if (res == SOCKET_ERROR && WSAGetLastError() == WSAEFAULT && addrlen) { - saddr = (wchar_t*)malloc((addrlen + 1) * sizeof(*saddr)); - WSAAddressToStringW(&adapter->address.addr, - ofi_sizeofaddr(&adapter->address.addr), - NULL, saddr, &addrlen); - } - else { - saddr = _wcsdup(L"unknown"); - } - - asprintf((char**)&adapter->name, "netdir-%S-%S-%p", - ofi_nd_filename(adapter->factory->module->path), - saddr, adapter); - free(saddr); - } - - return S_OK; -} - -static HRESULT ofi_nd_init(ofi_nd_adapter_cb_t cb) -{ - DWORD proto_len = 0; - HRESULT hr = ND_INTERNAL_ERROR; - int i, protonum, err; - size_t j, prov_count = 0; - WSAPROTOCOL_INFOW *proto = 0; - - memset(&ofi_nd_infra, 0, sizeof(ofi_nd_infra)); - - int ret = WSCEnumProtocols(NULL, NULL, &proto_len, &err); - if (ret != SOCKET_ERROR || err != WSAENOBUFS) { - hr = ND_NO_MEMORY; - goto fn_exit; - } - - proto = (WSAPROTOCOL_INFOW*)(malloc(proto_len)); - if (!proto) { - hr = ND_NO_MEMORY; - goto fn_exit; - } - - protonum = WSCEnumProtocols(NULL, proto, &proto_len, &err); - if (protonum == SOCKET_ERROR) { - hr = ND_INTERNAL_ERROR; - goto fn_protofail; - } - - /* calculating number of valid protocols. this number is used - as maximum of existing providers and class factories */ - for (i = 0; i < protonum; i++) { - if (ofi_nd_is_valid_proto(&proto[i])) - prov_count++; - } - - if (!prov_count) { - hr = E_NOINTERFACE; - goto fn_protofail; - } - - hr = ofi_nd_alloc_infra(prov_count); - if (hr != S_OK) - goto fn_protofail; - - for (i = 0; i < protonum; i++) { - if (ofi_nd_is_valid_proto(&proto[i])) - ofi_nd_create_factory(&proto[i]); - } - - free(proto); - - /* ok, factories are created, now list all available addresses, try to - create adapters & collect adapter's info */ - hr = ofi_nd_create_adapter(); - if (FAILED(hr)) - return hr; - - /* free all interfaces. we don't need it right now */ - ofi_nd_release_infra(); - - /* now call cb function to create info's */ - for (j = 0; j < ofi_nd_infra.adapters.count; j++) - cb(&ofi_nd_infra.adapters.adapter[j].info, - ofi_nd_infra.adapters.adapter[j].name); - - return hr; -fn_protofail: - free(proto); -fn_exit: - return hr; -} - -/* we don't need here exclusive execution because this function - * is called from OFI init routine which is single thread */ -HRESULT ofi_nd_startup(ofi_nd_adapter_cb_t cb) -{ - WSADATA data; - HRESULT hr; - int ret; - - assert(cb); - - if (ofi_nd_startup_done) - return S_OK; - - ND_LOG_INFO(FI_LOG_CORE, "ofi_nd_startup: starting initialization\n"); - - ret = WSAStartup(MAKEWORD(2, 2), &data); - if (ret) - return HRESULT_FROM_WIN32(ret); - - ND_LOG_DEBUG(FI_LOG_CORE, "ofi_nd_startup: WSAStartup complete\n"); - - hr = ofi_nd_init(cb); - - ofi_nd_startup_done = 1; - - return hr; -} - -HRESULT ofi_nd_shutdown(void) -{ - if (!ofi_nd_startup_done) - return S_OK; - - ND_LOG_INFO(FI_LOG_CORE, "ofi_nd_shutdown: shutdown WSA\n"); - - ofi_nd_free_infra(); - ofi_nd_startup_done = 0; - - return HRESULT_FROM_WIN32(WSACleanup()); -} - -int ofi_nd_lookup_adapter(const char *name, IND2Adapter **adapter, struct sockaddr** addr) -{ - size_t i; - - assert(name); - assert(adapter); - - if (!ofi_nd_startup_done) - return -FI_EOPBADSTATE; - - for (i = 0; i < ofi_nd_infra.adapters.count; i++) { - struct adapter_t *ada = &ofi_nd_infra.adapters.adapter[i]; - if (ada->name && !strcmp(ada->name, name)) { - HRESULT hr; - UINT64 adapter_id; - IClassFactory* factory = NULL; - IND2Provider *provider = NULL; - - /* ok, we found good adapter. try to initialize it */ - if (ada->adapter) { - *adapter = ada->adapter; - *addr = &ada->address.addr; - ada->adapter->lpVtbl->AddRef(ada->adapter); - return FI_SUCCESS; - } - - assert(ada->factory); - assert(ada->factory->module); - assert(ada->factory->module->get_class_object); - - hr = ada->factory->module->get_class_object( - &ada->factory->protocol.ProviderId, - &IID_IClassFactory, - (void**)&factory); - if (FAILED(hr)) - return H2F(hr); - assert(factory); - - hr = factory->lpVtbl->CreateInstance(factory, NULL, - &IID_IND2Provider, - (void**)&provider); - factory->lpVtbl->Release(factory); - if (FAILED(hr)) - return H2F(hr); - assert(provider); - - hr = provider->lpVtbl->ResolveAddress(provider, &ada->address.addr, - ofi_sizeofaddr(&ada->address.addr), - &adapter_id); - if (FAILED(hr)) { - provider->lpVtbl->Release(provider); - return H2F(hr); - } - - hr = provider->lpVtbl->OpenAdapter(provider, &IID_IND2Adapter, adapter_id, - (void**)&ada->adapter); - provider->lpVtbl->Release(provider); - if (FAILED(hr)) - return H2F(hr); - - *adapter = ada->adapter; - *addr = &ada->address.addr; - ada->adapter->lpVtbl->AddRef(ada->adapter); - - return FI_SUCCESS; - } - } - - return -FI_EINVAL; -} - -#endif /* _WIN32 */ - diff --git a/prov/netdir/src/netdir_ov.c b/prov/netdir/src/netdir_ov.c deleted file mode 100644 index c0736be2fa5..00000000000 --- a/prov/netdir/src/netdir_ov.c +++ /dev/null @@ -1,159 +0,0 @@ -/* -* Copyright (c) 2015-2016 Intel Corporation, Inc. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -*/ - -#ifdef _WIN32 - -#include -#define WIN32_NO_STATUS - -#include "netdir_ov.h" -#include "netdir_log.h" -#include "netdir_util.h" -#include "netdir_iface.h" - -#include "netdir_queue.h" - -OFI_ND_NB_BUF_IMP(nd_eq_event); -OFI_ND_NB_BUF_IMP(nd_buf_fini); - -volatile nd_buf_fini *nd_buf_fini_head = 0; - -LONG nd_async_progress = 0; - -void CALLBACK domain_io_cb(DWORD err, DWORD bytes, LPOVERLAPPED ov) -{ - assert(ov); - - InterlockedIncrement(&nd_async_progress); - - nd_event_base *base = container_of(ov, nd_event_base, ov); - - ND_LOG_DEBUG(FI_LOG_EP_CTRL, - "IO callback: err: %s, bytes: %d\n", - ofi_nd_error_str(err), bytes); - - if (err) { - assert(base->err_cb); - base->err_cb(base, bytes, err); - } - else { - assert(base->event_cb); - base->event_cb(base, bytes); - } - - InterlockedDecrement(&nd_async_progress); - - return; -} - -static void ofi_nd_util_mr_ov_free(struct nd_event_base* base) -{ - assert(base); - ofi_nd_util_ov *ov = container_of(base, ofi_nd_util_ov, base); - ov->cnt = 2; - ResetEvent(ov->base.ov.hEvent); - ND_BUF_FREE(ofi_nd_util_ov, ov); -} - -static void ofi_nd_util_mr_ov_event(struct nd_event_base* base, DWORD bytes) -{ - OFI_UNUSED(bytes); - - ofi_nd_util_ov *ov = container_of(base, ofi_nd_util_ov, base); - - if (!InterlockedDecrement(&ov->cnt)) - ov->base.free(&ov->base); -} - -static void ofi_nd_util_mr_ov_err(struct nd_event_base* base, DWORD bytes, - DWORD error) -{ - OFI_UNUSED(bytes); - OFI_UNUSED(error); - - ofi_nd_util_ov *ov = container_of(base, ofi_nd_util_ov, base); - - if (!InterlockedDecrement(&ov->cnt)) - ov->base.free(&ov->base); -} - -static ND_BUF_CHUNK(ofi_nd_util_ov) -*ofi_nd_alloc_ov_chunk(ND_BUF_FOOTER(ofi_nd_util_ov) *footer, size_t *count) -{ - OFI_UNUSED(footer); - - ND_BUF_CHUNK(ofi_nd_util_ov) *chunk = malloc(sizeof(*chunk)); - if (!chunk) - return 0; - assert(count); - *count = countof(chunk->item); - memset(chunk, 0, sizeof(*chunk)); - - size_t i; - for (i = 0; i < countof(chunk->item); i++) { - chunk->item[i].data.cnt = 2; - chunk->item[i].data.base.free = ofi_nd_util_mr_ov_free; - chunk->item[i].data.base.event_cb = ofi_nd_util_mr_ov_event; - chunk->item[i].data.base.err_cb = ofi_nd_util_mr_ov_err; - chunk->item[i].data.base.ov.hEvent = CreateEvent(0, TRUE, FALSE, 0); - if (!chunk->item[i].data.base.ov.hEvent || - chunk->item[i].data.base.ov.hEvent == INVALID_HANDLE_VALUE) - goto fn_fail; - } - - return chunk; - -fn_fail: - for (i = 0; i < countof(chunk->item); i++) { - if (chunk->item[i].data.base.ov.hEvent && - chunk->item[i].data.base.ov.hEvent != INVALID_HANDLE_VALUE) - CloseHandle(chunk->item[i].data.base.ov.hEvent); - } - free(chunk); - return 0; -} - -static void ofi_nd_free_ov_chunk(struct nd_buf_chunk_ofi_nd_util_ov *chunk) -{ - assert(chunk); - size_t i; - for(i = 0; i < countof(chunk->item); i++) - if (chunk->item[i].data.base.ov.hEvent && - chunk->item[i].data.base.ov.hEvent != INVALID_HANDLE_VALUE) - CloseHandle(chunk->item[i].data.base.ov.hEvent); - free(chunk); -} - -OFI_ND_NB_BUF_IMP_ALLOC(ofi_nd_util_ov, ofi_nd_alloc_ov_chunk, ofi_nd_free_ov_chunk); - -#endif /* _WIN32 */ - diff --git a/prov/netdir/src/netdir_ov.h b/prov/netdir/src/netdir_ov.h deleted file mode 100644 index ec002913bc8..00000000000 --- a/prov/netdir/src/netdir_ov.h +++ /dev/null @@ -1,249 +0,0 @@ -/* -* Copyright (c) 2015-2016 Intel Corporation, Inc. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -*/ - -#ifndef _FI_NETDIR_OV_H_ -#define _FI_NETDIR_OV_H_ - -#include -#include -#include - -#include "ndspi.h" - -#include "rdma/fabric.h" -#include "ofi_mem.h" - -#include "netdir.h" -#include "netdir_buf.h" -#include "netdir_log.h" -#include "netdir_iface.h" -#include "netdir_queue.h" - -#include "rdma/fi_eq.h" - -#ifdef __cplusplus -extern "C" { -#endif /* __cplusplus */ - -typedef enum ofi_nd_cq_state { - NORMAL_STATE = 0, - LARGE_MSG_RECV_REQ = 1, - LARGE_MSG_WAIT_ACK = 2, - MAX_STATE = 3 -} ofi_nd_cq_state; - -typedef enum ofi_nd_cq_event { - NORMAL_EVENT = 0, - LARGE_MSG_REQ = 1, - LARGE_MSG_ACK = 2, - MAX_EVENT = 3 -} ofi_nd_cq_event; - -typedef struct nd_eq_event { - OVERLAPPED ov; - int is_custom; - uint32_t eq_event; - union { - struct fi_eq_entry operation; - /* fi_eq_cm_entry could not be used here because it has - incomplete size */ - /*struct fi_eq_cm_entry connection;*/ - struct fi_eq_err_entry error; - }; - - /* connection data */ - void *data; - size_t len; -} nd_eq_event; - -typedef struct nd_send_entry nd_send_entry; - -typedef struct nd_cq_entry { - nd_event_base base; - struct nd_domain *domain; - struct nd_msgprefix *prefix; - struct nd_inlinebuf *inline_buf; - struct nd_notifybuf *notify_buf; - struct iovec iov[ND_MSG_IOV_LIMIT]; - size_t iov_cnt; - - /* used for RMA operations */ - size_t mr_count; - IND2MemoryRegion *mr[ND_MSG_IOV_LIMIT]; - ND2_RESULT result; - - uint64_t flags; - uint64_t seq; - void* buf; - size_t len; - uint64_t data; - struct nd_queue_item queue_item; - int completed; - void* context; - - struct { - struct nd_msg_location *locations; - /* != 0 only in case of large message - * receiving via RMA read */ - size_t count; - } rma_location; - struct { - /* these parameters are specified in - * parent's CQ entry to wait until all - * read/write operation will be completed */ - size_t comp_count; - size_t total_count; - - CRITICAL_SECTION comp_lock; - } wait_completion; - struct nd_cq_entry *aux_entry; - - ofi_nd_cq_state state; - ofi_nd_cq_event event; - nd_flow_cntrl_flags flow_cntrl_flags; - nd_send_entry *send_entry; -} nd_cq_entry; - -typedef struct nd_sge { - ND2_SGE entries[256]; - ULONG count; -} nd_sge; - -struct nd_send_entry { - struct nd_queue_item queue_item; - nd_sge *sge; - nd_cq_entry *cq_entry; - nd_cq_entry *prepost_entry; - struct nd_ep *ep; -}; - -typedef struct nd_buf_fini { - volatile struct nd_buf_fini *next; - void(*fini)(void); -} nd_buf_fini; - -#define ND_FI_CONTEXT(ptr) ((struct fi_context*)(ptr))->internal[0] - -OFI_ND_NB_BUF(nd_buf_fini); -OFI_ND_NB_BUF(nd_eq_event); -OFI_ND_NB_BUF(nd_cq_entry); -OFI_ND_NB_BUF(nd_sge); -OFI_ND_NB_BUF(nd_send_entry); - -extern LONG nd_async_progress; -extern volatile nd_buf_fini *nd_buf_fini_head; - -static inline void nd_buf_register_fini(void(*fini)(void)) -{ - assert(fini); - nd_buf_fini *fin = ND_BUF_ALLOC(nd_buf_fini); - if (fin) { - fin->fini = fini; - do { - fin->next = nd_buf_fini_head; - } while (InterlockedCompareExchangePointer( - (PVOID *)&nd_buf_fini_head, - fin, (PVOID)fin->next) != fin->next); - } - else { - ND_LOG_WARN(FI_LOG_CORE, "failed to allocate finalizer\n"); - } -} - -static inline void nd_buf_fini_apply() -{ - volatile nd_buf_fini *next = nd_buf_fini_head; - while (next) { - volatile nd_buf_fini *current = next; - next = current->next; - assert(current->fini); - current->fini(); - ND_BUF_FREE(nd_buf_fini, ((nd_buf_fini*)current)); - } -} - -#define ND_REGISTER_FINI(fini) \ -do { \ - static LONG init_done = 0; \ - if (!init_done) { \ - if (!InterlockedExchange(&init_done, 1)) \ - nd_buf_register_fini(fini); \ - } \ -} while (0) - -OFI_ND_NB_BUF(nd_event_base); - -static inline void ofi_nd_eq_free_event(nd_eq_event *ev) -{ - assert(ev); - - if (ev->data) - free(ev->data); - if (ev->eq_event == FI_CONNREQ) { - struct fi_eq_cm_entry *cm = (struct fi_eq_cm_entry*)&ev->operation; - if (cm->info) - fi_freeinfo(cm->info); - } - - ND_BUF_FREE(nd_eq_event, ev); -} - -void CALLBACK domain_io_cb(DWORD err, DWORD bytes, LPOVERLAPPED ov); - -static inline void ofi_nd_eq_push(struct nd_eq *eq, struct nd_eq_event *ev) -{ - assert(eq); - assert(ev); - - assert(eq->iocp); - PostQueuedCompletionStatus(eq->iocp, 0, 0, &ev->ov); - InterlockedIncrement(&eq->count); - WakeByAddressAll((void*)&eq->count); -} - -static inline void ofi_nd_eq_push_err(struct nd_eq *eq, struct nd_eq_event *ev) -{ - assert(eq); - assert(ev); - - assert(eq->err); - PostQueuedCompletionStatus(eq->err, 0, 0, &ev->ov); - InterlockedIncrement(&eq->count); - WakeByAddressAll((void*)&eq->count); -} - -#ifdef __cplusplus -} -#endif /* __cplusplus */ - -#endif /* _FI_NETDIR_OV_H_ */ - diff --git a/prov/netdir/src/netdir_pep.c b/prov/netdir/src/netdir_pep.c deleted file mode 100644 index 208fd024824..00000000000 --- a/prov/netdir/src/netdir_pep.c +++ /dev/null @@ -1,481 +0,0 @@ -/* -* Copyright (c) 2015-2016 Intel Corporation, Inc. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -*/ - -#ifdef _WIN32 - -#include -#include -#include - -#include "netdir.h" - -#include "ofi.h" -#include "ofi_osd.h" -#include "ofi_util.h" - -#include "netdir_ov.h" -#include "netdir_log.h" -#include "netdir_iface.h" - -static int ofi_nd_pep_getname(fid_t fid, void *addr, size_t *addrlen); -static int ofi_nd_pep_close(struct fid *fid); -static int ofi_nd_pep_listen(struct fid_pep *pep); -static int ofi_nd_pep_bind(struct fid *fid, struct fid *bfid, uint64_t flags); -static int ofi_nd_pep_reject(struct fid_pep *ppep, fid_t handle, - const void *param, size_t paramlen); - -static void ofi_nd_pep_connreq_free(nd_event_base *base); -static void ofi_nd_pep_connreq(nd_event_base *base, DWORD bytes); -static void ofi_nd_pep_connreq_err(nd_event_base *base, DWORD err, - DWORD bytes); -extern int ofi_nd_ep_getopt(struct fid *ep, int level, int optname, - void *optval, size_t *optlen); - -static struct fi_ops ofi_nd_fi_ops = { - .size = sizeof(ofi_nd_fi_ops), - .close = ofi_nd_pep_close, - .bind = ofi_nd_pep_bind, - .control = fi_no_control, - .ops_open = fi_no_ops_open, -}; - -static struct fid ofi_nd_fid = { - .fclass = FI_CLASS_PEP, - .context = NULL, - .ops = &ofi_nd_fi_ops -}; - -static struct fi_ops_cm ofi_nd_cm_ops = { - .size = sizeof(struct fi_ops_cm), - .setname = fi_no_setname, - .getname = ofi_nd_pep_getname, - .getpeer = fi_no_getpeer, - .connect = fi_no_connect, - .listen = ofi_nd_pep_listen, - .accept = fi_no_accept, - .reject = ofi_nd_pep_reject, - .shutdown = fi_no_shutdown, - .join = fi_no_join, -}; - -static struct fi_ops_ep ofi_nd_pep_ops = { - .size = sizeof(ofi_nd_pep_ops), - .cancel = fi_no_cancel, - .getopt = ofi_nd_ep_getopt, - .setopt = fi_no_setopt, - .tx_ctx = fi_no_tx_ctx, - .rx_ctx = fi_no_rx_ctx, - .rx_size_left = fi_no_rx_size_left, - .tx_size_left = fi_no_tx_size_left, -}; - -typedef struct nd_pep_connreq { - nd_event_base base; - struct nd_eq *eq; - struct fi_info *info; - IND2Connector *connector; - fid_t fid; -} nd_pep_connreq; - -static nd_event_base nd_pep_connreq_base_def = { - .free = ofi_nd_pep_connreq_free, - .event_cb = ofi_nd_pep_connreq, - .err_cb = ofi_nd_pep_connreq_err -}; - -OFI_ND_NB_BUF(nd_pep_connreq); -OFI_ND_NB_BUF_IMP(nd_pep_connreq); -OFI_ND_NB_BUF_TYPED(nd_connreq, struct nd_connreq); -OFI_ND_NB_BUF_IMP(nd_connreq); - -int ofi_nd_passive_endpoint(struct fid_fabric *fabric, struct fi_info *info, - struct fid_pep **ppep, void *context) -{ - OFI_UNUSED(context); - OFI_UNUSED(fabric); - - ND_REGISTER_FINI(ND_BUF_FINIPTR(nd_connreq)); - - assert(info); - assert(fabric); - assert(fabric->fid.fclass == FI_CLASS_FABRIC); - - struct nd_pep *pep = (struct nd_pep*)calloc(1, sizeof(*pep)); - if (!pep) - return -FI_ENOMEM; - - struct nd_pep def = { - .fid = { - .fid = { - .fclass = FI_CLASS_PEP, - .context = context, - .ops = &ofi_nd_fi_ops - }, - .ops = &ofi_nd_pep_ops, - .cm = &ofi_nd_cm_ops - }, - .info = fi_dupinfo(info) - }; - - *pep = def; - *ppep = &pep->fid; - - return FI_SUCCESS; -} - -static int ofi_nd_pep_getname(fid_t fid, void *addr, size_t *addrlen) -{ - assert(fid && fid->fclass == FI_CLASS_PEP); - - if (fid->fclass != FI_CLASS_PEP) - return -FI_EINVAL; - - HRESULT hr; - ULONG len = (ULONG)*addrlen; - struct nd_pep *pep = container_of(fid, struct nd_pep, fid.fid); - - if (!pep->listener) - return -FI_EOPBADSTATE; - - hr = pep->listener->lpVtbl->GetLocalAddress(pep->listener, - (struct sockaddr *)addr, - &len); - - if (*addrlen < len) { - ND_LOG_INFO(FI_LOG_EP_CTRL, - "Provided buffer (size = %"PRIu64") is too small, required = %"PRIu64, - addrlen, len); - *addrlen = (size_t)len; - return -FI_ETOOSMALL; - } - *addrlen = (size_t)len; - - return H2F(hr); -} - -static int ofi_nd_pep_close(struct fid *fid) -{ - assert(fid); - assert(fid->fclass == FI_CLASS_PEP); - - struct nd_pep *pep = container_of(fid, struct nd_pep, fid.fid); - - int ref; - if (pep->listener) { - ref = (int)pep->listener->lpVtbl->Release(pep->listener); - ND_LOG_DEBUG(FI_LOG_EP_CTRL, "pep->listener ref count: %d\n", ref); - } - if (pep->adapter) { - ref = (int)pep->adapter->lpVtbl->Release(pep->adapter); - ND_LOG_DEBUG(FI_LOG_EP_CTRL, "pep->adapter ref count: %d\n", ref); - } - if (pep->adapter_file && pep->adapter_file != INVALID_HANDLE_VALUE) - CloseHandle(pep->adapter_file); - if (pep->info) - fi_freeinfo(pep->info); - - free(pep); - - return FI_SUCCESS; -} - -static void ofi_nd_pep_connreq_free(nd_event_base *base) -{ - assert(base); - - nd_pep_connreq *connreq = container_of(base, nd_pep_connreq, base); - if (connreq->connector) - connreq->connector->lpVtbl->Release(connreq->connector); - ofi_nd_buf_free_nd_pep_connreq(connreq); -} - -static void ofi_nd_pep_connreq(nd_event_base *base, DWORD bytes) -{ - assert(base); - OFI_UNUSED(bytes); - - HRESULT hr; - ULONG len; - nd_pep_connreq *connreq = container_of(base, nd_pep_connreq, base); - struct nd_eq_event *err = 0; - - assert(connreq->connector); - assert(connreq->eq); - assert(connreq->fid); - assert(connreq->info); - - struct nd_eq_event *ev = ND_BUF_ALLOC(nd_eq_event); - if (!ev) { - ND_LOG_WARN(FI_LOG_EP_CTRL, "failed to allocate event\n"); - hr = ND_NO_MEMORY; - goto fn_fail_ev; - } - memset(ev, 0, sizeof(*ev)); - - ev->eq_event = FI_CONNREQ; - - struct fi_eq_cm_entry *cmev = (struct fi_eq_cm_entry*)&ev->operation; - cmev->fid = connreq->fid; - cmev->info = fi_dupinfo(connreq->info); - if (!cmev->info) { - ND_LOG_WARN(FI_LOG_EP_CTRL, "failed to copy info\n"); - hr = ND_NO_MEMORY; - goto fn_fail; - } - - struct nd_connreq *handle = ND_BUF_ALLOC(nd_connreq); - if (!handle) { - ND_LOG_WARN(FI_LOG_EP_CTRL, "failed to allocate handle\n"); - hr = ND_NO_MEMORY; - goto fn_fail; - } - memset(handle, 0, sizeof(*handle)); - handle->handle.fclass = FI_CLASS_CONNREQ; - handle->connector = connreq->connector; - handle->connector->lpVtbl->AddRef(handle->connector); - cmev->info->handle = &handle->handle; - - hr = connreq->connector->lpVtbl->GetPrivateData( - connreq->connector, NULL, &len); - if (FAILED(hr) && hr != ND_BUFFER_OVERFLOW) { - ND_LOG_WARN(FI_LOG_EP_CTRL, "failed to get private data\n"); - goto fn_fail_handle; - } - - if (len) { - ev->data = malloc(len); - if (!ev->data) { - ND_LOG_WARN(FI_LOG_EP_CTRL, "failed to allocate private data\n"); - ev->len = 0; - goto fn_fail_handle; - } - - hr = connreq->connector->lpVtbl->GetPrivateData( - connreq->connector, ev->data, &len); - if (FAILED(hr)) { - ND_LOG_WARN(FI_LOG_EP_CTRL, "failed to copy private data\n"); - free(ev->data); - ev->len = 0; - goto fn_fail_handle; - } - } - ev->len = (size_t)len; - - ofi_nd_eq_push(connreq->eq, ev); - ofi_nd_pep_connreq_free(&connreq->base); - return; - -fn_fail_handle: - handle->connector->lpVtbl->Release(handle->connector); - ND_BUF_FREE(nd_connreq, handle); -fn_fail: - ofi_nd_eq_free_event(ev); -fn_fail_ev: - err = ND_BUF_ALLOC(nd_eq_event); - if (!err) { - ND_LOG_WARN(FI_LOG_EP_CTRL, "failed to allocate error\n"); - return; - } - memset(err, 0, sizeof(*err)); - err->error.err = -H2F(hr); - err->error.prov_errno = (int)hr; - err->error.fid = connreq->fid; - ofi_nd_eq_push_err(connreq->eq, err); - ofi_nd_pep_connreq_free(&connreq->base); -} - -static void ofi_nd_pep_connreq_err(nd_event_base *base, DWORD error, DWORD bytes) -{ - assert(base); - OFI_UNUSED(bytes); - - nd_pep_connreq *connreq = container_of(base, nd_pep_connreq, base); - struct nd_eq_event *err = 0; - - assert(connreq->connector); - assert(connreq->eq); - assert(connreq->fid); - assert(connreq->info); - - err = ofi_nd_buf_alloc_nd_eq_event(); - - if (!err) { - ND_LOG_WARN(FI_LOG_EP_CTRL, "failed to allocate error\n"); - return; - } - memset(err, 0, sizeof(*err)); - err->error.err = FI_EOTHER; - err->error.prov_errno = (int)error; - err->error.fid = connreq->fid; - ofi_nd_eq_push_err(connreq->eq, err); - ofi_nd_pep_connreq_free(&connreq->base); -} - -static int ofi_nd_pep_listen(struct fid_pep *ppep) -{ - assert(ppep); - - int res = FI_SUCCESS; - HRESULT hr; - - if (ppep->fid.fclass != FI_CLASS_PEP) - return -FI_EINVAL; - - struct nd_pep *pep = container_of(ppep, struct nd_pep, fid); - - assert(pep->info); - assert(pep->info->domain_attr); - assert(pep->info->domain_attr->name); - - struct sockaddr* addr; - - if (!pep->adapter) { - struct sockaddr* listen_addr = NULL; - size_t listen_addr_len = 0; - - res = ofi_nd_lookup_adapter(pep->info->domain_attr->name, - &pep->adapter, &addr); - if (res != FI_SUCCESS) - return res; - assert(pep->adapter); - - hr = pep->adapter->lpVtbl->CreateOverlappedFile(pep->adapter, - &pep->adapter_file); - if (FAILED(hr)) - return H2F(hr); - assert(pep->adapter_file && - pep->adapter_file != INVALID_HANDLE_VALUE); - - BindIoCompletionCallback(pep->adapter_file, domain_io_cb, 0); - - hr = pep->adapter->lpVtbl->CreateListener(pep->adapter, - &IID_IND2Listener, - pep->adapter_file, - (void**)&pep->listener); - if (FAILED(hr)) - return H2F(hr); - assert(pep->listener); - - if (pep->info->src_addr) { - /* uses address that is specified in fi_info */ - listen_addr = pep->info->src_addr; - listen_addr_len = pep->info->src_addrlen; - } - else { - /* uses address on which provider are open */ - listen_addr = addr; - listen_addr_len = ofi_sizeofaddr(addr); - } - - hr = pep->listener->lpVtbl->Bind(pep->listener, - listen_addr, - (ULONG)sizeof(*listen_addr)); - if (FAILED(hr)) - return H2F(hr); - - hr = pep->listener->lpVtbl->Listen(pep->listener, 0); - if (FAILED(hr)) - return H2F(hr); - } - assert(pep->adapter); - - ND_REGISTER_FINI(ND_BUF_FINIPTR(nd_pep_connreq)); - - nd_pep_connreq *conn = ofi_nd_buf_alloc_nd_pep_connreq(); - if (!conn) - return -FI_ENOMEM; - memset(conn, 0, sizeof(*conn)); - - conn->base = nd_pep_connreq_base_def; - - hr = pep->adapter->lpVtbl->CreateConnector(pep->adapter, - &IID_IND2Connector, - pep->adapter_file, - (void**)&conn->connector); - if (FAILED(hr)) - return H2F(hr); - - conn->eq = pep->eq; - conn->info = pep->info; - conn->fid = &pep->fid.fid; - - hr = pep->listener->lpVtbl->GetConnectionRequest(pep->listener, - (IUnknown*)conn->connector, - &conn->base.ov); - if (FAILED(hr)) { - ND_LOG_WARN(FI_LOG_EP_CTRL, "failed to get connection request\n"); - } - - return H2F(hr); -} - -static int ofi_nd_pep_bind(struct fid *fid, struct fid *bfid, uint64_t flags) -{ - OFI_UNUSED(flags); - - if (fid->fclass != FI_CLASS_PEP) - return -FI_EINVAL; - if (bfid->fclass != FI_CLASS_EQ) - return -FI_EINVAL; - - struct nd_pep *pep = container_of(fid, struct nd_pep, fid.fid); - struct nd_eq *eq = container_of(bfid, struct nd_eq, fid.fid); - - pep->eq = eq; - - return FI_SUCCESS; -} - -static int ofi_nd_pep_reject(struct fid_pep *ppep, fid_t handle, - const void *param, size_t paramlen) -{ - assert(ppep); - - if (ppep->fid.fclass != FI_CLASS_PEP) - return -FI_EINVAL; - if (handle->fclass != FI_CLASS_CONNREQ) - return -FI_EINVAL; - - struct nd_connreq *connreq = container_of(handle, struct nd_connreq, handle); - - assert(connreq->connector); - connreq->connector->lpVtbl->Reject(connreq->connector, param, - (ULONG)paramlen); - - connreq->connector->lpVtbl->Release(connreq->connector); - ofi_nd_buf_free_nd_connreq(connreq); - - return FI_SUCCESS; -} - -#endif /* _WIN32 */ - diff --git a/prov/netdir/src/netdir_queue.h b/prov/netdir/src/netdir_queue.h deleted file mode 100644 index 2eebd9ddab9..00000000000 --- a/prov/netdir/src/netdir_queue.h +++ /dev/null @@ -1,187 +0,0 @@ -/* -* Copyright (c) 2015-2016 Intel Corporation, Inc. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -*/ - -#ifndef _FI_NETDIR_QUEUE_H_ -#define _FI_NETDIR_QUEUE_H_ - -#include - -#include "rdma/fabric.h" - -#include "ofi.h" -#include "ofi_osd.h" - -#include "netdir.h" - -#ifdef __cplusplus -extern "C" { -#endif /* __cplusplus */ - -struct nd_queue_item { - struct nd_queue_item *next; -}; - -__declspec(align(16)) struct nd_queue_queue { - union { - struct { - struct nd_queue_item *head; - struct nd_queue_item *tail; - }; - volatile LONG64 exchange[2]; - }; -}; - -/* push front call is non-blocking thread safe */ -static inline void ofi_nd_queue_push_front(struct nd_queue_queue *queue, - struct nd_queue_item *item) -{ - assert(queue); - - item->next = 0; - BOOLEAN success; - - struct { - struct nd_queue_item *head; - struct nd_queue_item *tail; - } src; - - do { - src.head = queue->head; - src.tail = queue->tail; - - LONG64 head = (LONG64)(src.head ? src.head : item); - LONG64 tail = (LONG64)item; - __declspec(align(16)) LONG64 compare[2] = { (LONG64)src.head, (LONG64)src.tail }; - success = InterlockedCompareExchange128( - queue->exchange, tail, head, compare); - } while (!success); - - if (src.tail) { - item->next = src.head; - src.head = item; - WakeByAddressAll(&src.head); - } -} - -/* push call is non-blocking thread safe */ -static inline void ofi_nd_queue_push(struct nd_queue_queue *queue, - struct nd_queue_item *item) -{ - assert(queue); - - item->next = 0; - BOOLEAN success; - - struct { - struct nd_queue_item *head; - struct nd_queue_item *tail; - } src; - - do { - src.head = queue->head; - src.tail = queue->tail; - - LONG64 head = (LONG64)(src.head ? src.head : item); - LONG64 tail = (LONG64)item; - __declspec(align(16)) LONG64 compare[2] = {(LONG64)src.head, (LONG64)src.tail}; - success = InterlockedCompareExchange128( - queue->exchange, tail, head, compare); - } while (!success); - - if (src.tail) { - src.tail->next = item; - WakeByAddressAll(&src.tail->next); - } -} - -/* pop call is NOT thread safe, it allows only one consumer, but it is - safe to be used with push operation without locks */ -static inline int ofi_nd_queue_pop(struct nd_queue_queue *queue, - struct nd_queue_item **item) -{ - assert(queue); - assert(item); - - BOOLEAN success; - struct { - struct nd_queue_item *head; - struct nd_queue_item *tail; - } src; - - do { - src.head = queue->head; - src.tail = queue->tail; - - if (!src.head) - return 0; - - /* here is potential thread race: object located at src.head - may be destroyed while we're waiting. that is why pop - operation is not thread safe */ - if (src.head != src.tail) { - /* in case if head and tail are not same - ensure that - head->next element is not NULL */ - void *zero = NULL; - while (!src.head->next) { - WaitOnAddress(&src.head->next, &zero, sizeof(zero), INFINITE); - } - } - - LONG64 head = (LONG64)src.head->next; - LONG64 tail = (LONG64)(src.head != src.tail ? src.tail : NULL); - __declspec(align(16)) LONG64 compare[2] = {(LONG64)src.head, (LONG64)src.tail}; - success = InterlockedCompareExchange128( - queue->exchange, tail, head, compare); - } while (!success); - - *item = src.head; - - return (*item) != NULL; -} - -/* peek call is NOT thread safe, it allows only one consumer */ -static inline int ofi_nd_queue_peek(struct nd_queue_queue *queue, - struct nd_queue_item **item) -{ - assert(queue); - assert(item); - - *item = queue->head; - return (*item) != 0; -} - -#ifdef __cplusplus -} -#endif /* __cplusplus */ - -#endif /* _FI_NETDIR_QUEUE_H_ */ - diff --git a/prov/netdir/src/netdir_unexp.c b/prov/netdir/src/netdir_unexp.c deleted file mode 100644 index 76b4a69d06c..00000000000 --- a/prov/netdir/src/netdir_unexp.c +++ /dev/null @@ -1,532 +0,0 @@ -/* -* Copyright (c) 2015-2016 Intel Corporation, Inc. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -*/ - -#ifdef _WIN32 - -#include -#define WIN32_NO_STATUS - -#include "ndspi.h" - -#include "netdir.h" -#include "netdir_cq.h" -#include "netdir_log.h" -#include "netdir_util.h" -#include "netdir_queue.h" -#include "netdir_iface.h" -#include "netdir_queue.h" -#include "netdir_unexp.h" - -#include - -#define PREPOSTLEN (sizeof(nd_unexpected_buf) + gl_data.inline_thr) - -static ND_BUF_CHUNK(nd_unexpected_entry) *ofi_nd_unexp_alloc_chunk( - ND_BUF_FOOTER(nd_unexpected_entry) *footer, size_t* count); -static void ofi_nd_unexp_free_chunk(ND_BUF_CHUNK(nd_unexpected_entry) *chunk); - -OFI_ND_NB_BUF(nd_unexpected_ctx); -OFI_ND_NB_BUF(nd_unexpected_entry); -OFI_ND_NB_BUF_IMP(nd_unexpected_ctx); -OFI_ND_NB_BUF_IMP_ALLOC(nd_unexpected_entry, - ofi_nd_unexp_alloc_chunk, - ofi_nd_unexp_free_chunk); - -HRESULT ofi_nd_unexp_init(struct nd_ep *ep) -{ - assert(ep); - assert(ep->fid.fid.fclass == FI_CLASS_EP); - - ND_REGISTER_FINI(ND_BUF_FINIPTR(nd_unexpected_ctx)); - ND_REGISTER_FINI(ND_BUF_FINIPTR(nd_unexpected_entry)); - - int i; - HRESULT hr; - int total_count = (gl_data.prepost_cnt + - gl_data.flow_control_cnt) * gl_data.prepost_buf_cnt; - - if (ep->unexpected.active) - return S_OK; - - dlist_init(&ep->unexpected.received); - - ep->unexpected.unexpected = malloc( - total_count * sizeof(*ep->unexpected.unexpected) - ); - if (!ep->unexpected.unexpected) - return ND_NO_MEMORY; - - size_t len = PREPOSTLEN * total_count; - - char* tmp = (char*)calloc(1, len); - if (!tmp) { - hr = ND_NO_MEMORY; - goto free_unexp; - } - - hr = ep->domain->adapter->lpVtbl->CreateMemoryRegion( - ep->domain->adapter, &IID_IND2MemoryRegion, - ep->domain->adapter_file, (void**)&ep->unexpected.mr); - if (FAILED(hr)) - goto free_tmp; - - hr = ofi_nd_util_register_mr(ep->unexpected.mr, tmp, - len, ND_MR_FLAG_ALLOW_LOCAL_WRITE); - if (FAILED(hr)) - goto release; - - ep->unexpected.token = - ep->unexpected.mr->lpVtbl->GetLocalToken(ep->unexpected.mr); - - for (i = 0; i < total_count; i++) { - ep->unexpected.unexpected[i] = - (struct nd_unexpected_buf*)(tmp + (PREPOSTLEN * i)); - } - - InterlockedIncrement(&ep->unexpected.active); - - return S_OK; - -release: - ep->unexpected.mr->lpVtbl->Release(ep->unexpected.mr); -free_tmp: - free(tmp); -free_unexp: - free(ep->unexpected.unexpected); - ep->unexpected.unexpected = NULL; - return hr; -} - -HRESULT ofi_nd_unexp_fini(struct nd_ep *ep) -{ - assert(ep); - assert(ep->fid.fid.fclass == FI_CLASS_EP); - - int total_count = (gl_data.prepost_cnt + - gl_data.flow_control_cnt) * gl_data.prepost_buf_cnt; - - if (InterlockedDecrement(&ep->unexpected.active)) - return S_OK; - - - ND_LOG_INFO(FI_LOG_EP_CTRL, "finalize unexpected queue\n" - "total_count = %d\n", - total_count); - - InterlockedAdd(&ep->shutdown, total_count); - if (ep->qp) { - ep->qp->lpVtbl->Flush(ep->qp); - /* wait until all preposted entries are canceled (GetResult() - * ND2_RESULT entries with Status == STATUS_CANCELLED) */ - while (InterlockedAdd(&ep->shutdown, 0) > 0) - /* yields execution to another thread - * that is ready to run on: */ - if (!SwitchToThread()) /* - the current processor */ - Sleep(0); /* - the another processor */ - } - - if (ep->unexpected.mr) { - ofi_nd_util_unregister_mr(ep->unexpected.mr); - ep->unexpected.mr->lpVtbl->Release(ep->unexpected.mr); - } - - if (ep->unexpected.unexpected) { - /* Free allocated a piece of memory for unexpected events */ - if (ep->unexpected.unexpected[0]) - free(ep->unexpected.unexpected[0]); - /* Free allocated set of unexpected entries */ - free(ep->unexpected.unexpected); - } - - return S_OK; -} - -static ND_BUF_CHUNK(nd_unexpected_entry) -*ofi_nd_unexp_alloc_chunk(ND_BUF_FOOTER(nd_unexpected_entry) *footer, size_t* count) -{ - OFI_UNUSED(footer); - - ND_BUF_CHUNK(nd_unexpected_entry) *chunk = malloc(sizeof(*chunk)); - if (!chunk) - return 0; - assert(count); - *count = countof(chunk->item); - memset(chunk, 0, sizeof(*chunk)); - - char *tmp = malloc(countof(chunk->item) * PREPOSTLEN); - if (!tmp) - goto fn_fail; - - size_t i; - for (i = 0; i < countof(chunk->item); i++) { - chunk->item[i].data.buf = (struct nd_unexpected_buf*)(tmp + (PREPOSTLEN * i)); - } - - return chunk; - -fn_fail: - free(chunk); - return 0; -} - -static void ofi_nd_unexp_free_chunk(ND_BUF_CHUNK(nd_unexpected_entry) *chunk) -{ - assert(chunk); - - if (chunk->item[0].data.buf) - free(chunk->item[0].data.buf); - free(chunk); -} - -static int ofi_nd_return_true(struct dlist_entry *item, const void *arg) -{ - OFI_UNUSED(item); - OFI_UNUSED(arg); - return 1; -} - -void ofi_nd_unexp_match(struct nd_ep *ep) -{ - assert(ep); - assert(ep->fid.fid.fclass == FI_CLASS_EP); - - int done = 0; - do { - nd_cq_entry *entry = NULL; - nd_cq_entry *ep_entry = NULL; - nd_cq_entry *srx_entry = NULL; - nd_unexpected_entry *unexp = NULL; - - if (ep->srx) - EnterCriticalSection(&ep->srx->prepost_lock); - else - EnterCriticalSection(&ep->prepost_lock); - - struct nd_queue_item *ep_qentry = NULL; - struct nd_queue_item *srx_qentry = NULL; - - struct dlist_entry *qunexp = NULL; - - if ((ofi_nd_queue_peek(&ep->prepost, &ep_qentry) || - (ep->srx && ofi_nd_queue_peek(&ep->srx->prepost, &srx_qentry))) && - (!dlist_empty(&ep->unexpected.received))) { - qunexp = dlist_find_first_match( - &ep->unexpected.received, ofi_nd_return_true, 0); - unexp = container_of( - qunexp, - nd_unexpected_entry, ep_list); - - if(ep_qentry) - ep_entry = container_of(ep_qentry, nd_cq_entry, queue_item); - if(srx_qentry) - srx_entry = container_of(srx_qentry, nd_cq_entry, queue_item); - - if (ep_entry && srx_entry) { - if (ep_entry->seq < srx_entry->seq) { - entry = ep_entry; - ofi_nd_queue_pop(&ep->prepost, &ep_qentry); - } - else { - entry = srx_entry; - ofi_nd_queue_pop(&ep->srx->prepost, &srx_qentry); - } - } - else if (ep_entry) { - entry = ep_entry; - ofi_nd_queue_pop(&ep->prepost, &ep_qentry); - } - else { - assert(ep->srx); - entry = srx_entry; - ofi_nd_queue_pop(&ep->srx->prepost, &srx_qentry); - } - - dlist_remove(qunexp); - /* remove element from srx queue */ - if(unexp->ep->srx) - dlist_remove(&unexp->srx_list); - } - else { - done = 1; - } - - if (ep->srx) - LeaveCriticalSection(&ep->srx->prepost_lock); - else - LeaveCriticalSection(&ep->prepost_lock); - if (!done) { - /* Set event that was received */ - entry->event = unexp->buf->header.event; - ND_LOG_EVENT_INFO(entry); - if (unexp->buf->header.flags.req_ack) - ofi_nd_send_ack(entry, ep); - ofi_nd_dispatch_cq_event(unexp->buf->header.event, entry, unexp); - } - } while (!done); -} - -void ofi_nd_srx_match(struct nd_srx *srx) -{ - assert(srx); - assert(srx->fid.fid.fclass == FI_CLASS_SRX_CTX); - - int done = 0; - do { - nd_cq_entry *entry = NULL; - nd_unexpected_entry *unexp = NULL; - - EnterCriticalSection(&srx->prepost_lock); - - struct nd_queue_item *qentry = NULL; - - if (ofi_nd_queue_peek(&srx->prepost, &qentry) && - !dlist_empty(&srx->received)) { - - entry = container_of(qentry, nd_cq_entry, queue_item); - ofi_nd_queue_pop(&srx->prepost, &qentry); - - struct dlist_entry *qunexp = dlist_remove_first_match( - &srx->received, ofi_nd_return_true, 0); - - unexp = container_of( - qunexp, nd_unexpected_entry, srx_list); - /* remove element from ep queue */ - dlist_remove(&unexp->ep_list); - } - else { - done = 1; - } - LeaveCriticalSection(&srx->prepost_lock); - - if (!done) { - if (unexp->buf->header.flags.req_ack) - ofi_nd_send_ack(entry, unexp->ep); - ofi_nd_dispatch_cq_event(unexp->buf->header.event, entry, unexp); - } - } while (!done); -} - -void ofi_nd_unexp_event(ND2_RESULT *result) -{ - assert(result); - assert(result->RequestType == Nd2RequestTypeReceive); - - struct nd_ep *ep = (struct nd_ep *)result->QueuePairContext; - assert(ep); - assert(ep->fid.fid.fclass == FI_CLASS_EP); - - nd_unexpected_ctx *ctx = (nd_unexpected_ctx *)result->RequestContext; - struct nd_unexpected_buf *buf = ctx->entry; - assert(ctx); - - if (ep->shutdown || result->Status == STATUS_CANCELLED) { - /* shutdown mode */ - ND_BUF_FREE(nd_unexpected_ctx, ctx); - InterlockedDecrement(&ep->shutdown); - return; - } - - if (ctx->entry->header.flags.ack) { - if (ctx->entry->header.flags.empty) { - /* Just drop this received unexpected entry - * since no CQ are posted to be cogherent with - * this unexp and this unexp isn't carrying - * payload data in itself */ - ND_BUF_FREE(nd_unexpected_ctx, ctx); - - ofi_nd_unexp_repost(ep, buf); - ep->send_op.flags.is_send_blocked = 0; - return; - } - - ep->send_op.flags.is_send_blocked = 0; - } - - nd_unexpected_entry *entry = ND_BUF_ALLOC(nd_unexpected_entry); - /* do NOT zero mem for entry: buf points to real buffer */ - if (!entry) { - ND_LOG_WARN(FI_LOG_EP_DATA, "Failed to allocate 'unexpected' buffer"); - return; - } - - entry->result = *result; - assert(entry->buf); - assert(result->BytesTransferred <= PREPOSTLEN); - memcpy(entry->buf, ctx->entry, result->BytesTransferred); - - entry->ep = ep; - - if (ep->srx) { - EnterCriticalSection(&ep->srx->prepost_lock); - dlist_insert_tail(&entry->srx_list, &ep->srx->received); - } - else { - EnterCriticalSection(&ep->prepost_lock); - } - dlist_insert_tail(&entry->ep_list, &ep->unexpected.received); - - if (ep->srx) { - LeaveCriticalSection(&ep->srx->prepost_lock); - } - else { - LeaveCriticalSection(&ep->prepost_lock); - } - - ND_BUF_FREE(nd_unexpected_ctx, ctx); - ofi_nd_unexp_repost(ep, buf); -#if 0 - ep->unexpected.used_counter++; - if (ep->unexpected.used_counter == (size_t)gl_data.total_avail) { - ep->unexpected.used_counter = 0; - ofi_nd_unexp_payload_run(ep); - } -#endif - ofi_nd_unexp_match(ep); -} - -void ofi_nd_unexp_service_event(ND2_RESULT *result) -{ - assert(result); - assert(result->RequestType == Nd2RequestTypeReceive); - - struct nd_ep *ep = (struct nd_ep *)result->QueuePairContext; - assert(ep); - assert(ep->fid.fid.fclass == FI_CLASS_EP); - - struct nd_queue_item *ep_qentry = NULL; - - nd_unexpected_ctx *ctx = (nd_unexpected_ctx *)result->RequestContext; - assert(ctx); - - if (ep->shutdown || result->Status == STATUS_CANCELLED) { - /* shutdown mode */ - ND_BUF_FREE(nd_unexpected_ctx, ctx); - InterlockedDecrement(&ep->shutdown); - return; - } - - struct nd_unexpected_buf *unexp_buf = ctx->entry; - assert(unexp_buf); - if (unexp_buf->header.flags.ack) - ep->send_op.flags.is_send_blocked = 0; - - if (OFI_ND_IS_SERVICE_EVENT(unexp_buf->header.event) && - (ofi_nd_queue_peek(&ep->internal_prepost, &ep_qentry))) { - if (ep_qentry) { - nd_unexpected_entry unexp = { 0 }; - nd_cq_entry *ep_entry = NULL; - - unexp.result = *result; - unexp.ep = ep; - - ND_LOG_DEBUG(FI_LOG_EP_CTRL, "Received internal event, let's " - "try to process it\n"); - ep_entry = container_of(ep_qentry, nd_cq_entry, queue_item); - ofi_nd_queue_pop(&ep->internal_prepost, &ep_qentry); - /* Set event that was received */ - ep_entry->event = unexp_buf->header.event; - ND_LOG_EVENT_INFO(ep_entry); - if (unexp_buf->header.flags.req_ack) - ofi_nd_send_ack(ep_entry, ep); - ofi_nd_dispatch_cq_event(unexp_buf->header.event, ep_entry, &unexp); - - /* just zero out it, because it should have been freed above */ - ep_entry = NULL; - } - else { - /* Shouldn't happen */ - ND_LOG_WARN(FI_LOG_EP_CTRL, "Received internal event, but " - "internal queue is empty\n"); - assert(0); - } - } - - ND_BUF_FREE(nd_unexpected_ctx, ctx); - ofi_nd_unexp_repost(ep, unexp_buf); -#if 0 - ep->unexpected.used_counter++; - if (ep->unexpected.used_counter == (size_t)gl_data.total_avail) { - ep->unexpected.used_counter = 0; - ofi_nd_unexp_payload_run(ep); - } -#endif -} - -HRESULT ofi_nd_unexp_repost(struct nd_ep *ep, struct nd_unexpected_buf *entry) -{ - assert(entry); - assert(ep); - assert(ep->fid.fid.fclass == FI_CLASS_EP); - - HRESULT hr; - - nd_unexpected_ctx *ctx = ND_BUF_ALLOC(nd_unexpected_ctx); - if (!ctx) - return ND_NO_MEMORY; - memset(ctx, 0, sizeof(*ctx)); - ctx->entry = entry; - ctx->ep = ep; - - assert(ep->unexpected.mr); - assert(ep->unexpected.token); - ND2_SGE sge = { - .Buffer = (void*)entry, - .BufferLength = (ULONG)(sizeof(nd_unexpected_buf) + - MIN(sizeof(struct nd_msg_location) * ND_MSG_IOV_LIMIT, - gl_data.inline_thr)), - .MemoryRegionToken = ep->unexpected.token - }; - - assert(ep->qp); - hr = ep->qp->lpVtbl->Receive(ep->qp, ctx, &sge, 1); - return hr; -} - -HRESULT ofi_nd_unexp_run(struct nd_ep *ep) -{ - int i; - int total_count = (gl_data.prepost_cnt + - gl_data.flow_control_cnt) * gl_data.prepost_buf_cnt; - - for (i = 0; i < total_count; i++) - ofi_nd_unexp_repost(ep, ep->unexpected.unexpected[i]); - - return S_OK; -} - -void ofi_nd_release_unexp_entry(nd_unexpected_entry *unexp) -{ - ND_BUF_FREE(nd_unexpected_entry, unexp); -} - -#endif /* _WIN32 */ - diff --git a/prov/netdir/src/netdir_unexp.h b/prov/netdir/src/netdir_unexp.h deleted file mode 100644 index 8801033f196..00000000000 --- a/prov/netdir/src/netdir_unexp.h +++ /dev/null @@ -1,84 +0,0 @@ -/* -* Copyright (c) 2015-2016 Intel Corporation, Inc. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -*/ - -#ifndef _FI_NETDIR_UNEXP_H_ -#define _FI_NETDIR_UNEXP_H_ - -#include - -#include "netdir_iface.h" -#include "netdir.h" - -#ifdef __cplusplus -extern "C" { -#endif /* __cplusplus */ - -#define OFI_ND_IS_SERVICE_EVENT(event) \ - ((event) == LARGE_MSG_ACK) - -typedef struct nd_unexpected_buf { - struct nd_msgheader header; - union received { - struct nd_msg_location locations[]; - char data[]; - } received_buf; -} nd_unexpected_buf; - -typedef struct nd_unexpected_ctx { - struct nd_ep *ep; - struct nd_unexpected_buf *entry; -} nd_unexpected_ctx; - -typedef struct nd_unexpected_entry { - struct dlist_entry ep_list; - struct dlist_entry srx_list; - ND2_RESULT result; - struct nd_unexpected_buf *buf; - struct nd_ep *ep; -} nd_unexpected_entry; - -HRESULT ofi_nd_unexp_init(struct nd_ep *ep); -HRESULT ofi_nd_unexp_fini(struct nd_ep *ep); -void ofi_nd_unexp_event(ND2_RESULT *result); -void ofi_nd_unexp_service_event(ND2_RESULT *result); -void ofi_nd_unexp_match(struct nd_ep *ep); -void ofi_nd_srx_match(struct nd_srx *srx); -HRESULT ofi_nd_unexp_repost(struct nd_ep *ep, struct nd_unexpected_buf *entry); -HRESULT ofi_nd_unexp_run(struct nd_ep *ep); -void ofi_nd_release_unexp_entry(nd_unexpected_entry *unexp); - -#ifdef __cplusplus -} -#endif /* __cplusplus */ - -#endif /* _FI_NETDIR_UNEXP_H_ */ - diff --git a/prov/netdir/src/netdir_util.h b/prov/netdir/src/netdir_util.h deleted file mode 100644 index a976eae8763..00000000000 --- a/prov/netdir/src/netdir_util.h +++ /dev/null @@ -1,176 +0,0 @@ -/* -* Copyright (c) 2015-2016 Intel Corporation, Inc. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -*/ - -#ifndef _FI_NETDIR_UTIL_H_ -#define _FI_NETDIR_UTIL_H_ - -#include -#include -#include -#include - -#include "ndspi.h" - -#include "rdma/fabric.h" -#include "ofi_mem.h" - -#include "netdir.h" -#include "netdir_buf.h" -#include "netdir_ov.h" -#include "netdir_log.h" -#include "netdir_iface.h" - -#ifdef __cplusplus -extern "C" { -#endif /* __cplusplus */ - -typedef struct ofi_nd_util_ov { - nd_event_base base; - LONG cnt; -} ofi_nd_util_ov; - -OFI_ND_NB_BUF(ofi_nd_util_ov); - -static HRESULT ofi_nd_util_ov_wait(void *overlapped, ofi_nd_util_ov *ov) -{ - assert(overlapped); - assert(ov); - - HRESULT hr = ((IND2Overlapped*)overlapped)->lpVtbl->GetOverlappedResult(overlapped, &ov->base.ov, TRUE); - if (!InterlockedDecrement(&ov->cnt)) - ov->base.free(&ov->base); - return hr; -} - -static inline HRESULT -ofi_nd_util_register_mr(IND2MemoryRegion *mr, const void *buffer, size_t len, DWORD flags) -{ - HRESULT hr = S_OK; - - ofi_nd_util_ov *ov = ND_BUF_ALLOC(ofi_nd_util_ov); - if (!ov) { - hr = ND_NO_MEMORY; - goto fn_fail; - } - - hr = mr->lpVtbl->Register(mr, buffer, len, flags, &ov->base.ov); - if (FAILED(hr)) - goto fn_fail_ov; - - hr = ofi_nd_util_ov_wait(mr, ov); - if (FAILED(hr)) - goto fn_fail_ov; - - return S_OK; - -fn_fail_ov: - ov->base.free(&ov->base); -fn_fail: - return hr; -} - -static inline HRESULT -ofi_nd_util_unregister_mr(IND2MemoryRegion *mr) -{ - HRESULT hr = S_OK; - - ofi_nd_util_ov *ov = ND_BUF_ALLOC(ofi_nd_util_ov); - if (!ov) { - hr = ND_NO_MEMORY; - goto fn_fail; - } - - hr = mr->lpVtbl->Deregister(mr, &ov->base.ov); - if (FAILED(hr)) - goto fn_fail_ov; - - hr = ofi_nd_util_ov_wait(mr, ov); - if (FAILED(hr)) - goto fn_fail_ov; - - return S_OK; - -fn_fail_ov: - ov->base.free(&ov->base); -fn_fail: - return hr; -} - -static inline int -ofi_nd_util_can_be_inlined(const struct iovec *iov, size_t iovlen) -{ - assert(iov); - - size_t i; - for (i = 0; i < iovlen; i++) - if (iov[i].iov_len > (size_t)gl_data.inline_thr) - return 0; - return 1; -} - -/* return 1 if notification should be */ -static inline int -ofi_nd_util_completion_blackmagic(uint64_t info_flags, - uint64_t cq_flags, - uint64_t op_flags) -{ - OFI_UNUSED(info_flags); - if ((op_flags & FI_COMPLETION) || - (op_flags & (FI_INJECT_COMPLETE | - FI_TRANSMIT_COMPLETE | - FI_DELIVERY_COMPLETE))) - return 1; - else if (op_flags & FI_INJECT) - return 0; - else if (!(cq_flags & FI_SELECTIVE_COMPLETION)) - return 1; - else - return 0; -} - -static inline -void ofi_nd_free_send_entry(nd_send_entry *entry) -{ - assert(entry); - - if (entry->sge) - ND_BUF_FREE(nd_sge, entry->sge); - - ND_BUF_FREE(nd_send_entry, entry); -} - -#ifdef __cplusplus -} -#endif /* __cplusplus */ - -#endif /* _FI_NETDIR_UTIL_H_ */ - diff --git a/prov/verbs/include/windows/verbs_nd.h b/prov/verbs/include/windows/verbs_nd.h index 27b0919c11d..1f91230a897 100644 --- a/prov/verbs/include/windows/verbs_nd.h +++ b/prov/verbs/include/windows/verbs_nd.h @@ -42,6 +42,12 @@ HRESULT nd_startup(); void nd_shutdown(); +int ofi_nd_is_valid_addr(const SOCKADDR *addr); +int ofi_nd_addr_cmp(const void *vaddr1, const void *vaddr2); +int ofi_nd_is_same_file(const wchar_t *path1, const wchar_t *path2); +int ofi_nd_file_exists(const wchar_t *path); +int ofi_nd_is_directory(const wchar_t *path); + char *ofi_nd_error_str(HRESULT hr); static inline int hresult2fi(HRESULT hr) { diff --git a/prov/netdir/src/netdir_addr.c b/prov/verbs/src/windows/verbs_nd_addr.c similarity index 97% rename from prov/netdir/src/netdir_addr.c rename to prov/verbs/src/windows/verbs_nd_addr.c index 3ae4a2fc673..1ec393580d6 100644 --- a/prov/netdir/src/netdir_addr.c +++ b/prov/verbs/src/windows/verbs_nd_addr.c @@ -30,13 +30,10 @@ * SOFTWARE. */ -#ifdef _WIN32 - #include #include #include -#include "netdir.h" int ofi_nd_is_valid_addr(const SOCKADDR *addr) { @@ -90,6 +87,3 @@ int ofi_nd_addr_cmp(const void* vaddr1, const void* vaddr2) return 0; } - -#endif /* _WIN32 */ - diff --git a/prov/netdir/src/netdir_fs.c b/prov/verbs/src/windows/verbs_nd_fs.c similarity index 98% rename from prov/netdir/src/netdir_fs.c rename to prov/verbs/src/windows/verbs_nd_fs.c index 3dcb56825fb..9bd5feb520d 100644 --- a/prov/netdir/src/netdir_fs.c +++ b/prov/verbs/src/windows/verbs_nd_fs.c @@ -30,12 +30,10 @@ * SOFTWARE. */ -#ifdef _WIN32 - -#include "netdir.h" #include + #define FI_ND_SHARE_ATTR (FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE) static inline DWORD ofi_nd_file_attributes(const wchar_t* path) @@ -110,6 +108,3 @@ const wchar_t *ofi_nd_filename(const wchar_t *path) return &path[i] + 1; return path; } - -#endif /* _WIN32 */ - diff --git a/prov/verbs/src/windows/verbs_nd_init.c b/prov/verbs/src/windows/verbs_nd_init.c index 43a1244e322..79311c3c41e 100644 --- a/prov/verbs/src/windows/verbs_nd_init.c +++ b/prov/verbs/src/windows/verbs_nd_init.c @@ -36,11 +36,6 @@ #include "ws2spi.h" #include "verbs_nd.h" -int ofi_nd_is_valid_addr(const SOCKADDR *addr); -int ofi_nd_addr_cmp(const void *vaddr1, const void *vaddr2); -int ofi_nd_is_same_file(const wchar_t *path1, const wchar_t *path2); -int ofi_nd_file_exists(const wchar_t *path); -int ofi_nd_is_directory(const wchar_t *path); /* Adapters must be sorted by nd_adapter::address. */ static size_t ofi_nd_remove_dups(struct nd_adapter *adapters, size_t num)