diff --git a/packaging/rhel/xrootd.spec.in b/packaging/rhel/xrootd.spec.in index 7dc5c963b8d..eb0c449bba9 100644 --- a/packaging/rhel/xrootd.spec.in +++ b/packaging/rhel/xrootd.spec.in @@ -654,6 +654,7 @@ fi %files libs %defattr(-,root,root,-) %{_libdir}/libXrdAppUtils.so.1* +%{_libdir}/libXrdClProxyPlugin-4.so %{_libdir}/libXrdCks*-4.so %{_libdir}/libXrdCrypto.so.1* %{_libdir}/libXrdCryptoLite.so.1* diff --git a/src/XrdApps.cmake b/src/XrdApps.cmake index 4b03418b895..59d6d77fdaf 100644 --- a/src/XrdApps.cmake +++ b/src/XrdApps.cmake @@ -1,6 +1,11 @@ include( XRootDCommon ) +#------------------------------------------------------------------------------- +# Modules +#------------------------------------------------------------------------------- +set( LIB_XRDCL_PROXY_PREFIX XrdClProxyPrefix-${PLUGIN_VERSION} ) + #------------------------------------------------------------------------------- # Shared library version #------------------------------------------------------------------------------- @@ -121,11 +126,29 @@ target_link_libraries( pthread ${EXTRA_LIBS} ) +#------------------------------------------------------------------------------- +# XrdClProxyPlugin library +#------------------------------------------------------------------------------- +add_library( + ${LIB_XRDCL_PROXY_PREFIX} + MODULE + XrdApps/XrdClProxyPlugin/ProxyPrefixPlugin.cc + XrdApps/XrdClProxyPlugin/ProxyPrefixFile.cc) + +target_link_libraries(${LIB_XRDCL_PROXY_PREFIX} XrdCl) + +set_target_properties( + ${LIB_XRDCL_PROXY_PREFIX} + PROPERTIES + INTERFACE_LINK_LIBRARIES "" + LINK_INTERFACE_LIBRARIES "" ) + #------------------------------------------------------------------------------- # Install #------------------------------------------------------------------------------- install( - TARGETS xrdadler32 cconfig mpxstats wait41 xrdcp-old XrdAppUtils xrdmapc xrdacctest + TARGETS xrdadler32 cconfig mpxstats wait41 xrdcp-old XrdAppUtils xrdmapc + xrdacctest ${LIB_XRDCL_PROXY_PREFIX} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} ) diff --git a/src/XrdApps/XrdClProxyPlugin/ProxyPrefixFile.cc b/src/XrdApps/XrdClProxyPlugin/ProxyPrefixFile.cc new file mode 100644 index 00000000000..b8ba392550b --- /dev/null +++ b/src/XrdApps/XrdClProxyPlugin/ProxyPrefixFile.cc @@ -0,0 +1,199 @@ +//------------------------------------------------------------------------------ +// Copyright (c) 2011-2017 by European Organization for Nuclear Research (CERN) +// Author: Elvin Sindrilaru +//------------------------------------------------------------------------------ +// This file is part of the XRootD software suite. +// +// XRootD is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// XRootD is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with XRootD. If not, see . +// +// In applying this licence, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +//------------------------------------------------------------------------------ + +#include "ProxyPrefixFile.hh" +#include +#include +#include +#include +#include +#include "XrdCl/XrdClLog.hh" + +namespace xrdcl_proxy +{ +//------------------------------------------------------------------------------ +// Constructor +//------------------------------------------------------------------------------ +ProxyPrefixFile::ProxyPrefixFile(): + mIsOpen(false), + pFile(0) +{} + +//------------------------------------------------------------------------------ +// Destructor +//------------------------------------------------------------------------------ +ProxyPrefixFile::~ProxyPrefixFile() +{ + if (pFile) { + delete pFile; + } +} + +//------------------------------------------------------------------------------ +// Open +//------------------------------------------------------------------------------ +XRootDStatus +ProxyPrefixFile::Open(const std::string& url, + OpenFlags::Flags flags, + Access::Mode mode, + ResponseHandler* handler, + uint16_t timeout) +{ + XRootDStatus st; + + if (mIsOpen) { + st = XRootDStatus(stError, errInvalidOp); + return st; + } + + pFile = new XrdCl::File(false); + std::string open_url = ConstructFinalUrl(url); + st = pFile->Open(open_url, flags, mode, handler, timeout); + + if (st.IsOK()) { + mIsOpen = true; + } + + return st; +} + +//------------------------------------------------------------------------------ +// Get proxy prefix Url +//------------------------------------------------------------------------------ +std::string +ProxyPrefixFile::GetPrefixUrl() const +{ + std::string url_prefix = (getenv("XROOT_PROXY") ? getenv("XROOT_PROXY") : ""); + + // Try out also the lower-case one + if (url_prefix.empty()) { + url_prefix = (getenv("xroot_proxy") ? getenv("xroot_proxy") : ""); + } + + return url_prefix; +} + +//------------------------------------------------------------------------------ +// Trim whitespaces from both ends for a string +//------------------------------------------------------------------------------ +std::string +ProxyPrefixFile::trim(const std::string& in) const +{ + std::string::const_iterator wsfront, wsback; + std::string::const_reverse_iterator rwsback; + wsfront = in.begin(); + rwsback = in.rbegin(); + + while (*wsfront == ' ') { + ++wsfront; + } + + while (*rwsback == ' ') { + ++rwsback; + } + + wsback = rwsback.base(); + return (wsback <= wsfront ? std::string() : std::string(wsfront, wsback)); + + /* TODO: To be used when C++11 is available + auto wsfront = std::find_if_not(in.begin(), in.end(), + [](int c) -> bool {return std::isspace(c);}); + auto wsback = std::find_if_not(in.rbegin(), in.rend(), + [](int c) -> bool {return std::isspace(c);}).base(); + return (wsback <= wsfront ? std::string() : std::string(wsfront, wsback)); + */ +} + +//------------------------------------------------------------------------------ +// Get list of domains which are NOT to be prefixed +//------------------------------------------------------------------------------ +std::list +ProxyPrefixFile::GetExclDomains() const +{ + std::string excl_domains = (getenv("XROOT_PROXY_EXCL_DOMAINS") ? + getenv("XROOT_PROXY_EXCL_DOMAINS") : ""); + + if (excl_domains.empty()) { + return std::list(); + } + + char delim = ','; + std::string item; + std::list lst; + std::stringstream ss(excl_domains); + + while (getline(ss, item, delim)) { + lst.push_back(trim(item)); + } + + return lst; +} + +//------------------------------------------------------------------------------ +// Construct final Url +//------------------------------------------------------------------------------ +std::string +ProxyPrefixFile::ConstructFinalUrl(const std::string& orig_surl) const +{ + std::string final_surl = orig_surl; + std::string url_prefix = GetPrefixUrl(); + XrdCl::Log* log = DefaultEnv::GetLog(); + log->Debug(1, "url=%s, url_prefix=%s", orig_surl.c_str(), url_prefix.c_str()); + + if (!url_prefix.empty()) { + bool exclude = false; + std::list lst_excl = GetExclDomains(); + XrdCl::URL orig_url(orig_surl); + std::string orig_host = orig_url.GetHostId(); + // Remove port if present + size_t pos = orig_host.find(':'); + + if (pos != std::string::npos) { + orig_host = orig_host.substr(0, pos); + } + + log->Debug(1, "orig_host=%s", orig_host.c_str()); + + for (std::list::iterator it_excl = lst_excl.begin(); + it_excl != lst_excl.end(); ++it_excl) { + if (url_prefix.size() < it_excl->size()) { + continue; + } + + if (std::equal(it_excl->rbegin(), it_excl->rend(), orig_host.rbegin())) { + exclude = true; + break; + } + } + + if (!exclude) { + final_surl.insert(0, url_prefix); + } + } + + log->Debug(1, "final_host=%s", final_surl.c_str()); + return final_surl; +} + +} // namespace xrdcl_proxy diff --git a/src/XrdApps/XrdClProxyPlugin/ProxyPrefixFile.hh b/src/XrdApps/XrdClProxyPlugin/ProxyPrefixFile.hh new file mode 100644 index 00000000000..dbdf2029e28 --- /dev/null +++ b/src/XrdApps/XrdClProxyPlugin/ProxyPrefixFile.hh @@ -0,0 +1,215 @@ +//------------------------------------------------------------------------------ +// Copyright (c) 2011-2017 by European Organization for Nuclear Research (CERN) +// Author: Elvin Sindrilaru +//------------------------------------------------------------------------------ +// This file is part of the XRootD software suite. +// +// XRootD is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// XRootD is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with XRootD. If not, see . +// +// In applying this licence, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +//------------------------------------------------------------------------------ + +#pragma once +#include "XrdCl/XrdClDefaultEnv.hh" +#include "XrdCl/XrdClPlugInInterface.hh" + +using namespace XrdCl; + +namespace xrdcl_proxy +{ +//------------------------------------------------------------------------------ +//! XrdClFile plugin that appends an URL prefix to the given URL. The URL +//! prefix is set as an environment variable XRD_URL_PREFIX. +//------------------------------------------------------------------------------ +class ProxyPrefixFile: public XrdCl::FilePlugIn +{ +public: + //---------------------------------------------------------------------------- + //! Constructor + //---------------------------------------------------------------------------- + ProxyPrefixFile(); + + //---------------------------------------------------------------------------- + //! Destructor + //---------------------------------------------------------------------------- + virtual ~ProxyPrefixFile(); + + //---------------------------------------------------------------------------- + //! Open + //---------------------------------------------------------------------------- + virtual XRootDStatus Open(const std::string& url, + OpenFlags::Flags flags, + Access::Mode mode, + ResponseHandler* handler, + uint16_t timeout); + + //---------------------------------------------------------------------------- + //! Close + //---------------------------------------------------------------------------- + virtual XRootDStatus Close(ResponseHandler* handler, + uint16_t timeout) + { + return pFile->Close(handler, timeout); + } + + //---------------------------------------------------------------------------- + //! Stat + //---------------------------------------------------------------------------- + virtual XRootDStatus Stat(bool force, + ResponseHandler* handler, + uint16_t timeout) + { + return pFile->Stat(force, handler, timeout); + } + + + //---------------------------------------------------------------------------- + //! Read + //---------------------------------------------------------------------------- + virtual XRootDStatus Read(uint64_t offset, + uint32_t size, + void* buffer, + ResponseHandler* handler, + uint16_t timeout) + { + return pFile->Read(offset, size, buffer, handler, timeout); + } + + //---------------------------------------------------------------------------- + //! Write + //---------------------------------------------------------------------------- + virtual XRootDStatus Write(uint64_t offset, + uint32_t size, + const void* buffer, + ResponseHandler* handler, + uint16_t timeout) + { + return pFile->Write(offset, size, buffer, handler, timeout); + } + + //---------------------------------------------------------------------------- + //! Sync + //---------------------------------------------------------------------------- + virtual XRootDStatus Sync(ResponseHandler* handler, + uint16_t timeout) + { + return pFile->Sync(handler, timeout); + } + + //---------------------------------------------------------------------------- + //! Truncate + //---------------------------------------------------------------------------- + virtual XRootDStatus Truncate(uint64_t size, + ResponseHandler* handler, + uint16_t timeout) + { + return pFile->Truncate(size, handler, timeout); + } + + //---------------------------------------------------------------------------- + //! VectorRead + //---------------------------------------------------------------------------- + virtual XRootDStatus VectorRead(const ChunkList& chunks, + void* buffer, + ResponseHandler* handler, + uint16_t timeout) + { + return pFile->VectorRead(chunks, buffer, handler, timeout); + } + + //---------------------------------------------------------------------------- + //! Fcntl + //---------------------------------------------------------------------------- + virtual XRootDStatus Fcntl(const Buffer& arg, + ResponseHandler* handler, + uint16_t timeout) + { + return pFile->Fcntl(arg, handler, timeout); + } + + //---------------------------------------------------------------------------- + //! Visa + //---------------------------------------------------------------------------- + virtual XRootDStatus Visa(ResponseHandler* handler, + uint16_t timeout) + { + return pFile->Visa(handler, timeout); + } + + //---------------------------------------------------------------------------- + //! IsOpen + //---------------------------------------------------------------------------- + virtual bool IsOpen() const + { + return pFile->IsOpen(); + } + + //---------------------------------------------------------------------------- + //! SetProperty + //---------------------------------------------------------------------------- + virtual bool SetProperty(const std::string& name, + const std::string& value) + { + return pFile->SetProperty(name, value); + } + + //---------------------------------------------------------------------------- + //! GetProperty + //---------------------------------------------------------------------------- + virtual bool GetProperty(const std::string& name, + std::string& value) const + { + return pFile->GetProperty(name, value); + } + +private: + + //---------------------------------------------------------------------------- + //! Trim whitespaces from both ends of a string + //! + //! @return trimmed string + //---------------------------------------------------------------------------- + inline std::string trim(const std::string& in) const; + + //---------------------------------------------------------------------------- + //! Get proxy prefix URL from the environment + //! + //! @return proxy prefix RUL + //---------------------------------------------------------------------------- + inline std::string GetPrefixUrl() const; + + //---------------------------------------------------------------------------- + //! Get list of domains which are NOT to be prefixed + //! + //! @return list of excluded domains + //---------------------------------------------------------------------------- + std::list GetExclDomains() const; + + //---------------------------------------------------------------------------- + //! Construct final URL if there is a proxy prefix URL specified and if the + //! exclusion list is satisfied + //! + //! @param orig_url original url + //! + //! @return final URL + //---------------------------------------------------------------------------- + std::string ConstructFinalUrl(const std::string& orig_url) const; + + bool mIsOpen; + XrdCl::File* pFile; +}; + +} // namespace xrdcl_proxy diff --git a/src/XrdApps/XrdClProxyPlugin/ProxyPrefixPlugin.cc b/src/XrdApps/XrdClProxyPlugin/ProxyPrefixPlugin.cc new file mode 100644 index 00000000000..f0a64324428 --- /dev/null +++ b/src/XrdApps/XrdClProxyPlugin/ProxyPrefixPlugin.cc @@ -0,0 +1,79 @@ +//------------------------------------------------------------------------------ +// Copyright (c) 2011-2017 by European Organization for Nuclear Research (CERN) +// Author: Elvin Sindrilaru +//------------------------------------------------------------------------------ +// This file is part of the XRootD software suite. +// +// XRootD is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// XRootD is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with XRootD. If not, see . +// +// In applying this licence, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +//------------------------------------------------------------------------------ + +#include "ProxyPrefixPlugin.hh" +#include "ProxyPrefixFile.hh" +#include "XrdVersion.hh" +#include "XrdSys/XrdSysDNS.hh" +#include "XrdCl/XrdClDefaultEnv.hh" +#include "XrdCl/XrdClLog.hh" +#include +#include + +XrdVERSIONINFO(XrdClGetPlugIn, XrdClGetPlugIn) + +extern "C" +{ + void* XrdClGetPlugIn(const void* arg) + { + return static_cast(new xrdcl_proxy::ProxyFactory()); + } +} + +namespace xrdcl_proxy +{ +//------------------------------------------------------------------------------ +// Construtor +//------------------------------------------------------------------------------ +ProxyFactory::ProxyFactory() +{ + //XrdCl::Log* log = XrdCl::DefaultEnv::GetLog(); + //log->Debug(1, "ProxyFactory constructor"); +} + +//------------------------------------------------------------------------------ +// Destructor +//------------------------------------------------------------------------------ +ProxyFactory::~ProxyFactory() {} + +//------------------------------------------------------------------------------ +// Create a file plug-in for the given URL +//------------------------------------------------------------------------------ +XrdCl::FilePlugIn* +ProxyFactory::CreateFile(const std::string& url) +{ + return static_cast(new ProxyPrefixFile()); +} + +//------------------------------------------------------------------------------ +// Create a file system plug-in for the given URL +//------------------------------------------------------------------------------ +XrdCl::FileSystemPlugIn* +ProxyFactory::CreateFileSystem(const std::string& url) +{ + XrdCl::Log* log = XrdCl::DefaultEnv::GetLog(); + log->Error(1, "FileSystem plugin implementation not suppoted"); + return static_cast(0); +} +} // namespace xrdcl_proxy diff --git a/src/XrdApps/XrdClProxyPlugin/ProxyPrefixPlugin.hh b/src/XrdApps/XrdClProxyPlugin/ProxyPrefixPlugin.hh new file mode 100644 index 00000000000..f102cf40c7a --- /dev/null +++ b/src/XrdApps/XrdClProxyPlugin/ProxyPrefixPlugin.hh @@ -0,0 +1,57 @@ +//------------------------------------------------------------------------------ +// Copyright (c) 2011-2017 by European Organization for Nuclear Research (CERN) +// Author: Elvin Sindrilaru +//------------------------------------------------------------------------------ +// This file is part of the XRootD software suite. +// +// XRootD is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// XRootD is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with XRootD. If not, see . +// +// In applying this licence, CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. +//------------------------------------------------------------------------------ + +#pragma once +#include "XrdCl/XrdClPlugInInterface.hh" + +namespace xrdcl_proxy +{ +//------------------------------------------------------------------------------ +//! XrdCl proxy prefix plugin factory +//------------------------------------------------------------------------------ +class ProxyFactory: public XrdCl::PlugInFactory +{ +public: + //---------------------------------------------------------------------------- + //! Construtor + //---------------------------------------------------------------------------- + ProxyFactory(); + + //---------------------------------------------------------------------------- + //! Destructor + //---------------------------------------------------------------------------- + virtual ~ProxyFactory(); + + //---------------------------------------------------------------------------- + //! Create a file plug-in for the given URL + //---------------------------------------------------------------------------- + virtual XrdCl::FilePlugIn* CreateFile(const std::string& url); + + //---------------------------------------------------------------------------- + //! Create a file system plug-in for the given URL + //---------------------------------------------------------------------------- + virtual XrdCl::FileSystemPlugIn* CreateFileSystem(const std::string& url); +}; + +} // namespace xrdcl_proxy diff --git a/src/XrdApps/XrdClProxyPlugin/README.md b/src/XrdApps/XrdClProxyPlugin/README.md new file mode 100644 index 00000000000..1cbd186f858 --- /dev/null +++ b/src/XrdApps/XrdClProxyPlugin/README.md @@ -0,0 +1,31 @@ +# XrdClProxyPrefix Plugin + +This XRootD Client Plugin can be used to tunnel traffic through an XRootD Proxy machine. The proxy endpoint is specifed as an environment variable. To enable this plugin the **XRD_PLUGIN** environment variable needs to point to the **libXrdClProxyPlugin.so** library. + +For example: + +```bash +XRD_PLUGIN=/usr/lib64/libXrdClProxyPlugin.so \ +XROOT_PROXY=root://esvm000:2010// \ +xrdcp -f -d 1 root://esvm000//tmp/file1.dat /tmp/dump +[1.812kB/1.812kB][100%][==================================================][1.812kB/s] +``` + +This will first redirect the client to the XRootD server on port 2010 which is a forwarding proxy and then the request will be served by the default XRootD server on port 1094. + +The user can also specify a list of exclusion domains, for which the original URL will not be modified even if the plugin is enabled. For example: + +```bash +XRD_PLUGIN=/usr/lib64/libXrdClProxyPlugin.so \ +XROOT_PROXY=root://esvm000.cern.ch:2010// \ +XROOT_PROXY_EXCL_DOMAINS="some.domain, some.other.domain, cern.ch " \ +xrdcp -f -d 1 root://esvm000.cern.ch//tmp/file1.dat /tmp/dump +``` + +This will not redirect the traffic since the original url "root://esmv000.cern.ch//" contains the "cern.ch" domain which is in the list of excluded domains. There are several environment variables that control the behaviour of this XRootD Client plugin: + +**XROOT_PROXY/xroot_proxy** - XRootD endpoint through which all traffic is tunnelled + +**XROOT_PROXY_EXCL_DOMAINS** - list of comma separated domains which are excluded from being tunnelled through the proxy endpoint + +**XRD_PLUGIN** - default environment variable used by the XRootD Client plugin loading mechanism which needs to point to the library implementation of the plugin