Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

Merge branch 'release/1.2'

  • Loading branch information...
commit de4089cc93d3b5c9df6fae6475a871725691b287 2 parents 6b86977 + 50331de
@ccutrer authored
Showing with 3,066 additions and 833 deletions.
  1. +16 −6 Makefile
  2. +77 −0 autoexp.dat
  3. +8 −0 mordor.xcodeproj/project.pbxproj
  4. +21 −1 mordor/atomic.h
  5. +102 −0 mordor/examples/decodebacktrace.cpp
  6. +100 −0 mordor/examples/decodebacktrace.vcxproj
  7. +6 −0 mordor/examples/decodebacktrace.vcxproj.filters
  8. +1 −0  mordor/examples/echoserver.cpp
  9. +38 −25 mordor/examples/tunnel.cpp
  10. +27 −3 mordor/examples/wget.cpp
  11. +6 −2 mordor/exception.cpp
  12. +1 −0  mordor/exception.h
  13. +72 −1 mordor/fiber.cpp
  14. +9 −0 mordor/fiber.h
  15. +118 −2 mordor/http/auth.cpp
  16. +19 −12 mordor/http/auth.h
  17. +8 −0 mordor/http/basic.cpp
  18. +7 −5 mordor/http/basic.h
  19. +291 −179 mordor/http/broker.cpp
  20. +91 −72 mordor/http/broker.h
  21. +131 −107 mordor/http/client.cpp
  22. +17 −14 mordor/http/client.h
  23. +5 −5 mordor/http/multipart.h
  24. +1 −0  mordor/http/negotiate.cpp
  25. +9 −2 mordor/http/negotiate.h
  26. +51 −13 mordor/http/oauth.cpp
  27. +7 −7 mordor/http/oauth.h
  28. +275 −138 mordor/http/proxy.cpp
  29. +36 −36 mordor/http/proxy.h
  30. +1 −0  mordor/http/server.cpp
  31. +18 −11 mordor/http/server.h
  32. +0 −30 mordor/http/tunnel.h
  33. +4 −1 mordor/iomanager_kqueue.cpp
  34. +19 −5 mordor/json.rl
  35. +18 −6 mordor/log.cpp
  36. +13 −5 mordor/mordor.vcproj
  37. +2 −1  mordor/mordor.vcxproj
  38. +6 −3 mordor/mordor.vcxproj.filters
  39. +11 −0 mordor/mordor2010.sln
  40. +1 −0  mordor/pch.h
  41. +1 −0  mordor/predef.h
  42. +1 −1  mordor/runtime_linking.h
  43. +144 −28 mordor/socket.cpp
  44. +15 −8 mordor/socket.h
  45. +135 −0 mordor/socks.cpp
  46. +45 −0 mordor/socks.h
  47. +1 −3 mordor/streams/filter.h
  48. +147 −71 mordor/streams/http.cpp
  49. +13 −2 mordor/streams/http.h
  50. +74 −0 mordor/string.cpp
  51. +12 −0 mordor/string.h
  52. +189 −0 mordor/test/antxmllistener.cpp
  53. +60 −0 mordor/test/antxmllistener.h
  54. +8 −0 mordor/test/mordortest.vcproj
  55. +2 −0  mordor/test/mordortest.vcxproj
  56. +6 −0 mordor/test/mordortest.vcxproj.filters
  57. +69 −1 mordor/test/test.cpp
  58. +52 −2 mordor/test/test.h
  59. +283 −11 mordor/tests/http.cpp
  60. +11 −0 mordor/tests/json.cpp
  61. +15 −4 mordor/tests/run_tests.cpp
  62. +1 −0  mordor/tests/tests.vcxproj
  63. +3 −0  mordor/tests/tests.vcxproj.filters
  64. +31 −0 mordor/tests/unicode.cpp
  65. +29 −0 mordor/tests/uri.cpp
  66. +2 −2 mordor/uri.h
  67. +22 −8 mordor/uri.rl
  68. +52 −0 mordor/util.h
View
22 Makefile
@@ -62,7 +62,7 @@ PLATFORMDIR := $(PLATFORM)/$(ARCH)
ifeq ($(PLATFORM), Darwin)
DARWIN := 1
- BOOST_EXT := -mt
+ BOOST_EXT :=
BOOST_LIB_FLAGS := -L/opt/local/lib
PQ_LIB_FLAGS := -L/opt/local/lib/postgresql83
IOMANAGER := kqueue
@@ -170,11 +170,12 @@ ifeq ($(RAGEL_MAJOR), 6)
RLCODEGEN :=
endif
+LIBS := $(BOOST_LIB_FLAGS) $(PQ_LIB_FLAGS) -lboost_thread$(BOOST_EXT) -lboost_program_options $(BOOST_EXT) -lboost_regex$(BOOST_EXT) -lboost_date_time$(BOOST_EXT) -lssl -lcrypto -lz -ldl
+
ifeq ($(PLATFORM), Darwin)
+ LIBS += -framework SystemConfiguration -framework CoreFoundation -framework CoreServices -framework Security
endif
-LIBS := $(BOOST_LIB_FLAGS) $(PQ_LIB_FLAGS) -lboost_thread$(BOOST_EXT) -lboost_regex$(BOOST_EXT) -lboost_date_time$(BOOST_EXT) -lssl -lcrypto -lz -ldl
-
ifeq ($(PLATFORM), FreeBSD)
LIBS += -lexecinfo
endif
@@ -390,9 +391,9 @@ endif
mordor/examples/wget: mordor/examples/wget.o \
mordor/libmordor.a
ifeq ($(Q),@)
- @echo ld $@
+ @echo ld $@ -lboost_program_options$(BOOST_EXT)
endif
- $(COMPLINK)
+ $(COMPLINK) -lboost_program_options$(BOOST_EXT)
mordor/streams/socket_stream.o: mordor/streams/socket.cpp
ifeq ($(Q),@)
@@ -401,6 +402,13 @@ endif
$(Q)mkdir -p $(@D)
$(Q)$(CXX) $(CXXFLAGS) -c -o $@ $<
+mordor/streams/http_stream.o: mordor/streams/http.cpp
+ifeq ($(Q),@)
+ @echo c++ $<
+endif
+ $(Q)mkdir -p $(@D)
+ $(Q)$(CXX) $(CXXFLAGS) -c -o $@ $<
+
LIBMORDOROBJECTS := \
mordor/assert.o \
@@ -429,6 +437,7 @@ LIBMORDOROBJECTS := \
mordor/semaphore.o \
mordor/sleep.o \
mordor/socket.o \
+ mordor/socks.o \
mordor/statistics.o \
mordor/streams/buffer.o \
mordor/streams/buffered.o \
@@ -436,7 +445,7 @@ LIBMORDOROBJECTS := \
mordor/streams/fd.o \
mordor/streams/file.o \
mordor/streams/hash.o \
- mordor/streams/http.o \
+ mordor/streams/http_stream.o \
mordor/streams/limited.o \
mordor/streams/memory.o \
mordor/streams/null.o \
@@ -471,6 +480,7 @@ endif
$(Q)$(AR) $(ARFLAGS) $@ $(filter %.o,$?)
LIBMORDORTESTOBJECTS := \
+ mordor/test/antxmllistener.o \
mordor/test/test.o \
mordor/test/stdoutlistener.o
View
77 autoexp.dat
@@ -0,0 +1,77 @@
+; Add to C:\Program Files (x86)\Microsoft Visual Studio 10.0\Common7\Packages\Debugger\autoexp.dat in the [Visualizer] section
+; The visualizer isn't perfect - it uses ; instead of :, and , instead of / between path segments
+; Also, it doesn't perform any encoding
+
+Mordor::URI{
+ preview (
+ #(
+ #if ($e.m_schemeDefined) ( #(
+ #if (($e.m_scheme._Myres) < ($e.m_scheme._BUF_SIZE)) ( [$e.m_scheme._Bx._Buf,sb] ) #else ( [$e.m_scheme._Bx._Ptr,sb] ),
+ ";")
+ ) #else ( "" ),
+ #if ($e.authority.m_hostDefined) ( #("//", $e.authority) ) #else ( "" ),
+ #if ($e.path.type == ABSOLUTE || $e.path.segments._Myfirst != $e.path.segments._Mylast) ( $e.path ) #else ( "" ),
+ #if ($e.m_queryDefined) ( #(
+ "?",
+ #if (($e.m_query._Myres) < ($e.m_query._BUF_SIZE)) ( [$e.m_query._Bx._Buf,sb] ) #else ( [$e.m_query._Bx._Ptr,sb] ))
+ ) #else ( "" ),
+ #if ($e.m_fragmentDefined) ( #(
+ "#",
+ #if (($e.m_fragment._Myres) < ($e.m_fragment._BUF_SIZE)) ( [$e.m_fragment._Bx._Buf,sb] ) #else ( [$e.m_fragment._Bx._Ptr,sb] ))
+ ) #else ( "" )
+ )
+ )
+ children (
+ #(
+ #if ($e.m_schemeDefined) ( #(scheme: $e.m_scheme) ) #else ( #array(expr: 0, size: 0) ),
+ #if ($e.authority.m_hostDefined) ( #(authority: $e.authority) ) #else ( #array(expr: 0, size: 0) ),
+ #if ($e.path.type == ABSOLUTE || $e.path.segments._Myfirst != $e.path.segments._Mylast) ( #(path: $e.path) ) #else ( #array(expr: 0, size: 0) ),
+ #if ($e.m_queryDefined) ( #(query: $e.m_query) ) #else ( #array(expr: 0, size: 0) ),
+ #if ($e.m_fragmentDefined) ( #(fragment: $e.m_fragment) ) #else ( #array(expr: 0, size: 0) ),
+ #(Actual Members: [$e,!])
+ )
+ )
+}
+
+Mordor::URI::Authority{
+ preview (
+ #(
+ #if ($e.m_userinfoDefined) ( #( #if (($e.m_userinfo._Myres) < ($e.m_userinfo._BUF_SIZE)) ( [$e.m_userinfo._Bx._Buf,sb] ) #else ( [$e.m_userinfo._Bx._Ptr,sb] ), "@") ) #else ( "" ),
+ #if ($e.m_hostDefined) ( #if (($e.m_host._Myres) < ($e.m_host._BUF_SIZE)) ( [$e.m_host._Bx._Buf,sb] ) #else ( [$e.m_host._Bx._Ptr,sb] ) ) #else ( "" ),
+ #if ($e.m_portDefined) ( #( ";", $e.m_port) ) #else ( "" )
+ )
+ )
+ children (
+ #(
+ #if ($e.m_userinfoDefined) ( #( userinfo: $e.m_userinfo) ) #else ( #array(expr: 0, size: 0) ),
+ #if ($e.m_hostDefined) ( #( host: $e.m_host) ) #else ( #array(expr: 0, size: 0) ),
+ #if ($e.m_portDefined) ( #( port: $e.m_port) ) #else ( #array(expr: 0, size: 0) ),
+ #(Actual Members: [$e,!])
+ )
+ )
+}
+
+Mordor::URI::Path{
+ preview (
+ #if ($e.type == RELATIVE && $e.segments._Myfirst == $e.segments._Mylast) (
+ ""
+ ) #else (
+ #(
+ #if ($e.type == ABSOLUTE) (
+ "/"
+ ) #else (
+ ""
+ ),
+ #array(
+ expr:
+ #if ((($e.segments._Myfirst[$i])._Myres) < (($e.segments._Myfirst[$i])._BUF_SIZE)) (
+ [($e.segments._Myfirst[$i])._Bx._Buf,sb]
+ ) #else (
+ [($e.segments._Myfirst[$i])._Bx._Ptr,sb]
+ ),
+ size: $e.segments._Mylast - $e.segments._Myfirst
+ )
+ )
+ )
+ )
+}
View
8 mordor.xcodeproj/project.pbxproj
@@ -61,6 +61,8 @@
3CAB29E9112B347400607AEE /* timer.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 3CAB2798112B096700607AEE /* timer.cpp */; };
3CAC4A341151622F00F85725 /* iomanager_kqueue.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 3CAB271B112B096700607AEE /* iomanager_kqueue.cpp */; };
3CC4CE561138646D001C21F2 /* progress.h in Headers */ = {isa = PBXBuildFile; fileRef = 3CC4CE551138646D001C21F2 /* progress.h */; };
+ C34893CA11BEB1300044E9C8 /* socks.cpp in Sources */ = {isa = PBXBuildFile; fileRef = C34893C811BEB1300044E9C8 /* socks.cpp */; };
+ C34893CB11BEB1300044E9C8 /* socks.h in Headers */ = {isa = PBXBuildFile; fileRef = C34893C911BEB1300044E9C8 /* socks.h */; };
E29FD2AA1136FD1D004BC293 /* timeout.cpp in Sources */ = {isa = PBXBuildFile; fileRef = E29FD2A81136FD1D004BC293 /* timeout.cpp */; };
E29FD2AB1136FD1D004BC293 /* timeout.h in Headers */ = {isa = PBXBuildFile; fileRef = E29FD2A91136FD1D004BC293 /* timeout.h */; };
/* End PBXBuildFile section */
@@ -279,6 +281,8 @@
3CAB27A1112B096700607AEE /* xml_parser.rl */ = {isa = PBXFileReference; explicitFileType = sourcecode.cpp.cpp.preprocessed; fileEncoding = 4; path = xml_parser.rl; sourceTree = "<group>"; };
3CAB29B1112B340600607AEE /* libmordor.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = libmordor.a; sourceTree = BUILT_PRODUCTS_DIR; };
3CC4CE551138646D001C21F2 /* progress.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = progress.h; sourceTree = "<group>"; };
+ C34893C811BEB1300044E9C8 /* socks.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = socks.cpp; sourceTree = "<group>"; };
+ C34893C911BEB1300044E9C8 /* socks.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = socks.h; sourceTree = "<group>"; };
E29FD2A81136FD1D004BC293 /* timeout.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = timeout.cpp; sourceTree = "<group>"; };
E29FD2A91136FD1D004BC293 /* timeout.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = timeout.h; sourceTree = "<group>"; };
/* End PBXFileReference section */
@@ -330,6 +334,8 @@
3CAB26D4112B096700607AEE /* mordor */ = {
isa = PBXGroup;
children = (
+ C34893C811BEB1300044E9C8 /* socks.cpp */,
+ C34893C911BEB1300044E9C8 /* socks.h */,
3CAB26D5112B096700607AEE /* .gitignore */,
3CAB26D6112B096700607AEE /* anymap.h */,
3CAB26D7112B096700607AEE /* assert.cpp */,
@@ -591,6 +597,7 @@
files = (
E29FD2AB1136FD1D004BC293 /* timeout.h in Headers */,
3CC4CE561138646D001C21F2 /* progress.h in Headers */,
+ C34893CB11BEB1300044E9C8 /* socks.h in Headers */,
);
runOnlyForDeploymentPostprocessing = 0;
};
@@ -692,6 +699,7 @@
3CAB29E9112B347400607AEE /* timer.cpp in Sources */,
E29FD2AA1136FD1D004BC293 /* timeout.cpp in Sources */,
3CAC4A341151622F00F85725 /* iomanager_kqueue.cpp in Sources */,
+ C34893CA11BEB1300044E9C8 /* socks.cpp in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
View
22 mordor/atomic.h
@@ -5,6 +5,10 @@
#include "predef.h"
+#if (__GNUC__ == 4 && __GNUC_MINOR__ >= 1 && defined(__arm__))
+#include <boost/smart_ptr/detail/sp_counted_base_spin.hpp>
+#endif
+
namespace Mordor {
#ifdef WINDOWS
@@ -166,7 +170,7 @@ atomicSwap(volatile T &t, T newvalue)
return comparand;
}
#endif
-#elif (__GNUC__ == 4 && __GNUC_MINOR__ >= 1)
+#elif (__GNUC__ == 4 && __GNUC_MINOR__ >= 1 && !defined(__arm__))
template <class T>
typename boost::enable_if_c<sizeof(T) <= sizeof(void *), T>::type
atomicDecrement(volatile T& t) { return __sync_sub_and_fetch(&t, 1); }
@@ -220,6 +224,22 @@ atomicIncrement(volatile T& t) { return __gnu_cxx::__exchange_and_add((_Atomic_w
template <class T>
typename boost::enable_if_c<sizeof(T) <= sizeof(_Atomic_word), T>::type
atomicAdd(volatile T& t, T v) { return __gnu_cxx::__exchange_and_add((_Atomic_word*)_&t, v) + v; }
+#elif (__GNUC__ == 4 && __GNUC_MINOR__ >= 1 && defined(__arm__))
+template <class T>
+typename boost::enable_if_c<sizeof(T) <= sizeof(int), T>::type
+atomicAdd(volatile T& t, T v) { return boost::detail::atomic_exchange_and_add((int *)&t, (int)v) + v; }
+template <class T>
+typename boost::enable_if_c<sizeof(T) <= sizeof(int), T>::type
+atomicIncrement(volatile T& t) { return atomicAdd(t, (T)1); }
+template <class T>
+typename boost::enable_if_c<sizeof(T) <= sizeof(int), T>::type
+atomicCompareAndSwap(volatile T &t, T newvalue, T comparand) {
+ ::boost::detail::spinlock_pool<1>::scoped_lock lock((void *)&t);
+ T oldvalue = t;
+ if (oldvalue == comparand)
+ t = newvalue;
+ return oldvalue;
+}
#endif
template <typename T>
View
102 mordor/examples/decodebacktrace.cpp
@@ -0,0 +1,102 @@
+// Copyright (c) 2009 - Mozy, Inc.
+
+#include "mordor/predef.h"
+
+#include <iostream>
+
+#include <DbgHelp.h>
+
+#include "mordor/config.h"
+#include "mordor/streams/buffered.h"
+#include "mordor/streams/std.h"
+
+using namespace Mordor;
+
+int main(int argc, const char *argv[])
+{
+ if (argc < 3) {
+ std::cerr << "Usage: " << argv[0] << " <symbolpath> <binary>..." << std::endl
+ << " Look for backtrace addresses on stdin, and attempt to convert them to" << std::endl
+ << " symbols of the given binaries." << std::endl << std::endl;
+ return 1;
+ }
+
+ std::vector<DWORD64> loadedModules;
+ try {
+ Config::loadFromEnvironment();
+ // Re-init symbols (it got initted statically in exception.cpp) with
+ // a symbol search path
+ if (!SymCleanup(GetCurrentProcess()))
+ MORDOR_THROW_EXCEPTION_FROM_LAST_ERROR_API("SymCleanup");
+ if (!SymInitialize(GetCurrentProcess(), argv[1], FALSE))
+ MORDOR_THROW_EXCEPTION_FROM_LAST_ERROR_API("SymInitialize");
+
+ // Load up the specified modules in their default load location
+ // Does *not* currently handle ASLR (basically, it would need to
+ // do two passes through the log, and calculate the actual load address
+ // of a module based on where the (limited) symbols that are in the
+ // file ended up; or, use a minidump to aid the process)
+ for (int i = 2; i < argc; ++i) {
+ DWORD64 baseAddress =
+ SymLoadModule64(GetCurrentProcess(), NULL, argv[i], NULL, 0, 0);
+ if (baseAddress == 0) {
+ if (GetLastError() == ERROR_SUCCESS)
+ continue;
+ MORDOR_THROW_EXCEPTION_FROM_LAST_ERROR_API("SymLoadModule64");
+ }
+ loadedModules.push_back(baseAddress);
+ }
+ char buf[sizeof(SYMBOL_INFO) + MAX_SYM_NAME - 1];
+ SYMBOL_INFO *symbol = (SYMBOL_INFO*)buf;
+ symbol->SizeOfStruct = sizeof(SYMBOL_INFO);
+ symbol->MaxNameLen = MAX_SYM_NAME;
+ DWORD64 displacement64 = 0;
+
+ Stream::ptr stream(new StdinStream());
+ stream.reset(new BufferedStream(stream));
+ StdoutStream output;
+ std::string line;
+ do {
+ line = stream->getDelimited('\n', true);
+ std::string copy(line);
+ if (copy.size() >= 35 &&
+ strncmp(copy.c_str(), "[struct Mordor::tag_backtrace *] = ", 35)
+ == 0)
+ copy = copy.substr(35);
+ char *end;
+ DWORD64 address = strtoull(copy.c_str(), &end, 16);
+ if ( (end - copy.c_str()) == 8 || (end - copy.c_str()) == 16) {
+ if (SymFromAddr(GetCurrentProcess(), address, &displacement64, symbol)) {
+ std::ostringstream os;
+ if (copy != line)
+ os << "[struct Mordor::tag_backtrace *] = ";
+ os << copy.substr(0, (end - copy.c_str())) << ": "
+ << symbol->Name << "+" << displacement64;
+ IMAGEHLP_LINE64 line;
+ line.SizeOfStruct = sizeof(IMAGEHLP_LINE64);
+ DWORD displacement = 0;
+ if (SymGetLineFromAddr64(GetCurrentProcess(),
+ address, &displacement, &line)) {
+ std::cout << ": " << line.FileName << "("
+ << line.LineNumber << ")+" << displacement;
+ }
+ os << std::endl;
+ std::string newline = os.str();
+ output.write(newline.c_str(), newline.size());
+ } else {
+ output.write(line.c_str(), line.size());
+ }
+ } else {
+ output.write(line.c_str(), line.size());
+ }
+ } while (!line.empty() && line.back() == '\n');
+ } catch (...) {
+ std::cerr << boost::current_exception_diagnostic_information() << std::endl;
+ for (size_t i = 0; i < loadedModules.size(); ++i)
+ SymUnloadModule64(GetCurrentProcess(), loadedModules[i]);
+ return 2;
+ }
+ for (size_t i = 0; i < loadedModules.size(); ++i)
+ SymUnloadModule64(GetCurrentProcess(), loadedModules[i]);
+ return 0;
+}
View
100 mordor/examples/decodebacktrace.vcxproj
@@ -0,0 +1,100 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <ItemGroup Label="ProjectConfigurations">
+ <ProjectConfiguration Include="Debug|Win32">
+ <Configuration>Debug</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Debug|x64">
+ <Configuration>Debug</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Release|Win32">
+ <Configuration>Release</Configuration>
+ <Platform>Win32</Platform>
+ </ProjectConfiguration>
+ <ProjectConfiguration Include="Release|x64">
+ <Configuration>Release</Configuration>
+ <Platform>x64</Platform>
+ </ProjectConfiguration>
+ </ItemGroup>
+ <PropertyGroup Label="Globals">
+ <ProjectGuid>{C7B7CBD7-5D43-4FA5-9970-8C4A09BFAF67}</ProjectGuid>
+ <RootNamespace>decodebacktrace</RootNamespace>
+ </PropertyGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
+ <PropertyGroup Label="Configuration">
+ <ConfigurationType>Application</ConfigurationType>
+ </PropertyGroup>
+ <PropertyGroup Condition="'$(Configuration)'=='Release'" Label="Configuration">
+ <WholeProgramOptimization>true</WholeProgramOptimization>
+ </PropertyGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" />
+ <ImportGroup Label="ExtensionSettings">
+ </ImportGroup>
+ <ImportGroup Label="PropertySheets">
+ <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" />
+ </ImportGroup>
+ <PropertyGroup Label="UserMacros" />
+ <PropertyGroup>
+ <_ProjectFileVersion>10.0.30319.1</_ProjectFileVersion>
+ <OutDir>$(SolutionDir)$(Platform)\$(Configuration)\</OutDir>
+ <IntDir>$(Platform)\$(Configuration)\$(ProjectName)\</IntDir>
+ </PropertyGroup>
+ <ItemDefinitionGroup>
+ <ClCompile>
+ <AdditionalIncludeDirectories>../..;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
+ <DebugInformationFormat>ProgramDatabase</DebugInformationFormat>
+ <WarningLevel>Level3</WarningLevel>
+ <TreatWarningAsError>true</TreatWarningAsError>
+ <DisableSpecificWarnings>4345</DisableSpecificWarnings>
+ </ClCompile>
+ <Link>
+ <GenerateDebugInformation>true</GenerateDebugInformation>
+ <SubSystem>Console</SubSystem>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)'=='Debug'">
+ <ClCompile>
+ <Optimization>Disabled</Optimization>
+ <MinimalRebuild>true</MinimalRebuild>
+ <BasicRuntimeChecks>EnableFastChecks</BasicRuntimeChecks>
+ <RuntimeLibrary>MultiThreadedDebugDLL</RuntimeLibrary>
+ <DebugInformationFormat Condition="'$(Platform)'=='Win32'">EditAndContinue</DebugInformationFormat>
+ </ClCompile>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Configuration)'=='Release'">
+ <ClCompile>
+ <Optimization>MaxSpeed</Optimization>
+ <IntrinsicFunctions>true</IntrinsicFunctions>
+ <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
+ <FunctionLevelLinking>true</FunctionLevelLinking>
+ </ClCompile>
+ <Link>
+ <OptimizeReferences>true</OptimizeReferences>
+ <EnableCOMDATFolding>true</EnableCOMDATFolding>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Platform)'=='Win32'">
+ <Link>
+ <TargetMachine>MachineX86</TargetMachine>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemDefinitionGroup Condition="'$(Platform)'=='x64'">
+ <Link>
+ <TargetMachine>MachineX64</TargetMachine>
+ </Link>
+ </ItemDefinitionGroup>
+ <ItemGroup>
+ <ClCompile Include="decodebacktrace.cpp" />
+ </ItemGroup>
+ <ItemGroup>
+ <ProjectReference Include="..\mordor.vcxproj">
+ <Project>{feac089a-cc93-49c3-8f22-a9ab96f6273a}</Project>
+ <ReferenceOutputAssembly>false</ReferenceOutputAssembly>
+ </ProjectReference>
+ </ItemGroup>
+ <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
+ <ImportGroup Label="ExtensionTargets">
+ </ImportGroup>
+</Project>
View
6 mordor/examples/decodebacktrace.vcxproj.filters
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <ItemGroup>
+ <ClCompile Include="decodebacktrace.cpp" />
+ </ItemGroup>
+</Project>
View
1  mordor/examples/echoserver.cpp
@@ -5,6 +5,7 @@
#include <boost/bind.hpp>
#include "mordor/config.h"
+#include "mordor/http/multipart.h"
#include "mordor/http/server.h"
#include "mordor/iomanager.h"
#include "mordor/socket.h"
View
63 mordor/examples/tunnel.cpp
@@ -6,7 +6,8 @@
#include "mordor/config.h"
#include "mordor/http/auth.h"
-#include "mordor/http/tunnel.h"
+#include "mordor/http/client.h"
+#include "mordor/http/proxy.h"
#include "mordor/iomanager.h"
#include "mordor/socket.h"
#include "mordor/streams/duplex.h"
@@ -70,28 +71,38 @@ static void outgoingConnection(Stream::ptr client, IOManager &ioManager,
}
}
-static
-HTTP::ClientConnection::ptr establishConn(IOManager &ioManager, const std::string &to)
+static bool getCredentials(HTTP::ClientRequest::ptr priorRequest,
+ std::string &scheme, std::string &username, std::string &password,
+ const std::string &user, const std::string &pass,
+ size_t attempts)
{
- std::vector<Address::ptr> addresses =
- Address::lookup(to, AF_UNSPEC, SOCK_STREAM);
- Socket::ptr sock;
- for (std::vector<Address::ptr>::const_iterator it(addresses.begin());
- it != addresses.end();
- ) {
- sock = (*it)->createSocket(ioManager);
- try {
- sock->connect(*it);
- break;
- } catch (std::exception &) {
- if (++it == addresses.end())
- throw;
- sock.reset();
- }
+ if (!priorRequest)
+ return false;
+ if (attempts > 1)
+ return false;
+ username = user;
+ password = pass;
+ const HTTP::ChallengeList &challengeList =
+ priorRequest->response().response.proxyAuthenticate;
+#ifdef WINDOWS
+ if (HTTP::isAcceptable(challengeList, "Negotiate")) {
+ scheme = "Negotiate";
+ return true;
+ }
+ if (HTTP::isAcceptable(challengeList, "NTLM")) {
+ scheme = "NTLM";
+ return true;
+ }
+#endif
+ if (HTTP::isAcceptable(challengeList, "Digest")) {
+ scheme = "Digest";
+ return true;
+ }
+ if (HTTP::isAcceptable(challengeList, "Basic")) {
+ scheme = "Basic";
+ return true;
}
- Stream::ptr stream(new SocketStream(sock));
- HTTP::ClientConnection::ptr conn(new HTTP::ClientConnection(stream));
- return conn;
+ return false;
}
static void outgoingProxyConnection(Stream::ptr client, IOManager &ioManager,
@@ -104,11 +115,13 @@ static void outgoingProxyConnection(Stream::ptr client, IOManager &ioManager,
std::vector<Address::ptr> addresses =
Address::lookup(proxy, AF_UNSPEC, SOCK_STREAM);
- HTTP::ClientAuthBroker authBroker(boost::bind(&establishConn,
- boost::ref(ioManager), proxy), "", "", username,
- password);
+ HTTP::RequestBrokerOptions options;
+ options.ioManager = &ioManager;
+ options.getProxyCredentialsDg = boost::bind(&getCredentials, _2, _3,
+ _5, _6, username, password, _7);
+ HTTP::RequestBroker::ptr requestBroker = HTTP::createRequestBroker(options).first;
- Stream::ptr server = HTTP::tunnel(authBroker, proxy, toConnectTo);
+ Stream::ptr server = HTTP::tunnel(requestBroker, proxy, toConnectTo);
if (ssl) {
SSLStream::ptr serverSSL(new SSLStream(server));
serverSSL->connect();
View
30 mordor/examples/wget.cpp
@@ -13,8 +13,10 @@
#include "mordor/http/auth.h"
#include "mordor/http/broker.h"
#include "mordor/http/client.h"
+#include "mordor/http/multipart.h"
#include "mordor/http/proxy.h"
#include "mordor/iomanager.h"
+#include "mordor/sleep.h"
#include "mordor/socket.h"
#include "mordor/streams/socket.h"
#include "mordor/streams/ssl.h"
@@ -98,12 +100,34 @@ int main(int argc, char *argv[])
if (vm.count("password")) password = vm["password"].as<std::string>();
if (vm.count("proxyusername")) proxyusername = vm["proxyusername"].as<std::string>();
if (vm.count("proxypassword")) proxypassword = vm["proxypassword"].as<std::string>();
- if (vm.count("username"))
- options.getCredentialsDg = boost::bind(&getCredentials, _2, _3, _5, _6,
- username, password, _7, false);
if (vm.count("proxyusername"))
options.getProxyCredentialsDg = boost::bind(&getCredentials, _2, _3, _5, _6,
proxyusername, proxypassword, _7, true);
+#ifdef OSX
+ else
+ options.getProxyCredentialsDg = &HTTP::getCredentialsFromKeychain;
+#endif
+ HTTP::RequestBroker::ptr proxyBroker =
+ HTTP::createRequestBroker(options).first;
+ if (vm.count("username"))
+ options.getCredentialsDg = boost::bind(&getCredentials, _2, _3, _5, _6,
+ username, password, _7, false);
+#ifdef OSX
+ else
+ options.getCredentialsDg = &HTTP::getCredentialsFromKeychain;
+#endif
+ options.proxyRequestBroker = proxyBroker;
+#ifdef WINDOWS
+ HTTP::ProxyCache proxyCache;
+ options.proxyForURIDg = boost::bind(
+ &HTTP::ProxyCache::proxyFromUserSettings, &proxyCache, _1);
+#elif defined (OSX)
+ HTTP::ProxyCache proxyCache(proxyBroker);
+ options.proxyForURIDg = boost::bind(
+ &HTTP::ProxyCache::proxyFromSystemConfiguration, &proxyCache, _1);
+#else
+ options.proxyForURIDg = &HTTP::proxyFromConfig;
+#endif
HTTP::RequestBroker::ptr requestBroker =
HTTP::createRequestBroker(options).first;
View
8 mordor/exception.cpp
@@ -47,14 +47,13 @@ static struct Initializer {
}
#endif
-std::string to_string( errinfo_backtrace const &bt )
+std::string to_string(const std::vector<void *> backtrace)
{
#ifdef WINDOWS
static boost::mutex s_mutex;
boost::mutex::scoped_lock lock(s_mutex);
#endif
std::ostringstream os;
- const std::vector<void *> &backtrace = bt.value();
#ifdef POSIX
boost::shared_ptr<char *> symbols(backtrace_symbols(&backtrace[0],
backtrace.size()), &free);
@@ -93,6 +92,11 @@ std::string to_string( errinfo_backtrace const &bt )
return os.str();
}
+std::string to_string( errinfo_backtrace const &bt )
+{
+ return to_string(bt.value());
+}
+
#ifdef WINDOWS
std::string to_string( errinfo_lasterror const &e)
{
View
1  mordor/exception.h
@@ -30,6 +30,7 @@ typedef errinfo_lasterror errinfo_nativeerror;
typedef boost::errinfo_errno errinfo_nativeerror;
#endif
+std::string to_string( const std::vector<void *> bt );
std::string to_string( errinfo_backtrace const &bt );
std::vector<void *> backtrace(int framesToSkip = 0);
View
73 mordor/fiber.cpp
@@ -506,13 +506,20 @@ Fiber::initStack()
m_env[9] = (int)m_stack + m_stacksize; // ESP
#elif defined(X86_64)
long long *env = (long long *)m_env;
- env[1] = 0xffffffffffffffffll; // RBP
+ env[1] = 0x0ll; // RBP
env[2] = (long long)m_stack + m_stacksize; // RSP
#elif defined(PPC)
m_env[0] = (int)m_stack;
#else
#error Architecture not supported
#endif
+#elif defined (LINUX)
+#ifdef ARM
+ int *env = (int *)m_env;
+ env[8] = (int)m_stack + m_stacksize;
+#else
+#error Platform not supported
+#endif
#else
#error Platform not supported
#endif
@@ -609,4 +616,68 @@ Fiber::flsGet(size_t key)
return self->m_fls[key];
}
+std::vector<void *>
+Fiber::backtrace()
+{
+ MORDOR_ASSERT(m_state != EXEC);
+ std::vector<void *> result;
+ if (m_state != HOLD)
+ return result;
+#ifdef WINDOWS
+ STACKFRAME64 frame;
+ DWORD type;
+ CONTEXT *context;
+#ifdef _M_IX86
+ context = (CONTEXT *)((char *)m_sp + 0x14);
+ type = IMAGE_FILE_MACHINE_I386;
+ frame.AddrPC.Offset = context->Eip;
+ frame.AddrPC.Mode = AddrModeFlat;
+ frame.AddrFrame.Offset = context->Ebp;
+ frame.AddrFrame.Mode = AddrModeFlat;
+ frame.AddrStack.Offset = context->Esp;
+ frame.AddrStack.Mode = AddrModeFlat;
+ context = NULL;
+#elif _M_X64
+ context = (CONTEXT *)((char *)m_sp + 0x30);
+ CONTEXT dupContext;
+ memcpy(&dupContext, context, sizeof(CONTEXT));
+ context = &dupContext;
+ type = IMAGE_FILE_MACHINE_AMD64;
+ frame.AddrPC.Offset = dupContext.Rip;
+ frame.AddrPC.Mode = AddrModeFlat;
+ frame.AddrFrame.Offset = dupContext.Rsp;
+ frame.AddrFrame.Mode = AddrModeFlat;
+ frame.AddrStack.Offset = dupContext.Rsp;
+ frame.AddrStack.Mode = AddrModeFlat;
+#else
+#error "Unsupported platform"
+#endif
+
+ while (result.size() < 64) {
+ if (!StackWalk64(type, GetCurrentProcess(), GetCurrentThread(),
+ &frame, context, NULL, &SymFunctionTableAccess64,
+ &SymGetModuleBase64, NULL)) {
+ DWORD lastError = GetLastError();
+ break;
+ }
+ if (frame.AddrPC.Offset != 0) {
+ result.push_back((void *)frame.AddrPC.Offset);
+ }
+ }
+#endif
+ return result;
+}
+
+void fiberBacktrace(Fiber *fiber)
+{
+ std::string bt = to_string(fiber->backtrace());
+#ifdef WINDOWS
+ OutputDebugStringA(bt.c_str());
+ OutputDebugStringA("\n");
+#else
+ std::cout << bt << std::endl;
+#endif
+}
+
+
}
View
9 mordor/fiber.h
@@ -152,6 +152,13 @@ class Fiber : public boost::enable_shared_from_this<Fiber>
/// The current execution state of the Fiber
State state();
+ /// Get the backtrace of a fiber
+
+ /// The fiber must not be currently executing. If it's in a state other
+ /// than HOLD, the backtrace will be empty.
+ /// @pre state() != EXEC
+ std::vector<void *> backtrace();
+
private:
void call(bool destructor);
Fiber::ptr yieldTo(bool yieldToCallerOnTerminate, State targetState);
@@ -194,6 +201,8 @@ class Fiber : public boost::enable_shared_from_this<Fiber>
std::vector<intptr_t> m_fls;
};
+void fiberBacktrace(Fiber *fiber);
+
template <class T>
class FiberLocalStorageBase : boost::noncopyable
{
View
120 mordor/http/auth.cpp
@@ -5,9 +5,18 @@
#include "auth.h"
#include "basic.h"
+#include "client.h"
#include "digest.h"
+#include "mordor/socket.h"
+
#ifdef WINDOWS
#include "negotiate.h"
+#elif defined (OSX)
+#include "mordor/util.h"
+
+#include <Security/SecKeychain.h>
+#include <Security/SecKeychainItem.h>
+#include <Security/SecKeychainSearch.h>
#endif
namespace Mordor {
@@ -79,10 +88,10 @@ ClientAuthBroker::request(Request &requestHeaders,
} else {
return request;
}
- } catch (SocketException) {
+ } catch (SocketException &) {
m_conn = m_dg();
continue;
- } catch (PriorRequestFailedException) {
+ } catch (PriorRequestFailedException &) {
m_conn = m_dg();
continue;
}
@@ -139,6 +148,17 @@ AuthRequestBroker::request(Request &requestHeaders, bool forceNewConnection,
m_getCredentialsDg(requestHeaders.requestLine.uri, priorRequest,
scheme, realm, username, password, attempts++)) {
#ifdef WINDOWS
+ MORDOR_ASSERT(
+ stricmp(scheme.c_str(), "Negotiate") == 0 ||
+ stricmp(scheme.c_str(), "NTLM") == 0 ||
+ stricmp(scheme.c_str(), "Digest") == 0 ||
+ stricmp(scheme.c_str(), "Basic") == 0);
+#else
+ MORDOR_ASSERT(
+ stricmp(scheme.c_str(), "Digest") == 0 ||
+ stricmp(scheme.c_str(), "Basic") == 0);
+#endif
+#ifdef WINDOWS
if (scheme == "Negotiate" || scheme == "NTLM") {
negotiateAuth.reset(new NegotiateAuth(username, password));
negotiateAuth->authorize(
@@ -167,6 +187,17 @@ AuthRequestBroker::request(Request &requestHeaders, bool forceNewConnection,
m_getProxyCredentialsDg(requestHeaders.requestLine.uri,
priorRequest, scheme, realm, username, password, proxyAttempts++)) {
#ifdef WINDOWS
+ MORDOR_ASSERT(
+ stricmp(scheme.c_str(), "Negotiate") == 0 ||
+ stricmp(scheme.c_str(), "NTLM") == 0 ||
+ stricmp(scheme.c_str(), "Digest") == 0 ||
+ stricmp(scheme.c_str(), "Basic") == 0);
+#else
+ MORDOR_ASSERT(
+ stricmp(scheme.c_str(), "Digest") == 0 ||
+ stricmp(scheme.c_str(), "Basic") == 0);
+#endif
+#ifdef WINDOWS
if (scheme == "Negotiate" || scheme == "NTLM") {
negotiateProxyAuth.reset(new NegotiateAuth(username, password));
negotiateProxyAuth->authorize(
@@ -218,4 +249,89 @@ AuthRequestBroker::request(Request &requestHeaders, bool forceNewConnection,
}
}
+#ifdef OSX
+bool getCredentialsFromKeychain(const URI &uri, ClientRequest::ptr priorRequest,
+ std::string &scheme, std::string &realm, std::string &username,
+ std::string &password, size_t attempts)
+{
+ if (attempts != 1)
+ return false;
+ bool proxy =
+ priorRequest->response().status.status == PROXY_AUTHENTICATION_REQUIRED;
+ const ChallengeList &challengeList = proxy ?
+ priorRequest->response().response.proxyAuthenticate :
+ priorRequest->response().response.wwwAuthenticate;
+ if (isAcceptable(challengeList, "Basic"))
+ scheme = "Basic";
+ else if (isAcceptable(challengeList, "Digest"))
+ scheme = "Digest";
+ else
+ return false;
+
+ std::vector<SecKeychainAttribute> attrVector;
+ std::string host = uri.authority.host();
+ attrVector.push_back((SecKeychainAttribute){kSecServerItemAttr, host.size(),
+ (void *)host.c_str()});
+
+ UInt32 port = 0;
+ if (uri.authority.portDefined()) {
+ port = uri.authority.port();
+ attrVector.push_back((SecKeychainAttribute){kSecPortItemAttr,
+ sizeof(UInt32), &port});
+ }
+ SecProtocolType protocol;
+ if (proxy && priorRequest->request().requestLine.method == CONNECT)
+ protocol = kSecProtocolTypeHTTPSProxy;
+ else if (proxy)
+ protocol = kSecProtocolTypeHTTPProxy;
+ else if (uri.scheme() == "https")
+ protocol = kSecProtocolTypeHTTPS;
+ else if (uri.scheme() == "http")
+ protocol = kSecProtocolTypeHTTP;
+ else
+ MORDOR_NOTREACHED();
+ attrVector.push_back((SecKeychainAttribute){kSecProtocolItemAttr,
+ sizeof(SecProtocolType), &protocol});
+
+ ScopedCFRef<SecKeychainSearchRef> search;
+ SecKeychainAttributeList attrList;
+ attrList.count = (UInt32)attrVector.size();
+ attrList.attr = &attrVector[0];
+
+ OSStatus status = SecKeychainSearchCreateFromAttributes(NULL,
+ kSecInternetPasswordItemClass, &attrList, &search);
+ if (status != 0)
+ return false;
+ ScopedCFRef<SecKeychainItemRef> item;
+ status = SecKeychainSearchCopyNext(search, &item);
+ if (status != 0)
+ return false;
+ SecKeychainAttributeInfo info;
+ SecKeychainAttrType tag = kSecAccountItemAttr;
+ CSSM_DB_ATTRIBUTE_FORMAT format = CSSM_DB_ATTRIBUTE_FORMAT_STRING;
+ info.count = 1;
+ info.tag = (UInt32 *)&tag;
+ info.format = (UInt32 *)&format;
+
+ SecKeychainAttributeList *attrs;
+ UInt32 passwordLength = 0;
+ void *passwordBytes = NULL;
+
+ status = SecKeychainItemCopyAttributesAndData(item, &info, NULL, &attrs,
+ &passwordLength, &passwordBytes);
+ if (status != 0)
+ return false;
+
+ try {
+ username.assign((const char *)attrs->attr[0].data, attrs->attr[0].length);
+ password.assign((const char *)passwordBytes, passwordLength);
+ } catch (...) {
+ SecKeychainItemFreeContent(attrs, passwordBytes);
+ throw;
+ }
+ SecKeychainItemFreeContent(attrs, passwordBytes);
+ return true;
+}
+#endif
+
}}
View
31 mordor/http/auth.h
@@ -7,7 +7,7 @@
#include <boost/scoped_ptr.hpp>
#include "broker.h"
-#include "client.h"
+#include "mordor/version.h"
namespace Mordor {
namespace HTTP {
@@ -16,7 +16,7 @@ namespace HTTP {
class ClientAuthBroker : public boost::noncopyable
{
public:
- ClientAuthBroker(boost::function<ClientConnection::ptr ()> dg,
+ ClientAuthBroker(boost::function<boost::shared_ptr<ClientConnection> ()> dg,
const std::string &username, const std::string &password,
const std::string &proxyUsername, const std::string &proxyPassword)
: m_dg(dg),
@@ -27,13 +27,13 @@ class ClientAuthBroker : public boost::noncopyable
{}
// optional dg is to provide the request body if necessary
- ClientRequest::ptr request(Request &requestHeaders,
- boost::function< void (ClientRequest::ptr)> dg = NULL);
+ boost::shared_ptr<ClientRequest> request(Request &requestHeaders,
+ boost::function< void (boost::shared_ptr<ClientRequest>)> dg = NULL);
private:
- boost::function<ClientConnection::ptr ()> m_dg;
+ boost::function<boost::shared_ptr<ClientConnection> ()> m_dg;
std::string m_username, m_password, m_proxyUsername, m_proxyPassword;
- ClientConnection::ptr m_conn;
+ boost::shared_ptr<ClientConnection> m_conn;
};
class AuthRequestBroker : public RequestBrokerFilter
@@ -41,13 +41,13 @@ class AuthRequestBroker : public RequestBrokerFilter
public:
AuthRequestBroker(RequestBroker::ptr parent,
boost::function<bool (const URI &,
- ClientRequest::ptr /* priorRequest = ClientRequest::ptr() */,
+ boost::shared_ptr<ClientRequest> /* priorRequest = ClientRequest::ptr() */,
std::string & /* scheme */, std::string & /* realm */,
std::string & /* username */, std::string & /* password */,
size_t /* attempts */)>
getCredentialsDg,
boost::function<bool (const URI &,
- ClientRequest::ptr /* priorRequest = ClientRequest::ptr() */,
+ boost::shared_ptr<ClientRequest> /* priorRequest = ClientRequest::ptr() */,
std::string & /* scheme */, std::string & /* realm */,
std::string & /* username */, std::string & /* password */,
size_t /* attempts */)>
@@ -57,15 +57,22 @@ class AuthRequestBroker : public RequestBrokerFilter
m_getProxyCredentialsDg(getProxyCredentialsDg)
{}
- ClientRequest::ptr request(Request &requestHeaders,
+ boost::shared_ptr<ClientRequest> request(Request &requestHeaders,
bool forceNewConnection = false,
- boost::function<void (ClientRequest::ptr)> bodyDg = NULL);
+ boost::function<void (boost::shared_ptr<ClientRequest>)> bodyDg = NULL);
private:
- boost::function<bool (const URI &, ClientRequest::ptr, std::string &,
- std::string &, std::string &, std::string &, size_t)>
+ boost::function<bool (const URI &, boost::shared_ptr<ClientRequest>,
+ std::string &, std::string &, std::string &, std::string &, size_t)>
m_getCredentialsDg, m_getProxyCredentialsDg;
};
+
+#ifdef OSX
+bool getCredentialsFromKeychain(const URI &uri,
+ boost::shared_ptr<ClientRequest> priorRequest,
+ std::string &scheme, std::string &realm, std::string &username,
+ std::string &password, size_t attempts);
+#endif
}}
View
8 mordor/http/basic.cpp
@@ -4,6 +4,7 @@
#include "basic.h"
+#include "http.h"
#include "mordor/string.h"
namespace Mordor {
@@ -18,4 +19,11 @@ void authorize(AuthParams &authorization, const std::string &username,
authorization.parameters.clear();
}
+void authorize(Request &nextRequest,
+ const std::string &username, const std::string &password,
+ bool proxy)
+{ authorize(proxy ? nextRequest.request.proxyAuthorization :
+ nextRequest.request.authorization, username, password); }
+
+
}}}
View
12 mordor/http/basic.h
@@ -2,21 +2,23 @@
#define __MORDOR_HTTP_BASIC_AUTH_H__
// Copyright (c) 2009 - Mozy, Inc.
-#include "http.h"
+#include <string>
namespace Mordor {
namespace HTTP {
+
+struct AuthParams;
+struct Request;
+
namespace BasicAuth {
void authorize(AuthParams &authorization, const std::string &username,
const std::string &password);
/// @deprecated
-inline void authorize(Request &nextRequest,
+void authorize(Request &nextRequest,
const std::string &username, const std::string &password,
- bool proxy = false)
-{ authorize(proxy ? nextRequest.request.proxyAuthorization :
- nextRequest.request.authorization, username, password); }
+ bool proxy = false);
}}}
View
470 mordor/http/broker.cpp
@@ -5,14 +5,17 @@
#include "broker.h"
#include "auth.h"
+#include "client.h"
#include "mordor/atomic.h"
#include "mordor/future.h"
+#include "mordor/socks.h"
#include "mordor/streams/buffered.h"
#include "mordor/streams/pipe.h"
#include "mordor/streams/socket.h"
#include "mordor/streams/ssl.h"
#include "mordor/streams/timeout.h"
#include "proxy.h"
+#include "server.h"
namespace Mordor {
namespace HTTP {
@@ -34,55 +37,32 @@ createRequestBroker(const RequestBrokerOptions &options)
streamBroker = options.customStreamBrokerFilter;
}
- SSLStreamBroker::ptr sslBroker(new SSLStreamBroker(streamBroker,
- options.sslCtx, options.verifySslCertificate,
- options.verifySslCertificateHost));
- sslBroker->timerManager(timerManager);
- sslBroker->readTimeout(options.sslConnectReadTimeout);
- sslBroker->writeTimeout(options.sslConnectWriteTimeout);
-
- ConnectionCache::ptr connectionCache(new ConnectionCache(sslBroker, 1u,
+ ConnectionCache::ptr connectionCache(new ConnectionCache(streamBroker,
timerManager));
- connectionCache->readTimeout(options.httpReadTimeout);
- connectionCache->writeTimeout(options.httpWriteTimeout);
-
- RequestBroker::ptr requestBroker;
- if (options.proxyForURIDg)
- requestBroker.reset(new BaseRequestBroker(
- ConnectionBroker::weak_ptr(connectionCache)));
- else
- requestBroker.reset(new BaseRequestBroker(
- boost::static_pointer_cast<ConnectionBroker>(connectionCache)));
-
- if (options.delayDg)
- requestBroker.reset(new RetryRequestBroker(requestBroker,
- options.delayDg));
- if ((options.proxyForURIDg && options.getProxyCredentialsDg) ||
- (!options.proxyForURIDg && options.getCredentialsDg))
+ connectionCache->httpReadTimeout(options.httpReadTimeout);
+ connectionCache->httpWriteTimeout(options.httpWriteTimeout);
+ connectionCache->sslReadTimeout(options.sslConnectReadTimeout);
+ connectionCache->sslWriteTimeout(options.sslConnectWriteTimeout);
+ connectionCache->sslCtx(options.sslCtx);
+ connectionCache->proxyForURI(options.proxyForURIDg);
+ connectionCache->proxyRequestBroker(options.proxyRequestBroker);
+ connectionCache->verifySslCertificate(options.verifySslCertificate);
+ connectionCache->verifySslCertificateHost(options.verifySslCertificateHost);
+
+ RequestBroker::ptr requestBroker(new BaseRequestBroker(
+ boost::static_pointer_cast<ConnectionBroker>(connectionCache)));
+
+ if (options.getCredentialsDg || options.getProxyCredentialsDg)
requestBroker.reset(new AuthRequestBroker(requestBroker,
- options.proxyForURIDg ? options.getCredentialsDg : NULL,
- options.proxyForURIDg ? options.getProxyCredentialsDg : NULL));
+ options.getCredentialsDg, options.getProxyCredentialsDg));
if (options.handleRedirects)
requestBroker.reset(new RedirectRequestBroker(requestBroker));
-
- if (options.proxyForURIDg) {
- ProxyStreamBroker::ptr proxyStreamBroker(new ProxyStreamBroker(socketBroker, requestBroker,
- options.proxyForURIDg));
- proxyStreamBroker->fallbackOnFailure(options.fallbackToDirectOnProxyFailure);
- sslBroker->parent(boost::static_pointer_cast<StreamBroker>(proxyStreamBroker));
- ProxyConnectionBroker::ptr connectionBroker(
- new ProxyConnectionBroker(connectionCache, options.proxyForURIDg));
- connectionBroker->fallbackOnFailure(options.fallbackToDirectOnProxyFailure);
- requestBroker.reset(new BaseRequestBroker(boost::static_pointer_cast<ConnectionBroker>(connectionBroker)));
- if (options.delayDg)
- requestBroker.reset(new RetryRequestBroker(requestBroker,
- options.delayDg));
- if (options.getCredentialsDg || options.getProxyCredentialsDg)
- requestBroker.reset(new AuthRequestBroker(requestBroker,
- options.getCredentialsDg, options.getProxyCredentialsDg));
- if (options.handleRedirects)
- requestBroker.reset(new RedirectRequestBroker(requestBroker));
- }
+ if (options.delayDg)
+ requestBroker.reset(new RetryRequestBroker(requestBroker,
+ options.delayDg));
+ if (!options.userAgent.empty())
+ requestBroker.reset(new UserAgentRequestBroker(requestBroker,
+ options.userAgent));
return std::make_pair(requestBroker, connectionCache);
}
@@ -178,40 +158,6 @@ SocketStreamBroker::cancelPending()
}
}
-Stream::ptr
-SSLStreamBroker::getStream(const URI &uri)
-{
- Stream::ptr result = parent()->getStream(uri);
- if (uri.schemeDefined() && uri.scheme() == "https") {
- TimeoutStream::ptr timeoutStream;
- if (m_timerManager) {
- timeoutStream.reset(new TimeoutStream(result, *m_timerManager));
- timeoutStream->readTimeout(m_readTimeout);
- timeoutStream->writeTimeout(m_writeTimeout);
- result = timeoutStream;
- }
- BufferedStream::ptr bufferedStream(new BufferedStream(result));
- bufferedStream->allowPartialReads(true);
- SSLStream::ptr sslStream(new SSLStream(bufferedStream, true, true, m_sslCtx));
- sslStream->connect();
- if (m_verifySslCertificate)
- sslStream->verifyPeerCertificate();
- if (m_verifySslCertificateHost)
- sslStream->verifyPeerCertificate(uri.authority.host());
- if (timeoutStream) {
- bufferedStream->parent(timeoutStream->parent());
- timeoutStream.reset();
- }
- bufferedStream.reset(new BufferedStream(sslStream));
- // Max data in each SSL record
- bufferedStream->bufferSize(16384);
- bufferedStream->flushMultiplesOfBuffer(true);
- bufferedStream->allowPartialReads(true);
- return bufferedStream;
- }
- return result;
-}
-
static bool least(const ClientConnection::ptr &lhs,
const ClientConnection::ptr &rhs)
{
@@ -228,105 +174,181 @@ static bool least(const ClientConnection::ptr &lhs,
std::pair<ClientConnection::ptr, bool>
ConnectionCache::getConnection(const URI &uri, bool forceNewConnection)
{
+ std::vector<URI> proxies;
+ if (m_proxyForURIDg)
+ proxies = m_proxyForURIDg(uri);
+ // Remove proxy types that aren't supported
+ for (std::vector<URI>::iterator it(proxies.begin());
+ it != proxies.end();
+ ++it) {
+ MORDOR_ASSERT(it->schemeDefined() || !it->isDefined());
+ if (!it->schemeDefined())
+ continue;
+ std::string scheme = it->scheme();
+ if (scheme != "http" && (scheme != "https" || !m_proxyBroker) &&
+ scheme != "socks")
+ it = proxies.erase(it, it);
+ }
URI schemeAndAuthority;
- std::map<URI, std::pair<ConnectionList,
- boost::shared_ptr<FiberCondition> > >::iterator it, it3;
- ConnectionList::iterator it2;
+ schemeAndAuthority = uri;
+ schemeAndAuthority.path = URI::Path();
+ schemeAndAuthority.queryDefined(false);
+ schemeAndAuthority.fragmentDefined(false);
std::pair<ClientConnection::ptr, bool> result;
- {
- FiberMutex::ScopedLock lock(m_mutex);
- if (m_closed)
- MORDOR_THROW_EXCEPTION(OperationAbortedException());
- // Clean out any dead conns
- for (it = m_conns.begin(); it != m_conns.end();) {
- for (it2 = it->second.first.begin();
- it2 != it->second.first.end();) {
- if (*it2 && !(*it2)->newRequestsAllowed()) {
- it2 = it->second.first.erase(it2);
- } else {
- ++it2;
- }
- }
- if (it->second.first.empty()) {
- it3 = it;
- ++it3;
- m_conns.erase(it);
- it = it3;
- } else {
- ++it;
- }
+
+ FiberMutex::ScopedLock lock(m_mutex);
+ if (m_closed)
+ MORDOR_THROW_EXCEPTION(OperationAbortedException());
+ // Clean out any dead conns
+ cleanOutDeadConns(m_conns);
+
+ if (!forceNewConnection) {
+ if (proxies.empty()) {
+ result = getConnectionViaProxyFromCache(schemeAndAuthority, URI());
+ if (result.first)
+ return result;
}
+ for (std::vector<URI>::const_iterator it(proxies.begin());
+ it != proxies.end();
+ ++it) {
+ result = getConnectionViaProxyFromCache(schemeAndAuthority, *it);
+ if (result.first)
+ return result;
+ }
+ }
- schemeAndAuthority = uri;
- schemeAndAuthority.path = URI::Path();
- schemeAndAuthority.queryDefined(false);
- schemeAndAuthority.fragmentDefined(false);
-
- if (!forceNewConnection) {
- while (true) {
- // Look for an existing connection
- it = m_conns.find(schemeAndAuthority);
- if (it != m_conns.end() &&
- !it->second.first.empty() &&
- it->second.first.size() >= m_connectionsPerHost) {
- ConnectionList &connsForThisUri = it->second.first;
- // Assign it2 to point to the connection with the
- // least number of outstanding requests
- it2 = std::min_element(connsForThisUri.begin(),
- connsForThisUri.end(), &least);
- // No connection has completed yet (but it's in progress)
- if (!*it2) {
- // Wait for somebody to let us try again
- it->second.second->wait();
- } else {
- // Return the existing, completed connection
- return std::make_pair(*it2, false);
- }
- } else {
- // No existing connections
- break;
- }
- }
+ // Create a new connection
+ if (proxies.empty())
+ return getConnectionViaProxy(schemeAndAuthority, URI(), lock);
+ std::vector<URI>::const_iterator it = proxies.begin();
+ while(true) {
+ try {
+ return getConnectionViaProxy(schemeAndAuthority, *it, lock);
+ } catch (SocketException &) {
+ if (++it == proxies.end())
+ throw;
+ } catch (HTTP::Exception &) {
+ if (++it == proxies.end())
+ throw;
+ } catch (UnexpectedEofException &) {
+ if (++it == proxies.end())
+ throw;
}
- // Create a new (blank) connection
- m_conns[schemeAndAuthority].first.push_back(ClientConnection::ptr());
- if (it == m_conns.end()) {
- // This is the first connection for this schemeAndAuthority
- it = m_conns.find(schemeAndAuthority);
- // (double-check)
- if (!it->second.second)
- // Create the condition variable for it
- it->second.second.reset(new FiberCondition(m_mutex));
+ }
+}
+
+std::pair<ClientConnection::ptr, bool>
+ConnectionCache::getConnectionViaProxyFromCache(const URI &uri, const URI &proxy)
+{
+ bool proxied = proxy.schemeDefined() && proxy.scheme() == "http";
+ const URI &endpoint = proxied ? proxy : uri;
+ CachedConnectionMap::iterator it = m_conns.find(endpoint);
+ ConnectionList::iterator it2;
+ while (true) {
+ if (it != m_conns.end() &&
+ !it->second.first.empty() &&
+ it->second.first.size() >= m_connectionsPerHost) {
+ ConnectionList &connsForThisUri = it->second.first;
+ // Assign it2 to point to the connection with the
+ // least number of outstanding requests
+ it2 = std::min_element(connsForThisUri.begin(),
+ connsForThisUri.end(), &least);
+ // No connection has completed yet (but it's in progress)
+ if (!*it2) {
+ // Wait for somebody to let us try again
+ it->second.second->wait();
+ // We let go of the mutex, and the last connection may have
+ // disappeared
+ it = m_conns.find(endpoint);
+ } else {
+ // Return the existing, completed connection
+ return std::make_pair(*it2, proxied);
+ }
+ } else {
+ // No existing connections
+ return std::make_pair(ClientConnection::ptr(), false);
}
}
+}
+
+std::pair<ClientConnection::ptr, bool>
+ConnectionCache::getConnectionViaProxy(const URI &uri, const URI &proxy,
+ FiberMutex::ScopedLock &lock)
+{
+ std::string proxyScheme;
+ if (proxy.schemeDefined())
+ proxyScheme = proxy.scheme();
+ bool proxied = proxyScheme == "http";
+ const URI &endpoint = proxied ? proxy : uri;
+
+ // Make sure we have a ConnectionList and mutex for this endpoint
+ CachedConnectionMap::iterator it = m_conns.find(endpoint);
+ if (it == m_conns.end()) {
+ it = m_conns.insert(std::make_pair(endpoint,
+ std::make_pair(ConnectionList(),
+ boost::shared_ptr<FiberCondition>()))).first;
+ it->second.second.reset(new FiberCondition(m_mutex));
+ }
+ // Add a placeholder for the new connection
+ it->second.first.push_back(ClientConnection::ptr());
+ lock.unlock();
+
+ ConnectionList::iterator it2;
+ std::pair<ClientConnection::ptr, bool> result;
// Establish a new connection
try {
- Stream::ptr stream = m_streamBroker->getStream(schemeAndAuthority);
- {
- FiberMutex::ScopedLock lock(m_mutex);
- result = std::make_pair(ClientConnection::ptr(
- new ClientConnection(stream, m_timerManager)), false);
- if (m_readTimeout != ~0ull)
- result.first->readTimeout(m_readTimeout);
- if (m_writeTimeout != ~0ull)
- result.first->writeTimeout(m_writeTimeout);
- // Assign this connection to the first blank connection for this
- // schemeAndAuthority
- // it should still be valid, even if the map changed
- for (it2 = it->second.first.begin();
- it2 != it->second.first.end();
- ++it2) {
- if (!*it2) {
- *it2 = result.first;
- break;
- }
+ Stream::ptr stream;
+ if (proxyScheme == "https") {
+ stream = tunnel(m_proxyBroker, proxy, uri);
+ } else if (proxyScheme == "socks") {
+ unsigned short port;
+ if (uri.authority.portDefined())
+ port = uri.authority.port();
+ else if (uri.scheme() == "http")
+ port = 80;
+ else if (uri.scheme() == "https")
+ port = 443;
+ else
+ // TODO: can this be looked up using the system? (getaddrinfo)
+ MORDOR_THROW_EXCEPTION(std::invalid_argument("Unknown protocol for proxying connection"));
+ stream = SOCKS::tunnel(m_streamBroker, proxy, IPAddress::ptr(),
+ uri.authority.host(), port);
+ } else {
+ stream = m_streamBroker->getStream(endpoint);
+ }
+ addSSL(endpoint, stream);
+ lock.lock();
+ // Somebody called abortConnections while we were unlocked; just throw
+ // this connection away
+ if (m_closed)
+ MORDOR_THROW_EXCEPTION(OperationAbortedException());
+ result = std::make_pair(ClientConnection::ptr(
+ new ClientConnection(stream, m_timerManager)), proxied);
+ if (m_httpReadTimeout != ~0ull)
+ result.first->readTimeout(m_httpReadTimeout);
+ if (m_httpWriteTimeout != ~0ull)
+ result.first->writeTimeout(m_httpWriteTimeout);
+ // Assign this connection to the first blank connection for this
+ // schemeAndAuthority
+ // it should still be valid, even if the map changed
+ for (it2 = it->second.first.begin();
+ it2 != it->second.first.end();
+ ++it2) {
+ if (!*it2) {
+ *it2 = result.first;
+ break;
}
- // Unblock all waiters for them to choose an existing connection
- it->second.second->broadcast();
}
+ // Unblock all waiters for them to choose an existing connection
+ it->second.second->broadcast();
} catch (...) {
- FiberMutex::ScopedLock lock(m_mutex);
+ lock.lock();
+ // Somebody called abortConnections while we were unlocked; no need to
+ // clean up the temporary spot for this connection, since it's gone;
+ // pass the original exception on, though
+ if (m_closed)
+ throw;
// This connection attempt failed; remove the first blank connection
// for this schemeAndAuthority to let someone else try to establish a
// connection
@@ -348,9 +370,39 @@ ConnectionCache::getConnection(const URI &uri, bool forceNewConnection)
}
void
-ConnectionCache::closeConnections()
+ConnectionCache::closeIdleConnections()
+{
+ FiberMutex::ScopedLock lock(m_mutex);
+ // We don't just clear the list, because there may be a connection in
+ // in progress that has an iterator into it
+ std::map<URI, std::pair<ConnectionList,
+ boost::shared_ptr<FiberCondition> > >::iterator it, extraIt;
+ for (it = m_conns.begin(); it != m_conns.end();) {
+ it->second.second->broadcast();
+ for (ConnectionList::iterator it2 = it->second.first.begin();
+ it2 != it->second.first.end();) {
+ if (*it2) {
+ Stream::ptr connStream = (*it2)->stream();
+ connStream->cancelRead();
+ connStream->cancelWrite();
+ it2 = it->second.first.erase(it2);
+ } else {
+ ++it2;
+ }
+ }
+ if (it->second.first.empty()) {
+ extraIt = it;
+ ++it;
+ m_conns.erase(extraIt);
+ } else {
+ ++it;
+ }
+ }
+}
+
+void
+ConnectionCache::abortConnections()
{
- m_streamBroker->cancelPending();
FiberMutex::ScopedLock lock(m_mutex);
m_closed = true;
std::map<URI, std::pair<ConnectionList,
@@ -368,6 +420,65 @@ ConnectionCache::closeConnections()
}
}
m_conns.clear();
+ lock.unlock();
+ m_streamBroker->cancelPending();
+}
+
+void
+ConnectionCache::cleanOutDeadConns(CachedConnectionMap &conns)
+{
+ CachedConnectionMap::iterator it, it3;
+ ConnectionList::iterator it2;
+ for (it = conns.begin(); it != conns.end();) {
+ for (it2 = it->second.first.begin();
+ it2 != it->second.first.end();) {
+ if (*it2 && !(*it2)->newRequestsAllowed()) {
+ it2 = it->second.first.erase(it2);
+ } else {
+ ++it2;
+ }
+ }
+ if (it->second.first.empty()) {
+ it3 = it;
+ ++it3;
+ conns.erase(it);
+ it = it3;
+ } else {
+ ++it;
+ }
+ }
+}
+
+void
+ConnectionCache::addSSL(const URI &uri, Stream::ptr &stream)
+{
+ if (uri.schemeDefined() && uri.scheme() == "https") {
+ TimeoutStream::ptr timeoutStream;
+ if (m_timerManager) {
+ timeoutStream.reset(new TimeoutStream(stream, *m_timerManager));
+ timeoutStream->readTimeout(m_sslReadTimeout);
+ timeoutStream->writeTimeout(m_sslWriteTimeout);
+ stream = timeoutStream;
+ }
+ BufferedStream::ptr bufferedStream(new BufferedStream(stream));
+ bufferedStream->allowPartialReads(true);
+ SSLStream::ptr sslStream(new SSLStream(bufferedStream, true, true, m_sslCtx));
+ sslStream->connect();
+ if (m_verifySslCertificate)
+ sslStream->verifyPeerCertificate();
+ if (m_verifySslCertificateHost)
+ sslStream->verifyPeerCertificate(uri.authority.host());
+ if (timeoutStream) {
+ bufferedStream->parent(timeoutStream->parent());
+ timeoutStream.reset();
+ }
+ bufferedStream.reset(new BufferedStream(sslStream));
+ // Max data in each SSL record
+ bufferedStream->bufferSize(16384);
+ bufferedStream->flushMultiplesOfBuffer(true);
+ bufferedStream->allowPartialReads(true);
+ stream = bufferedStream;
+ }
}
std::pair<ClientConnection::ptr, bool>
@@ -446,10 +557,13 @@ BaseRequestBroker::request(Request &requestHeaders, bool forceNewConnection,
bool connect = requestHeaders.requestLine.method == CONNECT;
MORDOR_ASSERT(connect || originalUri.authority.hostDefined());
MORDOR_ASSERT(!connect || !requestHeaders.request.host.empty());
- if (!connect)
+ if (!connect) {
requestHeaders.request.host = originalUri.authority.host();
- else
- originalUri = "http://" + requestHeaders.request.host;
+ } else {
+ MORDOR_ASSERT(originalUri.scheme() == "http");
+ MORDOR_ASSERT(originalUri.path.segments.size() == 1);
+ currentUri = originalUri.path.segments[0];
+ }
ConnectionBroker::ptr connectionBroker = m_connectionBroker;
if (!connectionBroker)
connectionBroker = m_weakConnectionBroker.lock();
@@ -458,13 +572,11 @@ BaseRequestBroker::request(Request &requestHeaders, bool forceNewConnection,
conn = connectionBroker->getConnection(
connect ? originalUri : currentUri, forceNewConnection);
} catch (boost::exception &ex) {
- if (!connect)
- currentUri = originalUri;
+ currentUri = originalUri;
ex << errinfo_source(CONNECTION);
throw;
} catch (...) {
- if (!connect)
- currentUri = originalUri;
+ currentUri = originalUri;
throw;
}
try {
@@ -483,6 +595,8 @@ BaseRequestBroker::request(Request &requestHeaders, bool forceNewConnection,
ClientRequest::ptr request;
try {
request = conn.first->request(requestHeaders);
+ if (!bodyDg)
+ request->doRequest();
} catch (boost::exception &ex) {
ex << errinfo_source(HTTP);
throw;
@@ -494,8 +608,7 @@ BaseRequestBroker::request(Request &requestHeaders, bool forceNewConnection,
Scheduler::getThis()->schedule(boost::bind(&doBody,
request, bodyDg, boost::ref(future), boost::ref(exception),
boost::ref(exceptionWasHttp)));
- if (!connect)
- currentUri = originalUri;
+ currentUri = originalUri;
try {
// Force reading the response here to check for connectivity problems
request->response();
@@ -526,8 +639,7 @@ BaseRequestBroker::request(Request &requestHeaders, bool forceNewConnection,
Mordor::rethrow_exception(exception);
return request;
} catch (...) {
- if (!connect)
- currentUri = originalUri;
+ currentUri = originalUri;
throw;
}
}
@@ -547,7 +659,7 @@ RetryRequestBroker::request(Request &requestHeaders, bool forceNewConnection,
return request;
} catch (SocketException &ex) {
const ExceptionSource *source = boost::get_error_info<errinfo_source>(ex);
- if (!source || *source != HTTP)
+ if (!source || (*source != HTTP && *source != CONNECTION))
throw;
if (m_delayDg && !m_delayDg(atomicIncrement(*retries)))
throw;
View
163 mordor/http/broker.h
@@ -4,13 +4,24 @@
#include <openssl/ssl.h>
-#include "client.h"
+#include "http.h"
+#include "mordor/iomanager.h"
#include "mordor/scheduler.h"
-#include "server.h"
namespace Mordor {
+
+class Socket;
+class Stream;
+class TimerManager;
+
namespace HTTP {
+class ClientConnection;
+class ClientRequest;
+class RequestBroker;
+class ServerConnection;
+class ServerRequest;
+
class StreamBroker
{
public:
@@ -20,7 +31,7 @@ class StreamBroker
public:
virtual ~StreamBroker() {}
- virtual Stream::ptr getStream(const URI &uri) = 0;
+ virtual boost::shared_ptr<Stream> getStream(const URI &uri) = 0;
virtual void cancelPending() {}
};
@@ -64,49 +75,18 @@ class SocketStreamBroker : public StreamBroker
void connectTimeout(unsigned long long timeout) { m_connectTimeout = timeout; }
- Stream::ptr getStream(const URI &uri);
+ boost::shared_ptr<Stream> getStream(const URI &uri);
void cancelPending();
private:
boost::mutex m_mutex;
bool m_cancelled;
- std::list<Socket::ptr> m_pending;
+ std::list<boost::shared_ptr<Socket> > m_pending;
IOManager *m_ioManager;
Scheduler *m_scheduler;
unsigned long long m_connectTimeout;
};
-class SSLStreamBroker : public StreamBrokerFilter
-{
-public:
- typedef boost::shared_ptr<SSLStreamBroker> ptr;
-
-public:
- SSLStreamBroker(StreamBroker::ptr parent,
- SSL_CTX *sslCtx = NULL, bool verifySslCertificate = false,
- bool verifySslCertificateHost = true)
- : StreamBrokerFilter(parent),
- m_sslCtx(sslCtx),
- m_timerManager(NULL),
- m_readTimeout(NULL),
- m_writeTimeout(NULL),
- m_verifySslCertificate(verifySslCertificate),
- m_verifySslCertificateHost(verifySslCertificate)
- {}
-
- void timerManager(TimerManager *timerManager) { m_timerManager = timerManager; }
- void readTimeout(unsigned long long timeout) { m_readTimeout = timeout; }
- void writeTimeout(unsigned long long timeout) { m_writeTimeout = timeout; }
-
- Stream::ptr getStream(const URI &uri);
-
-private:
- SSL_CTX *m_sslCtx;
- TimerManager *m_timerManager;
- unsigned long long m_readTimeout, m_writeTimeout;
- bool m_verifySslCertificate, m_verifySslCertificateHost;
-};
-
class ConnectionBroker
{
public:
@@ -116,7 +96,7 @@ class ConnectionBroker
public:
virtual ~ConnectionBroker() {}
- virtual std::pair<ClientConnection::ptr, bool>
+ virtual std::pair<boost::shared_ptr<ClientConnection>, bool>
getConnection(const URI &uri, bool forceNewConnection = false) = 0;
};
@@ -126,44 +106,80 @@ class ConnectionCache : public ConnectionBroker
typedef boost::shared_ptr<ConnectionCache> ptr;
public:
- ConnectionCache(StreamBroker::ptr streamBroker,
- size_t connectionsPerHost = 1, TimerManager *timerManager = NULL)
+ ConnectionCache(StreamBroker::ptr streamBroker, TimerManager *timerManager = NULL)
: m_streamBroker(streamBroker),
- m_connectionsPerHost(connectionsPerHost),
+ m_connectionsPerHost(1u),
m_closed(false),
- m_timerManager(NULL),
- m_readTimeout(~0ull),
- m_writeTimeout(~0ull)
+ m_verifySslCertificate(false),
+ m_verifySslCertificateHost(true),
+ m_timerManager(timerManager),
+ m_httpReadTimeout(~0ull),
+ m_httpWriteTimeout(~0ull),
+ m_sslReadTimeout(~0ull),
+ m_sslWriteTimeout(~0ull),
+ m_sslCtx(NULL)
{}
- void readTimeout(unsigned long long timeout) { m_readTimeout = timeout; }
- void writeTimeout(unsigned long long timeout) { m_writeTimeout = timeout; }
-
- std::pair<ClientConnection::ptr, bool>
+ void connectionsPerHost(size_t connections) { m_connectionsPerHost = connections; }
+ void httpReadTimeout(unsigned long long timeout) { m_httpReadTimeout = timeout; }
+ void httpWriteTimeout(unsigned long long timeout) { m_httpWriteTimeout = timeout; }
+ void sslReadTimeout(unsigned long long timeout) { m_sslReadTimeout = timeout; }
+ void sslWriteTimeout(unsigned long long timeout) { m_sslWriteTimeout = timeout; }
+ void sslCtx(SSL_CTX *ctx) { m_sslCtx = ctx; }
+ void verifySslCertificate(bool verify) { m_verifySslCertificate = verify; }
+ void verifySslCertificateHost(bool verify) { m_verifySslCertificateHost = verify; }
+ // Required to support any proxies
+ void proxyForURI(boost::function<std::vector<URI> (const URI &)> proxyForURIDg)
+ { m_proxyForURIDg = proxyForURIDg; }
+ // Required to support HTTPS proxies
+ void proxyRequestBroker(boost::shared_ptr<RequestBroker> broker)
+ { m_proxyBroker = broker; }
+
+ std::pair<boost::shared_ptr<ClientConnection>, bool>
getConnection(const URI &uri, bool forceNewConnection = false);
- void closeConnections();
+ void closeIdleConnections();
+ void abortConnections();
+
+private:
+ typedef std::list<boost::shared_ptr<ClientConnection> > ConnectionList;
+ typedef std::map<URI, std::pair<ConnectionList, boost::shared_ptr<FiberCondition> > >
+ CachedConnectionMap;
+
+private:
+ std::pair<boost::shared_ptr<ClientConnection>, bool>
+ getConnectionViaProxyFromCache(const URI &uri, const URI &proxy);
+ std::pair<boost::shared_ptr<ClientConnection>, bool>
+ getConnectionViaProxy(const URI &uri, const URI &proxy,
+ FiberMutex::ScopedLock &lock);
+ void cleanOutDeadConns(CachedConnectionMap &conns);
+ void addSSL(const URI &uri, boost::shared_ptr<Stream> &stream);
private:
FiberMutex m_mutex;
StreamBroker::ptr m_streamBroker;
size_t m_connectionsPerHost;
- typedef std::list<ClientConnection::ptr> ConnectionList;
- std::map<URI, std::pair<ConnectionList, boost::shared_ptr<FiberCondition> > > m_conns;
- bool m_closed;
+ CachedConnectionMap m_conns;
+ bool m_closed, m_verifySslCertificate, m_verifySslCertificateHost;
TimerManager *m_timerManager;
- unsigned long long m_readTimeout, m_writeTimeout;
+ unsigned long long m_httpReadTimeout, m_httpWriteTimeout, m_sslReadTimeout,
+ m_sslWriteTimeout;
+ SSL_CTX *m_sslCtx;
+ boost::function<std::vector<URI> (const URI &)> m_proxyForURIDg;
+ boost::shared_ptr<RequestBroker> m_proxyBroker;
};
class MockConnectionBroker : public ConnectionBroker
{
private:
typedef std::map<URI,
- std::pair<ClientConnection::ptr, ServerConnection::ptr> >
+ std::pair<boost::shared_ptr<ClientConnection>,
+ boost::shared_ptr<ServerConnection> > >
ConnectionCache;
public:
- MockConnectionBroker(boost::function<void (const URI &uri, ServerRequest::ptr)> dg,
+ MockConnectionBroker(boost::function<void (const URI &uri,
+ boost::shared_ptr<ServerRequest>)> dg,
TimerManager *timerManager = NULL, unsigned long long readTimeout = ~0ull,
unsigned long long writeTimeout = ~0ull)
: m_dg(dg),
@@ -172,11 +188,11 @@ class MockConnectionBroker : public ConnectionBroker
m_writeTimeout(writeTimeout)
{}
- std::pair<ClientConnection::ptr, bool>
+ std::pair<boost::shared_ptr<ClientConnection>, bool>
getConnection(const URI &uri, bool forceNewConnection = false);
private:
- boost::function<void (const URI &uri, ServerRequest::ptr)> m_dg;
+ boost::function<void (const URI &uri, boost::shared_ptr<ServerRequest>)> m_dg;
ConnectionCache m_conns;
TimerManager *m_timerManager;
unsigned long long m_readTimeout, m_writeTimeout;
@@ -191,9 +207,10 @@ class RequestBroker
public:
virtual ~RequestBroker() {}
- virtual ClientRequest::ptr request(Request &requestHeaders,
+ virtual boost::shared_ptr<ClientRequest> request(Request &requestHeaders,
bool forceNewConnection = false,
- boost::function<void (ClientRequest::ptr)> bodyDg = NULL) = 0;
+ boost::function<void (boost::shared_ptr<ClientRequest>)> bodyDg = NULL)
+ = 0;
};
class RequestBrokerFilter : public RequestBroker
@@ -207,9 +224,10 @@ class RequestBrokerFilter : public RequestBroker
RequestBroker::ptr parent();
- ClientRequest::ptr request(Request &requestHeaders,
+ boost::shared_ptr<ClientRequest> request(Request &requestHeaders,
bool forceNewConnection = false,
- boost::function<void (ClientRequest::ptr)> bodyDg = NULL) = 0;
+ boost::function<void (boost::shared_ptr<ClientRequest>)> bodyDg = NULL)
+ = 0;
private:
RequestBroker::ptr m_parent;
@@ -240,9 +258,9 @@ class BaseRequestBroker : public RequestBroker
: m_weakConnectionBroker(connectionBroker)
{}
- ClientRequest::ptr request(Request &requestHeaders,
+ boost::shared_ptr<ClientRequest> request(Request &requestHeaders,
bool forceNewConnection = false,
- boost::function<void (ClientRequest::ptr)> bodyDg = NULL);
+ boost::function<void (boost::shared_ptr<ClientRequest>)> bodyDg = NULL);
private:
ConnectionBroker::ptr m_connectionBroker;
@@ -265,9 +283,9 @@ class RetryRequestBroker : public RequestBrokerFilter
void sharedRetryCounter(size_t *retries) { mp_retries = retries; }
- ClientRequest::ptr request(Request &requestHeaders,
+ boost::shared_ptr<ClientRequest> request(Request &requestHeaders,
bool forceNewConnection = false,
- boost::function<void (ClientRequest::ptr)> bodyDg = NULL);
+ boost::function<void (boost::shared_ptr<ClientRequest>)> bodyDg = NULL);
private:
boost::function<bool (size_t)> m_delayDg;
@@ -305,9 +323,9 @@ class RedirectRequestBroker : public RequestBrokerFilter
void handleFound(bool handle) { m_handle302 = handle; }
void handleTemporaryRedirect(bool handle) { m_handle307 = handle; }
- ClientRequest::ptr request(Request &requestHeaders,
+ boost::shared_ptr<ClientRequest> request(Request &requestHeaders,
bool forceNewConnection = false,
- boost::function<void (ClientRequest::ptr)> bodyDg = NULL);
+ boost::function<void (boost::shared_ptr<ClientRequest>)> bodyDg = NULL);
private:
size_t m_maxRedirects;
@@ -323,9 +341,9 @@ class UserAgentRequestBroker : public RequestBrokerFilter
m_userAgent(userAgent)
{}
- ClientRequest::ptr request(Request &requestHeaders,
+ boost::shared_ptr<ClientRequest> request(Request &requestHeaders,
bool forceNewConnection = false,
- boost::function<void (ClientRequest::ptr)> bodyDg = NULL);
+ boost::function<void (boost::shared_ptr<ClientRequest>)> bodyDg = NULL);
private:
ProductAndCommentList m_userAgent;
@@ -343,7 +361,6 @@ struct RequestBrokerOptions
sslConnectWriteTimeout(~0ull),
httpReadTimeout(~0ull),
httpWriteTimeout(~0ull),
- fallbackToDirectOnProxyFailure(false),
sslCtx(NULL),
verifySslCertificate(false),
verifySslCertificateHost(true)
@@ -359,18 +376,20 @@ struct RequestBrokerOptions
unsigned long long sslConnectWriteTimeout;
unsigned long long httpReadTimeout;
unsigned long long httpWriteTimeout;
- boost::function<URI (const URI &)> proxyForURIDg;
+ boost::function<std::vector<URI> (const URI &)> proxyForURIDg;
+ /// Required to enable https proxy support
+ RequestBroker::ptr proxyRequestBroker;
boost::function<bool (const URI &,
- ClientRequest::ptr /* priorRequest = ClientRequest::ptr() */,
+ boost::shared_ptr<ClientRequest> /* priorRequest = ClientRequest::ptr() */,
std::string & /* scheme */, std::string & /* realm */,
std::string & /* username */, std::string & /* password */,
size_t /* attempts */)>
getCredentialsDg, getProxyCredentialsDg;
- bool fallbackToDirectOnProxyFailure;
StreamBrokerFilter::ptr customStreamBrokerFilter;
SSL_CTX *sslCtx;
bool verifySslCertificate;
bool verifySslCertificateHost;
+ ProductAndCommentList userAgent;
};
std::pair<RequestBroker::ptr, ConnectionCache::ptr>
View
238 mordor/http/client.cpp
@@ -10,6 +10,7 @@
#include "chunked.h"
#include "mordor/assert.h"
+#include "mordor/fiber.h"
#include "mordor/log.h"
#include "mordor/scheduler.h"
#include "mordor/streams/limited.h"
@@ -19,6 +20,7 @@
#include "mordor/streams/transfer.h"
#include "mordor/util.h"
#include "mordor/atomic.h"
+#include "multipart.h"
#include "parser.h"
namespace Mordor {
@@ -73,7 +75,7 @@ ClientRequest::ptr
ClientConnection::request(const Request &requestHeaders)
{
ClientRequest::ptr request(new ClientRequest(shared_from_this(), requestHeaders));
- request->doRequest();
+ request->waitForRequest();
return request;
}
@@ -136,8 +138,10 @@ ClientConnection::scheduleNextRequest(ClientRequest *request)
invariant();
MORDOR_ASSERT(m_currentRequest != m_pendingRequests.end());
MORDOR_ASSERT(request == *m_currentRequest);
- MORDOR_ASSERT(request->m_requestState == ClientRequest::BODY);
- MORDOR_LOG_TRACE(g_log) << m_connectionNumber << "-" << request->m_requestNumber << " request complete";
+ MORDOR_ASSERT(request->m_requestState == ClientRequest::BODY ||
+ request->m_requestState == ClientRequest::HEADERS);
+ MORDOR_LOG_TRACE(g_log) << m_connectionNumber << "-"
+ << request->m_requestNumber << " request complete";
std::list<ClientRequest *>::iterator it(m_currentRequest);
++it;
if (it == m_pendingRequests.end()) {
@@ -155,7 +159,8 @@ ClientConnection::scheduleNextRequest(ClientRequest *request)
request->m_requestState = ClientRequest::HEADERS;
MORDOR_ASSERT(request->m_scheduler);
MORDOR_ASSERT(request->m_fiber);
- MORDOR_LOG_TRACE(g_log) << m_connectionNumber << "-" << request->m_requestNumber << " scheduling request";
+ MORDOR_LOG_TRACE(g_log) << m_connectionNumber << "-"
+ << request->m_requestNumber << " scheduling request";
request->m_scheduler->schedule(request->m_fiber);
request->m_scheduler = NULL;
request->m_fiber.reset();
@@ -463,7 +468,7 @@ ClientRequest::~ClientRequest()
#endif
}
-const Request &
+Request &
ClientRequest::request()
{
return m_request;
@@ -481,6 +486,7 @@ ClientRequest::requestStream()
{
if (m_requestStream)
return m_requestStream;
+ doRequest();
MORDOR_ASSERT(!m_requestMultipart);
MORDOR_ASSERT(m_request.entity.contentType.type != "multipart");
if (!hasRequestBody()) {
@@ -500,6 +506,7 @@ ClientRequest::requestMultipart()
{
if (m_requestMultipart)
return m_requestMultipart;
+ doRequest();
MORDOR_ASSERT(m_request.entity.contentType.type == "multipart");
MORDOR_ASSERT(!m_requestStream);
MORDOR_ASSERT(m_requestState == BODY);
@@ -751,8 +758,80 @@ ClientRequest::finish()
}
void
+ClientRequest::waitForRequest()
+{
+ bool firstRequest;
+ // Put the request in the queue
+ {
+ boost::mutex::scoped_lock lock(m_conn->m_mutex);
+ m_conn->invariant();
+ if (!m_conn->m_allowNewRequests || m_conn->m_priorResponseClosed != ~0ull) {
+ m_requestState = m_responseState = ERROR;
+ MORDOR_THROW_EXCEPTION(ConnectionVoluntarilyClosedException());
+ }
+ if (m_conn->m_priorRequestFailed || m_conn->m_priorResponseFailed != ~0ull) {
+ m_requestState = m_responseState = ERROR;
+ MORDOR_THROW_EXCEPTION(PriorRequestFailedException());
+ }
+ if (m_conn->m_idleTimer) {
+ m_conn->m_idleTimer->cancel();
+ m_conn->m_idleTimer.reset();
+ }
+ firstRequest = m_conn->m_currentRequest == m_conn->m_pendingRequests.end();
+ m_requestNumber = ++m_conn->m_requestCount;
+ m_conn->m_pendingRequests.push_back(this);
+ if (firstRequest) {
+ m_conn->m_currentRequest = m_conn->m_pendingRequests.end();
+ --m_conn->m_currentRequest;
+ m_requestState = HEADERS;
+ // Disable read timeouts while a request is in progress
+ if (m_conn->m_timeoutStream)
+ m_conn->m_timeoutStream->readTimeout(~0ull);
+ MORDOR_LOG_TRACE(g_log) << m_conn->m_connectionNumber << "-" << m_requestNumber << " requesting";
+ } else {
+ m_scheduler = Scheduler::getThis();
+ m_fiber = Fiber::getThis();
+ MORDOR_ASSERT(m_scheduler);
+ MORDOR_ASSERT(m_fiber);
+ MORDOR_LOG_TRACE(g_log) << m_conn->m_connectionNumber << "-" << m_requestNumber << " waiting to request";
+ }
+ }
+ // If we weren't the first request in the queue, we have to wait for
+ // another request to schedule us
+ if (!firstRequest) {
+ Scheduler::yieldTo();
+ MORDOR_LOG_TRACE(g_log) << m_conn->m_connectionNumber << "-" << m_requestNumber << " requesting";
+ // Check for problems that occurred while we were waiting
+ boost::mutex::scoped_lock lock(m_conn->m_mutex);
+ m_conn->invariant();
+ if (m_conn->m_priorResponseClosed != ~0ull ||
+ m_conn->m_priorRequestFailed ||
+ m_conn->m_priorResponseFailed != ~0ull) {
+ if (m_requestState == HEADERS) {
+ MORDOR_ASSERT(m_conn->m_currentRequest !=
+ m_conn->m_pendingRequests.end());
+ MORDOR_ASSERT(*m_conn->m_currentRequest == this);
+ m_conn->m_currentRequest =
+ m_conn->m_pendingRequests.erase(m_conn->m_currentRequest);
+ MORDOR_ASSERT(m_conn->m_currentRequest ==
+ m_conn->m_pendingRequests.end());
+ }
+ m_requestState = m_responseState = ERROR;
+ if (m_conn->m_priorResponseClosed != ~0ull)
+ MORDOR_THROW_EXCEPTION(ConnectionVoluntarilyClosedException());
+ else
+ MORDOR_THROW_EXCEPTION(PriorRequestFailedException());
+ }
+ }
+ MORDOR_ASSERT(m_requestState == HEADERS);
+}
+
+void
ClientRequest::doRequest()
{
+ if (m_requestState > HEADERS)
+ return;
+
RequestLine &requestLine = m_request.requestLine;
// 1.0, 1.1, or defaulted
MORDOR_ASSERT(requestLine.ver == Version() ||
@@ -781,10 +860,10 @@ ClientRequest::doRequest()
} else