diff --git a/.github/mergeable.yml b/.github/mergeable.yml index 660d8cb4405cd..30692095c4640 100644 --- a/.github/mergeable.yml +++ b/.github/mergeable.yml @@ -1,6 +1,14 @@ mergeable: pull_requests: label: - must_include: - regex: "release notes: yes|release notes: no" - message: "Add release notes yes/no label. For yes, add lang label" + or: + - and: + - must_include: + regex: 'release notes: yes' + message: 'Please include release note: yes' + - must_include: + regex: '^lang\/' + message: 'Please include a language label' + - must_include: + regex: 'release notes: no' + message: 'Please include release note: no' diff --git a/CMakeLists.txt b/CMakeLists.txt index e8e65d4b71dc5..f242ee92bbf48 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -100,6 +100,8 @@ if (MSVC) add_definitions(-D_WIN32_WINNT=0x600 -D_SCL_SECURE_NO_WARNINGS -D_CRT_SECURE_NO_WARNINGS -D_WINSOCK_DEPRECATED_NO_WARNINGS) # needed to compile protobuf add_definitions(/wd4065 /wd4506) + # TODO(jtattermusch): revisit warnings that were silenced as part of upgrade to protobuf3.6.0 + add_definitions(/wd4200 /wd4291 /wd4244) # TODO(jtattermusch): revisit C4267 occurrences throughout the code add_definitions(/wd4267) # TODO(jtattermusch): needed to build boringssl with VS2017, revisit later @@ -660,12 +662,8 @@ add_dependencies(buildtests_cxx transport_security_common_api_test) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx writes_per_rpc_test) endif() -if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx resolver_component_test_unsecure) -endif() -if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx resolver_component_test) -endif() if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx resolver_component_tests_runner_invoker_unsecure) endif() @@ -674,9 +672,7 @@ add_dependencies(buildtests_cxx resolver_component_tests_runner_invoker) endif() add_dependencies(buildtests_cxx address_sorting_test_unsecure) add_dependencies(buildtests_cxx address_sorting_test) -if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_dependencies(buildtests_cxx cancel_ares_query_test) -endif() add_custom_target(buildtests DEPENDS buildtests_c buildtests_cxx) @@ -16211,7 +16207,6 @@ target_link_libraries(inproc_nosec_test endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) -if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_executable(resolver_component_test_unsecure test/cpp/naming/resolver_component_test.cc @@ -16251,10 +16246,8 @@ target_link_libraries(resolver_component_test_unsecure ${_gRPC_GFLAGS_LIBRARIES} ) -endif() endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) -if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_executable(resolver_component_test test/cpp/naming/resolver_component_test.cc @@ -16294,7 +16287,6 @@ target_link_libraries(resolver_component_test ${_gRPC_GFLAGS_LIBRARIES} ) -endif() endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) @@ -16465,7 +16457,6 @@ target_link_libraries(address_sorting_test endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) -if(_gRPC_PLATFORM_LINUX OR _gRPC_PLATFORM_MAC OR _gRPC_PLATFORM_POSIX) add_executable(cancel_ares_query_test test/cpp/naming/cancel_ares_query_test.cc @@ -16505,7 +16496,6 @@ target_link_libraries(cancel_ares_query_test ${_gRPC_GFLAGS_LIBRARIES} ) -endif() endif (gRPC_BUILD_TESTS) if (gRPC_BUILD_TESTS) diff --git a/bazel/grpc_deps.bzl b/bazel/grpc_deps.bzl index 7c349dbcdcf62..d7519b1453298 100644 --- a/bazel/grpc_deps.bzl +++ b/bazel/grpc_deps.bzl @@ -116,9 +116,9 @@ def grpc_deps(): if "com_google_protobuf" not in native.existing_rules(): native.http_archive( name = "com_google_protobuf", - strip_prefix = "protobuf-b5fbb742af122b565925987e65c08957739976a7", - url = "https://github.com/google/protobuf/archive/b5fbb742af122b565925987e65c08957739976a7.tar.gz", - ) + strip_prefix = "protobuf-48cb18e5c419ddd23d9badcfe4e9df7bde1979b2", + url = "https://github.com/google/protobuf/archive/48cb18e5c419ddd23d9badcfe4e9df7bde1979b2.tar.gz", + ) if "com_github_nanopb_nanopb" not in native.existing_rules(): native.new_http_archive( diff --git a/doc/csharp/server_reflection.md b/doc/csharp/server_reflection.md index 972168026947a..7827cbb21f020 100644 --- a/doc/csharp/server_reflection.md +++ b/doc/csharp/server_reflection.md @@ -30,7 +30,8 @@ server.Start(); ``` After starting the server, you can verify that the server reflection -is working properly by using the `grpc_cli` command line tool: +is working properly by using the [`grpc_cli` command line +tool](https://github.com/grpc/grpc/blob/master/doc/command_line_tool.md): ```sh $ grpc_cli ls localhost:50051 diff --git a/doc/python/server_reflection.md b/doc/python/server_reflection.md new file mode 100644 index 0000000000000..5a68e3c301909 --- /dev/null +++ b/doc/python/server_reflection.md @@ -0,0 +1,61 @@ +# gRPC Python Server Reflection + +This document shows how to use gRPC Server Reflection in gRPC Python. +Please see [C++ Server Reflection Tutorial](../server_reflection_tutorial.md) +for general information and more examples how to use server reflection. + +## Enable server reflection in Python servers + +gRPC Python Server Reflection is an add-on library. +To use it, first install the [grpcio-reflection](https://pypi.org/project/grpcio-reflection/) +PyPI package into your project. + +Note that with Python you need to manually register the service +descriptors with the reflection service implementation when creating a server +(this isn't necessary with e.g. C++ or Java) +```python +# add the following import statement to use server reflection +from grpc_reflection.v1alpha import reflection +# ... +def serve(): + server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) + helloworld_pb2_grpc.add_GreeterServicer_to_server(Greeter(), server) + # the reflection service will be aware of "Greeter" and "ServerReflection" services. + SERVICE_NAMES = ( + helloworld_pb2.DESCRIPTOR.services_by_name['Greeter'].full_name, + reflection.SERVICE_NAME, + ) + reflection.enable_server_reflection(SERVICE_NAMES, server) + server.add_insecure_port('[::]:50051') + server.start() +``` + +Please see +[greeter_server_with_reflection.py](https://github.com/grpc/grpc/blob/master/examples/python/helloworld/greeter_server_with_reflection.py) +in the examples directory for the full example, which extends the gRPC [Python +`Greeter` example](https://github.com/grpc/tree/master/examples/python/helloworld) on a +reflection-enabled server. + +After starting the server, you can verify that the server reflection +is working properly by using the [`grpc_cli` command line +tool](https://github.com/grpc/grpc/blob/master/doc/command_line_tool.md): + + ```sh + $ grpc_cli ls localhost:50051 + ``` + + output: + ```sh + grpc.reflection.v1alpha.ServerReflection + helloworld.Greeter + ``` + + For more examples and instructions how to use the `grpc_cli` tool, + please refer to the [`grpc_cli` documentation](../command_line_tool.md) + and the [C++ Server Reflection Tutorial](../server_reflection_tutorial.md). + +## Additional Resources + +The [Server Reflection Protocol](../server-reflection.md) provides detailed +information about how the server reflection works and describes the server reflection +protocol in detail. diff --git a/examples/python/helloworld/greeter_server_with_reflection.py b/examples/python/helloworld/greeter_server_with_reflection.py new file mode 100644 index 0000000000000..5ba8782dfca28 --- /dev/null +++ b/examples/python/helloworld/greeter_server_with_reflection.py @@ -0,0 +1,52 @@ +# Copyright 2018 The gRPC Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""The reflection-enabled version of gRPC helloworld.Greeter server.""" + +from concurrent import futures +import time + +import grpc +from grpc_reflection.v1alpha import reflection + +import helloworld_pb2 +import helloworld_pb2_grpc + +_ONE_DAY_IN_SECONDS = 60 * 60 * 24 + + +class Greeter(helloworld_pb2_grpc.GreeterServicer): + + def SayHello(self, request, context): + return helloworld_pb2.HelloReply(message='Hello, %s!' % request.name) + + +def serve(): + server = grpc.server(futures.ThreadPoolExecutor(max_workers=10)) + helloworld_pb2_grpc.add_GreeterServicer_to_server(Greeter(), server) + SERVICE_NAMES = ( + helloworld_pb2.DESCRIPTOR.services_by_name['Greeter'].full_name, + reflection.SERVICE_NAME, + ) + reflection.enable_server_reflection(SERVICE_NAMES, server) + server.add_insecure_port('[::]:50051') + server.start() + try: + while True: + time.sleep(_ONE_DAY_IN_SECONDS) + except KeyboardInterrupt: + server.stop(0) + + +if __name__ == '__main__': + serve() diff --git a/grpc.def b/grpc.def index 312e9166824bb..402c7ddefb516 100644 --- a/grpc.def +++ b/grpc.def @@ -42,6 +42,7 @@ EXPORTS grpc_census_call_get_context grpc_channel_get_target grpc_channel_get_info + grpc_channel_reset_connect_backoff grpc_insecure_channel_create grpc_lame_client_channel_create grpc_channel_destroy diff --git a/include/grpc/grpc.h b/include/grpc/grpc.h index c54e884439202..dc22008932d0a 100644 --- a/include/grpc/grpc.h +++ b/include/grpc/grpc.h @@ -274,6 +274,11 @@ GRPCAPI char* grpc_channel_get_target(grpc_channel* channel); GRPCAPI void grpc_channel_get_info(grpc_channel* channel, const grpc_channel_info* channel_info); +/** EXPERIMENTAL. Resets the channel's connect backoff. + TODO(roth): When we see whether this proves useful, either promote + to non-experimental or remove it. */ +GRPCAPI void grpc_channel_reset_connect_backoff(grpc_channel* channel); + /** Create a client channel to 'target'. Additional channel level configuration MAY be provided by grpc_channel_args, though the expectation is that most clients will want to simply pass NULL. The user data in 'args' need only diff --git a/include/grpcpp/channel.h b/include/grpcpp/channel.h index 4b45d5382cd66..fed02bf7bc7b5 100644 --- a/include/grpcpp/channel.h +++ b/include/grpcpp/channel.h @@ -30,6 +30,14 @@ struct grpc_channel; namespace grpc { + +namespace experimental { +/// Resets the channel's connection backoff. +/// TODO(roth): Once we see whether this proves useful, either create a gRFC +/// and change this to be a method of the Channel class, or remove it. +void ChannelResetConnectionBackoff(Channel* channel); +} // namespace experimental + /// Channels represent a connection to an endpoint. Created by \a CreateChannel. class Channel final : public ChannelInterface, public internal::CallHook, @@ -52,6 +60,7 @@ class Channel final : public ChannelInterface, private: template friend class internal::BlockingUnaryCallImpl; + friend void experimental::ChannelResetConnectionBackoff(Channel* channel); friend std::shared_ptr CreateChannelInternal( const grpc::string& host, grpc_channel* c_channel); Channel(const grpc::string& host, grpc_channel* c_channel); diff --git a/setup.py b/setup.py index b87843f335dd6..388e629ec2a16 100644 --- a/setup.py +++ b/setup.py @@ -276,11 +276,11 @@ def cython_extensions_and_necessity(): } INSTALL_REQUIRES = ( - "six>=1.5.2", - "futures>=2.2.0 ; python_version<'3.2'", - "enum34>=1.0.4 ; python_version<'3.4'" + 'six>=1.5.2', ) +if not PY3: + INSTALL_REQUIRES += ('futures>=2.2.0', 'enum34>=1.0.4') SETUP_REQUIRES = INSTALL_REQUIRES + ( 'sphinx>=1.3', diff --git a/src/core/ext/filters/client_channel/client_channel.cc b/src/core/ext/filters/client_channel/client_channel.cc index fead8feb17b81..b06f09d8c7b34 100644 --- a/src/core/ext/filters/client_channel/client_channel.cc +++ b/src/core/ext/filters/client_channel/client_channel.cc @@ -622,6 +622,17 @@ static void start_transport_op_locked(void* arg, grpc_error* error_ignored) { } GRPC_ERROR_UNREF(op->disconnect_with_error); } + + if (op->reset_connect_backoff) { + if (chand->resolver != nullptr) { + chand->resolver->ResetBackoffLocked(); + chand->resolver->RequestReresolutionLocked(); + } + if (chand->lb_policy != nullptr) { + chand->lb_policy->ResetBackoffLocked(); + } + } + GRPC_CHANNEL_STACK_UNREF(chand->owning_stack, "start_transport_op"); GRPC_CLOSURE_SCHED(op->on_consumed, GRPC_ERROR_NONE); diff --git a/src/core/ext/filters/client_channel/client_channel_plugin.cc b/src/core/ext/filters/client_channel/client_channel_plugin.cc index e0784b7e5c1f1..71da648660471 100644 --- a/src/core/ext/filters/client_channel/client_channel_plugin.cc +++ b/src/core/ext/filters/client_channel/client_channel_plugin.cc @@ -56,7 +56,7 @@ void grpc_client_channel_init(void) { grpc_register_http_proxy_mapper(); grpc_subchannel_index_init(); grpc_channel_init_register_stage( - GRPC_CLIENT_CHANNEL, GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, append_filter, + GRPC_CLIENT_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_MAX, append_filter, (void*)&grpc_client_channel_filter); grpc_http_connect_register_handshaker_factory(); } diff --git a/src/core/ext/filters/client_channel/lb_policy.h b/src/core/ext/filters/client_channel/lb_policy.h index 31c08246ae46b..3c0a9c1118020 100644 --- a/src/core/ext/filters/client_channel/lb_policy.h +++ b/src/core/ext/filters/client_channel/lb_policy.h @@ -144,7 +144,10 @@ class LoadBalancingPolicy /// consider whether this method is still needed. virtual void ExitIdleLocked() GRPC_ABSTRACT; - /// populates child_subchannels and child_channels with the uuids of this + /// Resets connection backoff. + virtual void ResetBackoffLocked() GRPC_ABSTRACT; + + /// Populates child_subchannels and child_channels with the uuids of this /// LB policy's referenced children. This is not invoked from the /// client_channel's combiner. The implementation is responsible for /// providing its own synchronization. diff --git a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc index 2d1f777474887..6581385ff9185 100644 --- a/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc +++ b/src/core/ext/filters/client_channel/lb_policy/grpclb/grpclb.cc @@ -134,6 +134,7 @@ class GrpcLb : public LoadBalancingPolicy { grpc_error** connectivity_error) override; void HandOffPendingPicksLocked(LoadBalancingPolicy* new_policy) override; void ExitIdleLocked() override; + void ResetBackoffLocked() override; void FillChildRefsForChannelz(ChildRefsList* child_subchannels, ChildRefsList* child_channels) override; @@ -1214,6 +1215,15 @@ void GrpcLb::ExitIdleLocked() { } } +void GrpcLb::ResetBackoffLocked() { + if (lb_channel_ != nullptr) { + grpc_channel_reset_connect_backoff(lb_channel_); + } + if (rr_policy_ != nullptr) { + rr_policy_->ResetBackoffLocked(); + } +} + bool GrpcLb::PickLocked(PickState* pick, grpc_error** error) { PendingPick* pp = PendingPickCreate(pick); bool pick_done = false; @@ -1880,7 +1890,7 @@ void grpc_lb_policy_grpclb_init() { grpc_core::UniquePtr( grpc_core::New())); grpc_channel_init_register_stage(GRPC_CLIENT_SUBCHANNEL, - GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, + GRPC_CHANNEL_INIT_PRIORITY_LOW, maybe_add_client_load_reporting_filter, (void*)&grpc_client_load_reporting_filter); } diff --git a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc index 46acbf628b0e3..2b6a9ba8c53ba 100644 --- a/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc +++ b/src/core/ext/filters/client_channel/lb_policy/pick_first/pick_first.cc @@ -57,6 +57,7 @@ class PickFirst : public LoadBalancingPolicy { grpc_error** connectivity_error) override; void HandOffPendingPicksLocked(LoadBalancingPolicy* new_policy) override; void ExitIdleLocked() override; + void ResetBackoffLocked() override; void FillChildRefsForChannelz(ChildRefsList* child_subchannels, ChildRefsList* ignored) override; @@ -259,6 +260,13 @@ void PickFirst::ExitIdleLocked() { } } +void PickFirst::ResetBackoffLocked() { + subchannel_list_->ResetBackoffLocked(); + if (latest_pending_subchannel_list_ != nullptr) { + latest_pending_subchannel_list_->ResetBackoffLocked(); + } +} + bool PickFirst::PickLocked(PickState* pick, grpc_error** error) { // If we have a selected subchannel already, return synchronously. if (selected_ != nullptr) { diff --git a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc index 9c3a15c67b010..fea84331d80f1 100644 --- a/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc +++ b/src/core/ext/filters/client_channel/lb_policy/round_robin/round_robin.cc @@ -68,6 +68,7 @@ class RoundRobin : public LoadBalancingPolicy { grpc_error** connectivity_error) override; void HandOffPendingPicksLocked(LoadBalancingPolicy* new_policy) override; void ExitIdleLocked() override; + void ResetBackoffLocked() override; void FillChildRefsForChannelz(ChildRefsList* child_subchannels, ChildRefsList* ignored) override; @@ -333,6 +334,13 @@ void RoundRobin::ExitIdleLocked() { } } +void RoundRobin::ResetBackoffLocked() { + subchannel_list_->ResetBackoffLocked(); + if (latest_pending_subchannel_list_ != nullptr) { + latest_pending_subchannel_list_->ResetBackoffLocked(); + } +} + bool RoundRobin::DoPickLocked(PickState* pick) { const size_t next_ready_index = subchannel_list_->GetNextReadySubchannelIndexLocked(); diff --git a/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h b/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h index 018ac3bb8684a..0fa2f04e73b28 100644 --- a/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h +++ b/src/core/ext/filters/client_channel/lb_policy/subchannel_list.h @@ -107,6 +107,11 @@ class SubchannelData { // being unreffed. virtual void UnrefSubchannelLocked(const char* reason); + // Resets the connection backoff. + // TODO(roth): This method should go away when we move the backoff + // code out of the subchannel and into the LB policies. + void ResetBackoffLocked(); + // Starts watching the connectivity state of the subchannel. // ProcessConnectivityChangeLocked() will be called when the // connectivity state changes. @@ -206,6 +211,11 @@ class SubchannelList LoadBalancingPolicy* policy() const { return policy_; } TraceFlag* tracer() const { return tracer_; } + // Resets connection backoff of all subchannels. + // TODO(roth): We will probably need to rethink this as part of moving + // the backoff code out of subchannels and into LB policies. + void ResetBackoffLocked(); + // Note: Caller must ensure that this is invoked inside of the combiner. void Orphan() override { ShutdownLocked(); @@ -298,6 +308,14 @@ void SubchannelData:: } } +template +void SubchannelData::ResetBackoffLocked() { + if (subchannel_ != nullptr) { + grpc_subchannel_reset_backoff(subchannel_); + } +} + template void SubchannelData::StartConnectivityWatchLocked() { @@ -544,6 +562,15 @@ void SubchannelList::ShutdownLocked() { } } +template +void SubchannelList::ResetBackoffLocked() { + for (size_t i = 0; i < subchannels_.size(); i++) { + SubchannelDataType* sd = &subchannels_[i]; + sd->ResetBackoffLocked(); + } +} + } // namespace grpc_core #endif /* GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_LB_POLICY_SUBCHANNEL_LIST_H */ diff --git a/src/core/ext/filters/client_channel/resolver.h b/src/core/ext/filters/client_channel/resolver.h index c7e37e4468ea3..48f2e890952b4 100644 --- a/src/core/ext/filters/client_channel/resolver.h +++ b/src/core/ext/filters/client_channel/resolver.h @@ -94,6 +94,14 @@ class Resolver : public InternallyRefCountedWithTracing { /// throw away unselected subchannels. virtual void RequestReresolutionLocked() GRPC_ABSTRACT; + /// Resets the re-resolution backoff, if any. + /// This needs to be implemented only by pull-based implementations; + /// for push-based implementations, it will be a no-op. + /// TODO(roth): Pull the backoff code out of resolver and into + /// client_channel, so that it can be shared across resolver + /// implementations. At that point, this method can go away. + virtual void ResetBackoffLocked() {} + void Orphan() override { // Invoke ShutdownAndUnrefLocked() inside of the combiner. GRPC_CLOSURE_SCHED( diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc index 7050e82121c67..f2bb5f3c71287 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/dns_resolver_ares.cc @@ -66,6 +66,8 @@ class AresDnsResolver : public Resolver { void RequestReresolutionLocked() override; + void ResetBackoffLocked() override; + void ShutdownLocked() override; private: @@ -187,6 +189,13 @@ void AresDnsResolver::RequestReresolutionLocked() { } } +void AresDnsResolver::ResetBackoffLocked() { + if (have_next_resolution_timer_) { + grpc_timer_cancel(&next_resolution_timer_); + } + backoff_.Reset(); +} + void AresDnsResolver::ShutdownLocked() { if (have_next_resolution_timer_) { grpc_timer_cancel(&next_resolution_timer_); diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc index 0068d0d5f4bd6..fdbd07ebf5172 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.cc @@ -74,6 +74,8 @@ struct grpc_ares_ev_driver { bool shutting_down; /** request object that's using this ev driver */ grpc_ares_request* request; + /** Owned by the ev_driver. Creates new GrpcPolledFd's */ + grpc_core::UniquePtr polled_fd_factory; }; static void grpc_ares_notify_on_event_locked(grpc_ares_ev_driver* ev_driver); @@ -93,7 +95,7 @@ static void grpc_ares_ev_driver_unref(grpc_ares_ev_driver* ev_driver) { GRPC_COMBINER_UNREF(ev_driver->combiner, "free ares event driver"); ares_destroy(ev_driver->channel); grpc_ares_complete_request_locked(ev_driver->request); - gpr_free(ev_driver); + grpc_core::Delete(ev_driver); } } @@ -118,13 +120,11 @@ grpc_error* grpc_ares_ev_driver_create_locked(grpc_ares_ev_driver** ev_driver, grpc_pollset_set* pollset_set, grpc_combiner* combiner, grpc_ares_request* request) { - *ev_driver = static_cast( - gpr_malloc(sizeof(grpc_ares_ev_driver))); + *ev_driver = grpc_core::New(); ares_options opts; memset(&opts, 0, sizeof(opts)); opts.flags |= ARES_FLAG_STAYOPEN; int status = ares_init_options(&(*ev_driver)->channel, &opts, ARES_OPT_FLAGS); - grpc_core::ConfigureAresChannelLocked(&(*ev_driver)->channel); gpr_log(GPR_DEBUG, "grpc_ares_ev_driver_create_locked"); if (status != ARES_SUCCESS) { char* err_msg; @@ -142,6 +142,10 @@ grpc_error* grpc_ares_ev_driver_create_locked(grpc_ares_ev_driver** ev_driver, (*ev_driver)->working = false; (*ev_driver)->shutting_down = false; (*ev_driver)->request = request; + (*ev_driver)->polled_fd_factory = + grpc_core::NewGrpcPolledFdFactory((*ev_driver)->combiner); + (*ev_driver) + ->polled_fd_factory->ConfigureAresChannelLocked((*ev_driver)->channel); return GRPC_ERROR_NONE; } @@ -245,8 +249,9 @@ static void grpc_ares_notify_on_event_locked(grpc_ares_ev_driver* ev_driver) { // Create a new fd_node if sock[i] is not in the fd_node list. if (fdn == nullptr) { fdn = static_cast(gpr_malloc(sizeof(fd_node))); - fdn->grpc_polled_fd = grpc_core::NewGrpcPolledFdLocked( - socks[i], ev_driver->pollset_set); + fdn->grpc_polled_fd = + ev_driver->polled_fd_factory->NewGrpcPolledFdLocked( + socks[i], ev_driver->pollset_set, ev_driver->combiner); gpr_log(GPR_DEBUG, "new fd: %s", fdn->grpc_polled_fd->GetName()); fdn->ev_driver = ev_driver; fdn->readable_registered = false; diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h index 2c9db7101160f..671c537fe7292 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h @@ -81,10 +81,24 @@ class GrpcPolledFd { GRPC_ABSTRACT_BASE_CLASS }; -/* Creates a new wrapped fd for the current platform */ -GrpcPolledFd* NewGrpcPolledFdLocked(ares_socket_t as, - grpc_pollset_set* driver_pollset_set); -void ConfigureAresChannelLocked(ares_channel* channel); +/* A GrpcPolledFdFactory is 1-to-1 with and owned by the + * ares event driver. It knows how to create GrpcPolledFd's + * for the current platform, and the ares driver uses it for all of + * its fd's. */ +class GrpcPolledFdFactory { + public: + virtual ~GrpcPolledFdFactory() {} + /* Creates a new wrapped fd for the current platform */ + virtual GrpcPolledFd* NewGrpcPolledFdLocked( + ares_socket_t as, grpc_pollset_set* driver_pollset_set, + grpc_combiner* combiner) GRPC_ABSTRACT; + /* Optionally configures the ares channel after creation */ + virtual void ConfigureAresChannelLocked(ares_channel channel) GRPC_ABSTRACT; + + GRPC_ABSTRACT_BASE_CLASS +}; + +UniquePtr NewGrpcPolledFdFactory(grpc_combiner* combiner); } // namespace grpc_core diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc index fffe9eda8e531..aa58e1aaf5067 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc @@ -86,12 +86,20 @@ class GrpcPolledFdPosix : public GrpcPolledFd { grpc_pollset_set* driver_pollset_set_; }; -GrpcPolledFd* NewGrpcPolledFdLocked(ares_socket_t as, - grpc_pollset_set* driver_pollset_set) { - return grpc_core::New(as, driver_pollset_set); -} +class GrpcPolledFdFactoryPosix : public GrpcPolledFdFactory { + public: + GrpcPolledFd* NewGrpcPolledFdLocked(ares_socket_t as, + grpc_pollset_set* driver_pollset_set, + grpc_combiner* combiner) override { + return New(as, driver_pollset_set); + } -void ConfigureAresChannelLocked(ares_channel* channel) {} + void ConfigureAresChannelLocked(ares_channel channel) override {} +}; + +UniquePtr NewGrpcPolledFdFactory(grpc_combiner* combiner) { + return UniquePtr(New()); +} } // namespace grpc_core diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc index 5d65ae3ab3745..02121aa0ab475 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc @@ -21,38 +21,516 @@ #if GRPC_ARES == 1 && defined(GPR_WINDOWS) #include + +#include +#include +#include +#include +#include #include +#include "src/core/lib/gpr/string.h" #include "src/core/lib/gprpp/memory.h" +#include "src/core/lib/iomgr/combiner.h" +#include "src/core/lib/iomgr/socket_windows.h" +#include "src/core/lib/iomgr/tcp_windows.h" +#include "src/core/lib/slice/slice_internal.h" #include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_ev_driver.h" +#include "src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h" + +/* TODO(apolcyn): remove this hack after fixing upstream. + * Our grpc/c-ares code on Windows uses the ares_set_socket_functions API, + * which uses "struct iovec" type, which on Windows is defined inside of + * a c-ares header that is not public. + * See https://github.com/c-ares/c-ares/issues/206. */ +struct iovec { + void* iov_base; + size_t iov_len; +}; namespace grpc_core { -/* TODO: fill in the body of GrpcPolledFdWindows to enable c-ares on Windows. - This dummy implementation only allows grpc to compile on windows with - GRPC_ARES=1. */ +/* c-ares creates its own sockets and is meant to read them when readable and + * write them when writeable. To fit this socket usage model into the grpc + * windows poller (which gives notifications when attempted reads and writes are + * actually fulfilled rather than possible), this GrpcPolledFdWindows class + * takes advantage of the ares_set_socket_functions API and acts as a virtual + * socket. It holds its own read and write buffers which are written to and read + * from c-ares and are used with the grpc windows poller, and it, e.g., + * manufactures virtual socket error codes when it e.g. needs to tell the c-ares + * library to wait for an async read. */ class GrpcPolledFdWindows : public GrpcPolledFd { public: - GrpcPolledFdWindows() { abort(); } - ~GrpcPolledFdWindows() { abort(); } + enum WriteState { + WRITE_IDLE, + WRITE_REQUESTED, + WRITE_PENDING, + WRITE_WAITING_FOR_VERIFICATION_UPON_RETRY, + }; + + GrpcPolledFdWindows(ares_socket_t as, grpc_combiner* combiner) + : read_buf_(grpc_empty_slice()), + write_buf_(grpc_empty_slice()), + write_state_(WRITE_IDLE), + gotten_into_driver_list_(false) { + gpr_asprintf(&name_, "c-ares socket: %" PRIdPTR, as); + winsocket_ = grpc_winsocket_create(as, name_); + combiner_ = GRPC_COMBINER_REF(combiner, name_); + GRPC_CLOSURE_INIT(&outer_read_closure_, + &GrpcPolledFdWindows::OnIocpReadable, this, + grpc_combiner_scheduler(combiner_)); + GRPC_CLOSURE_INIT(&outer_write_closure_, + &GrpcPolledFdWindows::OnIocpWriteable, this, + grpc_combiner_scheduler(combiner_)); + } + + ~GrpcPolledFdWindows() { + GRPC_COMBINER_UNREF(combiner_, name_); + grpc_slice_unref_internal(read_buf_); + grpc_slice_unref_internal(write_buf_); + GPR_ASSERT(read_closure_ == nullptr); + GPR_ASSERT(write_closure_ == nullptr); + grpc_winsocket_destroy(winsocket_); + gpr_free(name_); + } + + void ScheduleAndNullReadClosure(grpc_error* error) { + GRPC_CLOSURE_SCHED(read_closure_, error); + read_closure_ = nullptr; + } + + void ScheduleAndNullWriteClosure(grpc_error* error) { + GRPC_CLOSURE_SCHED(write_closure_, error); + write_closure_ = nullptr; + } + void RegisterForOnReadableLocked(grpc_closure* read_closure) override { - abort(); + GPR_ASSERT(read_closure_ == nullptr); + read_closure_ = read_closure; + GPR_ASSERT(GRPC_SLICE_LENGTH(read_buf_) == 0); + grpc_slice_unref_internal(read_buf_); + read_buf_ = GRPC_SLICE_MALLOC(4192); + WSABUF buffer; + buffer.buf = (char*)GRPC_SLICE_START_PTR(read_buf_); + buffer.len = GRPC_SLICE_LENGTH(read_buf_); + memset(&winsocket_->read_info.overlapped, 0, sizeof(OVERLAPPED)); + recv_from_source_addr_len_ = sizeof(recv_from_source_addr_); + DWORD flags = 0; + if (WSARecvFrom(grpc_winsocket_wrapped_socket(winsocket_), &buffer, 1, + nullptr, &flags, (sockaddr*)recv_from_source_addr_, + &recv_from_source_addr_len_, + &winsocket_->read_info.overlapped, nullptr)) { + char* msg = gpr_format_message(WSAGetLastError()); + grpc_error* error = GRPC_ERROR_CREATE_FROM_COPIED_STRING(msg); + GRPC_CARES_TRACE_LOG( + "RegisterForOnReadableLocked: WSARecvFrom error:|%s|. fd:|%s|", msg, + GetName()); + gpr_free(msg); + if (WSAGetLastError() != WSA_IO_PENDING) { + ScheduleAndNullReadClosure(error); + return; + } + } + grpc_socket_notify_on_read(winsocket_, &outer_read_closure_); } + void RegisterForOnWriteableLocked(grpc_closure* write_closure) override { + GRPC_CARES_TRACE_LOG( + "RegisterForOnWriteableLocked. fd:|%s|. Current write state: %d", + GetName(), write_state_); + GPR_ASSERT(write_closure_ == nullptr); + write_closure_ = write_closure; + switch (write_state_) { + case WRITE_IDLE: + ScheduleAndNullWriteClosure(GRPC_ERROR_NONE); + break; + case WRITE_REQUESTED: + write_state_ = WRITE_PENDING; + SendWriteBuf(nullptr, &winsocket_->write_info.overlapped); + grpc_socket_notify_on_write(winsocket_, &outer_write_closure_); + break; + case WRITE_PENDING: + case WRITE_WAITING_FOR_VERIFICATION_UPON_RETRY: + abort(); + } + } + + bool IsFdStillReadableLocked() override { + return GRPC_SLICE_LENGTH(read_buf_) > 0; + } + + void ShutdownLocked(grpc_error* error) override { + grpc_winsocket_shutdown(winsocket_); + } + + ares_socket_t GetWrappedAresSocketLocked() override { + return grpc_winsocket_wrapped_socket(winsocket_); + } + + const char* GetName() override { return name_; } + + ares_ssize_t RecvFrom(void* data, ares_socket_t data_len, int flags, + struct sockaddr* from, ares_socklen_t* from_len) { + GRPC_CARES_TRACE_LOG( + "RecvFrom called on fd:|%s|. Current read buf length:|%d|", GetName(), + GRPC_SLICE_LENGTH(read_buf_)); + if (GRPC_SLICE_LENGTH(read_buf_) == 0) { + WSASetLastError(WSAEWOULDBLOCK); + return -1; + } + ares_ssize_t bytes_read = 0; + for (size_t i = 0; i < GRPC_SLICE_LENGTH(read_buf_) && i < data_len; i++) { + ((char*)data)[i] = GRPC_SLICE_START_PTR(read_buf_)[i]; + bytes_read++; + } + read_buf_ = grpc_slice_sub_no_ref(read_buf_, bytes_read, + GRPC_SLICE_LENGTH(read_buf_)); + /* c-ares overloads this recv_from virtual socket function to receive + * data on both UDP and TCP sockets, and from is nullptr for TCP. */ + if (from != nullptr) { + GPR_ASSERT(*from_len <= recv_from_source_addr_len_); + memcpy(from, &recv_from_source_addr_, recv_from_source_addr_len_); + *from_len = recv_from_source_addr_len_; + } + return bytes_read; + } + + grpc_slice FlattenIovec(const struct iovec* iov, int iov_count) { + int total = 0; + for (int i = 0; i < iov_count; i++) { + total += iov[i].iov_len; + } + grpc_slice out = GRPC_SLICE_MALLOC(total); + size_t cur = 0; + for (int i = 0; i < iov_count; i++) { + for (int k = 0; k < iov[i].iov_len; k++) { + GRPC_SLICE_START_PTR(out)[cur++] = ((char*)iov[i].iov_base)[k]; + } + } + return out; + } + + int SendWriteBuf(LPDWORD bytes_sent_ptr, LPWSAOVERLAPPED overlapped) { + WSABUF buf; + buf.len = GRPC_SLICE_LENGTH(write_buf_); + buf.buf = (char*)GRPC_SLICE_START_PTR(write_buf_); + DWORD flags = 0; + int out = WSASend(grpc_winsocket_wrapped_socket(winsocket_), &buf, 1, + bytes_sent_ptr, flags, overlapped, nullptr); + GRPC_CARES_TRACE_LOG( + "WSASend: name:%s. buf len:%d. bytes sent: %d. overlapped %p. return " + "val: %d", + GetName(), buf.len, *bytes_sent_ptr, overlapped, out); + return out; + } + + ares_ssize_t TrySendWriteBufSyncNonBlocking() { + GPR_ASSERT(write_state_ == WRITE_IDLE); + ares_ssize_t total_sent; + DWORD bytes_sent = 0; + if (SendWriteBuf(&bytes_sent, nullptr) != 0) { + char* msg = gpr_format_message(WSAGetLastError()); + GRPC_CARES_TRACE_LOG( + "TrySendWriteBufSyncNonBlocking: SendWriteBuf error:|%s|. fd:|%s|", + msg, GetName()); + gpr_free(msg); + if (WSAGetLastError() == WSA_IO_PENDING) { + WSASetLastError(WSAEWOULDBLOCK); + write_state_ = WRITE_REQUESTED; + } + } + write_buf_ = grpc_slice_sub_no_ref(write_buf_, bytes_sent, + GRPC_SLICE_LENGTH(write_buf_)); + return bytes_sent; + } + + ares_ssize_t SendV(const struct iovec* iov, int iov_count) { + GRPC_CARES_TRACE_LOG("SendV called on fd:|%s|. Current write state: %d", + GetName(), write_state_); + switch (write_state_) { + case WRITE_IDLE: + GPR_ASSERT(GRPC_SLICE_LENGTH(write_buf_) == 0); + grpc_slice_unref_internal(write_buf_); + write_buf_ = FlattenIovec(iov, iov_count); + return TrySendWriteBufSyncNonBlocking(); + case WRITE_REQUESTED: + case WRITE_PENDING: + WSASetLastError(WSAEWOULDBLOCK); + return -1; + case WRITE_WAITING_FOR_VERIFICATION_UPON_RETRY: + grpc_slice currently_attempted = FlattenIovec(iov, iov_count); + GPR_ASSERT(GRPC_SLICE_LENGTH(currently_attempted) >= + GRPC_SLICE_LENGTH(write_buf_)); + ares_ssize_t total_sent = 0; + for (size_t i = 0; i < GRPC_SLICE_LENGTH(write_buf_); i++) { + GPR_ASSERT(GRPC_SLICE_START_PTR(currently_attempted)[i] == + GRPC_SLICE_START_PTR(write_buf_)[i]); + total_sent++; + } + grpc_slice_unref_internal(write_buf_); + write_buf_ = + grpc_slice_sub_no_ref(currently_attempted, total_sent, + GRPC_SLICE_LENGTH(currently_attempted)); + write_state_ = WRITE_IDLE; + total_sent += TrySendWriteBufSyncNonBlocking(); + return total_sent; + } abort(); } - bool IsFdStillReadableLocked() override { abort(); } - void ShutdownLocked(grpc_error* error) override { abort(); } - ares_socket_t GetWrappedAresSocketLocked() override { abort(); } - const char* GetName() override { abort(); } + + int Connect(const struct sockaddr* target, ares_socklen_t target_len) { + SOCKET s = grpc_winsocket_wrapped_socket(winsocket_); + GRPC_CARES_TRACE_LOG("Connect: fd:|%s|", GetName()); + int out = + WSAConnect(s, target, target_len, nullptr, nullptr, nullptr, nullptr); + if (out != 0) { + char* msg = gpr_format_message(WSAGetLastError()); + GRPC_CARES_TRACE_LOG("Connect error code:|%d|, msg:|%s|. fd:|%s|", + WSAGetLastError(), msg, GetName()); + gpr_free(msg); + // c-ares expects a posix-style connect API + out = -1; + } + return out; + } + + static void OnIocpReadable(void* arg, grpc_error* error) { + GrpcPolledFdWindows* polled_fd = static_cast(arg); + polled_fd->OnIocpReadableInner(error); + } + + void OnIocpReadableInner(grpc_error* error) { + if (error == GRPC_ERROR_NONE) { + if (winsocket_->read_info.wsa_error != 0) { + /* WSAEMSGSIZE would be due to receiving more data + * than our read buffer's fixed capacity. Assume that + * the connection is TCP and read the leftovers + * in subsequent c-ares reads. */ + if (winsocket_->read_info.wsa_error != WSAEMSGSIZE) { + GRPC_ERROR_UNREF(error); + char* msg = gpr_format_message(winsocket_->read_info.wsa_error); + GRPC_CARES_TRACE_LOG( + "OnIocpReadableInner. winsocket error:|%s|. fd:|%s|", msg, + GetName()); + error = GRPC_ERROR_CREATE_FROM_COPIED_STRING(msg); + gpr_free(msg); + } + } + } + if (error == GRPC_ERROR_NONE) { + read_buf_ = grpc_slice_sub_no_ref(read_buf_, 0, + winsocket_->read_info.bytes_transfered); + } else { + grpc_slice_unref_internal(read_buf_); + read_buf_ = grpc_empty_slice(); + } + GRPC_CARES_TRACE_LOG( + "OnIocpReadable finishing. read buf length now:|%d|. :fd:|%s|", + GRPC_SLICE_LENGTH(read_buf_), GetName()); + ScheduleAndNullReadClosure(error); + } + + static void OnIocpWriteable(void* arg, grpc_error* error) { + GrpcPolledFdWindows* polled_fd = static_cast(arg); + polled_fd->OnIocpWriteableInner(error); + } + + void OnIocpWriteableInner(grpc_error* error) { + GRPC_CARES_TRACE_LOG("OnIocpWriteableInner. fd:|%s|", GetName()); + if (error == GRPC_ERROR_NONE) { + if (winsocket_->write_info.wsa_error != 0) { + char* msg = gpr_format_message(winsocket_->write_info.wsa_error); + GRPC_CARES_TRACE_LOG( + "OnIocpWriteableInner. winsocket error:|%s|. fd:|%s|", msg, + GetName()); + GRPC_ERROR_UNREF(error); + error = GRPC_ERROR_CREATE_FROM_COPIED_STRING(msg); + gpr_free(msg); + } + } + GPR_ASSERT(write_state_ == WRITE_PENDING); + if (error == GRPC_ERROR_NONE) { + write_state_ = WRITE_WAITING_FOR_VERIFICATION_UPON_RETRY; + write_buf_ = grpc_slice_sub_no_ref( + write_buf_, 0, winsocket_->write_info.bytes_transfered); + } else { + grpc_slice_unref_internal(write_buf_); + write_buf_ = grpc_empty_slice(); + } + ScheduleAndNullWriteClosure(error); + } + + bool gotten_into_driver_list() const { return gotten_into_driver_list_; } + void set_gotten_into_driver_list() { gotten_into_driver_list_ = true; } + + grpc_combiner* combiner_; + char recv_from_source_addr_[200]; + ares_socklen_t recv_from_source_addr_len_; + grpc_slice read_buf_; + grpc_slice write_buf_; + grpc_closure* read_closure_ = nullptr; + grpc_closure* write_closure_ = nullptr; + grpc_closure outer_read_closure_; + grpc_closure outer_write_closure_; + grpc_winsocket* winsocket_; + WriteState write_state_; + char* name_ = nullptr; + bool gotten_into_driver_list_; }; -GrpcPolledFd* NewGrpcPolledFdLocked(ares_socket_t as, - grpc_pollset_set* driver_pollset_set) { - return nullptr; -} +struct SockToPolledFdEntry { + SockToPolledFdEntry(SOCKET s, GrpcPolledFdWindows* fd) + : socket(s), polled_fd(fd) {} + SOCKET socket; + GrpcPolledFdWindows* polled_fd; + SockToPolledFdEntry* next = nullptr; +}; + +/* A SockToPolledFdMap can make ares_socket_t types (SOCKET's on windows) + * to GrpcPolledFdWindow's, and is used to find the appropriate + * GrpcPolledFdWindows to handle a virtual socket call when c-ares makes that + * socket call on the ares_socket_t type. Instances are owned by and one-to-one + * with a GrpcPolledFdWindows factory and event driver */ +class SockToPolledFdMap { + public: + SockToPolledFdMap(grpc_combiner* combiner) { + combiner_ = GRPC_COMBINER_REF(combiner, "sock to polled fd map"); + } + + ~SockToPolledFdMap() { + GPR_ASSERT(head_ == nullptr); + GRPC_COMBINER_UNREF(combiner_, "sock to polled fd map"); + } + + void AddNewSocket(SOCKET s, GrpcPolledFdWindows* polled_fd) { + SockToPolledFdEntry* new_node = New(s, polled_fd); + new_node->next = head_; + head_ = new_node; + } + + GrpcPolledFdWindows* LookupPolledFd(SOCKET s) { + for (SockToPolledFdEntry* node = head_; node != nullptr; + node = node->next) { + if (node->socket == s) { + GPR_ASSERT(node->polled_fd != nullptr); + return node->polled_fd; + } + } + abort(); + } + + void RemoveEntry(SOCKET s) { + GPR_ASSERT(head_ != nullptr); + SockToPolledFdEntry** prev = &head_; + for (SockToPolledFdEntry* node = head_; node != nullptr; + node = node->next) { + if (node->socket == s) { + *prev = node->next; + Delete(node); + return; + } + prev = &node->next; + } + abort(); + } + + /* These virtual socket functions are called from within the c-ares + * library. These methods generally dispatch those socket calls to the + * appropriate methods. The virtual "socket" and "close" methods are + * special and instead create/add and remove/destroy GrpcPolledFdWindows + * objects. + */ + static ares_socket_t Socket(int af, int type, int protocol, void* user_data) { + SockToPolledFdMap* map = static_cast(user_data); + SOCKET s = WSASocket(af, type, protocol, nullptr, 0, WSA_FLAG_OVERLAPPED); + if (s == INVALID_SOCKET) { + return s; + } + grpc_tcp_set_non_block(s); + GrpcPolledFdWindows* polled_fd = + New(s, map->combiner_); + map->AddNewSocket(s, polled_fd); + return s; + } + + static int Connect(ares_socket_t as, const struct sockaddr* target, + ares_socklen_t target_len, void* user_data) { + SockToPolledFdMap* map = static_cast(user_data); + GrpcPolledFdWindows* polled_fd = map->LookupPolledFd(as); + return polled_fd->Connect(target, target_len); + } + + static ares_ssize_t SendV(ares_socket_t as, const struct iovec* iov, + int iovec_count, void* user_data) { + SockToPolledFdMap* map = static_cast(user_data); + GrpcPolledFdWindows* polled_fd = map->LookupPolledFd(as); + return polled_fd->SendV(iov, iovec_count); + } + + static ares_ssize_t RecvFrom(ares_socket_t as, void* data, size_t data_len, + int flags, struct sockaddr* from, + ares_socklen_t* from_len, void* user_data) { + SockToPolledFdMap* map = static_cast(user_data); + GrpcPolledFdWindows* polled_fd = map->LookupPolledFd(as); + return polled_fd->RecvFrom(data, data_len, flags, from, from_len); + } + + static int CloseSocket(SOCKET s, void* user_data) { + SockToPolledFdMap* map = static_cast(user_data); + GrpcPolledFdWindows* polled_fd = map->LookupPolledFd(s); + map->RemoveEntry(s); + // If a gRPC polled fd has not made it in to the driver's list yet, then + // the driver has not and will never see this socket. + if (!polled_fd->gotten_into_driver_list()) { + polled_fd->ShutdownLocked(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "Shut down c-ares fd before without it ever having made it into the " + "driver's list")); + return 0; + } + return 0; + } + + private: + SockToPolledFdEntry* head_ = nullptr; + grpc_combiner* combiner_; +}; + +const struct ares_socket_functions custom_ares_sock_funcs = { + &SockToPolledFdMap::Socket /* socket */, + &SockToPolledFdMap::CloseSocket /* close */, + &SockToPolledFdMap::Connect /* connect */, + &SockToPolledFdMap::RecvFrom /* recvfrom */, + &SockToPolledFdMap::SendV /* sendv */, +}; + +class GrpcPolledFdFactoryWindows : public GrpcPolledFdFactory { + public: + GrpcPolledFdFactoryWindows(grpc_combiner* combiner) + : sock_to_polled_fd_map_(combiner) {} + + GrpcPolledFd* NewGrpcPolledFdLocked(ares_socket_t as, + grpc_pollset_set* driver_pollset_set, + grpc_combiner* combiner) override { + GrpcPolledFdWindows* polled_fd = sock_to_polled_fd_map_.LookupPolledFd(as); + // Set a flag so that the virtual socket "close" method knows it + // doesn't need to call ShutdownLocked, since now the driver will. + polled_fd->set_gotten_into_driver_list(); + return polled_fd; + } -void ConfigureAresChannelLocked(ares_channel* channel) { abort(); } + void ConfigureAresChannelLocked(ares_channel channel) override { + ares_set_socket_functions(channel, &custom_ares_sock_funcs, + &sock_to_polled_fd_map_); + } + + private: + SockToPolledFdMap sock_to_polled_fd_map_; +}; + +UniquePtr NewGrpcPolledFdFactory(grpc_combiner* combiner) { + return UniquePtr( + New(combiner)); +} } // namespace grpc_core diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc index b3d6437e9a66b..485998f5e4460 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.cc @@ -49,6 +49,8 @@ static gpr_mu g_init_mu; grpc_core::TraceFlag grpc_trace_cares_address_sorting(false, "cares_address_sorting"); +grpc_core::TraceFlag grpc_trace_cares_resolver(false, "cares_resolver"); + struct grpc_ares_request { /** indicates the DNS server to use, if specified */ struct ares_addr_port_node dns_server_addr; diff --git a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h index 17eaa7ccf06aa..ca5779e1d79c6 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h +++ b/src/core/ext/filters/client_channel/resolver/dns/c_ares/grpc_ares_wrapper.h @@ -28,6 +28,13 @@ extern grpc_core::TraceFlag grpc_trace_cares_address_sorting; +extern grpc_core::TraceFlag grpc_trace_cares_resolver; + +#define GRPC_CARES_TRACE_LOG(format, ...) \ + if (grpc_trace_cares_resolver.enabled()) { \ + gpr_log(GPR_DEBUG, "(c-ares resolver) " format, __VA_ARGS__); \ + } + typedef struct grpc_ares_request grpc_ares_request; /* Asynchronously resolve \a name. Use \a default_port if a port isn't diff --git a/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc b/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc index fae4c33a17882..282caf215c7ac 100644 --- a/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc +++ b/src/core/ext/filters/client_channel/resolver/dns/native/dns_resolver.cc @@ -58,6 +58,8 @@ class NativeDnsResolver : public Resolver { void RequestReresolutionLocked() override; + void ResetBackoffLocked() override; + void ShutdownLocked() override; private: @@ -158,6 +160,13 @@ void NativeDnsResolver::RequestReresolutionLocked() { } } +void NativeDnsResolver::ResetBackoffLocked() { + if (have_next_resolution_timer_) { + grpc_timer_cancel(&next_resolution_timer_); + } + backoff_.Reset(); +} + void NativeDnsResolver::ShutdownLocked() { if (have_next_resolution_timer_) { grpc_timer_cancel(&next_resolution_timer_); diff --git a/src/core/ext/filters/client_channel/subchannel.cc b/src/core/ext/filters/client_channel/subchannel.cc index 71ef8c518bb3c..48c6030c895e6 100644 --- a/src/core/ext/filters/client_channel/subchannel.cc +++ b/src/core/ext/filters/client_channel/subchannel.cc @@ -132,6 +132,8 @@ struct grpc_subchannel { bool have_alarm; /** have we started the backoff loop */ bool backoff_begun; + // reset_backoff() was called while alarm was pending + bool deferred_reset_backoff; /** our alarm */ grpc_timer alarm; @@ -438,6 +440,9 @@ static void on_alarm(void* arg, grpc_error* error) { if (c->disconnected) { error = GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING("Disconnected", &error, 1); + } else if (c->deferred_reset_backoff) { + c->deferred_reset_backoff = false; + error = GRPC_ERROR_NONE; } else { GRPC_ERROR_REF(error); } @@ -675,6 +680,19 @@ static void on_subchannel_connected(void* arg, grpc_error* error) { grpc_channel_args_destroy(delete_channel_args); } +void grpc_subchannel_reset_backoff(grpc_subchannel* subchannel) { + gpr_mu_lock(&subchannel->mu); + if (subchannel->have_alarm) { + subchannel->deferred_reset_backoff = true; + grpc_timer_cancel(&subchannel->alarm); + } else { + subchannel->backoff_begun = false; + subchannel->backoff->Reset(); + maybe_start_connecting_locked(subchannel); + } + gpr_mu_unlock(&subchannel->mu); +} + /* * grpc_subchannel_call implementation */ diff --git a/src/core/ext/filters/client_channel/subchannel.h b/src/core/ext/filters/client_channel/subchannel.h index 9e53f7d5423b9..a135035d62a20 100644 --- a/src/core/ext/filters/client_channel/subchannel.h +++ b/src/core/ext/filters/client_channel/subchannel.h @@ -145,6 +145,13 @@ grpc_subchannel_get_connected_subchannel(grpc_subchannel* c); const grpc_subchannel_key* grpc_subchannel_get_key( const grpc_subchannel* subchannel); +// Resets the connection backoff of the subchannel. +// TODO(roth): Move connection backoff out of subchannels and up into LB +// policy code (probably by adding a SubchannelGroup between +// SubchannelList and SubchannelData), at which point this method can +// go away. +void grpc_subchannel_reset_backoff(grpc_subchannel* subchannel); + /** continue processing a transport op */ void grpc_subchannel_call_process_op(grpc_subchannel_call* subchannel_call, grpc_transport_stream_op_batch* op); diff --git a/src/core/ext/filters/deadline/deadline_filter.cc b/src/core/ext/filters/deadline/deadline_filter.cc index d23ad67ad51a0..3bd3059312213 100644 --- a/src/core/ext/filters/deadline/deadline_filter.cc +++ b/src/core/ext/filters/deadline/deadline_filter.cc @@ -379,10 +379,10 @@ static bool maybe_add_deadline_filter(grpc_channel_stack_builder* builder, void grpc_deadline_filter_init(void) { grpc_channel_init_register_stage( - GRPC_CLIENT_DIRECT_CHANNEL, GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, + GRPC_CLIENT_DIRECT_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_VERY_HIGH, maybe_add_deadline_filter, (void*)&grpc_client_deadline_filter); grpc_channel_init_register_stage( - GRPC_SERVER_CHANNEL, GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, + GRPC_SERVER_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_VERY_HIGH, maybe_add_deadline_filter, (void*)&grpc_server_deadline_filter); } diff --git a/src/core/ext/filters/http/client_authority_filter.cc b/src/core/ext/filters/http/client_authority_filter.cc index ddc939ed1217b..3c0ae47e8d596 100644 --- a/src/core/ext/filters/http/client_authority_filter.cc +++ b/src/core/ext/filters/http/client_authority_filter.cc @@ -146,12 +146,12 @@ static bool add_client_authority_filter(grpc_channel_stack_builder* builder, } void grpc_client_authority_filter_init(void) { - grpc_channel_init_register_stage(GRPC_CLIENT_SUBCHANNEL, INT_MAX, - add_client_authority_filter, - (void*)&grpc_client_authority_filter); - grpc_channel_init_register_stage(GRPC_CLIENT_DIRECT_CHANNEL, INT_MAX, - add_client_authority_filter, - (void*)&grpc_client_authority_filter); + grpc_channel_init_register_stage( + GRPC_CLIENT_SUBCHANNEL, GRPC_CHANNEL_INIT_PRIORITY_HIGH, + add_client_authority_filter, (void*)&grpc_client_authority_filter); + grpc_channel_init_register_stage( + GRPC_CLIENT_DIRECT_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_HIGH, + add_client_authority_filter, (void*)&grpc_client_authority_filter); } void grpc_client_authority_filter_shutdown(void) {} diff --git a/src/core/ext/filters/http/http_filters_plugin.cc b/src/core/ext/filters/http/http_filters_plugin.cc index f03fa0141dff2..38757710f3951 100644 --- a/src/core/ext/filters/http/http_filters_plugin.cc +++ b/src/core/ext/filters/http/http_filters_plugin.cc @@ -18,6 +18,7 @@ #include +#include #include #include "src/core/ext/filters/http/client/http_client_filter.h" @@ -51,15 +52,15 @@ static bool maybe_add_optional_filter(grpc_channel_stack_builder* builder, bool enable = grpc_channel_arg_get_bool( grpc_channel_args_find(channel_args, filtarg->control_channel_arg), !grpc_channel_args_want_minimal_stack(channel_args)); - return enable ? grpc_channel_stack_builder_prepend_filter( + return enable ? grpc_channel_stack_builder_append_filter( builder, filtarg->filter, nullptr, nullptr) : true; } -static bool maybe_add_required_filter(grpc_channel_stack_builder* builder, - void* arg) { +static bool maybe_append_required_filter(grpc_channel_stack_builder* builder, + void* arg) { return is_building_http_like_transport(builder) - ? grpc_channel_stack_builder_prepend_filter( + ? grpc_channel_stack_builder_append_filter( builder, static_cast(arg), nullptr, nullptr) : true; @@ -67,23 +68,23 @@ static bool maybe_add_required_filter(grpc_channel_stack_builder* builder, void grpc_http_filters_init(void) { grpc_channel_init_register_stage(GRPC_CLIENT_SUBCHANNEL, - GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, + GRPC_CHANNEL_INIT_PRIORITY_HIGH, maybe_add_optional_filter, &compress_filter); grpc_channel_init_register_stage(GRPC_CLIENT_DIRECT_CHANNEL, - GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, + GRPC_CHANNEL_INIT_PRIORITY_HIGH, maybe_add_optional_filter, &compress_filter); grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, - GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, + GRPC_CHANNEL_INIT_PRIORITY_HIGH, maybe_add_optional_filter, &compress_filter); grpc_channel_init_register_stage( - GRPC_CLIENT_SUBCHANNEL, GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, - maybe_add_required_filter, (void*)&grpc_http_client_filter); + GRPC_CLIENT_SUBCHANNEL, GRPC_CHANNEL_INIT_PRIORITY_HIGH, + maybe_append_required_filter, (void*)&grpc_http_client_filter); grpc_channel_init_register_stage( - GRPC_CLIENT_DIRECT_CHANNEL, GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, - maybe_add_required_filter, (void*)&grpc_http_client_filter); + GRPC_CLIENT_DIRECT_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_HIGH, + maybe_append_required_filter, (void*)&grpc_http_client_filter); grpc_channel_init_register_stage( - GRPC_SERVER_CHANNEL, GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, - maybe_add_required_filter, (void*)&grpc_http_server_filter); + GRPC_SERVER_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_HIGH, + maybe_append_required_filter, (void*)&grpc_http_server_filter); } void grpc_http_filters_shutdown(void) {} diff --git a/src/core/ext/filters/load_reporting/server_load_reporting_filter.cc b/src/core/ext/filters/load_reporting/server_load_reporting_filter.cc index 6529046a5e7fb..0c4ffea27bf16 100644 --- a/src/core/ext/filters/load_reporting/server_load_reporting_filter.cc +++ b/src/core/ext/filters/load_reporting/server_load_reporting_filter.cc @@ -345,7 +345,8 @@ struct ServerLoadReportingFilterStaticRegistrar { if (registered) return; RegisterChannelFilter( - "server_load_reporting", GRPC_SERVER_CHANNEL, INT_MAX, + "server_load_reporting", GRPC_SERVER_CHANNEL, + GRPC_CHANNEL_INIT_PRIORITY_LOW, true, MaybeAddServerLoadReportingFilter); // Access measures to ensure they are initialized. Otherwise, we can't // create any valid view before the first RPC. diff --git a/src/core/ext/filters/max_age/max_age_filter.cc b/src/core/ext/filters/max_age/max_age_filter.cc index 1fe8288bd0a53..7db30d5b48820 100644 --- a/src/core/ext/filters/max_age/max_age_filter.cc +++ b/src/core/ext/filters/max_age/max_age_filter.cc @@ -536,7 +536,7 @@ static bool maybe_add_max_age_filter(grpc_channel_stack_builder* builder, void grpc_max_age_filter_init(void) { grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, - GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, + GRPC_CHANNEL_INIT_PRIORITY_LOW, maybe_add_max_age_filter, nullptr); } diff --git a/src/core/ext/filters/message_size/message_size_filter.cc b/src/core/ext/filters/message_size/message_size_filter.cc index c7fc3f2e6275a..1bd9cf1426be7 100644 --- a/src/core/ext/filters/message_size/message_size_filter.cc +++ b/src/core/ext/filters/message_size/message_size_filter.cc @@ -311,13 +311,13 @@ static bool maybe_add_message_size_filter(grpc_channel_stack_builder* builder, void grpc_message_size_filter_init(void) { grpc_channel_init_register_stage(GRPC_CLIENT_SUBCHANNEL, - GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, + GRPC_CHANNEL_INIT_PRIORITY_LOW, maybe_add_message_size_filter, nullptr); grpc_channel_init_register_stage(GRPC_CLIENT_DIRECT_CHANNEL, - GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, + GRPC_CHANNEL_INIT_PRIORITY_LOW, maybe_add_message_size_filter, nullptr); grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, - GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, + GRPC_CHANNEL_INIT_PRIORITY_LOW, maybe_add_message_size_filter, nullptr); } diff --git a/src/core/lib/channel/connected_channel.cc b/src/core/lib/channel/connected_channel.cc index e2ea334dedf25..c78849a29b6f6 100644 --- a/src/core/lib/channel/connected_channel.cc +++ b/src/core/lib/channel/connected_channel.cc @@ -228,8 +228,8 @@ static void bind_transport(grpc_channel_stack* channel_stack, grpc_transport_stream_size(static_cast(t)); } -bool grpc_add_connected_filter(grpc_channel_stack_builder* builder, - void* arg_must_be_null) { +bool grpc_append_connected_filter(grpc_channel_stack_builder* builder, + void* arg_must_be_null) { GPR_ASSERT(arg_must_be_null == nullptr); grpc_transport* t = grpc_channel_stack_builder_get_transport(builder); GPR_ASSERT(t != nullptr); diff --git a/src/core/lib/channel/connected_channel.h b/src/core/lib/channel/connected_channel.h index faa1c73a21f13..280daf040da56 100644 --- a/src/core/lib/channel/connected_channel.h +++ b/src/core/lib/channel/connected_channel.h @@ -25,8 +25,8 @@ extern const grpc_channel_filter grpc_connected_filter; -bool grpc_add_connected_filter(grpc_channel_stack_builder* builder, - void* arg_must_be_null); +bool grpc_append_connected_filter(grpc_channel_stack_builder* builder, + void* arg_must_be_null); /* Debug helper to dig the transport stream out of a call element */ grpc_stream* grpc_connected_channel_get_stream(grpc_call_element* elem); diff --git a/src/core/lib/iomgr/iocp_windows.cc b/src/core/lib/iomgr/iocp_windows.cc index ce772310362c1..ad325fe215692 100644 --- a/src/core/lib/iomgr/iocp_windows.cc +++ b/src/core/lib/iomgr/iocp_windows.cc @@ -89,10 +89,15 @@ grpc_iocp_work_status grpc_iocp_work(grpc_millis deadline) { } else { abort(); } - success = WSAGetOverlappedResult(socket->socket, &info->overlapped, &bytes, - FALSE, &flags); - info->bytes_transfered = bytes; - info->wsa_error = success ? 0 : WSAGetLastError(); + if (socket->shutdown_called) { + info->bytes_transfered = 0; + info->wsa_error = WSA_OPERATION_ABORTED; + } else { + success = WSAGetOverlappedResult(socket->socket, &info->overlapped, &bytes, + FALSE, &flags); + info->bytes_transfered = bytes; + info->wsa_error = success ? 0 : WSAGetLastError(); + } GPR_ASSERT(overlapped == &info->overlapped); grpc_socket_become_ready(socket, info); return GRPC_IOCP_WORK_WORK; diff --git a/src/core/lib/iomgr/socket_windows.cc b/src/core/lib/iomgr/socket_windows.cc index 4ad31cb35d1ea..999c6646ad495 100644 --- a/src/core/lib/iomgr/socket_windows.cc +++ b/src/core/lib/iomgr/socket_windows.cc @@ -52,6 +52,10 @@ grpc_winsocket* grpc_winsocket_create(SOCKET socket, const char* name) { return r; } +SOCKET grpc_winsocket_wrapped_socket(grpc_winsocket* socket) { + return socket->socket; +} + /* Schedule a shutdown of the socket operations. Will call the pending operations to abort them. We need to do that this way because of the various callsites of that function, which happens to be in various diff --git a/src/core/lib/iomgr/socket_windows.h b/src/core/lib/iomgr/socket_windows.h index b09b9da5628d6..46d7d58356046 100644 --- a/src/core/lib/iomgr/socket_windows.h +++ b/src/core/lib/iomgr/socket_windows.h @@ -92,6 +92,8 @@ typedef struct grpc_winsocket { it will be responsible for closing it. */ grpc_winsocket* grpc_winsocket_create(SOCKET socket, const char* name); +SOCKET grpc_winsocket_wrapped_socket(grpc_winsocket* socket); + /* Initiate an asynchronous shutdown of the socket. Will call off any pending operation to cancel them. */ void grpc_winsocket_shutdown(grpc_winsocket* socket); diff --git a/src/core/lib/iomgr/tcp_windows.cc b/src/core/lib/iomgr/tcp_windows.cc index 5d316d477b7a2..b3cb442f18f83 100644 --- a/src/core/lib/iomgr/tcp_windows.cc +++ b/src/core/lib/iomgr/tcp_windows.cc @@ -53,7 +53,7 @@ extern grpc_core::TraceFlag grpc_tcp_trace; -static grpc_error* set_non_block(SOCKET sock) { +grpc_error* grpc_tcp_set_non_block(SOCKET sock) { int status; uint32_t param = 1; DWORD ret; @@ -90,7 +90,7 @@ static grpc_error* enable_loopback_fast_path(SOCKET sock) { grpc_error* grpc_tcp_prepare_socket(SOCKET sock) { grpc_error* err; - err = set_non_block(sock); + err = grpc_tcp_set_non_block(sock); if (err != GRPC_ERROR_NONE) return err; err = set_dualstack(sock); if (err != GRPC_ERROR_NONE) return err; diff --git a/src/core/lib/iomgr/tcp_windows.h b/src/core/lib/iomgr/tcp_windows.h index 161a545a2a5ea..04ef8102b65ce 100644 --- a/src/core/lib/iomgr/tcp_windows.h +++ b/src/core/lib/iomgr/tcp_windows.h @@ -46,6 +46,8 @@ grpc_endpoint* grpc_tcp_create(grpc_winsocket* socket, grpc_error* grpc_tcp_prepare_socket(SOCKET sock); +grpc_error* grpc_tcp_set_non_block(SOCKET sock); + #endif #endif /* GRPC_CORE_LIB_IOMGR_TCP_WINDOWS_H */ diff --git a/src/core/lib/surface/channel.cc b/src/core/lib/surface/channel.cc index 7cbd61adef52b..82635d3c21504 100644 --- a/src/core/lib/surface/channel.cc +++ b/src/core/lib/surface/channel.cc @@ -281,6 +281,17 @@ void grpc_channel_get_info(grpc_channel* channel, elem->filter->get_channel_info(elem, channel_info); } +void grpc_channel_reset_connect_backoff(grpc_channel* channel) { + grpc_core::ExecCtx exec_ctx; + GRPC_API_TRACE("grpc_channel_reset_connect_backoff(channel=%p)", 1, + (channel)); + grpc_transport_op* op = grpc_make_transport_op(nullptr); + op->reset_connect_backoff = true; + grpc_channel_element* elem = + grpc_channel_stack_element(CHANNEL_STACK_FROM_CHANNEL(channel), 0); + elem->filter->start_transport_op(elem, op); +} + static grpc_call* grpc_channel_create_call_internal( grpc_channel* channel, grpc_call* parent_call, uint32_t propagation_mask, grpc_completion_queue* cq, grpc_pollset_set* pollset_set_alternative, diff --git a/src/core/lib/surface/channel_init.h b/src/core/lib/surface/channel_init.h index f01852473ba30..6543796b4c867 100644 --- a/src/core/lib/surface/channel_init.h +++ b/src/core/lib/surface/channel_init.h @@ -21,11 +21,37 @@ #include +#include + #include "src/core/lib/channel/channel_stack_builder.h" #include "src/core/lib/surface/channel_stack_type.h" #include "src/core/lib/transport/transport.h" -#define GRPC_CHANNEL_INIT_BUILTIN_PRIORITY 10000 +// Priority for channel registration functions to be used in +// grpc_channel_init_register_stage(). The priority dictates the +// order in which the registration functions run. +// +// When used to register a filter, the filter can either be appended or +// prepended, thus dictating whether the filter goes at the top or bottom of +// the stack. Higher priority functions can get closer to the top or bottom +// of the stack than lower priority functions. +enum { + // Default level. Most of filters should use this level if their location in + // the stack does not matter. + GRPC_CHANNEL_INIT_PRIORITY_LOW = 0, + // For filters that should be added after the group of filters with default + // priority, such as auth filters. + GRPC_CHANNEL_INIT_PRIORITY_MED = 10000, + // For filters that need to be close to top or bottom, such as protocol-level + // filters (client_authority, http-client, http-server). + GRPC_CHANNEL_INIT_PRIORITY_HIGH = 20000, + // For filters that need to be very close to the wire or surface, such as + // stats filters (census). + GRPC_CHANNEL_INIT_PRIORITY_VERY_HIGH = 30000, + // For things that have to happen last, such as connected channel filter or + // surface server filter. Consider as reserved for gRPC internals. + GRPC_CHANNEL_INIT_PRIORITY_MAX = INT_MAX +}; /// This module provides a way for plugins (and the grpc core library itself) /// to register mutators for channel stacks. diff --git a/src/core/lib/surface/init.cc b/src/core/lib/surface/init.cc index 0ad82fed99ee4..7807b261d406b 100644 --- a/src/core/lib/surface/init.cc +++ b/src/core/lib/surface/init.cc @@ -70,11 +70,6 @@ static void do_basic_init(void) { g_initializations = 0; } -static bool append_filter(grpc_channel_stack_builder* builder, void* arg) { - return grpc_channel_stack_builder_append_filter( - builder, static_cast(arg), nullptr, nullptr); -} - static bool prepend_filter(grpc_channel_stack_builder* builder, void* arg) { return grpc_channel_stack_builder_prepend_filter( builder, static_cast(arg), nullptr, nullptr); @@ -82,19 +77,20 @@ static bool prepend_filter(grpc_channel_stack_builder* builder, void* arg) { static void register_builtin_channel_init() { grpc_channel_init_register_stage(GRPC_CLIENT_SUBCHANNEL, - GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, - grpc_add_connected_filter, nullptr); + GRPC_CHANNEL_INIT_PRIORITY_MAX, + grpc_append_connected_filter, nullptr); grpc_channel_init_register_stage(GRPC_CLIENT_DIRECT_CHANNEL, - GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, - grpc_add_connected_filter, nullptr); + GRPC_CHANNEL_INIT_PRIORITY_MAX, + grpc_append_connected_filter, nullptr); grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, - GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, - grpc_add_connected_filter, nullptr); + GRPC_CHANNEL_INIT_PRIORITY_MAX, + grpc_append_connected_filter, nullptr); grpc_channel_init_register_stage(GRPC_CLIENT_LAME_CHANNEL, - GRPC_CHANNEL_INIT_BUILTIN_PRIORITY, - append_filter, (void*)&grpc_lame_filter); - grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, INT_MAX, prepend_filter, - (void*)&grpc_server_top_filter); + GRPC_CHANNEL_INIT_PRIORITY_MAX, + prepend_filter, (void*)&grpc_lame_filter); + grpc_channel_init_register_stage( + GRPC_SERVER_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_MAX, prepend_filter, + (void*)&grpc_server_top_filter); } typedef struct grpc_plugin { diff --git a/src/core/lib/surface/init_secure.cc b/src/core/lib/surface/init_secure.cc index 28c6f7b121f24..8058aaa804d4d 100644 --- a/src/core/lib/surface/init_secure.cc +++ b/src/core/lib/surface/init_secure.cc @@ -67,14 +67,17 @@ static bool maybe_prepend_server_auth_filter( } void grpc_register_security_filters(void) { - // Register the auth client with a priority < INT_MAX to allow the authority + // Register the auth client with a medium priority to allow the authority // filter -on which the auth filter depends- to be higher on the channel // stack. - grpc_channel_init_register_stage(GRPC_CLIENT_SUBCHANNEL, INT_MAX - 1, + grpc_channel_init_register_stage(GRPC_CLIENT_SUBCHANNEL, + GRPC_CHANNEL_INIT_PRIORITY_MED, maybe_prepend_client_auth_filter, nullptr); - grpc_channel_init_register_stage(GRPC_CLIENT_DIRECT_CHANNEL, INT_MAX - 1, + grpc_channel_init_register_stage(GRPC_CLIENT_DIRECT_CHANNEL, + GRPC_CHANNEL_INIT_PRIORITY_MED, maybe_prepend_client_auth_filter, nullptr); - grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, INT_MAX, + grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, + GRPC_CHANNEL_INIT_PRIORITY_MED, maybe_prepend_server_auth_filter, nullptr); } diff --git a/src/core/lib/transport/transport.h b/src/core/lib/transport/transport.h index 585b9dfae9196..9e784635c690c 100644 --- a/src/core/lib/transport/transport.h +++ b/src/core/lib/transport/transport.h @@ -282,6 +282,8 @@ typedef struct grpc_transport_op { /** Called when the ping ack is received */ grpc_closure* on_ack; } send_ping; + // If true, will reset the channel's connection backoff. + bool reset_connect_backoff; /*************************************************************************** * remaining fields are initialized and used at the discretion of the diff --git a/src/cpp/client/channel_cc.cc b/src/cpp/client/channel_cc.cc index 867f31f0255f7..39b891c2e1f99 100644 --- a/src/cpp/client/channel_cc.cc +++ b/src/cpp/client/channel_cc.cc @@ -84,6 +84,14 @@ grpc::string Channel::GetServiceConfigJSON() const { &channel_info.service_config_json); } +namespace experimental { + +void ChannelResetConnectionBackoff(Channel* channel) { + grpc_channel_reset_connect_backoff(channel->c_channel_); +} + +} // namespace experimental + internal::Call Channel::CreateCall(const internal::RpcMethod& method, ClientContext* context, CompletionQueue* cq) { diff --git a/src/cpp/common/channel_filter.cc b/src/cpp/common/channel_filter.cc index 422e7bb65ee2c..0634b0416fde4 100644 --- a/src/cpp/common/channel_filter.cc +++ b/src/cpp/common/channel_filter.cc @@ -78,8 +78,13 @@ bool MaybeAddFilter(grpc_channel_stack_builder* builder, void* arg) { grpc_channel_stack_builder_get_channel_arguments(builder); if (!filter.include_filter(*args)) return true; } - return grpc_channel_stack_builder_prepend_filter(builder, &filter.filter, - nullptr, nullptr); + if (filter.prepend) { + return grpc_channel_stack_builder_prepend_filter(builder, &filter.filter, + nullptr, nullptr); + } else { + return grpc_channel_stack_builder_append_filter(builder, &filter.filter, + nullptr, nullptr); + } } } // namespace diff --git a/src/cpp/common/channel_filter.h b/src/cpp/common/channel_filter.h index 5e569c97e68dc..359c72737c2ac 100644 --- a/src/cpp/common/channel_filter.h +++ b/src/cpp/common/channel_filter.h @@ -36,7 +36,8 @@ /// \c ChannelData. Then register the filter using something like this: /// \code{.cpp} /// RegisterChannelFilter( -/// "name-of-filter", GRPC_SERVER_CHANNEL, INT_MAX, nullptr); +/// "name-of-filter", GRPC_SERVER_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_LOW, +/// true, nullptr); /// \endcode namespace grpc { @@ -351,6 +352,7 @@ class ChannelFilter final { struct FilterRecord { grpc_channel_stack_type stack_type; int priority; + bool prepend; std::function include_filter; grpc_channel_filter filter; }; @@ -363,12 +365,14 @@ void ChannelFilterPluginShutdown(); /// Registers a new filter. /// Must be called by only one thread at a time. +/// The \a prepend argument decides whether to prepend or append the filter. /// The \a include_filter argument specifies a function that will be called /// to determine at run-time whether or not to add the filter. If the /// value is nullptr, the filter will be added unconditionally. template void RegisterChannelFilter( const char* name, grpc_channel_stack_type stack_type, int priority, + bool prepend, std::function include_filter) { // If we haven't been called before, initialize channel_filters and // call grpc_register_plugin(). @@ -383,6 +387,7 @@ void RegisterChannelFilter( internal::FilterRecord filter_record = { stack_type, priority, + prepend, include_filter, {FilterType::StartTransportStreamOpBatch, FilterType::StartTransportOp, FilterType::call_data_size, FilterType::InitCallElement, diff --git a/src/cpp/ext/filters/census/grpc_plugin.cc b/src/cpp/ext/filters/census/grpc_plugin.cc index f978ed3bf51b8..f79e0e0e96023 100644 --- a/src/cpp/ext/filters/census/grpc_plugin.cc +++ b/src/cpp/ext/filters/census/grpc_plugin.cc @@ -32,10 +32,12 @@ namespace grpc { void RegisterOpenCensusPlugin() { RegisterChannelFilter( - "opencensus_client", GRPC_CLIENT_CHANNEL, INT_MAX /* priority */, + "opencensus_client", GRPC_CLIENT_CHANNEL, + GRPC_CHANNEL_INIT_PRIORITY_VERY_HIGH, true /* prepend */, nullptr /* condition function */); RegisterChannelFilter( - "opencensus_server", GRPC_SERVER_CHANNEL, INT_MAX /* priority */, + "opencensus_server", GRPC_SERVER_CHANNEL, + GRPC_CHANNEL_INIT_PRIORITY_VERY_HIGH, true /* prepend */, nullptr /* condition function */); // Access measures to ensure they are initialized. Otherwise, creating a view diff --git a/src/cpp/server/load_reporter/util.cc b/src/cpp/server/load_reporter/util.cc index a2f2f11e707e9..89bdf57049c72 100644 --- a/src/cpp/server/load_reporter/util.cc +++ b/src/cpp/server/load_reporter/util.cc @@ -20,6 +20,8 @@ #include +#include + #include namespace grpc { diff --git a/src/csharp/Grpc.Core/Version.csproj.include b/src/csharp/Grpc.Core/Version.csproj.include index 1a746acdd4fcd..6b0731eb401e6 100755 --- a/src/csharp/Grpc.Core/Version.csproj.include +++ b/src/csharp/Grpc.Core/Version.csproj.include @@ -2,6 +2,6 @@ 1.15.0-dev - 3.5.1 + 3.6.0 diff --git a/src/csharp/Grpc.IntegrationTesting/Control.cs b/src/csharp/Grpc.IntegrationTesting/Control.cs index f3284a5390fe0..6e00348451307 100644 --- a/src/csharp/Grpc.IntegrationTesting/Control.cs +++ b/src/csharp/Grpc.IntegrationTesting/Control.cs @@ -1039,14 +1039,14 @@ public sealed partial class ClientConfig : pb::IMessage { public ClientConfig(ClientConfig other) : this() { serverTargets_ = other.serverTargets_.Clone(); clientType_ = other.clientType_; - SecurityParams = other.securityParams_ != null ? other.SecurityParams.Clone() : null; + securityParams_ = other.securityParams_ != null ? other.securityParams_.Clone() : null; outstandingRpcsPerChannel_ = other.outstandingRpcsPerChannel_; clientChannels_ = other.clientChannels_; asyncClientThreads_ = other.asyncClientThreads_; rpcType_ = other.rpcType_; - LoadParams = other.loadParams_ != null ? other.LoadParams.Clone() : null; - PayloadConfig = other.payloadConfig_ != null ? other.PayloadConfig.Clone() : null; - HistogramParams = other.histogramParams_ != null ? other.HistogramParams.Clone() : null; + loadParams_ = other.loadParams_ != null ? other.loadParams_.Clone() : null; + payloadConfig_ = other.payloadConfig_ != null ? other.payloadConfig_.Clone() : null; + histogramParams_ = other.histogramParams_ != null ? other.histogramParams_.Clone() : null; coreList_ = other.coreList_.Clone(); coreLimit_ = other.coreLimit_; otherClientApi_ = other.otherClientApi_; @@ -1647,7 +1647,7 @@ public sealed partial class ClientStatus : pb::IMessage { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public ClientStatus(ClientStatus other) : this() { - Stats = other.stats_ != null ? other.Stats.Clone() : null; + stats_ = other.stats_ != null ? other.stats_.Clone() : null; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } @@ -2122,11 +2122,11 @@ public sealed partial class ServerConfig : pb::IMessage { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public ServerConfig(ServerConfig other) : this() { serverType_ = other.serverType_; - SecurityParams = other.securityParams_ != null ? other.SecurityParams.Clone() : null; + securityParams_ = other.securityParams_ != null ? other.securityParams_.Clone() : null; port_ = other.port_; asyncServerThreads_ = other.asyncServerThreads_; coreLimit_ = other.coreLimit_; - PayloadConfig = other.payloadConfig_ != null ? other.PayloadConfig.Clone() : null; + payloadConfig_ = other.payloadConfig_ != null ? other.payloadConfig_.Clone() : null; coreList_ = other.coreList_.Clone(); otherServerApi_ = other.otherServerApi_; threadsPerCq_ = other.threadsPerCq_; @@ -2758,7 +2758,7 @@ public sealed partial class ServerStatus : pb::IMessage { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public ServerStatus(ServerStatus other) : this() { - Stats = other.stats_ != null ? other.Stats.Clone() : null; + stats_ = other.stats_ != null ? other.stats_.Clone() : null; port_ = other.port_; cores_ = other.cores_; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); @@ -3293,9 +3293,9 @@ public sealed partial class Scenario : pb::IMessage { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public Scenario(Scenario other) : this() { name_ = other.name_; - ClientConfig = other.clientConfig_ != null ? other.ClientConfig.Clone() : null; + clientConfig_ = other.clientConfig_ != null ? other.clientConfig_.Clone() : null; numClients_ = other.numClients_; - ServerConfig = other.serverConfig_ != null ? other.ServerConfig.Clone() : null; + serverConfig_ = other.serverConfig_ != null ? other.serverConfig_.Clone() : null; numServers_ = other.numServers_; warmupSeconds_ = other.warmupSeconds_; benchmarkSeconds_ = other.benchmarkSeconds_; @@ -4422,12 +4422,12 @@ public sealed partial class ScenarioResult : pb::IMessage { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public ScenarioResult(ScenarioResult other) : this() { - Scenario = other.scenario_ != null ? other.Scenario.Clone() : null; - Latencies = other.latencies_ != null ? other.Latencies.Clone() : null; + scenario_ = other.scenario_ != null ? other.scenario_.Clone() : null; + latencies_ = other.latencies_ != null ? other.latencies_.Clone() : null; clientStats_ = other.clientStats_.Clone(); serverStats_ = other.serverStats_.Clone(); serverCores_ = other.serverCores_.Clone(); - Summary = other.summary_ != null ? other.Summary.Clone() : null; + summary_ = other.summary_ != null ? other.summary_.Clone() : null; clientSuccess_ = other.clientSuccess_.Clone(); serverSuccess_ = other.serverSuccess_.Clone(); requestResults_ = other.requestResults_.Clone(); diff --git a/src/csharp/Grpc.IntegrationTesting/EchoMessages.cs b/src/csharp/Grpc.IntegrationTesting/EchoMessages.cs index 39c3d76ce824f..80a1007e9a576 100644 --- a/src/csharp/Grpc.IntegrationTesting/EchoMessages.cs +++ b/src/csharp/Grpc.IntegrationTesting/EchoMessages.cs @@ -28,7 +28,7 @@ public static partial class EchoMessagesReflection { "DGdycGMudGVzdGluZyIyCglEZWJ1Z0luZm8SFQoNc3RhY2tfZW50cmllcxgB", "IAMoCRIOCgZkZXRhaWwYAiABKAkiUAoLRXJyb3JTdGF0dXMSDAoEY29kZRgB", "IAEoBRIVCg1lcnJvcl9tZXNzYWdlGAIgASgJEhwKFGJpbmFyeV9lcnJvcl9k", - "ZXRhaWxzGAMgASgJIuIDCg1SZXF1ZXN0UGFyYW1zEhUKDWVjaG9fZGVhZGxp", + "ZXRhaWxzGAMgASgJIv8DCg1SZXF1ZXN0UGFyYW1zEhUKDWVjaG9fZGVhZGxp", "bmUYASABKAgSHgoWY2xpZW50X2NhbmNlbF9hZnRlcl91cxgCIAEoBRIeChZz", "ZXJ2ZXJfY2FuY2VsX2FmdGVyX3VzGAMgASgFEhUKDWVjaG9fbWV0YWRhdGEY", "BCABKAgSGgoSY2hlY2tfYXV0aF9jb250ZXh0GAUgASgIEh8KF3Jlc3BvbnNl", @@ -39,18 +39,18 @@ public static partial class EchoMessagesReflection { "Zy5EZWJ1Z0luZm8SEgoKc2VydmVyX2RpZRgMIAEoCBIcChRiaW5hcnlfZXJy", "b3JfZGV0YWlscxgNIAEoCRIxCg5leHBlY3RlZF9lcnJvchgOIAEoCzIZLmdy", "cGMudGVzdGluZy5FcnJvclN0YXR1cxIXCg9zZXJ2ZXJfc2xlZXBfdXMYDyAB", - "KAUiSgoLRWNob1JlcXVlc3QSDwoHbWVzc2FnZRgBIAEoCRIqCgVwYXJhbRgC", - "IAEoCzIbLmdycGMudGVzdGluZy5SZXF1ZXN0UGFyYW1zIkYKDlJlc3BvbnNl", - "UGFyYW1zEhgKEHJlcXVlc3RfZGVhZGxpbmUYASABKAMSDAoEaG9zdBgCIAEo", - "CRIMCgRwZWVyGAMgASgJIkwKDEVjaG9SZXNwb25zZRIPCgdtZXNzYWdlGAEg", - "ASgJEisKBXBhcmFtGAIgASgLMhwuZ3JwYy50ZXN0aW5nLlJlc3BvbnNlUGFy", - "YW1zYgZwcm90bzM=")); + "KAUSGwoTYmFja2VuZF9jaGFubmVsX2lkeBgQIAEoBSJKCgtFY2hvUmVxdWVz", + "dBIPCgdtZXNzYWdlGAEgASgJEioKBXBhcmFtGAIgASgLMhsuZ3JwYy50ZXN0", + "aW5nLlJlcXVlc3RQYXJhbXMiRgoOUmVzcG9uc2VQYXJhbXMSGAoQcmVxdWVz", + "dF9kZWFkbGluZRgBIAEoAxIMCgRob3N0GAIgASgJEgwKBHBlZXIYAyABKAki", + "TAoMRWNob1Jlc3BvbnNlEg8KB21lc3NhZ2UYASABKAkSKwoFcGFyYW0YAiAB", + "KAsyHC5ncnBjLnRlc3RpbmcuUmVzcG9uc2VQYXJhbXNiBnByb3RvMw==")); descriptor = pbr::FileDescriptor.FromGeneratedCode(descriptorData, new pbr::FileDescriptor[] { }, new pbr::GeneratedClrTypeInfo(null, new pbr::GeneratedClrTypeInfo[] { new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.DebugInfo), global::Grpc.Testing.DebugInfo.Parser, new[]{ "StackEntries", "Detail" }, null, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.ErrorStatus), global::Grpc.Testing.ErrorStatus.Parser, new[]{ "Code", "ErrorMessage", "BinaryErrorDetails" }, null, null, null), - new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.RequestParams), global::Grpc.Testing.RequestParams.Parser, new[]{ "EchoDeadline", "ClientCancelAfterUs", "ServerCancelAfterUs", "EchoMetadata", "CheckAuthContext", "ResponseMessageLength", "EchoPeer", "ExpectedClientIdentity", "SkipCancelledCheck", "ExpectedTransportSecurityType", "DebugInfo", "ServerDie", "BinaryErrorDetails", "ExpectedError", "ServerSleepUs" }, null, null, null), + new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.RequestParams), global::Grpc.Testing.RequestParams.Parser, new[]{ "EchoDeadline", "ClientCancelAfterUs", "ServerCancelAfterUs", "EchoMetadata", "CheckAuthContext", "ResponseMessageLength", "EchoPeer", "ExpectedClientIdentity", "SkipCancelledCheck", "ExpectedTransportSecurityType", "DebugInfo", "ServerDie", "BinaryErrorDetails", "ExpectedError", "ServerSleepUs", "BackendChannelIdx" }, null, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.EchoRequest), global::Grpc.Testing.EchoRequest.Parser, new[]{ "Message", "Param" }, null, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.ResponseParams), global::Grpc.Testing.ResponseParams.Parser, new[]{ "RequestDeadline", "Host", "Peer" }, null, null, null), new pbr::GeneratedClrTypeInfo(typeof(global::Grpc.Testing.EchoResponse), global::Grpc.Testing.EchoResponse.Parser, new[]{ "Message", "Param" }, null, null, null) @@ -435,11 +435,12 @@ public sealed partial class RequestParams : pb::IMessage { expectedClientIdentity_ = other.expectedClientIdentity_; skipCancelledCheck_ = other.skipCancelledCheck_; expectedTransportSecurityType_ = other.expectedTransportSecurityType_; - DebugInfo = other.debugInfo_ != null ? other.DebugInfo.Clone() : null; + debugInfo_ = other.debugInfo_ != null ? other.debugInfo_.Clone() : null; serverDie_ = other.serverDie_; binaryErrorDetails_ = other.binaryErrorDetails_; - ExpectedError = other.expectedError_ != null ? other.ExpectedError.Clone() : null; + expectedError_ = other.expectedError_ != null ? other.expectedError_.Clone() : null; serverSleepUs_ = other.serverSleepUs_; + backendChannelIdx_ = other.backendChannelIdx_; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } @@ -622,6 +623,20 @@ public sealed partial class RequestParams : pb::IMessage { } } + /// Field number for the "backend_channel_idx" field. + public const int BackendChannelIdxFieldNumber = 16; + private int backendChannelIdx_; + /// + /// which backend to send request to + /// + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] + public int BackendChannelIdx { + get { return backendChannelIdx_; } + set { + backendChannelIdx_ = value; + } + } + [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public override bool Equals(object other) { return Equals(other as RequestParams); @@ -650,6 +665,7 @@ public sealed partial class RequestParams : pb::IMessage { if (BinaryErrorDetails != other.BinaryErrorDetails) return false; if (!object.Equals(ExpectedError, other.ExpectedError)) return false; if (ServerSleepUs != other.ServerSleepUs) return false; + if (BackendChannelIdx != other.BackendChannelIdx) return false; return Equals(_unknownFields, other._unknownFields); } @@ -671,6 +687,7 @@ public sealed partial class RequestParams : pb::IMessage { if (BinaryErrorDetails.Length != 0) hash ^= BinaryErrorDetails.GetHashCode(); if (expectedError_ != null) hash ^= ExpectedError.GetHashCode(); if (ServerSleepUs != 0) hash ^= ServerSleepUs.GetHashCode(); + if (BackendChannelIdx != 0) hash ^= BackendChannelIdx.GetHashCode(); if (_unknownFields != null) { hash ^= _unknownFields.GetHashCode(); } @@ -744,6 +761,10 @@ public sealed partial class RequestParams : pb::IMessage { output.WriteRawTag(120); output.WriteInt32(ServerSleepUs); } + if (BackendChannelIdx != 0) { + output.WriteRawTag(128, 1); + output.WriteInt32(BackendChannelIdx); + } if (_unknownFields != null) { _unknownFields.WriteTo(output); } @@ -797,6 +818,9 @@ public sealed partial class RequestParams : pb::IMessage { if (ServerSleepUs != 0) { size += 1 + pb::CodedOutputStream.ComputeInt32Size(ServerSleepUs); } + if (BackendChannelIdx != 0) { + size += 2 + pb::CodedOutputStream.ComputeInt32Size(BackendChannelIdx); + } if (_unknownFields != null) { size += _unknownFields.CalculateSize(); } @@ -859,6 +883,9 @@ public sealed partial class RequestParams : pb::IMessage { if (other.ServerSleepUs != 0) { ServerSleepUs = other.ServerSleepUs; } + if (other.BackendChannelIdx != 0) { + BackendChannelIdx = other.BackendChannelIdx; + } _unknownFields = pb::UnknownFieldSet.MergeFrom(_unknownFields, other._unknownFields); } @@ -936,6 +963,10 @@ public sealed partial class RequestParams : pb::IMessage { ServerSleepUs = input.ReadInt32(); break; } + case 128: { + BackendChannelIdx = input.ReadInt32(); + break; + } } } } @@ -968,7 +999,7 @@ public sealed partial class EchoRequest : pb::IMessage { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public EchoRequest(EchoRequest other) : this() { message_ = other.message_; - Param = other.param_ != null ? other.Param.Clone() : null; + param_ = other.param_ != null ? other.param_.Clone() : null; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } @@ -1316,7 +1347,7 @@ public sealed partial class EchoResponse : pb::IMessage { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public EchoResponse(EchoResponse other) : this() { message_ = other.message_; - Param = other.param_ != null ? other.Param.Clone() : null; + param_ = other.param_ != null ? other.param_.Clone() : null; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } diff --git a/src/csharp/Grpc.IntegrationTesting/Messages.cs b/src/csharp/Grpc.IntegrationTesting/Messages.cs index b5c93babd2096..35546f1b67134 100644 --- a/src/csharp/Grpc.IntegrationTesting/Messages.cs +++ b/src/csharp/Grpc.IntegrationTesting/Messages.cs @@ -77,7 +77,6 @@ public static partial class MessagesReflection { } #region Enums /// - /// DEPRECATED, don't use. To be removed shortly. /// The type of payload that should be returned. /// public enum PayloadType { @@ -269,7 +268,6 @@ public sealed partial class Payload : pb::IMessage { public const int TypeFieldNumber = 1; private global::Grpc.Testing.PayloadType type_ = 0; /// - /// DEPRECATED, don't use. To be removed shortly. /// The type of data in body. /// [global::System.Diagnostics.DebuggerNonUserCodeAttribute] @@ -585,12 +583,12 @@ public sealed partial class SimpleRequest : pb::IMessage { public SimpleRequest(SimpleRequest other) : this() { responseType_ = other.responseType_; responseSize_ = other.responseSize_; - Payload = other.payload_ != null ? other.Payload.Clone() : null; + payload_ = other.payload_ != null ? other.payload_.Clone() : null; fillUsername_ = other.fillUsername_; fillOauthScope_ = other.fillOauthScope_; - ResponseCompressed = other.responseCompressed_ != null ? other.ResponseCompressed.Clone() : null; - ResponseStatus = other.responseStatus_ != null ? other.ResponseStatus.Clone() : null; - ExpectCompressed = other.expectCompressed_ != null ? other.ExpectCompressed.Clone() : null; + responseCompressed_ = other.responseCompressed_ != null ? other.responseCompressed_.Clone() : null; + responseStatus_ = other.responseStatus_ != null ? other.responseStatus_.Clone() : null; + expectCompressed_ = other.expectCompressed_ != null ? other.expectCompressed_.Clone() : null; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } @@ -603,7 +601,6 @@ public sealed partial class SimpleRequest : pb::IMessage { public const int ResponseTypeFieldNumber = 1; private global::Grpc.Testing.PayloadType responseType_ = 0; /// - /// DEPRECATED, don't use. To be removed shortly. /// Desired payload type in the response from the server. /// If response_type is RANDOM, server randomly chooses one from other formats. /// @@ -964,7 +961,7 @@ public sealed partial class SimpleResponse : pb::IMessage { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public SimpleResponse(SimpleResponse other) : this() { - Payload = other.payload_ != null ? other.Payload.Clone() : null; + payload_ = other.payload_ != null ? other.payload_.Clone() : null; username_ = other.username_; oauthScope_ = other.oauthScope_; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); @@ -1168,8 +1165,8 @@ public sealed partial class StreamingInputCallRequest : pb::IMessage - /// DEPRECATED, don't use. To be removed shortly. /// Desired payload type in the response from the server. /// If response_type is RANDOM, the payload from each response in the stream /// might be of different types. This is to simulate a mixed type of payload @@ -1927,7 +1923,7 @@ public sealed partial class StreamingOutputCallResponse : pb::IMessage { totalCpuTime_ = other.totalCpuTime_; idleCpuTime_ = other.idleCpuTime_; cqPollCount_ = other.cqPollCount_; - CoreStats = other.coreStats_ != null ? other.CoreStats.Clone() : null; + coreStats_ = other.coreStats_ != null ? other.coreStats_.Clone() : null; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } @@ -993,13 +993,13 @@ public sealed partial class ClientStats : pb::IMessage { [global::System.Diagnostics.DebuggerNonUserCodeAttribute] public ClientStats(ClientStats other) : this() { - Latencies = other.latencies_ != null ? other.Latencies.Clone() : null; + latencies_ = other.latencies_ != null ? other.latencies_.Clone() : null; timeElapsed_ = other.timeElapsed_; timeUser_ = other.timeUser_; timeSystem_ = other.timeSystem_; requestResults_ = other.requestResults_.Clone(); cqPollCount_ = other.cqPollCount_; - CoreStats = other.coreStats_ != null ? other.CoreStats.Clone() : null; + coreStats_ = other.coreStats_ != null ? other.coreStats_.Clone() : null; _unknownFields = pb::UnknownFieldSet.Clone(other._unknownFields); } diff --git a/src/csharp/Grpc.Reflection/Reflection.cs b/src/csharp/Grpc.Reflection/Reflection.cs index 84b2a0a8421aa..e319be5bff74e 100644 --- a/src/csharp/Grpc.Reflection/Reflection.cs +++ b/src/csharp/Grpc.Reflection/Reflection.cs @@ -610,7 +610,7 @@ public sealed partial class ServerReflectionResponse : pb::IMessage=3.5.2.post1', + 'protobuf>=3.6.0', 'grpcio>={version}'.format(version=grpc_version.VERSION), ) diff --git a/src/python/grpcio_reflection/setup.py b/src/python/grpcio_reflection/setup.py index 589d0ff556557..b4087d87b49b7 100644 --- a/src/python/grpcio_reflection/setup.py +++ b/src/python/grpcio_reflection/setup.py @@ -58,7 +58,7 @@ def run(self): } INSTALL_REQUIRES = ( - 'protobuf>=3.5.2.post1', + 'protobuf>=3.6.0', 'grpcio>={version}'.format(version=grpc_version.VERSION), ) diff --git a/src/python/grpcio_testing/setup.py b/src/python/grpcio_testing/setup.py index eb480a5464a12..6ceb1fc5c9d6c 100644 --- a/src/python/grpcio_testing/setup.py +++ b/src/python/grpcio_testing/setup.py @@ -29,7 +29,7 @@ } INSTALL_REQUIRES = ( - 'protobuf>=3.5.2.post1', + 'protobuf>=3.6.0', 'grpcio>={version}'.format(version=grpc_version.VERSION), ) diff --git a/src/python/grpcio_tests/setup.py b/src/python/grpcio_tests/setup.py index 1262e485715dc..a94c0963ec140 100644 --- a/src/python/grpcio_tests/setup.py +++ b/src/python/grpcio_tests/setup.py @@ -41,8 +41,8 @@ 'grpcio>={version}'.format(version=grpc_version.VERSION), 'grpcio-tools>={version}'.format(version=grpc_version.VERSION), 'grpcio-health-checking>={version}'.format(version=grpc_version.VERSION), - 'oauth2client>=1.4.7', 'protobuf>=3.5.2.post1', 'six>=1.10', - 'google-auth>=1.0.0', 'requests>=2.14.2') + 'oauth2client>=1.4.7', 'protobuf>=3.6.0', 'six>=1.10', 'google-auth>=1.0.0', + 'requests>=2.14.2') if not PY3: INSTALL_REQUIRES += ('futures>=2.2.0',) diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.c b/src/ruby/ext/grpc/rb_grpc_imports.generated.c index 78090afd6c471..77f8ecf0b24f5 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.c +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.c @@ -65,6 +65,7 @@ grpc_census_call_set_context_type grpc_census_call_set_context_import; grpc_census_call_get_context_type grpc_census_call_get_context_import; grpc_channel_get_target_type grpc_channel_get_target_import; grpc_channel_get_info_type grpc_channel_get_info_import; +grpc_channel_reset_connect_backoff_type grpc_channel_reset_connect_backoff_import; grpc_insecure_channel_create_type grpc_insecure_channel_create_import; grpc_lame_client_channel_create_type grpc_lame_client_channel_create_import; grpc_channel_destroy_type grpc_channel_destroy_import; @@ -316,6 +317,7 @@ void grpc_rb_load_imports(HMODULE library) { grpc_census_call_get_context_import = (grpc_census_call_get_context_type) GetProcAddress(library, "grpc_census_call_get_context"); grpc_channel_get_target_import = (grpc_channel_get_target_type) GetProcAddress(library, "grpc_channel_get_target"); grpc_channel_get_info_import = (grpc_channel_get_info_type) GetProcAddress(library, "grpc_channel_get_info"); + grpc_channel_reset_connect_backoff_import = (grpc_channel_reset_connect_backoff_type) GetProcAddress(library, "grpc_channel_reset_connect_backoff"); grpc_insecure_channel_create_import = (grpc_insecure_channel_create_type) GetProcAddress(library, "grpc_insecure_channel_create"); grpc_lame_client_channel_create_import = (grpc_lame_client_channel_create_type) GetProcAddress(library, "grpc_lame_client_channel_create"); grpc_channel_destroy_import = (grpc_channel_destroy_type) GetProcAddress(library, "grpc_channel_destroy"); diff --git a/src/ruby/ext/grpc/rb_grpc_imports.generated.h b/src/ruby/ext/grpc/rb_grpc_imports.generated.h index 1807efa761e6a..0cb84c763c0c0 100644 --- a/src/ruby/ext/grpc/rb_grpc_imports.generated.h +++ b/src/ruby/ext/grpc/rb_grpc_imports.generated.h @@ -170,6 +170,9 @@ extern grpc_channel_get_target_type grpc_channel_get_target_import; typedef void(*grpc_channel_get_info_type)(grpc_channel* channel, const grpc_channel_info* channel_info); extern grpc_channel_get_info_type grpc_channel_get_info_import; #define grpc_channel_get_info grpc_channel_get_info_import +typedef void(*grpc_channel_reset_connect_backoff_type)(grpc_channel* channel); +extern grpc_channel_reset_connect_backoff_type grpc_channel_reset_connect_backoff_import; +#define grpc_channel_reset_connect_backoff grpc_channel_reset_connect_backoff_import typedef grpc_channel*(*grpc_insecure_channel_create_type)(const char* target, const grpc_channel_args* args, void* reserved); extern grpc_insecure_channel_create_type grpc_insecure_channel_create_import; #define grpc_insecure_channel_create grpc_insecure_channel_create_import diff --git a/templates/CMakeLists.txt.template b/templates/CMakeLists.txt.template index de13d02e2a399..2a514ed7b68ee 100644 --- a/templates/CMakeLists.txt.template +++ b/templates/CMakeLists.txt.template @@ -149,6 +149,8 @@ add_definitions(-D_WIN32_WINNT=0x600 -D_SCL_SECURE_NO_WARNINGS -D_CRT_SECURE_NO_WARNINGS -D_WINSOCK_DEPRECATED_NO_WARNINGS) # needed to compile protobuf add_definitions(/wd4065 /wd4506) + # TODO(jtattermusch): revisit warnings that were silenced as part of upgrade to protobuf3.6.0 + add_definitions(/wd4200 /wd4291 /wd4244) # TODO(jtattermusch): revisit C4267 occurrences throughout the code add_definitions(/wd4267) # TODO(jtattermusch): needed to build boringssl with VS2017, revisit later diff --git a/templates/src/csharp/Grpc.Core/Version.csproj.include.template b/templates/src/csharp/Grpc.Core/Version.csproj.include.template index 398b198dbd107..a950b1615d248 100755 --- a/templates/src/csharp/Grpc.Core/Version.csproj.include.template +++ b/templates/src/csharp/Grpc.Core/Version.csproj.include.template @@ -4,6 +4,6 @@ ${settings.csharp_version} - 3.5.1 + 3.6.0 diff --git a/templates/src/csharp/build_unitypackage.bat.template b/templates/src/csharp/build_unitypackage.bat.template index a53e385f9b148..76ec10dbd9069 100755 --- a/templates/src/csharp/build_unitypackage.bat.template +++ b/templates/src/csharp/build_unitypackage.bat.template @@ -70,7 +70,9 @@ copy /Y Grpc.HealthCheck\bin\Release\net45\Google.Protobuf.dll unitypackage\unitypackage_skeleton\Plugins\Google.Protobuf\lib\net45\Google.Protobuf.dll || goto :error @rem create a zipfile that will act as a Unity package - powershell -Command "Add-Type -Assembly 'System.IO.Compression.FileSystem'; [System.IO.Compression.ZipFile]::CreateFromDirectory('unitypackage\unitypackage_skeleton', 'grpc_unity_package.zip');" + cd unitypackage\unitypackage_skeleton + zip -r ..\..\grpc_unity_package.zip Plugins + cd ..\.. copy /Y grpc_unity_package.zip ..\..\artifacts\grpc_unity_package.%VERSION%.zip || goto :error goto :EOF diff --git a/templates/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec.template b/templates/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec.template index 5c1358f7c37f5..30b6c5684cc9c 100644 --- a/templates/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec.template +++ b/templates/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec.template @@ -103,7 +103,7 @@ s.preserve_paths = plugin # Restrict the protoc version to the one supported by this plugin. - s.dependency '!ProtoCompiler', '3.5.0' + s.dependency '!ProtoCompiler', '3.6.0' # For the Protobuf dependency not to complain: s.ios.deployment_target = '7.0' s.osx.deployment_target = '10.9' diff --git a/templates/test/cpp/naming/resolver_component_tests_defs.include b/templates/test/cpp/naming/resolver_component_tests_defs.include index bc981dc83e154..b34845e01a38c 100644 --- a/templates/test/cpp/naming/resolver_component_tests_defs.include +++ b/templates/test/cpp/naming/resolver_component_tests_defs.include @@ -22,6 +22,7 @@ import tempfile import os import time import signal +import platform argp = argparse.ArgumentParser(description='Run c-ares resolver tests') @@ -43,6 +44,11 @@ args = argp.parse_args() def test_runner_log(msg): sys.stderr.write('\n%s: %s\n' % (__file__, msg)) +def python_args(arg_list): + if platform.system() == 'Windows': + return [sys.executable] + arg_list + return arg_list + cur_resolver = os.environ.get('GRPC_DNS_RESOLVER') if cur_resolver and cur_resolver != 'ares': test_runner_log(('WARNING: cur resolver set to %s. This set of tests ' @@ -50,26 +56,27 @@ if cur_resolver and cur_resolver != 'ares': test_runner_log('Exit 1 without running tests.') sys.exit(1) os.environ.update({'GRPC_DNS_RESOLVER': 'ares'}) +os.environ.update({'GRPC_TRACE': 'cares_resolver'}) def wait_until_dns_server_is_up(args, dns_server_subprocess, dns_server_subprocess_output): for i in range(0, 30): test_runner_log('Health check: attempt to connect to DNS server over TCP.') - tcp_connect_subprocess = subprocess.Popen([ + tcp_connect_subprocess = subprocess.Popen(python_args([ args.tcp_connect_bin_path, '--server_host', '127.0.0.1', '--server_port', str(args.dns_server_port), - '--timeout', str(1)]) + '--timeout', str(1)])) tcp_connect_subprocess.communicate() if tcp_connect_subprocess.returncode == 0: test_runner_log(('Health check: attempt to make an A-record ' 'query to DNS server.')) - dns_resolver_subprocess = subprocess.Popen([ + dns_resolver_subprocess = subprocess.Popen(python_args([ args.dns_resolver_bin_path, '--qname', 'health-check-local-dns-server-is-alive.resolver-tests.grpctestingexp', '--server_host', '127.0.0.1', - '--server_port', str(args.dns_server_port)], + '--server_port', str(args.dns_server_port)]), stdout=subprocess.PIPE) dns_resolver_stdout, _ = dns_resolver_subprocess.communicate() if dns_resolver_subprocess.returncode == 0: @@ -91,10 +98,10 @@ def wait_until_dns_server_is_up(args, dns_server_subprocess_output = tempfile.mktemp() with open(dns_server_subprocess_output, 'w') as l: - dns_server_subprocess = subprocess.Popen([ + dns_server_subprocess = subprocess.Popen(python_args([ args.dns_server_bin_path, '--port', str(args.dns_server_port), - '--records_config_path', args.records_config_path], + '--records_config_path', args.records_config_path]), stdin=subprocess.PIPE, stdout=l, stderr=l) diff --git a/test/core/channel/minimal_stack_is_minimal_test.cc b/test/core/channel/minimal_stack_is_minimal_test.cc index e5953acedcf2b..5b651ed39b721 100644 --- a/test/core/channel/minimal_stack_is_minimal_test.cc +++ b/test/core/channel/minimal_stack_is_minimal_test.cc @@ -85,21 +85,21 @@ int main(int argc, char** argv) { // tests with a default stack errors += - CHECK_STACK("unknown", nullptr, GRPC_CLIENT_DIRECT_CHANNEL, "authority", - "message_size", "deadline", "connected", NULL); + CHECK_STACK("unknown", nullptr, GRPC_CLIENT_DIRECT_CHANNEL, "deadline", + "authority", "message_size", "connected", NULL); errors += CHECK_STACK("unknown", nullptr, GRPC_CLIENT_SUBCHANNEL, "authority", "message_size", "connected", NULL); errors += CHECK_STACK("unknown", nullptr, GRPC_SERVER_CHANNEL, "server", - "message_size", "deadline", "connected", NULL); + "deadline", "message_size", "connected", NULL); errors += CHECK_STACK("chttp2", nullptr, GRPC_CLIENT_DIRECT_CHANNEL, - "authority", "message_size", "deadline", "http-client", - "message_compress", "connected", NULL); + "deadline", "authority", "message_size", + "message_compress", "http-client", "connected", NULL); errors += CHECK_STACK("chttp2", nullptr, GRPC_CLIENT_SUBCHANNEL, "authority", - "message_size", "http-client", "message_compress", + "message_size", "message_compress", "http-client", "connected", NULL); errors += CHECK_STACK("chttp2", nullptr, GRPC_SERVER_CHANNEL, "server", - "message_size", "deadline", "http-server", - "message_compress", "connected", NULL); + "deadline", "message_size", "message_compress", + "http-server", "connected", NULL); errors += CHECK_STACK(nullptr, nullptr, GRPC_CLIENT_CHANNEL, "client-channel", NULL); diff --git a/test/core/end2end/tests/filter_call_init_fails.cc b/test/core/end2end/tests/filter_call_init_fails.cc index ab96879fe44b5..07e14214461d6 100644 --- a/test/core/end2end/tests/filter_call_init_fails.cc +++ b/test/core/end2end/tests/filter_call_init_fails.cc @@ -438,7 +438,6 @@ static bool maybe_add_server_channel_filter(grpc_channel_stack_builder* builder, // must be the last one. So we add it right before the last one. grpc_channel_stack_builder_iterator* it = grpc_channel_stack_builder_create_iterator_at_last(builder); - GPR_ASSERT(grpc_channel_stack_builder_move_prev(it)); const bool retval = grpc_channel_stack_builder_add_filter_before( it, &test_filter, nullptr, nullptr); grpc_channel_stack_builder_iterator_destroy(it); @@ -457,7 +456,6 @@ static bool maybe_add_client_channel_filter(grpc_channel_stack_builder* builder, // must be the last one. So we add it right before the last one. grpc_channel_stack_builder_iterator* it = grpc_channel_stack_builder_create_iterator_at_last(builder); - GPR_ASSERT(grpc_channel_stack_builder_move_prev(it)); const bool retval = grpc_channel_stack_builder_add_filter_before( it, &test_filter, nullptr, nullptr); grpc_channel_stack_builder_iterator_destroy(it); @@ -476,7 +474,6 @@ static bool maybe_add_client_subchannel_filter( // must be the last one. So we add it right before the last one. grpc_channel_stack_builder_iterator* it = grpc_channel_stack_builder_create_iterator_at_last(builder); - GPR_ASSERT(grpc_channel_stack_builder_move_prev(it)); const bool retval = grpc_channel_stack_builder_add_filter_before( it, &test_filter, nullptr, nullptr); grpc_channel_stack_builder_iterator_destroy(it); @@ -487,13 +484,17 @@ static bool maybe_add_client_subchannel_filter( } static void init_plugin(void) { - grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, INT_MAX, + grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, + GRPC_CHANNEL_INIT_PRIORITY_MAX, maybe_add_server_channel_filter, nullptr); - grpc_channel_init_register_stage(GRPC_CLIENT_CHANNEL, INT_MAX, + grpc_channel_init_register_stage(GRPC_CLIENT_CHANNEL, + GRPC_CHANNEL_INIT_PRIORITY_MAX, maybe_add_client_channel_filter, nullptr); - grpc_channel_init_register_stage(GRPC_CLIENT_SUBCHANNEL, INT_MAX, + grpc_channel_init_register_stage(GRPC_CLIENT_SUBCHANNEL, + GRPC_CHANNEL_INIT_PRIORITY_MAX, maybe_add_client_subchannel_filter, nullptr); - grpc_channel_init_register_stage(GRPC_CLIENT_DIRECT_CHANNEL, INT_MAX, + grpc_channel_init_register_stage(GRPC_CLIENT_DIRECT_CHANNEL, + GRPC_CHANNEL_INIT_PRIORITY_MAX, maybe_add_client_channel_filter, nullptr); } diff --git a/test/core/end2end/tests/filter_causes_close.cc b/test/core/end2end/tests/filter_causes_close.cc index a7f4268803f60..891c1b8c1fb26 100644 --- a/test/core/end2end/tests/filter_causes_close.cc +++ b/test/core/end2end/tests/filter_causes_close.cc @@ -261,8 +261,9 @@ static bool maybe_add_filter(grpc_channel_stack_builder* builder, void* arg) { } static void init_plugin(void) { - grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, 0, maybe_add_filter, - nullptr); + grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, + GRPC_CHANNEL_INIT_PRIORITY_HIGH, + maybe_add_filter, nullptr); } static void destroy_plugin(void) {} diff --git a/test/core/end2end/tests/filter_latency.cc b/test/core/end2end/tests/filter_latency.cc index a89db7b094ba4..02a4d0792779b 100644 --- a/test/core/end2end/tests/filter_latency.cc +++ b/test/core/end2end/tests/filter_latency.cc @@ -314,7 +314,6 @@ static bool maybe_add_filter(grpc_channel_stack_builder* builder, void* arg) { // must be the last one. So we add it right before the last one. grpc_channel_stack_builder_iterator* it = grpc_channel_stack_builder_create_iterator_at_last(builder); - GPR_ASSERT(grpc_channel_stack_builder_move_prev(it)); const bool retval = grpc_channel_stack_builder_add_filter_before( it, filter, nullptr, nullptr); grpc_channel_stack_builder_iterator_destroy(it); @@ -326,15 +325,15 @@ static bool maybe_add_filter(grpc_channel_stack_builder* builder, void* arg) { static void init_plugin(void) { gpr_mu_init(&g_mu); - grpc_channel_init_register_stage(GRPC_CLIENT_CHANNEL, INT_MAX, - maybe_add_filter, - (void*)&test_client_filter); - grpc_channel_init_register_stage(GRPC_CLIENT_DIRECT_CHANNEL, INT_MAX, - maybe_add_filter, - (void*)&test_client_filter); - grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, INT_MAX, - maybe_add_filter, - (void*)&test_server_filter); + grpc_channel_init_register_stage( + GRPC_CLIENT_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_MAX, maybe_add_filter, + (void*)&test_client_filter); + grpc_channel_init_register_stage( + GRPC_CLIENT_DIRECT_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_MAX, + maybe_add_filter, (void*)&test_client_filter); + grpc_channel_init_register_stage( + GRPC_SERVER_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_MAX, maybe_add_filter, + (void*)&test_server_filter); } static void destroy_plugin(void) { gpr_mu_destroy(&g_mu); } diff --git a/test/core/end2end/tests/filter_status_code.cc b/test/core/end2end/tests/filter_status_code.cc index ba3cbfa6d11d3..6ed1de15c62e7 100644 --- a/test/core/end2end/tests/filter_status_code.cc +++ b/test/core/end2end/tests/filter_status_code.cc @@ -333,7 +333,6 @@ static bool maybe_add_filter(grpc_channel_stack_builder* builder, void* arg) { // So we add it right before the last one. grpc_channel_stack_builder_iterator* it = grpc_channel_stack_builder_create_iterator_at_last(builder); - GPR_ASSERT(grpc_channel_stack_builder_move_prev(it)); const bool retval = grpc_channel_stack_builder_add_filter_before( it, filter, nullptr, nullptr); grpc_channel_stack_builder_iterator_destroy(it); @@ -350,15 +349,15 @@ static void init_plugin(void) { g_client_code_recv = false; g_server_code_recv = false; - grpc_channel_init_register_stage(GRPC_CLIENT_CHANNEL, INT_MAX, - maybe_add_filter, - (void*)&test_client_filter); - grpc_channel_init_register_stage(GRPC_CLIENT_DIRECT_CHANNEL, INT_MAX, - maybe_add_filter, - (void*)&test_client_filter); - grpc_channel_init_register_stage(GRPC_SERVER_CHANNEL, INT_MAX, - maybe_add_filter, - (void*)&test_server_filter); + grpc_channel_init_register_stage( + GRPC_CLIENT_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_MAX, maybe_add_filter, + (void*)&test_client_filter); + grpc_channel_init_register_stage( + GRPC_CLIENT_DIRECT_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_MAX, + maybe_add_filter, (void*)&test_client_filter); + grpc_channel_init_register_stage( + GRPC_SERVER_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_MAX, maybe_add_filter, + (void*)&test_server_filter); } static void destroy_plugin(void) { diff --git a/test/core/surface/public_headers_must_be_c89.c b/test/core/surface/public_headers_must_be_c89.c index 497f7194d57fe..ae9691435a847 100644 --- a/test/core/surface/public_headers_must_be_c89.c +++ b/test/core/surface/public_headers_must_be_c89.c @@ -104,6 +104,7 @@ int main(int argc, char **argv) { printf("%lx", (unsigned long) grpc_census_call_get_context); printf("%lx", (unsigned long) grpc_channel_get_target); printf("%lx", (unsigned long) grpc_channel_get_info); + printf("%lx", (unsigned long) grpc_channel_reset_connect_backoff); printf("%lx", (unsigned long) grpc_insecure_channel_create); printf("%lx", (unsigned long) grpc_lame_client_channel_create); printf("%lx", (unsigned long) grpc_channel_destroy); diff --git a/test/cpp/common/channel_filter_test.cc b/test/cpp/common/channel_filter_test.cc index 7bdd53f9e7ab6..9b603ca5b4840 100644 --- a/test/cpp/common/channel_filter_test.cc +++ b/test/cpp/common/channel_filter_test.cc @@ -50,7 +50,8 @@ class MyCallData : public CallData { // C-core, we don't accidentally break the C++ filter API. TEST(ChannelFilterTest, RegisterChannelFilter) { grpc::RegisterChannelFilter( - "myfilter", GRPC_CLIENT_CHANNEL, INT_MAX, nullptr); + "myfilter", GRPC_CLIENT_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_LOW, true, + nullptr); } // TODO(roth): When we have time, add tests for all methods of the diff --git a/test/cpp/end2end/async_end2end_test.cc b/test/cpp/end2end/async_end2end_test.cc index 3d31c9d81006c..c9246f0806872 100644 --- a/test/cpp/end2end/async_end2end_test.cc +++ b/test/cpp/end2end/async_end2end_test.cc @@ -1709,7 +1709,7 @@ TEST_P(AsyncEnd2endServerTryCancelTest, ServerBidiStreamingTryCancelAfter) { } std::vector CreateTestScenarios(bool test_secure, - int test_big_limit) { + bool test_message_size_limit) { std::vector scenarios; std::vector credentials_types; std::vector messages; @@ -1731,13 +1731,18 @@ std::vector CreateTestScenarios(bool test_secure, GPR_ASSERT(!credentials_types.empty()); messages.push_back("Hello"); - for (int sz = 1; sz <= test_big_limit; sz *= 32) { - grpc::string big_msg; - for (int i = 0; i < sz * 1024; i++) { - char c = 'a' + (i % 26); - big_msg += c; + if (test_message_size_limit) { + for (size_t k = 1; k < GRPC_DEFAULT_MAX_RECV_MESSAGE_LENGTH / 1024; + k *= 32) { + grpc::string big_msg; + for (size_t i = 0; i < k * 1024; ++i) { + char c = 'a' + (i % 26); + big_msg += c; + } + messages.push_back(big_msg); } - messages.push_back(big_msg); + messages.push_back( + grpc::string(GRPC_DEFAULT_MAX_RECV_MESSAGE_LENGTH - 10, 'a')); } // TODO (sreek) Renable tests with health check service after the issue @@ -1758,10 +1763,10 @@ std::vector CreateTestScenarios(bool test_secure, } INSTANTIATE_TEST_CASE_P(AsyncEnd2end, AsyncEnd2endTest, - ::testing::ValuesIn(CreateTestScenarios(true, 1024))); + ::testing::ValuesIn(CreateTestScenarios(true, true))); INSTANTIATE_TEST_CASE_P(AsyncEnd2endServerTryCancel, AsyncEnd2endServerTryCancelTest, - ::testing::ValuesIn(CreateTestScenarios(false, 0))); + ::testing::ValuesIn(CreateTestScenarios(false, false))); } // namespace } // namespace testing diff --git a/test/cpp/end2end/client_lb_end2end_test.cc b/test/cpp/end2end/client_lb_end2end_test.cc index c5a73a246986f..7fe0da8aaec04 100644 --- a/test/cpp/end2end/client_lb_end2end_test.cc +++ b/test/cpp/end2end/client_lb_end2end_test.cc @@ -408,6 +408,36 @@ TEST_F(ClientLbEnd2endTest, PickFirstBackOffMinReconnect) { gpr_atm_rel_store(&g_connection_delay_ms, 0); } +TEST_F(ClientLbEnd2endTest, PickFirstResetConnectionBackoff) { + ChannelArguments args; + constexpr int kInitialBackOffMs = 1000; + args.SetInt(GRPC_ARG_INITIAL_RECONNECT_BACKOFF_MS, kInitialBackOffMs); + const std::vector ports = {grpc_pick_unused_port_or_die()}; + auto channel = BuildChannel("pick_first", args); + auto stub = BuildStub(channel); + SetNextResolution(ports); + // The channel won't become connected (there's no server). + EXPECT_FALSE( + channel->WaitForConnected(grpc_timeout_milliseconds_to_deadline(10))); + // Bring up a server on the chosen port. + StartServers(1, ports); + const gpr_timespec t0 = gpr_now(GPR_CLOCK_MONOTONIC); + // Wait for connect, but not long enough. This proves that we're + // being throttled by initial backoff. + EXPECT_FALSE( + channel->WaitForConnected(grpc_timeout_milliseconds_to_deadline(10))); + // Reset connection backoff. + experimental::ChannelResetConnectionBackoff(channel.get()); + // Wait for connect. Should happen ~immediately. + EXPECT_TRUE( + channel->WaitForConnected(grpc_timeout_milliseconds_to_deadline(10))); + const gpr_timespec t1 = gpr_now(GPR_CLOCK_MONOTONIC); + const grpc_millis waited_ms = gpr_time_to_millis(gpr_time_sub(t1, t0)); + gpr_log(GPR_DEBUG, "Waited %" PRId64 " milliseconds", waited_ms); + // We should have waited less than kInitialBackOffMs. + EXPECT_LT(waited_ms, kInitialBackOffMs); +} + TEST_F(ClientLbEnd2endTest, PickFirstUpdates) { // Start servers and send one RPC per server. const int kNumServers = 3; diff --git a/test/cpp/end2end/filter_end2end_test.cc b/test/cpp/end2end/filter_end2end_test.cc index 88f8f380c3a75..a8022823b1015 100644 --- a/test/cpp/end2end/filter_end2end_test.cc +++ b/test/cpp/end2end/filter_end2end_test.cc @@ -323,7 +323,8 @@ TEST_F(FilterEnd2endTest, SimpleBidiStreaming) { void RegisterFilter() { grpc::RegisterChannelFilter( - "test-filter", GRPC_SERVER_CHANNEL, INT_MAX, nullptr); + "test-filter", GRPC_SERVER_CHANNEL, GRPC_CHANNEL_INIT_PRIORITY_LOW, true, + nullptr); } } // namespace diff --git a/test/cpp/naming/cancel_ares_query_test.cc b/test/cpp/naming/cancel_ares_query_test.cc index 0d59bf6fb60d1..dec7c171dc0e0 100644 --- a/test/cpp/naming/cancel_ares_query_test.cc +++ b/test/cpp/naming/cancel_ares_query_test.cc @@ -45,11 +45,14 @@ #include "test/core/util/port.h" #include "test/core/util/test_config.h" -// TODO: pull in different headers when enabling this -// test on windows. Also set BAD_SOCKET_RETURN_VAL -// to INVALID_SOCKET on windows. +#ifdef GPR_WINDOWS +#include "src/core/lib/iomgr/sockaddr_windows.h" +#include "src/core/lib/iomgr/socket_windows.h" +#define BAD_SOCKET_RETURN_VAL INVALID_SOCKET +#else #include "src/core/lib/iomgr/sockaddr_posix.h" #define BAD_SOCKET_RETURN_VAL -1 +#endif namespace { @@ -91,7 +94,13 @@ class FakeNonResponsiveDNSServer { abort(); } } - ~FakeNonResponsiveDNSServer() { close(socket_); } + ~FakeNonResponsiveDNSServer() { +#ifdef GPR_WINDOWS + closesocket(socket_); +#else + close(socket_); +#endif + } private: int socket_; @@ -193,6 +202,38 @@ TEST(CancelDuringAresQuery, TestCancelActiveDNSQuery) { TestCancelActiveDNSQuery(&args); } +#ifdef GPR_WINDOWS + +void MaybePollArbitraryPollsetTwice() { + grpc_pollset* pollset = (grpc_pollset*)gpr_zalloc(grpc_pollset_size()); + gpr_mu* mu; + grpc_pollset_init(pollset, &mu); + grpc_pollset_worker* worker = nullptr; + // Make a zero timeout poll + gpr_mu_lock(mu); + GRPC_LOG_IF_ERROR( + "pollset_work", + grpc_pollset_work(pollset, &worker, grpc_core::ExecCtx::Get()->Now())); + gpr_mu_unlock(mu); + grpc_core::ExecCtx::Get()->Flush(); + // Make a second zero-timeout poll (in case the first one + // short-circuited by picking up a previous "kick") + gpr_mu_lock(mu); + GRPC_LOG_IF_ERROR( + "pollset_work", + grpc_pollset_work(pollset, &worker, grpc_core::ExecCtx::Get()->Now())); + gpr_mu_unlock(mu); + grpc_core::ExecCtx::Get()->Flush(); + grpc_pollset_destroy(pollset); + gpr_free(pollset); +} + +#else + +void MaybePollArbitraryPollsetTwice() {} + +#endif + TEST(CancelDuringAresQuery, TestFdsAreDeletedFromPollsetSet) { grpc_core::ExecCtx exec_ctx; ArgsStruct args; @@ -209,6 +250,12 @@ TEST(CancelDuringAresQuery, TestFdsAreDeletedFromPollsetSet) { // this test. This test only cares about what happens to fd's that c-ares // opens. TestCancelActiveDNSQuery(&args); + // This test relies on the assumption that cancelling a c-ares query + // will flush out all callbacks on the current exec ctx, which is true + // on posix platforms but not on Windows, because fd shutdown on Windows + // requires a trip through the polling loop to schedule the callback. + // So we need to do extra polling work on Windows to free things up. + MaybePollArbitraryPollsetTwice(); EXPECT_EQ(grpc_iomgr_count_objects_for_testing(), 0u); grpc_pollset_set_destroy(fake_other_pollset_set); } diff --git a/test/cpp/naming/gen_build_yaml.py b/test/cpp/naming/gen_build_yaml.py index 5dad2ea7af6c5..1c9d0676b894b 100755 --- a/test/cpp/naming/gen_build_yaml.py +++ b/test/cpp/naming/gen_build_yaml.py @@ -68,7 +68,7 @@ def main(): 'gtest': False, 'run': False, 'src': ['test/cpp/naming/resolver_component_test.cc'], - 'platforms': ['linux', 'posix', 'mac'], + 'platforms': ['linux', 'posix', 'mac', 'windows'], 'deps': [ 'grpc++_test_util' + unsecure_build_config_suffix, 'grpc_test_util' + unsecure_build_config_suffix, @@ -129,7 +129,7 @@ def main(): 'gtest': True, 'run': True, 'src': ['test/cpp/naming/cancel_ares_query_test.cc'], - 'platforms': ['linux', 'posix', 'mac'], + 'platforms': ['linux', 'posix', 'mac', 'windows'], 'deps': [ 'grpc++_test_util', 'grpc_test_util', diff --git a/test/cpp/naming/manual_run_resolver_component_test.py b/test/cpp/naming/manual_run_resolver_component_test.py new file mode 100644 index 0000000000000..fb2157741a091 --- /dev/null +++ b/test/cpp/naming/manual_run_resolver_component_test.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python +# Copyright 2015 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import subprocess +import sys + +# The c-ares test suite doesn't get ran regularly on Windows, but +# this script provides a way to run a lot of the tests manually. +_MSBUILD_CONFIG = os.environ['CONFIG'] +os.chdir(os.path.join('..', '..', os.getcwd())) +# This port is arbitrary, but it needs to be available. +_DNS_SERVER_PORT = 15353 + +subprocess.call([ + sys.executable, + 'test\\cpp\\naming\\resolver_component_tests_runner.py', + '--test_bin_path', 'cmake\\build\\%s\\resolver_component_test.exe' % _MSBUILD_CONFIG, + '--dns_server_bin_path', 'test\\cpp\\naming\\utils\\dns_server.py', + '--records_config_path', 'test\\cpp\\naming\\resolver_test_record_groups.yaml', + '--dns_server_port', str(_DNS_SERVER_PORT), + '--dns_resolver_bin_path', 'test\\cpp\\naming\\utils\\dns_resolver.py', + '--tcp_connect_bin_path', 'test\\cpp\\naming\\utils\\tcp_connect.py', +]) diff --git a/test/cpp/naming/resolver_component_test.cc b/test/cpp/naming/resolver_component_test.cc index 6ac548120ca44..3dc6e7178cc34 100644 --- a/test/cpp/naming/resolver_component_test.cc +++ b/test/cpp/naming/resolver_component_test.cc @@ -16,6 +16,8 @@ * */ +#include + #include #include #include @@ -55,8 +57,15 @@ // TODO: pull in different headers when enabling this // test on windows. Also set BAD_SOCKET_RETURN_VAL // to INVALID_SOCKET on windows. +#ifdef GPR_WINDOWS +#include "src/core/lib/iomgr/sockaddr_windows.h" +#include "src/core/lib/iomgr/socket_windows.h" +#include "src/core/lib/iomgr/tcp_windows.h" +#define BAD_SOCKET_RETURN_VAL INVALID_SOCKET +#else #include "src/core/lib/iomgr/sockaddr_posix.h" #define BAD_SOCKET_RETURN_VAL -1 +#endif using grpc::SubProcess; using std::vector; @@ -241,6 +250,62 @@ void CheckLBPolicyResultLocked(grpc_channel_args* channel_args, } } +#ifdef GPR_WINDOWS +void OpenAndCloseSocketsStressLoop(int dummy_port, gpr_event* done_ev) { + sockaddr_in6 addr; + memset(&addr, 0, sizeof(addr)); + addr.sin6_family = AF_INET6; + addr.sin6_port = htons(dummy_port); + ((char*)&addr.sin6_addr)[15] = 1; + for (;;) { + if (gpr_event_get(done_ev)) { + return; + } + std::vector sockets; + for (size_t i = 0; i < 50; i++) { + SOCKET s = WSASocket(AF_INET6, SOCK_STREAM, IPPROTO_TCP, nullptr, 0, + WSA_FLAG_OVERLAPPED); + ASSERT_TRUE(s != BAD_SOCKET_RETURN_VAL) + << "Failed to create TCP ipv6 socket"; + gpr_log(GPR_DEBUG, "Opened socket: %d", s); + char val = 1; + ASSERT_TRUE(setsockopt(s, SOL_SOCKET, SO_REUSEADDR, &val, sizeof(val)) != + SOCKET_ERROR) + << "Failed to set socketopt reuseaddr. WSA error: " + + std::to_string(WSAGetLastError()); + ASSERT_TRUE(grpc_tcp_set_non_block(s) == GRPC_ERROR_NONE) + << "Failed to set socket non-blocking"; + ASSERT_TRUE(bind(s, (const sockaddr*)&addr, sizeof(addr)) != SOCKET_ERROR) + << "Failed to bind socket " + std::to_string(s) + + " to [::1]:" + std::to_string(dummy_port) + + ". WSA error: " + std::to_string(WSAGetLastError()); + ASSERT_TRUE(listen(s, 1) != SOCKET_ERROR) + << "Failed to listen on socket " + std::to_string(s) + + ". WSA error: " + std::to_string(WSAGetLastError()); + sockets.push_back(s); + } + // Do a non-blocking accept followed by a close on all of those sockets. + // Do this in a separate loop to try to induce a time window to hit races. + for (size_t i = 0; i < sockets.size(); i++) { + gpr_log(GPR_DEBUG, "non-blocking accept then close on %d", sockets[i]); + ASSERT_TRUE(accept(sockets[i], nullptr, nullptr) == INVALID_SOCKET) + << "Accept on dummy socket unexpectedly accepted actual connection."; + ASSERT_TRUE(WSAGetLastError() == WSAEWOULDBLOCK) + << "OpenAndCloseSocketsStressLoop accept on socket " + + std::to_string(sockets[i]) + + " failed in " + "an unexpected way. " + "WSA error: " + + std::to_string(WSAGetLastError()) + + ". Socket use-after-close bugs are likely."; + ASSERT_TRUE(closesocket(sockets[i]) != SOCKET_ERROR) + << "Failed to close socket: " + std::to_string(sockets[i]) + + ". WSA error: " + std::to_string(WSAGetLastError()); + } + } + return; +} +#else void OpenAndCloseSocketsStressLoop(int dummy_port, gpr_event* done_ev) { // The goal of this loop is to catch socket // "use after close" bugs within the c-ares resolver by acting @@ -311,6 +376,7 @@ void OpenAndCloseSocketsStressLoop(int dummy_port, gpr_event* done_ev) { } } } +#endif void CheckResolverResultLocked(void* argsp, grpc_error* err) { EXPECT_EQ(err, GRPC_ERROR_NONE); @@ -372,9 +438,9 @@ void RunResolvesRelevantRecordsTest(void (*OnDoneLocked)(void* arg, args.expected_lb_policy = FLAGS_expected_lb_policy; // maybe build the address with an authority char* whole_uri = nullptr; - GPR_ASSERT(asprintf(&whole_uri, "dns://%s/%s", - FLAGS_local_dns_server_address.c_str(), - FLAGS_target_name.c_str())); + GPR_ASSERT(gpr_asprintf(&whole_uri, "dns://%s/%s", + FLAGS_local_dns_server_address.c_str(), + FLAGS_target_name.c_str())); // create resolver and resolve grpc_core::OrphanablePtr resolver = grpc_core::ResolverRegistry::CreateResolver(whole_uri, nullptr, diff --git a/test/cpp/naming/resolver_component_tests_runner.py b/test/cpp/naming/resolver_component_tests_runner.py index 69386ebeb033c..1873eec35bd80 100755 --- a/test/cpp/naming/resolver_component_tests_runner.py +++ b/test/cpp/naming/resolver_component_tests_runner.py @@ -22,6 +22,7 @@ import os import time import signal +import platform argp = argparse.ArgumentParser(description='Run c-ares resolver tests') @@ -43,6 +44,11 @@ def test_runner_log(msg): sys.stderr.write('\n%s: %s\n' % (__file__, msg)) +def python_args(arg_list): + if platform.system() == 'Windows': + return [sys.executable] + arg_list + return arg_list + cur_resolver = os.environ.get('GRPC_DNS_RESOLVER') if cur_resolver and cur_resolver != 'ares': test_runner_log(('WARNING: cur resolver set to %s. This set of tests ' @@ -50,26 +56,27 @@ def test_runner_log(msg): test_runner_log('Exit 1 without running tests.') sys.exit(1) os.environ.update({'GRPC_DNS_RESOLVER': 'ares'}) +os.environ.update({'GRPC_TRACE': 'cares_resolver'}) def wait_until_dns_server_is_up(args, dns_server_subprocess, dns_server_subprocess_output): for i in range(0, 30): test_runner_log('Health check: attempt to connect to DNS server over TCP.') - tcp_connect_subprocess = subprocess.Popen([ + tcp_connect_subprocess = subprocess.Popen(python_args([ args.tcp_connect_bin_path, '--server_host', '127.0.0.1', '--server_port', str(args.dns_server_port), - '--timeout', str(1)]) + '--timeout', str(1)])) tcp_connect_subprocess.communicate() if tcp_connect_subprocess.returncode == 0: test_runner_log(('Health check: attempt to make an A-record ' 'query to DNS server.')) - dns_resolver_subprocess = subprocess.Popen([ + dns_resolver_subprocess = subprocess.Popen(python_args([ args.dns_resolver_bin_path, '--qname', 'health-check-local-dns-server-is-alive.resolver-tests.grpctestingexp', '--server_host', '127.0.0.1', - '--server_port', str(args.dns_server_port)], + '--server_port', str(args.dns_server_port)]), stdout=subprocess.PIPE) dns_resolver_stdout, _ = dns_resolver_subprocess.communicate() if dns_resolver_subprocess.returncode == 0: @@ -91,10 +98,10 @@ def wait_until_dns_server_is_up(args, dns_server_subprocess_output = tempfile.mktemp() with open(dns_server_subprocess_output, 'w') as l: - dns_server_subprocess = subprocess.Popen([ + dns_server_subprocess = subprocess.Popen(python_args([ args.dns_server_bin_path, '--port', str(args.dns_server_port), - '--records_config_path', args.records_config_path], + '--records_config_path', args.records_config_path]), stdin=subprocess.PIPE, stdout=l, stderr=l) @@ -112,6 +119,18 @@ def _quit_on_signal(signum, _frame): dns_server_subprocess_output) num_test_failures = 0 +test_runner_log('Run test with target: %s' % 'no-srv-ipv4-single-target.resolver-tests-version-4.grpctestingexp.') +current_test_subprocess = subprocess.Popen([ + args.test_bin_path, + '--target_name', 'no-srv-ipv4-single-target.resolver-tests-version-4.grpctestingexp.', + '--expected_addrs', '5.5.5.5:443,False', + '--expected_chosen_service_config', '', + '--expected_lb_policy', '', + '--local_dns_server_address', '127.0.0.1:%d' % args.dns_server_port]) +current_test_subprocess.communicate() +if current_test_subprocess.returncode != 0: + num_test_failures += 1 + test_runner_log('Run test with target: %s' % 'srv-ipv4-single-target.resolver-tests-version-4.grpctestingexp.') current_test_subprocess = subprocess.Popen([ args.test_bin_path, diff --git a/test/cpp/naming/resolver_test_record_groups.yaml b/test/cpp/naming/resolver_test_record_groups.yaml index 6c4f89d09b825..3c51a00c7b1cd 100644 --- a/test/cpp/naming/resolver_test_record_groups.yaml +++ b/test/cpp/naming/resolver_test_record_groups.yaml @@ -1,5 +1,13 @@ resolver_tests_common_zone_name: resolver-tests-version-4.grpctestingexp. resolver_component_tests: +- expected_addrs: + - {address: '5.5.5.5:443', is_balancer: false} + expected_chosen_service_config: null + expected_lb_policy: null + record_to_resolve: no-srv-ipv4-single-target + records: + no-srv-ipv4-single-target: + - {TTL: '2100', data: 5.5.5.5, type: A} - expected_addrs: - {address: '1.2.3.4:1234', is_balancer: true} expected_chosen_service_config: null diff --git a/test/cpp/util/cli_credentials.cc b/test/cpp/util/cli_credentials.cc index d14dc18f168cd..acf4ef8ef103e 100644 --- a/test/cpp/util/cli_credentials.cc +++ b/test/cpp/util/cli_credentials.cc @@ -20,8 +20,12 @@ #include -DEFINE_bool(enable_ssl, false, "Whether to use ssl/tls."); -DEFINE_bool(use_auth, false, "Whether to create default google credentials."); +DEFINE_bool( + enable_ssl, false, + "Whether to use ssl/tls. Deprecated. Use --channel_creds_type=ssl."); +DEFINE_bool(use_auth, false, + "Whether to create default google credentials. Deprecated. Use " + "--channel_creds_type=gdc."); DEFINE_string( access_token, "", "The access token that will be sent to the server to authenticate RPCs."); @@ -29,47 +33,109 @@ DEFINE_string( ssl_target, "", "If not empty, treat the server host name as this for ssl/tls certificate " "validation."); +DEFINE_string( + channel_creds_type, "", + "The channel creds type: insecure, ssl, gdc (Google Default Credentials) " + "or alts."); namespace grpc { namespace testing { -std::shared_ptr CliCredentials::GetCredentials() +grpc::string CliCredentials::GetDefaultChannelCredsType() const { + // Compatibility logic for --enable_ssl. + if (FLAGS_enable_ssl) { + fprintf(stderr, + "warning: --enable_ssl is deprecated. Use " + "--channel_creds_type=ssl.\n"); + return "ssl"; + } + // Compatibility logic for --use_auth. + if (FLAGS_access_token.empty() && FLAGS_use_auth) { + fprintf(stderr, + "warning: --use_auth is deprecated. Use " + "--channel_creds_type=gdc.\n"); + return "gdc"; + } + return "insecure"; +} + +std::shared_ptr +CliCredentials::GetChannelCredentials() const { + if (FLAGS_channel_creds_type.compare("insecure") == 0) { + return grpc::InsecureChannelCredentials(); + } else if (FLAGS_channel_creds_type.compare("ssl") == 0) { + return grpc::SslCredentials(grpc::SslCredentialsOptions()); + } else if (FLAGS_channel_creds_type.compare("gdc") == 0) { + return grpc::GoogleDefaultCredentials(); + } else if (FLAGS_channel_creds_type.compare("alts") == 0) { + return grpc::experimental::AltsCredentials( + grpc::experimental::AltsCredentialsOptions()); + } + fprintf(stderr, + "--channel_creds_type=%s invalid; must be insecure, ssl, gdc or " + "alts.\n", + FLAGS_channel_creds_type.c_str()); + return std::shared_ptr(); +} + +std::shared_ptr CliCredentials::GetCallCredentials() const { if (!FLAGS_access_token.empty()) { if (FLAGS_use_auth) { fprintf(stderr, "warning: use_auth is ignored when access_token is provided."); } - - return grpc::CompositeChannelCredentials( - grpc::SslCredentials(grpc::SslCredentialsOptions()), - grpc::AccessTokenCredentials(FLAGS_access_token)); + return grpc::AccessTokenCredentials(FLAGS_access_token); } + return std::shared_ptr(); +} - if (FLAGS_use_auth) { - return grpc::GoogleDefaultCredentials(); +std::shared_ptr CliCredentials::GetCredentials() + const { + if (FLAGS_channel_creds_type.empty()) { + FLAGS_channel_creds_type = GetDefaultChannelCredsType(); + } else if (FLAGS_enable_ssl && FLAGS_channel_creds_type.compare("ssl") != 0) { + fprintf(stderr, + "warning: ignoring --enable_ssl because " + "--channel_creds_type already set to %s.\n", + FLAGS_channel_creds_type.c_str()); + } else if (FLAGS_use_auth && FLAGS_channel_creds_type.compare("gdc") != 0) { + fprintf(stderr, + "warning: ignoring --use_auth because " + "--channel_creds_type already set to %s.\n", + FLAGS_channel_creds_type.c_str()); } - - if (FLAGS_enable_ssl) { - return grpc::SslCredentials(grpc::SslCredentialsOptions()); + // Legacy transport upgrade logic for insecure requests. + if (!FLAGS_access_token.empty() && + FLAGS_channel_creds_type.compare("insecure") == 0) { + fprintf(stderr, + "warning: --channel_creds_type=insecure upgraded to ssl because " + "an access token was provided.\n"); + FLAGS_channel_creds_type = "ssl"; } - - return grpc::InsecureChannelCredentials(); + std::shared_ptr channel_creds = + GetChannelCredentials(); + // Composite any call-type credentials on top of the base channel. + std::shared_ptr call_creds = GetCallCredentials(); + return (channel_creds == nullptr || call_creds == nullptr) + ? channel_creds + : grpc::CompositeChannelCredentials(channel_creds, call_creds); } const grpc::string CliCredentials::GetCredentialUsage() const { - return " --enable_ssl ; Set whether to use tls\n" + return " --enable_ssl ; Set whether to use ssl (deprecated)\n" " --use_auth ; Set whether to create default google" " credentials\n" " --access_token ; Set the access token in metadata," " overrides --use_auth\n" - " --ssl_target ; Set server host for tls validation\n"; + " --ssl_target ; Set server host for ssl validation\n" + " --channel_creds_type ; Set to insecure, ssl, gdc, or alts\n"; } const grpc::string CliCredentials::GetSslTargetNameOverride() const { - bool use_tls = - FLAGS_enable_ssl || (FLAGS_access_token.empty() && FLAGS_use_auth); - return use_tls ? FLAGS_ssl_target : ""; + bool use_ssl = FLAGS_channel_creds_type.compare("ssl") == 0 || + FLAGS_channel_creds_type.compare("gdc") == 0; + return use_ssl ? FLAGS_ssl_target : ""; } } // namespace testing diff --git a/test/cpp/util/cli_credentials.h b/test/cpp/util/cli_credentials.h index 8d662356de8d6..4636d3ca14912 100644 --- a/test/cpp/util/cli_credentials.h +++ b/test/cpp/util/cli_credentials.h @@ -28,9 +28,22 @@ namespace testing { class CliCredentials { public: virtual ~CliCredentials() {} - virtual std::shared_ptr GetCredentials() const; + std::shared_ptr GetCredentials() const; virtual const grpc::string GetCredentialUsage() const; virtual const grpc::string GetSslTargetNameOverride() const; + + protected: + // Returns the appropriate channel_creds_type value for the set of legacy + // flag arguments. + virtual grpc::string GetDefaultChannelCredsType() const; + // Returns the base transport channel credentials. Child classes can override + // to support additional channel_creds_types unknown to this base class. + virtual std::shared_ptr GetChannelCredentials() + const; + // Returns call credentials to composite onto the base transport channel + // credentials. Child classes can override to support additional + // authentication flags unknown to this base class. + virtual std::shared_ptr GetCallCredentials() const; }; } // namespace testing diff --git a/test/cpp/util/grpc_tool_test.cc b/test/cpp/util/grpc_tool_test.cc index 7e7f44551ef83..3aae090e81882 100644 --- a/test/cpp/util/grpc_tool_test.cc +++ b/test/cpp/util/grpc_tool_test.cc @@ -81,7 +81,7 @@ using grpc::testing::EchoResponse; " peer: \"peer\"\n" \ "}\n\n" -DECLARE_bool(enable_ssl); +DECLARE_string(channel_creds_type); DECLARE_string(ssl_target); namespace grpc { @@ -102,7 +102,8 @@ const int kServerDefaultResponseStreamsToSend = 3; class TestCliCredentials final : public grpc::testing::CliCredentials { public: TestCliCredentials(bool secure = false) : secure_(secure) {} - std::shared_ptr GetCredentials() const override { + std::shared_ptr GetChannelCredentials() + const override { if (!secure_) { return InsecureChannelCredentials(); } @@ -769,12 +770,12 @@ TEST_F(GrpcToolTest, CallCommandWithBadMetadata) { TEST_F(GrpcToolTest, ListCommand_OverrideSslHostName) { const grpc::string server_address = SetUpServer(true); - // Test input "grpc_cli ls localhost: --enable_ssl + // Test input "grpc_cli ls localhost: --channel_creds_type=ssl // --ssl_target=z.test.google.fr" std::stringstream output_stream; const char* argv[] = {"grpc_cli", "ls", server_address.c_str()}; FLAGS_l = false; - FLAGS_enable_ssl = true; + FLAGS_channel_creds_type = "ssl"; FLAGS_ssl_target = "z.test.google.fr"; EXPECT_TRUE( 0 == GrpcToolMainLib( @@ -784,7 +785,7 @@ TEST_F(GrpcToolTest, ListCommand_OverrideSslHostName) { "grpc.testing.EchoTestService\n" "grpc.reflection.v1alpha.ServerReflection\n")); - FLAGS_enable_ssl = false; + FLAGS_channel_creds_type = ""; FLAGS_ssl_target = ""; ShutdownServer(); } diff --git a/test/distrib/cpp/run_distrib_test_cmake.bat b/test/distrib/cpp/run_distrib_test_cmake.bat index 8eb3b201b130c..7658156be1db4 100644 --- a/test/distrib/cpp/run_distrib_test_cmake.bat +++ b/test/distrib/cpp/run_distrib_test_cmake.bat @@ -39,7 +39,7 @@ cd ../../.. cd third_party/protobuf/cmake mkdir build cd build -cmake -DCMAKE_INSTALL_PREFIX=%INSTALL_DIR% -Dprotobuf_MSVC_STATIC_RUNTIME=OFF -Dprotobuf_BUILD_TESTS=OFF .. +cmake -DCMAKE_INSTALL_PREFIX=%INSTALL_DIR% -DZLIB_ROOT=%INSTALL_DIR% -Dprotobuf_MSVC_STATIC_RUNTIME=OFF -Dprotobuf_BUILD_TESTS=OFF .. cmake --build . --config Release --target install || goto :error cd ../../../.. diff --git a/test/distrib/cpp/run_distrib_test_routeguide.sh b/test/distrib/cpp/run_distrib_test_routeguide.sh index dc69ab83773ee..a81692d920cdc 100755 --- a/test/distrib/cpp/run_distrib_test_routeguide.sh +++ b/test/distrib/cpp/run_distrib_test_routeguide.sh @@ -19,7 +19,7 @@ set -ex cd "$(dirname "$0")/../../.." cd third_party/protobuf && ./autogen.sh && \ -./configure && make -j4 && make check && make install && ldconfig +./configure && make -j4 && make install && ldconfig cd ../.. && make -j4 && make install diff --git a/third_party/protobuf b/third_party/protobuf index b5fbb742af122..48cb18e5c419d 160000 --- a/third_party/protobuf +++ b/third_party/protobuf @@ -1 +1 @@ -Subproject commit b5fbb742af122b565925987e65c08957739976a7 +Subproject commit 48cb18e5c419ddd23d9badcfe4e9df7bde1979b2 diff --git a/tools/codegen/core/gen_nano_proto.sh b/tools/codegen/core/gen_nano_proto.sh index 5c781761ba1a4..15495917467f8 100755 --- a/tools/codegen/core/gen_nano_proto.sh +++ b/tools/codegen/core/gen_nano_proto.sh @@ -68,7 +68,7 @@ popd # this should be the same version as the submodule we compile against # ideally we'd update this as a template to ensure that -pip install protobuf==3.5.2 +pip install protobuf==3.6.0 pushd "$(dirname $INPUT_PROTO)" > /dev/null diff --git a/tools/distrib/python/grpcio_tools/protoc_lib_deps.py b/tools/distrib/python/grpcio_tools/protoc_lib_deps.py index a0e1419ac1296..7d10db0329c64 100644 --- a/tools/distrib/python/grpcio_tools/protoc_lib_deps.py +++ b/tools/distrib/python/grpcio_tools/protoc_lib_deps.py @@ -14,10 +14,10 @@ # limitations under the License. # AUTO-GENERATED BY make_grpcio_tools.py! -CC_FILES=['google/protobuf/compiler/zip_writer.cc', 'google/protobuf/compiler/subprocess.cc', 'google/protobuf/compiler/ruby/ruby_generator.cc', 'google/protobuf/compiler/python/python_generator.cc', 'google/protobuf/compiler/plugin.pb.cc', 'google/protobuf/compiler/plugin.cc', 'google/protobuf/compiler/php/php_generator.cc', 'google/protobuf/compiler/objectivec/objectivec_primitive_field.cc', 'google/protobuf/compiler/objectivec/objectivec_oneof.cc', 'google/protobuf/compiler/objectivec/objectivec_message_field.cc', 'google/protobuf/compiler/objectivec/objectivec_message.cc', 'google/protobuf/compiler/objectivec/objectivec_map_field.cc', 'google/protobuf/compiler/objectivec/objectivec_helpers.cc', 'google/protobuf/compiler/objectivec/objectivec_generator.cc', 'google/protobuf/compiler/objectivec/objectivec_file.cc', 'google/protobuf/compiler/objectivec/objectivec_field.cc', 'google/protobuf/compiler/objectivec/objectivec_extension.cc', 'google/protobuf/compiler/objectivec/objectivec_enum_field.cc', 'google/protobuf/compiler/objectivec/objectivec_enum.cc', 'google/protobuf/compiler/js/well_known_types_embed.cc', 'google/protobuf/compiler/js/js_generator.cc', 'google/protobuf/compiler/javanano/javanano_primitive_field.cc', 'google/protobuf/compiler/javanano/javanano_message_field.cc', 'google/protobuf/compiler/javanano/javanano_message.cc', 'google/protobuf/compiler/javanano/javanano_map_field.cc', 'google/protobuf/compiler/javanano/javanano_helpers.cc', 'google/protobuf/compiler/javanano/javanano_generator.cc', 'google/protobuf/compiler/javanano/javanano_file.cc', 'google/protobuf/compiler/javanano/javanano_field.cc', 'google/protobuf/compiler/javanano/javanano_extension.cc', 'google/protobuf/compiler/javanano/javanano_enum_field.cc', 'google/protobuf/compiler/javanano/javanano_enum.cc', 'google/protobuf/compiler/java/java_string_field_lite.cc', 'google/protobuf/compiler/java/java_string_field.cc', 'google/protobuf/compiler/java/java_shared_code_generator.cc', 'google/protobuf/compiler/java/java_service.cc', 'google/protobuf/compiler/java/java_primitive_field_lite.cc', 'google/protobuf/compiler/java/java_primitive_field.cc', 'google/protobuf/compiler/java/java_name_resolver.cc', 'google/protobuf/compiler/java/java_message_lite.cc', 'google/protobuf/compiler/java/java_message_field_lite.cc', 'google/protobuf/compiler/java/java_message_field.cc', 'google/protobuf/compiler/java/java_message_builder_lite.cc', 'google/protobuf/compiler/java/java_message_builder.cc', 'google/protobuf/compiler/java/java_message.cc', 'google/protobuf/compiler/java/java_map_field_lite.cc', 'google/protobuf/compiler/java/java_map_field.cc', 'google/protobuf/compiler/java/java_lazy_message_field_lite.cc', 'google/protobuf/compiler/java/java_lazy_message_field.cc', 'google/protobuf/compiler/java/java_helpers.cc', 'google/protobuf/compiler/java/java_generator_factory.cc', 'google/protobuf/compiler/java/java_generator.cc', 'google/protobuf/compiler/java/java_file.cc', 'google/protobuf/compiler/java/java_field.cc', 'google/protobuf/compiler/java/java_extension_lite.cc', 'google/protobuf/compiler/java/java_extension.cc', 'google/protobuf/compiler/java/java_enum_lite.cc', 'google/protobuf/compiler/java/java_enum_field_lite.cc', 'google/protobuf/compiler/java/java_enum_field.cc', 'google/protobuf/compiler/java/java_enum.cc', 'google/protobuf/compiler/java/java_doc_comment.cc', 'google/protobuf/compiler/java/java_context.cc', 'google/protobuf/compiler/csharp/csharp_wrapper_field.cc', 'google/protobuf/compiler/csharp/csharp_source_generator_base.cc', 'google/protobuf/compiler/csharp/csharp_repeated_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_message_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_reflection_class.cc', 'google/protobuf/compiler/csharp/csharp_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_message_field.cc', 'google/protobuf/compiler/csharp/csharp_message.cc', 'google/protobuf/compiler/csharp/csharp_map_field.cc', 'google/protobuf/compiler/csharp/csharp_helpers.cc', 'google/protobuf/compiler/csharp/csharp_generator.cc', 'google/protobuf/compiler/csharp/csharp_field_base.cc', 'google/protobuf/compiler/csharp/csharp_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_enum.cc', 'google/protobuf/compiler/csharp/csharp_doc_comment.cc', 'google/protobuf/compiler/cpp/cpp_string_field.cc', 'google/protobuf/compiler/cpp/cpp_service.cc', 'google/protobuf/compiler/cpp/cpp_primitive_field.cc', 'google/protobuf/compiler/cpp/cpp_padding_optimizer.cc', 'google/protobuf/compiler/cpp/cpp_message_field.cc', 'google/protobuf/compiler/cpp/cpp_message.cc', 'google/protobuf/compiler/cpp/cpp_map_field.cc', 'google/protobuf/compiler/cpp/cpp_helpers.cc', 'google/protobuf/compiler/cpp/cpp_generator.cc', 'google/protobuf/compiler/cpp/cpp_file.cc', 'google/protobuf/compiler/cpp/cpp_field.cc', 'google/protobuf/compiler/cpp/cpp_extension.cc', 'google/protobuf/compiler/cpp/cpp_enum_field.cc', 'google/protobuf/compiler/cpp/cpp_enum.cc', 'google/protobuf/compiler/command_line_interface.cc', 'google/protobuf/compiler/code_generator.cc', 'google/protobuf/wrappers.pb.cc', 'google/protobuf/wire_format.cc', 'google/protobuf/util/type_resolver_util.cc', 'google/protobuf/util/time_util.cc', 'google/protobuf/util/message_differencer.cc', 'google/protobuf/util/json_util.cc', 'google/protobuf/util/internal/utility.cc', 'google/protobuf/util/internal/type_info_test_helper.cc', 'google/protobuf/util/internal/type_info.cc', 'google/protobuf/util/internal/protostream_objectwriter.cc', 'google/protobuf/util/internal/protostream_objectsource.cc', 'google/protobuf/util/internal/proto_writer.cc', 'google/protobuf/util/internal/object_writer.cc', 'google/protobuf/util/internal/json_stream_parser.cc', 'google/protobuf/util/internal/json_objectwriter.cc', 'google/protobuf/util/internal/json_escaping.cc', 'google/protobuf/util/internal/field_mask_utility.cc', 'google/protobuf/util/internal/error_listener.cc', 'google/protobuf/util/internal/default_value_objectwriter.cc', 'google/protobuf/util/internal/datapiece.cc', 'google/protobuf/util/field_mask_util.cc', 'google/protobuf/util/field_comparator.cc', 'google/protobuf/util/delimited_message_util.cc', 'google/protobuf/unknown_field_set.cc', 'google/protobuf/type.pb.cc', 'google/protobuf/timestamp.pb.cc', 'google/protobuf/text_format.cc', 'google/protobuf/stubs/substitute.cc', 'google/protobuf/stubs/mathlimits.cc', 'google/protobuf/struct.pb.cc', 'google/protobuf/source_context.pb.cc', 'google/protobuf/service.cc', 'google/protobuf/reflection_ops.cc', 'google/protobuf/message.cc', 'google/protobuf/map_field.cc', 'google/protobuf/io/zero_copy_stream_impl.cc', 'google/protobuf/io/tokenizer.cc', 'google/protobuf/io/strtod.cc', 'google/protobuf/io/printer.cc', 'google/protobuf/io/gzip_stream.cc', 'google/protobuf/generated_message_table_driven.cc', 'google/protobuf/generated_message_reflection.cc', 'google/protobuf/field_mask.pb.cc', 'google/protobuf/extension_set_heavy.cc', 'google/protobuf/empty.pb.cc', 'google/protobuf/dynamic_message.cc', 'google/protobuf/duration.pb.cc', 'google/protobuf/descriptor_database.cc', 'google/protobuf/descriptor.pb.cc', 'google/protobuf/descriptor.cc', 'google/protobuf/compiler/parser.cc', 'google/protobuf/compiler/importer.cc', 'google/protobuf/api.pb.cc', 'google/protobuf/any.pb.cc', 'google/protobuf/any.cc', 'google/protobuf/wire_format_lite.cc', 'google/protobuf/stubs/time.cc', 'google/protobuf/stubs/strutil.cc', 'google/protobuf/stubs/structurally_valid.cc', 'google/protobuf/stubs/stringprintf.cc', 'google/protobuf/stubs/stringpiece.cc', 'google/protobuf/stubs/statusor.cc', 'google/protobuf/stubs/status.cc', 'google/protobuf/stubs/once.cc', 'google/protobuf/stubs/io_win32.cc', 'google/protobuf/stubs/int128.cc', 'google/protobuf/stubs/common.cc', 'google/protobuf/stubs/bytestream.cc', 'google/protobuf/stubs/atomicops_internals_x86_msvc.cc', 'google/protobuf/stubs/atomicops_internals_x86_gcc.cc', 'google/protobuf/repeated_field.cc', 'google/protobuf/message_lite.cc', 'google/protobuf/io/zero_copy_stream_impl_lite.cc', 'google/protobuf/io/zero_copy_stream.cc', 'google/protobuf/io/coded_stream.cc', 'google/protobuf/generated_message_util.cc', 'google/protobuf/generated_message_table_driven_lite.cc', 'google/protobuf/extension_set.cc', 'google/protobuf/arenastring.cc', 'google/protobuf/arena.cc', 'google/protobuf/compiler/js/embed.cc'] +CC_FILES=['google/protobuf/compiler/zip_writer.cc', 'google/protobuf/compiler/subprocess.cc', 'google/protobuf/compiler/ruby/ruby_generator.cc', 'google/protobuf/compiler/python/python_generator.cc', 'google/protobuf/compiler/plugin.pb.cc', 'google/protobuf/compiler/plugin.cc', 'google/protobuf/compiler/php/php_generator.cc', 'google/protobuf/compiler/objectivec/objectivec_primitive_field.cc', 'google/protobuf/compiler/objectivec/objectivec_oneof.cc', 'google/protobuf/compiler/objectivec/objectivec_message_field.cc', 'google/protobuf/compiler/objectivec/objectivec_message.cc', 'google/protobuf/compiler/objectivec/objectivec_map_field.cc', 'google/protobuf/compiler/objectivec/objectivec_helpers.cc', 'google/protobuf/compiler/objectivec/objectivec_generator.cc', 'google/protobuf/compiler/objectivec/objectivec_file.cc', 'google/protobuf/compiler/objectivec/objectivec_field.cc', 'google/protobuf/compiler/objectivec/objectivec_extension.cc', 'google/protobuf/compiler/objectivec/objectivec_enum_field.cc', 'google/protobuf/compiler/objectivec/objectivec_enum.cc', 'google/protobuf/compiler/js/well_known_types_embed.cc', 'google/protobuf/compiler/js/js_generator.cc', 'google/protobuf/compiler/java/java_string_field_lite.cc', 'google/protobuf/compiler/java/java_string_field.cc', 'google/protobuf/compiler/java/java_shared_code_generator.cc', 'google/protobuf/compiler/java/java_service.cc', 'google/protobuf/compiler/java/java_primitive_field_lite.cc', 'google/protobuf/compiler/java/java_primitive_field.cc', 'google/protobuf/compiler/java/java_name_resolver.cc', 'google/protobuf/compiler/java/java_message_lite.cc', 'google/protobuf/compiler/java/java_message_field_lite.cc', 'google/protobuf/compiler/java/java_message_field.cc', 'google/protobuf/compiler/java/java_message_builder_lite.cc', 'google/protobuf/compiler/java/java_message_builder.cc', 'google/protobuf/compiler/java/java_message.cc', 'google/protobuf/compiler/java/java_map_field_lite.cc', 'google/protobuf/compiler/java/java_map_field.cc', 'google/protobuf/compiler/java/java_lazy_message_field_lite.cc', 'google/protobuf/compiler/java/java_lazy_message_field.cc', 'google/protobuf/compiler/java/java_helpers.cc', 'google/protobuf/compiler/java/java_generator_factory.cc', 'google/protobuf/compiler/java/java_generator.cc', 'google/protobuf/compiler/java/java_file.cc', 'google/protobuf/compiler/java/java_field.cc', 'google/protobuf/compiler/java/java_extension_lite.cc', 'google/protobuf/compiler/java/java_extension.cc', 'google/protobuf/compiler/java/java_enum_lite.cc', 'google/protobuf/compiler/java/java_enum_field_lite.cc', 'google/protobuf/compiler/java/java_enum_field.cc', 'google/protobuf/compiler/java/java_enum.cc', 'google/protobuf/compiler/java/java_doc_comment.cc', 'google/protobuf/compiler/java/java_context.cc', 'google/protobuf/compiler/csharp/csharp_wrapper_field.cc', 'google/protobuf/compiler/csharp/csharp_source_generator_base.cc', 'google/protobuf/compiler/csharp/csharp_repeated_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_message_field.cc', 'google/protobuf/compiler/csharp/csharp_repeated_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_reflection_class.cc', 'google/protobuf/compiler/csharp/csharp_primitive_field.cc', 'google/protobuf/compiler/csharp/csharp_message_field.cc', 'google/protobuf/compiler/csharp/csharp_message.cc', 'google/protobuf/compiler/csharp/csharp_map_field.cc', 'google/protobuf/compiler/csharp/csharp_helpers.cc', 'google/protobuf/compiler/csharp/csharp_generator.cc', 'google/protobuf/compiler/csharp/csharp_field_base.cc', 'google/protobuf/compiler/csharp/csharp_enum_field.cc', 'google/protobuf/compiler/csharp/csharp_enum.cc', 'google/protobuf/compiler/csharp/csharp_doc_comment.cc', 'google/protobuf/compiler/cpp/cpp_string_field.cc', 'google/protobuf/compiler/cpp/cpp_service.cc', 'google/protobuf/compiler/cpp/cpp_primitive_field.cc', 'google/protobuf/compiler/cpp/cpp_padding_optimizer.cc', 'google/protobuf/compiler/cpp/cpp_message_field.cc', 'google/protobuf/compiler/cpp/cpp_message.cc', 'google/protobuf/compiler/cpp/cpp_map_field.cc', 'google/protobuf/compiler/cpp/cpp_helpers.cc', 'google/protobuf/compiler/cpp/cpp_generator.cc', 'google/protobuf/compiler/cpp/cpp_file.cc', 'google/protobuf/compiler/cpp/cpp_field.cc', 'google/protobuf/compiler/cpp/cpp_extension.cc', 'google/protobuf/compiler/cpp/cpp_enum_field.cc', 'google/protobuf/compiler/cpp/cpp_enum.cc', 'google/protobuf/compiler/command_line_interface.cc', 'google/protobuf/compiler/code_generator.cc', 'google/protobuf/wrappers.pb.cc', 'google/protobuf/wire_format.cc', 'google/protobuf/util/type_resolver_util.cc', 'google/protobuf/util/time_util.cc', 'google/protobuf/util/message_differencer.cc', 'google/protobuf/util/json_util.cc', 'google/protobuf/util/internal/utility.cc', 'google/protobuf/util/internal/type_info_test_helper.cc', 'google/protobuf/util/internal/type_info.cc', 'google/protobuf/util/internal/protostream_objectwriter.cc', 'google/protobuf/util/internal/protostream_objectsource.cc', 'google/protobuf/util/internal/proto_writer.cc', 'google/protobuf/util/internal/object_writer.cc', 'google/protobuf/util/internal/json_stream_parser.cc', 'google/protobuf/util/internal/json_objectwriter.cc', 'google/protobuf/util/internal/json_escaping.cc', 'google/protobuf/util/internal/field_mask_utility.cc', 'google/protobuf/util/internal/error_listener.cc', 'google/protobuf/util/internal/default_value_objectwriter.cc', 'google/protobuf/util/internal/datapiece.cc', 'google/protobuf/util/field_mask_util.cc', 'google/protobuf/util/field_comparator.cc', 'google/protobuf/util/delimited_message_util.cc', 'google/protobuf/unknown_field_set.cc', 'google/protobuf/type.pb.cc', 'google/protobuf/timestamp.pb.cc', 'google/protobuf/text_format.cc', 'google/protobuf/stubs/substitute.cc', 'google/protobuf/stubs/mathlimits.cc', 'google/protobuf/struct.pb.cc', 'google/protobuf/source_context.pb.cc', 'google/protobuf/service.cc', 'google/protobuf/reflection_ops.cc', 'google/protobuf/message.cc', 'google/protobuf/map_field.cc', 'google/protobuf/io/zero_copy_stream_impl.cc', 'google/protobuf/io/tokenizer.cc', 'google/protobuf/io/strtod.cc', 'google/protobuf/io/printer.cc', 'google/protobuf/io/gzip_stream.cc', 'google/protobuf/generated_message_table_driven.cc', 'google/protobuf/generated_message_reflection.cc', 'google/protobuf/field_mask.pb.cc', 'google/protobuf/extension_set_heavy.cc', 'google/protobuf/empty.pb.cc', 'google/protobuf/dynamic_message.cc', 'google/protobuf/duration.pb.cc', 'google/protobuf/descriptor_database.cc', 'google/protobuf/descriptor.pb.cc', 'google/protobuf/descriptor.cc', 'google/protobuf/compiler/parser.cc', 'google/protobuf/compiler/importer.cc', 'google/protobuf/api.pb.cc', 'google/protobuf/any.pb.cc', 'google/protobuf/any.cc', 'google/protobuf/wire_format_lite.cc', 'google/protobuf/stubs/time.cc', 'google/protobuf/stubs/strutil.cc', 'google/protobuf/stubs/structurally_valid.cc', 'google/protobuf/stubs/stringprintf.cc', 'google/protobuf/stubs/stringpiece.cc', 'google/protobuf/stubs/statusor.cc', 'google/protobuf/stubs/status.cc', 'google/protobuf/stubs/io_win32.cc', 'google/protobuf/stubs/int128.cc', 'google/protobuf/stubs/common.cc', 'google/protobuf/stubs/bytestream.cc', 'google/protobuf/repeated_field.cc', 'google/protobuf/message_lite.cc', 'google/protobuf/io/zero_copy_stream_impl_lite.cc', 'google/protobuf/io/zero_copy_stream.cc', 'google/protobuf/io/coded_stream.cc', 'google/protobuf/implicit_weak_message.cc', 'google/protobuf/generated_message_util.cc', 'google/protobuf/generated_message_table_driven_lite.cc', 'google/protobuf/extension_set.cc', 'google/protobuf/arenastring.cc', 'google/protobuf/arena.cc'] PROTO_FILES=['google/protobuf/wrappers.proto', 'google/protobuf/type.proto', 'google/protobuf/timestamp.proto', 'google/protobuf/struct.proto', 'google/protobuf/source_context.proto', 'google/protobuf/field_mask.proto', 'google/protobuf/empty.proto', 'google/protobuf/duration.proto', 'google/protobuf/descriptor.proto', 'google/protobuf/compiler/plugin.proto', 'google/protobuf/api.proto', 'google/protobuf/any.proto'] CC_INCLUDE='third_party/protobuf/src' PROTO_INCLUDE='third_party/protobuf/src' -PROTOBUF_SUBMODULE_VERSION="b5fbb742af122b565925987e65c08957739976a7" +PROTOBUF_SUBMODULE_VERSION="48cb18e5c419ddd23d9badcfe4e9df7bde1979b2" diff --git a/tools/dockerfile/grpc_artifact_protoc/Dockerfile b/tools/dockerfile/grpc_artifact_protoc/Dockerfile index 33e2f2970c334..44fde4f99f02a 100644 --- a/tools/dockerfile/grpc_artifact_protoc/Dockerfile +++ b/tools/dockerfile/grpc_artifact_protoc/Dockerfile @@ -30,24 +30,34 @@ RUN yum install -y git \ glibc-devel \ glibc-devel.i686 -# Install GCC 4.7 to support -static-libstdc++ -RUN wget http://people.centos.org/tru/devtools-1.1/devtools-1.1.repo -P /etc/yum.repos.d -RUN bash -c 'echo "enabled=1" >> /etc/yum.repos.d/devtools-1.1.repo' -RUN bash -c "sed -e 's/\$basearch/i386/g' /etc/yum.repos.d/devtools-1.1.repo > /etc/yum.repos.d/devtools-i386-1.1.repo" -RUN sed -e 's/testing-/testing-i386-/g' -i /etc/yum.repos.d/devtools-i386-1.1.repo +# Install GCC 4.8 +RUN wget http://people.centos.org/tru/devtools-2/devtools-2.repo -P /etc/yum.repos.d +RUN bash -c 'echo "enabled=1" >> /etc/yum.repos.d/devtools-2.repo' +RUN bash -c "sed -e 's/\$basearch/i386/g' /etc/yum.repos.d/devtools-2.repo > /etc/yum.repos.d/devtools-i386-2.repo" +RUN sed -e 's/testing-/testing-i386-/g' -i /etc/yum.repos.d/devtools-i386-2.repo # We'll get and "Rpmdb checksum is invalid: dCDPT(pkg checksums)" error caused by # docker issue when using overlay storage driver, but all the stuff we need # will be installed, so for now we just ignore the error. # https://github.com/docker/docker/issues/10180 -RUN yum install -y devtoolset-1.1 \ - devtoolset-1.1-libstdc++-devel \ - devtoolset-1.1-libstdc++-devel.i686 || true +RUN yum install -y devtoolset-2-build \ + devtoolset-2-toolchain \ + devtoolset-2-binutils \ + devtoolset-2-gcc \ + devtoolset-2-gcc-c++ \ + devtoolset-2-libstdc++-devel \ + devtoolset-2-libstdc++-devel.i686 || true + +# Again, ignore the "Rpmdb checksum is invalid: dCDPT(pkg checksums)" error. +RUN yum install -y ca-certificates || true # renew certs to prevent download error for ius-release.rpm + +# TODO(jtattermusch): gRPC makefile uses "which" to detect the availability of gcc +RUN yum install -y which || true # Update Git to version >1.7 to allow cloning submodules with --reference arg. RUN yum remove -y git && yum clean all RUN yum install -y https://centos6.iuscommunity.org/ius-release.rpm && yum clean all RUN yum install -y git2u && yum clean all -# Start in devtoolset environment that uses GCC 4.7 -CMD ["scl", "enable", "devtoolset-1.1", "bash"] +# Start in devtoolset environment that uses GCC 4.8 +CMD ["scl", "enable", "devtoolset-2", "bash"] diff --git a/tools/internal_ci/helper_scripts/prepare_build_windows.bat b/tools/internal_ci/helper_scripts/prepare_build_windows.bat index 0164e4a56177d..f987f8a8cb5b4 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_windows.bat +++ b/tools/internal_ci/helper_scripts/prepare_build_windows.bat @@ -14,7 +14,7 @@ @rem make sure msys binaries are preferred over cygwin binaries @rem set path to python 2.7 -set PATH=C:\tools\msys64\usr\bin;C:\Python27;C:\Python37;%PATH% +set PATH=C:\tools\msys64\usr\bin;C:\Python27;%PATH% @rem If this is a PR using RUN_TESTS_FLAGS var, then add flags to filter tests if defined KOKORO_GITHUB_PULL_REQUEST_NUMBER if defined RUN_TESTS_FLAGS ( @@ -34,9 +34,6 @@ netsh interface ip add dnsservers "Local Area Connection 8" 8.8.4.4 index=3 @rem Needed for big_query_utils python -m pip install google-api-python-client -@rem Install Python 3.7 -chocolatey install -y -r python3 --version 3.7 - @rem Disable some unwanted dotnet options set NUGET_XMLDOC_MODE=skip set DOTNET_SKIP_FIRST_TIME_EXPERIENCE=true diff --git a/tools/internal_ci/linux/grpc_android.sh b/tools/internal_ci/linux/grpc_android.sh index 2220145dacc55..42c7f5fb0420b 100755 --- a/tools/internal_ci/linux/grpc_android.sh +++ b/tools/internal_ci/linux/grpc_android.sh @@ -25,13 +25,6 @@ git submodule update --init # Build protoc and grpc_cpp_plugin. Codegen is not cross-compiled to Android make HAS_SYSTEM_PROTOBUF=false -# TODO(ericgribkoff) Remove when this commit (already in master) is included in -# next protobuf release -cd third_party/protobuf -git fetch -git cherry-pick 7daa320065f3bea2b54bf983337d1724f153422d -m 1 - - # Build and run interop instrumentation tests on Firebase Test Lab cd "${REPO_ROOT}/src/android/test/interop/" diff --git a/tools/internal_ci/linux/grpc_asan_on_foundry.sh b/tools/internal_ci/linux/grpc_asan_on_foundry.sh index 5099fa03016c3..a6367ad962846 100644 --- a/tools/internal_ci/linux/grpc_asan_on_foundry.sh +++ b/tools/internal_ci/linux/grpc_asan_on_foundry.sh @@ -14,6 +14,6 @@ # limitations under the License. export UPLOAD_TEST_RESULTS=true -EXTRA_FLAGS="--copt=-gmlt --strip=never --copt=-fsanitize=address --linkopt=-fsanitize=address --test_timeout=3600 --runs_per_test_detects_flakes --runs_per_test=2" +EXTRA_FLAGS="--copt=-gmlt --strip=never --copt=-fsanitize=address --linkopt=-fsanitize=address --test_timeout=3600 --cache_test_results=no" github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" diff --git a/tools/internal_ci/linux/grpc_bazel_on_foundry_dbg.sh b/tools/internal_ci/linux/grpc_bazel_on_foundry_dbg.sh index 192d9d1b2d0bf..51cb66f5b866d 100644 --- a/tools/internal_ci/linux/grpc_bazel_on_foundry_dbg.sh +++ b/tools/internal_ci/linux/grpc_bazel_on_foundry_dbg.sh @@ -14,5 +14,5 @@ # limitations under the License. export UPLOAD_TEST_RESULTS=true -EXTRA_FLAGS="-c dbg --test_timeout=300,450,1200,3600 --runs_per_test_detects_flakes --runs_per_test=2" +EXTRA_FLAGS="-c dbg --test_timeout=300,450,1200,3600 --cache_test_results=no" github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" diff --git a/tools/internal_ci/linux/grpc_bazel_on_foundry_opt.sh b/tools/internal_ci/linux/grpc_bazel_on_foundry_opt.sh index 6fb3c77892e90..cbba9067ad3fe 100644 --- a/tools/internal_ci/linux/grpc_bazel_on_foundry_opt.sh +++ b/tools/internal_ci/linux/grpc_bazel_on_foundry_opt.sh @@ -14,5 +14,5 @@ # limitations under the License. export UPLOAD_TEST_RESULTS=true -EXTRA_FLAGS="-c opt --test_timeout=300,450,1200,3600 --runs_per_test_detects_flakes --runs_per_test=2" +EXTRA_FLAGS="-c opt --test_timeout=300,450,1200,3600 --cache_test_results=no" github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" diff --git a/tools/internal_ci/linux/grpc_tsan_on_foundry.sh b/tools/internal_ci/linux/grpc_tsan_on_foundry.sh index 25531ead2b56f..2ba7d469ecf2c 100644 --- a/tools/internal_ci/linux/grpc_tsan_on_foundry.sh +++ b/tools/internal_ci/linux/grpc_tsan_on_foundry.sh @@ -14,5 +14,5 @@ # limitations under the License. export UPLOAD_TEST_RESULTS=true -EXTRA_FLAGS="--copt=-gmlt --strip=never --copt=-fsanitize=thread --linkopt=-fsanitize=thread --test_timeout=3600 --action_env=TSAN_OPTIONS=suppressions=test/core/util/tsan_suppressions.txt:halt_on_error=1:second_deadlock_stack=1 --runs_per_test_detects_flakes --runs_per_test=2" +EXTRA_FLAGS="--copt=-gmlt --strip=never --copt=-fsanitize=thread --linkopt=-fsanitize=thread --test_timeout=3600 --action_env=TSAN_OPTIONS=suppressions=test/core/util/tsan_suppressions.txt:halt_on_error=1:second_deadlock_stack=1 --cache_test_results=no" github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" diff --git a/tools/internal_ci/linux/grpc_ubsan_on_foundry.sh b/tools/internal_ci/linux/grpc_ubsan_on_foundry.sh index d9b039f1ecd20..338b1b6a0d01d 100644 --- a/tools/internal_ci/linux/grpc_ubsan_on_foundry.sh +++ b/tools/internal_ci/linux/grpc_ubsan_on_foundry.sh @@ -61,8 +61,7 @@ source tools/internal_ci/helper_scripts/prepare_build_linux_rc --extra_execution_platforms=//third_party/toolchains:rbe_ubuntu1604 \ --host_platform=//third_party/toolchains:rbe_ubuntu1604 \ --platforms=//third_party/toolchains:rbe_ubuntu1604 \ - --runs_per_test_detects_flakes \ - --runs_per_test=2 \ + --cache_test_results=no \ -- //test/... || FAILED="true" # Sleep to let ResultStore finish writing results before querying diff --git a/tools/internal_ci/windows/grpc_build_artifacts.bat b/tools/internal_ci/windows/grpc_build_artifacts.bat index 4d528e0128523..0e02bdb6ecca8 100644 --- a/tools/internal_ci/windows/grpc_build_artifacts.bat +++ b/tools/internal_ci/windows/grpc_build_artifacts.bat @@ -18,6 +18,7 @@ rename C:\Python27_32bit Python27_32bits rename C:\Python34_32bit Python34_32bits rename C:\Python35_32bit Python35_32bits rename C:\Python36_32bit Python36_32bits +rename C:\Python37_32bit Python37_32bits @rem enter repo root cd /d %~dp0\..\..\.. diff --git a/tools/interop_matrix/README.md b/tools/interop_matrix/README.md index 40c02a1bc7a0a..db84d9b454c87 100644 --- a/tools/interop_matrix/README.md +++ b/tools/interop_matrix/README.md @@ -9,7 +9,7 @@ from specific releases/tag, are used to test version compatiblity between gRPC r We have continuous nightly test setup to test gRPC backward compatibility between old clients and latest server. When a gRPC developer creates a new gRPC release, s/he is also responsible to add the just-released gRPC client to the nightly test. The steps are: - Add (or update) an entry in `./client_matrix.py` file to reference the github tag for the release. - Build new client docker image(s). For example, for C and wrapper languages release `v1.9.9`, do - - `tools/interop_matrix/create_matrix_images.py --git_checkout --release=v1.9.9 --language cxx csharp python ruby php` + - `tools/interop_matrix/create_matrix_images.py --git_checkout --release=v1.9.9 --upload_images --language cxx csharp python ruby php` - Verify that the new docker image was built successfully and uploaded to GCR. For example, - `gcloud beta container images list --repository gcr.io/grpc-testing` shows image repos. - `gcloud beta container images list-tags gcr.io/grpc-testing/grpc_interop_java_oracle8` should show an image entry with tag `v1.9.9`. diff --git a/tools/interop_matrix/client_matrix.py b/tools/interop_matrix/client_matrix.py index 26bd4832032ed..dff33c88c02c8 100644 --- a/tools/interop_matrix/client_matrix.py +++ b/tools/interop_matrix/client_matrix.py @@ -96,6 +96,9 @@ def should_build_docker_interop_image_from_release_tag(lang): { 'v1.13.0': None }, + { + 'v1.14.0': None + }, ], 'go': [ { @@ -222,6 +225,9 @@ def should_build_docker_interop_image_from_release_tag(lang): { 'v1.13.0': None }, + { + 'v1.14.0': None + }, ], 'node': [ { @@ -307,6 +313,9 @@ def should_build_docker_interop_image_from_release_tag(lang): { 'v1.13.0': None }, + { + 'v1.14.0': None + }, ], 'php': [ { @@ -348,6 +357,9 @@ def should_build_docker_interop_image_from_release_tag(lang): { 'v1.13.0': None }, + { + 'v1.14.0': None + }, ], 'csharp': [ { @@ -394,6 +406,9 @@ def should_build_docker_interop_image_from_release_tag(lang): { 'v1.13.0': None }, + { + 'v1.14.0': None + }, ], } diff --git a/tools/profiling/ios_bin/binary_size.py b/tools/profiling/ios_bin/binary_size.py index 91c43f4424375..d4d134fef3725 100755 --- a/tools/profiling/ios_bin/binary_size.py +++ b/tools/profiling/ios_bin/binary_size.py @@ -75,6 +75,7 @@ def get_size(where, frameworks): def build(where, frameworks): + subprocess.check_call(['make', 'clean']) shutil.rmtree( 'src/objective-c/examples/Sample/Build/Build-%s' % where, ignore_errors=True) diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py index 01323b53266f5..a37098d9bf1ee 100644 --- a/tools/run_tests/artifacts/artifact_targets.py +++ b/tools/run_tests/artifacts/artifact_targets.py @@ -392,6 +392,7 @@ def targets(): PythonArtifact('windows', 'x86', 'Python34_32bits'), PythonArtifact('windows', 'x86', 'Python35_32bits'), PythonArtifact('windows', 'x86', 'Python36_32bits'), + PythonArtifact('windows', 'x86', 'Python37_32bits'), PythonArtifact('windows', 'x64', 'Python27'), PythonArtifact('windows', 'x64', 'Python34'), PythonArtifact('windows', 'x64', 'Python35'), diff --git a/tools/run_tests/artifacts/build_artifact_protoc.sh b/tools/run_tests/artifacts/build_artifact_protoc.sh index b531fc9a5d4ff..6d433f2dad41f 100755 --- a/tools/run_tests/artifacts/build_artifact_protoc.sh +++ b/tools/run_tests/artifacts/build_artifact_protoc.sh @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Use devtoolset environment that has GCC 4.7 before set -ex -source scl_source enable devtoolset-1.1 +# Use devtoolset environment that has GCC 4.8 before set -ex +source scl_source enable devtoolset-2 set -ex diff --git a/tools/run_tests/generated/tests.json b/tools/run_tests/generated/tests.json index 5815f82fefbb8..cf3b54e044057 100644 --- a/tools/run_tests/generated/tests.json +++ b/tools/run_tests/generated/tests.json @@ -5784,7 +5784,8 @@ "ci_platforms": [ "linux", "mac", - "posix" + "posix", + "windows" ], "cpu_cost": 1.0, "exclude_configs": [], @@ -5796,7 +5797,8 @@ "platforms": [ "linux", "mac", - "posix" + "posix", + "windows" ], "uses_polling": true }, diff --git a/tools/run_tests/sanity/check_submodules.sh b/tools/run_tests/sanity/check_submodules.sh index fb159bc2d2011..2c7c140716f38 100755 --- a/tools/run_tests/sanity/check_submodules.sh +++ b/tools/run_tests/sanity/check_submodules.sh @@ -36,7 +36,7 @@ cat << EOF | awk '{ print $1 }' | sort > "$want_submodules" ec44c6c1675c25b9827aacd08c02433cccde7780 third_party/googletest (release-1.8.0) 6599cac0965be8e5a835ab7a5684bbef033d5ad0 third_party/libcxx (heads/release_60) 9245d481eb3e890f708ff2d7dadf2a10c04748ba third_party/libcxxabi (heads/release_60) - b5fbb742af122b565925987e65c08957739976a7 third_party/protobuf (v3.3.1-641-gb5fbb74) + 48cb18e5c419ddd23d9badcfe4e9df7bde1979b2 third_party/protobuf (v3.6.0.1-37-g48cb18e5) cacf7f1d4e3d44d871b605da3b647f07d718623f third_party/zlib (v1.2.11) EOF