Skip to content

Commit

Permalink
locator: ec2_snitch: IMDSv2 support
Browse files Browse the repository at this point in the history 
Access to AWS Metadata may be configured in three distinct ways:
   1 - Optional HTTP tokens and HTTP endpoint enabled: The default as it works today
   2 - Required HTTP tokens and HTTP endpoint enabled: Which support is entirely missing today
   3 - HTTP endpoint disabled: Which effectively forbids one to use Ec2Snitch or Ec2MultiRegionSnitch

This commit makes the 2nd option the default which is not only AWS recommended option, but is also entirely compatible with the 1st option.
In addition, we now validate the HTTP response when querying the IMDS server. Therefore - should a HTTP 403 be received - Scylla will
properly notify users on what they are trying to do incorrectly in their setup.

The commit was tested under the following circumstances (covering all 3 variants):
 - Ec2Snitch: IMDSv2 optional & required, and HTTP server disabled.
 - Ec2MultiRegionSnitch: IMDSv2 optional & required, and HTTP server disabled.

Refs: https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html
      https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-options.html
      #9987
Fixes: #10490
Closes: #10490

Closes #11636
  • Loading branch information
@fee-mendes @avikivity
fee-mendes authored and avikivity committed Oct 4, 2022
1 parent c200ae2 commit f67bb43
Show file tree
Hide file tree
Showing 3 changed files with 32 additions and 13 deletions.
10 changes: 6 additions & 4 deletions locator/ec2_multi_region_snitch.cc
View file
Expand Up @@ -30,10 +30,12 @@ future<> ec2_multi_region_snitch::start() {
if (this_shard_id() == io_cpu_id()) {
inet_address local_public_address;

auto token = aws_api_call(AWS_QUERY_SERVER_ADDR, AWS_QUERY_SERVER_PORT, TOKEN_REQ_ENDPOINT, std::nullopt).get0();

try {
auto broadcast = utils::fb_utilities::get_broadcast_address();
if (broadcast.addr().is_ipv6()) {
auto macs = aws_api_call(AWS_QUERY_SERVER_ADDR, AWS_QUERY_SERVER_PORT, PRIVATE_MAC_QUERY).get0();
auto macs = aws_api_call(AWS_QUERY_SERVER_ADDR, AWS_QUERY_SERVER_PORT, PRIVATE_MAC_QUERY, token).get0();
// we should just get a single line, ending in '/'. If there are more than one mac, we should
// maybe try to loop the addresses and exclude local/link-local etc, but these addresses typically
// are already filtered by aws, so probably does not help. For now, just warn and pick first address.
Expand All @@ -42,11 +44,11 @@ future<> ec2_multi_region_snitch::start() {
if (i != std::string::npos && ++i != macs.size()) {
logger().warn("Ec2MultiRegionSnitch (ipv6): more than one MAC address listed ({}). Will use first.", macs);
}
auto ipv6 = aws_api_call(AWS_QUERY_SERVER_ADDR, AWS_QUERY_SERVER_PORT, format(PUBLIC_IPV6_QUERY_REQ, mac)).get0();
auto ipv6 = aws_api_call(AWS_QUERY_SERVER_ADDR, AWS_QUERY_SERVER_PORT, format(PUBLIC_IPV6_QUERY_REQ, mac), token).get0();
local_public_address = inet_address(ipv6);
_local_private_address = ipv6;
} else {
auto pub_addr = aws_api_call(AWS_QUERY_SERVER_ADDR, AWS_QUERY_SERVER_PORT, PUBLIC_IP_QUERY_REQ).get0();
auto pub_addr = aws_api_call(AWS_QUERY_SERVER_ADDR, AWS_QUERY_SERVER_PORT, PUBLIC_IP_QUERY_REQ, token).get0();
local_public_address = inet_address(pub_addr);
}
} catch (...) {
Expand All @@ -66,7 +68,7 @@ future<> ec2_multi_region_snitch::start() {
}

if (!local_public_address.addr().is_ipv6()) {
sstring priv_addr = aws_api_call(AWS_QUERY_SERVER_ADDR, AWS_QUERY_SERVER_PORT, PRIVATE_IP_QUERY_REQ).get0();
sstring priv_addr = aws_api_call(AWS_QUERY_SERVER_ADDR, AWS_QUERY_SERVER_PORT, PRIVATE_IP_QUERY_REQ, token).get0();
_local_private_address = priv_addr;
}

Expand Down
30 changes: 23 additions & 7 deletions locator/ec2_snitch.cc
View file
Expand Up @@ -21,7 +21,8 @@ future<> ec2_snitch::load_config(bool prefer_local) {
using namespace boost::algorithm;

if (this_shard_id() == io_cpu_id()) {
return aws_api_call(AWS_QUERY_SERVER_ADDR, AWS_QUERY_SERVER_PORT, ZONE_NAME_QUERY_REQ).then([this, prefer_local](sstring az) {
auto token = aws_api_call(AWS_QUERY_SERVER_ADDR, AWS_QUERY_SERVER_PORT, TOKEN_REQ_ENDPOINT, std::nullopt).get0();
return aws_api_call(AWS_QUERY_SERVER_ADDR, AWS_QUERY_SERVER_PORT, ZONE_NAME_QUERY_REQ, token).then([this, prefer_local](sstring az) {
assert(az.size());

std::vector<std::string> splits;
Expand Down Expand Up @@ -63,17 +64,26 @@ future<> ec2_snitch::start() {
});
}

future<sstring> ec2_snitch::aws_api_call(sstring addr, uint16_t port, sstring cmd) {
future<sstring> ec2_snitch::aws_api_call(sstring addr, uint16_t port, sstring cmd, std::optional<sstring> token) {
return connect(socket_address(inet_address{addr}, port))
.then([this, addr, cmd] (connected_socket fd) {
.then([this, addr, cmd, token] (connected_socket fd) {
_sd = std::move(fd);
_in = _sd.input();
_out = _sd.output();
_zone_req = sstring("GET ") + cmd +
sstring(" HTTP/1.1\r\nHost: ") +addr +
sstring("\r\n\r\n");

return _out.write(_zone_req.c_str()).then([this] {
if (token) {
_req = sstring("GET ") + cmd +
sstring(" HTTP/1.1\r\nHost: ") +addr +
sstring("\r\nX-aws-ec2-metadata-token: ") + *token +
sstring("\r\n\r\n");
} else {
_req = sstring("PUT ") + cmd +
sstring(" HTTP/1.1\r\nHost: ") + addr +
sstring("\r\nX-aws-ec2-metadata-token-ttl-seconds: 60") +
sstring("\r\n\r\n");
}

return _out.write(_req.c_str()).then([this] {
return _out.flush();
});
}).then([this] {
Expand All @@ -85,6 +95,12 @@ future<sstring> ec2_snitch::aws_api_call(sstring addr, uint16_t port, sstring cm

// Read HTTP response header first
auto _rsp = _parser.get_parsed_response();
auto rc = _rsp->_status_code;
// Verify EC2 instance metadata access
if (rc == 403) {
return make_exception_future<sstring>(std::runtime_error("Error: Unauthorized response received when trying to communicate with instance metadata service."));
}

auto it = _rsp->_headers.find("Content-Length");
if (it == _rsp->_headers.end()) {
return make_exception_future<sstring>("Error: HTTP response does not contain: Content-Length\n");
Expand Down
5 changes: 3 additions & 2 deletions locator/ec2_snitch.hh
View file
Expand Up @@ -13,6 +13,7 @@
namespace locator {
class ec2_snitch : public production_snitch_base {
public:
static constexpr const char* TOKEN_REQ_ENDPOINT = "/latest/api/token";
static constexpr const char* ZONE_NAME_QUERY_REQ = "/latest/meta-data/placement/availability-zone";
static constexpr const char* AWS_QUERY_SERVER_ADDR = "169.254.169.254";
static constexpr uint16_t AWS_QUERY_SERVER_PORT = 80;
Expand All @@ -24,13 +25,13 @@ public:
}
protected:
future<> load_config(bool prefer_local);
future<sstring> aws_api_call(sstring addr, uint16_t port, const sstring cmd);
future<sstring> aws_api_call(sstring addr, uint16_t port, const sstring cmd, std::optional<sstring> token);
future<sstring> read_property_file();
private:
connected_socket _sd;
input_stream<char> _in;
output_stream<char> _out;
http_response_parser _parser;
sstring _zone_req;
sstring _req;
};
} // namespace locator

0 comments on commit f67bb43

Please sign in to comment.