diff --git a/Cargo.lock b/Cargo.lock index 14488da8837..fe1cc390b81 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2338,11 +2338,23 @@ dependencies = [ name = "internal-dns-client" version = "0.1.0" dependencies = [ + "dropshot", + "futures", + "internal-dns", + "omicron-common 0.1.0", + "omicron-test-utils", "progenitor", "reqwest", "serde", "serde_json", + "sled", "slog", + "tempfile", + "thiserror", + "tokio", + "trust-dns-proto", + "trust-dns-resolver", + "uuid", ] [[package]] diff --git a/internal-dns-client/Cargo.toml b/internal-dns-client/Cargo.toml index af67e13d716..4872699610a 100644 --- a/internal-dns-client/Cargo.toml +++ b/internal-dns-client/Cargo.toml @@ -5,8 +5,22 @@ edition = "2021" license = "MPL-2.0" [dependencies] +futures = "0.3.21" +omicron-common = { path = "../common" } progenitor = { git = "https://github.com/oxidecomputer/progenitor" } +reqwest = { version = "0.11", features = ["json", "rustls-tls", "stream"] } serde = { version = "1.0", features = [ "derive" ] } serde_json = "1.0" slog = { version = "2.5.0", features = [ "max_level_trace", "release_max_level_debug" ] } -reqwest = { version = "0.11", features = ["json", "rustls-tls", "stream"] } +thiserror = "1.0" +trust-dns-proto = "0.21" +trust-dns-resolver = "0.21" +uuid = { version = "1.1.0", features = [ "v4", "serde" ] } + +[dev-dependencies] +dropshot = { git = "https://github.com/oxidecomputer/dropshot", branch = "main", features = [ "usdt-probes" ] } +internal-dns = { path = "../internal-dns" } +omicron-test-utils = { path = "../test-utils" } +sled = "0.34" +tempfile = "3.3" +tokio = { version = "1.18", features = [ "full" ] } diff --git a/internal-dns-client/src/lib.rs b/internal-dns-client/src/lib.rs index 49daa3d58ae..f7ce56f8521 100644 --- a/internal-dns-client/src/lib.rs +++ b/internal-dns-client/src/lib.rs @@ -16,3 +16,6 @@ progenitor::generate_api!( slog::debug!(log, "client response"; "result" => ?result); }), ); + +pub mod multiclient; +pub mod names; diff --git a/internal-dns-client/src/multiclient.rs b/internal-dns-client/src/multiclient.rs new file mode 100644 index 00000000000..2fc9089e334 --- /dev/null +++ b/internal-dns-client/src/multiclient.rs @@ -0,0 +1,596 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +use crate::types::{DnsKv, DnsRecord, DnsRecordKey, Srv}; +use futures::stream::{self, StreamExt, TryStreamExt}; +use omicron_common::address::{ + Ipv6Subnet, ReservedRackSubnet, AZ_PREFIX, DNS_PORT, DNS_SERVER_PORT, +}; +use slog::{info, Logger}; +use std::collections::HashMap; +use std::net::{IpAddr, Ipv6Addr, SocketAddr, SocketAddrV6}; +use trust_dns_resolver::config::{ + NameServerConfig, Protocol, ResolverConfig, ResolverOpts, +}; +use trust_dns_resolver::TokioAsyncResolver; + +pub type DnsError = crate::Error; + +pub type AAAARecord = (crate::names::AAAA, SocketAddrV6); + +/// Describes how to find the DNS servers. +/// +/// In production code, this is nearly always [`Ipv6Subnet`], +/// but it allows a point of dependency-injection for tests to supply their +/// own address lookups. +pub trait DnsAddressLookup { + fn dropshot_server_addrs(&self) -> Vec; + + fn dns_server_addrs(&self) -> Vec; +} + +fn subnet_to_ips( + subnet: Ipv6Subnet, +) -> impl Iterator { + ReservedRackSubnet::new(subnet) + .get_dns_subnets() + .into_iter() + .map(|dns_subnet| IpAddr::V6(dns_subnet.dns_address().ip())) +} + +impl DnsAddressLookup for Ipv6Subnet { + fn dropshot_server_addrs(&self) -> Vec { + subnet_to_ips(*self) + .map(|address| SocketAddr::new(address, DNS_SERVER_PORT)) + .collect() + } + + fn dns_server_addrs(&self) -> Vec { + subnet_to_ips(*self) + .map(|address| SocketAddr::new(address, DNS_PORT)) + .collect() + } +} + +/// A connection used to update multiple DNS servers. +pub struct Updater { + log: Logger, + clients: Vec, +} + +impl Updater { + pub fn new(address_getter: &impl DnsAddressLookup, log: Logger) -> Self { + let addrs = address_getter.dropshot_server_addrs(); + Self::new_from_addrs(addrs, log) + } + + fn new_from_addrs(addrs: Vec, log: Logger) -> Self { + let clients = addrs + .into_iter() + .map(|addr| { + info!(log, "Adding DNS server: {}", addr); + crate::Client::new(&format!("http://{}", addr), log.clone()) + }) + .collect::>(); + + Self { log, clients } + } + + /// Inserts all service records into the DNS server. + /// + /// Each SRV record should have one or more AAAA records. + pub async fn insert_dns_records( + &self, + records: &HashMap>, + ) -> Result<(), DnsError> { + for (srv, aaaa) in records.iter() { + info!(self.log, "Inserting DNS record: {:?}", srv); + + self.insert_dns_records_internal(aaaa, srv).await?; + } + Ok(()) + } + + // Utility function to insert: + // - A set of uniquely-named AAAA records, each corresponding to an address + // - An SRV record, pointing to each of the AAAA records. + async fn insert_dns_records_internal( + &self, + aaaa: &Vec, + srv_key: &crate::names::SRV, + ) -> Result<(), DnsError> { + let mut records = Vec::with_capacity(aaaa.len() + 1); + + // Add one DnsKv per AAAA, each with a single record. + records.extend(aaaa.iter().map(|(name, addr)| DnsKv { + key: DnsRecordKey { name: name.to_string() }, + records: vec![DnsRecord::Aaaa(*addr.ip())], + })); + + // Add the DnsKv for the SRV, with a record for each AAAA. + records.push(DnsKv { + key: DnsRecordKey { name: srv_key.to_string() }, + records: aaaa + .iter() + .map(|(name, addr)| { + DnsRecord::Srv(Srv { + prio: 0, + weight: 0, + port: addr.port(), + target: name.to_string(), + }) + }) + .collect::>(), + }); + self.dns_records_set(&records).await + } + + /// Sets a records on all DNS servers. + /// + /// Returns an error if setting the record fails on any server. + pub async fn dns_records_set<'a>( + &'a self, + body: &'a Vec, + ) -> Result<(), DnsError> { + stream::iter(&self.clients) + .map(Ok::<_, DnsError>) + .try_for_each_concurrent(None, |client| async move { + client.dns_records_set(body).await?; + Ok(()) + }) + .await?; + + Ok(()) + } + + /// Deletes records in all DNS servers. + /// + /// Returns an error if deleting the record fails on any server. + pub async fn dns_records_delete<'a>( + &'a self, + body: &'a Vec, + ) -> Result<(), DnsError> { + stream::iter(&self.clients) + .map(Ok::<_, DnsError>) + .try_for_each_concurrent(None, |client| async move { + client.dns_records_delete(body).await?; + Ok(()) + }) + .await?; + + Ok(()) + } +} + +#[derive(Debug, Clone, thiserror::Error)] +pub enum ResolveError { + #[error(transparent)] + Resolve(#[from] trust_dns_resolver::error::ResolveError), + + #[error("Record not found for SRV key: {0}")] + NotFound(crate::names::SRV), +} + +/// A wrapper around a DNS resolver, providing a way to conveniently +/// look up IP addresses of services based on their SRV keys. +pub struct Resolver { + inner: Box, +} + +impl Resolver { + pub fn new( + address_getter: &impl DnsAddressLookup, + ) -> Result { + let dns_addrs = address_getter.dns_server_addrs(); + Self::new_from_addrs(dns_addrs) + } + + fn new_from_addrs( + dns_addrs: Vec, + ) -> Result { + let mut rc = ResolverConfig::new(); + for socket_addr in dns_addrs.into_iter() { + rc.add_name_server(NameServerConfig { + socket_addr, + protocol: Protocol::Udp, + tls_dns_name: None, + trust_nx_responses: false, + bind_addr: None, + }); + } + let inner = + Box::new(TokioAsyncResolver::tokio(rc, ResolverOpts::default())?); + + Ok(Self { inner }) + } + + /// Convenience wrapper for [`Resolver::new`] which determines the subnet + /// based on a provided IP address. + pub fn new_from_ip(address: Ipv6Addr) -> Result { + let subnet = Ipv6Subnet::::new(address); + + Resolver::new(&subnet) + } + + /// Looks up a single [`Ipv6Addr`] based on the SRV name. + /// Returns an error if the record does not exist. + // TODO: There are lots of ways this API can expand: Caching, + // actually respecting TTL, looking up ports, etc. + // + // For now, however, it serves as a very simple "get everyone using DNS" + // API that can be improved upon later. + pub async fn lookup_ipv6( + &self, + srv: crate::names::SRV, + ) -> Result { + let response = self.inner.ipv6_lookup(&srv.to_string()).await?; + let address = response + .iter() + .next() + .ok_or_else(|| ResolveError::NotFound(srv))?; + Ok(*address) + } + + pub async fn lookup_ip( + &self, + srv: crate::names::SRV, + ) -> Result { + let response = self.inner.lookup_ip(&srv.to_string()).await?; + let address = response + .iter() + .next() + .ok_or_else(|| ResolveError::NotFound(srv))?; + Ok(address) + } +} + +#[cfg(test)] +mod test { + use super::*; + use crate::names::{BackendName, ServiceName, AAAA, SRV}; + use omicron_test_utils::dev::test_setup_log; + use std::str::FromStr; + use std::sync::Arc; + use tempfile::TempDir; + use uuid::Uuid; + + struct DnsServer { + _storage: TempDir, + dns_server: internal_dns::dns_server::Server, + dropshot_server: + dropshot::HttpServer>, + } + + impl DnsServer { + async fn create(log: &Logger) -> Self { + let storage = + TempDir::new().expect("Failed to create temporary directory"); + + let db = Arc::new(sled::open(&storage.path()).unwrap()); + + let dns_server = { + let db = db.clone(); + let log = log.clone(); + let dns_config = internal_dns::dns_server::Config { + bind_address: "[::1]:0".to_string(), + zone: crate::names::DNS_ZONE.into(), + }; + + internal_dns::dns_server::run(log, db, dns_config) + .await + .unwrap() + }; + + let config = internal_dns::Config { + log: dropshot::ConfigLogging::StderrTerminal { + level: dropshot::ConfigLoggingLevel::Info, + }, + dropshot: dropshot::ConfigDropshot { + bind_address: "[::1]:0".parse().unwrap(), + request_body_max_bytes: 1024, + ..Default::default() + }, + data: internal_dns::dns_data::Config { + nmax_messages: 16, + storage_path: storage.path().to_string_lossy().into(), + }, + }; + + let dropshot_server = + internal_dns::start_dropshot_server(config, log.clone(), db) + .await + .unwrap(); + + Self { _storage: storage, dns_server, dropshot_server } + } + + fn dns_server_address(&self) -> SocketAddr { + self.dns_server.address + } + + fn dropshot_server_address(&self) -> SocketAddr { + self.dropshot_server.local_addr() + } + } + + // A test-only way to infer DNS addresses. + // + // Rather than inferring DNS server addresses from the rack subnet, + // they may be explicitly supplied. This results in easier-to-test code. + #[derive(Default)] + struct LocalAddressGetter { + addrs: Vec<(SocketAddr, SocketAddr)>, + } + + impl LocalAddressGetter { + fn add_dns_server( + &mut self, + dns_address: SocketAddr, + server_address: SocketAddr, + ) { + self.addrs.push((dns_address, server_address)); + } + } + + impl DnsAddressLookup for LocalAddressGetter { + fn dropshot_server_addrs(&self) -> Vec { + self.addrs + .iter() + .map(|(_dns_address, dropshot_address)| *dropshot_address) + .collect() + } + + fn dns_server_addrs(&self) -> Vec { + self.addrs + .iter() + .map(|(dns_address, _dropshot_address)| *dns_address) + .collect() + } + } + + // The resolver cannot look up IPs before records have been inserted. + #[tokio::test] + async fn lookup_nonexistent_record_fails() { + let logctx = test_setup_log("lookup_nonexistent_record_fails"); + let dns_server = DnsServer::create(&logctx.log).await; + + let mut address_getter = LocalAddressGetter::default(); + address_getter.add_dns_server( + dns_server.dns_server_address(), + dns_server.dropshot_server_address(), + ); + + let resolver = Resolver::new(&address_getter) + .expect("Error creating localhost resolver"); + + let err = resolver + .lookup_ip(SRV::Service(ServiceName::Cockroach)) + .await + .expect_err("Looking up non-existent service should fail"); + + let dns_error = match err { + ResolveError::Resolve(err) => err, + _ => panic!("Unexpected error: {err}"), + }; + assert!( + matches!( + dns_error.kind(), + trust_dns_resolver::error::ResolveErrorKind::NoRecordsFound { .. }, + ), + "Saw error: {dns_error}", + ); + logctx.cleanup_successful(); + } + + // Insert and retreive a single DNS record. + #[tokio::test] + async fn insert_and_lookup_one_record() { + let logctx = test_setup_log("insert_and_lookup_one_record"); + let dns_server = DnsServer::create(&logctx.log).await; + + let mut address_getter = LocalAddressGetter::default(); + address_getter.add_dns_server( + dns_server.dns_server_address(), + dns_server.dropshot_server_address(), + ); + + let resolver = Resolver::new(&address_getter) + .expect("Error creating localhost resolver"); + let updater = Updater::new(&address_getter, logctx.log.clone()); + + let records = HashMap::from([( + SRV::Service(ServiceName::Cockroach), + vec![( + AAAA::Zone(Uuid::new_v4()), + SocketAddrV6::new( + Ipv6Addr::from_str("ff::01").unwrap(), + 12345, + 0, + 0, + ), + )], + )]); + updater.insert_dns_records(&records).await.unwrap(); + + let ip = resolver + .lookup_ipv6(SRV::Service(ServiceName::Cockroach)) + .await + .expect("Should have been able to look up IP address"); + assert_eq!( + &ip, + records[&SRV::Service(ServiceName::Cockroach)][0].1.ip() + ); + + logctx.cleanup_successful(); + } + + // Insert multiple DNS records of different types. + #[tokio::test] + async fn insert_and_lookup_multiple_records() { + let logctx = test_setup_log("insert_and_lookup_multiple_records"); + let dns_server = DnsServer::create(&logctx.log).await; + + let mut address_getter = LocalAddressGetter::default(); + address_getter.add_dns_server( + dns_server.dns_server_address(), + dns_server.dropshot_server_address(), + ); + + let resolver = Resolver::new(&address_getter) + .expect("Error creating localhost resolver"); + let updater = Updater::new(&address_getter, logctx.log.clone()); + + let cockroach_addrs = [ + SocketAddrV6::new( + Ipv6Addr::from_str("ff::01").unwrap(), + 1111, + 0, + 0, + ), + SocketAddrV6::new( + Ipv6Addr::from_str("ff::02").unwrap(), + 2222, + 0, + 0, + ), + SocketAddrV6::new( + Ipv6Addr::from_str("ff::03").unwrap(), + 3333, + 0, + 0, + ), + ]; + let clickhouse_addr = SocketAddrV6::new( + Ipv6Addr::from_str("fe::01").unwrap(), + 4444, + 0, + 0, + ); + let crucible_addr = SocketAddrV6::new( + Ipv6Addr::from_str("fd::02").unwrap(), + 5555, + 0, + 0, + ); + + let srv_crdb = SRV::Service(ServiceName::Cockroach); + let srv_clickhouse = SRV::Service(ServiceName::Clickhouse); + let srv_backend = SRV::Backend(BackendName::Crucible, Uuid::new_v4()); + + let records = HashMap::from([ + // Three Cockroach services + ( + srv_crdb.clone(), + vec![ + (AAAA::Zone(Uuid::new_v4()), cockroach_addrs[0]), + (AAAA::Zone(Uuid::new_v4()), cockroach_addrs[1]), + (AAAA::Zone(Uuid::new_v4()), cockroach_addrs[2]), + ], + ), + // One Clickhouse service + ( + srv_clickhouse.clone(), + vec![(AAAA::Zone(Uuid::new_v4()), clickhouse_addr)], + ), + // One Backend service + ( + srv_backend.clone(), + vec![(AAAA::Zone(Uuid::new_v4()), crucible_addr)], + ), + ]); + updater.insert_dns_records(&records).await.unwrap(); + + // Look up Cockroach + let ip = resolver + .lookup_ipv6(SRV::Service(ServiceName::Cockroach)) + .await + .expect("Should have been able to look up IP address"); + assert!(cockroach_addrs.iter().any(|addr| addr.ip() == &ip)); + + // Look up Clickhouse + let ip = resolver + .lookup_ipv6(SRV::Service(ServiceName::Clickhouse)) + .await + .expect("Should have been able to look up IP address"); + assert_eq!(&ip, clickhouse_addr.ip()); + + // Look up Backend Service + let ip = resolver + .lookup_ipv6(srv_backend) + .await + .expect("Should have been able to look up IP address"); + assert_eq!(&ip, crucible_addr.ip()); + + // If we remove the AAAA records for two of the CRDB services, + // only one will remain. + updater + .dns_records_delete(&vec![ + DnsRecordKey { name: records[&srv_crdb][0].0.to_string() }, + DnsRecordKey { name: records[&srv_crdb][1].0.to_string() }, + ]) + .await + .expect("Should have been able to delete record"); + let ip = resolver + .lookup_ipv6(SRV::Service(ServiceName::Cockroach)) + .await + .expect("Should have been able to look up IP address"); + assert_eq!(&ip, cockroach_addrs[2].ip()); + + logctx.cleanup_successful(); + } + + #[tokio::test] + async fn update_record() { + let logctx = test_setup_log("update_record"); + let dns_server = DnsServer::create(&logctx.log).await; + + let mut address_getter = LocalAddressGetter::default(); + address_getter.add_dns_server( + dns_server.dns_server_address(), + dns_server.dropshot_server_address(), + ); + + let resolver = Resolver::new(&address_getter) + .expect("Error creating localhost resolver"); + let updater = Updater::new(&address_getter, logctx.log.clone()); + + // Insert a record, observe that it exists. + let srv_crdb = SRV::Service(ServiceName::Cockroach); + let mut records = HashMap::from([( + srv_crdb.clone(), + vec![( + AAAA::Zone(Uuid::new_v4()), + SocketAddrV6::new( + Ipv6Addr::from_str("ff::01").unwrap(), + 12345, + 0, + 0, + ), + )], + )]); + updater.insert_dns_records(&records).await.unwrap(); + let ip = resolver + .lookup_ipv6(SRV::Service(ServiceName::Cockroach)) + .await + .expect("Should have been able to look up IP address"); + assert_eq!(&ip, records[&srv_crdb][0].1.ip()); + + // If we insert the same record with a new address, it should be + // updated. + records.get_mut(&srv_crdb).unwrap()[0].1 = SocketAddrV6::new( + Ipv6Addr::from_str("ee::02").unwrap(), + 54321, + 0, + 0, + ); + updater.insert_dns_records(&records).await.unwrap(); + let ip = resolver + .lookup_ipv6(SRV::Service(ServiceName::Cockroach)) + .await + .expect("Should have been able to look up IP address"); + assert_eq!(&ip, records[&srv_crdb][0].1.ip()); + + logctx.cleanup_successful(); + } +} diff --git a/internal-dns-client/src/names.rs b/internal-dns-client/src/names.rs new file mode 100644 index 00000000000..1b633f915e1 --- /dev/null +++ b/internal-dns-client/src/names.rs @@ -0,0 +1,152 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/. + +//! Naming scheme for Internal DNS names (RFD 248). + +use std::fmt; +use uuid::Uuid; + +pub(crate) const DNS_ZONE: &str = "control-plane.oxide.internal"; + +/// Names for services where backends are interchangeable. +#[derive(Clone, Debug, Hash, Eq, PartialEq, PartialOrd)] +pub enum ServiceName { + Clickhouse, + Cockroach, + InternalDNS, + Nexus, + Oximeter, +} + +impl fmt::Display for ServiceName { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match &self { + ServiceName::Clickhouse => write!(f, "clickhouse"), + ServiceName::Cockroach => write!(f, "cockroach"), + ServiceName::InternalDNS => write!(f, "internalDNS"), + ServiceName::Nexus => write!(f, "nexus"), + ServiceName::Oximeter => write!(f, "oximeter"), + } + } +} + +/// Names for services where backends are not interchangeable. +#[derive(Clone, Debug, Hash, Eq, PartialEq, PartialOrd)] +pub enum BackendName { + Crucible, + SledAgent, +} + +impl fmt::Display for BackendName { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match &self { + BackendName::Crucible => write!(f, "crucible"), + BackendName::SledAgent => write!(f, "sledagent"), + } + } +} + +#[derive(Clone, Debug, Hash, Eq, PartialEq, PartialOrd)] +pub enum SRV { + /// A service identified and accessed by name, such as "nexus", "CRDB", etc. + /// + /// This is used in cases where services are interchangeable. + Service(ServiceName), + + /// A service identified by name and a unique identifier. + /// + /// This is used in cases where services are not interchangeable, such as + /// for the Sled agent. + Backend(BackendName, Uuid), +} + +impl fmt::Display for SRV { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match &self { + SRV::Service(name) => { + write!(f, "_{}._tcp.{}", name, DNS_ZONE) + } + SRV::Backend(name, id) => { + write!(f, "_{}._tcp.{}.{}", name, id, DNS_ZONE) + } + } + } +} + +#[derive(Clone, Debug, PartialEq, PartialOrd)] +pub enum AAAA { + /// Identifies an AAAA record for a sled. + Sled(Uuid), + + /// Identifies an AAAA record for a zone within a sled. + Zone(Uuid), +} + +impl fmt::Display for AAAA { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match &self { + AAAA::Sled(id) => { + write!(f, "{}.sled.{}", id, DNS_ZONE) + } + AAAA::Zone(id) => { + write!(f, "{}.host.{}", id, DNS_ZONE) + } + } + } +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn display_srv_service() { + assert_eq!( + SRV::Service(ServiceName::Clickhouse).to_string(), + "_clickhouse._tcp.control-plane.oxide.internal", + ); + assert_eq!( + SRV::Service(ServiceName::Cockroach).to_string(), + "_cockroach._tcp.control-plane.oxide.internal", + ); + assert_eq!( + SRV::Service(ServiceName::InternalDNS).to_string(), + "_internalDNS._tcp.control-plane.oxide.internal", + ); + assert_eq!( + SRV::Service(ServiceName::Nexus).to_string(), + "_nexus._tcp.control-plane.oxide.internal", + ); + assert_eq!( + SRV::Service(ServiceName::Oximeter).to_string(), + "_oximeter._tcp.control-plane.oxide.internal", + ); + } + + #[test] + fn display_srv_backend() { + let uuid = Uuid::nil(); + assert_eq!( + SRV::Backend(BackendName::Crucible, uuid).to_string(), + "_crucible._tcp.00000000-0000-0000-0000-000000000000.control-plane.oxide.internal", + ); + assert_eq!( + SRV::Backend(BackendName::SledAgent, uuid).to_string(), + "_sledagent._tcp.00000000-0000-0000-0000-000000000000.control-plane.oxide.internal", + ); + } + + #[test] + fn display_aaaa() { + let uuid = Uuid::nil(); + assert_eq!( + AAAA::Sled(uuid).to_string(), + "00000000-0000-0000-0000-000000000000.sled.control-plane.oxide.internal", + ); + assert_eq!( + AAAA::Zone(uuid).to_string(), + "00000000-0000-0000-0000-000000000000.host.control-plane.oxide.internal", + ); + } +}