generated from ipfs/ipfs-repository-template
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #46 from ipfs/feat/improved-http-limits-and-dns-cache
feat: improved http limits and dns cache
- Loading branch information
Showing
7 changed files
with
164 additions
and
62 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,83 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "crypto/tls" | ||
| "net/http" | ||
| "net/url" | ||
| "time" | ||
|
|
||
| "github.com/filecoin-saturn/caboose" | ||
| blockstore "github.com/ipfs/go-ipfs-blockstore" | ||
| ) | ||
|
|
||
| func newCabooseBlockStore(orchestrator, loggingEndpoint string, cdns *cachedDNS) (blockstore.Blockstore, error) { | ||
| var ( | ||
| orchURL *url.URL | ||
| loggURL *url.URL | ||
| err error | ||
| ) | ||
|
|
||
| if orchestrator != "" { | ||
| orchURL, err = url.Parse(orchestrator) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| } | ||
|
|
||
| if loggingEndpoint != "" { | ||
| loggURL, err = url.Parse(loggingEndpoint) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| } | ||
|
|
||
| saturnServiceClient := &http.Client{ | ||
| Timeout: caboose.DefaultSaturnRequestTimeout, | ||
| Transport: &withUserAgent{ | ||
| RoundTripper: &http.Transport{ | ||
| DialContext: cdns.dialWithCachedDNS, | ||
| }, | ||
| }, | ||
| } | ||
|
|
||
| saturnRetrievalClient := &http.Client{ | ||
| Timeout: caboose.DefaultSaturnRequestTimeout, | ||
| Transport: &withUserAgent{ | ||
| RoundTripper: &http.Transport{ | ||
| // Increasing concurrency defaults from http.DefaultTransport | ||
| MaxIdleConns: 1000, | ||
| MaxConnsPerHost: 100, | ||
| MaxIdleConnsPerHost: 100, | ||
| IdleConnTimeout: 90 * time.Second, | ||
|
|
||
| DialContext: cdns.dialWithCachedDNS, | ||
|
|
||
| // Saturn Weltschmerz | ||
| TLSClientConfig: &tls.Config{ | ||
| // Saturn use TLS in controversial ways, which sooner or | ||
| // later will force them to switch away to different domain | ||
| // name and certs, in which case they will break us. Since | ||
| // we are fetching raw blocks and dont really care about | ||
| // TLS cert being legitimate, let's disable verification | ||
| // to save CPU and to avoid catastrophic failure when | ||
| // Saturn L1s suddenly switch to certs with different DNS name. | ||
| InsecureSkipVerify: true, | ||
| // ServerName: "strn.pl", | ||
| }, | ||
| }, | ||
| }, | ||
| } | ||
|
|
||
| return caboose.NewCaboose(&caboose.Config{ | ||
| OrchestratorEndpoint: orchURL, | ||
| OrchestratorClient: saturnServiceClient, | ||
|
|
||
| LoggingEndpoint: *loggURL, | ||
| LoggingClient: saturnServiceClient, | ||
| LoggingInterval: 5 * time.Second, | ||
|
|
||
| DoValidation: true, | ||
| PoolRefresh: 5 * time.Minute, | ||
| SaturnClient: saturnRetrievalClient, | ||
| }) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,68 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "context" | ||
| "net" | ||
| "time" | ||
|
|
||
| "github.com/rs/dnscache" | ||
| ) | ||
|
|
||
| // How often should we check for successful updates to cached entries | ||
| const dnsCacheRefreshInterval = 5 * time.Minute | ||
|
|
||
| // Local DNS cache because in this world things are ephemeral | ||
| type cachedDNS struct { | ||
| resolver *dnscache.Resolver | ||
| refresher *time.Ticker | ||
| } | ||
|
|
||
| func newCachedDNS(refreshInterval time.Duration) *cachedDNS { | ||
| cache := &cachedDNS{ | ||
| resolver: &dnscache.Resolver{}, | ||
| refresher: time.NewTicker(refreshInterval), | ||
| } | ||
|
|
||
| // Configure DNS cache to not remove stale records to protect gateway from | ||
| // catastrophic failures like https://github.com/ipfs/bifrost-gateway/issues/34 | ||
| options := dnscache.ResolverRefreshOptions{} | ||
| options.ClearUnused = false | ||
| options.PersistOnFailure = true | ||
|
|
||
| // Every refreshInterval we check for updates, but if there is | ||
| // none, or if domain disappears, we keep the last cached version | ||
| go func(cdns *cachedDNS) { | ||
| defer cdns.refresher.Stop() | ||
| for range cdns.refresher.C { | ||
| cdns.resolver.RefreshWithOptions(options) | ||
| } | ||
| }(cache) | ||
|
|
||
| return cache | ||
| } | ||
|
|
||
| // dialWithCachedDNS implements DialContext that uses cachedDNS | ||
| func (cdns *cachedDNS) dialWithCachedDNS(ctx context.Context, network string, addr string) (conn net.Conn, err error) { | ||
| host, port, err := net.SplitHostPort(addr) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| ips, err := cdns.resolver.LookupHost(ctx, host) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| // Try all IPs returned by DNS | ||
| for _, ip := range ips { | ||
| var dialer net.Dialer | ||
| conn, err = dialer.DialContext(ctx, network, net.JoinHostPort(ip, port)) | ||
| if err == nil { | ||
| break | ||
| } | ||
| } | ||
| return | ||
| } | ||
|
|
||
| func (cdns *cachedDNS) Close() error { | ||
| cdns.refresher.Stop() | ||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters