Out of memory errors for libcurl ≥ 7.71.0

[anthonynorth]

Out of memory errors are being thrown from libcurl, when request URL ⪆ 1MB and libcurl ≥ 7.71.0. Could be a breaking API change in curl?

Or it could be a bug, athough that seems less likely. I couldn't find any open "out of memory" issues in curl, but I did find this old issue which may be relevant: curl/curl/issues/6681.

Known breaking environments

I've been able to reproduce this behaviour in:

OS	R	{curl}	libcurl
Windows 10	4.3.1	5.1.0	8.3.0
Windows 11	4.2.2	5.1.0	7.84.0
Ubuntu 2204 (AWS)	4.3.1	5.1.0	7.81.0
Ubuntu 2204 (Desktop)	4.3.1	5.1.0	7.71.0, 7.81.0, 8.4.0 (see reprex below)

Reprex

I've run the same test -- make a dumb request with a url 1MB in size -- with 4 versions of libcurl and latest {curl} R package. I've built 7.70.0, 7.71.0 and 8.4.0 from source; 7.81.0 is the default libcurl on my test environment.

My 7.70.0 reprex is the only environment that successfully sent a request (with a 4xx response, which is expected).

The URL in the reprex is large because of the querystring. Problem still occurs if scheme + authority + path ⪆ 1MB. http / https is also immaterial. curl_interrupt = TRUE/FALSE is also immaterial.

The largest url that succeeds in all environments was 1048443 bytes.

7.70.0

packageVersion("curl")
#> [1] '5.1.0'
curl::curl_version()
#> $version
#> [1] "7.70.0"
#> 
#> $ssl_version
#> [1] "OpenSSL/3.0.2"
#> 
#> $libz_version
#> [1] "1.2.11"
#> 
#> $libssh_version
#> [1] NA
#> 
#> $libidn_version
#> [1] NA
#> 
#> $host
#> [1] "x86_64-pc-linux-gnu"
#> 
#> $protocols
#>  [1] "dict"   "file"   "ftp"    "ftps"   "gopher" "http"   "https"  "imap"  
#>  [9] "imaps"  "pop3"   "pop3s"  "rtsp"   "smb"    "smbs"   "smtp"   "smtps" 
#> [17] "telnet" "tftp"  
#> 
#> $ipv6
#> [1] TRUE
#> 
#> $http2
#> [1] FALSE
#> 
#> $idn
#> [1] FALSE

# nchar ⪆ 1MB -> Out of memory
url <- paste0(
  c("https://fakerapi.it/api/v1/custom?", rep.int(LETTERS, 50000)),
  collapse = ""
)

url <- substr(url, 1, 2**20)
nchar(url)
#> [1] 1048576

res <- curl::curl_fetch_memory(url)

^{Created on 2023-11-04 with reprex v2.0.2}

7.71.0

packageVersion("curl")
#> [1] '5.1.0'
curl::curl_version()
#> $version
#> [1] "7.71.0"
#> 
#> $ssl_version
#> [1] "OpenSSL/3.0.2"
#> 
#> $libz_version
#> [1] "1.2.11"
#> 
#> $libssh_version
#> [1] NA
#> 
#> $libidn_version
#> [1] NA
#> 
#> $host
#> [1] "x86_64-pc-linux-gnu"
#> 
#> $protocols
#>  [1] "dict"   "file"   "ftp"    "ftps"   "gopher" "http"   "https"  "imap"  
#>  [9] "imaps"  "pop3"   "pop3s"  "rtsp"   "smb"    "smbs"   "smtp"   "smtps" 
#> [17] "telnet" "tftp"  
#> 
#> $ipv6
#> [1] TRUE
#> 
#> $http2
#> [1] FALSE
#> 
#> $idn
#> [1] FALSE

# nchar ⪆ 1MB -> Out of memory
url <- paste0(
  c("https://fakerapi.it/api/v1/custom?", rep.int(LETTERS, 50000)),
  collapse = ""
)

url <- substr(url, 1, 2**20)
nchar(url)
#> [1] 1048576

res <- curl::curl_fetch_memory(url)
#> Error in curl::curl_fetch_memory(url): Out of memory

^{Created on 2023-11-04 with reprex v2.0.2}

7.81.0

packageVersion("curl")
#> [1] '5.1.0'
curl::curl_version()
#> $version
#> [1] "7.81.0"
#> 
#> $ssl_version
#> [1] "OpenSSL/3.0.2"
#> 
#> $libz_version
#> [1] "1.2.11"
#> 
#> $libssh_version
#> [1] "libssh/0.9.6/openssl/zlib"
#> 
#> $libidn_version
#> [1] "2.3.2"
#> 
#> $host
#> [1] "x86_64-pc-linux-gnu"
#> 
#> $protocols
#>  [1] "dict"    "file"    "ftp"     "ftps"    "gopher"  "gophers" "http"   
#>  [8] "https"   "imap"    "imaps"   "ldap"    "ldaps"   "mqtt"    "pop3"   
#> [15] "pop3s"   "rtmp"    "rtsp"    "scp"     "sftp"    "smb"     "smbs"   
#> [22] "smtp"    "smtps"   "telnet"  "tftp"   
#> 
#> $ipv6
#> [1] TRUE
#> 
#> $http2
#> [1] TRUE
#> 
#> $idn
#> [1] TRUE

# nchar ⪆ 1MB -> Out of memory
url <- paste0(
  c("https://fakerapi.it/api/v1/custom?", rep.int(LETTERS, 50000)),
  collapse = ""
)

url <- substr(url, 1, 2**20)
nchar(url)
#> [1] 1048576

res <- curl::curl_fetch_memory(url)
#> Error in curl::curl_fetch_memory(url): Out of memory

^{Created on 2023-11-04 with reprex v2.0.2}

8.4.0

packageVersion("curl")
#> [1] '5.1.0'
curl::curl_version()
#> $version
#> [1] "8.4.0"
#> 
#> $ssl_version
#> [1] "OpenSSL/3.0.2"
#> 
#> $libz_version
#> [1] "1.2.11"
#> 
#> $libssh_version
#> [1] NA
#> 
#> $libidn_version
#> [1] NA
#> 
#> $host
#> [1] "x86_64-pc-linux-gnu"
#> 
#> $protocols
#>  [1] "dict"    "file"    "ftp"     "ftps"    "gopher"  "gophers" "http"   
#>  [8] "https"   "imap"    "imaps"   "mqtt"    "pop3"    "pop3s"   "rtsp"   
#> [15] "smb"     "smbs"    "smtp"    "smtps"   "telnet"  "tftp"   
#> 
#> $ipv6
#> [1] TRUE
#> 
#> $http2
#> [1] FALSE
#> 
#> $idn
#> [1] FALSE
≥

# nchar ⪆ 1MB -> Out of memory
url <- paste0(
  c("https://fakerapi.it/api/v1/custom?", rep.int(LETTERS, 50000)),
  collapse = ""
)

url <- substr(url, 1, 2**20)
nchar(url)
#> [1] 1048576

res <- curl::curl_fetch_memory(url)
#> Error in curl::curl_fetch_memory(url): Out of memory

^{Created on 2023-11-04 with reprex v2.0.2}

[jeroen]

Thanks for reporting. I think this is an issue in libcurl. You can follow up here: curl/curl#12268