diff --git a/docs/cmdline-opts/ipfs-gateway.d b/docs/cmdline-opts/ipfs-gateway.d index 5d5f8b2d42effb..e6845b327a6b73 100644 --- a/docs/cmdline-opts/ipfs-gateway.d +++ b/docs/cmdline-opts/ipfs-gateway.d @@ -9,36 +9,24 @@ Category: ipfs Example: --ipfs-gateway $URL ipfs:// Multi: single --- -Specifies which gateway to use for IPFS and IPNS URLs. -Not specifying this argument will let cURL try to automatically -check if IPFS_GATEWAY environment variable is set, -or if ~/.ipfs/gateway plain text file exists. +Specify which gateway to use for IPFS and IPNS URLs. Not specifying this will +instead make curl check if the IPFS_GATEWAY environment variable is set, or if +a ~/.ipfs/gateway file holding the gateway URL exists. -If you run a local IPFS node, this gateway is by default -available under http://localhost:8080. A full example URL would -look like: +If you run a local IPFS node, this gateway is by default available under +http://localhost:8080. A full example URL would look like: curl --ipfs-gateway http://localhost:8080 ipfs://bafybeigagd5nmnn2iys2f3doro7ydrevyr2mzarwidgadawmamiteydbzi - -You can also specify publicly available gateways. One such -gateway is https://ipfs.io. A full example url would look like: - - curl --ipfs-gateway https://ipfs.io ipfs://bafybeigagd5nmnn2iys2f3doro7ydrevyr2mzarwidgadawmamiteydbzi - - -There are many public IPFS gateways. As a starting point to find -one that works for your case, consult this page: +There are many public IPFS gateways. See for example: https://ipfs.github.io/public-gateway-checker/ - -A word of caution! When you opt to go for a remote gateway you should -be aware that you completely trust the gateway. This is fine in local gateways -as you host it yourself. With remote gateways there could potentially be -a malicious actor returning you data that does not match the request you made, -inspect or even interfere with the request. You won't notice this when using cURL. -A mitigation could be to go for a "trustless" gateway. This means you -locally verify that the data. Consult the docs page on trusted vs trustless: +WARNING: If you opt to go for a remote gateway you should be aware that you +completely trust the gateway. This is fine in local gateways as you host it +yourself. With remote gateways there could potentially be a malicious actor +returning you data that does not match the request you made, inspect or even +interfere with the request. You will not notice this when using curl. A +mitigation could be to go for a "trustless" gateway. This means you locally +verify that the data. Consult the docs page on trusted vs trustless: https://docs.ipfs.tech/reference/http/gateway/#trusted-vs-trustless - diff --git a/src/Makefile.inc b/src/Makefile.inc index 25389353652b0a..c1d202a06192e6 100644 --- a/src/Makefile.inc +++ b/src/Makefile.inc @@ -79,6 +79,7 @@ CURL_CFILES = \ tool_help.c \ tool_helpers.c \ tool_hugehelp.c \ + tool_ipfs.c \ tool_libinfo.c \ tool_listhelp.c \ tool_main.c \ @@ -122,6 +123,7 @@ CURL_HFILES = \ tool_help.h \ tool_helpers.h \ tool_hugehelp.h \ + tool_ipfs.h \ tool_libinfo.h \ tool_main.h \ tool_msgs.h \ diff --git a/src/tool_ipfs.c b/src/tool_ipfs.c new file mode 100644 index 00000000000000..271fff2cd8b3a3 --- /dev/null +++ b/src/tool_ipfs.c @@ -0,0 +1,295 @@ +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) Daniel Stenberg, , et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at https://curl.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + * SPDX-License-Identifier: curl + * + ***************************************************************************/ +#include "tool_setup.h" + +#define ENABLE_CURLX_PRINTF +/* use our own printf() functions */ +#include "curlx.h" +#include "dynbuf.h" + +#include "tool_cfgable.h" +#include "tool_msgs.h" +#include "tool_ipfs.h" + +#include "memdebug.h" /* keep this as LAST include */ + +/* ensure input ends in slash */ +static CURLcode ensure_trailing_slash(char **input) +{ + if(*input && **input) { + size_t len = strlen(*input); + if(((*input)[len - 1] != '/')) { + struct curlx_dynbuf dyn; + curlx_dyn_init(&dyn, len + 2); + + if(curlx_dyn_addn(&dyn, *input, len)) { + Curl_safefree(*input); + return CURLE_OUT_OF_MEMORY; + } + + Curl_safefree(*input); + + if(curlx_dyn_addn(&dyn, "/", 1)) + return CURLE_OUT_OF_MEMORY; + + *input = curlx_dyn_ptr(&dyn); + } + } + + return CURLE_OK; +} + +static char *ipfs_gateway(void) +{ + char *ipfs_path = NULL; + char *gateway_composed_file_path = NULL; + FILE *gateway_file = NULL; + char *gateway = curlx_getenv("IPFS_GATEWAY"); + + /* Gateway is found from environment variable. */ + if(gateway) { + if(ensure_trailing_slash(&gateway)) + goto fail; + return gateway; + } + + /* Try to find the gateway in the IPFS data folder. */ + ipfs_path = curlx_getenv("IPFS_PATH"); + + if(!ipfs_path) { + char *home = curlx_getenv("HOME"); + if(home && *home) + ipfs_path = aprintf("%s/.ipfs/", home); + /* fallback to "~/.ipfs", as that's the default location. */ + + Curl_safefree(home); + } + + if(!ipfs_path || ensure_trailing_slash(&ipfs_path)) + goto fail; + + gateway_composed_file_path = aprintf("%sgateway", ipfs_path); + + if(!gateway_composed_file_path) + goto fail; + + gateway_file = fopen(gateway_composed_file_path, FOPEN_READTEXT); + Curl_safefree(gateway_composed_file_path); + + if(gateway_file) { + int c; + struct curlx_dynbuf dyn; + curlx_dyn_init(&dyn, MAX_GATEWAY_URL_LEN); + + /* get the first line of the gateway file, ignore the rest */ + while((c = getc(gateway_file)) != EOF && c != '\n' && c != '\r') { + if(curlx_dyn_addn(&dyn, &c, 1)) + goto fail; + } + + fclose(gateway_file); + gateway_file = NULL; + + if(curlx_dyn_len(&dyn)) + gateway = curlx_dyn_ptr(&dyn); + + if(gateway) + ensure_trailing_slash(&gateway); + + if(!gateway) + goto fail; + + Curl_safefree(ipfs_path); + + return gateway; + } +fail: + if(gateway_file) + fclose(gateway_file); + Curl_safefree(gateway); + Curl_safefree(ipfs_path); + return NULL; +} + +/* + * Rewrite ipfs:// and ipns:// to a HTTP(S) + * URL that can be handled by an IPFS gateway. + */ +CURLcode ipfs_url_rewrite(CURLU *uh, const char *protocol, char **url, + struct OperationConfig *config) +{ + CURLcode result = CURLE_URL_MALFORMAT; + CURLUcode getResult; + char *gateway = NULL; + char *gwhost = NULL; + char *gwpath = NULL; + char *gwquery = NULL; + char *gwscheme = NULL; + char *gwport = NULL; + char *inputpath = NULL; + char *cid = NULL; + char *pathbuffer = NULL; + char *cloneurl; + CURLU *gatewayurl = curl_url(); + + if(!gatewayurl) { + result = CURLE_FAILED_INIT; + goto clean; + } + + getResult = curl_url_get(uh, CURLUPART_HOST, &cid, CURLU_URLDECODE); + if(getResult || !cid) + goto clean; + + /* We might have a --ipfs-gateway argument. Check it first and use it. Error + * if we do have something but if it's an invalid url. + */ + if(config->ipfs_gateway) { + /* ensure the gateway ends in a trailing / */ + if(ensure_trailing_slash(&config->ipfs_gateway) != CURLE_OK) { + result = CURLE_OUT_OF_MEMORY; + goto clean; + } + + if(!curl_url_set(gatewayurl, CURLUPART_URL, config->ipfs_gateway, + CURLU_GUESS_SCHEME)) { + gateway = strdup(config->ipfs_gateway); + if(!gateway) { + result = CURLE_URL_MALFORMAT; + goto clean; + } + + } + else { + result = CURLE_BAD_FUNCTION_ARGUMENT; + goto clean; + } + } + else { + /* this is ensured to end in a trailing / within ipfs_gateway() */ + gateway = ipfs_gateway(); + if(!gateway) { + result = CURLE_FILE_COULDNT_READ_FILE; + goto clean; + } + + if(curl_url_set(gatewayurl, CURLUPART_URL, gateway, 0)) { + result = CURLE_URL_MALFORMAT; + goto clean; + } + } + + /* check for unsupported gateway parts */ + if(curl_url_get(gatewayurl, CURLUPART_QUERY, &gwquery, 0) + != CURLUE_NO_QUERY) { + result = CURLE_URL_MALFORMAT; + goto clean; + } + + /* get gateway parts */ + if(curl_url_get(gatewayurl, CURLUPART_HOST, + &gwhost, CURLU_URLDECODE)) { + goto clean; + } + + if(curl_url_get(gatewayurl, CURLUPART_SCHEME, + &gwscheme, CURLU_URLDECODE)) { + goto clean; + } + + curl_url_get(gatewayurl, CURLUPART_PORT, &gwport, CURLU_URLDECODE); + curl_url_get(gatewayurl, CURLUPART_PATH, &gwpath, CURLU_URLDECODE); + + /* get the path from user input */ + curl_url_get(uh, CURLUPART_PATH, &inputpath, CURLU_URLDECODE); + /* inputpath might be NULL or a valid pointer now */ + + /* set gateway parts in input url */ + if(curl_url_set(uh, CURLUPART_SCHEME, gwscheme, CURLU_URLENCODE) || + curl_url_set(uh, CURLUPART_HOST, gwhost, CURLU_URLENCODE) || + curl_url_set(uh, CURLUPART_PORT, gwport, CURLU_URLENCODE)) + goto clean; + + /* if the input path is just a slash, clear it */ + if(inputpath && (inputpath[0] == '/') && !inputpath[1]) + *inputpath = '\0'; + + /* ensure the gateway path ends with a trailing slash */ + ensure_trailing_slash(&gwpath); + + pathbuffer = aprintf("%s%s/%s%s", gwpath, protocol, cid, + inputpath ? inputpath : ""); + if(!pathbuffer) { + goto clean; + } + + if(curl_url_set(uh, CURLUPART_PATH, pathbuffer, CURLU_URLENCODE)) { + goto clean; + } + + /* Free whatever it has now, rewriting is next */ + Curl_safefree(*url); + + if(curl_url_get(uh, CURLUPART_URL, &cloneurl, CURLU_URLENCODE)) { + goto clean; + } + /* we need to strdup the URL so that we can call free() on it later */ + *url = strdup(cloneurl); + curl_free(cloneurl); + if(!*url) + goto clean; + + result = CURLE_OK; + +clean: + free(gateway); + curl_free(gwhost); + curl_free(gwpath); + curl_free(gwquery); + curl_free(inputpath); + curl_free(gwscheme); + curl_free(gwport); + curl_free(cid); + curl_free(pathbuffer); + curl_url_cleanup(gatewayurl); + { + const char *msg = NULL; + switch(result) { + case CURLE_URL_MALFORMAT: + msg = "malformed target URL"; + break; + case CURLE_FILE_COULDNT_READ_FILE: + msg = "IPFS automatic gateway detection failed"; + break; + case CURLE_BAD_FUNCTION_ARGUMENT: + msg = "--ipfs-gateway was given a malformed URL"; + break; + default: + break; + } + if(msg) + helpf(tool_stderr, msg); + } + return result; +} diff --git a/src/tool_ipfs.h b/src/tool_ipfs.h new file mode 100644 index 00000000000000..9c8a83e3b34ac0 --- /dev/null +++ b/src/tool_ipfs.h @@ -0,0 +1,33 @@ +#ifndef HEADER_CURL_TOOL_IPFS_H +#define HEADER_CURL_TOOL_IPFS_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) Daniel Stenberg, , et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at https://curl.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + * SPDX-License-Identifier: curl + * + ***************************************************************************/ +#include "tool_setup.h" + +#define MAX_GATEWAY_URL_LEN 10000 + +CURLcode ipfs_url_rewrite(CURLU *uh, const char *protocol, char **url, + struct OperationConfig *config); + +#endif /* HEADER_CURL_TOOL_IPFS_H */ diff --git a/src/tool_operate.c b/src/tool_operate.c index 88be9aaad727cd..47009c75efe2a5 100644 --- a/src/tool_operate.c +++ b/src/tool_operate.c @@ -81,6 +81,7 @@ #include "tool_help.h" #include "tool_hugehelp.h" #include "tool_progress.h" +#include "tool_ipfs.h" #include "dynbuf.h" #include "memdebug.h" /* keep this as LAST include */ @@ -697,296 +698,6 @@ static CURLcode post_per_transfer(struct GlobalConfig *global, return result; } -/* helper function to ensure input ends in val_to_ensure */ -static CURLcode ensure_trailing(char **input, const char val_to_ensure) -{ - if(*input && **input) { - size_t len = strlen(*input); - if(((*input)[len - 1] != val_to_ensure)) { - struct curlx_dynbuf dyn; - curlx_dyn_init(&dyn, len + 2); - - if(curlx_dyn_addn(&dyn, *input, len)) { - Curl_safefree(*input); - return CURLE_OUT_OF_MEMORY; - } - - Curl_safefree(*input); - - if(curlx_dyn_addn(&dyn, &val_to_ensure, 1)) - return CURLE_OUT_OF_MEMORY; - - *input = curlx_dyn_ptr(&dyn); - } - } - - return CURLE_OK; -} - -static char *ipfs_gateway(void) -{ - char *gateway = NULL; - char *ipfs_path = NULL; - char *gateway_composed_file_path = NULL; - FILE *gateway_file = NULL; - - gateway = curlx_getenv("IPFS_GATEWAY"); - - /* Gateway is found from environment variable. */ - if(gateway) { - if(ensure_trailing(&gateway, '/')) { - Curl_safefree(gateway); - return NULL; - } - return gateway; - } - - /* Try to find the gateway in the IPFS data folder. */ - ipfs_path = curlx_getenv("IPFS_PATH"); - - if(!ipfs_path) { - char *home = curlx_getenv("HOME"); - if(home && *home) - ipfs_path = aprintf("%s/.ipfs/", home); - /* fallback to "~/.ipfs", as that's the default location. */ - - Curl_safefree(home); - } - - if(!ipfs_path) { - Curl_safefree(gateway); - Curl_safefree(ipfs_path); - return NULL; - } - else { - if(ensure_trailing(&ipfs_path, '/')) { - Curl_safefree(gateway); - Curl_safefree(ipfs_path); - return NULL; - } - } - - gateway_composed_file_path = aprintf("%sgateway", ipfs_path); - - if(!gateway_composed_file_path) { - Curl_safefree(gateway); - Curl_safefree(ipfs_path); - return NULL; - } - - gateway_file = fopen(gateway_composed_file_path, FOPEN_READTEXT); - Curl_safefree(gateway_composed_file_path); - - if(gateway_file) { - int c; - struct curlx_dynbuf dyn; - curlx_dyn_init(&dyn, INT_MAX); - - /* get the first line of the gateway file, ignore the rest */ - while((c = getc(gateway_file)) != EOF && c != '\n' && c != '\r') { - if(curlx_dyn_addn(&dyn, &c, 1)) - break; - } - - if(gateway_file) - fclose(gateway_file); - - if(curlx_dyn_len(&dyn) > 0) - gateway = curlx_dyn_ptr(&dyn); - - if(gateway) - ensure_trailing(&gateway, '/'); - - if(!gateway) { - Curl_safefree(ipfs_path); - return NULL; - } - - Curl_safefree(ipfs_path); - - return gateway; - } - - Curl_safefree(gateway); - Curl_safefree(ipfs_path); - return NULL; -} - -/* - * Rewrite ipfs:// and ipns:// to a HTTP(S) - * URL that can be handled by an IPFS gateway. - */ -static CURLcode ipfs_url_rewrite(CURLU *uh, const char *protocol, char **url, - struct OperationConfig *config) -{ - CURLcode result = CURLE_URL_MALFORMAT; - CURLUcode getResult; - char *gateway = NULL; - char *gatewayhost = NULL; - char *gatewaypath = NULL; - char *gatewayquery = NULL; - char *inputpath = NULL; - char *gatewayscheme = NULL; - char *gatewayport = NULL; - char *cid = NULL; - char *pathbuffer = NULL; - CURLU *gatewaysurl = curl_url(); - - if(!gatewaysurl) { - result = CURLE_FAILED_INIT; - goto clean; - } - - getResult = curl_url_get(uh, CURLUPART_HOST, &cid, CURLU_URLDECODE); - - if(getResult) { - goto clean; - } - - if(!cid) { - goto clean; - } - - /* We might have a --ipfs-gateway argument. Check it first and use it. Error - * if we do have something but if it's an invalid url. - */ - if(config->ipfs_gateway) { - /* ensure the gateway ends in a trailing / */ - if(ensure_trailing(&config->ipfs_gateway, '/') != CURLE_OK) { - result = CURLE_OUT_OF_MEMORY; - goto clean; - } - - if(!curl_url_set(gatewaysurl, CURLUPART_URL, config->ipfs_gateway, - CURLU_GUESS_SCHEME)) { - gateway = strdup(config->ipfs_gateway); - if(!gateway) { - result = CURLE_URL_MALFORMAT; - goto clean; - } - - } - else { - result = CURLE_BAD_FUNCTION_ARGUMENT; - goto clean; - } - } - else { - /* this is ensured to end in a trailing / within ipfs_gateway() */ - gateway = ipfs_gateway(); - if(!gateway) { - result = CURLE_FILE_COULDNT_READ_FILE; - goto clean; - } - - if(curl_url_set(gatewaysurl, CURLUPART_URL, gateway, 0)) { - result = CURLE_URL_MALFORMAT; - goto clean; - } - } - - /* check for unsupported gateway parts */ - if(curl_url_get(gatewaysurl, CURLUPART_QUERY, &gatewayquery, 0) - != CURLUE_NO_QUERY) { - result = CURLE_URL_MALFORMAT; - goto clean; - } - - /* get gateway parts */ - if(curl_url_get(gatewaysurl, CURLUPART_HOST, - &gatewayhost, CURLU_URLDECODE)) { - goto clean; - } - - if(curl_url_get(gatewaysurl, CURLUPART_SCHEME, - &gatewayscheme, CURLU_URLDECODE)) { - goto clean; - } - - curl_url_get(gatewaysurl, CURLUPART_PORT, &gatewayport, CURLU_URLDECODE); - curl_url_get(gatewaysurl, CURLUPART_PATH, &gatewaypath, CURLU_URLDECODE); - - /* get the path from user input */ - curl_url_get(uh, CURLUPART_PATH, &inputpath, CURLU_URLDECODE); - /* inputpath might be NULL or a valid pointer now */ - - /* set gateway parts in input url */ - if(curl_url_set(uh, CURLUPART_SCHEME, gatewayscheme, CURLU_URLENCODE)) { - goto clean; - } - - if(curl_url_set(uh, CURLUPART_HOST, gatewayhost, CURLU_URLENCODE)) { - goto clean; - } - - if(curl_url_set(uh, CURLUPART_PORT, gatewayport, CURLU_URLENCODE)) { - goto clean; - } - - /* if the input path is just a slash, clear it */ - if(inputpath && (inputpath[0] == '/') && !inputpath[1]) - *inputpath = '\0'; - - /* ensure the gateway path ends with a trailing slash */ - ensure_trailing(&gatewaypath, '/'); - - pathbuffer = aprintf("%s%s/%s%s", gatewaypath, protocol, cid, - inputpath ? inputpath : ""); - if(!pathbuffer) { - goto clean; - } - - if(curl_url_set(uh, CURLUPART_PATH, pathbuffer, CURLU_URLENCODE)) { - goto clean; - } - - /* Free whatever it has now, rewriting is next */ - Curl_safefree(*url); - - if(curl_url_get(uh, CURLUPART_URL, url, CURLU_URLENCODE)) { - goto clean; - } - - result = CURLE_OK; - -clean: - free(gateway); - curl_free(gatewayhost); - curl_free(gatewaypath); - curl_free(gatewayquery); - curl_free(inputpath); - curl_free(gatewayscheme); - curl_free(gatewayport); - curl_free(cid); - curl_free(pathbuffer); - curl_url_cleanup(gatewaysurl); - - switch(result) { - case CURLE_URL_MALFORMAT: - helpf(tool_stderr, "malformed URL. Visit https://curl.se/" - "docs/ipfs.html#gateway-file-and-" - "environment-variable for more " - "information"); - break; - case CURLE_FILE_COULDNT_READ_FILE: - helpf(tool_stderr, "IPFS automatic gateway detection " - "failure. Visit https://curl.se/docs/" - "ipfs.html#malformed-gateway-url for " - "more information"); - break; - case CURLE_BAD_FUNCTION_ARGUMENT: - helpf(tool_stderr, "--ipfs-gateway argument results in " - "malformed URL. Visit https://curl.se/" - "docs/ipfs.html#malformed-gateway-url " - "for more information"); - break; - default: - break; - } - - return result; -} - /* * Return the protocol token for the scheme used in the given URL */ @@ -1010,13 +721,13 @@ static CURLcode url_proto(char **url, if(curl_strequal(schemep, proto_ipfs) || curl_strequal(schemep, proto_ipns)) { result = ipfs_url_rewrite(uh, schemep, url, config); - /* short-circuit proto_token, we know it's ipfs or ipns */ if(curl_strequal(schemep, proto_ipfs)) proto = proto_ipfs; else if(curl_strequal(schemep, proto_ipns)) proto = proto_ipns; - + if(result) + config->synthetic_error = TRUE; } else proto = proto_token(schemep);