From 8b4470fee09c7639392843b4f52c57ab73682d77 Mon Sep 17 00:00:00 2001 From: Mike Fulton Date: Fri, 20 Jan 2023 17:53:46 -0500 Subject: [PATCH 1/4] use /bin/cp instead of cp since I need to copy datasets and can't use Open Source 'cp' --- tools/src/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/src/Makefile b/tools/src/Makefile index efff77dca..3b22f0113 100644 --- a/tools/src/Makefile +++ b/tools/src/Makefile @@ -11,7 +11,7 @@ clean: rm -f *.o zopen-setup hwthic.h: - cp "//'sys1.sieahdrv.h(hwthic)'" hwthic.h + /bin/cp "//'sys1.sieahdrv.h(hwthic)'" hwthic.h headers: hwthic.h From 36e3ebd1f5fb25d6033100fe7cdd8382a4539a81 Mon Sep 17 00:00:00 2001 From: Mike Fulton Date: Fri, 20 Jan 2023 17:56:51 -0500 Subject: [PATCH 2/4] remove message for expected path where extra header (for OAUTH) added --- tools/src/download.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/src/download.c b/tools/src/download.c index 849d75804..4e9a6329c 100644 --- a/tools/src/download.c +++ b/tools/src/download.c @@ -2675,8 +2675,6 @@ int setRequestHeaders( HWTH_RETURNCODE_TYPE *rcPtr, trace( "Slist append failure" ); return -1; } /* endif slist append failed */ - else - trace( "Next request header appended" ); } /* endloop thru remaining headers */ if ( toolkitSetOption( rcPtr, From 692d6f7965042b69ae50c53e4684c09a849b4703 Mon Sep 17 00:00:00 2001 From: Mike Fulton Date: Fri, 20 Jan 2023 19:38:55 -0500 Subject: [PATCH 3/4] clean up download a bit --- tools/src/download.c | 54 +++++++++++++++++++++++--------------------- tools/src/httpsget.c | 8 ++++++- 2 files changed, 35 insertions(+), 27 deletions(-) diff --git a/tools/src/download.c b/tools/src/download.c index 4e9a6329c..b5e6dcb7e 100644 --- a/tools/src/download.c +++ b/tools/src/download.c @@ -142,9 +142,10 @@ char *DS_TYPE = "blocked"; /* TRSMAIN convention */ /**************************** * constants for http status ****************************/ -#define HTTP_RC_OK 200 -#define HTTP_RC_CREATED 201 -#define HTTP_RC_REDIRECT 302 /* @GG*/ +#define HTTP_RC_OK 200 +#define HTTP_RC_CREATED 201 +#define HTTP_RC_REDIRECT 302 +#define HTTP_RC_FORBIDDEN 403 /******************************** * variables and constants which @@ -341,7 +342,6 @@ int toolkitSetOption( HWTH_RETURNCODE_TYPE *rcPtr, uint32_t optionValueLength, HWTH_DIAGAREA_TYPE *diagAreaPtr ); -/* @GG start*/ int setRequestHeaders( HWTH_RETURNCODE_TYPE *rcPtr, HWTH_HANDLE_TYPE *requestHandlePtr, HWTH_DIAGAREA_TYPE *diagAreaPtr ); @@ -354,7 +354,6 @@ int toolkitSlistOperation( HWTH_RETURNCODE_TYPE *rcPtr, char **stringRef, uint32_t stringLength, HWTH_DIAGAREA_TYPE *diagAreaPtr ); -/* @GG end*/ #define MAX_TOKEN_PAYLOAD 16536 /* generous */ #define MAX_URI_LEN 511 /* generous */ @@ -364,6 +363,7 @@ int toolkitSlistOperation( HWTH_RETURNCODE_TYPE *rcPtr, * Callbacks **************/ HWTHRCVX recvexit; /* streaming receive */ + int httpStatusCode; /* http status */ HWTHHDRX rhdrexit; /* response headers */ #ifdef HAS_MAIN @@ -417,10 +417,9 @@ int toolkitSlistOperation( HWTH_RETURNCODE_TYPE *rcPtr, toolkitDisconnect( &connectHandle ); toolkitTerminate( &connectHandle ); - summarize( &downloadParms, - &receiveData, - toolkitRc ); - return toolkitRc; + summarize( &downloadParms, &receiveData, toolkitRc ); + + return checkHttpStatus(httpStatusCode); } /* end main */ @@ -713,7 +712,6 @@ int toolkitSlistOperation( HWTH_RETURNCODE_TYPE *rcPtr, return -1; } - /* @GG just in case, set max redirects to the max */ intOption = 50; if ( toolkitSetOption( &rc, connectHandlePtr, @@ -723,7 +721,7 @@ int toolkitSlistOperation( HWTH_RETURNCODE_TYPE *rcPtr, &diagArea ) ) return -1; - /* @GG allow for cross domain redirects */ + /* allow for cross domain redirects */ intOption = HWTH_XDOMAIN_REDIRS_ALLOWED; if ( toolkitSetOption( &rc, connectHandlePtr, @@ -800,7 +798,6 @@ int toolkitSlistOperation( HWTH_RETURNCODE_TYPE *rcPtr, &diagArea ) ) return -1; - /* @GG insert call to create request headers */ /********************************************* * Set the request headers. This involves * SList processing, done in called routines. @@ -975,13 +972,15 @@ int toolkitSlistOperation( HWTH_RETURNCODE_TYPE *rcPtr, *requestHandlePtr, &diagArea ); if ( rc != HWTH_OK ) { - /* @GG the HWTH_WARNING with reason code 1 is due to a redirect*/ - if (rc == HWTH_WARNING & diagArea.HWTH_reasonCode == 1) { - ; /* MSF - this is expected trace("*INFO*: The request was successful, however FYI it involved a redirect."); */ - rc = HWTH_OK; + /* The HWTH_WARNING with reason code 1 is due to a redirect */ + /* This is not an issue - redirects are normal - no message needed */ + if (rc == HWTH_WARNING && diagArea.HWTH_reasonCode == 1) { + rc = HWTH_OK; } else { - trace("hwthrqst did not return an acceptable RC"); - surfaceToolkitDiag( &rc, &diagArea ); + if (rc != HWTH_WARNING) { + trace("hwthrqst did not return an acceptable RC"); + surfaceToolkitDiag( &rc, &diagArea ); + } } } return rc; @@ -1362,8 +1361,8 @@ int toolkitSlistOperation( HWTH_RETURNCODE_TYPE *rcPtr, if ( pUserData->httpStatusCode == 0 ) { pStatusLine = pReceiveProgress->HWTH_responseStatusLine; pUserData->httpStatusCode = (int)(pStatusLine->HWTH_statusCode); - if ( checkHttpStatus( pUserData->httpStatusCode ) ) { - rxtrace( "aborting (unacceptable Http status)" ); + httpStatusCode = pUserData->httpStatusCode; + if (checkHttpStatus( pUserData->httpStatusCode ) ) { *pReceiveState = HWTH_STREAM_RECEIVE_ABORT; return; } /* endif unacceptable http status code */ @@ -1537,25 +1536,27 @@ int toolkitSlistOperation( HWTH_RETURNCODE_TYPE *rcPtr, * code (returned in the response headers) warrants * receipt of the response body. * - * Returns: 0 if acceptable status, -1 if not + * Returns: 0 if acceptable status, httpStatusCode if not *******************************************************/ int checkHttpStatus( int httpStatusCode ) { int rc = -1; char msgBuf[80]; - /* @GG - need to also handle the successful - case of redirect - */ switch ( httpStatusCode ) { case HTTP_RC_CREATED: case HTTP_RC_OK: case HTTP_RC_REDIRECT: - rc = 0; + rc = HTTP_RC_OK; /* map 'ok' stuff to rc_ok */ break; + case HTTP_RC_FORBIDDEN: + rc = HTTP_RC_FORBIDDEN; + break; /* this is an error - but we will not print a message about it */ default: sprintf( msgBuf, "Unexpected Http response code %d received", httpStatusCode ); + rc = httpStatusCode; + rxtrace( msgBuf ); break; } /* end switch */ @@ -2105,7 +2106,8 @@ int toolkitSlistOperation( HWTH_RETURNCODE_TYPE *rcPtr, * additional level(s) to increase the likelihood * of acceptance by your web server. **************************************************/ - //HWTH_SSLVERSION_DEFAULT @GG + /* HWTH_SSLVERSION_DEFAULT is the default */ + intOption = HWTH_SSLVERSION_TLSV12; if ( toolkitSetOption( &rc, connectHandlePtr, diff --git a/tools/src/httpsget.c b/tools/src/httpsget.c index 18fe6f487..4db95ba52 100644 --- a/tools/src/httpsget.c +++ b/tools/src/httpsget.c @@ -47,7 +47,13 @@ int httpsget(const char* host, const char* uri, const char* pem, const char* out } if (rc = download(host, uri, output, keydb, stashfile)) { - fprintf(stderr, "error downloading https://%s%s to %s: %d\n", host, uri, output, rc); + if (rc == 403) { + fprintf(stderr, "You have received a 403 Forbidden error from %s%s\n", host, uri); + fprintf(stderr, "This is likely because you have exceeded your download quota from github.\n"); + fprintf(stderr, "Please see: https://rebrand.ly/github-oauth for instructions on how to set up github.\n"); + } else { + fprintf(stderr, "error downloading https://%s%s to %s: %d\n", host, uri, output, rc); + } return rc; } From 9f8f4f04aca4359984cc904d2b223ec5871e3fd7 Mon Sep 17 00:00:00 2001 From: Mike Fulton Date: Fri, 20 Jan 2023 21:14:03 -0500 Subject: [PATCH 4/4] add special checks for 401 and 403 to give guidance --- tools/src/download.c | 14 +++++++++++--- tools/src/httpsget.c | 8 +++++++- tools/src/zopensetupmain.c | 8 ++------ 3 files changed, 20 insertions(+), 10 deletions(-) diff --git a/tools/src/download.c b/tools/src/download.c index b5e6dcb7e..b873a6e07 100644 --- a/tools/src/download.c +++ b/tools/src/download.c @@ -145,6 +145,7 @@ char *DS_TYPE = "blocked"; /* TRSMAIN convention */ #define HTTP_RC_OK 200 #define HTTP_RC_CREATED 201 #define HTTP_RC_REDIRECT 302 +#define HTTP_RC_UNAUTHORIZED 401 #define HTTP_RC_FORBIDDEN 403 /******************************** @@ -382,6 +383,7 @@ int toolkitSlistOperation( HWTH_RETURNCODE_TYPE *rcPtr, struct parmStruct downloadParms = { false, IOTYPE_FILE, SCHEME_HTTPS }; #endif char msgBuf[80]; + int rc; #ifdef HAS_MAIN if ( getDownloadParms( argc, argv, &downloadParms ) ) { @@ -419,7 +421,8 @@ int toolkitSlistOperation( HWTH_RETURNCODE_TYPE *rcPtr, summarize( &downloadParms, &receiveData, toolkitRc ); - return checkHttpStatus(httpStatusCode); + rc = checkHttpStatus(httpStatusCode); + return (rc == HTTP_RC_OK) ? 0 : rc; } /* end main */ @@ -1061,7 +1064,6 @@ int toolkitSlistOperation( HWTH_RETURNCODE_TYPE *rcPtr, if ( pRecvData->numBytesReceived ) downloadSize = pRecvData->numBytesReceived; else { - trace( "A full response was not received" ); if ( toolkitRc == HWTH_OK ) { sprintf( msgBuf, "Http response code %d was received", @@ -1069,10 +1071,13 @@ int toolkitSlistOperation( HWTH_RETURNCODE_TYPE *rcPtr, trace( msgBuf ); } else { + #if 0 + /* for things like forbidden or redirect, no message needed */ sprintf( msgBuf, "Toolkit failure rc %d was received", toolkitRc ); trace( msgBuf ); + #endif } downloadSize = pRecvData->knownContentLength; } /* endif lacking (full) response */ @@ -1362,7 +1367,7 @@ int toolkitSlistOperation( HWTH_RETURNCODE_TYPE *rcPtr, pStatusLine = pReceiveProgress->HWTH_responseStatusLine; pUserData->httpStatusCode = (int)(pStatusLine->HWTH_statusCode); httpStatusCode = pUserData->httpStatusCode; - if (checkHttpStatus( pUserData->httpStatusCode ) ) { + if (checkHttpStatus( pUserData->httpStatusCode ) != HTTP_RC_OK) { *pReceiveState = HWTH_STREAM_RECEIVE_ABORT; return; } /* endif unacceptable http status code */ @@ -1550,6 +1555,9 @@ int toolkitSlistOperation( HWTH_RETURNCODE_TYPE *rcPtr, break; case HTTP_RC_FORBIDDEN: rc = HTTP_RC_FORBIDDEN; + break; /* this is an error - but we will not print a message about it */ + case HTTP_RC_UNAUTHORIZED: + rc = HTTP_RC_UNAUTHORIZED; break; /* this is an error - but we will not print a message about it */ default: sprintf( msgBuf, diff --git a/tools/src/httpsget.c b/tools/src/httpsget.c index 4db95ba52..8bee19e4e 100644 --- a/tools/src/httpsget.c +++ b/tools/src/httpsget.c @@ -50,7 +50,13 @@ int httpsget(const char* host, const char* uri, const char* pem, const char* out if (rc == 403) { fprintf(stderr, "You have received a 403 Forbidden error from %s%s\n", host, uri); fprintf(stderr, "This is likely because you have exceeded your download quota from github.\n"); - fprintf(stderr, "Please see: https://rebrand.ly/github-oauth for instructions on how to set up github.\n"); + fprintf(stderr, "Please see: https://zosopentools.link/github-oauth for instructions on how to set up a github OAUTH id.\n"); + fprintf(stderr, "Once your OAUTH id is set, export ZOPEN_GIT_OAUTH_TOKEN= and then re-run zopen-setup.\n"); + } else if (rc == 401) { + fprintf(stderr, "You have received a 401 Unauthorized error from %s%s\n", host, uri); + fprintf(stderr, "This is likely because you have an expired or invalid github OAUTH id.\n"); + fprintf(stderr, "Please see: https://zosopentools.link/github-oauth for instructions on how to set up a new github OAUTH id.\n"); + fprintf(stderr, "Once your OAUTH id is reset, export ZOPEN_GIT_OAUTH_TOKEN= and then re-run zopen-setup.\n"); } else { fprintf(stderr, "error downloading https://%s%s to %s: %d\n", host, uri, output, rc); } diff --git a/tools/src/zopensetupmain.c b/tools/src/zopensetupmain.c index 9905617a0..56a4407d7 100644 --- a/tools/src/zopensetupmain.c +++ b/tools/src/zopensetupmain.c @@ -106,8 +106,8 @@ int main(int argc, char* argv[]) { return 4; } - if (!tmppem || genfilename("pem", tmppem, ZOPEN_PATH_MAX)) { - fprintf(stderr, "error acquiring storage\n"); + if (genfilename("pem", tmppem, ZOPEN_PATH_MAX)) { + /* genfilename issues specific errors */ return 4; } @@ -131,11 +131,9 @@ int main(int argc, char* argv[]) { fprintf(STDTRC, "Download %s into %s/%s\n", bootpkg[i], root, ZOPEN_BOOT); } if (getfilenamefrompkg(bootpkg[i], pkgsfx, tmppem, filename, ZOPEN_PATH_MAX)) { - fprintf(stderr, "error acquiring storage (3)\n"); return 4; } if (genfilenameinsubdir(root, ZOPEN_BOOT, filename, output, ZOPEN_PATH_MAX)) { - fprintf(stderr, "error acquiring storage (2)\n"); return 4; } if ((rc = snprintf(uri, sizeof(uri), "/%s/%s%s/%s/%s", ZOPEN_BOOT_URI_PREFIX, bootpkg[i], pkgsfx, ZOPEN_BOOT_URI_SUFFIX, filename)) > sizeof(uri)) { @@ -147,7 +145,6 @@ int main(int argc, char* argv[]) { return rc; } if (rc = unpaxandlink(root, ZOPEN_BOOT, output, bootpkg[i])) { - fprintf(stderr, "error unpaxing %s in directory %s/%s\n", output, root, ZOPEN_BOOT); return rc; } } @@ -156,7 +153,6 @@ int main(int argc, char* argv[]) { fprintf(STDTRC, "Create %s for source'ing in %s/%s\n", ZOPEN_BOOT_ENV, root, ZOPEN_BOOT); } if (rc = createbootenv(root, ZOPEN_BOOT, bootpkg)) { - fprintf(stderr, "error creating %s in directory %s/%s\n", ZOPEN_BOOT_ENV, root, ZOPEN_BOOT); return rc; }