Skip to content
This repository has been archived by the owner on Nov 8, 2022. It is now read-only.

Commit

Permalink
V11
Browse files Browse the repository at this point in the history 
2022/05/18 V11 :    
    > fix issue " Still some CRC errors occur when uploading on SDCard #21 ": remove sevbuf use to fix the last CRC errors and limit 
      to only 1 transfer for sdCard operations (simultaneous transfers leads in CRC errors)  
    > multi transfer enhancement : reduce open/close connection time with 
        - using a static preallocated array of threads (before it was allocated statically when creating the connection) 
        - manually set cpu and priority of transfer threads
    > now use also a small buffer for UL operations (before setting manually CPU and priority, a large buffer was used to let other 
      connections start)
    > tweak the buffers sizes used for DL / UL
      Use 25% DL vs 75% UL of the reserved memory for buffer socket operations (instead of 50/50)
    > This last setting increase DL speeds obtain when multi-transferring. Using a small buffer for upload reduced the "speed displayed" 
      on UL operation (now take the fwrite time into account). 
      This version gives max speeds over 4M/s on both ways and treat a large number of files more quickly.
  • Loading branch information
@Laf111
Laf111 committed May 18, 2022
1 parent 7ad14d1 commit b8285e4
Show file tree
Hide file tree
Showing 2 changed files with 51 additions and 43 deletions.
92 changes: 50 additions & 42 deletions src/ftp.c
View file
Expand Up @@ -61,8 +61,8 @@ static const uint32_t CRLF_LENGTH = 2;
// number of active connections
static uint32_t activeConnectionsNumber = 0;
static uint32_t activeTransfersNumber = 0;
static uint32_t activeTransfersOnSD = 0;
static const uint32_t maxTransfersOnSdCard = 1;
static uint32_t activeUploadsToSdCard = 0;
static const uint32_t maxUploadsOnSdCard = 1;

// unique client IP address
static char clientIp[15]="UNKNOWN_CLIENT";
Expand Down Expand Up @@ -259,10 +259,12 @@ int launchTransfer(int argc UNUSED, const char **argv)
writeToLog("C[%d] launching transfer for %s", activeConnection->index+1, activeConnection->fileName);
#endif


if (activeConnection->volPath == NULL) {
result = send_from_file(activeConnection->data_socket, activeConnection);

} else {

result = recv_to_file(activeConnection->data_socket, activeConnection);

// if cwd does not contain /sd/
Expand All @@ -285,17 +287,18 @@ int launchTransfer(int argc UNUSED, const char **argv)
return result;
}

// launch and monitor the transfer
static int32_t transfer(int32_t data_socket UNUSED, connection_t *connection) {
int32_t result = -EAGAIN;

// on the very first call
if (connection->dataTransferOffset == -1) {

// hard limit simultaneous transfers on SDCard
if (strstr(connection->cwd, "/sd/") != NULL) {
while (activeTransfersOnSD >= maxTransfersOnSdCard) OSSleepTicks(OSMillisecondsToTicks(100));
activeTransfersOnSD++;
}


// hard limit simultaneous uploads on SDCard
if ((connection->volPath != NULL) && (strstr(connection->cwd, "/sd/") != NULL)) {
if (activeUploadsToSdCard >= maxUploadsOnSdCard) return -EAGAIN;
activeUploadsToSdCard++;
}
activeTransfersNumber++;

// init bytes counter
Expand All @@ -308,7 +311,7 @@ static int32_t transfer(int32_t data_socket UNUSED, connection_t *connection) {
writeToLog("Using data_socket (%d) of C[%d] to transfer %s", data_socket, connection->index+1, connection->fileName);
// displayConnectionDetails(connection->index);
#endif


// Dispatching connections over CPUs :

Expand Down Expand Up @@ -359,52 +362,57 @@ static int32_t transfer(int32_t data_socket UNUSED, connection_t *connection) {
return result;
}

// this method is called on transfer success but also on transfer failure
static int32_t closeTransferredFile(connection_t *connection) {
int32_t result = 0;

activeTransfersNumber--;

// hard limit simultaneous transfers on SDCard
if (strstr(connection->cwd, "/sd/") != NULL) activeTransfersOnSD++;
activeTransfersNumber--;
if ((connection->volPath != NULL) && (strstr(connection->cwd, "/sd/") != NULL)) activeUploadsToSdCard--;

#ifdef LOG2FILE
writeToLog("CloseTransferredFile for C[%d] (file=%s)", connection->index+1, connection->fileName);
#endif

// free transfer buffer
// free transfer buffer if needed
if (connection->transferBuffer != NULL) MEMFreeToDefaultHeap(connection->transferBuffer);
connection->transferBuffer = NULL;


// close file if needed
if (connection->f != NULL) {
fclose(connection->f);
}

if (connection->volPath != NULL && connection->bytesTransferred == 0) {
// for all files except WiiUFtpServer_crc32_report.sfv
if ( calculateCrc32 && (strcmp(connection->fileName, "WiiUFtpServer_crc32_report.sfv") != 0) ) {

#ifdef LOG2FILE
display("C[%d] writing CRC32 after uploading %s", connection->index+1, connection->fileName);
#endif

// add the CRC32 value to the SFV file
writeCRC32('<', connection->cwd, connection->fileName, connection->crc32);
}
free(connection->volPath);
} else {
// for all files except WiiUFtpServer_crc32_report.sfv
if ( calculateCrc32 && (strcmp(connection->fileName, "WiiUFtpServer_crc32_report.sfv") != 0 ) ) {

#ifdef LOG2FILE
display("C[%d] writing CRC32 after downloading %s", connection->index+1, connection->fileName);
#endif

// add the CRC32 value to the SFV file
writeCRC32('>', connection->cwd, connection->fileName, connection->crc32);
// on success, if crc32 calc is enabled
if (connection->bytesTransferred == 0 && calculateCrc32) {

if (connection->volPath != NULL) {
// for all files except WiiUFtpServer_crc32_report.sfv
if ( strcmp(connection->fileName, "WiiUFtpServer_crc32_report.sfv") != 0 ) {

#ifdef LOG2FILE
display("C[%d] writing CRC32 after uploading %s", connection->index+1, connection->fileName);
#endif

// add the CRC32 value to the SFV file
writeCRC32('<', connection->cwd, connection->fileName, connection->crc32);
}
} else {
// for all files except WiiUFtpServer_crc32_report.sfv
if ( strcmp(connection->fileName, "WiiUFtpServer_crc32_report.sfv") != 0 ) {

#ifdef LOG2FILE
display("C[%d] writing CRC32 after downloading %s", connection->index+1, connection->fileName);
#endif

// add the CRC32 value to the SFV file
writeCRC32('>', connection->cwd, connection->fileName, connection->crc32);
}
}
}

if (strstr(connection->cwd, "/sd/") != NULL) activeTransfersOnSD--;
// if needed, free connection->volPath
if (connection->volPath != NULL) free(connection->volPath);

return result;
}
Expand Down Expand Up @@ -1565,7 +1573,7 @@ static int32_t ftp_RETR(connection_t *connection, char *path) {
return write_reply(connection, 550, strerror(lseek_error));
}
connection->volPath = NULL;

int32_t result = prepare_data_connection(connection, transfer, connection, closeTransferredFile);

if (result < 0) {
Expand Down Expand Up @@ -1625,7 +1633,7 @@ static int32_t stor_or_append(connection_t *connection, char *path, char mode[3]
if (folder != NULL) free(folder);
if (parentFolder != NULL) free(parentFolder);
if (folderName != NULL) free(folderName);

// allocate the buffer for transfering
connection->transferBuffer = MEMAllocFromDefaultHeapEx(UL_BUFFER_SIZE, 64);
if (connection->transferBuffer == NULL) {
Expand All @@ -1644,9 +1652,9 @@ static int32_t stor_or_append(connection_t *connection, char *path, char mode[3]
MEMFreeToDefaultHeap(connection->transferBuffer);
return write_reply(connection, 550, msg);
}

display("> C[%d] receiving %s...", connection->index+1, connection->fileName);

int32_t result = prepare_data_connection(connection, transfer, connection, closeTransferredFile);
if (result < 0) {
display("! ERROR : C[%d] prepare_data_connection failed in stor_or_append for %s", connection->index+1, path);
Expand Down
2 changes: 1 addition & 1 deletion src/net.h
View file
Expand Up @@ -77,7 +77,7 @@ extern "C"{
#define SOCKET_BUFFER_SIZE (16*UNSCALED_BUFFER_SIZE)

#define UL_BUFFER_SIZE (4*SOCKET_BUFFER_SIZE)
#define DL_BUFFER_SIZE 2*SOCKET_BUFFER_SIZE
#define DL_BUFFER_SIZE (2*SOCKET_BUFFER_SIZE)

// socket memory buffer size = (2*sndBuffSize+2*rcvBuffSize)
#define SOMEMOPT_BUFFER_SIZE (4*SOCKET_BUFFER_SIZE)
Expand Down

0 comments on commit b8285e4

Please sign in to comment.