diff --git a/CMakeLists.txt b/CMakeLists.txt index 1862f8c..da8af51 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ # Copyright © 2025 CCP ehf. cmake_minimum_required(VERSION 3.31.0) -project(resources VERSION 4.1.0) +project(resources VERSION 4.2.0) include(cmake/CcpTargetConfigurations.cmake) include(cmake/CcpDocsGenerator.cmake) diff --git a/cli/src/CreateResourceGroupFromFilterCliOperation.cpp b/cli/src/CreateResourceGroupFromFilterCliOperation.cpp index 3e64483..5ed94d0 100644 --- a/cli/src/CreateResourceGroupFromFilterCliOperation.cpp +++ b/cli/src/CreateResourceGroupFromFilterCliOperation.cpp @@ -23,7 +23,11 @@ CreateResourceGroupFromFilterCliOperation::CreateResourceGroupFromFilterCliOpera m_skipNonExistentInputDirectoriesId( "--skip-non-existent-input-directories" ), m_streamChunkSizeId( "--stream-chunk-size" ), m_remoteUrlToGetCompressionId( "--remote-url-to-attempt-to-get-compression-info" ), - m_skipBinaryOperationCalculationId( "--skip-binary-operation-calculation" ) + m_skipBinaryOperationCalculationId( "--skip-binary-operation-calculation" ), + m_numberOfThreadsId( "--number-of-threads" ), + m_networkRetryCountId( "--network-retry-count" ), + m_networkRetryBackoffMultiplierId( "--network-retry-backoff-multiplier" ) + { // Struct is inspected to ascertain default values @@ -47,19 +51,25 @@ CreateResourceGroupFromFilterCliOperation::CreateResourceGroupFromFilterCliOpera AddArgumentFlag( m_exportResourcesId, "Export resources after processing. see --export-resources-destination-type and --export-resources-destination-path" ); - AddArgument( m_exportResourcesDestinationTypeId, "Represents the type of repository where exported resources will be saved. Requires --export-resources", false, false, DestinationTypeToString( defaultImportParams.exportSettings.destinationSettings.destinationType ), ResourceDestinationTypeChoicesAsString() ); + AddArgument( m_exportResourcesDestinationTypeId, "Type of repository where exported resources will be saved. Requires --export-resources", false, false, DestinationTypeToString( defaultImportParams.exportSettings.destinationSettings.destinationType ), ResourceDestinationTypeChoicesAsString() ); - AddArgument( m_exportResourcesDestinationPathId, "Represents the base path where the exported resources will be saved. Requires --export-resources", false, false, defaultImportParams.exportSettings.destinationSettings.basePath.string() ); + AddArgument( m_exportResourcesDestinationPathId, "Base path where the exported resources will be saved. Requires --export-resources", false, false, defaultImportParams.exportSettings.destinationSettings.basePath.string() ); AddArgument( m_prefixMapBasepathId, "Base directory for prefix mappings defined in filter files.", false, false, "" ); AddArgumentFlag( m_skipNonExistentInputDirectoriesId, "Skips input directories specified that don't exist rather than error." ); - AddArgument( m_streamChunkSizeId, "Represents the chunks streamed in bytes when streaming data.", false, false, SizeToString( defaultImportParams.fileStreamChunkSize ) ); + AddArgument( m_streamChunkSizeId, "Chunks stream size in bytes for streaming data.", false, false, SizeToString( defaultImportParams.fileStreamChunkSize ) ); AddArgument( m_remoteUrlToGetCompressionId, "If supplied, url is checked to get compression information.", false, false, defaultImportParams.compressionCalculationSettings.remoteUrlToAttemptToGetCompression.string() ); AddArgumentFlag( m_skipBinaryOperationCalculationId, "Set skip to skip binary operation for resources" ); + + AddArgument( m_numberOfThreadsId, "Nnumber of threads to use for async processes.", false, false, SizeToString( defaultImportParams.asyncSettings.numberOfThreads ) ); + + AddArgument( m_networkRetryCountId, "Number of retries to attempt when encountering a failed download.", false, false, SizeToString( defaultImportParams.compressionCalculationSettings.downloadSettings.retryCount ) ); + + AddArgument( m_networkRetryBackoffMultiplierId, "Multiplier in seconds to wait for when retrying, value will multiply on each retry to backoff.", false, false, SizeToString( defaultImportParams.compressionCalculationSettings.downloadSettings.retrySeconds.count() ) ); } bool CreateResourceGroupFromFilterCliOperation::Execute( std::string& returnErrorMessage ) const @@ -79,6 +89,62 @@ bool CreateResourceGroupFromFilterCliOperation::Execute( std::string& returnErro return false; } + try + { + unsigned long long numberOfThreadsUnsignedLongLong = std::stoull( m_argumentParser->get( m_numberOfThreadsId ) ); + + if( numberOfThreadsUnsignedLongLong > std::numeric_limits::max() ) + { + return false; + } + + createResourceGroupParams.asyncSettings.numberOfThreads = static_cast( numberOfThreadsUnsignedLongLong ); + + } + catch( std::invalid_argument& ) + { + return false; + } + catch( std::out_of_range& ) + { + return false; + } + + try + { + unsigned long long networkRetryCountUnsignedLongLong = std::stoull( m_argumentParser->get( m_networkRetryCountId ) ); + + if( networkRetryCountUnsignedLongLong > std::numeric_limits::max() ) + { + return false; + } + + createResourceGroupParams.compressionCalculationSettings.downloadSettings.retryCount = static_cast( networkRetryCountUnsignedLongLong ); + } + catch( std::invalid_argument& ) + { + return false; + } + catch( std::out_of_range& ) + { + return false; + } + + try + { + unsigned long long networkRetryBackoffMultiplierLongLong = std::stoull( m_argumentParser->get( m_networkRetryBackoffMultiplierId ) ); + + createResourceGroupParams.compressionCalculationSettings.downloadSettings.retrySeconds = std::chrono::seconds( networkRetryBackoffMultiplierLongLong ); + } + catch( std::invalid_argument& ) + { + return false; + } + catch( std::out_of_range& ) + { + return false; + } + bool versionIsValid = ParseDocumentVersion( m_argumentParser->get( m_documentVersionId ), createResourceGroupParams.outputDocumentVersion ); if( !versionIsValid ) @@ -176,7 +242,7 @@ void CreateResourceGroupFromFilterCliOperation::PrintStartBanner( const std::filesystem::path& basePathToFilterFiles, const std::filesystem::path& basePathRespourceFiles ) const { - std::cout << "---Creating Resource Group---" << std::endl; + std::cout << "---Creating Resource Groups From Filters---" << std::endl; PrintCommonOperationHeaderInformation(); @@ -235,10 +301,16 @@ void CreateResourceGroupFromFilterCliOperation::PrintStartBanner( if( createResourceGroupFromFilterParams.compressionCalculationSettings.remoteUrlToAttemptToGetCompression != "" ) { std::cout << "Compression info check url: " << createResourceGroupFromFilterParams.compressionCalculationSettings.remoteUrlToAttemptToGetCompression << std::endl; - } + + std::cout << "Network retry count: " << createResourceGroupFromFilterParams.compressionCalculationSettings.downloadSettings.retryCount << std::endl; + + std::cout << "Network retry backoff multiplier ( Seconds ): " << createResourceGroupFromFilterParams.compressionCalculationSettings.downloadSettings.retrySeconds.count() << std::endl; + } std::cout << "File stream chunk Size: " << createResourceGroupFromFilterParams.fileStreamChunkSize << " Bytes" << std::endl; + std::cout << "Number of threads for async operations: " << createResourceGroupFromFilterParams.asyncSettings.numberOfThreads << std::endl; + std::cout << "----------------------------\n" << std::endl; } @@ -247,6 +319,7 @@ bool CreateResourceGroupFromFilterCliOperation::CreateResourceGroups( CarbonResources::CreateResourceGroupFromFilterParams& createResourceGroupFromFilterParams, std::vector>& exportParams ) const { + createResourceGroupFromFilterParams.callbackSettings.statusCallback = GetStatusCallback(); createResourceGroupFromFilterParams.callbackSettings.verbosityLevel = GetVerbosityLevel(); @@ -289,6 +362,7 @@ bool CreateResourceGroupFromFilterCliOperation::CreateResourceGroups( exportParams.callbackSettings.statusCallback = GetStatusCallback(); exportParams.callbackSettings.verbosityLevel = GetVerbosityLevel(); + exportParams.callbackSettings.verbosityLevel = exportParams.callbackSettings.verbosityLevel > 2 || exportParams.callbackSettings.verbosityLevel == -1 ? 2 : exportParams.callbackSettings.verbosityLevel; CarbonResources::Result exportToFileResult = ( *resourceIter )->ExportToFile( exportParams ); diff --git a/cli/src/CreateResourceGroupFromFilterCliOperation.h b/cli/src/CreateResourceGroupFromFilterCliOperation.h index b3e224d..0f99238 100644 --- a/cli/src/CreateResourceGroupFromFilterCliOperation.h +++ b/cli/src/CreateResourceGroupFromFilterCliOperation.h @@ -58,6 +58,12 @@ class CreateResourceGroupFromFilterCliOperation : public CliOperation std::string m_remoteUrlToGetCompressionId; std::string m_skipBinaryOperationCalculationId; + + std::string m_numberOfThreadsId; + + std::string m_networkRetryCountId; + + std::string m_networkRetryBackoffMultiplierId; }; #endif // CreateResourceGroupFromFilterCliOperation_H \ No newline at end of file diff --git a/include/Enums.h b/include/Enums.h index cd8a0a1..294e274 100644 --- a/include/Enums.h +++ b/include/Enums.h @@ -182,6 +182,12 @@ struct Result ResultType type = ResultType::SUCCESS; std::string info = ""; + + bool operator!=( const Result& other ) const + { + return type != other.type; + } + }; /** Converts ResultType to string diff --git a/include/ResourceGroup.h b/include/ResourceGroup.h index 416e361..a30a300 100644 --- a/include/ResourceGroup.h +++ b/include/ResourceGroup.h @@ -10,6 +10,7 @@ #include #include #include +#include namespace CarbonResources @@ -315,6 +316,16 @@ struct ExportResourceSettings }; }; +/** @struct AsyncSettings + * @brief Settings related to async processing + * @var AsyncSettings::numberOfThreads + * Number of threads to use for async processes, passing 0 will run all on main thread + */ +struct AsyncSettings +{ + uint32_t numberOfThreads = std::thread::hardware_concurrency(); +}; + /** @struct CompressionCalculationSettings * @brief Function Parameters related to resource compression calculation * @var CompressionCalculationSettings::calculateCompressions @@ -353,6 +364,8 @@ struct CompressionCalculationSettings * Settings related to filtering * @var CreateResourceGroupFromFilterParams::calculateBinaryOperation * Set true to include calculation of binary operation + * @var CreateResourceGroupFromFilterParams::asyncSettings + * Settings related to async setup */ struct CreateResourceGroupFromFilterParams { @@ -373,6 +386,8 @@ struct CreateResourceGroupFromFilterParams FilterSettings filterSettings; bool calculateBinaryOperation = true; + + AsyncSettings asyncSettings; }; /** @struct ResourceGroupMergeParams diff --git a/src/BundleResourceGroupImpl.cpp b/src/BundleResourceGroupImpl.cpp index 3b910a7..ad799d4 100644 --- a/src/BundleResourceGroupImpl.cpp +++ b/src/BundleResourceGroupImpl.cpp @@ -280,7 +280,7 @@ Result BundleResourceGroup::BundleResourceGroupImpl::Unpack( const BundleUnpackP // Validate the resource data std::string recreatedResourceChecksum; - if (!resourceChecksumStream.FinishAndRetrieve(recreatedResourceChecksum)) + if (!resourceChecksumStream.Retrieve(recreatedResourceChecksum)) { return Result{ ResultType::FAILED_TO_GENERATE_CHECKSUM }; } diff --git a/src/PatchResourceGroupImpl.cpp b/src/PatchResourceGroupImpl.cpp index a7ee6ea..33e66d3 100644 --- a/src/PatchResourceGroupImpl.cpp +++ b/src/PatchResourceGroupImpl.cpp @@ -630,7 +630,7 @@ Result PatchResourceGroup::PatchResourceGroupImpl::Apply( const PatchApplyParams std::string patchedFileChecksum; - if (!patchedFileChecksumStream.FinishAndRetrieve(patchedFileChecksum)) + if (!patchedFileChecksumStream.Retrieve(patchedFileChecksum)) { return Result{ ResultType::FAILED_TO_GENERATE_CHECKSUM }; } diff --git a/src/ResourceGroupImpl.cpp b/src/ResourceGroupImpl.cpp index 21cb785..6708775 100644 --- a/src/ResourceGroupImpl.cpp +++ b/src/ResourceGroupImpl.cpp @@ -23,6 +23,7 @@ #include "ChunkIndex.h" #include "ResourceGroupFactory.h" #include "ResourceFilter.h" +#include "ThreadPool.h" namespace CarbonResources { @@ -217,7 +218,7 @@ Result ResourceGroup::ResourceGroupImpl::CreateFromDirectory( const CreateResour std::string checksum; - if( !checksumStream.FinishAndRetrieve( checksum ) ) + if( !checksumStream.Retrieve( checksum ) ) { return Result{ ResultType::FAILED_TO_GENERATE_CHECKSUM }; } @@ -333,588 +334,841 @@ Result ResourceGroup::ResourceGroupImpl::CreateFromDirectory( const CreateResour return Result{ ResultType::SUCCESS }; } -Result ResourceGroup::ResourceGroupImpl::CreateFromFilter( const CreateResourceGroupFromFilterParams& params, StatusSettings& statusSettings ) +// Thread structs for filtering +struct ResourceGroupFromFilterPoolArguments { - // Update status - statusSettings.Update( StatusProgressType::PERCENTAGE, 0, 5, "Creating resource group from filters" ); + bool backoff = false; - // Ensure document version is valid - VersionInternal documentVersion( params.outputDocumentVersion ); + std::mutex backoffMutex; + + std::condition_variable backoffConditional; + + std::mutex resourceGroupMutex; + + std::mutex checkedRelativePathsMutex; + + std::map checkedRelativePaths; + + std::vector> filterGroups; + + bool ignoreCase; + + const CreateResourceGroupFromFilterParams* params; + + Result status = Result{ ResultType::SUCCESS }; + + bool checkForDuplicateRelativePaths = false; - bool ignoreCase = false; + int totalNumberOfResources = 0; - if( documentVersion == VersionInternal( S_CSV_DOCUMENT_VERSION ) ) + std::atomic numberOfResourcesProcessed = 0; + + std::atomic numberOfResourcesSkipped = 0; + + std::atomic numberOfResourcesCompressed = 0; + + std::mutex statusUpdateMutex; + + StatusSettings statusSettings; + + void RunStatusUpdate() { - ignoreCase = true; + if (statusSettings.RequiresStatusUpdates()) + { + { + std::unique_lock lock( statusUpdateMutex ); + if ((numberOfResourcesProcessed % 1000) == 0) + { + + float step = static_cast( 100.0 / totalNumberOfResources ); + float progress = step * numberOfResourcesProcessed; + + std::stringstream ss; + + ss << "Processed: " << std::to_string( numberOfResourcesProcessed ) + << ", Skipped: " << std::to_string( numberOfResourcesSkipped ) + << ", Compressed:" << std::to_string( numberOfResourcesCompressed ); + + statusSettings.Update( StatusProgressType::PERCENTAGE, progress, step, ss.str() ); + } + } + } } - if( !documentVersion.isVersionValid() ) - { - return Result{ ResultType::DOCUMENT_VERSION_UNSUPPORTED }; - } + void ShowStatusWarning(const std::string& warningString) + { + { + std::unique_lock lock( statusUpdateMutex ); - statusSettings.Update( StatusProgressType::PERCENTAGE, 5, 10, "Loading filter files" ); + statusSettings.Update( StatusProgressType::WARNING, 0, 0, warningString ); + } + } +}; - // Initialise Resource Filters - std::vector> filterGroups; +struct ResourceGroupFromFilterThreadArguments +{ + std::filesystem::path inputDirectory; - for( auto& filter : params.filterSettings.filters ) + std::vector entryPaths; + + std::filesystem::path searchPath; + + ResourceTools::Downloader downloader; +}; + +void CreateResourceGroupsFromFilterWorker( std::shared_ptr commonArguments, std::shared_ptr threadArguments ) +{ + ResourceTools::FileDataStreamIn fileStreamIn( commonArguments->params->fileStreamChunkSize ); + + ResourceTools::Md5ChecksumStream checksumStream; + + for( auto entry : threadArguments->entryPaths ) { - if( filter->filterFilePaths.empty() ) + + commonArguments->RunStatusUpdate(); + + std::filesystem::path entryPath = entry.path(); + + uintmax_t entrySize = entry.file_size(); + + commonArguments->numberOfResourcesProcessed++; + + // Process file + // Check each filter group + std::vector> matchedFilters; + + std::filesystem::path filePathRelativeToInputDirectory = std::filesystem::relative( entryPath, commonArguments->params->filterSettings.prefixMapBasePath ); + + std::string normalisedRelativePath = filePathRelativeToInputDirectory.generic_string(); + + // Filtering is not case sensitive + if( commonArguments->ignoreCase ) { - std::string errorMsg = "No filter files provided."; - return Result{ ResultType::FAILED_TO_INITIALIZE_RESOURCE_FILTER, errorMsg }; + std::transform( normalisedRelativePath.begin(), normalisedRelativePath.end(), normalisedRelativePath.begin(), []( unsigned char c ) { return std::tolower( c ); } ); } - std::shared_ptr filterGroup = std::make_shared(); + for( auto& filterGroup : commonArguments->filterGroups ) + { - filterGroup->resourceGroup = filter->outputResourceGroup->m_impl; + // Check each filter in filter group + for( auto& filter : filterGroup->filters ) + { - for( auto filterPath : filter->filterFilePaths ) + // Check that the current search path is relevant to the current filter + const std::vector& filterSearchPaths = filter->GetPrefixPaths(); + + bool searchPathInFilter = false; + + for( auto& filterPath : filterSearchPaths ) + { + if( threadArguments->searchPath == filterPath ) + { + searchPathInFilter = true; + break; + } + } + + if( !searchPathInFilter ) + { + // The current search path isn't one that is present in the current filter + continue; + } + + if( filter->CheckPath( normalisedRelativePath ) ) + { + matchedFilters.push_back( filterGroup ); + + break; + } + } + } + + // Check if any filters were matched + if( matchedFilters.empty() ) { - // Get filter file data - std::string filterData; + //File doesn't meet any of the filters + commonArguments->numberOfResourcesSkipped++; - if( !ResourceTools::GetLocalFileData( filterPath, filterData ) ) + continue; + } + + // Process the file data via streaming + checksumStream.Start(); + + std::filesystem::path resourceRelativePath = std::filesystem::relative( entryPath, threadArguments->inputDirectory ); + + std::string resourceRelativePathString = resourceRelativePath.generic_string(); + + if( commonArguments->ignoreCase ) + { + std::transform( resourceRelativePathString.begin(), resourceRelativePathString.end(), resourceRelativePathString.begin(), []( unsigned char c ) { return std::tolower( c ); } ); + } + + // Check to ensure path hasn't been checked before + bool duplicateFound = false; + + if (commonArguments->checkForDuplicateRelativePaths) + { { - std::string errorMsg = "Failed to open filter file: " + filterPath.string(); - return Result{ ResultType::FAILED_TO_OPEN_FILE, errorMsg }; + std::unique_lock lock( commonArguments->checkedRelativePathsMutex ); + duplicateFound = commonArguments->checkedRelativePaths.find( resourceRelativePathString ) != commonArguments->checkedRelativePaths.end(); } + } - ResourceTools::FilterFile fileData; + if (duplicateFound) + { + commonArguments->numberOfResourcesSkipped++; + continue; + } - try + // Add path to checked paths + { + std::unique_lock lock( commonArguments->checkedRelativePathsMutex ); + commonArguments->checkedRelativePaths[resourceRelativePathString] = 1; + } + + if( !fileStreamIn.StartRead( entryPath ) ) + { { - ResourceTools::FilterFileReader::LoadFromIniFileData( filterData.data(), filterData.size(), fileData, ignoreCase ); + std::unique_lock lock( commonArguments->resourceGroupMutex ); + commonArguments->status = Result{ ResultType::FAILED_TO_OPEN_FILE_STREAM }; } - catch( const std::exception& e ) + return; + } + + // Calculate without compression to get checksum + while( !fileStreamIn.IsFinished() ) + { + std::string fileData; + + if( !( fileStreamIn >> fileData ) ) { - std::string errorMsg = "Failed to read filter file: " + std::string( e.what() ); + { + std::unique_lock lock( commonArguments->resourceGroupMutex ); + commonArguments->status = Result{ ResultType::FAILED_TO_READ_FROM_STREAM }; + } + return; + } - return Result{ ResultType::FAILED_TO_INITIALIZE_RESOURCE_FILTER, errorMsg }; + if( !( checksumStream << fileData ) ) + { + { + std::unique_lock lock( commonArguments->resourceGroupMutex ); + commonArguments->status = Result{ ResultType::FAILED_TO_GENERATE_CHECKSUM }; + } + return; } + } - std::unique_ptr filter = std::make_unique(); + // Get Checksum + std::string checksum; - if( !filter->SetFromFilterFileData( fileData ) ) + if( !checksumStream.Retrieve( checksum ) ) + { { - return Result{ ResultType::FAILED_TO_INITIALIZE_RESOURCE_FILTER }; + std::unique_lock lock( commonArguments->resourceGroupMutex ); + commonArguments->status = Result{ ResultType::FAILED_TO_GENERATE_CHECKSUM }; } - - filterGroup->filters.push_back( std::move( filter ) ); + return; } - filterGroups.push_back( std::move( filterGroup ) ); - } + // Create resource from parameters + ResourceInfoParams resourceParams; - // Populate all search paths - std::vector searchPaths; + resourceParams.relativePath = resourceRelativePathString; - for( auto& filterGroup : filterGroups ) - { - for( auto& filter : filterGroup->filters ) + resourceParams.prefix = commonArguments->params->resourcePrefix; + + resourceParams.uncompressedSize = entrySize; + + resourceParams.compressedSize = 0; // If required this will be calculated later + + resourceParams.checksum = checksum; + + if( commonArguments->params->calculateBinaryOperation ) { - const std::vector& filterPaths = filter->GetPrefixPaths(); + resourceParams.binaryOperation = ResourceTools::CalculateBinaryOperation( entryPath ); + } - for( auto& filterPath : filterPaths ) + Location location; + + Result calculateLocationResult = location.SetFromRelativePathAndDataChecksum( resourceParams.relativePath, resourceParams.checksum, resourceParams.prefix ); + + if( calculateLocationResult.type != ResultType::SUCCESS ) + { { - if( std::find( searchPaths.begin(), searchPaths.end(), filterPath ) == searchPaths.end() ) - { - searchPaths.push_back( filterPath ); - } + std::unique_lock lock( commonArguments->resourceGroupMutex ); + commonArguments->status = calculateLocationResult; } + return; } - } - // Process files in search paths - { - StatusSettings inputDirectoryStatus; - statusSettings.Update( StatusProgressType::PERCENTAGE, 10, 90, "Creating resource group from directories", &inputDirectoryStatus ); + resourceParams.location = location.ToString(); - std::map checkedRelativePaths; - std::string matchSection = ""; - std::string matchPath = ""; - std::string filterMatchInformation = ""; - ResourceTools::Downloader downloader; - ResourceTools::FileDataStreamIn fileStreamIn( params.fileStreamChunkSize ); + ResourceInfo resource( resourceParams ); - int i = 0; - for( auto searchPath : searchPaths ) - { - std::filesystem::path inputDirectory = params.filterSettings.prefixMapBasePath / searchPath; + // Setup export + std::filesystem::path resourceDestinationPath; - StatusSettings fileProcessingInnerStatusSettings; + Result constructDestinationPathResult = resource.GetDestinationPath( commonArguments->params->exportSettings.destinationSettings, resourceDestinationPath ); - if( inputDirectoryStatus.RequiresStatusUpdates() ) + if( constructDestinationPathResult.type != ResultType::SUCCESS ) + { { - float step = static_cast( 100.0 / searchPaths.size() ); - float progress = static_cast( i * step ); - i++; - inputDirectoryStatus.Update( StatusProgressType::PERCENTAGE, progress, step, "Processing Directory: " + inputDirectory.string(), &fileProcessingInnerStatusSettings ); + std::unique_lock lock( commonArguments->resourceGroupMutex ); + commonArguments->status = constructDestinationPathResult; } + return; + } - // Check validity of search path - if( !std::filesystem::exists( inputDirectory ) ) + // If further calculation is required on the data then calculate here + if( commonArguments->params->compressionCalculationSettings.calculateCompressions || commonArguments->params->exportSettings.enabled ) + { + if( !fileStreamIn.StartRead( entryPath ) ) { - if( params.skipNonExistentInputDirectories ) { - inputDirectoryStatus.Update( StatusProgressType::WARNING, 0, 0, "Skipping input directory as it doesn't exist." ); - - continue; + std::unique_lock lock( commonArguments->resourceGroupMutex ); + commonArguments->status = Result{ ResultType::FAILED_TO_OPEN_FILE_STREAM }; } - else + return; + } + + ResourceTools::FileDataStreamOut exportResourceDataStreamOut; + + std::filesystem::path resourceDestinationPath; + + Result getDestionationResult = resource.GetDestinationPath( commonArguments->params->exportSettings.destinationSettings, resourceDestinationPath ); + + if( getDestionationResult.type != ResultType::SUCCESS ) + { { - return Result{ ResultType::INPUT_DIRECTORY_DOESNT_EXIST, inputDirectory.string() }; + std::unique_lock lock( commonArguments->resourceGroupMutex ); + commonArguments->status = getDestionationResult; } + return; } - // Walk directory - auto recursiveDirectoryIter = std::filesystem::recursive_directory_iterator( inputDirectory ); + std::string compressedData; + + uintmax_t compressedDataSize = 0; + + ResourceTools::GzipCompressionStream compressionStream( &compressedData ); + + // Compression will need to be calculated if specified or an exported resource requires it + bool calculateCompressions = commonArguments->params->compressionCalculationSettings.calculateCompressions; + bool exportResource = commonArguments->params->exportSettings.enabled; + + // Check url for compression information if supplied + // If argument set to skip if file already exists in destination then + // Check for file and if to skip + if( commonArguments->params->compressionCalculationSettings.remoteUrlToAttemptToGetCompression != "" ) { - // Walk entries - for( const std::filesystem::directory_entry& entry : recursiveDirectoryIter ) - { - if( !entry.is_regular_file() ) - { - // Not a file - continue; - } + std::string resourceUrl = commonArguments->params->compressionCalculationSettings.remoteUrlToAttemptToGetCompression.string() + "/" + resourceParams.location; - std::filesystem::path entryPath = entry.path(); + // Check if file exists in remote location + std::string response; - std::string entryPathString; + ResourceTools::Response downloadResponse = ResourceTools::Response::NONE; - if (inputDirectoryStatus.RequiresStatusUpdates()) - { - entryPathString = entryPath.string(); - } + { + std::unique_lock lock( commonArguments->backoffMutex ); - // Process file - // Check each filter group - std::vector> matchedFilters; + commonArguments->backoffConditional.wait( lock, [commonArguments] { + return !commonArguments->backoff; + } ); + } - std::filesystem::path filePathRelativeToInputDirectory = std::filesystem::relative( entryPath, params.filterSettings.prefixMapBasePath ); + downloadResponse = threadArguments->downloader.GetHeader( resourceUrl, response ); - std::string normalisedRelativePath = filePathRelativeToInputDirectory.generic_string(); + auto retryCount = commonArguments->params->compressionCalculationSettings.downloadSettings.retryCount; + auto retrySeconds = commonArguments->params->compressionCalculationSettings.downloadSettings.retrySeconds; - // Filtering is not case sensitive - if (ignoreCase) + if( downloadResponse == ResourceTools::Response::DOWNLOAD_ERROR ) + { { - std::transform( normalisedRelativePath.begin(), normalisedRelativePath.end(), normalisedRelativePath.begin(), []( unsigned char c ) { return std::tolower( c ); } ); - } + std::unique_lock lock( commonArguments->backoffMutex ); - if( fileProcessingInnerStatusSettings.RequiresStatusUpdates() ) - { - filterMatchInformation = ""; - } + std::string warningMessage = "Download Info: Network error, requests backing off."; - for( auto& filterGroup : filterGroups ) - { - if( fileProcessingInnerStatusSettings.RequiresStatusUpdates() ) - { - filterMatchInformation = filterMatchInformation + "Filter:"; - } + commonArguments->ShowStatusWarning( warningMessage ); - // Check each filter in filter group - for( auto& filter : filterGroup->filters ) + commonArguments->backoff = true; + + for( int i = 0; i < retryCount; i++ ) { - // Check that the current search path is relevant to the current filter - const std::vector& filterSearchPaths = filter->GetPrefixPaths(); + downloadResponse = threadArguments->downloader.GetHeader( resourceUrl, response ); - bool searchPathInFilter = false; + std::chrono::seconds backoffTime = retrySeconds * ( i + 1 ); - for( auto& filterPath : filterSearchPaths ) + if( downloadResponse != ResourceTools::Response::DOWNLOAD_ERROR ) { - if( searchPath == filterPath ) - { - searchPathInFilter = true; - break; - } - } + std::string warningMessage = "Download Info: Network requests resuming."; - if( !searchPathInFilter ) - { - // The current search path isn't one that is present in the current filter - continue; - } + commonArguments->ShowStatusWarning( warningMessage ); - if( filter->CheckPath( normalisedRelativePath, matchSection, matchPath ) ) - { - matchedFilters.push_back( filterGroup ); + commonArguments->backoff = false; + + commonArguments->backoffConditional.notify_all(); - filterMatchInformation += " matched section filter - "; - filterMatchInformation += matchSection; - filterMatchInformation += ",matched path - "; - filterMatchInformation += matchPath; break; } + else + { + std::stringstream ss; + + ss << "Download Info: Network error, retry "; + ss << i << " failed. requests backing off for "; + ss << backoffTime.count(); + ss << " seconds."; + + std::string warningMessage = ss.str(); + + commonArguments->ShowStatusWarning( warningMessage ); + } + + std::this_thread::sleep_for( backoffTime ); + } + // No matter what unblock other threads anyway to continue execution. + // The download may have still errored at this point + commonArguments->backoff = false; + + commonArguments->backoffConditional.notify_all(); + } + } + + if( downloadResponse == ResourceTools::Response::SUCCESS ) + { + // Parse the header for the content size + std::string contentLengthStr; - // Check if any filters were matched - if( matchedFilters.empty() ) + if( ResourceTools::Downloader::GetAttributeValueFromHeader( response, "Content-Length", contentLengthStr ) ) { - //File doesn't meet any of the filters - fileProcessingInnerStatusSettings.Update( CarbonResources::StatusProgressType::UNBOUNDED, 0, 0, "Skipping File that didn't match filters: " + entryPathString ); - continue; + // Parse content length + try + { + unsigned long in = std::stoul( contentLengthStr ); + if( in > std::numeric_limits::max() ) + { + std::string warningMessage = "Invalid compression data from header information, compression will be calculated." + resourceUrl; + + commonArguments->ShowStatusWarning( warningMessage ); + } + else + { + // Compression is from the existing file rather than new one. + compressedDataSize = static_cast( in ); + + calculateCompressions = false; + + exportResource = false; + } + } + catch( std::invalid_argument& ) + { + std::string warningMessage = "Invalid compression data from header information, compression will be calculated." + resourceUrl; + + commonArguments->ShowStatusWarning( warningMessage ); + } + catch( std::out_of_range& ) + { + std::string warningMessage = "Invalid compression data from header information, compression will be calculated." + resourceUrl; + + commonArguments->ShowStatusWarning( warningMessage ); + } } + } + else if( downloadResponse == ResourceTools::Response::DOWNLOAD_ERROR ) + { + // If download error persists after set retries then the resource + // is treated as if new and computation continues - // Create resource std::stringstream ss; - if( fileProcessingInnerStatusSettings.RequiresStatusUpdates() ) - { - ss << "Processing file: " - << filePathRelativeToInputDirectory.string() - << " # " - << filterMatchInformation; - } + ss << "Error while downloading header from: " + << resourceUrl + << " After " << retryCount << " retries. Continuing and treating resource as new."; - StatusSettings resourceProcessGranular; - fileProcessingInnerStatusSettings.Update( CarbonResources::StatusProgressType::UNBOUNDED, 0, 0, ss.str(), &resourceProcessGranular ); - - // Process the file data via streaming - ResourceTools::Md5ChecksumStream checksumStream; + std::string warningMessage = ss.str(); - std::filesystem::path resourceRelativePath = std::filesystem::relative( entryPath, inputDirectory ); + commonArguments->ShowStatusWarning( warningMessage ); - std::string resourceRelativePathString = resourceRelativePath.generic_string(); + } + } + - if( ignoreCase ) + if( exportResource ) + { + if( !exportResourceDataStreamOut.StartWrite( resourceDestinationPath ) ) + { { - std::transform( resourceRelativePathString.begin(), resourceRelativePathString.end(), resourceRelativePathString.begin(), []( unsigned char c ) { return std::tolower( c ); } ); + std::unique_lock lock( commonArguments->resourceGroupMutex ); + commonArguments->status = Result{ ResultType::FAILED_TO_OPEN_FILE_STREAM }; } + return; + } + } + + if( ( exportResource && commonArguments->params->exportSettings.destinationSettings.destinationType == ResourceDestinationType::REMOTE_CDN ) ) + { + calculateCompressions = true; + } - if( !fileStreamIn.StartRead( entryPath ) ) + if( calculateCompressions ) + { + commonArguments->numberOfResourcesCompressed++; + + if( !compressionStream.Start() ) + { { - return Result{ ResultType::FAILED_TO_OPEN_FILE_STREAM }; + std::unique_lock lock( commonArguments->resourceGroupMutex ); + commonArguments->status = Result{ ResultType::FAILED_TO_COMPRESS_DATA }; } + return; + } + } - // Calculate without compression to get checksum - while( !fileStreamIn.IsFinished() ) - { - // Update status - if( resourceProcessGranular.RequiresStatusUpdates() ) - { - float step = static_cast( 100.0 / fileStreamIn.Size() ); - float percentage = static_cast( fileStreamIn.GetCurrentPosition() * step ); - resourceProcessGranular.Update( CarbonResources::StatusProgressType::PERCENTAGE, percentage, step, "Stream processing file attributes: " + entryPathString ); - } + if( calculateCompressions || exportResource ) + { + while( !fileStreamIn.IsFinished() ) + { - std::string fileData; - if( !( fileStreamIn >> fileData ) ) + std::string fileData; + + if( !( fileStreamIn >> fileData ) ) + { { - return Result{ ResultType::FAILED_TO_READ_FROM_STREAM }; + std::unique_lock lock( commonArguments->resourceGroupMutex ); + commonArguments->status = Result{ ResultType::FAILED_TO_READ_FROM_STREAM }; } + return; + } - if( !( checksumStream << fileData ) ) + if( calculateCompressions ) + { + if( !( compressionStream << &fileData ) ) { - return Result{ ResultType::FAILED_TO_GENERATE_CHECKSUM }; + { + std::unique_lock lock( commonArguments->resourceGroupMutex ); + commonArguments->status = Result{ ResultType::FAILED_TO_COMPRESS_DATA }; + } + return; } } - // Get Checksum - std::string checksum; - - if( !checksumStream.FinishAndRetrieve( checksum ) ) + // If exporting resource then export the correct data + if( exportResource ) { - return Result{ ResultType::FAILED_TO_GENERATE_CHECKSUM }; + if( commonArguments->params->exportSettings.destinationSettings.destinationType == ResourceDestinationType::LOCAL_CDN || commonArguments->params->exportSettings.destinationSettings.destinationType == ResourceDestinationType::LOCAL_RELATIVE ) + { + // Output Uncompressed Data + if( !( exportResourceDataStreamOut << fileData ) ) + { + { + std::unique_lock lock( commonArguments->resourceGroupMutex ); + commonArguments->status = Result{ ResultType::FAILED_TO_WRITE_TO_STREAM }; + } + return; + } + } + else + { + // Output Compressed Data + if( !( exportResourceDataStreamOut << compressedData ) ) + { + { + std::unique_lock lock( commonArguments->resourceGroupMutex ); + commonArguments->status = Result{ ResultType::FAILED_TO_WRITE_TO_STREAM }; + } + return; + } + } } - // Create resource from parameters - ResourceInfoParams resourceParams; + compressedDataSize += compressedData.size(); + compressedData.clear(); + } + } - resourceParams.relativePath = resourceRelativePathString; + // Finish stream read in + fileStreamIn.Finish(); - resourceParams.prefix = params.resourcePrefix; + if( calculateCompressions ) + { + if( !compressionStream.Finish() ) + { + { + std::unique_lock lock( commonArguments->resourceGroupMutex ); + commonArguments->status = Result{ ResultType::FAILED_TO_WRITE_TO_STREAM }; + } + return; + } - resourceParams.uncompressedSize = entry.file_size(); + // Add remaining compression data + compressedDataSize += compressedData.size(); + } - resourceParams.compressedSize = 0; // If required this will be calculated later + if( commonArguments->params->compressionCalculationSettings.calculateCompressions ) + { + // Set compressed size only if compression is calculated + // If export requires compression but calculation is skipped + // Then the calculations are not set as this may be unexpected + resource.SetCompressedSize( compressedDataSize ); + } - resourceParams.checksum = checksum; + if( exportResource ) + { + if( commonArguments->params->exportSettings.destinationSettings.destinationType == ResourceDestinationType::REMOTE_CDN ) + { + // Add remaining compression data + if( !( exportResourceDataStreamOut << compressedData ) ) + { + { + std::unique_lock lock( commonArguments->resourceGroupMutex ); + commonArguments->status = Result{ ResultType::FAILED_TO_WRITE_TO_STREAM }; + } + return; + } + } - if( params.calculateBinaryOperation ) + if( !exportResourceDataStreamOut.Finish() ) + { { - resourceParams.binaryOperation = ResourceTools::CalculateBinaryOperation( entryPath ); + std::unique_lock lock( commonArguments->resourceGroupMutex ); + commonArguments->status = Result{ ResultType::FAILED_TO_WRITE_TO_STREAM }; } + return; + } + } + } - Location location; + { + std::unique_lock lock( commonArguments->resourceGroupMutex ); - Result calculateLocationResult = location.SetFromRelativePathAndDataChecksum( resourceParams.relativePath, resourceParams.checksum, resourceParams.prefix ); + // Add entry to all relevant resource groups + for( auto& filterGroup : matchedFilters ) + { + // Resource group takes ownership of resource info + filterGroup->resourceGroup->AddResource( new ResourceInfo( resource ) ); + } + } - if( calculateLocationResult.type != ResultType::SUCCESS ) - { - return calculateLocationResult; - } + } - resourceParams.location = location.ToString(); + return; +} - ResourceInfo resource( resourceParams ); +Result ResourceGroup::ResourceGroupImpl::CreateFromFilter( const CreateResourceGroupFromFilterParams& params, StatusSettings& statusSettings ) +{ + ResourceTools::Downloader downloader; - // If resource already exists in other include path then skip this resource - if( checkedRelativePaths.find( resourceRelativePathString ) != checkedRelativePaths.end() ) - { - resourceProcessGranular.Update( CarbonResources::StatusProgressType::WARNING, 0, 0, "Skipping file as it was found in multiple paths: " + entryPathString ); - continue; - } - else - { - checkedRelativePaths[resourceRelativePathString] = 1; - } + // Update status + statusSettings.Update( StatusProgressType::PERCENTAGE, 0, 5, "Creating resource group from filters" ); - // Setup export - std::filesystem::path resourceDestinationPath; + std::shared_ptr poolArguments = std::make_shared(); + poolArguments->ignoreCase = false; + poolArguments->params = ¶ms; - Result constructDestinationPathResult = resource.GetDestinationPath( params.exportSettings.destinationSettings, resourceDestinationPath ); + // Ensure document version is valid + VersionInternal documentVersion( params.outputDocumentVersion ); - if( constructDestinationPathResult.type != ResultType::SUCCESS ) - { - return constructDestinationPathResult; - } + if( documentVersion == VersionInternal( S_CSV_DOCUMENT_VERSION ) ) + { + poolArguments->ignoreCase = true; + } - // If further calculation is required on the data then calculate here - if( params.compressionCalculationSettings.calculateCompressions || params.exportSettings.enabled ) - { - if( !fileStreamIn.StartRead( entryPath ) ) - { - return Result{ ResultType::FAILED_TO_OPEN_FILE_STREAM }; - } + if( !documentVersion.isVersionValid() ) + { + return Result{ ResultType::DOCUMENT_VERSION_UNSUPPORTED }; + } - ResourceTools::FileDataStreamOut exportResourceDataStreamOut; + statusSettings.Update( StatusProgressType::PERCENTAGE, 5, 5, "Loading filter files" ); - std::filesystem::path resourceDestinationPath; + // Initialise Resource Filters + for( auto& filter : params.filterSettings.filters ) + { + if( filter->filterFilePaths.empty() ) + { + std::string errorMsg = "No filter files provided."; + return Result{ ResultType::FAILED_TO_INITIALIZE_RESOURCE_FILTER, errorMsg }; + } - Result getDestionationResult = resource.GetDestinationPath( params.exportSettings.destinationSettings, resourceDestinationPath ); + std::shared_ptr filterGroup = std::make_shared(); - if( getDestionationResult.type != ResultType::SUCCESS ) - { - return getDestionationResult; - } + filterGroup->resourceGroup = filter->outputResourceGroup->m_impl; - std::string compressedData; + for( auto filterPath : filter->filterFilePaths ) + { + // Get filter file data + std::string filterData; - uintmax_t compressedDataSize = 0; + if( !ResourceTools::GetLocalFileData( filterPath, filterData ) ) + { + std::string errorMsg = "Failed to open filter file: " + filterPath.string(); + return Result{ ResultType::FAILED_TO_OPEN_FILE, errorMsg }; + } - ResourceTools::GzipCompressionStream compressionStream( &compressedData ); + ResourceTools::FilterFile fileData; - // Compression will need to be calculated if specified or an exported resource requires it - bool calculateCompressions = params.compressionCalculationSettings.calculateCompressions; - bool exportResource = params.exportSettings.enabled; + try + { + ResourceTools::FilterFileReader::LoadFromIniFileData( filterData.data(), filterData.size(), fileData, poolArguments->ignoreCase ); + } + catch( const std::exception& e ) + { + std::string errorMsg = "Failed to read filter file: " + std::string( e.what() ); - // Check url for compression information if supplied - // If argument set to skip if file already exists in destination then - // Check for file and if to skip - if( params.compressionCalculationSettings.remoteUrlToAttemptToGetCompression != "" ) - { + return Result{ ResultType::FAILED_TO_INITIALIZE_RESOURCE_FILTER, errorMsg }; + } - std::string resourceUrl = params.compressionCalculationSettings.remoteUrlToAttemptToGetCompression.string() + "/" + resourceParams.location; + std::unique_ptr filter = std::make_unique(); - // Check if file exists in remote location - std::string response; + if( !filter->SetFromFilterFileData( fileData ) ) + { + return Result{ ResultType::FAILED_TO_INITIALIZE_RESOURCE_FILTER }; + } - ResourceTools::Response downloadResponse; + filterGroup->filters.push_back( std::move( filter ) ); + } - downloadResponse = downloader.GetHeader( resourceUrl, params.compressionCalculationSettings.downloadSettings.retryCount, params.compressionCalculationSettings.downloadSettings.retrySeconds, response ); + poolArguments->filterGroups.push_back( std::move( filterGroup ) ); + } - if( downloadResponse == ResourceTools::Response::SUCCESS ) - { - resourceProcessGranular.Update( CarbonResources::StatusProgressType::WARNING, 0, 0, "Attempting to get compression data from remote: " + resourceUrl ); + // Populate all search paths + std::vector searchPaths; - // Parse the header for the content size - std::string contentLengthStr; + for( auto& filterGroup : poolArguments->filterGroups ) + { + for( auto& filter : filterGroup->filters ) + { + const std::vector& filterPaths = filter->GetPrefixPaths(); - if( ResourceTools::Downloader::GetAttributeValueFromHeader( response, "Content-Length", contentLengthStr ) ) - { - // Parse content length - try - { - unsigned long in = std::stoul( contentLengthStr ); - if( in > std::numeric_limits::max() ) - { - resourceProcessGranular.Update( CarbonResources::StatusProgressType::WARNING, 0, 0, "Invalid compression data from header information, compression will be calculated." + resourceUrl ); - } - else - { - // Compression is from the existing file rather than new one. - compressedDataSize = static_cast( in ); - calculateCompressions = false; - exportResource = false; - } - } - catch( std::invalid_argument& ) - { - resourceProcessGranular.Update( CarbonResources::StatusProgressType::WARNING, 0, 0, "Invalid compression data from header information, compression will be calculated." + resourceUrl ); - } - catch( std::out_of_range& ) - { - resourceProcessGranular.Update( CarbonResources::StatusProgressType::WARNING, 0, 0, "Invalid compression data from header information, compression will be calculated." + resourceUrl ); - } - } - } - else if( downloadResponse == ResourceTools::Response::DOWNLOAD_ERROR ) - { - std::stringstream ss; + for( auto& filterPath : filterPaths ) + { + if( std::find( searchPaths.begin(), searchPaths.end(), filterPath ) == searchPaths.end() ) + { + searchPaths.push_back( filterPath ); + } + } + } + } - ss << "Error while downloading header from: " - << resourceUrl - << " Response:\n" - << response; + // Process files in search paths + { + StatusSettings directoryStatusSettings; + statusSettings.Update( StatusProgressType::PERCENTAGE, 10, 90, "Creating resource group from directories", &directoryStatusSettings ); - return Result{ ResultType::FAILED_TO_DOWNLOAD_FILE, ss.str() }; - } + // num threads must at least be 1 + // If zero threads are passed via async settings then + // the job is done on the main thread + int numThreads = params.asyncSettings.numberOfThreads + 1; - if( calculateCompressions ) - { - resourceProcessGranular.Update( CarbonResources::StatusProgressType::WARNING, 0, 0, "Resource not found at remote location, compression calculation will continue: " + resourceUrl ); - } - } + // numThreads-1 as the value passed is the number of threads to create + // If numThreads-1==0 then the tasks for the pool will all execute on this thread + ThreadPool, std::shared_ptr> threadPool( numThreads - 1 ); - if( exportResource ) - { - if( !exportResourceDataStreamOut.StartWrite( resourceDestinationPath ) ) - { - return Result{ ResultType::FAILED_TO_OPEN_FILE_STREAM }; - } - } + threadPool.Start(); - if( ( exportResource && params.exportSettings.destinationSettings.destinationType == ResourceDestinationType::REMOTE_CDN ) ) - { - calculateCompressions = true; - } + int searchPathIndex = 0; - if( calculateCompressions ) - { - if( !compressionStream.Start() ) - { - return Result{ ResultType::FAILED_TO_COMPRESS_DATA }; - } - } + float statusProgressStep = static_cast(100.0 / searchPaths.size()); - if( calculateCompressions || exportResource ) - { - while( !fileStreamIn.IsFinished() ) - { - // Update status - if( resourceProcessGranular.RequiresStatusUpdates() ) - { - float step = static_cast( 100.0 / fileStreamIn.Size() ); - float percentage = static_cast( fileStreamIn.GetCurrentPosition() * step ); + for( auto searchPath : searchPaths ) + { + std::filesystem::path inputDirectory = params.filterSettings.prefixMapBasePath / searchPath; + + if (directoryStatusSettings.RequiresStatusUpdates()) + { + float progress = statusProgressStep * searchPathIndex; + searchPathIndex++; + directoryStatusSettings.Update( StatusProgressType::PERCENTAGE, progress, statusProgressStep, "Checking directory: " + inputDirectory.lexically_normal().generic_string(), &poolArguments->statusSettings ); + } - std::string info; - if( calculateCompressions ) - { - info += "Compressing"; - } + if( !std::filesystem::exists( inputDirectory ) ) + { + if( params.skipNonExistentInputDirectories ) + { + directoryStatusSettings.Update( StatusProgressType::WARNING, 0, 0, "Skipping input directory as it doesn't exist." ); - if( exportResource ) - { - if( info != "" ) - { - info += " & "; - } + continue; + } + else + { + return Result{ ResultType::INPUT_DIRECTORY_DOESNT_EXIST, inputDirectory.string() }; + } + } - info += "Exporting"; - } + //Create thread structure + std::vector> threadArguments; - info += " Resource: "; + for( int i = 0; i < numThreads; i++ ) + { + std::shared_ptr arguments = std::make_shared(); - resourceProcessGranular.Update( CarbonResources::StatusProgressType::PERCENTAGE, percentage, step, info + entryPathString ); - } + arguments->inputDirectory = inputDirectory; - std::string fileData; + arguments->searchPath = searchPath; - if( !( fileStreamIn >> fileData ) ) - { - return Result{ ResultType::FAILED_TO_READ_FROM_STREAM }; - } + threadArguments.push_back( std::move( arguments ) ); + } - if( calculateCompressions ) - { - if( !( compressionStream << &fileData ) ) - { - return Result{ ResultType::FAILED_TO_COMPRESS_DATA }; - } - } + int threadId = 0; - // If exporting resource then export the correct data - if( exportResource ) - { - if( params.exportSettings.destinationSettings.destinationType == ResourceDestinationType::LOCAL_CDN || params.exportSettings.destinationSettings.destinationType == ResourceDestinationType::LOCAL_RELATIVE ) - { - // Output Uncompressed Data - if( !( exportResourceDataStreamOut << fileData ) ) - { - return Result{ ResultType::FAILED_TO_WRITE_TO_STREAM }; - } - } - else - { - // Output Compressed Data - if( !( exportResourceDataStreamOut << compressedData ) ) - { - return Result{ ResultType::FAILED_TO_WRITE_TO_STREAM }; - } - } - } + // Walk directory + auto recursiveDirectoryIter = std::filesystem::recursive_directory_iterator( inputDirectory ); + { - compressedDataSize += compressedData.size(); - compressedData.clear(); - } - } + // Walk entries + for( const std::filesystem::directory_entry& entry : recursiveDirectoryIter ) + { - // Finish stream read in - fileStreamIn.Finish(); + if( !entry.is_regular_file() ) + { + // Not a file + continue; + } - if( calculateCompressions ) - { - if( !compressionStream.Finish() ) - { - return Result{ ResultType::FAILED_TO_WRITE_TO_STREAM }; - } + std::filesystem::path entryPath = entry.path(); - // Add remaining compression data - compressedDataSize += compressedData.size(); - } + - if( params.compressionCalculationSettings.calculateCompressions ) - { - // Set compressed size only if compression is calculated - // If export requires compression but calculation is skipped - // Then the calculations are not set as this may be unexpected - resource.SetCompressedSize( compressedDataSize ); - } + threadArguments.at( threadId )->entryPaths.push_back( entry ); - if( exportResource ) - { - if( params.exportSettings.destinationSettings.destinationType == ResourceDestinationType::REMOTE_CDN ) - { - // Add remaining compression data - if( !( exportResourceDataStreamOut << compressedData ) ) - { - return Result{ ResultType::FAILED_TO_WRITE_TO_STREAM }; - } - } + poolArguments->totalNumberOfResources++; - if( !exportResourceDataStreamOut.Finish() ) - { - return Result{ ResultType::FAILED_TO_WRITE_TO_STREAM }; - } - } - } + threadId++; - // Add entry to all relevant resource groups - for( auto& filterGroup : matchedFilters ) + if( threadId >= numThreads ) { - // Resource group takes ownership of resource info - filterGroup->resourceGroup->AddResource( new ResourceInfo( resource ) ); + threadId = 0; } } + + for( int i = 0; i < numThreads; i++ ) + { + threadPool.AddTask( CreateResourceGroupsFromFilterWorker, poolArguments, threadArguments.at( i ) ); + } + + threadPool.Join(); + + if( poolArguments->status.type != ResultType::SUCCESS ) + { + return poolArguments->status; + } + + // Subsequent paths need to have an added check to ensure they have + // not already been processed + // this is not needed for the first path as files will never have been + // encountered before so for efficiency is skipped. + poolArguments->checkForDuplicateRelativePaths = true; } + + poolArguments->statusSettings.Update( StatusProgressType::END, 100, 0, "process complete" ); } } @@ -1635,7 +1889,7 @@ Result ResourceGroup::ResourceGroupImpl::ProcessChunk( ResourceTools::GetChunk& } } - if( !checksumStream.FinishAndRetrieve( checksum ) ) + if( !checksumStream.Retrieve( checksum ) ) { return Result( { ResultType::FAILED_TO_GENERATE_CHECKSUM } ); } diff --git a/src/ResourceInfo/ResourceInfo.cpp b/src/ResourceInfo/ResourceInfo.cpp index f11f8b9..dea9ebb 100644 --- a/src/ResourceInfo/ResourceInfo.cpp +++ b/src/ResourceInfo/ResourceInfo.cpp @@ -1142,7 +1142,7 @@ Result ResourceInfo::SetParametersFromSourceStream( ResourceTools::FileDataStrea } } - if( !md5ChecksumStream.FinishAndRetrieve( checksum ) ) + if( !md5ChecksumStream.Retrieve( checksum ) ) { stream.Seek( start ); return Result{ ResultType::FAILED_TO_GENERATE_CHECKSUM }; diff --git a/src/StatusSettings.cpp b/src/StatusSettings.cpp index 918b6e9..6e24563 100644 --- a/src/StatusSettings.cpp +++ b/src/StatusSettings.cpp @@ -14,7 +14,10 @@ StatusSettings::StatusSettings() : StatusSettings ::~StatusSettings() { - Update( StatusProgressType::END, 100, 0, "Process complete." ); + if (m_lastUpdate.progress < 100) + { + Update( StatusProgressType::END, 100, 0, "Process complete." ); + } } void StatusSettings::SetCallbackSettings( const CallbackSettings& callbackSettings ) diff --git a/tests/src/ResourceToolsFilterTest.cpp b/tests/src/ResourceToolsFilterTest.cpp index 941672e..7459cf0 100644 --- a/tests/src/ResourceToolsFilterTest.cpp +++ b/tests/src/ResourceToolsFilterTest.cpp @@ -123,7 +123,7 @@ TEST_F( ResourceToolsTest, Filtering_AsteriskPatternMatch ) std::filesystem::path validPath = "File"; - ASSERT_TRUE( resourceFilter.CheckPath( validPath ) ); + ASSERT_TRUE( resourceFilter.CheckPath( validPath.generic_string() ) ); } @@ -142,7 +142,7 @@ TEST_F( ResourceToolsTest, Filtering_AsteriskPatternMissmatch ) std::filesystem::path validPath = "Subfolder/File"; - ASSERT_FALSE( resourceFilter.CheckPath( validPath ) ); + ASSERT_FALSE( resourceFilter.CheckPath( validPath.generic_string() ) ); } TEST_F( ResourceToolsTest, Filtering_ElipsisPatternMatch ) @@ -165,7 +165,7 @@ TEST_F( ResourceToolsTest, Filtering_ElipsisPatternMatch ) for (auto path : validPaths) { - ASSERT_TRUE( resourceFilter.CheckPath( path ) ); + ASSERT_TRUE( resourceFilter.CheckPath( path.generic_string() ) ); } } @@ -183,11 +183,11 @@ TEST_F( ResourceToolsTest, Filtering_SpecificFile ) std::filesystem::path validPath = "File"; - ASSERT_TRUE( resourceFilter.CheckPath( validPath ) ); + ASSERT_TRUE( resourceFilter.CheckPath( validPath.generic_string() ) ); std::filesystem::path invalidPath = "NonMatching"; - ASSERT_FALSE( resourceFilter.CheckPath( invalidPath ) ); + ASSERT_FALSE( resourceFilter.CheckPath( invalidPath.generic_string() ) ); } TEST_F( ResourceToolsTest, Filtering_FilterIncludeRule ) @@ -205,11 +205,11 @@ TEST_F( ResourceToolsTest, Filtering_FilterIncludeRule ) std::filesystem::path validPath = "File.type1"; - ASSERT_TRUE( resourceFilter.CheckPath( validPath ) ); + ASSERT_TRUE( resourceFilter.CheckPath( validPath.generic_string() ) ); std::filesystem::path invalidPath = "File.type2"; - ASSERT_FALSE( resourceFilter.CheckPath( invalidPath ) ); + ASSERT_FALSE( resourceFilter.CheckPath( invalidPath.generic_string() ) ); } TEST_F( ResourceToolsTest, Filtering_FilterExcludeRule ) @@ -227,11 +227,11 @@ TEST_F( ResourceToolsTest, Filtering_FilterExcludeRule ) std::filesystem::path validPath = "File.type2"; - ASSERT_TRUE( resourceFilter.CheckPath( validPath ) ); + ASSERT_TRUE( resourceFilter.CheckPath( validPath.generic_string() ) ); std::filesystem::path invalidPath = "File.type1"; - ASSERT_FALSE( resourceFilter.CheckPath( invalidPath ) ); + ASSERT_FALSE( resourceFilter.CheckPath( invalidPath.generic_string() ) ); } @@ -250,11 +250,11 @@ TEST_F( ResourceToolsTest, Filtering_FilterOverlappingIncludeExcludeRule1 ) std::filesystem::path invalidPath = "File.type1"; - ASSERT_FALSE( resourceFilter.CheckPath( invalidPath ) ); + ASSERT_FALSE( resourceFilter.CheckPath( invalidPath.generic_string() ) ); std::filesystem::path invalidPath2 = "File.type2"; - ASSERT_FALSE( resourceFilter.CheckPath( invalidPath2 ) ); + ASSERT_FALSE( resourceFilter.CheckPath( invalidPath2.generic_string() ) ); } @@ -273,11 +273,11 @@ TEST_F( ResourceToolsTest, Filtering_FilterOverlappingIncludeExcludeRule2 ) std::filesystem::path invalidPath = "File.type1"; - ASSERT_FALSE( resourceFilter.CheckPath( invalidPath ) ); + ASSERT_FALSE( resourceFilter.CheckPath( invalidPath.generic_string() ) ); std::filesystem::path invalidPath2 = "File.type2"; - ASSERT_FALSE( resourceFilter.CheckPath( invalidPath2 ) ); + ASSERT_FALSE( resourceFilter.CheckPath( invalidPath2.generic_string() ) ); } TEST_F( ResourceToolsTest, Filtering_FilterRespathExcludeRule ) @@ -294,11 +294,11 @@ TEST_F( ResourceToolsTest, Filtering_FilterRespathExcludeRule ) std::filesystem::path invalidPath = "File"; - ASSERT_FALSE( resourceFilter.CheckPath( invalidPath ) ); + ASSERT_FALSE( resourceFilter.CheckPath( invalidPath.generic_string() ) ); std::filesystem::path validPath = "Another"; - ASSERT_TRUE( resourceFilter.CheckPath( validPath ) ); + ASSERT_TRUE( resourceFilter.CheckPath( validPath.generic_string() ) ); } TEST_F( ResourceToolsTest, Filtering_FilterRespathIncludeRule ) @@ -315,11 +315,11 @@ TEST_F( ResourceToolsTest, Filtering_FilterRespathIncludeRule ) std::filesystem::path validPath = "File"; - ASSERT_TRUE( resourceFilter.CheckPath( validPath ) ); + ASSERT_TRUE( resourceFilter.CheckPath( validPath.generic_string() ) ); std::filesystem::path invalidPath = "Another"; - ASSERT_FALSE( resourceFilter.CheckPath( invalidPath ) ); + ASSERT_FALSE( resourceFilter.CheckPath( invalidPath.generic_string() ) ); } TEST_F( ResourceToolsTest, Filtering_FilterIncludeRuleWithOverlappingRespathExcludeRule ) @@ -337,11 +337,11 @@ TEST_F( ResourceToolsTest, Filtering_FilterIncludeRuleWithOverlappingRespathExcl std::filesystem::path invalidPath = "File"; - ASSERT_FALSE( resourceFilter.CheckPath( invalidPath ) ); + ASSERT_FALSE( resourceFilter.CheckPath( invalidPath.generic_string() ) ); std::filesystem::path invalidPath2 = "Another"; - ASSERT_FALSE( resourceFilter.CheckPath( invalidPath2 ) ); + ASSERT_FALSE( resourceFilter.CheckPath( invalidPath2.generic_string() ) ); } TEST_F( ResourceToolsTest, Filtering_FilterExcludeRuleWithOverlappingRespathIncludeRule ) @@ -359,11 +359,11 @@ TEST_F( ResourceToolsTest, Filtering_FilterExcludeRuleWithOverlappingRespathIncl std::filesystem::path invalidPath = "File"; - ASSERT_FALSE( resourceFilter.CheckPath( invalidPath ) ); + ASSERT_FALSE( resourceFilter.CheckPath( invalidPath.generic_string() ) ); std::filesystem::path invalidPath2 = "Another"; - ASSERT_FALSE( resourceFilter.CheckPath( invalidPath2 ) ); + ASSERT_FALSE( resourceFilter.CheckPath( invalidPath2.generic_string() ) ); } TEST_F( ResourceToolsTest, Filtering_MultiPrefixMatch ) @@ -381,11 +381,11 @@ TEST_F( ResourceToolsTest, Filtering_MultiPrefixMatch ) std::filesystem::path prefix1ValidPath = "Path1/File"; - ASSERT_TRUE( resourceFilter.CheckPath( prefix1ValidPath ) ); + ASSERT_TRUE( resourceFilter.CheckPath( prefix1ValidPath.generic_string() ) ); std::filesystem::path prefix2ValidPath = "Path2/File"; - ASSERT_TRUE( resourceFilter.CheckPath( prefix2ValidPath ) ); + ASSERT_TRUE( resourceFilter.CheckPath( prefix2ValidPath.generic_string() ) ); } TEST_F( ResourceToolsTest, Filtering_MultiPrefixExcludeAppliesAcrossBoth ) @@ -404,19 +404,19 @@ TEST_F( ResourceToolsTest, Filtering_MultiPrefixExcludeAppliesAcrossBoth ) std::filesystem::path prefix1InvalidPath = "Path1/File"; - ASSERT_FALSE( resourceFilter.CheckPath( prefix1InvalidPath ) ); + ASSERT_FALSE( resourceFilter.CheckPath( prefix1InvalidPath.generic_string() ) ); std::filesystem::path prefix2InvalidPath = "Path2/File"; - ASSERT_FALSE( resourceFilter.CheckPath( prefix2InvalidPath ) ); + ASSERT_FALSE( resourceFilter.CheckPath( prefix2InvalidPath.generic_string() ) ); std::filesystem::path prefix1ValidPath = "Path1/Another"; - ASSERT_TRUE( resourceFilter.CheckPath( prefix1ValidPath ) ); + ASSERT_TRUE( resourceFilter.CheckPath( prefix1ValidPath.generic_string() ) ); std::filesystem::path prefix2ValidPath = "Path2/Another"; - ASSERT_TRUE( resourceFilter.CheckPath( prefix2ValidPath ) ); + ASSERT_TRUE( resourceFilter.CheckPath( prefix2ValidPath.generic_string() ) ); } TEST_F( ResourceToolsTest, Filtering_MultiPrefixIncludeAppliesAcrossBoth ) @@ -435,19 +435,19 @@ TEST_F( ResourceToolsTest, Filtering_MultiPrefixIncludeAppliesAcrossBoth ) std::filesystem::path prefix1ValidPath = "Path1/File"; - ASSERT_TRUE( resourceFilter.CheckPath( prefix1ValidPath ) ); + ASSERT_TRUE( resourceFilter.CheckPath( prefix1ValidPath.generic_string() ) ); std::filesystem::path prefix2ValidPath = "Path2/File"; - ASSERT_TRUE( resourceFilter.CheckPath( prefix2ValidPath ) ); + ASSERT_TRUE( resourceFilter.CheckPath( prefix2ValidPath.generic_string() ) ); std::filesystem::path prefix1InvalidPath = "Path1/Another"; - ASSERT_FALSE( resourceFilter.CheckPath( prefix1InvalidPath ) ); + ASSERT_FALSE( resourceFilter.CheckPath( prefix1InvalidPath.generic_string() ) ); std::filesystem::path prefix2inValidPath = "Path2/Another"; - ASSERT_FALSE( resourceFilter.CheckPath( prefix2inValidPath ) ); + ASSERT_FALSE( resourceFilter.CheckPath( prefix2inValidPath.generic_string() ) ); } TEST_F( ResourceToolsTest, Filtering_MultiPrefixFilterIncludeOverrideExcludeAppliesAccrossBoth ) @@ -466,19 +466,19 @@ TEST_F( ResourceToolsTest, Filtering_MultiPrefixFilterIncludeOverrideExcludeAppl std::filesystem::path prefix1InvalidPath = "Path1/File"; - ASSERT_FALSE( resourceFilter.CheckPath( prefix1InvalidPath ) ); + ASSERT_FALSE( resourceFilter.CheckPath( prefix1InvalidPath.generic_string() ) ); std::filesystem::path prefix2InvalidPath = "Path2/File"; - ASSERT_FALSE( resourceFilter.CheckPath( prefix2InvalidPath ) ); + ASSERT_FALSE( resourceFilter.CheckPath( prefix2InvalidPath.generic_string() ) ); std::filesystem::path prefix1InvalidPath2 = "Path1/Another"; - ASSERT_FALSE( resourceFilter.CheckPath( prefix1InvalidPath2 ) ); + ASSERT_FALSE( resourceFilter.CheckPath( prefix1InvalidPath2.generic_string() ) ); std::filesystem::path prefix2InvalidPath2 = "Path2/Another"; - ASSERT_FALSE( resourceFilter.CheckPath( prefix2InvalidPath2 ) ); + ASSERT_FALSE( resourceFilter.CheckPath( prefix2InvalidPath2.generic_string() ) ); } TEST_F( ResourceToolsTest, Filtering_MultiPrefixFilterIncludeAddedPathIncludeAppliesToLaterPaths ) @@ -497,11 +497,11 @@ TEST_F( ResourceToolsTest, Filtering_MultiPrefixFilterIncludeAddedPathIncludeApp std::filesystem::path prefix1ValidPath = "Path1/File.type1"; - ASSERT_TRUE( resourceFilter.CheckPath( prefix1ValidPath ) ); + ASSERT_TRUE( resourceFilter.CheckPath( prefix1ValidPath.generic_string() ) ); std::filesystem::path prefix2ValidPath = "Path2/File.type2"; - ASSERT_TRUE( resourceFilter.CheckPath( prefix2ValidPath ) ); + ASSERT_TRUE( resourceFilter.CheckPath( prefix2ValidPath.generic_string() ) ); } TEST_F( ResourceToolsTest, Filtering_MultiPrefixFilterIncludeWithLaterRespathExclude ) @@ -521,11 +521,11 @@ TEST_F( ResourceToolsTest, Filtering_MultiPrefixFilterIncludeWithLaterRespathExc std::filesystem::path prefix1ValidPath = "Path1/File.type1"; - ASSERT_TRUE( resourceFilter.CheckPath( prefix1ValidPath ) ); + ASSERT_TRUE( resourceFilter.CheckPath( prefix1ValidPath.generic_string() ) ); std::filesystem::path prefix2InvalidPath = "Path2/File.type1"; - ASSERT_FALSE( resourceFilter.CheckPath( prefix2InvalidPath ) ); + ASSERT_FALSE( resourceFilter.CheckPath( prefix2InvalidPath.generic_string() ) ); } TEST_F( ResourceToolsTest, Filtering_SpecificFileWithInclude ) @@ -543,7 +543,7 @@ TEST_F( ResourceToolsTest, Filtering_SpecificFileWithInclude ) std::filesystem::path invalidPath = "File"; - ASSERT_FALSE( resourceFilter.CheckPath( invalidPath ) ); + ASSERT_FALSE( resourceFilter.CheckPath( invalidPath.generic_string() ) ); } TEST_F( ResourceToolsTest, Filtering_SpecificFileWithNonMatchingExclude ) @@ -563,11 +563,11 @@ TEST_F( ResourceToolsTest, Filtering_SpecificFileWithNonMatchingExclude ) std::filesystem::path invalidPath = "File"; - ASSERT_FALSE( resourceFilter.CheckPath( invalidPath ) ); + ASSERT_FALSE( resourceFilter.CheckPath( invalidPath.generic_string() ) ); std::filesystem::path validPath = "File2"; - ASSERT_TRUE( resourceFilter.CheckPath( validPath ) ); + ASSERT_TRUE( resourceFilter.CheckPath( validPath.generic_string() ) ); } TEST_F( ResourceToolsTest, Filtering_SpecificFileWithOverlappingExcludeRule ) @@ -585,7 +585,7 @@ TEST_F( ResourceToolsTest, Filtering_SpecificFileWithOverlappingExcludeRule ) std::filesystem::path validPath = "File"; - ASSERT_TRUE( resourceFilter.CheckPath( validPath ) ); + ASSERT_TRUE( resourceFilter.CheckPath( validPath.generic_string() ) ); } TEST_F( ResourceToolsTest, Filtering_FilterGlobalIncludeRule ) @@ -603,9 +603,9 @@ TEST_F( ResourceToolsTest, Filtering_FilterGlobalIncludeRule ) std::filesystem::path validPath = "File.type1"; - ASSERT_TRUE( resourceFilter.CheckPath( validPath ) ); + ASSERT_TRUE( resourceFilter.CheckPath( validPath.generic_string() ) ); std::filesystem::path invalidPath = "File.type2"; - ASSERT_FALSE( resourceFilter.CheckPath( invalidPath ) ); + ASSERT_FALSE( resourceFilter.CheckPath( invalidPath.generic_string() ) ); } \ No newline at end of file diff --git a/tests/src/ResourceToolsLibraryTest.cpp b/tests/src/ResourceToolsLibraryTest.cpp index b7cd4a5..98ce94d 100644 --- a/tests/src/ResourceToolsLibraryTest.cpp +++ b/tests/src/ResourceToolsLibraryTest.cpp @@ -138,10 +138,8 @@ TEST_F( ResourceToolsTest, DownloadHeader ) std::string sourcePathString( sourcePath.string() ); std::string url = "file://" + sourcePathString; - std::chrono::seconds retrySeconds{ 0 }; - int retryCount = 3; std::string response = ""; - EXPECT_EQ( downloader.GetHeader( url, retryCount, retrySeconds, response ), ResourceTools::Response::SUCCESS ); + EXPECT_EQ( downloader.GetHeader( url, response ), ResourceTools::Response::SUCCESS ); EXPECT_NE( response, "" ); @@ -455,7 +453,7 @@ TEST_F( ResourceToolsTest, ResourceChunking ) std::string reconstitutedChecksum; - EXPECT_TRUE( reconstitutedResource1ChecksumStream.FinishAndRetrieve( reconstitutedChecksum ) ); + EXPECT_TRUE( reconstitutedResource1ChecksumStream.Retrieve( reconstitutedChecksum ) ); EXPECT_EQ( reconsititedResource.expectedChecksum, reconstitutedChecksum ); } @@ -867,11 +865,11 @@ TEST_F( ResourceToolsTest, GzipStreams ) ResourceTools::GetLocalFileData( testFile, originalData ); std::string originalChecksum; - EXPECT_TRUE( originalMd5Stream.FinishAndRetrieve( originalChecksum ) ); + EXPECT_TRUE( originalMd5Stream.Retrieve( originalChecksum ) ); ResourceTools::Md5ChecksumStream uncompressedMd5Stream; uncompressedMd5Stream << uncompressedData; std::string uncompressedChecksum; - EXPECT_TRUE( uncompressedMd5Stream.FinishAndRetrieve( uncompressedChecksum ) ); + EXPECT_TRUE( uncompressedMd5Stream.Retrieve( uncompressedChecksum ) ); EXPECT_EQ( uncompressedChecksum, originalChecksum ); } diff --git a/tests/src/ResourcesCliTest.cpp b/tests/src/ResourcesCliTest.cpp index d4b99ff..b8782ab 100644 --- a/tests/src/ResourcesCliTest.cpp +++ b/tests/src/ResourcesCliTest.cpp @@ -756,6 +756,10 @@ TEST_F( ResourcesCliTest, CreateResourceGroupFromFilter ) arguments.push_back( prefixBasePath.string() ); + arguments.push_back( "--number-of-threads" ); + + arguments.push_back( "0" ); + int res = RunCli( arguments, output ); ASSERT_EQ( res, 0 ); @@ -816,6 +820,10 @@ TEST_F( ResourcesCliTest, CreateResourceGroupFromFilterExportResources ) arguments.push_back( exportedResourcesPath.string() ); + arguments.push_back( "--number-of-threads" ); + + arguments.push_back( "0" ); + int res = RunCli( arguments, output ); ASSERT_EQ( res, 0 ); diff --git a/tests/src/ResourcesLibraryTest.cpp b/tests/src/ResourcesLibraryTest.cpp index 400b68c..fb9e05c 100644 --- a/tests/src/ResourcesLibraryTest.cpp +++ b/tests/src/ResourcesLibraryTest.cpp @@ -1737,6 +1737,8 @@ TEST_F( ResourcesLibraryTest, CreateResourceGroupFromFilter ) params.callbackSettings.statusCallback = StatusUpdate; + params.asyncSettings.numberOfThreads = 0; + EXPECT_EQ( resourceGroup.CreateFromFilter( params ).type, CarbonResources::ResultType::SUCCESS ); EXPECT_TRUE( StatusIsValid() ); @@ -1783,6 +1785,8 @@ TEST_F( ResourcesLibraryTest, CreateResourceGroupFromFilterExportResources ) params.callbackSettings.statusCallback = StatusUpdate; + params.asyncSettings.numberOfThreads = 0; + EXPECT_EQ( resourceGroup.CreateFromFilter( params ).type, CarbonResources::ResultType::SUCCESS ); EXPECT_TRUE( StatusIsValid() ); diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 4e672a9..cea4b90 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -23,6 +23,7 @@ set(SRC_FILES include/ResourceTools.h include/RollingChecksum.h include/ScopedFile.h + include/ThreadPool.h src/BundleStreamIn.cpp src/BundleStreamOut.cpp diff --git a/tools/include/Downloader.h b/tools/include/Downloader.h index db174cd..9578cd8 100644 --- a/tools/include/Downloader.h +++ b/tools/include/Downloader.h @@ -12,6 +12,7 @@ namespace ResourceTools enum class Response { + NONE, SUCCESS, FILE_NOT_FOUND, DOWNLOAD_ERROR, @@ -23,10 +24,12 @@ class Downloader { public: Downloader(); + ~Downloader(); + bool DownloadFile( const std::string& url, const std::filesystem::path& outputPath, const std::chrono::seconds& retrySeconds ); - Response GetHeader( const std::string& url, uintmax_t retryCount, const std::chrono::seconds& retrySeconds, std::string& response ); + Response GetHeader( const std::string& url, std::string& response ); static bool GetAttributeValueFromHeader( const std::string& header, const std::string& attributeName, std::string& value ); diff --git a/tools/include/Md5ChecksumStream.h b/tools/include/Md5ChecksumStream.h index 17012d3..126a4f3 100644 --- a/tools/include/Md5ChecksumStream.h +++ b/tools/include/Md5ChecksumStream.h @@ -27,7 +27,9 @@ class Md5ChecksumStream ~Md5ChecksumStream(); - bool FinishAndRetrieve( std::string& checksum ); + void Start(); + + bool Retrieve( std::string& checksum ); bool operator<<( const std::string& data ); diff --git a/tools/include/ResourceFilter.h b/tools/include/ResourceFilter.h index de4dadd..ce98857 100644 --- a/tools/include/ResourceFilter.h +++ b/tools/include/ResourceFilter.h @@ -22,6 +22,11 @@ struct FilterPath std::string path; + // Stored to speed up comparison during checks + bool pathContainsWildcard; + + uintmax_t pathLength; + std::string matchPattern; std::regex matchPatternRegex; @@ -42,14 +47,12 @@ class ResourceFilter bool SetFromFilterFileData( const FilterFile& fileData ); - bool CheckPath( const std::filesystem::path& path ) const; - - bool CheckPath( const std::filesystem::path& path, std::string& matchSectionId, std::string& matchPath ) const; + bool CheckPath( const std::string& path, std::string* matchSectionId = nullptr, std::string* matchPath = nullptr ) const; const std::vector& GetPrefixPaths() const; private: - void ConvertResPathToPattern( std::string resPath, std::string& pattern ) const; + void ConvertResPathToPattern( std::string resPath, std::string& pattern, bool& pathContainsWildcard ) const; private: std::map>> m_paths; diff --git a/tools/include/ThreadPool.h b/tools/include/ThreadPool.h new file mode 100644 index 0000000..d59b83c --- /dev/null +++ b/tools/include/ThreadPool.h @@ -0,0 +1,176 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +template +class ThreadPool +{ +public: + + ThreadPool(size_t numberOfThreads = std::thread::hardware_concurrency()): + m_maxNumberOfThreads(numberOfThreads), + m_numberOfActiveThreads(0), + m_stop(false), + m_numTasks(0) + { + + } + + ~ThreadPool() + { + m_stop = true; + + m_conditional.notify_all(); + + while (IsRunning()) {} + + for (std::thread* t : m_threads) + { + t->join(); + delete t; + } + } + + void Start( ) + { + // Create threads + for (size_t i = 0; i < m_maxNumberOfThreads; ++i) + { + m_threads.push_back(new std::thread(&ThreadPool::Worker, this)); + + m_numberOfActiveThreads++; + } + } + + int GetNumberOfPendingTasks() + { + return m_numTasks; + } + + void Join() + { + if (m_maxNumberOfThreads == 0) + { + while (!m_tasks.empty()) + { + std::unique_ptr task = std::move( m_tasks.front() ); + + m_tasks.pop(); + + std::apply( [task = task.get()]( auto&&... args ) { task->function( args... ); }, task->arguments ); + + m_numTasks--; + } + } + else + { + std::unique_lock lock( m_waitMutex ); + + m_joinConditional.wait( lock, [this] { + return m_numTasks == 0; + } ); + } + + } + + void AddTask( std::function function, ArgumentTypes... arguments ) + { + std::unique_ptr task = std::make_unique(); + + task->arguments = std::make_tuple(arguments...); + + task->function = function; + + std::unique_lock lock(m_queueMutex); + + m_tasks.push(std::move(task)); + + m_numTasks++; + + if (IsRunning()) + { + m_conditional.notify_one(); + } + } + +private: + + struct Task + { + std::tuple arguments; + std::function function; + }; + + void Worker() + { + while (true) + { + std::unique_ptr task; + + { + std::unique_lock lock(m_queueMutex); + + m_conditional.wait(lock, [this] { + return !m_tasks.empty() || m_stop == true; + }); + + + if (m_stop) + { + // Stop thread + m_numberOfActiveThreads--; + return; + } + + task = std::move(m_tasks.front()); + + m_tasks.pop(); + } + + std::apply( [task = task.get() ]( auto&&... args ) { task->function( args... ); }, task->arguments ); + + m_numTasks--; + + m_joinConditional.notify_all(); + } + } + + bool NumberOfTasksPending() const + { + return m_numTasks > 0; + } + + bool IsRunning() const + { + return m_numberOfActiveThreads > 0; + } + +private: + + size_t m_maxNumberOfThreads; + + std::atomic m_numberOfActiveThreads; + + std::vector m_threads; + + std::mutex m_queueMutex; + + std::queue> m_tasks; + + std::atomic m_numTasks; + + std::condition_variable m_conditional; + + std::mutex m_waitMutex; + + std::condition_variable m_joinConditional; + + bool m_stop; + +}; diff --git a/tools/src/Downloader.cpp b/tools/src/Downloader.cpp index b32f057..10de492 100644 --- a/tools/src/Downloader.cpp +++ b/tools/src/Downloader.cpp @@ -76,7 +76,7 @@ Downloader::~Downloader() } } -Response Downloader::GetHeader( const std::string& url, uintmax_t retryCount, const std::chrono::seconds& retrySeconds, std::string& response ) +Response Downloader::GetHeader( const std::string& url, std::string& response ) { std::stringstream out; curl_easy_setopt( m_curlHandle, CURLOPT_URL, url.c_str() ); @@ -85,33 +85,29 @@ Response Downloader::GetHeader( const std::string& url, uintmax_t retryCount, co curl_easy_setopt( m_curlHandle, CURLOPT_WRITEDATA, &out ); curl_easy_setopt( m_curlHandle, CURLOPT_WRITEFUNCTION, WriteToFileStringCallback ); - for( unsigned int i = 0; i < retryCount; i++ ) - { - CURLcode cc = curl_easy_perform( m_curlHandle ); - - if( cc == CURLE_OK ) - { - // File exists - response = out.str(); - - return Response::SUCCESS; - } - - if( cc == CURLE_REMOTE_FILE_NOT_FOUND ) - { - // File doesn't exist - response = out.str(); + + CURLcode cc = curl_easy_perform( m_curlHandle ); - return Response::FILE_NOT_FOUND; - } + if( cc == CURLE_OK ) + { + // File exists + response = out.str(); - // Wait and retry with simple backoff - std::this_thread::sleep_for( retrySeconds * ( i + 1 ) ); + return Response::SUCCESS; } + else if( cc == CURLE_REMOTE_FILE_NOT_FOUND ) + { + // File doesn't exist + response = out.str(); - response = out.str(); - - return Response::DOWNLOAD_ERROR; + return Response::FILE_NOT_FOUND; + } + else + { + curl_easy_reset( m_curlHandle ); + return Response::DOWNLOAD_ERROR; + } + } bool Downloader::DownloadFile( const std::string& url, const std::filesystem::path& outputPath, const std::chrono::seconds& retrySeconds ) diff --git a/tools/src/Md5ChecksumStream.cpp b/tools/src/Md5ChecksumStream.cpp index dfb5a92..81839c9 100644 --- a/tools/src/Md5ChecksumStream.cpp +++ b/tools/src/Md5ChecksumStream.cpp @@ -22,6 +22,12 @@ Md5ChecksumStream::~Md5ChecksumStream() Finish(); } +void Md5ChecksumStream::Start() +{ + m_ss.str( std::string() ); + m_hash->Restart(); +} + void Md5ChecksumStream::Finish() { if( m_encoder ) @@ -54,7 +60,7 @@ bool Md5ChecksumStream::operator<<( const std::string& data ) return true; } -bool Md5ChecksumStream::FinishAndRetrieve( std::string& checksum ) +bool Md5ChecksumStream::Retrieve( std::string& checksum ) { if( !m_hash || !m_encoder ) { @@ -76,7 +82,6 @@ bool Md5ChecksumStream::FinishAndRetrieve( std::string& checksum ) checksum = "0" + checksum; } - Finish(); return true; } diff --git a/tools/src/ResourceFilter.cpp b/tools/src/ResourceFilter.cpp index 5624f49..9182919 100644 --- a/tools/src/ResourceFilter.cpp +++ b/tools/src/ResourceFilter.cpp @@ -55,6 +55,8 @@ bool ResourceFilter::SetFromFilterFileData( const FilterFile& fileData ) // ResolvePath filterPath->path = prefixPathStr + resPath->path.string(); + filterPath->pathContainsWildcard = false; + std::replace( filterPath->path.begin(), filterPath->path.end(), '\\', '/' ); // Remove leading slash @@ -63,9 +65,11 @@ bool ResourceFilter::SetFromFilterFileData( const FilterFile& fileData ) filterPath->path = filterPath->path.substr( 1 ); } + filterPath->pathLength = filterPath->path.size(); + bool wildcardReplacedInMatchPattern = false; - ConvertResPathToPattern( filterPath->path, filterPath->matchPattern); + ConvertResPathToPattern( filterPath->path, filterPath->matchPattern, filterPath->pathContainsWildcard ); if( wildcardReplacedInMatchPattern ) { @@ -96,7 +100,7 @@ bool ResourceFilter::SetFromFilterFileData( const FilterFile& fileData ) return true; } -void ResourceFilter::ConvertResPathToPattern( std::string resPath, std::string& pattern ) const +void ResourceFilter::ConvertResPathToPattern( std::string resPath, std::string& pattern, bool& pathContainsWildcard ) const { // Replace any "..." with a unique token (RECURSIVE_FOLDER_ELLIPSES_WILDCARD) @@ -113,10 +117,12 @@ void ResourceFilter::ConvertResPathToPattern( std::string resPath, std::string& if( resPath[i] == '*' ) { pattern += "[^/]*"; + pathContainsWildcard = true; } else if( resPath[i] == RECURSIVE_FOLDER_ELLIPSES_WILDCARD ) { pattern += ".*"; + pathContainsWildcard = true; } else if( std::string( ".^$|()[]{}+?\\" ).find( resPath[i] ) != std::string::npos ) { @@ -131,17 +137,8 @@ void ResourceFilter::ConvertResPathToPattern( std::string resPath, std::string& } } -bool ResourceFilter::CheckPath( const std::filesystem::path& path ) const -{ - std::string sectionId; - std::string matchPath; - return CheckPath( path, sectionId, matchPath ); -} - -bool ResourceFilter::CheckPath( const std::filesystem::path& path, std::string& matchSectionId, std::string& matchPath ) const +bool ResourceFilter::CheckPath( const std::string& path, std::string* matchSectionId /*= nullptr*/, std::string* matchPath /*= nullptr*/ ) const { - std::string resolvedPathStr = path.generic_string(); - for( auto& sectionFilterPath : m_paths ) { bool includeOrExcludeRulesFailedForSection = false; @@ -149,7 +146,15 @@ bool ResourceFilter::CheckPath( const std::filesystem::path& path, std::string& for( auto& filterPath : sectionFilterPath.second ) { // Check for directly specified file - bool specificFileMatch = filterPath->path == resolvedPathStr; + bool specificFileMatch = false; + + if( !filterPath->pathContainsWildcard ) // If path contains a wildcard it cannot be an exact match + { + if( filterPath->pathLength == path.size() ) // If path size doesn't match it doesn't match, early out + { + specificFileMatch = filterPath->path == path; // Finally do string comparison + } + } if( specificFileMatch ) { @@ -162,29 +167,44 @@ bool ResourceFilter::CheckPath( const std::filesystem::path& path, std::string& else { // Ignore all other rules and match this file - matchSectionId = filterPath->sectionId; - matchPath = filterPath->path; + if( matchSectionId ) + { + ( *matchSectionId ) = filterPath->sectionId; + } + if( matchPath ) + { + ( *matchPath ) = filterPath->path; + } + return true; } } else { - // If a previous include or exclude rule was failed - // Then the rest will also fail - // The section cannot be completely skipped as - // There may be specific files specified by full path - // That don't have to match filter rules. - if (includeOrExcludeRulesFailedForSection) - { + if( !filterPath->pathContainsWildcard ) + { + // Filter path is not a wildcard + // And it didn't directly match + // Therefore the remaining checks can be skipped continue; - } + } + + // If a previous include or exclude rule was failed + // Then the rest will also fail + // The section cannot be completely skipped as + // There may be specific files specified by full path + // That don't have to match filter rules. + if( includeOrExcludeRulesFailedForSection ) + { + continue; + } // Check exclude rules bool excludeRulesPassed = true; for( auto& excludeRule : filterPath->excludeRules ) { - if( resolvedPathStr.find( excludeRule ) != std::string::npos ) + if( path.find( excludeRule ) != std::string::npos ) { // Exclude rule met excludeRulesPassed = false; @@ -206,7 +226,7 @@ bool ResourceFilter::CheckPath( const std::filesystem::path& path, std::string& for( auto& includeRule : filterPath->includeRules ) { - if( resolvedPathStr.find( includeRule ) != std::string::npos ) + if( path.find( includeRule ) != std::string::npos ) { includeRulesPassed = true; break; @@ -224,32 +244,36 @@ bool ResourceFilter::CheckPath( const std::filesystem::path& path, std::string& // Perform regex on filter pattern try { - if( !std::regex_match( resolvedPathStr, filterPath->matchPatternRegex ) ) + if( !std::regex_match( path, filterPath->matchPatternRegex ) ) { continue; } else { - matchSectionId = filterPath->sectionId; - - matchPath = filterPath->path; + if( matchSectionId ) + { + ( *matchSectionId ) = filterPath->sectionId; + } + if( matchPath ) + { + ( *matchPath ) = filterPath->path; + } return true; } } catch( const std::regex_error& e ) { - std::string errorMsg = "Regex Exception during WildcardMatching - regexPattern: " + filterPath->matchPattern + " checkString: " + resolvedPathStr + " - error details: " + e.what(); + std::string errorMsg = "Regex Exception during WildcardMatching - regexPattern: " + filterPath->matchPattern + " checkString: " + path + " - error details: " + e.what(); throw std::runtime_error( errorMsg ); } catch( const std::exception& e ) { - std::string errorMsg = "Standard Exception during WildcardMatching - regexPattern: " + filterPath->matchPattern + " checkString: " + resolvedPathStr + " - error details: " + e.what(); + std::string errorMsg = "Standard Exception during WildcardMatching - regexPattern: " + filterPath->matchPattern + " checkString: " + path + " - error details: " + e.what(); throw std::runtime_error( errorMsg ); } } } - } return false; diff --git a/tools/src/ResourceTools.cpp b/tools/src/ResourceTools.cpp index 7b9ef23..cd71104 100644 --- a/tools/src/ResourceTools.cpp +++ b/tools/src/ResourceTools.cpp @@ -36,7 +36,7 @@ bool GenerateMd5Checksum( const std::filesystem::path& path, std::string& checks { md5Stream << temp; } - md5Stream.FinishAndRetrieve( checksum ); + md5Stream.Retrieve( checksum ); return true; } @@ -56,7 +56,7 @@ bool GenerateMd5Checksum( const std::string& data, std::string& checksum ) md5ChecksumStream << data; - if( md5ChecksumStream.FinishAndRetrieve( checksum ) ) + if( md5ChecksumStream.Retrieve( checksum ) ) { return true; }