Release v79.0 (#71)

1. 优化基础实现、关闭不必要的Log输出
2. 优化对依赖加载材质的额外Shader编译， 支持`NoPostLoadCacheDDC`参数控制（UE4中需要修改引擎才能使用该优化）
3. 优化CookCluster的分配策略
4. 优化Pak文件列表收集
5. 优化资源分析性能
6. 优化模块的可扩展性
7. 修复跨引擎版本的兼容性问题
8. 修复Shipping的C4172 ERROR
9. 修复不支持WP导致的基础包打包错误（`Found mor than one redistered Cook Package Splitter`）
10.  支持`PakSaveDirRegular`，自定义Pak的输出路径
11. 支持PakPreset/CookAndPak以统一的方式执行
12. 支持全局的AllowCookPlatforms

HotPatcher/Source/HotPatcherCore/Classes/Commandlets/CommandletHelper.cpp

@@ -88,7 +88,9 @@ void CommandletHelper::MainTick(TFunction<bool()> IsRequestExit)
 	// main loop
 	FDateTime LastConnectionTime = FDateTime::UtcNow();
 
-	while (GIsRunning && !IsRequestingExit() && !IsRequestExit())
+	while (GIsRunning &&
+		// !IsRequestingExit() &&
+		!IsRequestExit())
 	{
 		GEngine->UpdateTimeAndHandleMaxTickRate();
 		GEngine->Tick(FApp::GetDeltaTime(), false);

HotPatcher/Source/HotPatcherCore/Classes/Commandlets/HotPatcherCommandlet.cpp

@@ -68,7 +68,7 @@ int32 UHotPatcherCommandlet::Main(const FString& Params)
 
 		FString FinalConfig;
 		THotPatcherTemplateHelper::TSerializeStructAsJsonString(*ExportPatchSetting,FinalConfig);
-		UE_LOG(LogHotPatcherCommandlet, Display, TEXT("%s"), *FinalConfig);
+		// UE_LOG(LogHotPatcherCommandlet, Display, TEXT("%s"), *FinalConfig);
 
 
 		UPatcherProxy* PatcherProxy = NewObject<UPatcherProxy>();

HotPatcher/Source/HotPatcherCore/HotPatcherCore.Build.cs

@@ -190,7 +190,7 @@ public HotPatcherCore(ReadOnlyTargetRules Target) : base(Target)
 		PublicDefinitions.AddRange(new string[]
 		{
 			"TOOL_NAME=\"HotPatcher\"",
-			"CURRENT_VERSION_ID=78",
+			"CURRENT_VERSION_ID=79",
 			"CURRENT_PATCH_ID=0",
 			"REMOTE_VERSION_FILE=\"https://imzlp.com/opensource/version.json\""
 		});

HotPatcher/Source/HotPatcherCore/Private/Cooker/MultiCooker/SingleCookerProxy.cpp

@@ -49,14 +49,21 @@ void FFreezePackageTracker::NotifyUObjectCreated(const UObjectBase* Object, int3
 		}
 	}
 }
-
+#define NO_POSTLOAD_CACHE_DDC_OPTION TEXT("-NoPostLoadCacheDDC")
 void USingleCookerProxy::Init(FPatcherEntitySettingBase* InSetting)
 {
 	SCOPED_NAMED_EVENT_TEXT("Init",FColor::Red);
 	Super::Init(InSetting);
 	// IConsoleVariable* StreamableDelegateDelayFrames = IConsoleManager::Get().FindConsoleVariable(TEXT("s.StreamableDelegateDelayFrames"));
 	// StreamableDelegateDelayFrames->Set(0);
-
+	UFlibHotPatcherCoreHelper::DumpActiveTargetPlatforms();
+
+	FString Cmdline = FCommandLine::Get();
+	if(!Cmdline.Contains(NO_POSTLOAD_CACHE_DDC_OPTION,ESearchCase::IgnoreCase))
+	{
+		FCommandLine::Append(*FString::Printf(TEXT(" %s"),NO_POSTLOAD_CACHE_DDC_OPTION));
+		UE_LOG(LogHotPatcher,Display,TEXT("Append %s to Cmdline."),NO_POSTLOAD_CACHE_DDC_OPTION);
+	}
 #if WITH_PACKAGE_CONTEXT
 	if(GetSettingObject()->bOverrideSavePackageContext)
 	{
@@ -112,62 +119,52 @@ int32 USingleCookerProxy::MakeCookQueue(FCookCluster& InCluser)
 
 	const int32 NumberOfAssetsPerFrame = GetSettingObject()->GetNumberOfAssetsPerFrame();
 
-	if(GetSettingObject()->bForceCookInOneFrame)
+	for(auto Class:GetPreCacheClasses())
 	{
-		CookCluserQueue.Enqueue(InCluser);
-		++MakeClusterCount;
-		return MakeClusterCount;
-	}
-	else
-	{
-		for(auto Class:GetPreCacheClasses())
+		TArray<FAssetDetail> ObjectAssets = UFlibAssetManageHelper::GetAssetDetailsByClass(InCluser.AssetDetails,Class,true);
+		if(ObjectAssets.Num())
 		{
-			TArray<FAssetDetail> ObjectAssets = UFlibAssetManageHelper::GetAssetDetailsByClass(InCluser.AssetDetails,Class,true);
-			if(ObjectAssets.Num())
-			{
-				DumpCookerInfo.Append(FString::Printf(TEXT("\t%s -- %d\n"),*Class->GetName(),ObjectAssets.Num()));
-			}
-			int32 ClassesNumberOfAssetsPerFrame = GetClassAssetNumOfPerCluster(Class);
+			DumpCookerInfo.Append(FString::Printf(TEXT("\t%s -- %d\n"),*Class->GetName(),ObjectAssets.Num()));
+		}
+		int32 ClassesNumberOfAssetsPerFrame = GetClassAssetNumOfPerCluster(Class);
 
-			while(ObjectAssets.Num())
-			{
-				int32 ClusterAssetNum = ClassesNumberOfAssetsPerFrame < 1 ? ObjectAssets.Num() : ClassesNumberOfAssetsPerFrame;
-				int32 NewClusterAssetNum = FMath::Min(ClusterAssetNum,ObjectAssets.Num());
+		while(ObjectAssets.Num())
+		{
+			int32 ClusterAssetNum = ClassesNumberOfAssetsPerFrame < 1 ? ObjectAssets.Num() : ClassesNumberOfAssetsPerFrame;
+			int32 NewClusterAssetNum = FMath::Min(ClusterAssetNum,ObjectAssets.Num());
 
-				TArray<FAssetDetail> CulsterObjectAssets(ObjectAssets.GetData(),NewClusterAssetNum);
-				FCookCluster NewCluster;
-				NewCluster.AssetDetails = std::move(CulsterObjectAssets);
-				ObjectAssets.RemoveAt(0,NewClusterAssetNum);
-				NewCluster.Platforms = GetSettingObject()->CookTargetPlatforms;
-				NewCluster.bPreGeneratePlatformData = GetSettingObject()->bPreGeneratePlatformData;
-
-				NewCluster.CookActionCallback.OnAssetCooked = GetOnPackageSavedCallback();
-				NewCluster.CookActionCallback.OnCookBegin = GetOnCookAssetBeginCallback();
-				CookCluserQueue.Enqueue(NewCluster);
-				++MakeClusterCount;
-			}
+			TArray<FAssetDetail> CulsterObjectAssets(ObjectAssets.GetData(),NewClusterAssetNum);
+			FCookCluster NewCluster;
+			NewCluster.AssetDetails = std::move(CulsterObjectAssets);
+			ObjectAssets.RemoveAt(0,NewClusterAssetNum);
+			NewCluster.Platforms = GetSettingObject()->CookTargetPlatforms;
+			NewCluster.bPreGeneratePlatformData = GetSettingObject()->bPreGeneratePlatformData;
+			NewCluster.CookActionCallback.OnAssetCooked = GetOnPackageSavedCallback();
+			NewCluster.CookActionCallback.OnCookBegin = GetOnCookAssetBeginCallback();
+			CookCluserQueue.Enqueue(NewCluster);
+			++MakeClusterCount;
 		}
-
-		if(InCluser.AssetDetails.Num())
+	}
+	if(InCluser.AssetDetails.Num())
+	{
+		int32 OtherAssetNumPerFrame = NumberOfAssetsPerFrame < 1 ? InCluser.AssetDetails.Num() : NumberOfAssetsPerFrame;
+		int32 SplitNum = (InCluser.AssetDetails.Num() / OtherAssetNumPerFrame) + 1;
+
+		const TArray<TArray<FAssetDetail>> SplitedAssets= THotPatcherTemplateHelper::SplitArray(InCluser.AssetDetails,SplitNum);
+		for(const auto& AssetDetails:SplitedAssets)
 		{
-			int32 OtherAssetNumPerFrame = NumberOfAssetsPerFrame < 1 ? InCluser.AssetDetails.Num() : NumberOfAssetsPerFrame;
-			int32 SplitNum = (InCluser.AssetDetails.Num() / OtherAssetNumPerFrame) + 1;
-
-			const TArray<TArray<FAssetDetail>> SplitedAssets= THotPatcherTemplateHelper::SplitArray(InCluser.AssetDetails,SplitNum);
-			for(const auto& AssetDetails:SplitedAssets)
-			{
-				FCookCluster NewCluster;
-				NewCluster.AssetDetails = std::move(AssetDetails);
-				NewCluster.Platforms = GetSettingObject()->CookTargetPlatforms;
-				NewCluster.bPreGeneratePlatformData = GetSettingObject()->bPreGeneratePlatformData;
-				NewCluster.CookActionCallback.OnAssetCooked = GetOnPackageSavedCallback();
-				NewCluster.CookActionCallback.OnCookBegin = GetOnCookAssetBeginCallback();
-				CookCluserQueue.Enqueue(NewCluster);
-				++MakeClusterCount;
-			}
-			DumpCookerInfo.Append(FString::Printf(TEXT("\tOther Assets -- %d, make %d cluster.\n"),InCluser.AssetDetails.Num(),SplitedAssets.Num()));
+			FCookCluster NewCluster;
+			NewCluster.AssetDetails = std::move(AssetDetails);
+			NewCluster.Platforms = GetSettingObject()->CookTargetPlatforms;
+			NewCluster.bPreGeneratePlatformData = GetSettingObject()->bPreGeneratePlatformData;
+			NewCluster.CookActionCallback.OnAssetCooked = GetOnPackageSavedCallback();
+			NewCluster.CookActionCallback.OnCookBegin = GetOnCookAssetBeginCallback();
+			CookCluserQueue.Enqueue(NewCluster);
+			++MakeClusterCount;
 		}
+		DumpCookerInfo.Append(FString::Printf(TEXT("\tOther Assets -- %d, make %d cluster.\n"),InCluser.AssetDetails.Num(),SplitedAssets.Num()));
 	}
+
 	DumpCookerInfo.Append(TEXT("---------------------------------------------------------------------\n"));
 	UE_LOG(LogHotPatcher,Display,TEXT("%s"),*DumpCookerInfo);
 	return MakeClusterCount;
@@ -294,15 +291,17 @@ void USingleCookerProxy::ExecCookCluster(const FCookCluster& CookCluster)
 		return;
 	}
 
+	bool bContainShaderCluster = UFlibHotPatcherCoreHelper::AssetDetailsHasClasses(CookCluster.AssetDetails,UFlibHotPatcherCoreHelper::GetAllMaterialClassesNames());
 	DumpCluster(CookCluster,GCookLog);
-
 	TArray<ITargetPlatform*> TargetPlatforms = UFlibHotPatcherCoreHelper::GetTargetPlatformsByNames(CookCluster.Platforms);
 	FString CookBaseDir = GetSettingObject()->GetStorageCookedAbsDir();
 	CleanOldCooked(CookBaseDir,CookCluster.AsSoftObjectPaths(),CookCluster.Platforms);
 
 #if WITH_PACKAGE_CONTEXT
 	TMap<FString, FSavePackageContext*> SavePackageContextsNameMapping = GetPlatformSavePackageContextsNameMapping();
 #endif
+
+	TSharedPtr<FClassesPackageTracker> ClassesPackageTracker = MakeShareable(new FClassesPackageTracker);
 	TArray<UPackage*> PreCachePackages = UFlibAssetManageHelper::LoadPackagesForCooking(CookCluster.AsSoftObjectPaths(),GetSettingObject()->bConcurrentSave);
 
 	bool bCanConcurrentSave = GetSettingObject()->bConcurrentSave && CookCluster.bPreGeneratePlatformData;
@@ -371,7 +370,6 @@ void USingleCookerProxy::ExecCookCluster(const FCookCluster& CookCluster)
 
 	// clean cached ddd / release memory
 	// CleanClusterCachedPlatformData(CookCluster);
-
 	UFlibShaderCodeLibraryHelper::WaitShaderCompilingComplete();
 	UFlibHotPatcherCoreHelper::WaitForAsyncFileWrites();
 	// for GC
@@ -603,10 +601,14 @@ FCookCluster USingleCookerProxy::GetPackageTrackerAsCluster()
 	if(PackageTracker && GetSettingObject()->bCookPackageTrackerAssets)
 	{
 		PackageTrackerCluster.AssetDetails.Empty();
-		for(FName PackagePath:PackageTracker->GetPendingPackageSet())
+		for(FName LongPackageName:PackageTracker->GetPendingPackageSet())
 		{
 			// make asset data to asset registry
-			FSoftObjectPath ObjectPath(PackagePath.ToString());
+			FSoftObjectPath ObjectPath(
+				UFlibAssetManageHelper::LongPackageNameToPackagePath(LongPackageName.ToString())
+			);
+			UFlibAssetManageHelper::UpdateAssetRegistryData(ObjectPath.GetLongPackageName());
+
 			FAssetData AssetData;
 			if(UAssetManager::Get().GetAssetDataForPath(ObjectPath,AssetData))
 			{
@@ -618,7 +620,7 @@ FCookCluster USingleCookerProxy::GetPackageTrackerAsCluster()
 			}
 			else
 			{
-				UE_LOG(LogHotPatcher,Warning,TEXT("[GetPackageTrackerAsCluster] Get %s AssetData Failed!"),*PackagePath.ToString());
+				UE_LOG(LogHotPatcher,Warning,TEXT("[GetPackageTrackerAsCluster] Get %s AssetData Failed!"),*LongPackageName.ToString());
 			}
 		}
 	}
@@ -695,6 +697,7 @@ bool USingleCookerProxy::DoExport()
 
 void USingleCookerProxy::CleanOldCooked(const FString& CookBaseDir,const TArray<FSoftObjectPath>& ObjectPaths,const TArray<ETargetPlatform>& CookPlatforms)
 {
+	SCOPED_NAMED_EVENT_TEXT("CleanOldCooked",FColor::Red);
 	TArray<ITargetPlatform*> CookPlatfotms = UFlibHotPatcherCoreHelper::GetTargetPlatformsByNames(CookPlatforms);
 	{
 		SCOPED_NAMED_EVENT_TEXT("Delete Old Cooked Files",FColor::Red);
@@ -724,7 +727,6 @@ void USingleCookerProxy::CleanOldCooked(const FString& CookBaseDir,const TArray<
 	}
 }
 
-
 bool USingleCookerProxy::HasError()
 {
 	SCOPED_NAMED_EVENT_TEXT("USingleCookerProxy::HasError",FColor::Red);

HotPatcher/Source/HotPatcherCore/Private/Cooker/PackageWriter/HotWorldPartitionCookPackageSplitter.cpp

@@ -15,7 +15,7 @@
 #include "Editor.h"
 
 // Register FHotWorldPartitionCookPackageSplitter for UWorld class
-REGISTER_COOKPACKAGE_SPLITTER(FHotWorldPartitionCookPackageSplitter, UWorld);
+// REGISTER_COOKPACKAGE_SPLITTER(FHotWorldPartitionCookPackageSplitter, UWorld);
 
 bool FHotWorldPartitionCookPackageSplitter::ShouldSplit(UObject* SplitData)
 {

HotPatcher/Source/HotPatcherCore/Private/CreatePatch/PatcherProxy.cpp

@@ -158,6 +158,7 @@ namespace PatchWorker
 
 void UPatcherProxy::Init(FPatcherEntitySettingBase* InSetting)
 {
+	SCOPED_NAMED_EVENT_TEXT("UPatcherProxy::Init",FColor::Red);
 	Super::Init(InSetting);
 #if WITH_PACKAGE_CONTEXT
 	PlatformSavePackageContexts = UFlibHotPatcherCoreHelper::CreatePlatformsPackageContexts(
@@ -200,6 +201,7 @@ void UPatcherProxy::Shutdown()
 
 bool UPatcherProxy::DoExport()
 {
+	SCOPED_NAMED_EVENT_TEXT("UPatcherProxy::DoExport",FColor::Red);
 	PatchContext = MakeShareable(new FHotPatcherPatchContext);
 	PatchContext->PatchProxy = this;
 	PatchContext->OnPaking.AddLambda([this](const FString& One,const FString& Msg){this->OnPaking.Broadcast(One,Msg);});
@@ -584,9 +586,17 @@ namespace PatchWorker
 						EmptySetting.bConcurrentSave = false;
 						// for current impl arch
 						EmptySetting.bForceCookInOneFrame = true;
+						EmptySetting.NumberOfAssetsPerFrame = 200;
 						EmptySetting.bDisplayConfig = false;
 						EmptySetting.StorageCookedDir = Context.GetSettingObject()->GetStorageCookedDir();//FPaths::Combine(FPaths::ConvertRelativePathToFull(FPaths::ProjectSavedDir()),TEXT("Cooked"));
-						EmptySetting.StorageMetadataDir = FPaths::Combine(Context.GetSettingObject()->GetSaveAbsPath(),Context.CurrentVersion.VersionId,TEXT("Metadatas"),Chunk.ChunkName);
+						FReplacePakRegular PakSaveDirRegular{
+							Context.CurrentVersion.VersionId,
+							Context.CurrentVersion.BaseVersionId,
+							Chunk.ChunkName,
+							PlatformName
+						};
+						FString ReplacedPakSaveDirRegular = UFlibHotPatcherCoreHelper::ReplacePakRegular(PakSaveDirRegular,Context.GetSettingObject()->GetPakSaveDirRegular());
+						EmptySetting.StorageMetadataDir = FPaths::Combine(Context.GetSettingObject()->GetSaveAbsPath(),ReplacedPakSaveDirRegular,TEXT("Metadatas"));
 #if WITH_PACKAGE_CONTEXT
 						EmptySetting.bOverrideSavePackageContext = true;
 						EmptySetting.PlatformSavePackageContexts = Context.PatchProxy->GetPlatformSavePackageContexts();
@@ -865,7 +875,14 @@ namespace PatchWorker
 					Context.OnPaking.Broadcast(TEXT("ExportPatch"),*Dialog.ToString());
 					Context.UnrealPakSlowTask->EnterProgressFrame(1.0, Dialog);
 				}
-				FString ChunkSaveBasePath = FPaths::Combine(Context.GetSettingObject()->GetSaveAbsPath(), Context.CurrentVersion.VersionId, PlatformName);
+				FReplacePakRegular PakSaveDirRegular{
+					Context.CurrentVersion.VersionId,
+					Context.CurrentVersion.BaseVersionId,
+					Chunk.ChunkName,
+					PlatformName
+				};
+				FString ReplacedPakSaveDirRegular = UFlibHotPatcherCoreHelper::ReplacePakRegular(PakSaveDirRegular,Context.GetSettingObject()->GetPakSaveDirRegular());
+				FString ChunkSaveBasePath = FPaths::Combine(Context.GetSettingObject()->GetSaveAbsPath(),ReplacedPakSaveDirRegular);
 
 				TArray<FPakCommand> ChunkPakListCommands;
 				{
@@ -961,8 +978,8 @@ namespace PatchWorker
 					SinglePakForChunk.PakCommands = ChunkPakListCommands;
 					// add extern file to pak(version file)
 					SinglePakForChunk.PakCommands.Append(Context.AdditionalFileToPak);
-					
-					const FString ChunkPakName = UFlibHotPatcherCoreHelper::MakePakShortName(Context.CurrentVersion,Chunk,PlatformName,Context.GetSettingObject()->GetPakNameRegular());
+
+					const FString ChunkPakName = UFlibHotPatcherCoreHelper::ReplacePakRegular(PakSaveDirRegular,Context.GetSettingObject()->GetPakNameRegular());
 					SinglePakForChunk.ChunkStoreName = ChunkPakName;
 					SinglePakForChunk.StorageDirectory = ChunkSaveBasePath;
 					Chunk.GetPakFileProxys().Add(SinglePakForChunk);