diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/Bundle.cpp b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/Bundle.cpp index ff33131160..379119164e 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/Bundle.cpp +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/Bundle.cpp @@ -6,6 +6,9 @@ #include "KBDebug.h" #include "ObjectPool.h" +namespace KBEngine +{ + static ObjectPool _g_bundlePool; Bundle::Bundle(): @@ -292,4 +295,6 @@ void Bundle::writeVector4(const FVector4& v) { checkStream(16); (*pCurrPacket_).writeVector4(v); +} + } \ No newline at end of file diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/Bundle.h b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/Bundle.h index ba8bd0af0b..3056c3d1a9 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/Bundle.h +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/Bundle.h @@ -4,6 +4,9 @@ #include "KBECommon.h" +namespace KBEngine +{ + class MemoryStream; class NetworkInterfaceBase; class Message; @@ -145,3 +148,5 @@ class KBENGINEPLUGINS_API Bundle Message* pMsgtype_; int curMsgStreamIndex_; }; + +} \ No newline at end of file diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/ClientSDKUpdateUI.cpp b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/ClientSDKUpdateUI.cpp new file mode 100644 index 0000000000..3775b033d1 --- /dev/null +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/ClientSDKUpdateUI.cpp @@ -0,0 +1,62 @@ +#include "ClientSDKUpdateUI.h" +#include "Engine.h" +#include "KBEvent.h" +#include "KBEngine.h" +#include "KBEventTypes.h" +#include "SlateOptMacros.h" + +BEGIN_SLATE_FUNCTION_BUILD_OPTIMIZATION +void SClientSDKUpdateUI::Construct(const FArguments& args) +{ + FTextBlockStyle textStyle = FCoreStyle::Get().GetWidgetStyle< FTextBlockStyle >("NormalText"); + textStyle.SetFont(FSlateFontInfo("Veranda", 26)); + textStyle.SetColorAndOpacity(FLinearColor::White); + + ChildSlot + [ + SNew(SOverlay) + /*+ SOverlay::Slot() + .HAlign(HAlign_Center) + .VAlign(VAlign_Top) + [ + SNew(STextBlock) + .ColorAndOpacity(FLinearColor::White) + .ShadowColorAndOpacity(FLinearColor::Black) + .ShadowOffset(FIntPoint(-1, 1)) + .Font(FSlateFontInfo("Arial", 26)) + .Text(FText::FromString("Main Menu")) + ]*/ + + SOverlay::Slot() + .HAlign(HAlign_Center) + .VAlign(VAlign_Center) + [ + SNew(SVerticalBox) + + SVerticalBox::Slot() + .Padding(20) + [ + SNew(SButton) + .Text(FText::FromString("Update SDK")) + .TextStyle(&textStyle) + .ContentPadding(FMargin(30.0, 15.0)) + .ButtonColorAndOpacity(FLinearColor::Green) + .HAlign(HAlign_Center) + .VAlign(VAlign_Top) + .OnClicked(this, &SClientSDKUpdateUI::UpdateSDKClicked) + ] + ] + ]; +} +END_SLATE_FUNCTION_BUILD_OPTIMIZATION + +FReply SClientSDKUpdateUI::UpdateSDKClicked() +{ + if (GEngine) + { + GEngine->AddOnScreenDebugMessage(-1, 3.f, FColor::Yellow, TEXT("Update SDK!")); + } + + UKBEventData_onDownloadSDK* pEventData = NewObject(); + pEventData->isDownload = true; + KBENGINE_EVENT_FIRE(KBEngine::KBEventTypes::onDownloadSDK, pEventData); + return FReply::Handled(); +} diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/ClientSDKUpdateUI.h b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/ClientSDKUpdateUI.h new file mode 100644 index 0000000000..02e04aced4 --- /dev/null +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/ClientSDKUpdateUI.h @@ -0,0 +1,19 @@ +#pragma once + +#include "KBEnginePluginsPrivatePCH.h" + + +class KBENGINEPLUGINS_API SClientSDKUpdateUI : public SCompoundWidget +{ + +public: + SLATE_BEGIN_ARGS(SClientSDKUpdateUI){} + + SLATE_END_ARGS() + + void Construct(const FArguments& args); + + FReply UpdateSDKClicked(); + + +}; diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/ClientSDKUpdater.cpp b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/ClientSDKUpdater.cpp index 6df2902eca..f6345adef0 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/ClientSDKUpdater.cpp +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/ClientSDKUpdater.cpp @@ -9,8 +9,8 @@ #include "FileHelper.h" #include "OutPutDeviceDebug.h" -using namespace std; - +namespace KBEngine +{ ClientSDKUpdater::ClientSDKUpdater() { @@ -246,7 +246,7 @@ void ClientSDKUpdater::moveToFile(FString fromFileName, FString toFileName) PlatformFile.MoveFile(*toFileName, *fromFileName); } -void ClientSDKUpdater::copyDirectory(FString fromDicPath, FString toDicPath) +void ClientSDKUpdater::copyDirectory(FString fromDicPath, FString toDicPath) { FPaths::NormalizeDirectoryName(fromDicPath); FPaths::NormalizeDirectoryName(toDicPath); @@ -254,3 +254,4 @@ void ClientSDKUpdater::copyDirectory(FString fromDicPath, FString toDicPath) PlatformFile.CopyDirectoryTree(*toDicPath, *fromDicPath, true); } +} diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/ClientSDKUpdater.h b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/ClientSDKUpdater.h index 4fc946f3bf..a358880cb7 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/ClientSDKUpdater.h +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/ClientSDKUpdater.h @@ -18,6 +18,9 @@ /** * */ +namespace KBEngine +{ + class KBENGINEPLUGINS_API ClientSDKUpdater { public: @@ -53,3 +56,5 @@ class KBENGINEPLUGINS_API ClientSDKUpdater MemoryStream* pSdkFileStream = nullptr; }; + +} \ No newline at end of file diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/DataTypes.cpp b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/DataTypes.cpp index 4c483211a1..fe2d5955dd 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/DataTypes.cpp +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/DataTypes.cpp @@ -4,6 +4,10 @@ #include "EntityDef.h" #include "KBDebug.h" #include "Runtime/Core/Public/Misc/Variant.h" +#include "Bundle.h" + +namespace KBEngine +{ KBVar* DATATYPE_INT8::createFromStream(MemoryStream& stream) { @@ -589,3 +593,5 @@ bool DATATYPE_FIXED_DICT::isSameType(KBVar& v) return true; } + +} \ No newline at end of file diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/DataTypes.h b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/DataTypes.h index f60ca3a75c..99f437cdfa 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/DataTypes.h +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/DataTypes.h @@ -5,13 +5,17 @@ #include "KBECommon.h" #include "KBVar.h" -class Bundle; -class MemoryStream; /* entitydef所支持的基本数据类型 改模块中的类抽象出了所有的支持类型并提供了这些类型的数据序列化成二进制数据与反序列化操作（主要用于网络通讯的打包与解包） */ +namespace KBEngine +{ + +class Bundle; +class MemoryStream; + class KBENGINEPLUGINS_API DATATYPE_BASE { public: @@ -88,7 +92,7 @@ class KBENGINEPLUGINS_API DATATYPE_INT32 : public DATATYPE_BASE { public: virtual KBVar* createFromStream(MemoryStream& stream) override; - virtual void addToStream(Bundle& stream, KBVar& v) override; + virtual void addToStream(KBEngine::Bundle& stream, KBVar& v) override; virtual KBVar* parseDefaultValStr(const FString& v) override; @@ -382,15 +386,4 @@ class KBENGINEPLUGINS_API DATATYPE_FIXED_DICT : public DATATYPE_BASE TMap dicttype_map; }; - - - - - - - - - - - - +} diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/EncryptionFilter.cpp b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/EncryptionFilter.cpp index d164861dd2..f3ec5215eb 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/EncryptionFilter.cpp +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/EncryptionFilter.cpp @@ -5,35 +5,40 @@ #include "PacketSenderKCP.h" #include "Engine/KBDebug.h" -#ifndef KBENGINE_NO_CRYPTO -#include "cryptlib.h" -#include "rdrand.h" -#include "secblock.h" -#endif +namespace KBEngine +{ + +EncryptionFilter::~EncryptionFilter() +{ +} -BlowfishFilter::BlowfishFilter(int keySize): +BlowfishFilter::BlowfishFilter(int keySize) : isGood_(false), pPacket_(new MemoryStream()), pEncryptStream_(new MemoryStream()), packetLen_(0), - padSize_(0) + padSize_(0), + key_(), + keySize_(0), + pBlowFishKey_(NULL) { -#ifndef KBENGINE_NO_CRYPTO - key_.Init(0, keySize); + unsigned char buf[20] = ""; + RAND_bytes(buf, 20); + key_ = (char *)buf; + keySize_ = key_.Len(); - CryptoPP::RDRAND rng; - rng.GenerateBlock(key_.GetData(), key_.Num()); init(); -#endif } -BlowfishFilter::BlowfishFilter(const TArray& key): +BlowfishFilter::BlowfishFilter(const FString & key) : isGood_(false), - key_(key), pPacket_(new MemoryStream()), pEncryptStream_(new MemoryStream()), packetLen_(0), - padSize_(0) + padSize_(0), + key_(key), + keySize_(key_.Len()), + pBlowFishKey_(NULL) { init(); } @@ -42,23 +47,25 @@ BlowfishFilter::~BlowfishFilter() { KBE_SAFE_RELEASE(pPacket_); KBE_SAFE_RELEASE(pEncryptStream_); + KBE_SAFE_RELEASE(pBlowFishKey_); } bool BlowfishFilter::init() { -#ifndef KBENGINE_NO_CRYPTO - if (key_.Num() >= encripter.MinKeyLength() && key_.Num() <= encripter.MaxKeyLength()) + pBlowFishKey_ = new BF_KEY; + + if (MIN_KEY_SIZE <= keySize_ && keySize_ <= MAX_KEY_SIZE) { - encripter.SetKey(key_.GetData(), key_.Num()); - decripter.SetKey(key_.GetData(), key_.Num()); + + BF_set_key(this->pBlowFishKey(), key_.Len(), reinterpret_cast(TCHAR_TO_ANSI(*key_))); isGood_ = true; } else { - ERROR_MSG("BlowfishFilter::init: invalid length %d", key_.Num()); + ERROR_MSG("BlowfishFilter::init: invalid length %d", key_.Len()); isGood_ = false; } -#endif + return isGood_; } @@ -66,7 +73,6 @@ void BlowfishFilter::encrypt(MemoryStream *pMemoryStream) { // BlowFish 每次只能加密和解密8字节数据 // 不足8字节则填充0 -#ifndef KBENGINE_NO_CRYPTO uint8 padSize = 0; if (pMemoryStream->length() % BLOCK_SIZE != 0) @@ -86,12 +92,10 @@ void BlowfishFilter::encrypt(MemoryStream *pMemoryStream) pMemoryStream->swap(*pEncryptStream_); pEncryptStream_->clear(false); -#endif } void BlowfishFilter::encrypt(uint8 *buf, MessageLengthEx len) { -#ifndef KBENGINE_NO_CRYPTO if (len % BLOCK_SIZE != 0) { ERROR_MSG("BlowfishFilter::encrypt: Input length (%d) is not a multiple of block size ", len); @@ -113,9 +117,8 @@ void BlowfishFilter::encrypt(uint8 *buf, MessageLengthEx len) prevBlock = *(uint64*)(data + i); } - encripter.ProcessData(data + i, data + i, BLOCK_SIZE); + BF_ecb_encrypt(data + i, data + i, this->pBlowFishKey(), BF_ENCRYPT); } -#endif } void BlowfishFilter::encrypt(uint8 *buf, MessageLengthEx offset, MessageLengthEx len) @@ -130,7 +133,6 @@ void BlowfishFilter::decrypt(MemoryStream *pMemoryStream) void BlowfishFilter::decrypt(uint8 *buf, MessageLengthEx len) { -#ifndef KBENGINE_NO_CRYPTO if (len % BLOCK_SIZE != 0) { ERROR_MSG("BlowfishFilter::decrypt: Input length (%d) is not a multiple of block size ", len); @@ -141,16 +143,15 @@ void BlowfishFilter::decrypt(uint8 *buf, MessageLengthEx len) uint64 prevBlock = 0; for (uint32 i = 0; i < len; i += BLOCK_SIZE) { - decripter.ProcessData(data + i, data + i, BLOCK_SIZE); + BF_ecb_encrypt(data + i, data + i, this->pBlowFishKey(), BF_DECRYPT); if (prevBlock != 0) { *(uint64*)(data + i) = *(uint64*)(data + i) ^ (prevBlock); } - + prevBlock = *(uint64*)(data + i); } -#endif } void BlowfishFilter::decrypt(uint8 *buf, MessageLengthEx offset, MessageLengthEx len) @@ -160,7 +161,6 @@ void BlowfishFilter::decrypt(uint8 *buf, MessageLengthEx offset, MessageLengthEx bool BlowfishFilter::send(PacketSenderBase* pPacketSender, MemoryStream *pPacket) { -#ifndef KBENGINE_NO_CRYPTO if (!isGood_) { ERROR_MSG("BlowfishFilter::send: Dropping packet due to invalid filter"); @@ -168,13 +168,11 @@ bool BlowfishFilter::send(PacketSenderBase* pPacketSender, MemoryStream *pPacket } encrypt(pPacket); -#endif return pPacketSender->send(pPacket);; } bool BlowfishFilter::recv(MessageReader* pMessageReader, MemoryStream *pPacket) { -#ifndef KBENGINE_NO_CRYPTO if (!isGood_) { ERROR_MSG("BlowfishFilter::recv: Dropping packet due to invalid filter"); @@ -185,7 +183,7 @@ bool BlowfishFilter::recv(MessageReader* pMessageReader, MemoryStream *pPacket) uint32 len = pPacket->length(); uint16 packeLen = pPacket->readUint16(); - if ( 0 == pPacket_->length() && len > MIN_PACKET_SIZE && packeLen - 1 == len - 3) + if (0 == pPacket_->length() && len > MIN_PACKET_SIZE && packeLen - 1 == len - 3) { int padSize = pPacket->readUint8(); decrypt(pPacket); @@ -267,12 +265,7 @@ bool BlowfishFilter::recv(MessageReader* pMessageReader, MemoryStream *pPacket) packetLen_ = 0; padSize_ = 0; } -#else - if (pMessageReader) - { - pMessageReader->process(pPacket->data() + pPacket->rpos(), 0, pPacket->length()); - } -#endif - return true; } + +} \ No newline at end of file diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/EncryptionFilter.h b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/EncryptionFilter.h index 1a224f549c..5bf4820a02 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/EncryptionFilter.h +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/EncryptionFilter.h @@ -1,14 +1,24 @@ #pragma once #include "KBECommon.h" -#ifndef KBENGINE_NO_CRYPTO -// https://stackoverflow.com/questions/51416259/unreal-engine-4-20-build-error-in-plugin-adaptive-unity-build-disabling-pch-f -#pragma warning(disable:4668) // x is not defined as a preprocessor macro, replacing with '0' for '#if/#elif' -#include "blowfish.h" -#pragma warning(default:4668) -#include "modes.h" +#if PLATFORM_WINDOWS +#include "WindowsHWrapper.h" +#include "AllowWindowsPlatformTypes.h" #endif +#define UI UI_ST +THIRD_PARTY_INCLUDES_START +#include "openssl/rand.h" +#include "openssl/blowfish.h" +THIRD_PARTY_INCLUDES_END +#undef UI + +#if PLATFORM_WINDOWS +#include "HideWindowsPlatformTypes.h" +#endif + +namespace KBEngine +{ class MemoryStream; class PacketSenderBase; @@ -17,8 +27,9 @@ class MessageReader; class EncryptionFilter { public: - virtual ~EncryptionFilter() {} + EncryptionFilter() {} + virtual ~EncryptionFilter(); virtual void encrypt(MemoryStream *pMemoryStream) = 0; virtual void encrypt(uint8 *buf, MessageLengthEx len) = 0; virtual void encrypt(uint8 *buf, MessageLengthEx offset, MessageLengthEx len) = 0; @@ -39,11 +50,20 @@ class BlowfishFilter : public EncryptionFilter static const uint32 BLOCK_SIZE = 64 / 8; static const uint32 MIN_PACKET_SIZE = (sizeof(MessageLength) + 1 + BLOCK_SIZE); - BlowfishFilter(int keySize=16); - BlowfishFilter(const TArray& key); + + // key的最小和最大大小 + static const int MIN_KEY_SIZE = 32 / 8; + static const int MAX_KEY_SIZE = 448 / 8; + + // 默认key的大小 + static const int DEFAULT_KEY_SIZE = 128 / 8; + + BlowfishFilter(const FString & key); + BlowfishFilter(int keySize = DEFAULT_KEY_SIZE); + virtual ~BlowfishFilter(); - virtual void encrypt(MemoryStream *pMemoryStream) ; + virtual void encrypt(MemoryStream *pMemoryStream); virtual void encrypt(uint8 *buf, MessageLengthEx len); virtual void encrypt(uint8 *buf, MessageLengthEx offset, MessageLengthEx len); @@ -54,8 +74,16 @@ class BlowfishFilter : public EncryptionFilter virtual bool send(PacketSenderBase *pPacketSender, MemoryStream *pPacket); virtual bool recv(MessageReader *pMessageReader, MemoryStream *pPacket); - TArray& key() { - return key_; + + BF_KEY * pBlowFishKey() { return (BF_KEY*)pBlowFishKey_; } + + TArray key() + { + TArray keyArray; + keyArray.SetNum(key_.Len()); + memcpy(keyArray.GetData(), TCHAR_TO_ANSI(*key_), key_.Len()); + + return keyArray; } private: @@ -63,13 +91,14 @@ class BlowfishFilter : public EncryptionFilter private: bool isGood_; - TArray key_; MemoryStream* pPacket_; MemoryStream* pEncryptStream_; MessageLength packetLen_; uint8 padSize_; -#ifndef KBENGINE_NO_CRYPTO - CryptoPP::ECB_Mode::Encryption encripter; - CryptoPP::ECB_Mode::Decryption decripter; -#endif + + FString key_; + int keySize_; + void * pBlowFishKey_; }; + +} diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/Entity.cpp b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/Entity.cpp index 4067fe17cb..6b822821c2 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/Entity.cpp +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/Entity.cpp @@ -10,6 +10,8 @@ #include "Property.h" #include "KBDebug.h" +namespace KBEngine +{ Entity::Entity(): id_(0), @@ -178,3 +180,5 @@ void Entity::onDirectionChanged(const FVector& oldValue) KBENGINE_EVENT_FIRE(KBEventTypes::set_direction, pEventData); } } + +} \ No newline at end of file diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/Entity.h b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/Entity.h index 9eb511a294..e37903e252 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/Entity.h +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/Entity.h @@ -5,6 +5,9 @@ #include "KBVar.h" #include "KBECommon.h" +namespace KBEngine +{ + class Method; class Property; class EntityCall; @@ -213,6 +216,7 @@ class KBENGINEPLUGINS_API Entity //EntityCall* cellEntityCall = null; }; +} diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/EntityCall.cpp b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/EntityCall.cpp index c3dcc76521..516c09f0a9 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/EntityCall.cpp +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/EntityCall.cpp @@ -8,6 +8,9 @@ #include "Method.h" #include "ScriptModule.h" +namespace KBEngine +{ + EntityCall::EntityCall(int32 eid, const FString& ename): id(eid), className(ename), @@ -78,4 +81,6 @@ void EntityCall::sendCall(Bundle* inBundle) if (inBundle == pBundle) pBundle = NULL; +} + } \ No newline at end of file diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/EntityCall.h b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/EntityCall.h index 81028924cb..239d891659 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/EntityCall.h +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/EntityCall.h @@ -4,13 +4,16 @@ #include "KBECommon.h" -class Bundle; - /* 实体的EntityCall 关于EntityCall请参考API手册中对它的描述 https://github.com/kbengine/kbengine/tree/master/docs/api */ +namespace KBEngine +{ + +class Bundle; + class KBENGINEPLUGINS_API EntityCall { public: @@ -51,3 +54,5 @@ class KBENGINEPLUGINS_API EntityCall ENTITYCALL_TYPE type; Bundle* pBundle; }; + +} \ No newline at end of file diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/EntityComponent.cpp b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/EntityComponent.cpp index a99b7877aa..f115088e7e 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/EntityComponent.cpp +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/EntityComponent.cpp @@ -4,6 +4,9 @@ #include "KBDebug.h" #include "MemoryStream.h" +namespace KBEngine +{ + EntityComponent::EntityComponent(): entityComponentPropertyID(0), componentType(0), @@ -30,3 +33,4 @@ void EntityComponent::createFromStream(MemoryStream& stream) onUpdatePropertys(0, stream, count); } +} diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/EntityComponent.h b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/EntityComponent.h index 264666e613..7191923678 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/EntityComponent.h +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/EntityComponent.h @@ -4,6 +4,9 @@ #include "KBECommon.h" +namespace KBEngine +{ + class Entity; class MemoryStream; class ScriptModule; @@ -59,3 +62,5 @@ class KBENGINEPLUGINS_API EntityComponent ENTITY_ID ownerID; Entity* pOwner; }; + +} \ No newline at end of file diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/Interfaces.h b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/Interfaces.h index afa2f6b885..37e531c034 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/Interfaces.h +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/Interfaces.h @@ -4,6 +4,9 @@ #include "KBECommon.h" +namespace KBEngine +{ + /* 接口模块 用于声明和实现某些回调统一接口 @@ -16,3 +19,5 @@ class KBENGINEPLUGINS_API InterfaceConnect }; + +} \ No newline at end of file diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBDebug.h b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBDebug.h index dbfcb0e127..6331a2a706 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBDebug.h +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBDebug.h @@ -1,6 +1,9 @@ // Copyright 1998-2016 Epic Games, Inc. All Rights Reserved. #pragma once +namespace KBEngine +{ + #define INFO_MSG(Format, ...) \ { \ SET_WARN_COLOR(COLOR_CYAN); \ @@ -57,4 +60,6 @@ { \ const FString Msg = FString::Printf(TEXT(Format), ##__VA_ARGS__); \ GEngine->AddOnScreenDebugMessage(-1, 10000.f, FColor::White, Msg); \ +} + } \ No newline at end of file diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBECommon.cpp b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBECommon.cpp index dbc5079652..73a0216337 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBECommon.cpp +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBECommon.cpp @@ -3,8 +3,13 @@ #include "KBECommon.h" #include "KBDebug.h" +namespace KBEngine +{ + DEFINE_LOG_CATEGORY(LogKBEngine); +} + double getTimeSeconds() { return FPlatformTime::Seconds(); diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBECommon.h b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBECommon.h index 538dcf65a1..c57973151d 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBECommon.h +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBECommon.h @@ -9,8 +9,12 @@ #include "GameFramework/Actor.h" #include "KBECommon.generated.h" +namespace KBEngine +{ + DECLARE_LOG_CATEGORY_EXTERN(LogKBEngine, Log, All); +} #define KBE_ASSERT check typedef uint16 MessageID; @@ -20,8 +24,8 @@ typedef int32 ENTITY_ID; typedef uint32 SPACE_ID; typedef uint64 DBID; typedef TArray ByteArray; -typedef TMap KB_FIXED_DICT; -typedef TArray KB_ARRAY; +typedef TMap KB_FIXED_DICT; +typedef TArray KB_ARRAY; #define KBE_FLT_MAX FLT_MAX @@ -170,7 +174,7 @@ inline bool almostEqual(float f1, float f2, float epsilon) return FMath::Abs(f1 - f2) < epsilon; } -inline bool isNumeric(KBVar& v) +inline bool isNumeric(KBEngine::KBVar& v) { return v.GetType() == EKBVarTypes::Bool || v.GetType() == EKBVarTypes::Double || diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBEMain.cpp b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBEMain.cpp index b98bd5831f..64d82fe9f0 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBEMain.cpp +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBEMain.cpp @@ -51,8 +51,7 @@ void UKBEMain::UninitializeComponent() void UKBEMain::BeginPlay() { Super::BeginPlay(); - - KBEngineArgs* pArgs = new KBEngineArgs(); + KBEngine::KBEngineArgs* pArgs = new KBEngine::KBEngineArgs(); pArgs->ip = ip; pArgs->port = port; pArgs->syncPlayerMS = syncPlayerMS; @@ -67,19 +66,10 @@ void UKBEMain::BeginPlay() pArgs->UDP_SEND_BUFFER_MAX = UDP_SEND_BUFFER_MAX; pArgs->UDP_RECV_BUFFER_MAX = UDP_RECV_BUFFER_MAX; - KBEngineApp::destroyKBEngineApp(); - if (!KBEngineApp::getSingleton().initialize(pArgs)) + if(!KBEngine::KBEngineApp::getSingleton().initialize(pArgs)) delete pArgs; installEvents(); - -#ifdef KBENGINE_NO_CRYPTO - if (pArgs->networkEncryptType == NETWORK_ENCRYPT_TYPE::ENCRYPT_TYPE_BLOWFISH) - { - pArgs->networkEncryptType = NETWORK_ENCRYPT_TYPE::ENCRYPT_TYPE_NONE; - ERROR_MSG("No module CryptoPP! Please use unreal engine source code to install"); - } -#endif } void UKBEMain::EndPlay(const EEndPlayReason::Type EndPlayReason) @@ -90,6 +80,7 @@ void UKBEMain::EndPlay(const EEndPlayReason::Type EndPlayReason) pUpdaterObj = nullptr; } + ClientSDKUpdateUI.Reset(); deregisterEvents(); Super::EndPlay(EndPlayReason); } @@ -102,16 +93,18 @@ void UKBEMain::TickComponent( float DeltaTime, ELevelTick TickType, FActorCompon void UKBEMain::installEvents() { - KBENGINE_REGISTER_EVENT(KBEventTypes::onScriptVersionNotMatch, onScriptVersionNotMatch); - KBENGINE_REGISTER_EVENT(KBEventTypes::onVersionNotMatch, onVersionNotMatch); - KBENGINE_REGISTER_EVENT(KBEventTypes::onImportClientSDKSuccessfully, onImportClientSDKSuccessfully); + KBENGINE_REGISTER_EVENT(KBEngine::KBEventTypes::onScriptVersionNotMatch, onScriptVersionNotMatch); + KBENGINE_REGISTER_EVENT(KBEngine::KBEventTypes::onVersionNotMatch, onVersionNotMatch); + KBENGINE_REGISTER_EVENT(KBEngine::KBEventTypes::onImportClientSDKSuccessfully, onImportClientSDKSuccessfully); + KBENGINE_REGISTER_EVENT(KBEngine::KBEventTypes::onDownloadSDK, onDownloadSDK); } void UKBEMain::deregisterEvents() { - KBENGINE_DEREGISTER_EVENT(KBEventTypes::onScriptVersionNotMatch); - KBENGINE_DEREGISTER_EVENT(KBEventTypes::onVersionNotMatch); - KBENGINE_DEREGISTER_EVENT(KBEventTypes::onImportClientSDKSuccessfully); + KBENGINE_DEREGISTER_EVENT(KBEngine::KBEventTypes::onScriptVersionNotMatch); + KBENGINE_DEREGISTER_EVENT(KBEngine::KBEventTypes::onVersionNotMatch); + KBENGINE_DEREGISTER_EVENT(KBEngine::KBEventTypes::onImportClientSDKSuccessfully); + KBENGINE_DEREGISTER_EVENT(KBEngine::KBEventTypes::onDownloadSDK); } void UKBEMain::onVersionNotMatch(const UKBEventData* pEventData) @@ -126,26 +119,63 @@ void UKBEMain::onScriptVersionNotMatch(const UKBEventData* pEventData) bool UKBEMain::isUpdateSDK() { - #if WITH_EDITOR - return automaticallyUpdateSDK; - #endif - return false; +#if WITH_EDITOR + return automaticallyUpdateSDK; +#endif + + return false; } void UKBEMain::downloadSDKFromServer() { + if (GEngine->IsValidLowLevel()) + { + GEngine->GameViewport->RemoveAllViewportWidgets(); + } + if (isUpdateSDK()) + { + ClientSDKUpdateUI = SNew(SClientSDKUpdateUI); + + if (GEngine->IsValidLowLevel()) + { + GEngine->GameViewport->AddViewportWidgetContent(SNew(SWeakWidget).PossiblyNullContent(ClientSDKUpdateUI.ToSharedRef())); + } + + if (ClientSDKUpdateUI.IsValid()) + { + ClientSDKUpdateUI->SetVisibility(EVisibility::Visible); + } + + } +} + +void UKBEMain::onImportClientSDKSuccessfully(const UKBEventData* pEventData) +{ + UKismetSystemLibrary::QuitGame(this, nullptr, EQuitPreference::Quit, true); +} + +void UKBEMain::onDownloadSDK(const UKBEventData* pEventData) +{ + ClientSDKUpdateUI.Reset(); + if (GEngine->IsValidLowLevel()) + { + GEngine->GameViewport->RemoveAllViewportWidgets(); + } + + const UKBEventData_onDownloadSDK* pData = Cast(pEventData); + if(pData->isDownload) { if (pUpdaterObj == nullptr) { - pUpdaterObj = new ClientSDKUpdater(); + pUpdaterObj = new KBEngine::ClientSDKUpdater(); } pUpdaterObj->downloadSDKFromServer(); } else { - if(pUpdaterObj != nullptr) + if (pUpdaterObj != nullptr) { delete pUpdaterObj; pUpdaterObj = nullptr; @@ -153,91 +183,86 @@ void UKBEMain::downloadSDKFromServer() } } -void UKBEMain::onImportClientSDKSuccessfully(const UKBEventData* pEventData) -{ - UKismetSystemLibrary::QuitGame(this, nullptr, EQuitPreference::Quit, true); -} - FString UKBEMain::getClientVersion() { - if (!KBEngineApp::getSingleton().isInitialized()) + if (!KBEngine::KBEngineApp::getSingleton().isInitialized()) return TEXT(""); - return KBEngineApp::getSingleton().clientVersion(); + return KBEngine::KBEngineApp::getSingleton().clientVersion(); } FString UKBEMain::getClientScriptVersion() { - if (!KBEngineApp::getSingleton().isInitialized()) + if (!KBEngine::KBEngineApp::getSingleton().isInitialized()) return TEXT(""); - return KBEngineApp::getSingleton().clientScriptVersion(); + return KBEngine::KBEngineApp::getSingleton().clientScriptVersion(); } FString UKBEMain::getServerVersion() { - if (!KBEngineApp::getSingleton().isInitialized()) + if (!KBEngine::KBEngineApp::getSingleton().isInitialized()) return TEXT(""); - return KBEngineApp::getSingleton().serverVersion(); + return KBEngine::KBEngineApp::getSingleton().serverVersion(); } FString UKBEMain::getServerScriptVersion() { - if (!KBEngineApp::getSingleton().isInitialized()) + if (!KBEngine::KBEngineApp::getSingleton().isInitialized()) return TEXT(""); - return KBEngineApp::getSingleton().serverScriptVersion(); + return KBEngine::KBEngineApp::getSingleton().serverScriptVersion(); } FString UKBEMain::getComponentName() { - if (!KBEngineApp::getSingleton().isInitialized()) + if (!KBEngine::KBEngineApp::getSingleton().isInitialized()) return TEXT(""); - return KBEngineApp::getSingleton().component(); + return KBEngine::KBEngineApp::getSingleton().component(); } bool UKBEMain::destroyKBEngine() { - if (!KBEngineApp::getSingleton().isInitialized()) + if (!KBEngine::KBEngineApp::getSingleton().isInitialized()) return false; - KBEngineApp::getSingleton().destroy(); + KBEngine::KBEngineApp::getSingleton().destroy(); KBENGINE_EVENT_CLEAR(); return true; } bool UKBEMain::login(FString username, FString password, TArray datas) { - if (!KBEngineApp::getSingleton().isInitialized()) + if (!KBEngine::KBEngineApp::getSingleton().isInitialized()) { return false; } - KBEngineApp::getSingleton().reset(); + KBEngine::KBEngineApp::getSingleton().reset(); UKBEventData_login* pEventData = NewObject(); pEventData->username = username; pEventData->password = password; pEventData->datas = datas; - KBENGINE_EVENT_FIRE(KBEventTypes::login, pEventData); + KBENGINE_EVENT_FIRE(KBEngine::KBEventTypes::login, pEventData); return true; } bool UKBEMain::createAccount(FString username, FString password, const TArray& datas) { - if (!KBEngineApp::getSingleton().isInitialized()) + if (!KBEngine::KBEngineApp::getSingleton().isInitialized()) { return false; } - KBEngineApp::getSingleton().reset(); + KBEngine::KBEngineApp::getSingleton().reset(); UKBEventData_createAccount* pEventData = NewObject(); pEventData->username = username; pEventData->password = password; pEventData->datas = datas; - KBENGINE_EVENT_FIRE(KBEventTypes::createAccount, pEventData); + KBENGINE_EVENT_FIRE(KBEngine::KBEventTypes::createAccount, pEventData); return true; } \ No newline at end of file diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBEMain.h b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBEMain.h index 1590049715..db88ebfd35 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBEMain.h +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBEMain.h @@ -5,6 +5,7 @@ #include "KBECommon.h" #include "KBEvent.h" #include "ClientSDKUpdater.h" +#include "ClientSDKUpdateUI.h" #include "Components/ActorComponent.h" #include "KBEMain.generated.h" @@ -58,26 +59,27 @@ class KBENGINEPLUGINS_API UKBEMain : public UActorComponent bool isUpdateSDK(); void downloadSDKFromServer(); + void onDownloadSDK(const UKBEventData* pEventData); void onImportClientSDKSuccessfully(const UKBEventData* pEventData); UFUNCTION(BlueprintCallable, Category = "KBEngine") - FString getClientVersion(); + static FString getClientVersion(); UFUNCTION(BlueprintCallable, Category = "KBEngine") - FString getClientScriptVersion(); + static FString getClientScriptVersion(); UFUNCTION(BlueprintCallable, Category = "KBEngine") - FString getServerVersion(); + static FString getServerVersion(); UFUNCTION(BlueprintCallable, Category = "KBEngine") - FString getServerScriptVersion(); + static FString getServerScriptVersion(); /* 客户端属于KBE框架中的一个功能组件，这里获取将固定返回client */ UFUNCTION(BlueprintCallable, Category = "KBEngine") - FString getComponentName(); + static FString getComponentName(); /** 在程序关闭时需要主动调用, 彻底销毁KBEngine @@ -136,6 +138,8 @@ class KBENGINEPLUGINS_API UKBEMain : public UActorComponent UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = KBEngine) bool automaticallyUpdateSDK; - ClientSDKUpdater* pUpdaterObj; + KBEngine::ClientSDKUpdater* pUpdaterObj; + + TSharedPtr ClientSDKUpdateUI; }; diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBETicker.cpp b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBETicker.cpp index a37637f9e3..36e21806b0 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBETicker.cpp +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBETicker.cpp @@ -6,9 +6,15 @@ #include "Entity.h" #include "KBEngine.h" +#if WITH_EDITOR +#include "Editor.h" +#endif UKBETicker::UKBETicker() { +#if WITH_EDITOR + FEditorDelegates::EndPIE.AddUObject(this, &UKBETicker::OnEndPIE); +#endif } UKBETicker::~UKBETicker() @@ -20,7 +26,7 @@ void UKBETicker::Tick(float DeltaTime) KBEvent::processOutEvents(); APawn* ue4_player = UGameplayStatics::GetPlayerPawn(GetWorld(), 0); - Entity* kbe_player = KBEngineApp::getSingleton().player(); + KBEngine::Entity* kbe_player = KBEngine::KBEngineApp::getSingleton().player(); // 每个tick将UE4的玩家坐标写入到KBE插件中的玩家实体坐标，插件会定期同步给服务器 if (kbe_player && ue4_player) @@ -31,7 +37,7 @@ void UKBETicker::Tick(float DeltaTime) kbe_player->isOnGround(ue4_player->GetMovementComponent() && ue4_player->GetMovementComponent()->IsMovingOnGround()); } - KBEngineApp::getSingleton().process(); + KBEngine::KBEngineApp::getSingleton().process(); } bool UKBETicker::IsTickable() const @@ -47,11 +53,10 @@ TStatId UKBETicker::GetStatId() const UWorld* UKBETicker::GetWorld() const { UWorld* World = (GetOuter() != nullptr) ? GetOuter()->GetWorld() : GWorld; - if (World == nullptr) + if (World == nullptr) { - World = GWorld; + World = GWorld; } - return World; } @@ -68,4 +73,11 @@ bool UKBETicker::IsTickableInEditor() const UWorld* UKBETicker::GetTickableGameObjectWorld() const { return GetWorld(); +} + +void UKBETicker::OnEndPIE(const bool data) +{ +#if WITH_EDITOR + KBEngine::KBEngineApp::destroyKBEngineApp(); +#endif } \ No newline at end of file diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBETicker.h b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBETicker.h index 3a7b8dda93..b4e8c2c456 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBETicker.h +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBETicker.h @@ -28,4 +28,6 @@ class KBENGINEPLUGINS_API UKBETicker : public UObject, public FTickableGameObjec virtual void Tick(float DeltaTime) override; virtual bool IsTickable() const override; virtual TStatId GetStatId() const override; + + void OnEndPIE(const bool); }; diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBEngine.cpp b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBEngine.cpp index baadd72648..1fbf9dc16f 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBEngine.cpp +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBEngine.cpp @@ -1,2338 +1,2343 @@ - -#include "KBEngine.h" -#include "KBEngineArgs.h" -#include "Entity.h" -#include "EntityDef.h" -#include "Messages.h" -#include "NetworkInterfaceTcp.h" -#include "NetworkInterfaceKcp.h" -#include "Bundle.h" -#include "MemoryStream.h" -#include "DataTypes.h" -#include "ScriptModule.h" -#include "Property.h" -#include "Method.h" -#include "EntityCall.h" -#include "Regex.h" -#include "KBDebug.h" -#include "KBEvent.h" -#include "EncryptionFilter.h" - -ServerErrorDescrs KBEngineApp::serverErrs_; - -KBEngineApp::KBEngineApp() : - pArgs_(NULL), - pNetworkInterface_(NULL), - username_(TEXT("")), - password_(TEXT("")), - baseappIP_(TEXT("")), - baseappTcpPort_(0), - baseappUdpPort_(0), - currserver_(TEXT("")), - currstate_(TEXT("")), - serverdatas_(), - clientdatas_(), - encryptedKey_(), - serverVersion_(TEXT("")), - clientVersion_(TEXT("")), - serverScriptVersion_(TEXT("")), - clientScriptVersion_(TEXT("")), - serverProtocolMD5_(TEXT("@{KBE_SERVER_PROTO_MD5}")), - serverEntitydefMD5_(TEXT("@{KBE_SERVER_ENTITYDEF_MD5}")), - entity_uuid_(0), - entity_id_(0), - entity_type_(TEXT("")), - useAliasEntityID_(@{KBE_USE_ALIAS_ENTITYID}), - controlledEntities_(), - entityServerPos_(), - spacedatas_(), - entities_(), - entityIDAliasIDList_(), - bufferedCreateEntityMessages_(), - lastTickTime_(0.0), - lastTickCBTime_(0.0), - lastUpdateToServerTime_(0.0), - spaceID_(0), - spaceResPath_(TEXT("")), - isLoadedGeometry_(false), - component_(TEXT("client")), - pFilter_(NULL), - pUKBETicker_(nullptr) -{ - INFO_MSG("KBEngineApp::KBEngineApp(): hello!"); - installUKBETicker(); -} - -KBEngineApp::KBEngineApp(KBEngineArgs* pArgs): - pArgs_(NULL), - pNetworkInterface_(NULL), - username_(TEXT("")), - password_(TEXT("")), - baseappIP_(TEXT("")), - baseappTcpPort_(0), - baseappUdpPort_(0), - currserver_(TEXT("")), - currstate_(TEXT("")), - serverdatas_(), - clientdatas_(), - encryptedKey_(), - serverVersion_(TEXT("")), - clientVersion_(TEXT("")), - serverScriptVersion_(TEXT("")), - clientScriptVersion_(TEXT("")), - serverProtocolMD5_(TEXT("@{KBE_SERVER_PROTO_MD5}")), - serverEntitydefMD5_(TEXT("@{KBE_SERVER_ENTITYDEF_MD5}")), - entity_uuid_(0), - entity_id_(0), - entity_type_(TEXT("")), - useAliasEntityID_(@{KBE_USE_ALIAS_ENTITYID}), - controlledEntities_(), - entityServerPos_(), - spacedatas_(), - entities_(), - entityIDAliasIDList_(), - bufferedCreateEntityMessages_(), - lastTickTime_(0.0), - lastTickCBTime_(0.0), - lastUpdateToServerTime_(0.0), - spaceID_(0), - spaceResPath_(TEXT("")), - isLoadedGeometry_(false), - component_(TEXT("client")), - pFilter_(NULL), - pUKBETicker_(nullptr) -{ - INFO_MSG("KBEngineApp::KBEngineApp(): hello!"); - initialize(pArgs); - installUKBETicker(); -} - -KBEngineApp::~KBEngineApp() -{ - destroy(); - INFO_MSG("KBEngineApp::~KBEngineApp(): destructed!"); -} - -KBEngineApp* pKBEngineApp = nullptr; - -KBEngineApp& KBEngineApp::getSingleton() -{ - if(!pKBEngineApp) - pKBEngineApp = new KBEngineApp(); - - return *pKBEngineApp; -} - -void KBEngineApp::destroyKBEngineApp() -{ - if(pKBEngineApp) - { - delete pKBEngineApp; - pKBEngineApp = nullptr; - KBEvent::clear(); - } -} - -bool KBEngineApp::initialize(KBEngineArgs* pArgs) -{ - if (isInitialized()) - return false; - - EntityDef::initialize(); - - // 注册事件 - installEvents(); - - pArgs_ = pArgs; - reset(); - return true; -} - -void KBEngineApp::installEvents() -{ - KBENGINE_REGISTER_EVENT_OVERRIDE_FUNC(KBEventTypes::login, KBEventTypes::login, [this](const UKBEventData* pEventData) - { - const UKBEventData_login& data = static_cast(*pEventData); - login(data.username, data.password, data.datas); - }); - - KBENGINE_REGISTER_EVENT_OVERRIDE_FUNC(KBEventTypes::logout, KBEventTypes::logout, [this](const UKBEventData* pEventData) - { - logout(); - }); - - KBENGINE_REGISTER_EVENT_OVERRIDE_FUNC(KBEventTypes::createAccount, KBEventTypes::createAccount, [this](const UKBEventData* pEventData) - { - const UKBEventData_createAccount& data = static_cast(*pEventData); - createAccount(data.username, data.password, data.datas); - }); - - KBENGINE_REGISTER_EVENT_OVERRIDE_FUNC(KBEventTypes::reloginBaseapp, KBEventTypes::reloginBaseapp, [this](const UKBEventData* pEventData) - { - reloginBaseapp(); - }); - - KBENGINE_REGISTER_EVENT_OVERRIDE_FUNC(KBEventTypes::resetPassword, KBEventTypes::resetPassword, [this](const UKBEventData* pEventData) - { - const UKBEventData_resetPassword& data = static_cast(*pEventData); - resetPassword(data.username); - }); - - KBENGINE_REGISTER_EVENT_OVERRIDE_FUNC(KBEventTypes::bindAccountEmail, KBEventTypes::bindAccountEmail, [this](const UKBEventData* pEventData) - { - const UKBEventData_bindAccountEmail& data = static_cast(*pEventData); - bindAccountEmail(data.email); - }); - - KBENGINE_REGISTER_EVENT_OVERRIDE_FUNC(KBEventTypes::newPassword, KBEventTypes::newPassword, [this](const UKBEventData* pEventData) - { - const UKBEventData_newPassword& data = static_cast(*pEventData); - newPassword(data.old_password, data.new_password); - }); - - // 内部事件 - KBENGINE_REGISTER_EVENT_OVERRIDE_FUNC("_closeNetwork", "_closeNetwork", [this](const UKBEventData* pEventData) - { - _closeNetwork(); - }); -} - -void KBEngineApp::destroy() -{ - reset(); - KBENGINE_DEREGISTER_ALL_EVENT(); - resetMessages(); - - KBE_SAFE_RELEASE(pArgs_); - KBE_SAFE_RELEASE(pNetworkInterface_); - KBE_SAFE_RELEASE(pFilter_); - uninstallUKBETicker(); -} - -void KBEngineApp::resetMessages() -{ - serverErrs_.Clear(); - - Messages::clear(); - EntityDef::clear(); - Entity::clear(); - - INFO_MSG("KBEngineApp::resetMessages(): done!"); -} - -void KBEngineApp::reset() -{ - KBEvent::clearFiredEvents(); - - clearEntities(true); - - currserver_ = TEXT(""); - currstate_ = TEXT(""); - - serverdatas_.Empty(); - - serverVersion_ = TEXT(""); - clientVersion_ = TEXT("@{KBE_VERSION}"); - serverScriptVersion_ = TEXT(""); - clientScriptVersion_ = TEXT("@{KBE_SCRIPT_VERSION}"); - - entity_uuid_ = 0; - entity_id_ = 0; - entity_type_ = TEXT(""); - - entityIDAliasIDList_.Empty(); - bufferedCreateEntityMessages_.Empty(); - - lastTickTime_ = getTimeSeconds(); - lastTickCBTime_ = getTimeSeconds(); - lastUpdateToServerTime_ = getTimeSeconds(); - - spacedatas_.Empty(); - - spaceID_ = 0; - spaceResPath_ = TEXT(""); - isLoadedGeometry_ = false; - - baseappUdpPort_ = 0; - - initNetwork(); -} - -void KBEngineApp::installUKBETicker() -{ - if (pUKBETicker_ == nullptr) - { - pUKBETicker_ = NewObject(); - pUKBETicker_->AddToRoot(); - } -} - -void KBEngineApp::uninstallUKBETicker() -{ - if (pUKBETicker_) - { - pUKBETicker_->RemoveFromRoot(); - pUKBETicker_ = nullptr; - } -} - -bool KBEngineApp::initNetwork() -{ - KBE_SAFE_RELEASE(pFilter_); - - if (pNetworkInterface_) - delete pNetworkInterface_; - - Messages::initialize(); - - if (pArgs_) - { - if (pArgs_->forceDisableUDP || baseappUdpPort_ == 0) - pNetworkInterface_ = new NetworkInterfaceTCP(); - else - pNetworkInterface_ = new NetworkInterfaceKCP(); - } - - return true; -} - -void KBEngineApp::_closeNetwork() -{ - if (pNetworkInterface_) - pNetworkInterface_->close(); -} - -bool KBEngineApp::validEmail(const FString& strEmail) -{ - const FRegexPattern spattern(FString(TEXT("[a-z0-9._%+-]+@[a-z0-9.-]+\\.[a-z]{2,4}"))); - - FRegexMatcher fMatcher(spattern, strEmail); - - if (fMatcher.FindNext()) { - return true; - } - - return false; -} - -void KBEngineApp::process() -{ - // 处理网络 - if (pNetworkInterface_) - pNetworkInterface_->process(); - - // 处理外层抛入的事件 - KBEvent::processInEvents(); - - // 向服务端发送心跳以及同步角色信息到服务端 - sendTick(); -} - -void KBEngineApp::sendTick() -{ - if (!pNetworkInterface_ || !pNetworkInterface_->valid()) - return; - - double span = getTimeSeconds() - lastTickTime_; - - // 更新玩家的位置与朝向到服务端 - updatePlayerToServer(); - - if (pArgs_->serverHeartbeatTick > 0 && span > pArgs_->serverHeartbeatTick) - { - span = lastTickCBTime_ - lastTickTime_; - - // 如果心跳回调接收时间小于心跳发送时间，说明没有收到回调 - // 此时应该通知客户端掉线了 - if (span < 0) - { - SCREEN_ERROR_MSG("KBEngineApp::sendTick(): Receive appTick timeout!"); - pNetworkInterface_->close(); - return; - } - - Message** Loginapp_onClientActiveTickMsgFind = Messages::messages.Find("Loginapp_onClientActiveTick"); - Message** Baseapp_onClientActiveTickMsgFind = Messages::messages.Find("Baseapp_onClientActiveTick"); - - if (currserver_ == TEXT("loginapp")) - { - if (Loginapp_onClientActiveTickMsgFind) - { - Bundle* pBundle = Bundle::createObject(); - pBundle->newMessage(*Loginapp_onClientActiveTickMsgFind); - pBundle->send(pNetworkInterface_); - } - } - else - { - if (Baseapp_onClientActiveTickMsgFind) - { - Bundle* pBundle = Bundle::createObject(); - pBundle->newMessage(*Baseapp_onClientActiveTickMsgFind); - pBundle->send(pNetworkInterface_); - } - } - - lastTickTime_ = getTimeSeconds(); - } -} - -Entity* KBEngineApp::player() -{ - return findEntity(entity_id_); -} - -Entity* KBEngineApp::findEntity(int32 entityID) -{ - Entity** pEntity = entities_.Find(entityID); - if (pEntity == nullptr) - return NULL; - - return *pEntity; -} - -FString KBEngineApp::serverErr(uint16 id) -{ - return serverErrs_.ServerErrStr(id); -} - -void KBEngineApp::updatePlayerToServer() -{ - if (pArgs_->syncPlayerMS <= 0 || spaceID_ == 0) - return; - - double tnow = getTimeSeconds(); - double span = tnow - lastUpdateToServerTime_; - - if (span < ((double)pArgs_->syncPlayerMS / 1000.0)) - return; - - Entity* pPlayerEntity = player(); - if (!pPlayerEntity || !pPlayerEntity->inWorld() || pPlayerEntity->isControlled()) - return; - - lastUpdateToServerTime_ = tnow; - const FVector& position = pPlayerEntity->position; - const FVector& direction = pPlayerEntity->direction; - - bool posHasChanged = (pPlayerEntity->entityLastLocalPos - position).Size() > 0.001f; - bool dirHasChanged = (pPlayerEntity->entityLastLocalDir - direction).Size() > 0.001f; - - if (posHasChanged || dirHasChanged) - { - pPlayerEntity->entityLastLocalPos = position; - pPlayerEntity->entityLastLocalDir = direction; - - Bundle* pBundle = Bundle::createObject(); - pBundle->newMessage(Messages::messages[TEXT("Baseapp_onUpdateDataFromClient"])); - (*pBundle) << position.X; - (*pBundle) << position.Y; - (*pBundle) << position.Z; - - (*pBundle) << direction.X; - (*pBundle) << direction.Y; - (*pBundle) << direction.Z; - - (*pBundle) << (uint8)pPlayerEntity->isOnGround(); - (*pBundle) << spaceID_; - - pBundle->send(pNetworkInterface_); - } - - // 开始同步所有被控制了的entity的位置 - for(auto& item : controlledEntities_) - { - Entity* pEntity = item; - const FVector& e_position = pEntity->position; - const FVector& e_direction = pEntity->direction; - - posHasChanged = (pEntity->entityLastLocalPos - e_position).Size() > 0.001f; - dirHasChanged = (pEntity->entityLastLocalDir - e_direction).Size() > 0.001f; - - if (posHasChanged || dirHasChanged) - { - pEntity->entityLastLocalPos = e_position; - pEntity->entityLastLocalDir = e_direction; - - Bundle* pBundle = Bundle::createObject(); - pBundle->newMessage(Messages::messages[TEXT("Baseapp_onUpdateDataFromClientForControlledEntity"])); - (*pBundle) << pEntity->id(); - (*pBundle) << e_position.X; - (*pBundle) << e_position.Y; - (*pBundle) << e_position.Z; - - (*pBundle) << e_direction.X; - (*pBundle) << e_direction.Y; - (*pBundle) << e_direction.Z; - - (*pBundle) << (uint8)pEntity->isOnGround(); - (*pBundle) << spaceID_; - - pBundle->send(pNetworkInterface_); - } - } - -} - -void KBEngineApp::Client_onAppActiveTickCB() -{ - lastTickCBTime_ = getTimeSeconds(); -} - -void KBEngineApp::hello() -{ - Bundle* pBundle = Bundle::createObject(); - if (currserver_ == TEXT("loginapp")) - pBundle->newMessage(Messages::messages[TEXT("Loginapp_hello")]); - else - pBundle->newMessage(Messages::messages[TEXT("Baseapp_hello")]); - - KBE_SAFE_RELEASE(pFilter_); - - if (pArgs_->networkEncryptType == NETWORK_ENCRYPT_TYPE::ENCRYPT_TYPE_BLOWFISH) - { - pFilter_ = new BlowfishFilter(); - encryptedKey_ = ((BlowfishFilter*)pFilter_)->key(); - pNetworkInterface_->setFilter(NULL); - } - - (*pBundle) << clientVersion_; - (*pBundle) << clientScriptVersion_; - pBundle->appendBlob(encryptedKey_); - pBundle->send(pNetworkInterface_); -} - -void KBEngineApp::Client_onHelloCB(MemoryStream& stream) -{ - FString str_serverVersion; - stream >> str_serverVersion; - stream >> serverScriptVersion_; - - FString serverProtocolMD5; - stream >> serverProtocolMD5; - - FString serverEntitydefMD5; - stream >> serverEntitydefMD5; - - int32 ctype; - stream >> ctype; - - INFO_MSG("KBEngineApp::Client_onHelloCB(): verInfo(%s), scriptVersion(%s), srvProtocolMD5(%s), srvEntitydefMD5(%s), ctype(%d)!", - *str_serverVersion, *serverScriptVersion_, *serverProtocolMD5_, *serverEntitydefMD5_, ctype); - - if(str_serverVersion != "Getting") - { - serverVersion_ = str_serverVersion; - - /* - if(serverProtocolMD5_ != serverProtocolMD5) - { - ERROR_MSG("KBEngineApp::Client_onHelloCB(): digest not match! serverProtocolMD5=%s(server: %s)", *serverProtocolMD5_, *serverProtocolMD5); - - UKBEventData_onVersionNotMatch* pEventData = NewObject(); - pEventData->clientVersion = clientVersion_; - pEventData->serverVersion = serverVersion_; - KBENGINE_EVENT_FIRE(KBEventTypes::onVersionNotMatch, pEventData); - return; - } - */ - - if(serverEntitydefMD5_ != serverEntitydefMD5) - { - ERROR_MSG("KBEngineApp::Client_onHelloCB(): digest not match! serverEntitydefMD5=%s(server: %s)", *serverEntitydefMD5_, *serverEntitydefMD5); - - UKBEventData_onVersionNotMatch* pEventData = NewObject(); - pEventData->clientVersion = clientVersion_; - pEventData->serverVersion = serverVersion_; - KBENGINE_EVENT_FIRE(KBEventTypes::onVersionNotMatch, pEventData); - return; - } - } - - if (pArgs_->networkEncryptType == NETWORK_ENCRYPT_TYPE::ENCRYPT_TYPE_BLOWFISH) - { - pNetworkInterface_->setFilter(pFilter_); - pFilter_ = NULL; - } - - onServerDigest(); - - if (currserver_ == TEXT("baseapp")) - { - onLogin_baseapp(); - } - else - { - onLogin_loginapp(); - } -} - -void KBEngineApp::Client_onVersionNotMatch(MemoryStream& stream) -{ - stream >> serverVersion_; - - ERROR_MSG("KBEngineApp::Client_onVersionNotMatch(): verInfo=%s(server: %s)", *clientVersion_, *serverVersion_); - - UKBEventData_onVersionNotMatch* pEventData = NewObject(); - pEventData->clientVersion = clientVersion_; - pEventData->serverVersion = serverVersion_; - KBENGINE_EVENT_FIRE(KBEventTypes::onVersionNotMatch, pEventData); -} - -void KBEngineApp::Client_onScriptVersionNotMatch(MemoryStream& stream) -{ - stream >> serverScriptVersion_; - - ERROR_MSG("KBEngineApp::Client_onScriptVersionNotMatch(): verInfo=%s(server: %s)", *clientScriptVersion_, *serverScriptVersion_); - - UKBEventData_onScriptVersionNotMatch* pEventData = NewObject(); - pEventData->clientScriptVersion = clientScriptVersion_; - pEventData->serverScriptVersion = serverScriptVersion_; - KBENGINE_EVENT_FIRE(KBEventTypes::onScriptVersionNotMatch, pEventData); -} - -void KBEngineApp::Client_onImportClientSDK(MemoryStream& stream) -{ - UKBEventData_onImportClientSDK* pEventData = NewObject(); - - pEventData->remainingFiles = stream.readInt32(); - pEventData->fileName = stream.readString(); - pEventData->fileSize = stream.readInt32(); - stream.readBlob(pEventData->fileDatas); - - KBENGINE_EVENT_FIRE("onImportClientSDK", pEventData); -} - -void KBEngineApp::Client_onKicked(uint16 failedcode) -{ - DEBUG_MSG("KBEngineApp::Client_onKicked(): failedcode=%d, %s", failedcode, *serverErr(failedcode)); - - UKBEventData_onKicked* pEventData = NewObject(); - pEventData->failedcode = failedcode; - pEventData->errorStr = serverErr(failedcode); - KBENGINE_EVENT_FIRE(KBEventTypes::onKicked, pEventData); -} - -void KBEngineApp::onServerDigest() -{ -} - -void KBEngineApp::onConnectCallback(FString ip, uint16 port, bool success, int userdata) -{ - if (userdata == 0) - { - onConnectTo_loginapp_login_callback(ip, port, success); - } - else if (userdata == 1) - { - onConnectTo_loginapp_create_callback(ip, port, success); - } - else if (userdata == 2) - { - onConnectTo_baseapp_callback(ip, port, success); - } - else if (userdata == 3) - { - onReloginTo_baseapp_callback(ip, port, success); - } - else if (userdata == 4) - { - onConnectTo_resetpassword_callback(ip, port, success); - } - else if (userdata == 5) - { - //onConnectTo_resetpassword_callback(ip, port, success); - } - else - { - check(false); - } -} - -bool KBEngineApp::login(const FString& username, const FString& password, const TArray& datas) -{ - if (username.Len() == 0) - { - ERROR_MSG("KBEngineApp::login(): username is empty!"); - return false; - } - - if (password.Len() == 0) - { - ERROR_MSG("KBEngineApp::login(): password is empty!"); - return false; - } - - username_ = username; - password_ = password; - clientdatas_ = datas; - - login_loginapp(true); - return true; -} - -bool KBEngineApp::logout() -{ - if (currserver_ != TEXT("baseapp")) - return false; - - INFO_MSG("KBEngineApp::logout()"); - Bundle* pBundle = Bundle::createObject(); - pBundle->newMessage(Messages::messages[TEXT("Baseapp_logoutBaseapp"])); - (*pBundle) << entity_uuid_; - (*pBundle) << entity_id_; - pBundle->send(pNetworkInterface_); - return true; -} - -void KBEngineApp::login_loginapp(bool noconnect) -{ - if (noconnect) - { - reset(); - pNetworkInterface_->connectTo(pArgs_->ip, pArgs_->port, this, 0); - } - else - { - INFO_MSG("KBEngineApp::login_loginapp(): send login! username=%s", *username_); - Bundle* pBundle = Bundle::createObject(); - pBundle->newMessage(Messages::messages[TEXT("Loginapp_login"])); - (*pBundle) << (uint8)pArgs_->clientType; - pBundle->appendBlob(clientdatas_); - (*pBundle) << username_; - (*pBundle) << password_; - pBundle->send(pNetworkInterface_); - } -} - -void KBEngineApp::onConnectTo_loginapp_login_callback(FString ip, uint16 port, bool success) -{ - if (!success) - { - ERROR_MSG("KBEngineApp::onConnectTo_loginapp_login_callback(): connect %s:%d is error!", *ip, port); - return; - } - - currserver_ = TEXT("loginapp"); - currstate_ = TEXT("login"); - - INFO_MSG("KBEngineApp::onConnectTo_loginapp_login_callback(): connect %s:%d is success!", *ip, port); - - hello(); -} - -void KBEngineApp::onLogin_loginapp() -{ - lastTickCBTime_ = getTimeSeconds(); - login_loginapp(false); -} - -void KBEngineApp::Client_onLoginFailed(MemoryStream& stream) -{ - uint16 failedcode = 0; - stream >> failedcode; - stream.readBlob(serverdatas_); - ERROR_MSG("KBEngineApp::Client_onLoginFailed(): failedcode(%d:%s), datas(%d)!", failedcode, *serverErr(failedcode), serverdatas_.Num()); - - UKBEventData_onLoginFailed* pEventData = NewObject(); - pEventData->failedcode = failedcode; - pEventData->errorStr = serverErr(failedcode); - KBENGINE_EVENT_FIRE(KBEventTypes::onLoginFailed, pEventData); -} - -void KBEngineApp::Client_onLoginSuccessfully(MemoryStream& stream) -{ - FString accountName; - stream >> accountName; - username_ = accountName; - stream >> baseappIP_; - stream >> baseappTcpPort_; - stream >> baseappUdpPort_; - stream.readBlob(serverdatas_); - - DEBUG_MSG("KBEngineApp::Client_onLoginSuccessfully(): accountName(%s), addr(" - "%s:%d:%d), datas(%d)!", *accountName, *baseappIP_, baseappTcpPort_, baseappUdpPort_, serverdatas_.Num()); - - login_baseapp(true); -} - -void KBEngineApp::login_baseapp(bool noconnect) -{ - if (noconnect) - { - KBENGINE_EVENT_FIRE(KBEventTypes::onLoginBaseapp, NewObject()); - - pNetworkInterface_->destroy(); - pNetworkInterface_ = NULL; - initNetwork(); - pNetworkInterface_->connectTo(baseappIP_, (!pArgs_->forceDisableUDP && baseappUdpPort_ > 0) ? baseappUdpPort_ : baseappTcpPort_, this, 2); - } - else - { - Bundle* pBundle = Bundle::createObject(); - pBundle->newMessage(Messages::messages[TEXT("Baseapp_loginBaseapp"])); - (*pBundle) << username_; - (*pBundle) << password_; - pBundle->send(pNetworkInterface_); - } -} - -void KBEngineApp::onConnectTo_baseapp_callback(FString ip, uint16 port, bool success) -{ - lastTickCBTime_ = getTimeSeconds(); - - if (!success) - { - ERROR_MSG("KBEngineApp::onConnectTo_baseapp_callback(): connect %s:%d is error!", *ip, port); - return; - } - - currserver_ = TEXT("baseapp"); - currstate_ = TEXT(""); - - DEBUG_MSG("KBEngineApp::onConnectTo_baseapp_callback(): connect %s:%d is successfully!", *ip, port); - - hello(); -} - -void KBEngineApp::onLogin_baseapp() -{ - lastTickCBTime_ = getTimeSeconds(); - login_baseapp(false); -} - -void KBEngineApp::reloginBaseapp() -{ - lastTickTime_ = getTimeSeconds(); - lastTickCBTime_ = getTimeSeconds(); - - if(pNetworkInterface_->valid()) - return; - - UKBEventData_onReloginBaseapp* pEventData = NewObject(); - KBENGINE_EVENT_FIRE(KBEventTypes::onReloginBaseapp, pEventData); - - pNetworkInterface_->connectTo(baseappIP_, (!pArgs_->forceDisableUDP && baseappUdpPort_ > 0) ? baseappUdpPort_ : baseappTcpPort_, this, 3); -} - -void KBEngineApp::onReloginTo_baseapp_callback(FString ip, uint16 port, bool success) -{ - if (!success) - { - ERROR_MSG("KBEngineApp::onReloginTo_baseapp_callback(): connect %s:%d is error!", *ip, port); - return; - } - - INFO_MSG("KBEngineApp::onReloginTo_baseapp_callback(): connect %s:%d is success!", *ip, port); - - Bundle* pBundle = Bundle::createObject(); - pBundle->newMessage(Messages::messages[TEXT("Baseapp_reloginBaseapp"])); - (*pBundle) << username_; - (*pBundle) << password_; - (*pBundle) << entity_uuid_; - (*pBundle) << entity_id_; - pBundle->send(pNetworkInterface_); - - lastTickCBTime_ = getTimeSeconds(); -} - -void KBEngineApp::Client_onLoginBaseappFailed(uint16 failedcode) -{ - ERROR_MSG("KBEngineApp::Client_onLoginBaseappFailed(): failedcode(%d:%s)!", failedcode, *serverErr(failedcode)); - - UKBEventData_onLoginBaseappFailed* pEventData = NewObject(); - pEventData->failedcode = failedcode; - pEventData->errorStr = serverErr(failedcode); - KBENGINE_EVENT_FIRE(KBEventTypes::onLoginBaseappFailed, pEventData); -} - -void KBEngineApp::Client_onReloginBaseappFailed(uint16 failedcode) -{ - ERROR_MSG("KBEngineApp::Client_onReloginBaseappFailed(): failedcode(%d:%s)!", failedcode, *serverErr(failedcode)); - - UKBEventData_onReloginBaseappFailed* pEventData = NewObject(); - pEventData->failedcode = failedcode; - pEventData->errorStr = serverErr(failedcode); - KBENGINE_EVENT_FIRE(KBEventTypes::onReloginBaseappFailed, pEventData); -} - -void KBEngineApp::Client_onReloginBaseappSuccessfully(MemoryStream& stream) -{ - stream >> entity_uuid_; - ERROR_MSG("KBEngineApp::Client_onReloginBaseappSuccessfully(): name(%s)!", *username_); - UKBEventData_onReloginBaseappSuccessfully* pEventData = NewObject(); - KBENGINE_EVENT_FIRE(KBEventTypes::onReloginBaseappSuccessfully, pEventData); -} - -void KBEngineApp::Client_onCreatedProxies(uint64 rndUUID, int32 eid, FString& entityType) -{ - DEBUG_MSG("KBEngineApp::Client_onCreatedProxies(): eid(%d), entityType(%s)!", eid, *entityType); - - entity_uuid_ = rndUUID; - entity_id_ = eid; - entity_type_ = entityType; - - if (!entities_.Contains(eid)) - { - ScriptModule** pModuleFind = EntityDef::moduledefs.Find(entityType); - if (!pModuleFind) - { - SCREEN_ERROR_MSG("KBEngineApp::Client_onCreatedProxies(): not found ScriptModule(%s)!", *entityType); - return; - } - - ScriptModule* pModule = *pModuleFind; - - Entity* pEntity = pModule->createEntity(); - pEntity->id(eid); - pEntity->className(entityType); - pEntity->onGetBase(); - - entities_.Add(eid, pEntity); - - MemoryStream** entityMessageFind = bufferedCreateEntityMessages_.Find(eid); - if (entityMessageFind) - { - MemoryStream* entityMessage = *entityMessageFind; - Client_onUpdatePropertys(*entityMessage); - bufferedCreateEntityMessages_.Remove(eid); - MemoryStream::reclaimObject(entityMessage); - } - - pEntity->__init__(); - pEntity->attachComponents(); - pEntity->inited(true); - - if (pArgs_->isOnInitCallPropertysSetMethods) - pEntity->callPropertysSetMethods(); - } - else - { - MemoryStream** entityMessageFind = bufferedCreateEntityMessages_.Find(eid); - if (entityMessageFind) - { - MemoryStream* entityMessage = *entityMessageFind; - Client_onUpdatePropertys(*entityMessage); - bufferedCreateEntityMessages_.Remove(eid); - MemoryStream::reclaimObject(entityMessage); - } - } -} - -ENTITY_ID KBEngineApp::getViewEntityIDFromStream(MemoryStream& stream) -{ - ENTITY_ID id = 0; - - if (!pArgs_->useAliasEntityID) - { - stream >> id; - return id; - } - - if (entityIDAliasIDList_.Num()> 255) - { - stream >> id; - } - else - { - uint8 aliasID = 0; - stream >> aliasID; - - // 如果为0且客户端上一步是重登陆或者重连操作并且服务端entity在断线期间一直处于在线状态 - // 则可以忽略这个错误, 因为cellapp可能一直在向baseapp发送同步消息， 当客户端重连上时未等 - // 服务端初始化步骤开始则收到同步信息, 此时这里就会出错。 - if (entityIDAliasIDList_.Num() <= aliasID) - return 0; - - id = entityIDAliasIDList_[aliasID]; - } - - return id; -} - -void KBEngineApp::Client_onUpdatePropertysOptimized(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - onUpdatePropertys_(eid, stream); -} - -void KBEngineApp::Client_onUpdatePropertys(MemoryStream& stream) -{ - ENTITY_ID eid; - stream >> eid; - onUpdatePropertys_(eid, stream); -} - -void KBEngineApp::onUpdatePropertys_(ENTITY_ID eid, MemoryStream& stream) -{ - Entity** pEntityFind = entities_.Find(eid); - - if (!pEntityFind) - { - MemoryStream** entityMessageFind = bufferedCreateEntityMessages_.Find(eid); - if (entityMessageFind) - { - ERROR_MSG("KBEngineApp::onUpdatePropertys_(): entity(%d) not found!", eid); - return; - } - - MemoryStream* stream1 = MemoryStream::createObject(); - stream1->append(stream); - stream1->rpos(stream.rpos() - 4); - bufferedCreateEntityMessages_.Add(eid, stream1); - return; - } - - Entity* pEntity = *pEntityFind; - - pEntity->onUpdatePropertys(stream); -} - -void KBEngineApp::Client_onEntityDestroyed(int32 eid) -{ - DEBUG_MSG("KBEngineApp::Client_onEntityDestroyed(): entity(%d)!", eid); - - Entity** pEntityFind = entities_.Find(eid); - - if (!pEntityFind) - { - ERROR_MSG("KBEngineApp::Client_onEntityDestroyed(): entity(%d) not found!", eid); - return; - } - - Entity* pEntity = (*pEntityFind); - - if (pEntity->inWorld()) - { - if (entity_id_ == eid) - clearSpace(false); - - (*pEntityFind)->leaveWorld(); - } - - if (controlledEntities_.Contains(pEntity)) - { - controlledEntities_.Remove(pEntity); - - UKBEventData_onLoseControlledEntity* pEventData = NewObject(); - pEventData->entityID = pEntity->id(); - KBENGINE_EVENT_FIRE(KBEventTypes::onLoseControlledEntity, pEventData); - } - - entities_.Remove(eid); - pEntity->destroy(); -} - -void KBEngineApp::clearSpace(bool isall) -{ - entityIDAliasIDList_.Empty(); - spacedatas_.Empty(); - clearEntities(isall); - isLoadedGeometry_ = false; - spaceID_ = 0; -} - -void KBEngineApp::clearEntities(bool isall) -{ - controlledEntities_.Empty(); - - if (!isall) - { - Entity* pEntity = player(); - - for(auto& item : entities_) - { - if (item.Key == pEntity->id()) - continue; - - if (item.Value->inWorld()) - item.Value->leaveWorld(); - - item.Value->destroy(); - } - - entities_.Empty(); - entities_.Add(pEntity->id(), pEntity); - } - else - { - for (auto& item : entities_) - { - if (item.Value->inWorld()) - item.Value->leaveWorld(); - - item.Value->destroy(); - } - - entities_.Empty(); - } -} - -void KBEngineApp::Client_initSpaceData(MemoryStream& stream) -{ - clearSpace(false); - stream >> spaceID_; - - while (stream.length() > 0) - { - FString key; - FString val; - - stream >> key >> val; - Client_setSpaceData(spaceID_, key, val); - } - - DEBUG_MSG("KBEngineApp::Client_initSpaceData(): spaceID(%d), size(%d)!", spaceID_, spacedatas_.Num()); -} - -void KBEngineApp::Client_setSpaceData(uint32 spaceID, const FString& key, const FString& value) -{ - DEBUG_MSG("KBEngineApp::Client_setSpaceData(): spaceID(%d), key(%s), value(%s)!", spaceID_, *key, *value); - spacedatas_.Add(key, value); - - if (key == TEXT("_mapping")) - addSpaceGeometryMapping(spaceID, value); - - UKBEventData_onSetSpaceData* pEventData = NewObject(); - pEventData->spaceID = spaceID_; - pEventData->key = key; - pEventData->value = value; - KBENGINE_EVENT_FIRE(KBEventTypes::onSetSpaceData, pEventData); -} - -void KBEngineApp::Client_delSpaceData(uint32 spaceID, const FString& key) -{ - DEBUG_MSG("KBEngineApp::Client_delSpaceData(): spaceID(%d), key(%s)!", spaceID_, *key); - - spacedatas_.Remove(key); - - UKBEventData_onDelSpaceData* pEventData = NewObject(); - pEventData->spaceID = spaceID_; - pEventData->key = key; - KBENGINE_EVENT_FIRE(KBEventTypes::onDelSpaceData, pEventData); -} - -void KBEngineApp::addSpaceGeometryMapping(uint32 uspaceID, const FString& respath) -{ - DEBUG_MSG("KBEngineApp::addSpaceGeometryMapping(): spaceID(%d), respath(%s)!", spaceID_, *respath); - - isLoadedGeometry_ = true; - spaceID_ = uspaceID; - spaceResPath_ = respath; - - UKBEventData_addSpaceGeometryMapping* pEventData = NewObject(); - pEventData->spaceResPath = spaceResPath_; - KBENGINE_EVENT_FIRE(KBEventTypes::addSpaceGeometryMapping, pEventData); -} - -FString KBEngineApp::getSpaceData(const FString& key) -{ - FString* valFind = spacedatas_.Find(key); - - if(!valFind) - return FString(); - - return (*valFind); -} - -void KBEngineApp::resetPassword(const FString& username) -{ - username_ = username; - resetpassword_loginapp(true); -} - -void KBEngineApp::resetpassword_loginapp(bool noconnect) -{ - if (noconnect) - { - reset(); - pNetworkInterface_->connectTo(pArgs_->ip, pArgs_->port, this, 4); - } - else - { - INFO_MSG("KBEngineApp::resetpassword_loginapp(): send resetpassword! username=%s", *username_); - Bundle* pBundle = Bundle::createObject(); - pBundle->newMessage(Messages::messages[TEXT("Loginapp_reqAccountResetPassword"])); - (*pBundle) << username_; - pBundle->send(pNetworkInterface_); - } -} - -void KBEngineApp::onOpenLoginapp_resetpassword() -{ - DEBUG_MSG("KBEngineApp::onOpenLoginapp_resetpassword(): successfully!"); - currserver_ = "loginapp"; - currstate_ = "resetpassword"; - lastTickCBTime_ = getTimeSeconds(); - - resetpassword_loginapp(false); -} - -void KBEngineApp::onConnectTo_resetpassword_callback(FString ip, uint16 port, bool success) -{ - lastTickCBTime_ = getTimeSeconds(); - - if (!success) - { - ERROR_MSG("KBEngineApp::onConnectTo_resetpassword_callback(): connect %s:%d is error!", *ip, port); - return; - } - - INFO_MSG("KBEngineApp::onConnectTo_resetpassword_callback(): connect %s:%d is success!", *ip, port); - - onOpenLoginapp_resetpassword(); -} - -void KBEngineApp::Client_onReqAccountResetPasswordCB(uint16 failcode) -{ - if (failcode != 0) - { - ERROR_MSG("KBEngineApp::Client_onReqAccountResetPasswordCB(): reset failed! code=%d, error=%s! username=%s", failcode, *serverErr(failcode), *username_); - return; - } - - DEBUG_MSG("KBEngineApp::Client_onReqAccountResetPasswordCB(): successfully! username=%s", *username_); -} - -bool KBEngineApp::createAccount(const FString& username, const FString& password, const TArray& datas) -{ - if (username.Len() == 0) - { - ERROR_MSG("KBEngineApp::createAccount(): username is empty!"); - return false; - } - - if (password.Len() == 0) - { - ERROR_MSG("KBEngineApp::createAccount(): password is empty!"); - return false; - } - - username_ = username; - password_ = password; - clientdatas_ = datas; - - createAccount_loginapp(true); - return true; -} - -void KBEngineApp::createAccount_loginapp(bool noconnect) -{ - if (noconnect) - { - reset(); - pNetworkInterface_->connectTo(pArgs_->ip, pArgs_->port, this, 1); - } - else - { - INFO_MSG("KBEngineApp::createAccount_loginapp(): send create! username=%s", *username_); - Bundle* pBundle = Bundle::createObject(); - pBundle->newMessage(Messages::messages[TEXT("Loginapp_reqCreateAccount"])); - (*pBundle) << username_; - (*pBundle) << password_; - pBundle->appendBlob(clientdatas_); - pBundle->send(pNetworkInterface_); - } -} - -void KBEngineApp::onOpenLoginapp_createAccount() -{ - DEBUG_MSG("KBEngineApp::onOpenLoginapp_createAccount(): successfully!"); - - currserver_ = TEXT("loginapp"); - currstate_ = TEXT("createAccount"); - lastTickCBTime_ = getTimeSeconds(); - - createAccount_loginapp(false); -} - -void KBEngineApp::onConnectTo_loginapp_create_callback(FString ip, uint16 port, bool success) -{ - lastTickCBTime_ = getTimeSeconds(); - - if (!success) - { - ERROR_MSG("KBEngineApp::onConnectTo_loginapp_create_callback(): connect %s:%d is error!", *ip, port); - return; - } - - INFO_MSG("KBEngineApp::onConnectTo_loginapp_create_callback(): connect %s:%d is success!", *ip, port); - - onOpenLoginapp_createAccount(); -} - -void KBEngineApp::Client_onCreateAccountResult(MemoryStream& stream) -{ - uint16 retcode; - stream >> retcode; - - TArray datas; - stream.readBlob(datas); - - UKBEventData_onCreateAccountResult* pEventData = NewObject(); - pEventData->errorCode = retcode; - pEventData->errorStr = serverErr(retcode); - pEventData->datas = datas; - KBENGINE_EVENT_FIRE(KBEventTypes::onCreateAccountResult, pEventData); - - if (retcode != 0) - { - WARNING_MSG("KBEngineApp::Client_onCreateAccountResult(): create(%s) failed! error=%d(%s)!", *username_, retcode, *serverErr(retcode)); - return; - } - - DEBUG_MSG("KBEngineApp::Client_onCreateAccountResult(): create(%s) is successfully!", *username_); -} - -void KBEngineApp::bindAccountEmail(const FString& emailAddress) -{ - INFO_MSG("KBEngineApp::bindAccountEmail(): send bindAccountEmail! username=%s", *username_); - Bundle* pBundle = Bundle::createObject(); - pBundle->newMessage(Messages::messages[TEXT("Baseapp_reqAccountBindEmail"])); - (*pBundle) << entity_id_; - (*pBundle) << password_; - (*pBundle) << emailAddress; - pBundle->send(pNetworkInterface_); -} - -void KBEngineApp::Client_onReqAccountBindEmailCB(uint16 failcode) -{ - if (failcode != 0) - { - ERROR_MSG("KBEngineApp::Client_onReqAccountBindEmailCB(): bind failed! code=%d, error=%s! username=%s", failcode, *serverErr(failcode), *username_); - return; - } - - DEBUG_MSG("KBEngineApp::Client_onReqAccountBindEmailCB(): successfully! username=%s", *username_); -} - -void KBEngineApp::newPassword(const FString& old_password, const FString& new_password) -{ - INFO_MSG("KBEngineApp::newPassword(): send newPassword! username=%s", *username_); - Bundle* pBundle = Bundle::createObject(); - pBundle->newMessage(Messages::messages[TEXT("Baseapp_reqAccountNewPassword"])); - (*pBundle) << entity_id_; - (*pBundle) << password_; - (*pBundle) << old_password; - (*pBundle) << new_password; - pBundle->send(pNetworkInterface_); -} - -void KBEngineApp::Client_onReqAccountNewPasswordCB(uint16 failcode) -{ - if (failcode != 0) - { - ERROR_MSG("KBEngineApp::Client_onReqAccountNewPasswordCB(): newPassword failed! code=%d, error=%s! username=%s", failcode, *serverErr(failcode), *username_); - return; - } - - DEBUG_MSG("KBEngineApp::Client_onReqAccountNewPasswordCB(): successfully! username=%s", *username_); -} - -void KBEngineApp::Client_onRemoteMethodCallOptimized(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - onRemoteMethodCall_(eid, stream); -} - -void KBEngineApp::Client_onRemoteMethodCall(MemoryStream& stream) -{ - ENTITY_ID eid = 0; - stream >> eid; - onRemoteMethodCall_(eid, stream); -} - -void KBEngineApp::onRemoteMethodCall_(ENTITY_ID eid, MemoryStream& stream) -{ - Entity** pEntityFind = entities_.Find(eid); - - if (!pEntityFind) - { - ERROR_MSG("KBEngineApp::onRemoteMethodCall_(): entity(%d) not found!", eid); - return; - } - - Entity* pEntity = *pEntityFind; - pEntity->onRemoteMethodCall(stream); -} - -void KBEngineApp::Client_onControlEntity(ENTITY_ID eid, int8 isControlled) -{ - Entity** pEntityFind = entities_.Find(eid); - - if (!pEntityFind) - { - ERROR_MSG("KBEngineApp::Client_onControlEntity(): entity(%d) not found!", eid); - return; - } - - bool isCont = isControlled != 0; - - if (isCont) - { - // 如果被控制者是玩家自己，那表示玩家自己被其它人控制了 - // 所以玩家自己不应该进入这个被控制列表 - if (entity_id_ != (*pEntityFind)->id()) - { - controlledEntities_.Add((*pEntityFind)); - } - } - else - { - controlledEntities_.Remove((*pEntityFind)); - } - - (*pEntityFind)->isControlled(isCont); - (*pEntityFind)->onControlled(isCont); - - UKBEventData_onControlled* pEventData = NewObject(); - pEventData->entityID = (*pEntityFind)->id(); - pEventData->isControlled = isCont; - KBENGINE_EVENT_FIRE(KBEventTypes::onControlled, pEventData); -} - -void KBEngineApp::Client_onStreamDataStarted(int16 id, uint32 datasize, FString descr) -{ - UKBEventData_onStreamDataStarted* pEventData = NewObject(); - pEventData->resID = id; - pEventData->dataSize = datasize; - pEventData->dataDescr = descr; - KBENGINE_EVENT_FIRE(KBEventTypes::onStreamDataStarted, pEventData); -} - -void KBEngineApp::Client_onStreamDataRecv(MemoryStream& stream) -{ - UKBEventData_onStreamDataRecv* pEventData = NewObject(); - - uint16 id = stream.read(); - pEventData->resID = id; - stream.readBlob(pEventData->data); - - KBENGINE_EVENT_FIRE(KBEventTypes::onStreamDataRecv, pEventData); -} - -void KBEngineApp::Client_onStreamDataCompleted(int16 id) -{ - UKBEventData_onStreamDataCompleted* pEventData = NewObject(); - pEventData->resID = id; - KBENGINE_EVENT_FIRE(KBEventTypes::onStreamDataCompleted, pEventData); -} - -void KBEngineApp::Client_onEntityEnterWorld(MemoryStream& stream) -{ - ENTITY_ID eid; - stream >> eid; - - if (entity_id_ > 0 && entity_id_ != eid) - entityIDAliasIDList_.Add(eid); - - uint16 uEntityType; - - if (EntityDef::idmoduledefs.Num() > 255) - uEntityType = stream.read(); - else - uEntityType = stream.read(); - - int8 isOnGround = 1; - - if (stream.length() > 0) - isOnGround = stream.read(); - - ScriptModule** pScriptModuleFind = EntityDef::idmoduledefs.Find(uEntityType); - if (!pScriptModuleFind) - { - SCREEN_ERROR_MSG("KBEngineApp::Client_onEntityEnterWorld(): not found ScriptModule(utype = %d)!", uEntityType); - return; - } - - ScriptModule* pScriptModule = *pScriptModuleFind; - FString entityType = pScriptModule->name; - // DEBUG_MSG("KBEngineApp::Client_onEntityEnterWorld(): %s(%d), spaceID(%d)!", *entityType, eid, spaceID_); - - Entity** pEntityFind = entities_.Find(eid); - - if (!pEntityFind) - { - MemoryStream** entityMessageFind = bufferedCreateEntityMessages_.Find(eid); - if (!entityMessageFind) - { - ERROR_MSG("KBEngineApp::Client_onEntityEnterWorld(): entity(%d) not found!", eid); - return; - } - - pScriptModuleFind = EntityDef::moduledefs.Find(entityType); - if (!pScriptModuleFind) - { - SCREEN_ERROR_MSG("KBEngineApp::Client_onEntityEnterWorld(): not found ScriptModule(%s)!", *entityType); - return; - } - - ScriptModule* pModule = *pScriptModuleFind; - - Entity* pEntity = pModule->createEntity(); - pEntity->id(eid); - pEntity->className(entityType); - pEntity->onGetCell(); - - entities_.Add(eid, pEntity); - - Client_onUpdatePropertys(*(*entityMessageFind)); - MemoryStream::reclaimObject((*entityMessageFind)); - bufferedCreateEntityMessages_.Remove(eid); - - pEntity->isOnGround(isOnGround > 0); - pEntity->onDirectionChanged(pEntity->direction); - pEntity->onPositionChanged(pEntity->position); - - pEntity->__init__(); - pEntity->attachComponents(); - pEntity->inited(true); - pEntity->inWorld(true); - pEntity->enterWorld(); - - if (pArgs_->isOnInitCallPropertysSetMethods) - pEntity->callPropertysSetMethods(); - } - else - { - Entity* pEntity = (*pEntityFind); - - if (!pEntity->inWorld()) - { - // 安全起见， 这里清空一下 - // 如果服务端上使用giveClientTo切换控制权 - // 之前的实体已经进入世界， 切换后的实体也进入世界， 这里可能会残留之前那个实体进入世界的信息 - entityIDAliasIDList_.Empty(); - clearEntities(false); - entities_.Add(pEntity->id(), pEntity); - - pEntity->onGetCell(); - - pEntity->onDirectionChanged(pEntity->direction); - pEntity->onPositionChanged(pEntity->position); - - entityServerPos_ = pEntity->position; - pEntity->isOnGround(isOnGround > 0); - pEntity->inWorld(true); - pEntity->enterWorld(); - - if (pArgs_->isOnInitCallPropertysSetMethods) - pEntity->callPropertysSetMethods(); - } - } -} - -void KBEngineApp::Client_onEntityLeaveWorldOptimized(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - Client_onEntityLeaveWorld(eid); -} - -void KBEngineApp::Client_onEntityLeaveWorld(ENTITY_ID eid) -{ - Entity** pEntityFind = entities_.Find(eid); - - if (!pEntityFind) - { - ERROR_MSG("KBEngineApp::Client_onEntityLeaveWorld(): entity(%d) not found!", eid); - return; - } - - Entity* pEntity = *pEntityFind; - - if (pEntity->inWorld()) - pEntity->leaveWorld(); - - if (entity_id_ == eid) - { - clearSpace(false); - pEntity->onLoseCell(); - } - else - { - if (controlledEntities_.Contains(pEntity)) - { - controlledEntities_.Remove(pEntity); - - UKBEventData_onLoseControlledEntity* pEventData = NewObject(); - pEventData->entityID = pEntity->id(); - KBENGINE_EVENT_FIRE(KBEventTypes::onLoseControlledEntity, pEventData); - } - - entities_.Remove(eid); - pEntity->destroy(); - entityIDAliasIDList_.Remove(eid); - } -} - -void KBEngineApp::Client_onEntityEnterSpace(MemoryStream& stream) -{ - ENTITY_ID eid = stream.read(); - spaceID_ = stream.read(); - - int8 isOnGround = 1; - - if (stream.length() > 0) - isOnGround = stream.read(); - - Entity** pEntityFind = entities_.Find(eid); - - if (!pEntityFind) - { - ERROR_MSG("KBEngineApp::Client_onEntityEnterSpace(): entity(%d) not found!", eid); - return; - } - - Entity* pEntity = *pEntityFind; - pEntity->isOnGround(isOnGround > 0); - entityServerPos_ = pEntity->position; - pEntity->enterSpace(); -} - -void KBEngineApp::Client_onEntityLeaveSpace(ENTITY_ID eid) -{ - Entity** pEntityFind = entities_.Find(eid); - - if (!pEntityFind) - { - ERROR_MSG("KBEngineApp::Client_onEntityLeaveSpace(): entity(%d) not found!", eid); - return; - } - - Entity* pEntity = *pEntityFind; - pEntity->leaveSpace(); - - clearSpace(false); -} - -void KBEngineApp::Client_onUpdateBasePos(float x, float y, float z) -{ - entityServerPos_.X = x; - entityServerPos_.Y = y; - entityServerPos_.Z = z; - - Entity* pEntity = player(); - if (pEntity && pEntity->isControlled()) - { - pEntity->position.Set(entityServerPos_.X, entityServerPos_.Y, entityServerPos_.Z); - - UKBEventData_updatePosition* pEventData = NewObject(); - KBPos2UE4Pos(pEventData->position, entityServerPos_); - KBDir2UE4Dir(pEventData->direction, pEntity->direction); - pEventData->entityID = pEntity->id(); - pEventData->moveSpeed = pEntity->velocity(); - KBENGINE_EVENT_FIRE(KBEventTypes::updatePosition, pEventData); - - pEntity->onUpdateVolatileData(); - } -} - -void KBEngineApp::Client_onUpdateBasePosXZ(float x, float z) -{ - entityServerPos_.X = x; - entityServerPos_.Z = z; - - Entity* pEntity = player(); - if (pEntity && pEntity->isControlled()) - { - pEntity->position.X = entityServerPos_.X; - pEntity->position.Z = entityServerPos_.Z; - - UKBEventData_updatePosition* pEventData = NewObject(); - KBPos2UE4Pos(pEventData->position, entityServerPos_); - KBDir2UE4Dir(pEventData->direction, pEntity->direction); - pEventData->entityID = pEntity->id(); - pEventData->moveSpeed = pEntity->velocity(); - KBENGINE_EVENT_FIRE(KBEventTypes::updatePosition, pEventData); - - pEntity->onUpdateVolatileData(); - } -} - -void KBEngineApp::Client_onUpdateBaseDir(MemoryStream& stream) -{ - float yaw, pitch, roll; - stream >> yaw >> pitch >> roll; - - Entity* pEntity = player(); - if (pEntity && pEntity->isControlled()) - { - pEntity->direction.Set(roll, pitch, yaw); - - UKBEventData_set_direction* pEventData = NewObject(); - KBDir2UE4Dir(pEventData->direction, pEntity->direction); - pEventData->entityID = pEntity->id(); - KBENGINE_EVENT_FIRE(KBEventTypes::set_direction, pEventData); - - pEntity->onUpdateVolatileData(); - } -} - -void KBEngineApp::Client_onUpdateData(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - Entity** pEntityFind = entities_.Find(eid); - - if (!pEntityFind) - { - ERROR_MSG("KBEngineApp::Client_onUpdateData(): entity(%d) not found!", eid); - return; - } -} - -void KBEngineApp::Client_onSetEntityPosAndDir(MemoryStream& stream) -{ - ENTITY_ID eid; - stream >> eid; - - Entity** pEntityFind = entities_.Find(eid); - - if (!pEntityFind) - { - ERROR_MSG("KBEngineApp::Client_onSetEntityPosAndDir(): entity(%d) not found!", eid); - return; - } - - Entity& entity = *(*pEntityFind); - - FVector old_position = entity.position; - FVector old_direction = entity.direction; - - entity.position.X = stream.read(); - entity.position.Y = stream.read(); - entity.position.Z = stream.read(); - - entity.direction.X = stream.read(); - entity.direction.Y = stream.read(); - entity.direction.Z = stream.read(); - - entity.entityLastLocalPos = entity.position; - entity.entityLastLocalDir = entity.direction; - - entity.onDirectionChanged(old_direction); - entity.onPositionChanged(old_position); -} - -void KBEngineApp::Client_onUpdateData_ypr(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - float y = stream.read(); - float p = stream.read(); - float r = stream.read(); - - _updateVolatileData(eid, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, y, p, r, -1, false); -} - -void KBEngineApp::Client_onUpdateData_yp(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - float y = stream.read(); - float p = stream.read(); - - _updateVolatileData(eid, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, y, p, KBE_FLT_MAX, -1, false); -} - -void KBEngineApp::Client_onUpdateData_yr(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - float y = stream.read(); - float r = stream.read(); - - _updateVolatileData(eid, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, y, KBE_FLT_MAX, r, -1, false); -} - -void KBEngineApp::Client_onUpdateData_pr(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - float p = stream.read(); - float r = stream.read(); - - _updateVolatileData(eid, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, p, r, -1, false); -} - -void KBEngineApp::Client_onUpdateData_y(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - float y = stream.read(); - - _updateVolatileData(eid, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, y, KBE_FLT_MAX, KBE_FLT_MAX, -1, false); -} - -void KBEngineApp::Client_onUpdateData_p(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - float p = stream.read(); - - _updateVolatileData(eid, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, p, KBE_FLT_MAX, -1, false); -} - -void KBEngineApp::Client_onUpdateData_r(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - float r = stream.read(); - - _updateVolatileData(eid, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, r, -1, false); -} - -void KBEngineApp::Client_onUpdateData_xz(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - float x = stream.read(); - float z = stream.read(); - - _updateVolatileData(eid, x, KBE_FLT_MAX, z, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, 1, false); -} - -void KBEngineApp::Client_onUpdateData_xz_ypr(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - float x = stream.read(); - float z = stream.read(); - - float y = stream.read(); - float p = stream.read(); - float r = stream.read(); - - _updateVolatileData(eid, x, KBE_FLT_MAX, z, y, p, r, 1, false); -} - -void KBEngineApp::Client_onUpdateData_xz_yp(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - float x = stream.read(); - float z = stream.read(); - - float y = stream.read(); - float p = stream.read(); - - _updateVolatileData(eid, x, KBE_FLT_MAX, z, y, p, KBE_FLT_MAX, 1, false); -} - -void KBEngineApp::Client_onUpdateData_xz_yr(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - float x = stream.read(); - float z = stream.read(); - - float y = stream.read(); - float r = stream.read(); - - _updateVolatileData(eid, x, KBE_FLT_MAX, z, y, KBE_FLT_MAX, r, 1, false); -} - -void KBEngineApp::Client_onUpdateData_xz_pr(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - float x = stream.read(); - float z = stream.read(); - - float p = stream.read(); - float r = stream.read(); - - _updateVolatileData(eid, x, KBE_FLT_MAX, z, KBE_FLT_MAX, p, r, 1, false); -} - -void KBEngineApp::Client_onUpdateData_xz_y(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - float x = stream.read(); - float z = stream.read(); - - float y = stream.read(); - - _updateVolatileData(eid, x, KBE_FLT_MAX, z, y, KBE_FLT_MAX, KBE_FLT_MAX, 1, false); -} - -void KBEngineApp::Client_onUpdateData_xz_p(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - float x = stream.read(); - float z = stream.read(); - - float p = stream.read(); - - _updateVolatileData(eid, x, KBE_FLT_MAX, z, KBE_FLT_MAX, p, KBE_FLT_MAX, 1, false); -} - -void KBEngineApp::Client_onUpdateData_xz_r(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - float x = stream.read(); - float z = stream.read(); - - float r = stream.read(); - - _updateVolatileData(eid, x, KBE_FLT_MAX, z, KBE_FLT_MAX, KBE_FLT_MAX, r, 1, false); -} - -void KBEngineApp::Client_onUpdateData_xyz(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - float x = stream.read(); - float y = stream.read(); - float z = stream.read(); - - _updateVolatileData(eid, x, y, z, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, 0, false); -} - -void KBEngineApp::Client_onUpdateData_xyz_ypr(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - float x = stream.read(); - float y = stream.read(); - float z = stream.read(); - - float yaw = stream.read(); - float p = stream.read(); - float r = stream.read(); - - _updateVolatileData(eid, x, y, z, yaw, p, r, 0, false); -} - -void KBEngineApp::Client_onUpdateData_xyz_yp(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - float x = stream.read(); - float y = stream.read(); - float z = stream.read(); - - float yaw = stream.read(); - float p = stream.read(); - - _updateVolatileData(eid, x, y, z, yaw, p, KBE_FLT_MAX, 0, false); -} - -void KBEngineApp::Client_onUpdateData_xyz_yr(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - float x = stream.read(); - float y = stream.read(); - float z = stream.read(); - - float yaw = stream.read(); - float r = stream.read(); - - _updateVolatileData(eid, x, y, z, yaw, KBE_FLT_MAX, r, 0, false); -} - -void KBEngineApp::Client_onUpdateData_xyz_pr(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - float x = stream.read(); - float y = stream.read(); - float z = stream.read(); - - float p = stream.read(); - float r = stream.read(); - - _updateVolatileData(eid, x, y, z, KBE_FLT_MAX, p, r, 0, false); -} - -void KBEngineApp::Client_onUpdateData_xyz_y(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - float x = stream.read(); - float y = stream.read(); - float z = stream.read(); - - float yaw = stream.read(); - - _updateVolatileData(eid, x, y, z, yaw, KBE_FLT_MAX, KBE_FLT_MAX, 0, false); -} - -void KBEngineApp::Client_onUpdateData_xyz_p(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - float x = stream.read(); - float y = stream.read(); - float z = stream.read(); - - float p = stream.read(); - - _updateVolatileData(eid, x, y, z, KBE_FLT_MAX, p, KBE_FLT_MAX, 0, false); -} - -void KBEngineApp::Client_onUpdateData_xyz_r(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - float x = stream.read(); - float y = stream.read(); - float z = stream.read(); - - float r = stream.read(); - - _updateVolatileData(eid, x, y, z, KBE_FLT_MAX, KBE_FLT_MAX, r, 0, false); -} - -void KBEngineApp::Client_onUpdateData_ypr_optimized(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - int8 y = stream.read(); - int8 p = stream.read(); - int8 r = stream.read(); - - _updateVolatileData(eid, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, y, p, r, -1, true); -} - -void KBEngineApp::Client_onUpdateData_yp_optimized(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - int8 y = stream.read(); - int8 p = stream.read(); - - _updateVolatileData(eid, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, y, p, KBE_FLT_MAX, -1, true); -} - -void KBEngineApp::Client_onUpdateData_yr_optimized(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - int8 y = stream.read(); - int8 r = stream.read(); - - _updateVolatileData(eid, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, y, KBE_FLT_MAX, r, -1, true); -} - -void KBEngineApp::Client_onUpdateData_pr_optimized(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - int8 p = stream.read(); - int8 r = stream.read(); - - _updateVolatileData(eid, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, p, r, -1, true); -} - -void KBEngineApp::Client_onUpdateData_y_optimized(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - int8 y = stream.read(); - - _updateVolatileData(eid, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, y, KBE_FLT_MAX, KBE_FLT_MAX, -1, true); -} - -void KBEngineApp::Client_onUpdateData_p_optimized(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - int8 p = stream.read(); - - _updateVolatileData(eid, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, p, KBE_FLT_MAX, -1, true); -} - -void KBEngineApp::Client_onUpdateData_r_optimized(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - int8 r = stream.read(); - - _updateVolatileData(eid, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, r, -1, true); -} - -void KBEngineApp::Client_onUpdateData_xz_optimized(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - FVector xz; - stream.readPackXZ(xz.X, xz.Z); - - _updateVolatileData(eid, xz.X, KBE_FLT_MAX, xz.Z, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, 1, true); -} - -void KBEngineApp::Client_onUpdateData_xz_ypr_optimized(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - FVector xz; - stream.readPackXZ(xz.X, xz.Z); - - int8 y = stream.read(); - int8 p = stream.read(); - int8 r = stream.read(); - - _updateVolatileData(eid, xz.X, KBE_FLT_MAX, xz.Z, y, p, r, 1, true); -} - -void KBEngineApp::Client_onUpdateData_xz_yp_optimized(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - FVector xz; - stream.readPackXZ(xz.X, xz.Z); - - int8 y = stream.read(); - int8 p = stream.read(); - - _updateVolatileData(eid, xz.X, KBE_FLT_MAX, xz.Z, y, p, KBE_FLT_MAX, 1, true); -} - -void KBEngineApp::Client_onUpdateData_xz_yr_optimized(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - FVector xz; - stream.readPackXZ(xz.X, xz.Z); - - int8 y = stream.read(); - int8 r = stream.read(); - - _updateVolatileData(eid, xz.X, KBE_FLT_MAX, xz.Z, y, KBE_FLT_MAX, r, 1, true); -} - -void KBEngineApp::Client_onUpdateData_xz_pr_optimized(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - FVector xz; - stream.readPackXZ(xz.X, xz.Z); - - int8 p = stream.read(); - int8 r = stream.read(); - - _updateVolatileData(eid, xz.X, KBE_FLT_MAX, xz.Z, KBE_FLT_MAX, p, r, 1, true); -} - -void KBEngineApp::Client_onUpdateData_xz_y_optimized(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - FVector xz; - stream.readPackXZ(xz.X, xz.Z); - - int8 y = stream.read(); - - _updateVolatileData(eid, xz.X, KBE_FLT_MAX, xz.Z, y, KBE_FLT_MAX, KBE_FLT_MAX, 1, true); -} - -void KBEngineApp::Client_onUpdateData_xz_p_optimized(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - FVector xz; - stream.readPackXZ(xz.X, xz.Z); - - int8 p = stream.read(); - - _updateVolatileData(eid, xz.X, KBE_FLT_MAX, xz.Z, KBE_FLT_MAX, p, KBE_FLT_MAX, 1, true); -} - -void KBEngineApp::Client_onUpdateData_xz_r_optimized(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - FVector xz; - stream.readPackXZ(xz.X, xz.Z); - - int8 r = stream.read(); - - _updateVolatileData(eid, xz.X, KBE_FLT_MAX, xz.Z, KBE_FLT_MAX, KBE_FLT_MAX, r, 1, true); -} - -void KBEngineApp::Client_onUpdateData_xyz_optimized(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - FVector xz; - stream.readPackXZ(xz.X, xz.Z); - stream.readPackY(xz.Y); - - _updateVolatileData(eid, xz.X, xz.Y, xz.Z, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, 0, true); -} - -void KBEngineApp::Client_onUpdateData_xyz_ypr_optimized(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - FVector xz; - stream.readPackXZ(xz.X, xz.Z); - stream.readPackY(xz.Y); - - int8 y = stream.read(); - int8 p = stream.read(); - int8 r = stream.read(); - - _updateVolatileData(eid, xz.X, xz.Y, xz.Z, y, p, r, 0, true); -} - -void KBEngineApp::Client_onUpdateData_xyz_yp_optimized(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - FVector xz; - stream.readPackXZ(xz.X, xz.Z); - stream.readPackY(xz.Y); - - int8 y = stream.read(); - int8 p = stream.read(); - - _updateVolatileData(eid, xz.X, xz.Y, xz.Z, y, p, KBE_FLT_MAX, 0, true); -} - -void KBEngineApp::Client_onUpdateData_xyz_yr_optimized(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - FVector xz; - stream.readPackXZ(xz.X, xz.Z); - stream.readPackY(xz.Y); - - int8 y = stream.read(); - int8 r = stream.read(); - - _updateVolatileData(eid, xz.X, xz.Y, xz.Z, y, KBE_FLT_MAX, r, 0, true); -} - -void KBEngineApp::Client_onUpdateData_xyz_pr_optimized(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - FVector xz; - stream.readPackXZ(xz.X, xz.Z); - stream.readPackY(xz.Y); - - int8 p = stream.read(); - int8 r = stream.read(); - - _updateVolatileData(eid, xz.X, xz.Y, xz.Z, KBE_FLT_MAX, p, r, 0, true); -} - -void KBEngineApp::Client_onUpdateData_xyz_y_optimized(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - FVector xz; - stream.readPackXZ(xz.X, xz.Z); - stream.readPackY(xz.Y); - - int8 y = stream.read(); - - _updateVolatileData(eid, xz.X, xz.Y, xz.Z, y, KBE_FLT_MAX, KBE_FLT_MAX, 0, true); -} - -void KBEngineApp::Client_onUpdateData_xyz_p_optimized(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - FVector xz; - stream.readPackXZ(xz.X, xz.Z); - stream.readPackY(xz.Y); - - int8 p = stream.read(); - - _updateVolatileData(eid, xz.X, xz.Y, xz.Z, KBE_FLT_MAX, p, KBE_FLT_MAX, 0, true); -} - -void KBEngineApp::Client_onUpdateData_xyz_r_optimized(MemoryStream& stream) -{ - ENTITY_ID eid = getViewEntityIDFromStream(stream); - - FVector xz; - stream.readPackXZ(xz.X, xz.Z); - stream.readPackY(xz.Y); - - int8 r = stream.read(); - - _updateVolatileData(eid, xz.X, xz.Y, xz.Z, KBE_FLT_MAX, KBE_FLT_MAX, r, 0, true); -} - -void KBEngineApp::_updateVolatileData(ENTITY_ID entityID, float x, float y, float z, float yaw, float pitch, float roll, int8 isOnGround, bool isOptimized) -{ - Entity** pEntityFind = entities_.Find(entityID); - - if (!pEntityFind) - { - // 如果为0且客户端上一步是重登陆或者重连操作并且服务端entity在断线期间一直处于在线状态 - // 则可以忽略这个错误, 因为cellapp可能一直在向baseapp发送同步消息， 当客户端重连上时未等 - // 服务端初始化步骤开始则收到同步信息, 此时这里就会出错。 - ERROR_MSG("KBEngineApp::_updateVolatileData(): entity(%d) not found!", entityID); - return; - } - - Entity& entity = *(*pEntityFind); - - // 小于0不设置 - if (isOnGround >= 0) - { - entity.isOnGround(isOnGround > 0); - } - - bool changeDirection = false; - - if (roll != KBE_FLT_MAX) - { - changeDirection = true; - entity.direction.X = int82angle((int8)roll, false); - } - - if (pitch != KBE_FLT_MAX) - { - changeDirection = true; - entity.direction.Y = int82angle((int8)pitch, false); - } - - if (yaw != KBE_FLT_MAX) - { - changeDirection = true; - entity.direction.Z = int82angle((int8)yaw, false); - } - - bool done = false; - if (changeDirection == true) - { - UKBEventData_set_direction* pEventData = NewObject(); - KBDir2UE4Dir(pEventData->direction, entity.direction); - pEventData->entityID = entity.id(); - KBENGINE_EVENT_FIRE(KBEventTypes::set_direction, pEventData); - - done = true; - } - - bool positionChanged = x != KBE_FLT_MAX || y != KBE_FLT_MAX || z != KBE_FLT_MAX; - if (x == KBE_FLT_MAX) x = 0.0f; - if (y == KBE_FLT_MAX) y = 0.0f; - if (z == KBE_FLT_MAX) z = 0.0f; - - if (positionChanged) - { - entity.position = isOptimized ? FVector(x + entityServerPos_.X, y + entityServerPos_.Y, z + entityServerPos_.Z) : FVector(x, y, z); - done = true; - - UKBEventData_updatePosition* pEventData = NewObject(); - KBPos2UE4Pos(pEventData->position, entity.position); - KBDir2UE4Dir(pEventData->direction, entity.direction); - pEventData->entityID = entity.id(); - pEventData->moveSpeed = entity.velocity(); - pEventData->isOnGround = entity.isOnGround(); - KBENGINE_EVENT_FIRE(KBEventTypes::updatePosition, pEventData); - } - - if (done) - entity.onUpdateVolatileData(); + +#include "KBEngine.h" +#include "KBEngineArgs.h" +#include "Entity.h" +#include "EntityDef.h" +#include "Messages.h" +#include "NetworkInterfaceTcp.h" +#include "NetworkInterfaceKcp.h" +#include "Bundle.h" +#include "MemoryStream.h" +#include "DataTypes.h" +#include "ScriptModule.h" +#include "Property.h" +#include "Method.h" +#include "EntityCall.h" +#include "Regex.h" +#include "KBDebug.h" +#include "KBEvent.h" +#include "EncryptionFilter.h" + +namespace KBEngine +{ + +ServerErrorDescrs KBEngineApp::serverErrs_; + +KBEngineApp::KBEngineApp() : + pArgs_(NULL), + pNetworkInterface_(NULL), + username_(TEXT("")), + password_(TEXT("")), + baseappIP_(TEXT("")), + baseappTcpPort_(0), + baseappUdpPort_(0), + currserver_(TEXT("")), + currstate_(TEXT("")), + serverdatas_(), + clientdatas_(), + encryptedKey_(), + serverVersion_(TEXT("")), + clientVersion_(TEXT("")), + serverScriptVersion_(TEXT("")), + clientScriptVersion_(TEXT("")), + serverProtocolMD5_(TEXT("@{KBE_SERVER_PROTO_MD5}")), + serverEntitydefMD5_(TEXT("@{KBE_SERVER_ENTITYDEF_MD5}")), + entity_uuid_(0), + entity_id_(0), + entity_type_(TEXT("")), + useAliasEntityID_(@{KBE_USE_ALIAS_ENTITYID}), + controlledEntities_(), + entityServerPos_(), + spacedatas_(), + entities_(), + entityIDAliasIDList_(), + bufferedCreateEntityMessages_(), + lastTickTime_(0.0), + lastTickCBTime_(0.0), + lastUpdateToServerTime_(0.0), + spaceID_(0), + spaceResPath_(TEXT("")), + isLoadedGeometry_(false), + component_(TEXT("client")), + pFilter_(NULL), + pUKBETicker_(nullptr) +{ + INFO_MSG("KBEngineApp::KBEngineApp(): hello!"); + installUKBETicker(); +} + +KBEngineApp::KBEngineApp(KBEngineArgs* pArgs): + pArgs_(NULL), + pNetworkInterface_(NULL), + username_(TEXT("")), + password_(TEXT("")), + baseappIP_(TEXT("")), + baseappTcpPort_(0), + baseappUdpPort_(0), + currserver_(TEXT("")), + currstate_(TEXT("")), + serverdatas_(), + clientdatas_(), + encryptedKey_(), + serverVersion_(TEXT("")), + clientVersion_(TEXT("")), + serverScriptVersion_(TEXT("")), + clientScriptVersion_(TEXT("")), + serverProtocolMD5_(TEXT("@{KBE_SERVER_PROTO_MD5}")), + serverEntitydefMD5_(TEXT("@{KBE_SERVER_ENTITYDEF_MD5}")), + entity_uuid_(0), + entity_id_(0), + entity_type_(TEXT("")), + useAliasEntityID_(@{KBE_USE_ALIAS_ENTITYID}), + controlledEntities_(), + entityServerPos_(), + spacedatas_(), + entities_(), + entityIDAliasIDList_(), + bufferedCreateEntityMessages_(), + lastTickTime_(0.0), + lastTickCBTime_(0.0), + lastUpdateToServerTime_(0.0), + spaceID_(0), + spaceResPath_(TEXT("")), + isLoadedGeometry_(false), + component_(TEXT("client")), + pFilter_(NULL), + pUKBETicker_(nullptr) +{ + INFO_MSG("KBEngineApp::KBEngineApp(): hello!"); + initialize(pArgs); + installUKBETicker(); +} + +KBEngineApp::~KBEngineApp() +{ + destroy(); + INFO_MSG("KBEngineApp::~KBEngineApp(): destructed!"); +} + +KBEngineApp* pKBEngineApp = nullptr; + +KBEngineApp& KBEngineApp::getSingleton() +{ + if(!pKBEngineApp) + pKBEngineApp = new KBEngineApp(); + + return *pKBEngineApp; +} + +void KBEngineApp::destroyKBEngineApp() +{ + if(pKBEngineApp) + { + delete pKBEngineApp; + pKBEngineApp = nullptr; + KBEvent::clear(); + } +} + +bool KBEngineApp::initialize(KBEngineArgs* pArgs) +{ + if (isInitialized()) + return false; + + EntityDef::initialize(); + + // 注册事件 + installEvents(); + + pArgs_ = pArgs; + reset(); + return true; +} + +void KBEngineApp::installEvents() +{ + KBENGINE_REGISTER_EVENT_OVERRIDE_FUNC(KBEventTypes::login, KBEventTypes::login, [this](const UKBEventData* pEventData) + { + const UKBEventData_login& data = static_cast(*pEventData); + login(data.username, data.password, data.datas); + }); + + KBENGINE_REGISTER_EVENT_OVERRIDE_FUNC(KBEventTypes::logout, KBEventTypes::logout, [this](const UKBEventData* pEventData) + { + logout(); + }); + + KBENGINE_REGISTER_EVENT_OVERRIDE_FUNC(KBEventTypes::createAccount, KBEventTypes::createAccount, [this](const UKBEventData* pEventData) + { + const UKBEventData_createAccount& data = static_cast(*pEventData); + createAccount(data.username, data.password, data.datas); + }); + + KBENGINE_REGISTER_EVENT_OVERRIDE_FUNC(KBEventTypes::reloginBaseapp, KBEventTypes::reloginBaseapp, [this](const UKBEventData* pEventData) + { + reloginBaseapp(); + }); + + KBENGINE_REGISTER_EVENT_OVERRIDE_FUNC(KBEventTypes::resetPassword, KBEventTypes::resetPassword, [this](const UKBEventData* pEventData) + { + const UKBEventData_resetPassword& data = static_cast(*pEventData); + resetPassword(data.username); + }); + + KBENGINE_REGISTER_EVENT_OVERRIDE_FUNC(KBEventTypes::bindAccountEmail, KBEventTypes::bindAccountEmail, [this](const UKBEventData* pEventData) + { + const UKBEventData_bindAccountEmail& data = static_cast(*pEventData); + bindAccountEmail(data.email); + }); + + KBENGINE_REGISTER_EVENT_OVERRIDE_FUNC(KBEventTypes::newPassword, KBEventTypes::newPassword, [this](const UKBEventData* pEventData) + { + const UKBEventData_newPassword& data = static_cast(*pEventData); + newPassword(data.old_password, data.new_password); + }); + + // 内部事件 + KBENGINE_REGISTER_EVENT_OVERRIDE_FUNC("_closeNetwork", "_closeNetwork", [this](const UKBEventData* pEventData) + { + _closeNetwork(); + }); +} + +void KBEngineApp::destroy() +{ + reset(); + KBENGINE_DEREGISTER_ALL_EVENT(); + resetMessages(); + + KBE_SAFE_RELEASE(pArgs_); + KBE_SAFE_RELEASE(pNetworkInterface_); + KBE_SAFE_RELEASE(pFilter_); + uninstallUKBETicker(); +} + +void KBEngineApp::resetMessages() +{ + serverErrs_.Clear(); + + Messages::clear(); + EntityDef::clear(); + Entity::clear(); + + INFO_MSG("KBEngineApp::resetMessages(): done!"); +} + +void KBEngineApp::reset() +{ + KBEvent::clearFiredEvents(); + + clearEntities(true); + + currserver_ = TEXT(""); + currstate_ = TEXT(""); + + serverdatas_.Empty(); + + serverVersion_ = TEXT(""); + clientVersion_ = TEXT("@{KBE_VERSION}"); + serverScriptVersion_ = TEXT(""); + clientScriptVersion_ = TEXT("@{KBE_SCRIPT_VERSION}"); + + entity_uuid_ = 0; + entity_id_ = 0; + entity_type_ = TEXT(""); + + entityIDAliasIDList_.Empty(); + bufferedCreateEntityMessages_.Empty(); + + lastTickTime_ = getTimeSeconds(); + lastTickCBTime_ = getTimeSeconds(); + lastUpdateToServerTime_ = getTimeSeconds(); + + spacedatas_.Empty(); + + spaceID_ = 0; + spaceResPath_ = TEXT(""); + isLoadedGeometry_ = false; + + baseappUdpPort_ = 0; + + initNetwork(); +} + +void KBEngineApp::installUKBETicker() +{ + if (pUKBETicker_ == nullptr) + { + pUKBETicker_ = NewObject(); + pUKBETicker_->AddToRoot(); + } +} + +void KBEngineApp::uninstallUKBETicker() +{ + if (pUKBETicker_) + { + pUKBETicker_->RemoveFromRoot(); + pUKBETicker_->ConditionalBeginDestroy(); + pUKBETicker_ = nullptr; + } +} + +bool KBEngineApp::initNetwork() +{ + KBE_SAFE_RELEASE(pFilter_); + + if (pNetworkInterface_) + delete pNetworkInterface_; + + Messages::initialize(); + + if (pArgs_) + { + if (pArgs_->forceDisableUDP || baseappUdpPort_ == 0) + pNetworkInterface_ = new NetworkInterfaceTCP(); + else + pNetworkInterface_ = new NetworkInterfaceKCP(); + } + + return true; +} + +void KBEngineApp::_closeNetwork() +{ + if (pNetworkInterface_) + pNetworkInterface_->close(); +} + +bool KBEngineApp::validEmail(const FString& strEmail) +{ + const FRegexPattern spattern(FString(TEXT("[a-z0-9._%+-]+@[a-z0-9.-]+\\.[a-z]{2,4}"))); + + FRegexMatcher fMatcher(spattern, strEmail); + + if (fMatcher.FindNext()) { + return true; + } + + return false; +} + +void KBEngineApp::process() +{ + // 处理网络 + if (pNetworkInterface_) + pNetworkInterface_->process(); + + // 处理外层抛入的事件 + KBEvent::processInEvents(); + + // 向服务端发送心跳以及同步角色信息到服务端 + sendTick(); +} + +void KBEngineApp::sendTick() +{ + if (!pNetworkInterface_ || !pNetworkInterface_->valid()) + return; + + double span = getTimeSeconds() - lastTickTime_; + + // 更新玩家的位置与朝向到服务端 + updatePlayerToServer(); + + if (pArgs_->serverHeartbeatTick > 0 && span > pArgs_->serverHeartbeatTick) + { + span = lastTickCBTime_ - lastTickTime_; + + // 如果心跳回调接收时间小于心跳发送时间，说明没有收到回调 + // 此时应该通知客户端掉线了 + if (span < 0) + { + SCREEN_ERROR_MSG("KBEngineApp::sendTick(): Receive appTick timeout!"); + pNetworkInterface_->close(); + return; + } + + Message** Loginapp_onClientActiveTickMsgFind = Messages::messages.Find("Loginapp_onClientActiveTick"); + Message** Baseapp_onClientActiveTickMsgFind = Messages::messages.Find("Baseapp_onClientActiveTick"); + + if (currserver_ == TEXT("loginapp")) + { + if (Loginapp_onClientActiveTickMsgFind) + { + Bundle* pBundle = Bundle::createObject(); + pBundle->newMessage(*Loginapp_onClientActiveTickMsgFind); + pBundle->send(pNetworkInterface_); + } + } + else + { + if (Baseapp_onClientActiveTickMsgFind) + { + Bundle* pBundle = Bundle::createObject(); + pBundle->newMessage(*Baseapp_onClientActiveTickMsgFind); + pBundle->send(pNetworkInterface_); + } + } + + lastTickTime_ = getTimeSeconds(); + } +} + +Entity* KBEngineApp::player() +{ + return findEntity(entity_id_); +} + +Entity* KBEngineApp::findEntity(int32 entityID) +{ + Entity** pEntity = entities_.Find(entityID); + if (pEntity == nullptr) + return NULL; + + return *pEntity; +} + +FString KBEngineApp::serverErr(uint16 id) +{ + return serverErrs_.ServerErrStr(id); +} + +void KBEngineApp::updatePlayerToServer() +{ + if (pArgs_->syncPlayerMS <= 0 || spaceID_ == 0) + return; + + double tnow = getTimeSeconds(); + double span = tnow - lastUpdateToServerTime_; + + if (span < ((double)pArgs_->syncPlayerMS / 1000.0)) + return; + + Entity* pPlayerEntity = player(); + if (!pPlayerEntity || !pPlayerEntity->inWorld() || pPlayerEntity->isControlled()) + return; + + lastUpdateToServerTime_ = tnow; + const FVector& position = pPlayerEntity->position; + const FVector& direction = pPlayerEntity->direction; + + bool posHasChanged = (pPlayerEntity->entityLastLocalPos - position).Size() > 0.001f; + bool dirHasChanged = (pPlayerEntity->entityLastLocalDir - direction).Size() > 0.001f; + + if (posHasChanged || dirHasChanged) + { + pPlayerEntity->entityLastLocalPos = position; + pPlayerEntity->entityLastLocalDir = direction; + + Bundle* pBundle = Bundle::createObject(); + pBundle->newMessage(Messages::messages[TEXT("Baseapp_onUpdateDataFromClient"])); + (*pBundle) << position.X; + (*pBundle) << position.Y; + (*pBundle) << position.Z; + + (*pBundle) << direction.X; + (*pBundle) << direction.Y; + (*pBundle) << direction.Z; + + (*pBundle) << (uint8)pPlayerEntity->isOnGround(); + (*pBundle) << spaceID_; + + pBundle->send(pNetworkInterface_); + } + + // 开始同步所有被控制了的entity的位置 + for(auto& item : controlledEntities_) + { + Entity* pEntity = item; + const FVector& e_position = pEntity->position; + const FVector& e_direction = pEntity->direction; + + posHasChanged = (pEntity->entityLastLocalPos - e_position).Size() > 0.001f; + dirHasChanged = (pEntity->entityLastLocalDir - e_direction).Size() > 0.001f; + + if (posHasChanged || dirHasChanged) + { + pEntity->entityLastLocalPos = e_position; + pEntity->entityLastLocalDir = e_direction; + + Bundle* pBundle = Bundle::createObject(); + pBundle->newMessage(Messages::messages[TEXT("Baseapp_onUpdateDataFromClientForControlledEntity"])); + (*pBundle) << pEntity->id(); + (*pBundle) << e_position.X; + (*pBundle) << e_position.Y; + (*pBundle) << e_position.Z; + + (*pBundle) << e_direction.X; + (*pBundle) << e_direction.Y; + (*pBundle) << e_direction.Z; + + (*pBundle) << (uint8)pEntity->isOnGround(); + (*pBundle) << spaceID_; + + pBundle->send(pNetworkInterface_); + } + } + +} + +void KBEngineApp::Client_onAppActiveTickCB() +{ + lastTickCBTime_ = getTimeSeconds(); +} + +void KBEngineApp::hello() +{ + Bundle* pBundle = Bundle::createObject(); + if (currserver_ == TEXT("loginapp")) + pBundle->newMessage(Messages::messages[TEXT("Loginapp_hello")]); + else + pBundle->newMessage(Messages::messages[TEXT("Baseapp_hello")]); + + KBE_SAFE_RELEASE(pFilter_); + + if (pArgs_->networkEncryptType == NETWORK_ENCRYPT_TYPE::ENCRYPT_TYPE_BLOWFISH) + { + pFilter_ = new BlowfishFilter(); + encryptedKey_ = ((BlowfishFilter*)pFilter_)->key(); + pNetworkInterface_->setFilter(NULL); + } + + (*pBundle) << clientVersion_; + (*pBundle) << clientScriptVersion_; + pBundle->appendBlob(encryptedKey_); + pBundle->send(pNetworkInterface_); +} + +void KBEngineApp::Client_onHelloCB(MemoryStream& stream) +{ + FString str_serverVersion; + stream >> str_serverVersion; + stream >> serverScriptVersion_; + + FString serverProtocolMD5; + stream >> serverProtocolMD5; + + FString serverEntitydefMD5; + stream >> serverEntitydefMD5; + + int32 ctype; + stream >> ctype; + + INFO_MSG("KBEngineApp::Client_onHelloCB(): verInfo(%s), scriptVersion(%s), srvProtocolMD5(%s), srvEntitydefMD5(%s), ctype(%d)!", + *str_serverVersion, *serverScriptVersion_, *serverProtocolMD5_, *serverEntitydefMD5_, ctype); + + if(str_serverVersion != "Getting") + { + serverVersion_ = str_serverVersion; + + /* + if(serverProtocolMD5_ != serverProtocolMD5) + { + ERROR_MSG("KBEngineApp::Client_onHelloCB(): digest not match! serverProtocolMD5=%s(server: %s)", *serverProtocolMD5_, *serverProtocolMD5); + + UKBEventData_onVersionNotMatch* pEventData = NewObject(); + pEventData->clientVersion = clientVersion_; + pEventData->serverVersion = serverVersion_; + KBENGINE_EVENT_FIRE(KBEventTypes::onVersionNotMatch, pEventData); + return; + } + */ + + if(serverEntitydefMD5_ != serverEntitydefMD5) + { + ERROR_MSG("KBEngineApp::Client_onHelloCB(): digest not match! serverEntitydefMD5=%s(server: %s)", *serverEntitydefMD5_, *serverEntitydefMD5); + + UKBEventData_onVersionNotMatch* pEventData = NewObject(); + pEventData->clientVersion = clientVersion_; + pEventData->serverVersion = serverVersion_; + KBENGINE_EVENT_FIRE(KBEventTypes::onVersionNotMatch, pEventData); + return; + } + } + + if (pArgs_->networkEncryptType == NETWORK_ENCRYPT_TYPE::ENCRYPT_TYPE_BLOWFISH) + { + pNetworkInterface_->setFilter(pFilter_); + pFilter_ = NULL; + } + + onServerDigest(); + + if (currserver_ == TEXT("baseapp")) + { + onLogin_baseapp(); + } + else + { + onLogin_loginapp(); + } +} + +void KBEngineApp::Client_onVersionNotMatch(MemoryStream& stream) +{ + stream >> serverVersion_; + + ERROR_MSG("KBEngineApp::Client_onVersionNotMatch(): verInfo=%s(server: %s)", *clientVersion_, *serverVersion_); + + UKBEventData_onVersionNotMatch* pEventData = NewObject(); + pEventData->clientVersion = clientVersion_; + pEventData->serverVersion = serverVersion_; + KBENGINE_EVENT_FIRE(KBEventTypes::onVersionNotMatch, pEventData); +} + +void KBEngineApp::Client_onScriptVersionNotMatch(MemoryStream& stream) +{ + stream >> serverScriptVersion_; + + ERROR_MSG("KBEngineApp::Client_onScriptVersionNotMatch(): verInfo=%s(server: %s)", *clientScriptVersion_, *serverScriptVersion_); + + UKBEventData_onScriptVersionNotMatch* pEventData = NewObject(); + pEventData->clientScriptVersion = clientScriptVersion_; + pEventData->serverScriptVersion = serverScriptVersion_; + KBENGINE_EVENT_FIRE(KBEventTypes::onScriptVersionNotMatch, pEventData); +} + +void KBEngineApp::Client_onImportClientSDK(MemoryStream& stream) +{ + UKBEventData_onImportClientSDK* pEventData = NewObject(); + + pEventData->remainingFiles = stream.readInt32(); + pEventData->fileName = stream.readString(); + pEventData->fileSize = stream.readInt32(); + stream.readBlob(pEventData->fileDatas); + + KBENGINE_EVENT_FIRE("onImportClientSDK", pEventData); +} + +void KBEngineApp::Client_onKicked(uint16 failedcode) +{ + DEBUG_MSG("KBEngineApp::Client_onKicked(): failedcode=%d, %s", failedcode, *serverErr(failedcode)); + + UKBEventData_onKicked* pEventData = NewObject(); + pEventData->failedcode = failedcode; + pEventData->errorStr = serverErr(failedcode); + KBENGINE_EVENT_FIRE(KBEventTypes::onKicked, pEventData); +} + +void KBEngineApp::onServerDigest() +{ +} + +void KBEngineApp::onConnectCallback(FString ip, uint16 port, bool success, int userdata) +{ + if (userdata == 0) + { + onConnectTo_loginapp_login_callback(ip, port, success); + } + else if (userdata == 1) + { + onConnectTo_loginapp_create_callback(ip, port, success); + } + else if (userdata == 2) + { + onConnectTo_baseapp_callback(ip, port, success); + } + else if (userdata == 3) + { + onReloginTo_baseapp_callback(ip, port, success); + } + else if (userdata == 4) + { + onConnectTo_resetpassword_callback(ip, port, success); + } + else if (userdata == 5) + { + //onConnectTo_resetpassword_callback(ip, port, success); + } + else + { + check(false); + } +} + +bool KBEngineApp::login(const FString& username, const FString& password, const TArray& datas) +{ + if (username.Len() == 0) + { + ERROR_MSG("KBEngineApp::login(): username is empty!"); + return false; + } + + if (password.Len() == 0) + { + ERROR_MSG("KBEngineApp::login(): password is empty!"); + return false; + } + + username_ = username; + password_ = password; + clientdatas_ = datas; + + login_loginapp(true); + return true; +} + +bool KBEngineApp::logout() +{ + if (currserver_ != TEXT("baseapp")) + return false; + + INFO_MSG("KBEngineApp::logout()"); + Bundle* pBundle = Bundle::createObject(); + pBundle->newMessage(Messages::messages[TEXT("Baseapp_logoutBaseapp"])); + (*pBundle) << entity_uuid_; + (*pBundle) << entity_id_; + pBundle->send(pNetworkInterface_); + return true; +} + +void KBEngineApp::login_loginapp(bool noconnect) +{ + if (noconnect) + { + reset(); + pNetworkInterface_->connectTo(pArgs_->ip, pArgs_->port, this, 0); + } + else + { + INFO_MSG("KBEngineApp::login_loginapp(): send login! username=%s", *username_); + Bundle* pBundle = Bundle::createObject(); + pBundle->newMessage(Messages::messages[TEXT("Loginapp_login"])); + (*pBundle) << (uint8)pArgs_->clientType; + pBundle->appendBlob(clientdatas_); + (*pBundle) << username_; + (*pBundle) << password_; + pBundle->send(pNetworkInterface_); + } +} + +void KBEngineApp::onConnectTo_loginapp_login_callback(FString ip, uint16 port, bool success) +{ + if (!success) + { + ERROR_MSG("KBEngineApp::onConnectTo_loginapp_login_callback(): connect %s:%d is error!", *ip, port); + return; + } + + currserver_ = TEXT("loginapp"); + currstate_ = TEXT("login"); + + INFO_MSG("KBEngineApp::onConnectTo_loginapp_login_callback(): connect %s:%d is success!", *ip, port); + + hello(); +} + +void KBEngineApp::onLogin_loginapp() +{ + lastTickCBTime_ = getTimeSeconds(); + login_loginapp(false); +} + +void KBEngineApp::Client_onLoginFailed(MemoryStream& stream) +{ + uint16 failedcode = 0; + stream >> failedcode; + stream.readBlob(serverdatas_); + ERROR_MSG("KBEngineApp::Client_onLoginFailed(): failedcode(%d:%s), datas(%d)!", failedcode, *serverErr(failedcode), serverdatas_.Num()); + + UKBEventData_onLoginFailed* pEventData = NewObject(); + pEventData->failedcode = failedcode; + pEventData->errorStr = serverErr(failedcode); + KBENGINE_EVENT_FIRE(KBEventTypes::onLoginFailed, pEventData); +} + +void KBEngineApp::Client_onLoginSuccessfully(MemoryStream& stream) +{ + FString accountName; + stream >> accountName; + username_ = accountName; + stream >> baseappIP_; + stream >> baseappTcpPort_; + stream >> baseappUdpPort_; + stream.readBlob(serverdatas_); + + DEBUG_MSG("KBEngineApp::Client_onLoginSuccessfully(): accountName(%s), addr(" + "%s:%d:%d), datas(%d)!", *accountName, *baseappIP_, baseappTcpPort_, baseappUdpPort_, serverdatas_.Num()); + + login_baseapp(true); +} + +void KBEngineApp::login_baseapp(bool noconnect) +{ + if (noconnect) + { + KBENGINE_EVENT_FIRE(KBEventTypes::onLoginBaseapp, NewObject()); + + pNetworkInterface_->destroy(); + pNetworkInterface_ = NULL; + initNetwork(); + pNetworkInterface_->connectTo(baseappIP_, (!pArgs_->forceDisableUDP && baseappUdpPort_ > 0) ? baseappUdpPort_ : baseappTcpPort_, this, 2); + } + else + { + Bundle* pBundle = Bundle::createObject(); + pBundle->newMessage(Messages::messages[TEXT("Baseapp_loginBaseapp"])); + (*pBundle) << username_; + (*pBundle) << password_; + pBundle->send(pNetworkInterface_); + } +} + +void KBEngineApp::onConnectTo_baseapp_callback(FString ip, uint16 port, bool success) +{ + lastTickCBTime_ = getTimeSeconds(); + + if (!success) + { + ERROR_MSG("KBEngineApp::onConnectTo_baseapp_callback(): connect %s:%d is error!", *ip, port); + return; + } + + currserver_ = TEXT("baseapp"); + currstate_ = TEXT(""); + + DEBUG_MSG("KBEngineApp::onConnectTo_baseapp_callback(): connect %s:%d is successfully!", *ip, port); + + hello(); +} + +void KBEngineApp::onLogin_baseapp() +{ + lastTickCBTime_ = getTimeSeconds(); + login_baseapp(false); +} + +void KBEngineApp::reloginBaseapp() +{ + lastTickTime_ = getTimeSeconds(); + lastTickCBTime_ = getTimeSeconds(); + + if(pNetworkInterface_->valid()) + return; + + UKBEventData_onReloginBaseapp* pEventData = NewObject(); + KBENGINE_EVENT_FIRE(KBEventTypes::onReloginBaseapp, pEventData); + + pNetworkInterface_->connectTo(baseappIP_, (!pArgs_->forceDisableUDP && baseappUdpPort_ > 0) ? baseappUdpPort_ : baseappTcpPort_, this, 3); +} + +void KBEngineApp::onReloginTo_baseapp_callback(FString ip, uint16 port, bool success) +{ + if (!success) + { + ERROR_MSG("KBEngineApp::onReloginTo_baseapp_callback(): connect %s:%d is error!", *ip, port); + return; + } + + INFO_MSG("KBEngineApp::onReloginTo_baseapp_callback(): connect %s:%d is success!", *ip, port); + + Bundle* pBundle = Bundle::createObject(); + pBundle->newMessage(Messages::messages[TEXT("Baseapp_reloginBaseapp"])); + (*pBundle) << username_; + (*pBundle) << password_; + (*pBundle) << entity_uuid_; + (*pBundle) << entity_id_; + pBundle->send(pNetworkInterface_); + + lastTickCBTime_ = getTimeSeconds(); +} + +void KBEngineApp::Client_onLoginBaseappFailed(uint16 failedcode) +{ + ERROR_MSG("KBEngineApp::Client_onLoginBaseappFailed(): failedcode(%d:%s)!", failedcode, *serverErr(failedcode)); + + UKBEventData_onLoginBaseappFailed* pEventData = NewObject(); + pEventData->failedcode = failedcode; + pEventData->errorStr = serverErr(failedcode); + KBENGINE_EVENT_FIRE(KBEventTypes::onLoginBaseappFailed, pEventData); +} + +void KBEngineApp::Client_onReloginBaseappFailed(uint16 failedcode) +{ + ERROR_MSG("KBEngineApp::Client_onReloginBaseappFailed(): failedcode(%d:%s)!", failedcode, *serverErr(failedcode)); + + UKBEventData_onReloginBaseappFailed* pEventData = NewObject(); + pEventData->failedcode = failedcode; + pEventData->errorStr = serverErr(failedcode); + KBENGINE_EVENT_FIRE(KBEventTypes::onReloginBaseappFailed, pEventData); +} + +void KBEngineApp::Client_onReloginBaseappSuccessfully(MemoryStream& stream) +{ + stream >> entity_uuid_; + ERROR_MSG("KBEngineApp::Client_onReloginBaseappSuccessfully(): name(%s)!", *username_); + UKBEventData_onReloginBaseappSuccessfully* pEventData = NewObject(); + KBENGINE_EVENT_FIRE(KBEventTypes::onReloginBaseappSuccessfully, pEventData); +} + +void KBEngineApp::Client_onCreatedProxies(uint64 rndUUID, int32 eid, FString& entityType) +{ + DEBUG_MSG("KBEngineApp::Client_onCreatedProxies(): eid(%d), entityType(%s)!", eid, *entityType); + + entity_uuid_ = rndUUID; + entity_id_ = eid; + entity_type_ = entityType; + + if (!entities_.Contains(eid)) + { + ScriptModule** pModuleFind = EntityDef::moduledefs.Find(entityType); + if (!pModuleFind) + { + SCREEN_ERROR_MSG("KBEngineApp::Client_onCreatedProxies(): not found ScriptModule(%s)!", *entityType); + return; + } + + ScriptModule* pModule = *pModuleFind; + + Entity* pEntity = pModule->createEntity(); + pEntity->id(eid); + pEntity->className(entityType); + pEntity->onGetBase(); + + entities_.Add(eid, pEntity); + + MemoryStream** entityMessageFind = bufferedCreateEntityMessages_.Find(eid); + if (entityMessageFind) + { + MemoryStream* entityMessage = *entityMessageFind; + Client_onUpdatePropertys(*entityMessage); + bufferedCreateEntityMessages_.Remove(eid); + MemoryStream::reclaimObject(entityMessage); + } + + pEntity->__init__(); + pEntity->attachComponents(); + pEntity->inited(true); + + if (pArgs_->isOnInitCallPropertysSetMethods) + pEntity->callPropertysSetMethods(); + } + else + { + MemoryStream** entityMessageFind = bufferedCreateEntityMessages_.Find(eid); + if (entityMessageFind) + { + MemoryStream* entityMessage = *entityMessageFind; + Client_onUpdatePropertys(*entityMessage); + bufferedCreateEntityMessages_.Remove(eid); + MemoryStream::reclaimObject(entityMessage); + } + } +} + +ENTITY_ID KBEngineApp::getViewEntityIDFromStream(MemoryStream& stream) +{ + ENTITY_ID id = 0; + + if (!pArgs_->useAliasEntityID) + { + stream >> id; + return id; + } + + if (entityIDAliasIDList_.Num()> 255) + { + stream >> id; + } + else + { + uint8 aliasID = 0; + stream >> aliasID; + + // 如果为0且客户端上一步是重登陆或者重连操作并且服务端entity在断线期间一直处于在线状态 + // 则可以忽略这个错误, 因为cellapp可能一直在向baseapp发送同步消息， 当客户端重连上时未等 + // 服务端初始化步骤开始则收到同步信息, 此时这里就会出错。 + if (entityIDAliasIDList_.Num() <= aliasID) + return 0; + + id = entityIDAliasIDList_[aliasID]; + } + + return id; +} + +void KBEngineApp::Client_onUpdatePropertysOptimized(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + onUpdatePropertys_(eid, stream); +} + +void KBEngineApp::Client_onUpdatePropertys(MemoryStream& stream) +{ + ENTITY_ID eid; + stream >> eid; + onUpdatePropertys_(eid, stream); +} + +void KBEngineApp::onUpdatePropertys_(ENTITY_ID eid, MemoryStream& stream) +{ + Entity** pEntityFind = entities_.Find(eid); + + if (!pEntityFind) + { + MemoryStream** entityMessageFind = bufferedCreateEntityMessages_.Find(eid); + if (entityMessageFind) + { + ERROR_MSG("KBEngineApp::onUpdatePropertys_(): entity(%d) not found!", eid); + return; + } + + MemoryStream* stream1 = MemoryStream::createObject(); + stream1->append(stream); + stream1->rpos(stream.rpos() - 4); + bufferedCreateEntityMessages_.Add(eid, stream1); + return; + } + + Entity* pEntity = *pEntityFind; + + pEntity->onUpdatePropertys(stream); +} + +void KBEngineApp::Client_onEntityDestroyed(int32 eid) +{ + DEBUG_MSG("KBEngineApp::Client_onEntityDestroyed(): entity(%d)!", eid); + + Entity** pEntityFind = entities_.Find(eid); + + if (!pEntityFind) + { + ERROR_MSG("KBEngineApp::Client_onEntityDestroyed(): entity(%d) not found!", eid); + return; + } + + Entity* pEntity = (*pEntityFind); + + if (pEntity->inWorld()) + { + if (entity_id_ == eid) + clearSpace(false); + + (*pEntityFind)->leaveWorld(); + } + + if (controlledEntities_.Contains(pEntity)) + { + controlledEntities_.Remove(pEntity); + + UKBEventData_onLoseControlledEntity* pEventData = NewObject(); + pEventData->entityID = pEntity->id(); + KBENGINE_EVENT_FIRE(KBEventTypes::onLoseControlledEntity, pEventData); + } + + entities_.Remove(eid); + pEntity->destroy(); +} + +void KBEngineApp::clearSpace(bool isall) +{ + entityIDAliasIDList_.Empty(); + spacedatas_.Empty(); + clearEntities(isall); + isLoadedGeometry_ = false; + spaceID_ = 0; +} + +void KBEngineApp::clearEntities(bool isall) +{ + controlledEntities_.Empty(); + + if (!isall) + { + Entity* pEntity = player(); + + for(auto& item : entities_) + { + if (item.Key == pEntity->id()) + continue; + + if (item.Value->inWorld()) + item.Value->leaveWorld(); + + item.Value->destroy(); + } + + entities_.Empty(); + entities_.Add(pEntity->id(), pEntity); + } + else + { + for (auto& item : entities_) + { + if (item.Value->inWorld()) + item.Value->leaveWorld(); + + item.Value->destroy(); + } + + entities_.Empty(); + } +} + +void KBEngineApp::Client_initSpaceData(MemoryStream& stream) +{ + clearSpace(false); + stream >> spaceID_; + + while (stream.length() > 0) + { + FString key; + FString val; + + stream >> key >> val; + Client_setSpaceData(spaceID_, key, val); + } + + DEBUG_MSG("KBEngineApp::Client_initSpaceData(): spaceID(%d), size(%d)!", spaceID_, spacedatas_.Num()); +} + +void KBEngineApp::Client_setSpaceData(uint32 spaceID, const FString& key, const FString& value) +{ + DEBUG_MSG("KBEngineApp::Client_setSpaceData(): spaceID(%d), key(%s), value(%s)!", spaceID_, *key, *value); + spacedatas_.Add(key, value); + + if (key == TEXT("_mapping")) + addSpaceGeometryMapping(spaceID, value); + + UKBEventData_onSetSpaceData* pEventData = NewObject(); + pEventData->spaceID = spaceID_; + pEventData->key = key; + pEventData->value = value; + KBENGINE_EVENT_FIRE(KBEventTypes::onSetSpaceData, pEventData); +} + +void KBEngineApp::Client_delSpaceData(uint32 spaceID, const FString& key) +{ + DEBUG_MSG("KBEngineApp::Client_delSpaceData(): spaceID(%d), key(%s)!", spaceID_, *key); + + spacedatas_.Remove(key); + + UKBEventData_onDelSpaceData* pEventData = NewObject(); + pEventData->spaceID = spaceID_; + pEventData->key = key; + KBENGINE_EVENT_FIRE(KBEventTypes::onDelSpaceData, pEventData); +} + +void KBEngineApp::addSpaceGeometryMapping(uint32 uspaceID, const FString& respath) +{ + DEBUG_MSG("KBEngineApp::addSpaceGeometryMapping(): spaceID(%d), respath(%s)!", spaceID_, *respath); + + isLoadedGeometry_ = true; + spaceID_ = uspaceID; + spaceResPath_ = respath; + + UKBEventData_addSpaceGeometryMapping* pEventData = NewObject(); + pEventData->spaceResPath = spaceResPath_; + KBENGINE_EVENT_FIRE(KBEventTypes::addSpaceGeometryMapping, pEventData); +} + +FString KBEngineApp::getSpaceData(const FString& key) +{ + FString* valFind = spacedatas_.Find(key); + + if(!valFind) + return FString(); + + return (*valFind); +} + +void KBEngineApp::resetPassword(const FString& username) +{ + username_ = username; + resetpassword_loginapp(true); +} + +void KBEngineApp::resetpassword_loginapp(bool noconnect) +{ + if (noconnect) + { + reset(); + pNetworkInterface_->connectTo(pArgs_->ip, pArgs_->port, this, 4); + } + else + { + INFO_MSG("KBEngineApp::resetpassword_loginapp(): send resetpassword! username=%s", *username_); + Bundle* pBundle = Bundle::createObject(); + pBundle->newMessage(Messages::messages[TEXT("Loginapp_reqAccountResetPassword"])); + (*pBundle) << username_; + pBundle->send(pNetworkInterface_); + } +} + +void KBEngineApp::onOpenLoginapp_resetpassword() +{ + DEBUG_MSG("KBEngineApp::onOpenLoginapp_resetpassword(): successfully!"); + currserver_ = "loginapp"; + currstate_ = "resetpassword"; + lastTickCBTime_ = getTimeSeconds(); + + resetpassword_loginapp(false); +} + +void KBEngineApp::onConnectTo_resetpassword_callback(FString ip, uint16 port, bool success) +{ + lastTickCBTime_ = getTimeSeconds(); + + if (!success) + { + ERROR_MSG("KBEngineApp::onConnectTo_resetpassword_callback(): connect %s:%d is error!", *ip, port); + return; + } + + INFO_MSG("KBEngineApp::onConnectTo_resetpassword_callback(): connect %s:%d is success!", *ip, port); + + onOpenLoginapp_resetpassword(); +} + +void KBEngineApp::Client_onReqAccountResetPasswordCB(uint16 failcode) +{ + if (failcode != 0) + { + ERROR_MSG("KBEngineApp::Client_onReqAccountResetPasswordCB(): reset failed! code=%d, error=%s! username=%s", failcode, *serverErr(failcode), *username_); + return; + } + + DEBUG_MSG("KBEngineApp::Client_onReqAccountResetPasswordCB(): successfully! username=%s", *username_); +} + +bool KBEngineApp::createAccount(const FString& username, const FString& password, const TArray& datas) +{ + if (username.Len() == 0) + { + ERROR_MSG("KBEngineApp::createAccount(): username is empty!"); + return false; + } + + if (password.Len() == 0) + { + ERROR_MSG("KBEngineApp::createAccount(): password is empty!"); + return false; + } + + username_ = username; + password_ = password; + clientdatas_ = datas; + + createAccount_loginapp(true); + return true; +} + +void KBEngineApp::createAccount_loginapp(bool noconnect) +{ + if (noconnect) + { + reset(); + pNetworkInterface_->connectTo(pArgs_->ip, pArgs_->port, this, 1); + } + else + { + INFO_MSG("KBEngineApp::createAccount_loginapp(): send create! username=%s", *username_); + Bundle* pBundle = Bundle::createObject(); + pBundle->newMessage(Messages::messages[TEXT("Loginapp_reqCreateAccount"])); + (*pBundle) << username_; + (*pBundle) << password_; + pBundle->appendBlob(clientdatas_); + pBundle->send(pNetworkInterface_); + } +} + +void KBEngineApp::onOpenLoginapp_createAccount() +{ + DEBUG_MSG("KBEngineApp::onOpenLoginapp_createAccount(): successfully!"); + + currserver_ = TEXT("loginapp"); + currstate_ = TEXT("createAccount"); + lastTickCBTime_ = getTimeSeconds(); + + createAccount_loginapp(false); +} + +void KBEngineApp::onConnectTo_loginapp_create_callback(FString ip, uint16 port, bool success) +{ + lastTickCBTime_ = getTimeSeconds(); + + if (!success) + { + ERROR_MSG("KBEngineApp::onConnectTo_loginapp_create_callback(): connect %s:%d is error!", *ip, port); + return; + } + + INFO_MSG("KBEngineApp::onConnectTo_loginapp_create_callback(): connect %s:%d is success!", *ip, port); + + onOpenLoginapp_createAccount(); +} + +void KBEngineApp::Client_onCreateAccountResult(MemoryStream& stream) +{ + uint16 retcode; + stream >> retcode; + + TArray datas; + stream.readBlob(datas); + + UKBEventData_onCreateAccountResult* pEventData = NewObject(); + pEventData->errorCode = retcode; + pEventData->errorStr = serverErr(retcode); + pEventData->datas = datas; + KBENGINE_EVENT_FIRE(KBEventTypes::onCreateAccountResult, pEventData); + + if (retcode != 0) + { + WARNING_MSG("KBEngineApp::Client_onCreateAccountResult(): create(%s) failed! error=%d(%s)!", *username_, retcode, *serverErr(retcode)); + return; + } + + DEBUG_MSG("KBEngineApp::Client_onCreateAccountResult(): create(%s) is successfully!", *username_); +} + +void KBEngineApp::bindAccountEmail(const FString& emailAddress) +{ + INFO_MSG("KBEngineApp::bindAccountEmail(): send bindAccountEmail! username=%s", *username_); + Bundle* pBundle = Bundle::createObject(); + pBundle->newMessage(Messages::messages[TEXT("Baseapp_reqAccountBindEmail"])); + (*pBundle) << entity_id_; + (*pBundle) << password_; + (*pBundle) << emailAddress; + pBundle->send(pNetworkInterface_); +} + +void KBEngineApp::Client_onReqAccountBindEmailCB(uint16 failcode) +{ + if (failcode != 0) + { + ERROR_MSG("KBEngineApp::Client_onReqAccountBindEmailCB(): bind failed! code=%d, error=%s! username=%s", failcode, *serverErr(failcode), *username_); + return; + } + + DEBUG_MSG("KBEngineApp::Client_onReqAccountBindEmailCB(): successfully! username=%s", *username_); +} + +void KBEngineApp::newPassword(const FString& old_password, const FString& new_password) +{ + INFO_MSG("KBEngineApp::newPassword(): send newPassword! username=%s", *username_); + Bundle* pBundle = Bundle::createObject(); + pBundle->newMessage(Messages::messages[TEXT("Baseapp_reqAccountNewPassword"])); + (*pBundle) << entity_id_; + (*pBundle) << old_password; + (*pBundle) << new_password; + pBundle->send(pNetworkInterface_); +} + +void KBEngineApp::Client_onReqAccountNewPasswordCB(uint16 failcode) +{ + if (failcode != 0) + { + ERROR_MSG("KBEngineApp::Client_onReqAccountNewPasswordCB(): newPassword failed! code=%d, error=%s! username=%s", failcode, *serverErr(failcode), *username_); + return; + } + + DEBUG_MSG("KBEngineApp::Client_onReqAccountNewPasswordCB(): successfully! username=%s", *username_); +} + +void KBEngineApp::Client_onRemoteMethodCallOptimized(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + onRemoteMethodCall_(eid, stream); +} + +void KBEngineApp::Client_onRemoteMethodCall(MemoryStream& stream) +{ + ENTITY_ID eid = 0; + stream >> eid; + onRemoteMethodCall_(eid, stream); +} + +void KBEngineApp::onRemoteMethodCall_(ENTITY_ID eid, MemoryStream& stream) +{ + Entity** pEntityFind = entities_.Find(eid); + + if (!pEntityFind) + { + ERROR_MSG("KBEngineApp::onRemoteMethodCall_(): entity(%d) not found!", eid); + return; + } + + Entity* pEntity = *pEntityFind; + pEntity->onRemoteMethodCall(stream); +} + +void KBEngineApp::Client_onControlEntity(ENTITY_ID eid, int8 isControlled) +{ + Entity** pEntityFind = entities_.Find(eid); + + if (!pEntityFind) + { + ERROR_MSG("KBEngineApp::Client_onControlEntity(): entity(%d) not found!", eid); + return; + } + + bool isCont = isControlled != 0; + + if (isCont) + { + // 如果被控制者是玩家自己，那表示玩家自己被其它人控制了 + // 所以玩家自己不应该进入这个被控制列表 + if (entity_id_ != (*pEntityFind)->id()) + { + controlledEntities_.Add((*pEntityFind)); + } + } + else + { + controlledEntities_.Remove((*pEntityFind)); + } + + (*pEntityFind)->isControlled(isCont); + (*pEntityFind)->onControlled(isCont); + + UKBEventData_onControlled* pEventData = NewObject(); + pEventData->entityID = (*pEntityFind)->id(); + pEventData->isControlled = isCont; + KBENGINE_EVENT_FIRE(KBEventTypes::onControlled, pEventData); +} + +void KBEngineApp::Client_onStreamDataStarted(int16 id, uint32 datasize, FString descr) +{ + UKBEventData_onStreamDataStarted* pEventData = NewObject(); + pEventData->resID = id; + pEventData->dataSize = datasize; + pEventData->dataDescr = descr; + KBENGINE_EVENT_FIRE(KBEventTypes::onStreamDataStarted, pEventData); +} + +void KBEngineApp::Client_onStreamDataRecv(MemoryStream& stream) +{ + UKBEventData_onStreamDataRecv* pEventData = NewObject(); + + uint16 id = stream.read(); + pEventData->resID = id; + stream.readBlob(pEventData->data); + + KBENGINE_EVENT_FIRE(KBEventTypes::onStreamDataRecv, pEventData); +} + +void KBEngineApp::Client_onStreamDataCompleted(int16 id) +{ + UKBEventData_onStreamDataCompleted* pEventData = NewObject(); + pEventData->resID = id; + KBENGINE_EVENT_FIRE(KBEventTypes::onStreamDataCompleted, pEventData); +} + +void KBEngineApp::Client_onEntityEnterWorld(MemoryStream& stream) +{ + ENTITY_ID eid; + stream >> eid; + + if (entity_id_ > 0 && entity_id_ != eid) + entityIDAliasIDList_.Add(eid); + + uint16 uEntityType; + + if (EntityDef::idmoduledefs.Num() > 255) + uEntityType = stream.read(); + else + uEntityType = stream.read(); + + int8 isOnGround = 1; + + if (stream.length() > 0) + isOnGround = stream.read(); + + ScriptModule** pScriptModuleFind = EntityDef::idmoduledefs.Find(uEntityType); + if (!pScriptModuleFind) + { + SCREEN_ERROR_MSG("KBEngineApp::Client_onEntityEnterWorld(): not found ScriptModule(utype = %d)!", uEntityType); + return; + } + + ScriptModule* pScriptModule = *pScriptModuleFind; + FString entityType = pScriptModule->name; + // DEBUG_MSG("KBEngineApp::Client_onEntityEnterWorld(): %s(%d), spaceID(%d)!", *entityType, eid, spaceID_); + + Entity** pEntityFind = entities_.Find(eid); + + if (!pEntityFind) + { + MemoryStream** entityMessageFind = bufferedCreateEntityMessages_.Find(eid); + if (!entityMessageFind) + { + ERROR_MSG("KBEngineApp::Client_onEntityEnterWorld(): entity(%d) not found!", eid); + return; + } + + pScriptModuleFind = EntityDef::moduledefs.Find(entityType); + if (!pScriptModuleFind) + { + SCREEN_ERROR_MSG("KBEngineApp::Client_onEntityEnterWorld(): not found ScriptModule(%s)!", *entityType); + return; + } + + ScriptModule* pModule = *pScriptModuleFind; + + Entity* pEntity = pModule->createEntity(); + pEntity->id(eid); + pEntity->className(entityType); + pEntity->onGetCell(); + + entities_.Add(eid, pEntity); + + Client_onUpdatePropertys(*(*entityMessageFind)); + MemoryStream::reclaimObject((*entityMessageFind)); + bufferedCreateEntityMessages_.Remove(eid); + + pEntity->isOnGround(isOnGround > 0); + pEntity->onDirectionChanged(pEntity->direction); + pEntity->onPositionChanged(pEntity->position); + + pEntity->__init__(); + pEntity->attachComponents(); + pEntity->inited(true); + pEntity->inWorld(true); + pEntity->enterWorld(); + + if (pArgs_->isOnInitCallPropertysSetMethods) + pEntity->callPropertysSetMethods(); + } + else + { + Entity* pEntity = (*pEntityFind); + + if (!pEntity->inWorld()) + { + // 安全起见， 这里清空一下 + // 如果服务端上使用giveClientTo切换控制权 + // 之前的实体已经进入世界， 切换后的实体也进入世界， 这里可能会残留之前那个实体进入世界的信息 + entityIDAliasIDList_.Empty(); + clearEntities(false); + entities_.Add(pEntity->id(), pEntity); + + pEntity->onGetCell(); + + pEntity->onDirectionChanged(pEntity->direction); + pEntity->onPositionChanged(pEntity->position); + + entityServerPos_ = pEntity->position; + pEntity->isOnGround(isOnGround > 0); + pEntity->inWorld(true); + pEntity->enterWorld(); + + if (pArgs_->isOnInitCallPropertysSetMethods) + pEntity->callPropertysSetMethods(); + } + } +} + +void KBEngineApp::Client_onEntityLeaveWorldOptimized(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + Client_onEntityLeaveWorld(eid); +} + +void KBEngineApp::Client_onEntityLeaveWorld(ENTITY_ID eid) +{ + Entity** pEntityFind = entities_.Find(eid); + + if (!pEntityFind) + { + ERROR_MSG("KBEngineApp::Client_onEntityLeaveWorld(): entity(%d) not found!", eid); + return; + } + + Entity* pEntity = *pEntityFind; + + if (pEntity->inWorld()) + pEntity->leaveWorld(); + + if (entity_id_ == eid) + { + clearSpace(false); + pEntity->onLoseCell(); + } + else + { + if (controlledEntities_.Contains(pEntity)) + { + controlledEntities_.Remove(pEntity); + + UKBEventData_onLoseControlledEntity* pEventData = NewObject(); + pEventData->entityID = pEntity->id(); + KBENGINE_EVENT_FIRE(KBEventTypes::onLoseControlledEntity, pEventData); + } + + entities_.Remove(eid); + pEntity->destroy(); + entityIDAliasIDList_.Remove(eid); + } +} + +void KBEngineApp::Client_onEntityEnterSpace(MemoryStream& stream) +{ + ENTITY_ID eid = stream.read(); + spaceID_ = stream.read(); + + int8 isOnGround = 1; + + if (stream.length() > 0) + isOnGround = stream.read(); + + Entity** pEntityFind = entities_.Find(eid); + + if (!pEntityFind) + { + ERROR_MSG("KBEngineApp::Client_onEntityEnterSpace(): entity(%d) not found!", eid); + return; + } + + Entity* pEntity = *pEntityFind; + pEntity->isOnGround(isOnGround > 0); + entityServerPos_ = pEntity->position; + pEntity->enterSpace(); +} + +void KBEngineApp::Client_onEntityLeaveSpace(ENTITY_ID eid) +{ + Entity** pEntityFind = entities_.Find(eid); + + if (!pEntityFind) + { + ERROR_MSG("KBEngineApp::Client_onEntityLeaveSpace(): entity(%d) not found!", eid); + return; + } + + Entity* pEntity = *pEntityFind; + pEntity->leaveSpace(); + + clearSpace(false); +} + +void KBEngineApp::Client_onUpdateBasePos(float x, float y, float z) +{ + entityServerPos_.X = x; + entityServerPos_.Y = y; + entityServerPos_.Z = z; + + Entity* pEntity = player(); + if (pEntity && pEntity->isControlled()) + { + pEntity->position.Set(entityServerPos_.X, entityServerPos_.Y, entityServerPos_.Z); + + UKBEventData_updatePosition* pEventData = NewObject(); + KBPos2UE4Pos(pEventData->position, entityServerPos_); + KBDir2UE4Dir(pEventData->direction, pEntity->direction); + pEventData->entityID = pEntity->id(); + pEventData->moveSpeed = pEntity->velocity(); + KBENGINE_EVENT_FIRE(KBEventTypes::updatePosition, pEventData); + + pEntity->onUpdateVolatileData(); + } +} + +void KBEngineApp::Client_onUpdateBasePosXZ(float x, float z) +{ + entityServerPos_.X = x; + entityServerPos_.Z = z; + + Entity* pEntity = player(); + if (pEntity && pEntity->isControlled()) + { + pEntity->position.X = entityServerPos_.X; + pEntity->position.Z = entityServerPos_.Z; + + UKBEventData_updatePosition* pEventData = NewObject(); + KBPos2UE4Pos(pEventData->position, entityServerPos_); + KBDir2UE4Dir(pEventData->direction, pEntity->direction); + pEventData->entityID = pEntity->id(); + pEventData->moveSpeed = pEntity->velocity(); + KBENGINE_EVENT_FIRE(KBEventTypes::updatePosition, pEventData); + + pEntity->onUpdateVolatileData(); + } +} + +void KBEngineApp::Client_onUpdateBaseDir(MemoryStream& stream) +{ + float yaw, pitch, roll; + stream >> yaw >> pitch >> roll; + + Entity* pEntity = player(); + if (pEntity && pEntity->isControlled()) + { + pEntity->direction.Set(roll, pitch, yaw); + + UKBEventData_set_direction* pEventData = NewObject(); + KBDir2UE4Dir(pEventData->direction, pEntity->direction); + pEventData->entityID = pEntity->id(); + KBENGINE_EVENT_FIRE(KBEventTypes::set_direction, pEventData); + + pEntity->onUpdateVolatileData(); + } +} + +void KBEngineApp::Client_onUpdateData(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + Entity** pEntityFind = entities_.Find(eid); + + if (!pEntityFind) + { + ERROR_MSG("KBEngineApp::Client_onUpdateData(): entity(%d) not found!", eid); + return; + } +} + +void KBEngineApp::Client_onSetEntityPosAndDir(MemoryStream& stream) +{ + ENTITY_ID eid; + stream >> eid; + + Entity** pEntityFind = entities_.Find(eid); + + if (!pEntityFind) + { + ERROR_MSG("KBEngineApp::Client_onSetEntityPosAndDir(): entity(%d) not found!", eid); + return; + } + + Entity& entity = *(*pEntityFind); + + FVector old_position = entity.position; + FVector old_direction = entity.direction; + + entity.position.X = stream.read(); + entity.position.Y = stream.read(); + entity.position.Z = stream.read(); + + entity.direction.X = stream.read(); + entity.direction.Y = stream.read(); + entity.direction.Z = stream.read(); + + entity.entityLastLocalPos = entity.position; + entity.entityLastLocalDir = entity.direction; + + entity.onDirectionChanged(old_direction); + entity.onPositionChanged(old_position); +} + +void KBEngineApp::Client_onUpdateData_ypr(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + float y = stream.read(); + float p = stream.read(); + float r = stream.read(); + + _updateVolatileData(eid, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, y, p, r, -1, false); +} + +void KBEngineApp::Client_onUpdateData_yp(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + float y = stream.read(); + float p = stream.read(); + + _updateVolatileData(eid, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, y, p, KBE_FLT_MAX, -1, false); +} + +void KBEngineApp::Client_onUpdateData_yr(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + float y = stream.read(); + float r = stream.read(); + + _updateVolatileData(eid, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, y, KBE_FLT_MAX, r, -1, false); +} + +void KBEngineApp::Client_onUpdateData_pr(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + float p = stream.read(); + float r = stream.read(); + + _updateVolatileData(eid, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, p, r, -1, false); +} + +void KBEngineApp::Client_onUpdateData_y(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + float y = stream.read(); + + _updateVolatileData(eid, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, y, KBE_FLT_MAX, KBE_FLT_MAX, -1, false); +} + +void KBEngineApp::Client_onUpdateData_p(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + float p = stream.read(); + + _updateVolatileData(eid, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, p, KBE_FLT_MAX, -1, false); +} + +void KBEngineApp::Client_onUpdateData_r(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + float r = stream.read(); + + _updateVolatileData(eid, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, r, -1, false); +} + +void KBEngineApp::Client_onUpdateData_xz(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + float x = stream.read(); + float z = stream.read(); + + _updateVolatileData(eid, x, KBE_FLT_MAX, z, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, 1, false); +} + +void KBEngineApp::Client_onUpdateData_xz_ypr(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + float x = stream.read(); + float z = stream.read(); + + float y = stream.read(); + float p = stream.read(); + float r = stream.read(); + + _updateVolatileData(eid, x, KBE_FLT_MAX, z, y, p, r, 1, false); +} + +void KBEngineApp::Client_onUpdateData_xz_yp(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + float x = stream.read(); + float z = stream.read(); + + float y = stream.read(); + float p = stream.read(); + + _updateVolatileData(eid, x, KBE_FLT_MAX, z, y, p, KBE_FLT_MAX, 1, false); +} + +void KBEngineApp::Client_onUpdateData_xz_yr(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + float x = stream.read(); + float z = stream.read(); + + float y = stream.read(); + float r = stream.read(); + + _updateVolatileData(eid, x, KBE_FLT_MAX, z, y, KBE_FLT_MAX, r, 1, false); +} + +void KBEngineApp::Client_onUpdateData_xz_pr(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + float x = stream.read(); + float z = stream.read(); + + float p = stream.read(); + float r = stream.read(); + + _updateVolatileData(eid, x, KBE_FLT_MAX, z, KBE_FLT_MAX, p, r, 1, false); +} + +void KBEngineApp::Client_onUpdateData_xz_y(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + float x = stream.read(); + float z = stream.read(); + + float y = stream.read(); + + _updateVolatileData(eid, x, KBE_FLT_MAX, z, y, KBE_FLT_MAX, KBE_FLT_MAX, 1, false); +} + +void KBEngineApp::Client_onUpdateData_xz_p(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + float x = stream.read(); + float z = stream.read(); + + float p = stream.read(); + + _updateVolatileData(eid, x, KBE_FLT_MAX, z, KBE_FLT_MAX, p, KBE_FLT_MAX, 1, false); +} + +void KBEngineApp::Client_onUpdateData_xz_r(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + float x = stream.read(); + float z = stream.read(); + + float r = stream.read(); + + _updateVolatileData(eid, x, KBE_FLT_MAX, z, KBE_FLT_MAX, KBE_FLT_MAX, r, 1, false); +} + +void KBEngineApp::Client_onUpdateData_xyz(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + float x = stream.read(); + float y = stream.read(); + float z = stream.read(); + + _updateVolatileData(eid, x, y, z, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, 0, false); +} + +void KBEngineApp::Client_onUpdateData_xyz_ypr(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + float x = stream.read(); + float y = stream.read(); + float z = stream.read(); + + float yaw = stream.read(); + float p = stream.read(); + float r = stream.read(); + + _updateVolatileData(eid, x, y, z, yaw, p, r, 0, false); +} + +void KBEngineApp::Client_onUpdateData_xyz_yp(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + float x = stream.read(); + float y = stream.read(); + float z = stream.read(); + + float yaw = stream.read(); + float p = stream.read(); + + _updateVolatileData(eid, x, y, z, yaw, p, KBE_FLT_MAX, 0, false); +} + +void KBEngineApp::Client_onUpdateData_xyz_yr(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + float x = stream.read(); + float y = stream.read(); + float z = stream.read(); + + float yaw = stream.read(); + float r = stream.read(); + + _updateVolatileData(eid, x, y, z, yaw, KBE_FLT_MAX, r, 0, false); +} + +void KBEngineApp::Client_onUpdateData_xyz_pr(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + float x = stream.read(); + float y = stream.read(); + float z = stream.read(); + + float p = stream.read(); + float r = stream.read(); + + _updateVolatileData(eid, x, y, z, KBE_FLT_MAX, p, r, 0, false); +} + +void KBEngineApp::Client_onUpdateData_xyz_y(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + float x = stream.read(); + float y = stream.read(); + float z = stream.read(); + + float yaw = stream.read(); + + _updateVolatileData(eid, x, y, z, yaw, KBE_FLT_MAX, KBE_FLT_MAX, 0, false); +} + +void KBEngineApp::Client_onUpdateData_xyz_p(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + float x = stream.read(); + float y = stream.read(); + float z = stream.read(); + + float p = stream.read(); + + _updateVolatileData(eid, x, y, z, KBE_FLT_MAX, p, KBE_FLT_MAX, 0, false); +} + +void KBEngineApp::Client_onUpdateData_xyz_r(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + float x = stream.read(); + float y = stream.read(); + float z = stream.read(); + + float r = stream.read(); + + _updateVolatileData(eid, x, y, z, KBE_FLT_MAX, KBE_FLT_MAX, r, 0, false); +} + +void KBEngineApp::Client_onUpdateData_ypr_optimized(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + int8 y = stream.read(); + int8 p = stream.read(); + int8 r = stream.read(); + + _updateVolatileData(eid, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, y, p, r, -1, true); +} + +void KBEngineApp::Client_onUpdateData_yp_optimized(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + int8 y = stream.read(); + int8 p = stream.read(); + + _updateVolatileData(eid, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, y, p, KBE_FLT_MAX, -1, true); +} + +void KBEngineApp::Client_onUpdateData_yr_optimized(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + int8 y = stream.read(); + int8 r = stream.read(); + + _updateVolatileData(eid, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, y, KBE_FLT_MAX, r, -1, true); +} + +void KBEngineApp::Client_onUpdateData_pr_optimized(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + int8 p = stream.read(); + int8 r = stream.read(); + + _updateVolatileData(eid, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, p, r, -1, true); +} + +void KBEngineApp::Client_onUpdateData_y_optimized(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + int8 y = stream.read(); + + _updateVolatileData(eid, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, y, KBE_FLT_MAX, KBE_FLT_MAX, -1, true); +} + +void KBEngineApp::Client_onUpdateData_p_optimized(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + int8 p = stream.read(); + + _updateVolatileData(eid, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, p, KBE_FLT_MAX, -1, true); +} + +void KBEngineApp::Client_onUpdateData_r_optimized(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + int8 r = stream.read(); + + _updateVolatileData(eid, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, r, -1, true); +} + +void KBEngineApp::Client_onUpdateData_xz_optimized(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + FVector xz; + stream.readPackXZ(xz.X, xz.Z); + + _updateVolatileData(eid, xz.X, KBE_FLT_MAX, xz.Z, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, 1, true); +} + +void KBEngineApp::Client_onUpdateData_xz_ypr_optimized(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + FVector xz; + stream.readPackXZ(xz.X, xz.Z); + + int8 y = stream.read(); + int8 p = stream.read(); + int8 r = stream.read(); + + _updateVolatileData(eid, xz.X, KBE_FLT_MAX, xz.Z, y, p, r, 1, true); +} + +void KBEngineApp::Client_onUpdateData_xz_yp_optimized(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + FVector xz; + stream.readPackXZ(xz.X, xz.Z); + + int8 y = stream.read(); + int8 p = stream.read(); + + _updateVolatileData(eid, xz.X, KBE_FLT_MAX, xz.Z, y, p, KBE_FLT_MAX, 1, true); +} + +void KBEngineApp::Client_onUpdateData_xz_yr_optimized(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + FVector xz; + stream.readPackXZ(xz.X, xz.Z); + + int8 y = stream.read(); + int8 r = stream.read(); + + _updateVolatileData(eid, xz.X, KBE_FLT_MAX, xz.Z, y, KBE_FLT_MAX, r, 1, true); +} + +void KBEngineApp::Client_onUpdateData_xz_pr_optimized(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + FVector xz; + stream.readPackXZ(xz.X, xz.Z); + + int8 p = stream.read(); + int8 r = stream.read(); + + _updateVolatileData(eid, xz.X, KBE_FLT_MAX, xz.Z, KBE_FLT_MAX, p, r, 1, true); +} + +void KBEngineApp::Client_onUpdateData_xz_y_optimized(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + FVector xz; + stream.readPackXZ(xz.X, xz.Z); + + int8 y = stream.read(); + + _updateVolatileData(eid, xz.X, KBE_FLT_MAX, xz.Z, y, KBE_FLT_MAX, KBE_FLT_MAX, 1, true); +} + +void KBEngineApp::Client_onUpdateData_xz_p_optimized(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + FVector xz; + stream.readPackXZ(xz.X, xz.Z); + + int8 p = stream.read(); + + _updateVolatileData(eid, xz.X, KBE_FLT_MAX, xz.Z, KBE_FLT_MAX, p, KBE_FLT_MAX, 1, true); +} + +void KBEngineApp::Client_onUpdateData_xz_r_optimized(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + FVector xz; + stream.readPackXZ(xz.X, xz.Z); + + int8 r = stream.read(); + + _updateVolatileData(eid, xz.X, KBE_FLT_MAX, xz.Z, KBE_FLT_MAX, KBE_FLT_MAX, r, 1, true); +} + +void KBEngineApp::Client_onUpdateData_xyz_optimized(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + FVector xz; + stream.readPackXZ(xz.X, xz.Z); + stream.readPackY(xz.Y); + + _updateVolatileData(eid, xz.X, xz.Y, xz.Z, KBE_FLT_MAX, KBE_FLT_MAX, KBE_FLT_MAX, 0, true); +} + +void KBEngineApp::Client_onUpdateData_xyz_ypr_optimized(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + FVector xz; + stream.readPackXZ(xz.X, xz.Z); + stream.readPackY(xz.Y); + + int8 y = stream.read(); + int8 p = stream.read(); + int8 r = stream.read(); + + _updateVolatileData(eid, xz.X, xz.Y, xz.Z, y, p, r, 0, true); +} + +void KBEngineApp::Client_onUpdateData_xyz_yp_optimized(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + FVector xz; + stream.readPackXZ(xz.X, xz.Z); + stream.readPackY(xz.Y); + + int8 y = stream.read(); + int8 p = stream.read(); + + _updateVolatileData(eid, xz.X, xz.Y, xz.Z, y, p, KBE_FLT_MAX, 0, true); +} + +void KBEngineApp::Client_onUpdateData_xyz_yr_optimized(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + FVector xz; + stream.readPackXZ(xz.X, xz.Z); + stream.readPackY(xz.Y); + + int8 y = stream.read(); + int8 r = stream.read(); + + _updateVolatileData(eid, xz.X, xz.Y, xz.Z, y, KBE_FLT_MAX, r, 0, true); +} + +void KBEngineApp::Client_onUpdateData_xyz_pr_optimized(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + FVector xz; + stream.readPackXZ(xz.X, xz.Z); + stream.readPackY(xz.Y); + + int8 p = stream.read(); + int8 r = stream.read(); + + _updateVolatileData(eid, xz.X, xz.Y, xz.Z, KBE_FLT_MAX, p, r, 0, true); +} + +void KBEngineApp::Client_onUpdateData_xyz_y_optimized(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + FVector xz; + stream.readPackXZ(xz.X, xz.Z); + stream.readPackY(xz.Y); + + int8 y = stream.read(); + + _updateVolatileData(eid, xz.X, xz.Y, xz.Z, y, KBE_FLT_MAX, KBE_FLT_MAX, 0, true); +} + +void KBEngineApp::Client_onUpdateData_xyz_p_optimized(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + FVector xz; + stream.readPackXZ(xz.X, xz.Z); + stream.readPackY(xz.Y); + + int8 p = stream.read(); + + _updateVolatileData(eid, xz.X, xz.Y, xz.Z, KBE_FLT_MAX, p, KBE_FLT_MAX, 0, true); +} + +void KBEngineApp::Client_onUpdateData_xyz_r_optimized(MemoryStream& stream) +{ + ENTITY_ID eid = getViewEntityIDFromStream(stream); + + FVector xz; + stream.readPackXZ(xz.X, xz.Z); + stream.readPackY(xz.Y); + + int8 r = stream.read(); + + _updateVolatileData(eid, xz.X, xz.Y, xz.Z, KBE_FLT_MAX, KBE_FLT_MAX, r, 0, true); +} + +void KBEngineApp::_updateVolatileData(ENTITY_ID entityID, float x, float y, float z, float yaw, float pitch, float roll, int8 isOnGround, bool isOptimized) +{ + Entity** pEntityFind = entities_.Find(entityID); + + if (!pEntityFind) + { + // 如果为0且客户端上一步是重登陆或者重连操作并且服务端entity在断线期间一直处于在线状态 + // 则可以忽略这个错误, 因为cellapp可能一直在向baseapp发送同步消息， 当客户端重连上时未等 + // 服务端初始化步骤开始则收到同步信息, 此时这里就会出错。 + ERROR_MSG("KBEngineApp::_updateVolatileData(): entity(%d) not found!", entityID); + return; + } + + Entity& entity = *(*pEntityFind); + + // 小于0不设置 + if (isOnGround >= 0) + { + entity.isOnGround(isOnGround > 0); + } + + bool changeDirection = false; + + if (roll != KBE_FLT_MAX) + { + changeDirection = true; + entity.direction.X = int82angle((int8)roll, false); + } + + if (pitch != KBE_FLT_MAX) + { + changeDirection = true; + entity.direction.Y = int82angle((int8)pitch, false); + } + + if (yaw != KBE_FLT_MAX) + { + changeDirection = true; + entity.direction.Z = int82angle((int8)yaw, false); + } + + bool done = false; + if (changeDirection == true) + { + UKBEventData_set_direction* pEventData = NewObject(); + KBDir2UE4Dir(pEventData->direction, entity.direction); + pEventData->entityID = entity.id(); + KBENGINE_EVENT_FIRE(KBEventTypes::set_direction, pEventData); + + done = true; + } + + bool positionChanged = x != KBE_FLT_MAX || y != KBE_FLT_MAX || z != KBE_FLT_MAX; + if (x == KBE_FLT_MAX) x = 0.0f; + if (y == KBE_FLT_MAX) y = 0.0f; + if (z == KBE_FLT_MAX) z = 0.0f; + + if (positionChanged) + { + entity.position = isOptimized ? FVector(x + entityServerPos_.X, y + entityServerPos_.Y, z + entityServerPos_.Z) : FVector(x, y, z); + done = true; + + UKBEventData_updatePosition* pEventData = NewObject(); + KBPos2UE4Pos(pEventData->position, entity.position); + KBDir2UE4Dir(pEventData->direction, entity.direction); + pEventData->entityID = entity.id(); + pEventData->moveSpeed = entity.velocity(); + pEventData->isOnGround = entity.isOnGround(); + KBENGINE_EVENT_FIRE(KBEventTypes::updatePosition, pEventData); + } + + if (done) + entity.onUpdateVolatileData(); +} + } \ No newline at end of file diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBEngine.h b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBEngine.h index 7670f36096..5462a1150d 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBEngine.h +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBEngine.h @@ -7,6 +7,9 @@ #include "Interfaces.h" #include "KBETicker.h" +namespace KBEngine +{ + class KBEngineArgs; class Entity; class NetworkInterfaceBase; @@ -518,3 +521,4 @@ class KBENGINEPLUGINS_API KBEngineApp : public InterfaceConnect }; +} diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBEngineArgs.cpp b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBEngineArgs.cpp index 8ab0c7d860..c239713286 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBEngineArgs.cpp +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBEngineArgs.cpp @@ -2,6 +2,9 @@ #include "KBEngineArgs.h" #include "KBDebug.h" +namespace KBEngine +{ + KBEngineArgs::KBEngineArgs(): ip(TEXT("127.0.0.1")), port(20013), @@ -41,4 +44,6 @@ int KBEngineArgs::getUDPRecvBufferSize() int KBEngineArgs::getUDPSendBufferSize() { return (int)UDP_SEND_BUFFER_MAX; +} + } \ No newline at end of file diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBEngineArgs.h b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBEngineArgs.h index 6b1ad4265b..935a5f4739 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBEngineArgs.h +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBEngineArgs.h @@ -8,6 +8,9 @@ /* 初始化KBEngine的参数类 */ +namespace KBEngine +{ + class KBENGINEPLUGINS_API KBEngineArgs { public: @@ -48,3 +51,5 @@ class KBENGINEPLUGINS_API KBEngineArgs MessageLengthEx UDP_RECV_BUFFER_MAX; }; + +} \ No newline at end of file diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBEnginePluginsPrivatePCH.h b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBEnginePluginsPrivatePCH.h index 801faac386..ffe7fb40b3 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBEnginePluginsPrivatePCH.h +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBEnginePluginsPrivatePCH.h @@ -1,65 +1,5 @@ +#pragma once + // Copyright 1998-2016 Epic Games, Inc. All Rights Reserved. #include "Engine.h" #include "KBEnginePlugins.h" - -// You should place include statements to your module's private header files here. You only need to -// add includes for headers that are used in most of your module's source files though. - - -#define INFO_MSG(Format, ...) \ -{ \ - SET_WARN_COLOR(COLOR_CYAN); \ - const FString Msg = FString::Printf(TEXT(Format), ##__VA_ARGS__); \ - UE_LOG(LogKBEngine, Log, TEXT("%s"), *Msg); \ - CLEAR_WARN_COLOR(); \ -} - -#define DEBUG_MSG(Format, ...) \ -{ \ - SET_WARN_COLOR(COLOR_CYAN); \ - const FString Msg = FString::Printf(TEXT(Format), ##__VA_ARGS__); \ - UE_LOG(LogKBEngine, Log, TEXT("**DEBUG** %s"), *Msg); \ - CLEAR_WARN_COLOR(); \ -} - -#define WARNING_MSG(Format, ...) \ -{ \ - SET_WARN_COLOR(COLOR_YELLOW); \ - const FString Msg = FString::Printf(TEXT(Format), ##__VA_ARGS__); \ - UE_LOG(LogKBEngine, Log, TEXT("**WARNING** %s"), *Msg); \ - CLEAR_WARN_COLOR(); \ -} - -#define SCREEN_WARNING_MSG(Format, ...) \ -{ \ - SET_WARN_COLOR(COLOR_YELLOW); \ - const FString Msg = FString::Printf(TEXT(Format), ##__VA_ARGS__); \ - const FString NewMsg = FString::Printf(TEXT("**WARNING** %s"), *Msg); \ - UE_LOG(LogKBEngine, Log, TEXT("%s"), *NewMsg); \ - CLEAR_WARN_COLOR(); \ - GEngine->AddOnScreenDebugMessage(-1, 5.f, FColor::Yellow, NewMsg); \ -} - -#define ERROR_MSG(Format, ...) \ -{ \ - SET_WARN_COLOR(COLOR_RED); \ - const FString Msg = FString::Printf(TEXT(Format), ##__VA_ARGS__); \ - UE_LOG(LogKBEngine, Log, TEXT("**ERROR** %s"), *Msg); \ - CLEAR_WARN_COLOR(); \ -} - -#define SCREEN_ERROR_MSG(Format, ...) \ -{ \ - SET_WARN_COLOR(COLOR_RED); \ - const FString Msg = FString::Printf(TEXT(Format), ##__VA_ARGS__); \ - const FString NewMsg = FString::Printf(TEXT("**ERROR** %s"), *Msg); \ - UE_LOG(LogKBEngine, Log, TEXT("%s"), *NewMsg); \ - CLEAR_WARN_COLOR(); \ - GEngine->AddOnScreenDebugMessage(-1, 15.f, FColor::Red, NewMsg); \ -} - -#define SCREENDEBUG(Format, ...) \ -{ \ - const FString Msg = FString::Printf(TEXT(Format), ##__VA_ARGS__); \ - GEngine->AddOnScreenDebugMessage(-1, 10000.f, FColor::White, Msg); \ -} \ No newline at end of file diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBEvent.h b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBEvent.h index 7c5f2fda1f..2376caeb3a 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBEvent.h +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBEvent.h @@ -634,3 +634,14 @@ class KBENGINEPLUGINS_API UKBEventData_onImportClientSDKSuccessfully : public UK public: }; + +UCLASS(Blueprintable, BlueprintType) +class KBENGINEPLUGINS_API UKBEventData_onDownloadSDK : public UKBEventData +{ + GENERATED_BODY() + +public: + UPROPERTY(EditAnywhere, BlueprintReadWrite, Category = KBEngine) + bool isDownload; +}; + diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBEventTypes.cpp b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBEventTypes.cpp index a450a2cee4..1955033d7c 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBEventTypes.cpp +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBEventTypes.cpp @@ -1,5 +1,8 @@ #include "KBEventTypes.h" +namespace KBEngine +{ + const FString KBEventTypes::createAccount = "createAccount"; const FString KBEventTypes::login = "login"; const FString KBEventTypes::logout = "logout"; @@ -44,4 +47,7 @@ const FString KBEventTypes::onStreamDataRecv = "onStreamDataRecv"; const FString KBEventTypes::onStreamDataCompleted = "onStreamDataCompleted"; // ------------------------------------SDK更新相关------------------------------------- -const FString KBEventTypes::onImportClientSDKSuccessfully = "onImportClientSDKSuccessfully"; \ No newline at end of file +const FString KBEventTypes::onImportClientSDKSuccessfully = "onImportClientSDKSuccessfully"; +const FString KBEventTypes::onDownloadSDK = "onDownloadSDK"; + +} \ No newline at end of file diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBEventTypes.h b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBEventTypes.h index 1ce930ba3b..3e24053a6a 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBEventTypes.h +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBEventTypes.h @@ -2,6 +2,9 @@ #include "KBECommon.h" +namespace KBEngine +{ + struct KBENGINEPLUGINS_API KBEventTypes { // Create new account. @@ -158,7 +161,10 @@ struct KBENGINEPLUGINS_API KBEventTypes // param1(uint16): resouce id static const FString onStreamDataCompleted; - // ------------------------------------SDK更新相关----------------------------------- static const FString onImportClientSDKSuccessfully; -}; \ No newline at end of file + static const FString onDownloadSDK; + +}; + +} \ No newline at end of file diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBVar.cpp b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBVar.cpp index 3490e3718c..d3e9cdbd3b 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBVar.cpp +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBVar.cpp @@ -1,6 +1,10 @@ #include "KBVar.h" #include "KBDebug.h" +#include "KBECommon.h" + +namespace KBEngine +{ void KBVar::ErrorLog(const FString& errstr) const { @@ -13,3 +17,5 @@ void KBVar::ErrorLog(const FString& errstr) const GEngine->AddOnScreenDebugMessage(-1, 15.f, FColor::Red, NewMsg); } +} + diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBVar.h b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBVar.h index 5195848398..ae8c72e4aa 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBVar.h +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/KBVar.h @@ -45,6 +45,9 @@ namespace EKBVarTypes * * @param T The type to be used in KBVar. */ +namespace KBEngine +{ + template struct TKBVariantTraits { static int32 GetType() @@ -864,3 +867,4 @@ template<> struct TKBVariantTraits> static int32 GetType() { return EKBVarTypes::KBVarMap; } }; +} \ No newline at end of file diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/MemoryStream.cpp b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/MemoryStream.cpp index 9dd5a69807..cb4c10ec72 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/MemoryStream.cpp +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/MemoryStream.cpp @@ -3,6 +3,9 @@ #include "KBDebug.h" #include "ObjectPool.h" +namespace KBEngine +{ + static ObjectPool _g_memoryStreamPool; MemoryStream* MemoryStream::createObject() @@ -57,3 +60,5 @@ void MemoryStream::hexlike() rpos_ = trpos; } + +} \ No newline at end of file diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/MemoryStream.h b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/MemoryStream.h index 8ca1e99d59..c53b89bad9 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/MemoryStream.h +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/MemoryStream.h @@ -39,6 +39,9 @@ namespace MemoryStreamConverter } } +namespace KBEngine +{ + template inline void EndianConvert(T& val) { if(!FGenericPlatformProperties::IsLittleEndian()) @@ -916,3 +919,5 @@ inline void MemoryStream::read_skip() { read_skip(); } + +} \ No newline at end of file diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/MessageReader.cpp b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/MessageReader.cpp index 6b97b3d6d8..db19cbbe5a 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/MessageReader.cpp +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/MessageReader.cpp @@ -4,6 +4,9 @@ #include "KBDebug.h" #include "MemoryStream.h" +namespace KBEngine +{ + MessageReader::MessageReader(): msgid_(0), msglen_(0), @@ -155,3 +158,5 @@ void MessageReader::process(const uint8* datas, MessageLengthEx offset, MessageL } } } + +} \ No newline at end of file diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/MessageReader.h b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/MessageReader.h index 04d2b1e917..316400b43d 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/MessageReader.h +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/MessageReader.h @@ -4,6 +4,9 @@ #include "KBECommon.h" +namespace KBEngine +{ + class MemoryStream; /* @@ -43,3 +46,5 @@ class KBENGINEPLUGINS_API MessageReader MemoryStream* pMemoryStream_; }; + +} \ No newline at end of file diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/Method.cpp b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/Method.cpp index b8a0b50542..8b4ba327b8 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/Method.cpp +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/Method.cpp @@ -3,6 +3,9 @@ #include "DataTypes.h" #include "KBDebug.h" +namespace KBEngine +{ + Method::Method(): name(TEXT("")), methodUtype(0), @@ -14,3 +17,5 @@ Method::Method(): Method::~Method() { } + +} \ No newline at end of file diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/Method.h b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/Method.h index 7eb9c9bfbc..5b432bfe27 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/Method.h +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/Method.h @@ -4,6 +4,9 @@ #include "KBVar.h" #include "KBECommon.h" +namespace KBEngine +{ + class DATATYPE_BASE; /* @@ -24,3 +27,5 @@ class KBENGINEPLUGINS_API Method TArray args; }; + +} \ No newline at end of file diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/NetworkInterfaceBase.cpp b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/NetworkInterfaceBase.cpp index 2e11c969a7..3f243aa93e 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/NetworkInterfaceBase.cpp +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/NetworkInterfaceBase.cpp @@ -8,6 +8,9 @@ #include "Interfaces.h" #include "KBEngine.h" +namespace KBEngine +{ + NetworkInterfaceBase::NetworkInterfaceBase(): socket_(NULL), pPacketSender_(NULL), @@ -201,3 +204,5 @@ void NetworkInterfaceBase::tickConnecting() } } } + +} \ No newline at end of file diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/NetworkInterfaceBase.h b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/NetworkInterfaceBase.h index 89e880c35f..2ecd10d54d 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/NetworkInterfaceBase.h +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/NetworkInterfaceBase.h @@ -10,19 +10,23 @@ #include "Runtime/Sockets/Public/Sockets.h" #include "EncryptionFilter.h" -class PacketSenderBase; -class PacketReceiverBase; -class MemoryStream; -class InterfaceConnect; + /* 网络模块 处理连接、收发数据 */ +namespace KBEngine +{ + +class PacketSenderBase; +class PacketReceiverBase; +class MemoryStream; +class InterfaceConnect; class KBENGINEPLUGINS_API NetworkInterfaceBase { public: - NetworkInterfaceBase(); + NetworkInterfaceBase(); virtual ~NetworkInterfaceBase(); const FString UDP_HELLO = TEXT("62a559f3fa7748bc22f8e0766019d498"); @@ -79,3 +83,5 @@ class KBENGINEPLUGINS_API NetworkInterfaceBase EncryptionFilter *pFilter_; }; + +} \ No newline at end of file diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/NetworkInterfaceKCP.cpp b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/NetworkInterfaceKCP.cpp index 4b4db912d5..ce3bdab111 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/NetworkInterfaceKCP.cpp +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/NetworkInterfaceKCP.cpp @@ -11,6 +11,9 @@ #include "PacketSenderKCP.h" #include "PacketReceiverKCP.h" +namespace KBEngine +{ + NetworkInterfaceKCP::NetworkInterfaceKCP(): NetworkInterfaceBase(), pKCP_(NULL), @@ -197,3 +200,5 @@ void NetworkInterfaceKCP::process() NetworkInterfaceBase::process(); } + +} \ No newline at end of file diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/NetworkInterfaceKCP.h b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/NetworkInterfaceKCP.h index d81a07365a..b879e74b21 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/NetworkInterfaceKCP.h +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/NetworkInterfaceKCP.h @@ -15,6 +15,9 @@ 网络模块 处理连接、收发数据 */ +namespace KBEngine +{ + class KBENGINEPLUGINS_API NetworkInterfaceKCP : public NetworkInterfaceBase { public: @@ -58,3 +61,5 @@ class KBENGINEPLUGINS_API NetworkInterfaceKCP : public NetworkInterfaceBase TSharedRef addr_; }; + +} \ No newline at end of file diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/NetworkInterfaceTCP.cpp b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/NetworkInterfaceTCP.cpp index 850fd76d94..fc07835817 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/NetworkInterfaceTCP.cpp +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/NetworkInterfaceTCP.cpp @@ -8,6 +8,9 @@ #include "PacketSenderTCP.h" #include "PacketReceiverTCP.h" +namespace KBEngine +{ + NetworkInterfaceTCP::NetworkInterfaceTCP(): NetworkInterfaceBase() { @@ -27,3 +30,4 @@ PacketReceiverBase* NetworkInterfaceTCP::createPacketReceiver() return new PacketReceiverTCP(this); } +} diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/NetworkInterfaceTCP.h b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/NetworkInterfaceTCP.h index f808353254..dd722be697 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/NetworkInterfaceTCP.h +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/NetworkInterfaceTCP.h @@ -14,6 +14,9 @@ 网络模块 处理连接、收发数据 */ +namespace KBEngine +{ + class KBENGINEPLUGINS_API NetworkInterfaceTCP : public NetworkInterfaceBase { public: @@ -25,3 +28,5 @@ class KBENGINEPLUGINS_API NetworkInterfaceTCP : public NetworkInterfaceBase PacketReceiverBase* createPacketReceiver() override; }; + +} diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/ObjectPool.h b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/ObjectPool.h index 46492399ef..2709bca5f4 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/ObjectPool.h +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/ObjectPool.h @@ -1,5 +1,8 @@ #pragma once +namespace KBEngine +{ + template class ObjectPool { @@ -42,3 +45,5 @@ class ObjectPool private: OBJECT_LIST objects_; }; + +} \ No newline at end of file diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketReceiverBase.cpp b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketReceiverBase.cpp index d06a14f745..3780e9101f 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketReceiverBase.cpp +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketReceiverBase.cpp @@ -6,6 +6,9 @@ #include "KBDebug.h" #include "MemoryStream.h" +namespace KBEngine +{ + PacketReceiverBase::PacketReceiverBase(NetworkInterfaceBase* pNetworkInterface): pNetworkInterface_(pNetworkInterface), pMessageReader_(new MessageReader()), @@ -24,3 +27,5 @@ PacketReceiverBase::~PacketReceiverBase() void PacketReceiverBase::process() { } + +} diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketReceiverBase.h b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketReceiverBase.h index 98a503d127..c57bbeaab8 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketReceiverBase.h +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketReceiverBase.h @@ -4,6 +4,9 @@ #include "KBECommon.h" +namespace KBEngine +{ + class NetworkInterfaceBase; class MessageReader; class MemoryStream; @@ -26,3 +29,5 @@ class KBENGINEPLUGINS_API PacketReceiverBase MessageReader* pMessageReader_; MemoryStream* pBuffer_; }; + +} \ No newline at end of file diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketReceiverKCP.cpp b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketReceiverKCP.cpp index f19125a193..5645da3a73 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketReceiverKCP.cpp +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketReceiverKCP.cpp @@ -6,6 +6,9 @@ #include "KBDebug.h" #include "MemoryStream.h" +namespace KBEngine +{ + PacketReceiverKCP::PacketReceiverKCP(NetworkInterfaceBase* pNetworkInterface): PacketReceiverBase(pNetworkInterface), remoteAddr_(ISocketSubsystem::Get(PLATFORM_SOCKETSUBSYSTEM)->CreateInternetAddr()) @@ -67,4 +70,6 @@ void PacketReceiverKCP::process() } } } +} + } \ No newline at end of file diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketReceiverKCP.h b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketReceiverKCP.h index 18b5bbf329..39684b81b5 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketReceiverKCP.h +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketReceiverKCP.h @@ -6,6 +6,9 @@ #include "PacketReceiverBase.h" #include "Runtime/Sockets/Public/SocketSubsystem.h" +namespace KBEngine +{ + class NetworkInterfaceBase; class MessageReader; class MemoryStream; @@ -26,3 +29,5 @@ class KBENGINEPLUGINS_API PacketReceiverKCP : public PacketReceiverBase protected: TSharedRef remoteAddr_; }; + +} diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketReceiverTCP.cpp b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketReceiverTCP.cpp index da6e515d1c..f4d92af84f 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketReceiverTCP.cpp +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketReceiverTCP.cpp @@ -6,6 +6,9 @@ #include "KBDebug.h" #include "MemoryStream.h" +namespace KBEngine +{ + PacketReceiverTCP::PacketReceiverTCP(NetworkInterfaceBase* pNetworkInterface): PacketReceiverBase(pNetworkInterface) { @@ -39,4 +42,6 @@ void PacketReceiverTCP::process() } } } +} + } \ No newline at end of file diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketReceiverTCP.h b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketReceiverTCP.h index f3c9c918a3..c8c8b5d6b6 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketReceiverTCP.h +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketReceiverTCP.h @@ -5,6 +5,9 @@ #include "KBECommon.h" #include "PacketReceiverBase.h" +namespace KBEngine +{ + class NetworkInterfaceBase; class MessageReader; class MemoryStream; @@ -25,3 +28,5 @@ class KBENGINEPLUGINS_API PacketReceiverTCP : public PacketReceiverBase protected: }; + +} \ No newline at end of file diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketSenderBase.cpp b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketSenderBase.cpp index 08533e4f40..9528e3e6df 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketSenderBase.cpp +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketSenderBase.cpp @@ -4,6 +4,9 @@ #include "KBDebug.h" #include "NetworkInterfaceBase.h" +namespace KBEngine +{ + PacketSenderBase::PacketSenderBase(NetworkInterfaceBase* pNetworkInterface) : pNetworkInterface_(pNetworkInterface) { @@ -16,4 +19,6 @@ PacketSenderBase::~PacketSenderBase() bool PacketSenderBase::send(MemoryStream* pMemoryStream) { return true; +} + } \ No newline at end of file diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketSenderBase.h b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketSenderBase.h index ea683ce6ea..defa207c0b 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketSenderBase.h +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketSenderBase.h @@ -4,6 +4,9 @@ #include "KBECommon.h" +namespace KBEngine +{ + class NetworkInterfaceBase; class MemoryStream; @@ -23,3 +26,5 @@ class KBENGINEPLUGINS_API PacketSenderBase protected: NetworkInterfaceBase * pNetworkInterface_; }; + +} \ No newline at end of file diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketSenderKCP.cpp b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketSenderKCP.cpp index aa7c44a7c2..ee4ce8c42e 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketSenderKCP.cpp +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketSenderKCP.cpp @@ -4,6 +4,9 @@ #include "KBDebug.h" #include "NetworkInterfaceKCP.h" +namespace KBEngine +{ + PacketSenderKCP::PacketSenderKCP(NetworkInterfaceBase* pNetworkInterface) : PacketSenderBase(pNetworkInterface) { @@ -28,4 +31,6 @@ bool PacketSenderKCP::send(MemoryStream* pMemoryStream) } return true; +} + } \ No newline at end of file diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketSenderKCP.h b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketSenderKCP.h index 2138e6b96c..5d917de0ab 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketSenderKCP.h +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketSenderKCP.h @@ -5,6 +5,9 @@ #include "KBECommon.h" #include "PacketSenderBase.h" +namespace KBEngine +{ + class NetworkInterfaceBase; class MemoryStream; @@ -24,3 +27,5 @@ class KBENGINEPLUGINS_API PacketSenderKCP : public PacketSenderBase protected: }; + +} \ No newline at end of file diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketSenderTCP.cpp b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketSenderTCP.cpp index 792287f62a..2140cfc587 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketSenderTCP.cpp +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketSenderTCP.cpp @@ -4,6 +4,9 @@ #include "KBDebug.h" #include "NetworkInterfaceBase.h" +namespace KBEngine +{ + PacketSenderTCP::PacketSenderTCP(NetworkInterfaceBase* pNetworkInterface) : PacketSenderBase(pNetworkInterface) { @@ -17,4 +20,6 @@ bool PacketSenderTCP::send(MemoryStream* pMemoryStream) { int32 sent = 0; return pNetworkInterface_->socket()->Send(pMemoryStream->data() + pMemoryStream->rpos(), pMemoryStream->length(), sent); +} + } \ No newline at end of file diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketSenderTCP.h b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketSenderTCP.h index 04d127b9ca..71c488d41b 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketSenderTCP.h +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/PacketSenderTCP.h @@ -5,6 +5,9 @@ #include "KBECommon.h" #include "PacketSenderBase.h" +namespace KBEngine +{ + class NetworkInterfaceBase; class MemoryStream; @@ -24,3 +27,5 @@ class KBENGINEPLUGINS_API PacketSenderTCP : public PacketSenderBase protected: }; + +} \ No newline at end of file diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/Property.cpp b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/Property.cpp index 85cefca820..d65e39caf7 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/Property.cpp +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/Property.cpp @@ -2,6 +2,9 @@ #include "Property.h" #include "KBDebug.h" +namespace KBEngine +{ + Property::Property(): name(TEXT("")), pUtype(NULL), @@ -17,3 +20,5 @@ Property::~Property() { KBE_SAFE_RELEASE(pDefaultVal); } + +} \ No newline at end of file diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/Property.h b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/Property.h index 6d42d18c9d..62463d28b0 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/Property.h +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/Property.h @@ -4,6 +4,10 @@ #include "KBVar.h" #include "KBECommon.h" + +namespace KBEngine +{ + class DATATYPE_BASE; /* @@ -45,3 +49,5 @@ class KBENGINEPLUGINS_API Property KBVar* pDefaultVal; }; + +} \ No newline at end of file diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/ScriptModule.cpp b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/ScriptModule.cpp index 181ed022c3..9562014dd2 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/ScriptModule.cpp +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/ScriptModule.cpp @@ -6,6 +6,9 @@ #include "EntityDef.h" #include "KBDebug.h" +namespace KBEngine +{ + ScriptModule::ScriptModule(const FString& moduleName, int type): name(moduleName), usePropertyDescrAlias(false), @@ -30,3 +33,5 @@ Entity* ScriptModule::createEntity() { return EntityDef::createEntity(utype); } + +} \ No newline at end of file diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/ScriptModule.h b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/ScriptModule.h index 78efec249c..bd81dbce3c 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/ScriptModule.h +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/Engine/ScriptModule.h @@ -3,6 +3,10 @@ #pragma once #include "KBECommon.h" + +namespace KBEngine +{ + class Property; class Method; class Entity; @@ -37,3 +41,5 @@ class KBENGINEPLUGINS_API ScriptModule uint16 utype; }; + +} diff --git a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/KBEnginePlugins.Build.cs b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/KBEnginePlugins.Build.cs index a524d68b05..6dae3152b7 100644 --- a/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/KBEnginePlugins.Build.cs +++ b/kbe/res/sdk_templates/client/ue4/Source/KBEnginePlugins/KBEnginePlugins.Build.cs @@ -12,19 +12,9 @@ public KBEnginePlugins(ReadOnlyTargetRules Target) : base(Target) bEnableUndefinedIdentifierWarnings = false; - string CryptoPPPath = Target.UEThirdPartySourceDirectory + "CryptoPP/5.6.5/lib/"; - string[] PrivateModules = new string[] { "CoreUObject", "Engine", "Slate", "SlateCore", "Networking", "Sockets" }; - - if (Target.Platform == UnrealTargetPlatform.Win64 && Directory.Exists(CryptoPPPath)) - { - List PrivateModuleList = new List(PrivateModules); - PrivateModuleList.Add("CryptoPP"); - PrivateModules = PrivateModuleList.ToArray(); - } - else - { - PublicDefinitions.Add("KBENGINE_NO_CRYPTO"); - } + string[] PrivateModules = new string[] { "Slate", "SlateCore", "Networking", "Sockets", "OpenSSL" }; + string[] PublicModules = new string[] { "Core", "CoreUObject", "Engine"}; + List PublicModulesList = new List(PublicModules); PublicIncludePaths.AddRange( new string[] { @@ -39,14 +29,14 @@ public KBEnginePlugins(ReadOnlyTargetRules Target) : base(Target) "KBEnginePlugins/Scripts", } ); - - PublicDependencyModuleNames.AddRange( - new string[] - { - "Core", - // ... add other public dependencies that you statically link with here ... - } - ); + + if (Target.bBuildEditor) + { + PublicModulesList.Add("UnrealEd"); + } + + PublicModules = PublicModulesList.ToArray(); + PublicDependencyModuleNames.AddRange(PublicModules); PrivateDependencyModuleNames.AddRange(PrivateModules); @@ -55,6 +45,6 @@ public KBEnginePlugins(ReadOnlyTargetRules Target) : base(Target) { // ... add any modules that your module loads dynamically here ... } - ); + ); } } diff --git a/kbe/src/kbengine.sln b/kbe/src/kbengine.sln index 8c000a7611..c3ec2eacbc 100644 --- a/kbe/src/kbengine.sln +++ b/kbe/src/kbengine.sln @@ -1,707 +1,696 @@ - -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 15 -VisualStudioVersion = 15.0.27703.2047 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "dependencies", "dependencies", "{8BF90F4F-766B-4B4D-B7F3-BF4189A29F4E}" -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "libs", "libs", "{C57D8946-CC54-4360-89FA-DAC04E93423A}" -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "srv-apps", "srv-apps", "{89F7F2F7-EB4D-4810-AA06-A96D592678E6}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "entitydef", "lib\entitydef\entitydef.vcxproj", "{44FA54E8-0A60-4A6C-A449-118872E75502}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "helper", "lib\helper\helper.vcxproj", "{0E032FA8-BB7B-40F6-8CB1-15F204113B24}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "math", "lib\math\math.vcxproj", "{F67B2C56-D1B1-4EA7-B16E-EF8E7F1B6C5F}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pyscript", "lib\pyscript\pyscript.vcxproj", "{7A411D5D-DA3B-474D-B08B-3DD787A1F375}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "xml", "lib\xml\xml.vcxproj", "{2992ABDB-0853-4C43-B9E2-98D4211192EE}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "server", "lib\server\server.vcxproj", "{56D453A7-787B-49E3-84BC-465EED38DE8A}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "network", "lib\network\network.vcxproj", "{5EF24499-4F74-4AF6-8048-650BE7BD7808}" - ProjectSection(ProjectDependencies) = postProject - {70ABE75C-FBC2-4C82-9103-B952FA5CD168} = {70ABE75C-FBC2-4C82-9103-B952FA5CD168} - {DA6F56B4-06A4-441D-AD70-AC5A7D51FADB} = {DA6F56B4-06A4-441D-AD70-AC5A7D51FADB} - {24265BD3-3CF3-41A9-B137-53DBEC0F53BF} = {24265BD3-3CF3-41A9-B137-53DBEC0F53BF} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "thread", "lib\thread\thread.vcxproj", "{863B8043-533C-499D-B48E-F5EC91A6CB9B}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "cellapp", "server\cellapp\cellapp.vcxproj", "{04C620E9-BE02-498E-A713-B9E470D75F15}" - ProjectSection(ProjectDependencies) = postProject - {745DEC58-EBB3-47A9-A9B8-4C6627C01BF8} = {745DEC58-EBB3-47A9-A9B8-4C6627C01BF8} - {42FC1F89-7136-45E8-94E3-594C173C2937} = {42FC1F89-7136-45E8-94E3-594C173C2937} - {896616D2-A5D2-4548-859E-D59905F46140} = {896616D2-A5D2-4548-859E-D59905F46140} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "machine", "server\machine\machine.vcxproj", "{89DA21A1-625B-4482-B386-D7C0AF97E477}" - ProjectSection(ProjectDependencies) = postProject - {561FA9F7-14D0-4D94-822C-AAE9B08AACEF} = {561FA9F7-14D0-4D94-822C-AAE9B08AACEF} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "baseappmgr", "server\baseappmgr\baseappmgr.vcxproj", "{1BAD0518-F2BB-42F5-83E7-9E1804FAD7C1}" - ProjectSection(ProjectDependencies) = postProject - {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26} = {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26} - {561FA9F7-14D0-4D94-822C-AAE9B08AACEF} = {561FA9F7-14D0-4D94-822C-AAE9B08AACEF} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "cellappmgr", "server\cellappmgr\cellappmgr.vcxproj", "{FE88370E-80F7-49BD-A8D1-E7A2CA699E44}" - ProjectSection(ProjectDependencies) = postProject - {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26} = {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26} - {561FA9F7-14D0-4D94-822C-AAE9B08AACEF} = {561FA9F7-14D0-4D94-822C-AAE9B08AACEF} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "dbmgr", "server\dbmgr\dbmgr.vcxproj", "{9EA2E328-E6BE-4DF0-A8DC-05C08A0B822B}" - ProjectSection(ProjectDependencies) = postProject - {13E85053-54B3-487B-8DDB-3430B1C1B3BF} = {13E85053-54B3-487B-8DDB-3430B1C1B3BF} - {70E890EA-9496-4F7C-B903-5FCF87A73AC4} = {70E890EA-9496-4F7C-B903-5FCF87A73AC4} - {561FA9F7-14D0-4D94-822C-AAE9B08AACEF} = {561FA9F7-14D0-4D94-822C-AAE9B08AACEF} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "baseapp", "server\baseapp\baseapp.vcxproj", "{69C18C83-5753-4F36-BF5A-6EE217304D2D}" - ProjectSection(ProjectDependencies) = postProject - {45A08C1B-D249-42C4-932C-615ED3924CBB} = {45A08C1B-D249-42C4-932C-615ED3924CBB} - {745DEC58-EBB3-47A9-A9B8-4C6627C01BF8} = {745DEC58-EBB3-47A9-A9B8-4C6627C01BF8} - {896616D2-A5D2-4548-859E-D59905F46140} = {896616D2-A5D2-4548-859E-D59905F46140} - {561FA9F7-14D0-4D94-822C-AAE9B08AACEF} = {561FA9F7-14D0-4D94-822C-AAE9B08AACEF} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "loginapp", "server\loginapp\loginapp.vcxproj", "{4AF743A4-8C2F-4424-BB4F-A5B0B5C128BD}" - ProjectSection(ProjectDependencies) = postProject - {896616D2-A5D2-4548-859E-D59905F46140} = {896616D2-A5D2-4548-859E-D59905F46140} - {561FA9F7-14D0-4D94-822C-AAE9B08AACEF} = {561FA9F7-14D0-4D94-822C-AAE9B08AACEF} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bots", "server\tools\bots\bots.vcxproj", "{E17527D0-CCA7-4063-924B-C09A1BD43EC3}" - ProjectSection(ProjectDependencies) = postProject - {561FA9F7-14D0-4D94-822C-AAE9B08AACEF} = {561FA9F7-14D0-4D94-822C-AAE9B08AACEF} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "client_lib", "lib\client_lib\client_lib.vcxproj", "{896616D2-A5D2-4548-859E-D59905F46140}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "guiconsole", "server\tools\guiconsole\guiconsole.vcxproj", "{78EE8B08-4E07-4C07-9592-BC4D09503155}" - ProjectSection(ProjectDependencies) = postProject - {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26} = {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26} - {561FA9F7-14D0-4D94-822C-AAE9B08AACEF} = {561FA9F7-14D0-4D94-822C-AAE9B08AACEF} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "resmgr", "lib\resmgr\resmgr.vcxproj", "{3F8B2057-492F-4332-99DE-3E12F20ED489}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "db_interface", "lib\db_interface\db_interface.vcxproj", "{6C92BA78-CFAA-4524-A636-F044B0280AB0}" - ProjectSection(ProjectDependencies) = postProject - {24CB1A43-C6E1-442E-AF99-A91D26BA8FA0} = {24CB1A43-C6E1-442E-AF99-A91D26BA8FA0} - {70E890EA-9496-4F7C-B903-5FCF87A73AC4} = {70E890EA-9496-4F7C-B903-5FCF87A73AC4} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "db_mysql", "lib\db_mysql\db_mysql.vcxproj", "{24CB1A43-C6E1-442E-AF99-A91D26BA8FA0}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "logger", "server\tools\logger\logger.vcxproj", "{E999D336-32D1-48CC-A71F-FC552BD5093B}" - ProjectSection(ProjectDependencies) = postProject - {7A411D5D-DA3B-474D-B08B-3DD787A1F375} = {7A411D5D-DA3B-474D-B08B-3DD787A1F375} - {561FA9F7-14D0-4D94-822C-AAE9B08AACEF} = {561FA9F7-14D0-4D94-822C-AAE9B08AACEF} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "interfaces", "server\tools\interfaces\interfaces.vcxproj", "{7817D13F-1FAA-47D8-8C9F-B790FE2D6B0A}" - ProjectSection(ProjectDependencies) = postProject - {7A411D5D-DA3B-474D-B08B-3DD787A1F375} = {7A411D5D-DA3B-474D-B08B-3DD787A1F375} - {561FA9F7-14D0-4D94-822C-AAE9B08AACEF} = {561FA9F7-14D0-4D94-822C-AAE9B08AACEF} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "jwsmtp", "lib\dependencies\jwsmtp\jwsmtp.vcxproj", "{45A08C1B-D249-42C4-932C-615ED3924CBB}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "navigation", "lib\navigation\navigation.vcxproj", "{9085A36E-CAF8-4E86-93AF-373D2554026B}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libeay32", "lib\dependencies\openssl\vcbuild\libeay32\libeay32.vcxproj", "{70ABE75C-FBC2-4C82-9103-B952FA5CD168}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ssleay32", "lib\dependencies\openssl\vcbuild\ssleay32\ssleay32.vcxproj", "{24265BD3-3CF3-41A9-B137-53DBEC0F53BF}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "apr", "lib\dependencies\apr\apr.vcxproj", "{C6C836E6-2899-458A-90BA-184192342D30}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "aprutil", "lib\dependencies\apr-util\aprutil.vcxproj", "{7CDBBFFF-3B84-413E-BA1A-47CB8D454A46}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "log4cxx", "lib\dependencies\log4cxx\projects\log4cxx.vcxproj", "{561FA9F7-14D0-4D94-822C-AAE9B08AACEF}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "tmxparser", "lib\dependencies\tmxparser\tmxparser\tmxparser.vcxproj", "{42FC1F89-7136-45E8-94E3-594C173C2937}" -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "log4cxx", "log4cxx", "{67707698-A79D-4C72-8F99-3B96BA970D0D}" -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "openssl", "openssl", "{B82CBA7A-6640-47BD-94C9-9EEE37CECBE0}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "expat", "lib\dependencies\expat\lib\expat_static.vcxproj", "{30042FAB-AD01-BF78-3C79-F1F6316BC947}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "common", "lib\common\common.vcxproj", "{A04109A7-46C9-42F9-AB29-8E3D84450172}" -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "srv-tools", "srv-tools", "{54612744-437F-4400-AE6C-BE4CD68C8507}" -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "python", "python", "{0E68FE27-FE42-4AA2-BE7A-C267B8F5FD2E}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "hiredis", "lib\dependencies\hiredis\redis-win\msvs\hiredis\hiredis.vcxproj", "{13E85053-54B3-487B-8DDB-3430B1C1B3BF}" -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "hiredis", "hiredis", "{F4FD40AD-244F-4162-88D7-FFD4722FAFAE}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "db_redis", "lib\db_redis\db_redis.vcxproj", "{70E890EA-9496-4F7C-B903-5FCF87A73AC4}" - ProjectSection(ProjectDependencies) = postProject - {13E85053-54B3-487B-8DDB-3430B1C1B3BF} = {13E85053-54B3-487B-8DDB-3430B1C1B3BF} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "kbcmd", "server\tools\kbcmd\kbcmd.vcxproj", "{938D17CB-3F34-46A4-AF64-2099B7FF6902}" - ProjectSection(ProjectDependencies) = postProject - {7A411D5D-DA3B-474D-B08B-3DD787A1F375} = {7A411D5D-DA3B-474D-B08B-3DD787A1F375} - {44FA54E8-0A60-4A6C-A449-118872E75502} = {44FA54E8-0A60-4A6C-A449-118872E75502} - EndProjectSection -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fmt", "lib\dependencies\fmt\fmt.vcxproj", "{13E95053-54B3-497B-9DDB-3430B1C1B6BF}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libcurl", "lib\dependencies\curl\projects\Windows\VC14\lib\libcurl.vcxproj", "{DA6F56B4-06A4-441D-AD70-AC5A7D51FADB}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pythoncore", "lib\python\PCbuild\pythoncore.vcxproj", "{CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}" - ProjectSection(ProjectDependencies) = postProject - {745DEC58-EBB3-47A9-A9B8-4C6627C01BF8} = {745DEC58-EBB3-47A9-A9B8-4C6627C01BF8} - {70ABE75C-FBC2-4C82-9103-B952FA5CD168} = {70ABE75C-FBC2-4C82-9103-B952FA5CD168} - {24265BD3-3CF3-41A9-B137-53DBEC0F53BF} = {24265BD3-3CF3-41A9-B137-53DBEC0F53BF} - EndProjectSection -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "pyd", "pyd", "{04274E83-D752-4CC9-BFDE-A717D4E6ABAA}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_socket", "lib\python\PCbuild\_socket.vcxproj", "{86937F53-C189-40EF-8CE8-8759D8E7D480}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "select", "lib\python\PCbuild\select.vcxproj", "{18CAE28C-B454-46C1-87A0-493D91D97F03}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_ctypes", "lib\python\PCbuild\_ctypes.vcxproj", "{0E9791DB-593A-465F-98BC-681011311618}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_elementtree", "lib\python\PCbuild\_elementtree.vcxproj", "{17E1E049-C309-4D79-843F-AE483C264AEA}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pyexpat", "lib\python\PCbuild\pyexpat.vcxproj", "{D06B6426-4762-44CC-8BAD-D79052507F2F}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "unicodedata", "lib\python\PCbuild\unicodedata.vcxproj", "{ECC7CEAC-A5E5-458E-BB9E-2413CC847881}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_ssl", "lib\python\PCbuild\_ssl.vcxproj", "{C6E20F84-3247-4AD6-B051-B073268F73BA}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_hashlib", "lib\python\PCbuild\_hashlib.vcxproj", "{447F05A8-F581-4CAC-A466-5AC7936E207E}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zlib", "lib\dependencies\zlib\contrib\vstudio\vc14\zlibstat.vcxproj", "{745DEC58-EBB3-47A9-A9B8-4C6627C01BF8}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_asyncio", "lib\python\PCbuild\_asyncio.vcxproj", "{384C224A-7474-476E-A01B-750EA7DE918C}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_contextvars", "lib\python\PCbuild\_contextvars.vcxproj", "{B8BF1D81-09DC-42D4-B406-4F868B33A89E}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_decimal", "lib\python\PCbuild\_decimal.vcxproj", "{0E9791DB-593A-465F-98BC-681011311617}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_multiprocessing", "lib\python\PCbuild\_multiprocessing.vcxproj", "{9E48B300-37D1-11DD-8C41-005056C00008}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_overlapped", "lib\python\PCbuild\_overlapped.vcxproj", "{EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}" -EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_queue", "lib\python\PCbuild\_queue.vcxproj", "{78D80A15-BD8C-44E2-B49E-1F05B0A0A687}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Win32 = Debug|Win32 - Debug|Win64 = Debug|Win64 - Release|Win32 = Release|Win32 - Release|Win64 = Release|Win64 - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {44FA54E8-0A60-4A6C-A449-118872E75502}.Debug|Win32.ActiveCfg = Debug|Win32 - {44FA54E8-0A60-4A6C-A449-118872E75502}.Debug|Win32.Build.0 = Debug|Win32 - {44FA54E8-0A60-4A6C-A449-118872E75502}.Debug|Win64.ActiveCfg = Debug|x64 - {44FA54E8-0A60-4A6C-A449-118872E75502}.Debug|Win64.Build.0 = Debug|x64 - {44FA54E8-0A60-4A6C-A449-118872E75502}.Release|Win32.ActiveCfg = Release|Win32 - {44FA54E8-0A60-4A6C-A449-118872E75502}.Release|Win32.Build.0 = Release|Win32 - {44FA54E8-0A60-4A6C-A449-118872E75502}.Release|Win64.ActiveCfg = Release|x64 - {44FA54E8-0A60-4A6C-A449-118872E75502}.Release|Win64.Build.0 = Release|x64 - {0E032FA8-BB7B-40F6-8CB1-15F204113B24}.Debug|Win32.ActiveCfg = Debug|Win32 - {0E032FA8-BB7B-40F6-8CB1-15F204113B24}.Debug|Win32.Build.0 = Debug|Win32 - {0E032FA8-BB7B-40F6-8CB1-15F204113B24}.Debug|Win64.ActiveCfg = Debug|x64 - {0E032FA8-BB7B-40F6-8CB1-15F204113B24}.Debug|Win64.Build.0 = Debug|x64 - {0E032FA8-BB7B-40F6-8CB1-15F204113B24}.Release|Win32.ActiveCfg = Release|Win32 - {0E032FA8-BB7B-40F6-8CB1-15F204113B24}.Release|Win32.Build.0 = Release|Win32 - {0E032FA8-BB7B-40F6-8CB1-15F204113B24}.Release|Win64.ActiveCfg = Release|x64 - {0E032FA8-BB7B-40F6-8CB1-15F204113B24}.Release|Win64.Build.0 = Release|x64 - {F67B2C56-D1B1-4EA7-B16E-EF8E7F1B6C5F}.Debug|Win32.ActiveCfg = Debug|Win32 - {F67B2C56-D1B1-4EA7-B16E-EF8E7F1B6C5F}.Debug|Win32.Build.0 = Debug|Win32 - {F67B2C56-D1B1-4EA7-B16E-EF8E7F1B6C5F}.Debug|Win64.ActiveCfg = Debug|x64 - {F67B2C56-D1B1-4EA7-B16E-EF8E7F1B6C5F}.Debug|Win64.Build.0 = Debug|x64 - {F67B2C56-D1B1-4EA7-B16E-EF8E7F1B6C5F}.Release|Win32.ActiveCfg = Release|Win32 - {F67B2C56-D1B1-4EA7-B16E-EF8E7F1B6C5F}.Release|Win32.Build.0 = Release|Win32 - {F67B2C56-D1B1-4EA7-B16E-EF8E7F1B6C5F}.Release|Win64.ActiveCfg = Release|x64 - {F67B2C56-D1B1-4EA7-B16E-EF8E7F1B6C5F}.Release|Win64.Build.0 = Release|x64 - {7A411D5D-DA3B-474D-B08B-3DD787A1F375}.Debug|Win32.ActiveCfg = Debug|Win32 - {7A411D5D-DA3B-474D-B08B-3DD787A1F375}.Debug|Win32.Build.0 = Debug|Win32 - {7A411D5D-DA3B-474D-B08B-3DD787A1F375}.Debug|Win64.ActiveCfg = Debug|x64 - {7A411D5D-DA3B-474D-B08B-3DD787A1F375}.Debug|Win64.Build.0 = Debug|x64 - {7A411D5D-DA3B-474D-B08B-3DD787A1F375}.Release|Win32.ActiveCfg = Release|Win32 - {7A411D5D-DA3B-474D-B08B-3DD787A1F375}.Release|Win32.Build.0 = Release|Win32 - {7A411D5D-DA3B-474D-B08B-3DD787A1F375}.Release|Win64.ActiveCfg = Release|x64 - {7A411D5D-DA3B-474D-B08B-3DD787A1F375}.Release|Win64.Build.0 = Release|x64 - {2992ABDB-0853-4C43-B9E2-98D4211192EE}.Debug|Win32.ActiveCfg = Debug|Win32 - {2992ABDB-0853-4C43-B9E2-98D4211192EE}.Debug|Win32.Build.0 = Debug|Win32 - {2992ABDB-0853-4C43-B9E2-98D4211192EE}.Debug|Win64.ActiveCfg = Debug|x64 - {2992ABDB-0853-4C43-B9E2-98D4211192EE}.Debug|Win64.Build.0 = Debug|x64 - {2992ABDB-0853-4C43-B9E2-98D4211192EE}.Release|Win32.ActiveCfg = Release|Win32 - {2992ABDB-0853-4C43-B9E2-98D4211192EE}.Release|Win32.Build.0 = Release|Win32 - {2992ABDB-0853-4C43-B9E2-98D4211192EE}.Release|Win64.ActiveCfg = Release|x64 - {2992ABDB-0853-4C43-B9E2-98D4211192EE}.Release|Win64.Build.0 = Release|x64 - {56D453A7-787B-49E3-84BC-465EED38DE8A}.Debug|Win32.ActiveCfg = Debug|Win32 - {56D453A7-787B-49E3-84BC-465EED38DE8A}.Debug|Win32.Build.0 = Debug|Win32 - {56D453A7-787B-49E3-84BC-465EED38DE8A}.Debug|Win64.ActiveCfg = Debug|x64 - {56D453A7-787B-49E3-84BC-465EED38DE8A}.Debug|Win64.Build.0 = Debug|x64 - {56D453A7-787B-49E3-84BC-465EED38DE8A}.Release|Win32.ActiveCfg = Release|Win32 - {56D453A7-787B-49E3-84BC-465EED38DE8A}.Release|Win32.Build.0 = Release|Win32 - {56D453A7-787B-49E3-84BC-465EED38DE8A}.Release|Win64.ActiveCfg = Release|x64 - {56D453A7-787B-49E3-84BC-465EED38DE8A}.Release|Win64.Build.0 = Release|x64 - {5EF24499-4F74-4AF6-8048-650BE7BD7808}.Debug|Win32.ActiveCfg = Debug|Win32 - {5EF24499-4F74-4AF6-8048-650BE7BD7808}.Debug|Win32.Build.0 = Debug|Win32 - {5EF24499-4F74-4AF6-8048-650BE7BD7808}.Debug|Win64.ActiveCfg = Debug|x64 - {5EF24499-4F74-4AF6-8048-650BE7BD7808}.Debug|Win64.Build.0 = Debug|x64 - {5EF24499-4F74-4AF6-8048-650BE7BD7808}.Release|Win32.ActiveCfg = Release|Win32 - {5EF24499-4F74-4AF6-8048-650BE7BD7808}.Release|Win32.Build.0 = Release|Win32 - {5EF24499-4F74-4AF6-8048-650BE7BD7808}.Release|Win64.ActiveCfg = Release|x64 - {5EF24499-4F74-4AF6-8048-650BE7BD7808}.Release|Win64.Build.0 = Release|x64 - {863B8043-533C-499D-B48E-F5EC91A6CB9B}.Debug|Win32.ActiveCfg = Debug|Win32 - {863B8043-533C-499D-B48E-F5EC91A6CB9B}.Debug|Win32.Build.0 = Debug|Win32 - {863B8043-533C-499D-B48E-F5EC91A6CB9B}.Debug|Win64.ActiveCfg = Debug|x64 - {863B8043-533C-499D-B48E-F5EC91A6CB9B}.Debug|Win64.Build.0 = Debug|x64 - {863B8043-533C-499D-B48E-F5EC91A6CB9B}.Release|Win32.ActiveCfg = Release|Win32 - {863B8043-533C-499D-B48E-F5EC91A6CB9B}.Release|Win32.Build.0 = Release|Win32 - {863B8043-533C-499D-B48E-F5EC91A6CB9B}.Release|Win64.ActiveCfg = Release|x64 - {863B8043-533C-499D-B48E-F5EC91A6CB9B}.Release|Win64.Build.0 = Release|x64 - {04C620E9-BE02-498E-A713-B9E470D75F15}.Debug|Win32.ActiveCfg = Debug|Win32 - {04C620E9-BE02-498E-A713-B9E470D75F15}.Debug|Win32.Build.0 = Debug|Win32 - {04C620E9-BE02-498E-A713-B9E470D75F15}.Debug|Win64.ActiveCfg = Debug|x64 - {04C620E9-BE02-498E-A713-B9E470D75F15}.Debug|Win64.Build.0 = Debug|x64 - {04C620E9-BE02-498E-A713-B9E470D75F15}.Release|Win32.ActiveCfg = Release|Win32 - {04C620E9-BE02-498E-A713-B9E470D75F15}.Release|Win32.Build.0 = Release|Win32 - {04C620E9-BE02-498E-A713-B9E470D75F15}.Release|Win64.ActiveCfg = Release|x64 - {04C620E9-BE02-498E-A713-B9E470D75F15}.Release|Win64.Build.0 = Release|x64 - {89DA21A1-625B-4482-B386-D7C0AF97E477}.Debug|Win32.ActiveCfg = Debug|Win32 - {89DA21A1-625B-4482-B386-D7C0AF97E477}.Debug|Win32.Build.0 = Debug|Win32 - {89DA21A1-625B-4482-B386-D7C0AF97E477}.Debug|Win64.ActiveCfg = Debug|x64 - {89DA21A1-625B-4482-B386-D7C0AF97E477}.Debug|Win64.Build.0 = Debug|x64 - {89DA21A1-625B-4482-B386-D7C0AF97E477}.Release|Win32.ActiveCfg = Release|Win32 - {89DA21A1-625B-4482-B386-D7C0AF97E477}.Release|Win32.Build.0 = Release|Win32 - {89DA21A1-625B-4482-B386-D7C0AF97E477}.Release|Win64.ActiveCfg = Release|x64 - {89DA21A1-625B-4482-B386-D7C0AF97E477}.Release|Win64.Build.0 = Release|x64 - {1BAD0518-F2BB-42F5-83E7-9E1804FAD7C1}.Debug|Win32.ActiveCfg = Debug|Win32 - {1BAD0518-F2BB-42F5-83E7-9E1804FAD7C1}.Debug|Win32.Build.0 = Debug|Win32 - {1BAD0518-F2BB-42F5-83E7-9E1804FAD7C1}.Debug|Win64.ActiveCfg = Debug|x64 - {1BAD0518-F2BB-42F5-83E7-9E1804FAD7C1}.Debug|Win64.Build.0 = Debug|x64 - {1BAD0518-F2BB-42F5-83E7-9E1804FAD7C1}.Release|Win32.ActiveCfg = Release|Win32 - {1BAD0518-F2BB-42F5-83E7-9E1804FAD7C1}.Release|Win32.Build.0 = Release|Win32 - {1BAD0518-F2BB-42F5-83E7-9E1804FAD7C1}.Release|Win64.ActiveCfg = Release|x64 - {1BAD0518-F2BB-42F5-83E7-9E1804FAD7C1}.Release|Win64.Build.0 = Release|x64 - {FE88370E-80F7-49BD-A8D1-E7A2CA699E44}.Debug|Win32.ActiveCfg = Debug|Win32 - {FE88370E-80F7-49BD-A8D1-E7A2CA699E44}.Debug|Win32.Build.0 = Debug|Win32 - {FE88370E-80F7-49BD-A8D1-E7A2CA699E44}.Debug|Win64.ActiveCfg = Debug|x64 - {FE88370E-80F7-49BD-A8D1-E7A2CA699E44}.Debug|Win64.Build.0 = Debug|x64 - {FE88370E-80F7-49BD-A8D1-E7A2CA699E44}.Release|Win32.ActiveCfg = Release|Win32 - {FE88370E-80F7-49BD-A8D1-E7A2CA699E44}.Release|Win32.Build.0 = Release|Win32 - {FE88370E-80F7-49BD-A8D1-E7A2CA699E44}.Release|Win64.ActiveCfg = Release|x64 - {FE88370E-80F7-49BD-A8D1-E7A2CA699E44}.Release|Win64.Build.0 = Release|x64 - {9EA2E328-E6BE-4DF0-A8DC-05C08A0B822B}.Debug|Win32.ActiveCfg = Debug|Win32 - {9EA2E328-E6BE-4DF0-A8DC-05C08A0B822B}.Debug|Win32.Build.0 = Debug|Win32 - {9EA2E328-E6BE-4DF0-A8DC-05C08A0B822B}.Debug|Win64.ActiveCfg = Debug|x64 - {9EA2E328-E6BE-4DF0-A8DC-05C08A0B822B}.Debug|Win64.Build.0 = Debug|x64 - {9EA2E328-E6BE-4DF0-A8DC-05C08A0B822B}.Release|Win32.ActiveCfg = Release|Win32 - {9EA2E328-E6BE-4DF0-A8DC-05C08A0B822B}.Release|Win32.Build.0 = Release|Win32 - {9EA2E328-E6BE-4DF0-A8DC-05C08A0B822B}.Release|Win64.ActiveCfg = Release|x64 - {9EA2E328-E6BE-4DF0-A8DC-05C08A0B822B}.Release|Win64.Build.0 = Release|x64 - {69C18C83-5753-4F36-BF5A-6EE217304D2D}.Debug|Win32.ActiveCfg = Debug|Win32 - {69C18C83-5753-4F36-BF5A-6EE217304D2D}.Debug|Win32.Build.0 = Debug|Win32 - {69C18C83-5753-4F36-BF5A-6EE217304D2D}.Debug|Win64.ActiveCfg = Debug|x64 - {69C18C83-5753-4F36-BF5A-6EE217304D2D}.Debug|Win64.Build.0 = Debug|x64 - {69C18C83-5753-4F36-BF5A-6EE217304D2D}.Release|Win32.ActiveCfg = Release|Win32 - {69C18C83-5753-4F36-BF5A-6EE217304D2D}.Release|Win32.Build.0 = Release|Win32 - {69C18C83-5753-4F36-BF5A-6EE217304D2D}.Release|Win64.ActiveCfg = Release|x64 - {69C18C83-5753-4F36-BF5A-6EE217304D2D}.Release|Win64.Build.0 = Release|x64 - {4AF743A4-8C2F-4424-BB4F-A5B0B5C128BD}.Debug|Win32.ActiveCfg = Debug|Win32 - {4AF743A4-8C2F-4424-BB4F-A5B0B5C128BD}.Debug|Win32.Build.0 = Debug|Win32 - {4AF743A4-8C2F-4424-BB4F-A5B0B5C128BD}.Debug|Win64.ActiveCfg = Debug|x64 - {4AF743A4-8C2F-4424-BB4F-A5B0B5C128BD}.Debug|Win64.Build.0 = Debug|x64 - {4AF743A4-8C2F-4424-BB4F-A5B0B5C128BD}.Release|Win32.ActiveCfg = Release|Win32 - {4AF743A4-8C2F-4424-BB4F-A5B0B5C128BD}.Release|Win32.Build.0 = Release|Win32 - {4AF743A4-8C2F-4424-BB4F-A5B0B5C128BD}.Release|Win64.ActiveCfg = Release|x64 - {4AF743A4-8C2F-4424-BB4F-A5B0B5C128BD}.Release|Win64.Build.0 = Release|x64 - {E17527D0-CCA7-4063-924B-C09A1BD43EC3}.Debug|Win32.ActiveCfg = Debug|Win32 - {E17527D0-CCA7-4063-924B-C09A1BD43EC3}.Debug|Win32.Build.0 = Debug|Win32 - {E17527D0-CCA7-4063-924B-C09A1BD43EC3}.Debug|Win64.ActiveCfg = Debug|x64 - {E17527D0-CCA7-4063-924B-C09A1BD43EC3}.Debug|Win64.Build.0 = Debug|x64 - {E17527D0-CCA7-4063-924B-C09A1BD43EC3}.Release|Win32.ActiveCfg = Release|Win32 - {E17527D0-CCA7-4063-924B-C09A1BD43EC3}.Release|Win32.Build.0 = Release|Win32 - {E17527D0-CCA7-4063-924B-C09A1BD43EC3}.Release|Win64.ActiveCfg = Release|x64 - {E17527D0-CCA7-4063-924B-C09A1BD43EC3}.Release|Win64.Build.0 = Release|x64 - {896616D2-A5D2-4548-859E-D59905F46140}.Debug|Win32.ActiveCfg = Debug|Win32 - {896616D2-A5D2-4548-859E-D59905F46140}.Debug|Win32.Build.0 = Debug|Win32 - {896616D2-A5D2-4548-859E-D59905F46140}.Debug|Win64.ActiveCfg = Debug|x64 - {896616D2-A5D2-4548-859E-D59905F46140}.Debug|Win64.Build.0 = Debug|x64 - {896616D2-A5D2-4548-859E-D59905F46140}.Release|Win32.ActiveCfg = Release|Win32 - {896616D2-A5D2-4548-859E-D59905F46140}.Release|Win32.Build.0 = Release|Win32 - {896616D2-A5D2-4548-859E-D59905F46140}.Release|Win64.ActiveCfg = Release|x64 - {896616D2-A5D2-4548-859E-D59905F46140}.Release|Win64.Build.0 = Release|x64 - {78EE8B08-4E07-4C07-9592-BC4D09503155}.Debug|Win32.ActiveCfg = Debug|Win32 - {78EE8B08-4E07-4C07-9592-BC4D09503155}.Debug|Win32.Build.0 = Debug|Win32 - {78EE8B08-4E07-4C07-9592-BC4D09503155}.Debug|Win64.ActiveCfg = Debug|x64 - {78EE8B08-4E07-4C07-9592-BC4D09503155}.Debug|Win64.Build.0 = Debug|x64 - {78EE8B08-4E07-4C07-9592-BC4D09503155}.Release|Win32.ActiveCfg = Release|Win32 - {78EE8B08-4E07-4C07-9592-BC4D09503155}.Release|Win32.Build.0 = Release|Win32 - {78EE8B08-4E07-4C07-9592-BC4D09503155}.Release|Win64.ActiveCfg = Release|x64 - {78EE8B08-4E07-4C07-9592-BC4D09503155}.Release|Win64.Build.0 = Release|x64 - {3F8B2057-492F-4332-99DE-3E12F20ED489}.Debug|Win32.ActiveCfg = Debug|Win32 - {3F8B2057-492F-4332-99DE-3E12F20ED489}.Debug|Win32.Build.0 = Debug|Win32 - {3F8B2057-492F-4332-99DE-3E12F20ED489}.Debug|Win64.ActiveCfg = Debug|x64 - {3F8B2057-492F-4332-99DE-3E12F20ED489}.Debug|Win64.Build.0 = Debug|x64 - {3F8B2057-492F-4332-99DE-3E12F20ED489}.Release|Win32.ActiveCfg = Release|Win32 - {3F8B2057-492F-4332-99DE-3E12F20ED489}.Release|Win32.Build.0 = Release|Win32 - {3F8B2057-492F-4332-99DE-3E12F20ED489}.Release|Win64.ActiveCfg = Release|x64 - {3F8B2057-492F-4332-99DE-3E12F20ED489}.Release|Win64.Build.0 = Release|x64 - {6C92BA78-CFAA-4524-A636-F044B0280AB0}.Debug|Win32.ActiveCfg = Debug|Win32 - {6C92BA78-CFAA-4524-A636-F044B0280AB0}.Debug|Win32.Build.0 = Debug|Win32 - {6C92BA78-CFAA-4524-A636-F044B0280AB0}.Debug|Win64.ActiveCfg = Debug|x64 - {6C92BA78-CFAA-4524-A636-F044B0280AB0}.Debug|Win64.Build.0 = Debug|x64 - {6C92BA78-CFAA-4524-A636-F044B0280AB0}.Release|Win32.ActiveCfg = Release|Win32 - {6C92BA78-CFAA-4524-A636-F044B0280AB0}.Release|Win32.Build.0 = Release|Win32 - {6C92BA78-CFAA-4524-A636-F044B0280AB0}.Release|Win64.ActiveCfg = Release|x64 - {6C92BA78-CFAA-4524-A636-F044B0280AB0}.Release|Win64.Build.0 = Release|x64 - {24CB1A43-C6E1-442E-AF99-A91D26BA8FA0}.Debug|Win32.ActiveCfg = Debug|Win32 - {24CB1A43-C6E1-442E-AF99-A91D26BA8FA0}.Debug|Win32.Build.0 = Debug|Win32 - {24CB1A43-C6E1-442E-AF99-A91D26BA8FA0}.Debug|Win64.ActiveCfg = Debug|x64 - {24CB1A43-C6E1-442E-AF99-A91D26BA8FA0}.Debug|Win64.Build.0 = Debug|x64 - {24CB1A43-C6E1-442E-AF99-A91D26BA8FA0}.Release|Win32.ActiveCfg = Release|Win32 - {24CB1A43-C6E1-442E-AF99-A91D26BA8FA0}.Release|Win32.Build.0 = Release|Win32 - {24CB1A43-C6E1-442E-AF99-A91D26BA8FA0}.Release|Win64.ActiveCfg = Release|x64 - {24CB1A43-C6E1-442E-AF99-A91D26BA8FA0}.Release|Win64.Build.0 = Release|x64 - {E999D336-32D1-48CC-A71F-FC552BD5093B}.Debug|Win32.ActiveCfg = Debug|Win32 - {E999D336-32D1-48CC-A71F-FC552BD5093B}.Debug|Win32.Build.0 = Debug|Win32 - {E999D336-32D1-48CC-A71F-FC552BD5093B}.Debug|Win64.ActiveCfg = Debug|x64 - {E999D336-32D1-48CC-A71F-FC552BD5093B}.Debug|Win64.Build.0 = Debug|x64 - {E999D336-32D1-48CC-A71F-FC552BD5093B}.Release|Win32.ActiveCfg = Release|Win32 - {E999D336-32D1-48CC-A71F-FC552BD5093B}.Release|Win32.Build.0 = Release|Win32 - {E999D336-32D1-48CC-A71F-FC552BD5093B}.Release|Win64.ActiveCfg = Release|x64 - {E999D336-32D1-48CC-A71F-FC552BD5093B}.Release|Win64.Build.0 = Release|x64 - {7817D13F-1FAA-47D8-8C9F-B790FE2D6B0A}.Debug|Win32.ActiveCfg = Debug|Win32 - {7817D13F-1FAA-47D8-8C9F-B790FE2D6B0A}.Debug|Win32.Build.0 = Debug|Win32 - {7817D13F-1FAA-47D8-8C9F-B790FE2D6B0A}.Debug|Win64.ActiveCfg = Debug|x64 - {7817D13F-1FAA-47D8-8C9F-B790FE2D6B0A}.Debug|Win64.Build.0 = Debug|x64 - {7817D13F-1FAA-47D8-8C9F-B790FE2D6B0A}.Release|Win32.ActiveCfg = Release|Win32 - {7817D13F-1FAA-47D8-8C9F-B790FE2D6B0A}.Release|Win32.Build.0 = Release|Win32 - {7817D13F-1FAA-47D8-8C9F-B790FE2D6B0A}.Release|Win64.ActiveCfg = Release|x64 - {7817D13F-1FAA-47D8-8C9F-B790FE2D6B0A}.Release|Win64.Build.0 = Release|x64 - {45A08C1B-D249-42C4-932C-615ED3924CBB}.Debug|Win32.ActiveCfg = Debug|Win32 - {45A08C1B-D249-42C4-932C-615ED3924CBB}.Debug|Win32.Build.0 = Debug|Win32 - {45A08C1B-D249-42C4-932C-615ED3924CBB}.Debug|Win64.ActiveCfg = Debug|x64 - {45A08C1B-D249-42C4-932C-615ED3924CBB}.Debug|Win64.Build.0 = Debug|x64 - {45A08C1B-D249-42C4-932C-615ED3924CBB}.Release|Win32.ActiveCfg = Release|Win32 - {45A08C1B-D249-42C4-932C-615ED3924CBB}.Release|Win32.Build.0 = Release|Win32 - {45A08C1B-D249-42C4-932C-615ED3924CBB}.Release|Win64.ActiveCfg = Release|x64 - {45A08C1B-D249-42C4-932C-615ED3924CBB}.Release|Win64.Build.0 = Release|x64 - {9085A36E-CAF8-4E86-93AF-373D2554026B}.Debug|Win32.ActiveCfg = Debug|Win32 - {9085A36E-CAF8-4E86-93AF-373D2554026B}.Debug|Win32.Build.0 = Debug|Win32 - {9085A36E-CAF8-4E86-93AF-373D2554026B}.Debug|Win64.ActiveCfg = Debug|x64 - {9085A36E-CAF8-4E86-93AF-373D2554026B}.Debug|Win64.Build.0 = Debug|x64 - {9085A36E-CAF8-4E86-93AF-373D2554026B}.Release|Win32.ActiveCfg = Release|Win32 - {9085A36E-CAF8-4E86-93AF-373D2554026B}.Release|Win32.Build.0 = Release|Win32 - {9085A36E-CAF8-4E86-93AF-373D2554026B}.Release|Win64.ActiveCfg = Release|x64 - {9085A36E-CAF8-4E86-93AF-373D2554026B}.Release|Win64.Build.0 = Release|x64 - {70ABE75C-FBC2-4C82-9103-B952FA5CD168}.Debug|Win32.ActiveCfg = Debug|Win32 - {70ABE75C-FBC2-4C82-9103-B952FA5CD168}.Debug|Win32.Build.0 = Debug|Win32 - {70ABE75C-FBC2-4C82-9103-B952FA5CD168}.Debug|Win64.ActiveCfg = Debug|x64 - {70ABE75C-FBC2-4C82-9103-B952FA5CD168}.Debug|Win64.Build.0 = Debug|x64 - {70ABE75C-FBC2-4C82-9103-B952FA5CD168}.Release|Win32.ActiveCfg = Release|Win32 - {70ABE75C-FBC2-4C82-9103-B952FA5CD168}.Release|Win32.Build.0 = Release|Win32 - {70ABE75C-FBC2-4C82-9103-B952FA5CD168}.Release|Win64.ActiveCfg = Release|x64 - {70ABE75C-FBC2-4C82-9103-B952FA5CD168}.Release|Win64.Build.0 = Release|x64 - {24265BD3-3CF3-41A9-B137-53DBEC0F53BF}.Debug|Win32.ActiveCfg = Debug|Win32 - {24265BD3-3CF3-41A9-B137-53DBEC0F53BF}.Debug|Win32.Build.0 = Debug|Win32 - {24265BD3-3CF3-41A9-B137-53DBEC0F53BF}.Debug|Win64.ActiveCfg = Debug|x64 - {24265BD3-3CF3-41A9-B137-53DBEC0F53BF}.Debug|Win64.Build.0 = Debug|x64 - {24265BD3-3CF3-41A9-B137-53DBEC0F53BF}.Release|Win32.ActiveCfg = Release|Win32 - {24265BD3-3CF3-41A9-B137-53DBEC0F53BF}.Release|Win32.Build.0 = Release|Win32 - {24265BD3-3CF3-41A9-B137-53DBEC0F53BF}.Release|Win64.ActiveCfg = Release|x64 - {24265BD3-3CF3-41A9-B137-53DBEC0F53BF}.Release|Win64.Build.0 = Release|x64 - {C6C836E6-2899-458A-90BA-184192342D30}.Debug|Win32.ActiveCfg = Debug|Win32 - {C6C836E6-2899-458A-90BA-184192342D30}.Debug|Win32.Build.0 = Debug|Win32 - {C6C836E6-2899-458A-90BA-184192342D30}.Debug|Win64.ActiveCfg = Debug|x64 - {C6C836E6-2899-458A-90BA-184192342D30}.Debug|Win64.Build.0 = Debug|x64 - {C6C836E6-2899-458A-90BA-184192342D30}.Release|Win32.ActiveCfg = Release|Win32 - {C6C836E6-2899-458A-90BA-184192342D30}.Release|Win32.Build.0 = Release|Win32 - {C6C836E6-2899-458A-90BA-184192342D30}.Release|Win64.ActiveCfg = Release|x64 - {C6C836E6-2899-458A-90BA-184192342D30}.Release|Win64.Build.0 = Release|x64 - {7CDBBFFF-3B84-413E-BA1A-47CB8D454A46}.Debug|Win32.ActiveCfg = Debug|Win32 - {7CDBBFFF-3B84-413E-BA1A-47CB8D454A46}.Debug|Win32.Build.0 = Debug|Win32 - {7CDBBFFF-3B84-413E-BA1A-47CB8D454A46}.Debug|Win64.ActiveCfg = Debug|x64 - {7CDBBFFF-3B84-413E-BA1A-47CB8D454A46}.Debug|Win64.Build.0 = Debug|x64 - {7CDBBFFF-3B84-413E-BA1A-47CB8D454A46}.Release|Win32.ActiveCfg = Release|Win32 - {7CDBBFFF-3B84-413E-BA1A-47CB8D454A46}.Release|Win32.Build.0 = Release|Win32 - {7CDBBFFF-3B84-413E-BA1A-47CB8D454A46}.Release|Win64.ActiveCfg = Release|x64 - {7CDBBFFF-3B84-413E-BA1A-47CB8D454A46}.Release|Win64.Build.0 = Release|x64 - {561FA9F7-14D0-4D94-822C-AAE9B08AACEF}.Debug|Win32.ActiveCfg = Debug|Win32 - {561FA9F7-14D0-4D94-822C-AAE9B08AACEF}.Debug|Win32.Build.0 = Debug|Win32 - {561FA9F7-14D0-4D94-822C-AAE9B08AACEF}.Debug|Win64.ActiveCfg = Debug|x64 - {561FA9F7-14D0-4D94-822C-AAE9B08AACEF}.Debug|Win64.Build.0 = Debug|x64 - {561FA9F7-14D0-4D94-822C-AAE9B08AACEF}.Release|Win32.ActiveCfg = Release|Win32 - {561FA9F7-14D0-4D94-822C-AAE9B08AACEF}.Release|Win32.Build.0 = Release|Win32 - {561FA9F7-14D0-4D94-822C-AAE9B08AACEF}.Release|Win64.ActiveCfg = Release|x64 - {561FA9F7-14D0-4D94-822C-AAE9B08AACEF}.Release|Win64.Build.0 = Release|x64 - {42FC1F89-7136-45E8-94E3-594C173C2937}.Debug|Win32.ActiveCfg = Debug|Win32 - {42FC1F89-7136-45E8-94E3-594C173C2937}.Debug|Win32.Build.0 = Debug|Win32 - {42FC1F89-7136-45E8-94E3-594C173C2937}.Debug|Win64.ActiveCfg = Debug|x64 - {42FC1F89-7136-45E8-94E3-594C173C2937}.Debug|Win64.Build.0 = Debug|x64 - {42FC1F89-7136-45E8-94E3-594C173C2937}.Release|Win32.ActiveCfg = Release|Win32 - {42FC1F89-7136-45E8-94E3-594C173C2937}.Release|Win32.Build.0 = Release|Win32 - {42FC1F89-7136-45E8-94E3-594C173C2937}.Release|Win64.ActiveCfg = Release|x64 - {42FC1F89-7136-45E8-94E3-594C173C2937}.Release|Win64.Build.0 = Release|x64 - {30042FAB-AD01-BF78-3C79-F1F6316BC947}.Debug|Win32.ActiveCfg = Debug|Win32 - {30042FAB-AD01-BF78-3C79-F1F6316BC947}.Debug|Win32.Build.0 = Debug|Win32 - {30042FAB-AD01-BF78-3C79-F1F6316BC947}.Debug|Win64.ActiveCfg = Debug|x64 - {30042FAB-AD01-BF78-3C79-F1F6316BC947}.Debug|Win64.Build.0 = Debug|x64 - {30042FAB-AD01-BF78-3C79-F1F6316BC947}.Release|Win32.ActiveCfg = Release|Win32 - {30042FAB-AD01-BF78-3C79-F1F6316BC947}.Release|Win32.Build.0 = Release|Win32 - {30042FAB-AD01-BF78-3C79-F1F6316BC947}.Release|Win64.ActiveCfg = Release|x64 - {30042FAB-AD01-BF78-3C79-F1F6316BC947}.Release|Win64.Build.0 = Release|x64 - {A04109A7-46C9-42F9-AB29-8E3D84450172}.Debug|Win32.ActiveCfg = Debug|Win32 - {A04109A7-46C9-42F9-AB29-8E3D84450172}.Debug|Win32.Build.0 = Debug|Win32 - {A04109A7-46C9-42F9-AB29-8E3D84450172}.Debug|Win64.ActiveCfg = Debug|x64 - {A04109A7-46C9-42F9-AB29-8E3D84450172}.Debug|Win64.Build.0 = Debug|x64 - {A04109A7-46C9-42F9-AB29-8E3D84450172}.Release|Win32.ActiveCfg = Release|Win32 - {A04109A7-46C9-42F9-AB29-8E3D84450172}.Release|Win32.Build.0 = Release|Win32 - {A04109A7-46C9-42F9-AB29-8E3D84450172}.Release|Win64.ActiveCfg = Release|x64 - {A04109A7-46C9-42F9-AB29-8E3D84450172}.Release|Win64.Build.0 = Release|x64 - {13E85053-54B3-487B-8DDB-3430B1C1B3BF}.Debug|Win32.ActiveCfg = Debug|Win32 - {13E85053-54B3-487B-8DDB-3430B1C1B3BF}.Debug|Win32.Build.0 = Debug|Win32 - {13E85053-54B3-487B-8DDB-3430B1C1B3BF}.Debug|Win64.ActiveCfg = Debug|x64 - {13E85053-54B3-487B-8DDB-3430B1C1B3BF}.Debug|Win64.Build.0 = Debug|x64 - {13E85053-54B3-487B-8DDB-3430B1C1B3BF}.Release|Win32.ActiveCfg = Release|Win32 - {13E85053-54B3-487B-8DDB-3430B1C1B3BF}.Release|Win32.Build.0 = Release|Win32 - {13E85053-54B3-487B-8DDB-3430B1C1B3BF}.Release|Win64.ActiveCfg = Release|x64 - {13E85053-54B3-487B-8DDB-3430B1C1B3BF}.Release|Win64.Build.0 = Release|x64 - {70E890EA-9496-4F7C-B903-5FCF87A73AC4}.Debug|Win32.ActiveCfg = Debug|Win32 - {70E890EA-9496-4F7C-B903-5FCF87A73AC4}.Debug|Win32.Build.0 = Debug|Win32 - {70E890EA-9496-4F7C-B903-5FCF87A73AC4}.Debug|Win64.ActiveCfg = Debug|x64 - {70E890EA-9496-4F7C-B903-5FCF87A73AC4}.Debug|Win64.Build.0 = Debug|x64 - {70E890EA-9496-4F7C-B903-5FCF87A73AC4}.Release|Win32.ActiveCfg = Release|Win32 - {70E890EA-9496-4F7C-B903-5FCF87A73AC4}.Release|Win32.Build.0 = Release|Win32 - {70E890EA-9496-4F7C-B903-5FCF87A73AC4}.Release|Win64.ActiveCfg = Release|x64 - {70E890EA-9496-4F7C-B903-5FCF87A73AC4}.Release|Win64.Build.0 = Release|x64 - {938D17CB-3F34-46A4-AF64-2099B7FF6902}.Debug|Win32.ActiveCfg = Debug|Win32 - {938D17CB-3F34-46A4-AF64-2099B7FF6902}.Debug|Win32.Build.0 = Debug|Win32 - {938D17CB-3F34-46A4-AF64-2099B7FF6902}.Debug|Win64.ActiveCfg = Debug|x64 - {938D17CB-3F34-46A4-AF64-2099B7FF6902}.Debug|Win64.Build.0 = Debug|x64 - {938D17CB-3F34-46A4-AF64-2099B7FF6902}.Release|Win32.ActiveCfg = Release|Win32 - {938D17CB-3F34-46A4-AF64-2099B7FF6902}.Release|Win32.Build.0 = Release|Win32 - {938D17CB-3F34-46A4-AF64-2099B7FF6902}.Release|Win64.ActiveCfg = Release|x64 - {938D17CB-3F34-46A4-AF64-2099B7FF6902}.Release|Win64.Build.0 = Release|x64 - {13E95053-54B3-497B-9DDB-3430B1C1B6BF}.Debug|Win32.ActiveCfg = Debug|Win32 - {13E95053-54B3-497B-9DDB-3430B1C1B6BF}.Debug|Win32.Build.0 = Debug|Win32 - {13E95053-54B3-497B-9DDB-3430B1C1B6BF}.Debug|Win64.ActiveCfg = Debug|x64 - {13E95053-54B3-497B-9DDB-3430B1C1B6BF}.Debug|Win64.Build.0 = Debug|x64 - {13E95053-54B3-497B-9DDB-3430B1C1B6BF}.Release|Win32.ActiveCfg = Release|Win32 - {13E95053-54B3-497B-9DDB-3430B1C1B6BF}.Release|Win32.Build.0 = Release|Win32 - {13E95053-54B3-497B-9DDB-3430B1C1B6BF}.Release|Win64.ActiveCfg = Release|x64 - {13E95053-54B3-497B-9DDB-3430B1C1B6BF}.Release|Win64.Build.0 = Release|x64 - {DA6F56B4-06A4-441D-AD70-AC5A7D51FADB}.Debug|Win32.ActiveCfg = LIB Debug - LIB OpenSSL|Win32 - {DA6F56B4-06A4-441D-AD70-AC5A7D51FADB}.Debug|Win32.Build.0 = LIB Debug - LIB OpenSSL|Win32 - {DA6F56B4-06A4-441D-AD70-AC5A7D51FADB}.Debug|Win64.ActiveCfg = LIB Debug - LIB OpenSSL|x64 - {DA6F56B4-06A4-441D-AD70-AC5A7D51FADB}.Debug|Win64.Build.0 = LIB Debug - LIB OpenSSL|x64 - {DA6F56B4-06A4-441D-AD70-AC5A7D51FADB}.Release|Win32.ActiveCfg = LIB Release - LIB OpenSSL|Win32 - {DA6F56B4-06A4-441D-AD70-AC5A7D51FADB}.Release|Win32.Build.0 = LIB Release - LIB OpenSSL|Win32 - {DA6F56B4-06A4-441D-AD70-AC5A7D51FADB}.Release|Win64.ActiveCfg = LIB Release - LIB OpenSSL|x64 - {DA6F56B4-06A4-441D-AD70-AC5A7D51FADB}.Release|Win64.Build.0 = LIB Release - LIB OpenSSL|x64 - {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Debug|Win32.ActiveCfg = Debug|Win32 - {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Debug|Win32.Build.0 = Debug|Win32 - {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Debug|Win64.ActiveCfg = Debug|x64 - {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Debug|Win64.Build.0 = Debug|x64 - {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Release|Win32.ActiveCfg = Release|Win32 - {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Release|Win32.Build.0 = Release|Win32 - {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Release|Win64.ActiveCfg = Release|x64 - {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Release|Win64.Build.0 = Release|x64 - {86937F53-C189-40EF-8CE8-8759D8E7D480}.Debug|Win32.ActiveCfg = Debug|Win32 - {86937F53-C189-40EF-8CE8-8759D8E7D480}.Debug|Win32.Build.0 = Debug|Win32 - {86937F53-C189-40EF-8CE8-8759D8E7D480}.Debug|Win64.ActiveCfg = Debug|x64 - {86937F53-C189-40EF-8CE8-8759D8E7D480}.Debug|Win64.Build.0 = Debug|x64 - {86937F53-C189-40EF-8CE8-8759D8E7D480}.Release|Win32.ActiveCfg = Release|Win32 - {86937F53-C189-40EF-8CE8-8759D8E7D480}.Release|Win32.Build.0 = Release|Win32 - {86937F53-C189-40EF-8CE8-8759D8E7D480}.Release|Win64.ActiveCfg = Release|x64 - {86937F53-C189-40EF-8CE8-8759D8E7D480}.Release|Win64.Build.0 = Release|x64 - {18CAE28C-B454-46C1-87A0-493D91D97F03}.Debug|Win32.ActiveCfg = Debug|Win32 - {18CAE28C-B454-46C1-87A0-493D91D97F03}.Debug|Win32.Build.0 = Debug|Win32 - {18CAE28C-B454-46C1-87A0-493D91D97F03}.Debug|Win64.ActiveCfg = Debug|x64 - {18CAE28C-B454-46C1-87A0-493D91D97F03}.Debug|Win64.Build.0 = Debug|x64 - {18CAE28C-B454-46C1-87A0-493D91D97F03}.Release|Win32.ActiveCfg = Release|Win32 - {18CAE28C-B454-46C1-87A0-493D91D97F03}.Release|Win32.Build.0 = Release|Win32 - {18CAE28C-B454-46C1-87A0-493D91D97F03}.Release|Win64.ActiveCfg = Release|x64 - {18CAE28C-B454-46C1-87A0-493D91D97F03}.Release|Win64.Build.0 = Release|x64 - {0E9791DB-593A-465F-98BC-681011311618}.Debug|Win32.ActiveCfg = Debug|Win32 - {0E9791DB-593A-465F-98BC-681011311618}.Debug|Win32.Build.0 = Debug|Win32 - {0E9791DB-593A-465F-98BC-681011311618}.Debug|Win64.ActiveCfg = Debug|x64 - {0E9791DB-593A-465F-98BC-681011311618}.Debug|Win64.Build.0 = Debug|x64 - {0E9791DB-593A-465F-98BC-681011311618}.Release|Win32.ActiveCfg = Release|Win32 - {0E9791DB-593A-465F-98BC-681011311618}.Release|Win32.Build.0 = Release|Win32 - {0E9791DB-593A-465F-98BC-681011311618}.Release|Win64.ActiveCfg = Release|x64 - {0E9791DB-593A-465F-98BC-681011311618}.Release|Win64.Build.0 = Release|x64 - {17E1E049-C309-4D79-843F-AE483C264AEA}.Debug|Win32.ActiveCfg = Debug|Win32 - {17E1E049-C309-4D79-843F-AE483C264AEA}.Debug|Win32.Build.0 = Debug|Win32 - {17E1E049-C309-4D79-843F-AE483C264AEA}.Debug|Win64.ActiveCfg = Debug|x64 - {17E1E049-C309-4D79-843F-AE483C264AEA}.Debug|Win64.Build.0 = Debug|x64 - {17E1E049-C309-4D79-843F-AE483C264AEA}.Release|Win32.ActiveCfg = Release|Win32 - {17E1E049-C309-4D79-843F-AE483C264AEA}.Release|Win32.Build.0 = Release|Win32 - {17E1E049-C309-4D79-843F-AE483C264AEA}.Release|Win64.ActiveCfg = Release|x64 - {17E1E049-C309-4D79-843F-AE483C264AEA}.Release|Win64.Build.0 = Release|x64 - {D06B6426-4762-44CC-8BAD-D79052507F2F}.Debug|Win32.ActiveCfg = Debug|Win32 - {D06B6426-4762-44CC-8BAD-D79052507F2F}.Debug|Win32.Build.0 = Debug|Win32 - {D06B6426-4762-44CC-8BAD-D79052507F2F}.Debug|Win64.ActiveCfg = Debug|x64 - {D06B6426-4762-44CC-8BAD-D79052507F2F}.Debug|Win64.Build.0 = Debug|x64 - {D06B6426-4762-44CC-8BAD-D79052507F2F}.Release|Win32.ActiveCfg = Release|Win32 - {D06B6426-4762-44CC-8BAD-D79052507F2F}.Release|Win32.Build.0 = Release|Win32 - {D06B6426-4762-44CC-8BAD-D79052507F2F}.Release|Win64.ActiveCfg = Release|x64 - {D06B6426-4762-44CC-8BAD-D79052507F2F}.Release|Win64.Build.0 = Release|x64 - {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Debug|Win32.ActiveCfg = Debug|Win32 - {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Debug|Win32.Build.0 = Debug|Win32 - {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Debug|Win64.ActiveCfg = Debug|x64 - {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Debug|Win64.Build.0 = Debug|x64 - {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Release|Win32.ActiveCfg = Release|Win32 - {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Release|Win32.Build.0 = Release|Win32 - {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Release|Win64.ActiveCfg = Release|x64 - {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Release|Win64.Build.0 = Release|x64 - {C6E20F84-3247-4AD6-B051-B073268F73BA}.Debug|Win32.ActiveCfg = Debug|Win32 - {C6E20F84-3247-4AD6-B051-B073268F73BA}.Debug|Win32.Build.0 = Debug|Win32 - {C6E20F84-3247-4AD6-B051-B073268F73BA}.Debug|Win64.ActiveCfg = Debug|x64 - {C6E20F84-3247-4AD6-B051-B073268F73BA}.Debug|Win64.Build.0 = Debug|x64 - {C6E20F84-3247-4AD6-B051-B073268F73BA}.Release|Win32.ActiveCfg = Release|Win32 - {C6E20F84-3247-4AD6-B051-B073268F73BA}.Release|Win32.Build.0 = Release|Win32 - {C6E20F84-3247-4AD6-B051-B073268F73BA}.Release|Win64.ActiveCfg = Release|x64 - {C6E20F84-3247-4AD6-B051-B073268F73BA}.Release|Win64.Build.0 = Release|x64 - {447F05A8-F581-4CAC-A466-5AC7936E207E}.Debug|Win32.ActiveCfg = Debug|Win32 - {447F05A8-F581-4CAC-A466-5AC7936E207E}.Debug|Win32.Build.0 = Debug|Win32 - {447F05A8-F581-4CAC-A466-5AC7936E207E}.Debug|Win64.ActiveCfg = Debug|x64 - {447F05A8-F581-4CAC-A466-5AC7936E207E}.Debug|Win64.Build.0 = Debug|x64 - {447F05A8-F581-4CAC-A466-5AC7936E207E}.Release|Win32.ActiveCfg = Release|Win32 - {447F05A8-F581-4CAC-A466-5AC7936E207E}.Release|Win32.Build.0 = Release|Win32 - {447F05A8-F581-4CAC-A466-5AC7936E207E}.Release|Win64.ActiveCfg = Release|x64 - {447F05A8-F581-4CAC-A466-5AC7936E207E}.Release|Win64.Build.0 = Release|x64 - {745DEC58-EBB3-47A9-A9B8-4C6627C01BF8}.Debug|Win32.ActiveCfg = Debug|Win32 - {745DEC58-EBB3-47A9-A9B8-4C6627C01BF8}.Debug|Win32.Build.0 = Debug|Win32 - {745DEC58-EBB3-47A9-A9B8-4C6627C01BF8}.Debug|Win64.ActiveCfg = Debug|x64 - {745DEC58-EBB3-47A9-A9B8-4C6627C01BF8}.Debug|Win64.Build.0 = Debug|x64 - {745DEC58-EBB3-47A9-A9B8-4C6627C01BF8}.Release|Win32.ActiveCfg = Release|Win32 - {745DEC58-EBB3-47A9-A9B8-4C6627C01BF8}.Release|Win32.Build.0 = Release|Win32 - {745DEC58-EBB3-47A9-A9B8-4C6627C01BF8}.Release|Win64.ActiveCfg = Release|x64 - {745DEC58-EBB3-47A9-A9B8-4C6627C01BF8}.Release|Win64.Build.0 = Release|x64 - {384C224A-7474-476E-A01B-750EA7DE918C}.Debug|Win32.ActiveCfg = Debug|Win32 - {384C224A-7474-476E-A01B-750EA7DE918C}.Debug|Win32.Build.0 = Debug|Win32 - {384C224A-7474-476E-A01B-750EA7DE918C}.Debug|Win64.ActiveCfg = Debug|x64 - {384C224A-7474-476E-A01B-750EA7DE918C}.Debug|Win64.Build.0 = Debug|x64 - {384C224A-7474-476E-A01B-750EA7DE918C}.Release|Win32.ActiveCfg = Release|Win32 - {384C224A-7474-476E-A01B-750EA7DE918C}.Release|Win32.Build.0 = Release|Win32 - {384C224A-7474-476E-A01B-750EA7DE918C}.Release|Win64.ActiveCfg = Release|x64 - {384C224A-7474-476E-A01B-750EA7DE918C}.Release|Win64.Build.0 = Release|x64 - {B8BF1D81-09DC-42D4-B406-4F868B33A89E}.Debug|Win32.ActiveCfg = Debug|Win32 - {B8BF1D81-09DC-42D4-B406-4F868B33A89E}.Debug|Win32.Build.0 = Debug|Win32 - {B8BF1D81-09DC-42D4-B406-4F868B33A89E}.Debug|Win64.ActiveCfg = Debug|x64 - {B8BF1D81-09DC-42D4-B406-4F868B33A89E}.Debug|Win64.Build.0 = Debug|x64 - {B8BF1D81-09DC-42D4-B406-4F868B33A89E}.Release|Win32.ActiveCfg = Release|Win32 - {B8BF1D81-09DC-42D4-B406-4F868B33A89E}.Release|Win32.Build.0 = Release|Win32 - {B8BF1D81-09DC-42D4-B406-4F868B33A89E}.Release|Win64.ActiveCfg = Release|x64 - {B8BF1D81-09DC-42D4-B406-4F868B33A89E}.Release|Win64.Build.0 = Release|x64 - {0E9791DB-593A-465F-98BC-681011311617}.Debug|Win32.ActiveCfg = Debug|Win32 - {0E9791DB-593A-465F-98BC-681011311617}.Debug|Win32.Build.0 = Debug|Win32 - {0E9791DB-593A-465F-98BC-681011311617}.Debug|Win64.ActiveCfg = Debug|x64 - {0E9791DB-593A-465F-98BC-681011311617}.Debug|Win64.Build.0 = Debug|x64 - {0E9791DB-593A-465F-98BC-681011311617}.Release|Win32.ActiveCfg = Release|Win32 - {0E9791DB-593A-465F-98BC-681011311617}.Release|Win32.Build.0 = Release|Win32 - {0E9791DB-593A-465F-98BC-681011311617}.Release|Win64.ActiveCfg = Release|x64 - {0E9791DB-593A-465F-98BC-681011311617}.Release|Win64.Build.0 = Release|x64 - {9E48B300-37D1-11DD-8C41-005056C00008}.Debug|Win32.ActiveCfg = Debug|Win32 - {9E48B300-37D1-11DD-8C41-005056C00008}.Debug|Win32.Build.0 = Debug|Win32 - {9E48B300-37D1-11DD-8C41-005056C00008}.Debug|Win64.ActiveCfg = Debug|x64 - {9E48B300-37D1-11DD-8C41-005056C00008}.Debug|Win64.Build.0 = Debug|x64 - {9E48B300-37D1-11DD-8C41-005056C00008}.Release|Win32.ActiveCfg = Release|Win32 - {9E48B300-37D1-11DD-8C41-005056C00008}.Release|Win32.Build.0 = Release|Win32 - {9E48B300-37D1-11DD-8C41-005056C00008}.Release|Win64.ActiveCfg = Release|x64 - {9E48B300-37D1-11DD-8C41-005056C00008}.Release|Win64.Build.0 = Release|x64 - {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Debug|Win32.ActiveCfg = Debug|Win32 - {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Debug|Win32.Build.0 = Debug|Win32 - {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Debug|Win64.ActiveCfg = Debug|x64 - {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Debug|Win64.Build.0 = Debug|x64 - {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Release|Win32.ActiveCfg = Release|Win32 - {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Release|Win32.Build.0 = Release|Win32 - {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Release|Win64.ActiveCfg = Release|x64 - {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Release|Win64.Build.0 = Release|x64 - {78D80A15-BD8C-44E2-B49E-1F05B0A0A687}.Debug|Win32.ActiveCfg = Debug|Win32 - {78D80A15-BD8C-44E2-B49E-1F05B0A0A687}.Debug|Win32.Build.0 = Debug|Win32 - {78D80A15-BD8C-44E2-B49E-1F05B0A0A687}.Debug|Win64.ActiveCfg = Debug|x64 - {78D80A15-BD8C-44E2-B49E-1F05B0A0A687}.Debug|Win64.Build.0 = Debug|x64 - {78D80A15-BD8C-44E2-B49E-1F05B0A0A687}.Release|Win32.ActiveCfg = Release|Win32 - {78D80A15-BD8C-44E2-B49E-1F05B0A0A687}.Release|Win32.Build.0 = Release|Win32 - {78D80A15-BD8C-44E2-B49E-1F05B0A0A687}.Release|Win64.ActiveCfg = Release|x64 - {78D80A15-BD8C-44E2-B49E-1F05B0A0A687}.Release|Win64.Build.0 = Release|x64 - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection - GlobalSection(NestedProjects) = preSolution - {44FA54E8-0A60-4A6C-A449-118872E75502} = {C57D8946-CC54-4360-89FA-DAC04E93423A} - {0E032FA8-BB7B-40F6-8CB1-15F204113B24} = {C57D8946-CC54-4360-89FA-DAC04E93423A} - {F67B2C56-D1B1-4EA7-B16E-EF8E7F1B6C5F} = {C57D8946-CC54-4360-89FA-DAC04E93423A} - {7A411D5D-DA3B-474D-B08B-3DD787A1F375} = {C57D8946-CC54-4360-89FA-DAC04E93423A} - {2992ABDB-0853-4C43-B9E2-98D4211192EE} = {C57D8946-CC54-4360-89FA-DAC04E93423A} - {56D453A7-787B-49E3-84BC-465EED38DE8A} = {C57D8946-CC54-4360-89FA-DAC04E93423A} - {5EF24499-4F74-4AF6-8048-650BE7BD7808} = {C57D8946-CC54-4360-89FA-DAC04E93423A} - {863B8043-533C-499D-B48E-F5EC91A6CB9B} = {C57D8946-CC54-4360-89FA-DAC04E93423A} - {04C620E9-BE02-498E-A713-B9E470D75F15} = {89F7F2F7-EB4D-4810-AA06-A96D592678E6} - {89DA21A1-625B-4482-B386-D7C0AF97E477} = {89F7F2F7-EB4D-4810-AA06-A96D592678E6} - {1BAD0518-F2BB-42F5-83E7-9E1804FAD7C1} = {89F7F2F7-EB4D-4810-AA06-A96D592678E6} - {FE88370E-80F7-49BD-A8D1-E7A2CA699E44} = {89F7F2F7-EB4D-4810-AA06-A96D592678E6} - {9EA2E328-E6BE-4DF0-A8DC-05C08A0B822B} = {89F7F2F7-EB4D-4810-AA06-A96D592678E6} - {69C18C83-5753-4F36-BF5A-6EE217304D2D} = {89F7F2F7-EB4D-4810-AA06-A96D592678E6} - {4AF743A4-8C2F-4424-BB4F-A5B0B5C128BD} = {89F7F2F7-EB4D-4810-AA06-A96D592678E6} - {E17527D0-CCA7-4063-924B-C09A1BD43EC3} = {54612744-437F-4400-AE6C-BE4CD68C8507} - {896616D2-A5D2-4548-859E-D59905F46140} = {C57D8946-CC54-4360-89FA-DAC04E93423A} - {78EE8B08-4E07-4C07-9592-BC4D09503155} = {54612744-437F-4400-AE6C-BE4CD68C8507} - {3F8B2057-492F-4332-99DE-3E12F20ED489} = {C57D8946-CC54-4360-89FA-DAC04E93423A} - {6C92BA78-CFAA-4524-A636-F044B0280AB0} = {C57D8946-CC54-4360-89FA-DAC04E93423A} - {24CB1A43-C6E1-442E-AF99-A91D26BA8FA0} = {C57D8946-CC54-4360-89FA-DAC04E93423A} - {E999D336-32D1-48CC-A71F-FC552BD5093B} = {54612744-437F-4400-AE6C-BE4CD68C8507} - {7817D13F-1FAA-47D8-8C9F-B790FE2D6B0A} = {54612744-437F-4400-AE6C-BE4CD68C8507} - {45A08C1B-D249-42C4-932C-615ED3924CBB} = {8BF90F4F-766B-4B4D-B7F3-BF4189A29F4E} - {9085A36E-CAF8-4E86-93AF-373D2554026B} = {C57D8946-CC54-4360-89FA-DAC04E93423A} - {70ABE75C-FBC2-4C82-9103-B952FA5CD168} = {B82CBA7A-6640-47BD-94C9-9EEE37CECBE0} - {24265BD3-3CF3-41A9-B137-53DBEC0F53BF} = {B82CBA7A-6640-47BD-94C9-9EEE37CECBE0} - {C6C836E6-2899-458A-90BA-184192342D30} = {67707698-A79D-4C72-8F99-3B96BA970D0D} - {7CDBBFFF-3B84-413E-BA1A-47CB8D454A46} = {67707698-A79D-4C72-8F99-3B96BA970D0D} - {561FA9F7-14D0-4D94-822C-AAE9B08AACEF} = {67707698-A79D-4C72-8F99-3B96BA970D0D} - {42FC1F89-7136-45E8-94E3-594C173C2937} = {8BF90F4F-766B-4B4D-B7F3-BF4189A29F4E} - {67707698-A79D-4C72-8F99-3B96BA970D0D} = {8BF90F4F-766B-4B4D-B7F3-BF4189A29F4E} - {B82CBA7A-6640-47BD-94C9-9EEE37CECBE0} = {8BF90F4F-766B-4B4D-B7F3-BF4189A29F4E} - {30042FAB-AD01-BF78-3C79-F1F6316BC947} = {67707698-A79D-4C72-8F99-3B96BA970D0D} - {A04109A7-46C9-42F9-AB29-8E3D84450172} = {C57D8946-CC54-4360-89FA-DAC04E93423A} - {0E68FE27-FE42-4AA2-BE7A-C267B8F5FD2E} = {8BF90F4F-766B-4B4D-B7F3-BF4189A29F4E} - {13E85053-54B3-487B-8DDB-3430B1C1B3BF} = {F4FD40AD-244F-4162-88D7-FFD4722FAFAE} - {F4FD40AD-244F-4162-88D7-FFD4722FAFAE} = {8BF90F4F-766B-4B4D-B7F3-BF4189A29F4E} - {70E890EA-9496-4F7C-B903-5FCF87A73AC4} = {C57D8946-CC54-4360-89FA-DAC04E93423A} - {938D17CB-3F34-46A4-AF64-2099B7FF6902} = {54612744-437F-4400-AE6C-BE4CD68C8507} - {13E95053-54B3-497B-9DDB-3430B1C1B6BF} = {8BF90F4F-766B-4B4D-B7F3-BF4189A29F4E} - {DA6F56B4-06A4-441D-AD70-AC5A7D51FADB} = {8BF90F4F-766B-4B4D-B7F3-BF4189A29F4E} - {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26} = {0E68FE27-FE42-4AA2-BE7A-C267B8F5FD2E} - {04274E83-D752-4CC9-BFDE-A717D4E6ABAA} = {0E68FE27-FE42-4AA2-BE7A-C267B8F5FD2E} - {86937F53-C189-40EF-8CE8-8759D8E7D480} = {04274E83-D752-4CC9-BFDE-A717D4E6ABAA} - {18CAE28C-B454-46C1-87A0-493D91D97F03} = {04274E83-D752-4CC9-BFDE-A717D4E6ABAA} - {0E9791DB-593A-465F-98BC-681011311618} = {04274E83-D752-4CC9-BFDE-A717D4E6ABAA} - {17E1E049-C309-4D79-843F-AE483C264AEA} = {04274E83-D752-4CC9-BFDE-A717D4E6ABAA} - {D06B6426-4762-44CC-8BAD-D79052507F2F} = {04274E83-D752-4CC9-BFDE-A717D4E6ABAA} - {ECC7CEAC-A5E5-458E-BB9E-2413CC847881} = {04274E83-D752-4CC9-BFDE-A717D4E6ABAA} - {C6E20F84-3247-4AD6-B051-B073268F73BA} = {04274E83-D752-4CC9-BFDE-A717D4E6ABAA} - {447F05A8-F581-4CAC-A466-5AC7936E207E} = {04274E83-D752-4CC9-BFDE-A717D4E6ABAA} - {745DEC58-EBB3-47A9-A9B8-4C6627C01BF8} = {8BF90F4F-766B-4B4D-B7F3-BF4189A29F4E} - {384C224A-7474-476E-A01B-750EA7DE918C} = {04274E83-D752-4CC9-BFDE-A717D4E6ABAA} - {B8BF1D81-09DC-42D4-B406-4F868B33A89E} = {04274E83-D752-4CC9-BFDE-A717D4E6ABAA} - {0E9791DB-593A-465F-98BC-681011311617} = {04274E83-D752-4CC9-BFDE-A717D4E6ABAA} - {9E48B300-37D1-11DD-8C41-005056C00008} = {04274E83-D752-4CC9-BFDE-A717D4E6ABAA} - {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E} = {04274E83-D752-4CC9-BFDE-A717D4E6ABAA} - {78D80A15-BD8C-44E2-B49E-1F05B0A0A687} = {04274E83-D752-4CC9-BFDE-A717D4E6ABAA} - EndGlobalSection - GlobalSection(ExtensibilityGlobals) = postSolution - SolutionGuid = {98BC2259-260D-4CB1-B4A6-FAAA2DC3E91B} - EndGlobalSection -EndGlobal + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 15 +VisualStudioVersion = 15.0.27703.2047 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "dependencies", "dependencies", "{8BF90F4F-766B-4B4D-B7F3-BF4189A29F4E}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "libs", "libs", "{C57D8946-CC54-4360-89FA-DAC04E93423A}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "srv-apps", "srv-apps", "{89F7F2F7-EB4D-4810-AA06-A96D592678E6}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "entitydef", "lib\entitydef\entitydef.vcxproj", "{44FA54E8-0A60-4A6C-A449-118872E75502}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "helper", "lib\helper\helper.vcxproj", "{0E032FA8-BB7B-40F6-8CB1-15F204113B24}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "math", "lib\math\math.vcxproj", "{F67B2C56-D1B1-4EA7-B16E-EF8E7F1B6C5F}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pyscript", "lib\pyscript\pyscript.vcxproj", "{7A411D5D-DA3B-474D-B08B-3DD787A1F375}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "xml", "lib\xml\xml.vcxproj", "{2992ABDB-0853-4C43-B9E2-98D4211192EE}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "server", "lib\server\server.vcxproj", "{56D453A7-787B-49E3-84BC-465EED38DE8A}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "network", "lib\network\network.vcxproj", "{5EF24499-4F74-4AF6-8048-650BE7BD7808}" + ProjectSection(ProjectDependencies) = postProject + {70ABE75C-FBC2-4C82-9103-B952FA5CD168} = {70ABE75C-FBC2-4C82-9103-B952FA5CD168} + {DA6F56B4-06A4-441D-AD70-AC5A7D51FADB} = {DA6F56B4-06A4-441D-AD70-AC5A7D51FADB} + {24265BD3-3CF3-41A9-B137-53DBEC0F53BF} = {24265BD3-3CF3-41A9-B137-53DBEC0F53BF} + EndProjectSection +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "thread", "lib\thread\thread.vcxproj", "{863B8043-533C-499D-B48E-F5EC91A6CB9B}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "cellapp", "server\cellapp\cellapp.vcxproj", "{04C620E9-BE02-498E-A713-B9E470D75F15}" + ProjectSection(ProjectDependencies) = postProject + {745DEC58-EBB3-47A9-A9B8-4C6627C01BF8} = {745DEC58-EBB3-47A9-A9B8-4C6627C01BF8} + {42FC1F89-7136-45E8-94E3-594C173C2937} = {42FC1F89-7136-45E8-94E3-594C173C2937} + {896616D2-A5D2-4548-859E-D59905F46140} = {896616D2-A5D2-4548-859E-D59905F46140} + EndProjectSection +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "machine", "server\machine\machine.vcxproj", "{89DA21A1-625B-4482-B386-D7C0AF97E477}" + ProjectSection(ProjectDependencies) = postProject + {561FA9F7-14D0-4D94-822C-AAE9B08AACEF} = {561FA9F7-14D0-4D94-822C-AAE9B08AACEF} + EndProjectSection +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "baseappmgr", "server\baseappmgr\baseappmgr.vcxproj", "{1BAD0518-F2BB-42F5-83E7-9E1804FAD7C1}" + ProjectSection(ProjectDependencies) = postProject + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26} = {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26} + {561FA9F7-14D0-4D94-822C-AAE9B08AACEF} = {561FA9F7-14D0-4D94-822C-AAE9B08AACEF} + EndProjectSection +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "cellappmgr", "server\cellappmgr\cellappmgr.vcxproj", "{FE88370E-80F7-49BD-A8D1-E7A2CA699E44}" + ProjectSection(ProjectDependencies) = postProject + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26} = {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26} + {561FA9F7-14D0-4D94-822C-AAE9B08AACEF} = {561FA9F7-14D0-4D94-822C-AAE9B08AACEF} + EndProjectSection +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "dbmgr", "server\dbmgr\dbmgr.vcxproj", "{9EA2E328-E6BE-4DF0-A8DC-05C08A0B822B}" + ProjectSection(ProjectDependencies) = postProject + {13E85053-54B3-487B-8DDB-3430B1C1B3BF} = {13E85053-54B3-487B-8DDB-3430B1C1B3BF} + {70E890EA-9496-4F7C-B903-5FCF87A73AC4} = {70E890EA-9496-4F7C-B903-5FCF87A73AC4} + {561FA9F7-14D0-4D94-822C-AAE9B08AACEF} = {561FA9F7-14D0-4D94-822C-AAE9B08AACEF} + EndProjectSection +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "baseapp", "server\baseapp\baseapp.vcxproj", "{69C18C83-5753-4F36-BF5A-6EE217304D2D}" + ProjectSection(ProjectDependencies) = postProject + {45A08C1B-D249-42C4-932C-615ED3924CBB} = {45A08C1B-D249-42C4-932C-615ED3924CBB} + {745DEC58-EBB3-47A9-A9B8-4C6627C01BF8} = {745DEC58-EBB3-47A9-A9B8-4C6627C01BF8} + {896616D2-A5D2-4548-859E-D59905F46140} = {896616D2-A5D2-4548-859E-D59905F46140} + {561FA9F7-14D0-4D94-822C-AAE9B08AACEF} = {561FA9F7-14D0-4D94-822C-AAE9B08AACEF} + EndProjectSection +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "loginapp", "server\loginapp\loginapp.vcxproj", "{4AF743A4-8C2F-4424-BB4F-A5B0B5C128BD}" + ProjectSection(ProjectDependencies) = postProject + {896616D2-A5D2-4548-859E-D59905F46140} = {896616D2-A5D2-4548-859E-D59905F46140} + {561FA9F7-14D0-4D94-822C-AAE9B08AACEF} = {561FA9F7-14D0-4D94-822C-AAE9B08AACEF} + EndProjectSection +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bots", "server\tools\bots\bots.vcxproj", "{E17527D0-CCA7-4063-924B-C09A1BD43EC3}" + ProjectSection(ProjectDependencies) = postProject + {561FA9F7-14D0-4D94-822C-AAE9B08AACEF} = {561FA9F7-14D0-4D94-822C-AAE9B08AACEF} + EndProjectSection +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "client_lib", "lib\client_lib\client_lib.vcxproj", "{896616D2-A5D2-4548-859E-D59905F46140}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "guiconsole", "server\tools\guiconsole\guiconsole.vcxproj", "{78EE8B08-4E07-4C07-9592-BC4D09503155}" + ProjectSection(ProjectDependencies) = postProject + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26} = {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26} + {561FA9F7-14D0-4D94-822C-AAE9B08AACEF} = {561FA9F7-14D0-4D94-822C-AAE9B08AACEF} + EndProjectSection +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "resmgr", "lib\resmgr\resmgr.vcxproj", "{3F8B2057-492F-4332-99DE-3E12F20ED489}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "db_interface", "lib\db_interface\db_interface.vcxproj", "{6C92BA78-CFAA-4524-A636-F044B0280AB0}" + ProjectSection(ProjectDependencies) = postProject + {24CB1A43-C6E1-442E-AF99-A91D26BA8FA0} = {24CB1A43-C6E1-442E-AF99-A91D26BA8FA0} + {70E890EA-9496-4F7C-B903-5FCF87A73AC4} = {70E890EA-9496-4F7C-B903-5FCF87A73AC4} + EndProjectSection +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "db_mysql", "lib\db_mysql\db_mysql.vcxproj", "{24CB1A43-C6E1-442E-AF99-A91D26BA8FA0}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "logger", "server\tools\logger\logger.vcxproj", "{E999D336-32D1-48CC-A71F-FC552BD5093B}" + ProjectSection(ProjectDependencies) = postProject + {7A411D5D-DA3B-474D-B08B-3DD787A1F375} = {7A411D5D-DA3B-474D-B08B-3DD787A1F375} + {561FA9F7-14D0-4D94-822C-AAE9B08AACEF} = {561FA9F7-14D0-4D94-822C-AAE9B08AACEF} + EndProjectSection +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "interfaces", "server\tools\interfaces\interfaces.vcxproj", "{7817D13F-1FAA-47D8-8C9F-B790FE2D6B0A}" + ProjectSection(ProjectDependencies) = postProject + {7A411D5D-DA3B-474D-B08B-3DD787A1F375} = {7A411D5D-DA3B-474D-B08B-3DD787A1F375} + {561FA9F7-14D0-4D94-822C-AAE9B08AACEF} = {561FA9F7-14D0-4D94-822C-AAE9B08AACEF} + EndProjectSection +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "jwsmtp", "lib\dependencies\jwsmtp\jwsmtp.vcxproj", "{45A08C1B-D249-42C4-932C-615ED3924CBB}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "navigation", "lib\navigation\navigation.vcxproj", "{9085A36E-CAF8-4E86-93AF-373D2554026B}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libeay32", "lib\dependencies\openssl\vcbuild\libeay32\libeay32.vcxproj", "{70ABE75C-FBC2-4C82-9103-B952FA5CD168}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ssleay32", "lib\dependencies\openssl\vcbuild\ssleay32\ssleay32.vcxproj", "{24265BD3-3CF3-41A9-B137-53DBEC0F53BF}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "apr", "lib\dependencies\apr\apr.vcxproj", "{C6C836E6-2899-458A-90BA-184192342D30}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "aprutil", "lib\dependencies\apr-util\aprutil.vcxproj", "{7CDBBFFF-3B84-413E-BA1A-47CB8D454A46}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "log4cxx", "lib\dependencies\log4cxx\projects\log4cxx.vcxproj", "{561FA9F7-14D0-4D94-822C-AAE9B08AACEF}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "tmxparser", "lib\dependencies\tmxparser\tmxparser\tmxparser.vcxproj", "{42FC1F89-7136-45E8-94E3-594C173C2937}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "log4cxx", "log4cxx", "{67707698-A79D-4C72-8F99-3B96BA970D0D}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "openssl", "openssl", "{B82CBA7A-6640-47BD-94C9-9EEE37CECBE0}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "expat", "lib\dependencies\expat\lib\expat_static.vcxproj", "{30042FAB-AD01-BF78-3C79-F1F6316BC947}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "common", "lib\common\common.vcxproj", "{A04109A7-46C9-42F9-AB29-8E3D84450172}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "srv-tools", "srv-tools", "{54612744-437F-4400-AE6C-BE4CD68C8507}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "python", "python", "{0E68FE27-FE42-4AA2-BE7A-C267B8F5FD2E}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "hiredis", "lib\dependencies\hiredis\redis-win\msvs\hiredis\hiredis.vcxproj", "{13E85053-54B3-487B-8DDB-3430B1C1B3BF}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "hiredis", "hiredis", "{F4FD40AD-244F-4162-88D7-FFD4722FAFAE}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "db_redis", "lib\db_redis\db_redis.vcxproj", "{70E890EA-9496-4F7C-B903-5FCF87A73AC4}" + ProjectSection(ProjectDependencies) = postProject + {13E85053-54B3-487B-8DDB-3430B1C1B3BF} = {13E85053-54B3-487B-8DDB-3430B1C1B3BF} + EndProjectSection +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "kbcmd", "server\tools\kbcmd\kbcmd.vcxproj", "{938D17CB-3F34-46A4-AF64-2099B7FF6902}" + ProjectSection(ProjectDependencies) = postProject + {7A411D5D-DA3B-474D-B08B-3DD787A1F375} = {7A411D5D-DA3B-474D-B08B-3DD787A1F375} + {44FA54E8-0A60-4A6C-A449-118872E75502} = {44FA54E8-0A60-4A6C-A449-118872E75502} + EndProjectSection +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "fmt", "lib\dependencies\fmt\fmt.vcxproj", "{13E95053-54B3-497B-9DDB-3430B1C1B6BF}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "libcurl", "lib\dependencies\curl\projects\Windows\VC14\lib\libcurl.vcxproj", "{DA6F56B4-06A4-441D-AD70-AC5A7D51FADB}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pythoncore", "lib\python\PCbuild\pythoncore.vcxproj", "{CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}" + ProjectSection(ProjectDependencies) = postProject + {745DEC58-EBB3-47A9-A9B8-4C6627C01BF8} = {745DEC58-EBB3-47A9-A9B8-4C6627C01BF8} + {70ABE75C-FBC2-4C82-9103-B952FA5CD168} = {70ABE75C-FBC2-4C82-9103-B952FA5CD168} + {24265BD3-3CF3-41A9-B137-53DBEC0F53BF} = {24265BD3-3CF3-41A9-B137-53DBEC0F53BF} + EndProjectSection +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "pyd", "pyd", "{04274E83-D752-4CC9-BFDE-A717D4E6ABAA}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_socket", "lib\python\PCbuild\_socket.vcxproj", "{86937F53-C189-40EF-8CE8-8759D8E7D480}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "select", "lib\python\PCbuild\select.vcxproj", "{18CAE28C-B454-46C1-87A0-493D91D97F03}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_ctypes", "lib\python\PCbuild\_ctypes.vcxproj", "{0E9791DB-593A-465F-98BC-681011311618}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_elementtree", "lib\python\PCbuild\_elementtree.vcxproj", "{17E1E049-C309-4D79-843F-AE483C264AEA}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pyexpat", "lib\python\PCbuild\pyexpat.vcxproj", "{D06B6426-4762-44CC-8BAD-D79052507F2F}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "unicodedata", "lib\python\PCbuild\unicodedata.vcxproj", "{ECC7CEAC-A5E5-458E-BB9E-2413CC847881}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_ssl", "lib\python\PCbuild\_ssl.vcxproj", "{C6E20F84-3247-4AD6-B051-B073268F73BA}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_hashlib", "lib\python\PCbuild\_hashlib.vcxproj", "{447F05A8-F581-4CAC-A466-5AC7936E207E}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zlib", "lib\dependencies\zlib\contrib\vstudio\vc14\zlibstat.vcxproj", "{745DEC58-EBB3-47A9-A9B8-4C6627C01BF8}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_asyncio", "lib\python\PCbuild\_asyncio.vcxproj", "{384C224A-7474-476E-A01B-750EA7DE918C}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_decimal", "lib\python\PCbuild\_decimal.vcxproj", "{0E9791DB-593A-465F-98BC-681011311617}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_multiprocessing", "lib\python\PCbuild\_multiprocessing.vcxproj", "{9E48B300-37D1-11DD-8C41-005056C00008}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_overlapped", "lib\python\PCbuild\_overlapped.vcxproj", "{EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_queue", "lib\python\PCbuild\_queue.vcxproj", "{78D80A15-BD8C-44E2-B49E-1F05B0A0A687}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Win32 = Debug|Win32 + Debug|Win64 = Debug|Win64 + Release|Win32 = Release|Win32 + Release|Win64 = Release|Win64 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {44FA54E8-0A60-4A6C-A449-118872E75502}.Debug|Win32.ActiveCfg = Debug|Win32 + {44FA54E8-0A60-4A6C-A449-118872E75502}.Debug|Win32.Build.0 = Debug|Win32 + {44FA54E8-0A60-4A6C-A449-118872E75502}.Debug|Win64.ActiveCfg = Debug|x64 + {44FA54E8-0A60-4A6C-A449-118872E75502}.Debug|Win64.Build.0 = Debug|x64 + {44FA54E8-0A60-4A6C-A449-118872E75502}.Release|Win32.ActiveCfg = Release|Win32 + {44FA54E8-0A60-4A6C-A449-118872E75502}.Release|Win32.Build.0 = Release|Win32 + {44FA54E8-0A60-4A6C-A449-118872E75502}.Release|Win64.ActiveCfg = Release|x64 + {44FA54E8-0A60-4A6C-A449-118872E75502}.Release|Win64.Build.0 = Release|x64 + {0E032FA8-BB7B-40F6-8CB1-15F204113B24}.Debug|Win32.ActiveCfg = Debug|Win32 + {0E032FA8-BB7B-40F6-8CB1-15F204113B24}.Debug|Win32.Build.0 = Debug|Win32 + {0E032FA8-BB7B-40F6-8CB1-15F204113B24}.Debug|Win64.ActiveCfg = Debug|x64 + {0E032FA8-BB7B-40F6-8CB1-15F204113B24}.Debug|Win64.Build.0 = Debug|x64 + {0E032FA8-BB7B-40F6-8CB1-15F204113B24}.Release|Win32.ActiveCfg = Release|Win32 + {0E032FA8-BB7B-40F6-8CB1-15F204113B24}.Release|Win32.Build.0 = Release|Win32 + {0E032FA8-BB7B-40F6-8CB1-15F204113B24}.Release|Win64.ActiveCfg = Release|x64 + {0E032FA8-BB7B-40F6-8CB1-15F204113B24}.Release|Win64.Build.0 = Release|x64 + {F67B2C56-D1B1-4EA7-B16E-EF8E7F1B6C5F}.Debug|Win32.ActiveCfg = Debug|Win32 + {F67B2C56-D1B1-4EA7-B16E-EF8E7F1B6C5F}.Debug|Win32.Build.0 = Debug|Win32 + {F67B2C56-D1B1-4EA7-B16E-EF8E7F1B6C5F}.Debug|Win64.ActiveCfg = Debug|x64 + {F67B2C56-D1B1-4EA7-B16E-EF8E7F1B6C5F}.Debug|Win64.Build.0 = Debug|x64 + {F67B2C56-D1B1-4EA7-B16E-EF8E7F1B6C5F}.Release|Win32.ActiveCfg = Release|Win32 + {F67B2C56-D1B1-4EA7-B16E-EF8E7F1B6C5F}.Release|Win32.Build.0 = Release|Win32 + {F67B2C56-D1B1-4EA7-B16E-EF8E7F1B6C5F}.Release|Win64.ActiveCfg = Release|x64 + {F67B2C56-D1B1-4EA7-B16E-EF8E7F1B6C5F}.Release|Win64.Build.0 = Release|x64 + {7A411D5D-DA3B-474D-B08B-3DD787A1F375}.Debug|Win32.ActiveCfg = Debug|Win32 + {7A411D5D-DA3B-474D-B08B-3DD787A1F375}.Debug|Win32.Build.0 = Debug|Win32 + {7A411D5D-DA3B-474D-B08B-3DD787A1F375}.Debug|Win64.ActiveCfg = Debug|x64 + {7A411D5D-DA3B-474D-B08B-3DD787A1F375}.Debug|Win64.Build.0 = Debug|x64 + {7A411D5D-DA3B-474D-B08B-3DD787A1F375}.Release|Win32.ActiveCfg = Release|Win32 + {7A411D5D-DA3B-474D-B08B-3DD787A1F375}.Release|Win32.Build.0 = Release|Win32 + {7A411D5D-DA3B-474D-B08B-3DD787A1F375}.Release|Win64.ActiveCfg = Release|x64 + {7A411D5D-DA3B-474D-B08B-3DD787A1F375}.Release|Win64.Build.0 = Release|x64 + {2992ABDB-0853-4C43-B9E2-98D4211192EE}.Debug|Win32.ActiveCfg = Debug|Win32 + {2992ABDB-0853-4C43-B9E2-98D4211192EE}.Debug|Win32.Build.0 = Debug|Win32 + {2992ABDB-0853-4C43-B9E2-98D4211192EE}.Debug|Win64.ActiveCfg = Debug|x64 + {2992ABDB-0853-4C43-B9E2-98D4211192EE}.Debug|Win64.Build.0 = Debug|x64 + {2992ABDB-0853-4C43-B9E2-98D4211192EE}.Release|Win32.ActiveCfg = Release|Win32 + {2992ABDB-0853-4C43-B9E2-98D4211192EE}.Release|Win32.Build.0 = Release|Win32 + {2992ABDB-0853-4C43-B9E2-98D4211192EE}.Release|Win64.ActiveCfg = Release|x64 + {2992ABDB-0853-4C43-B9E2-98D4211192EE}.Release|Win64.Build.0 = Release|x64 + {56D453A7-787B-49E3-84BC-465EED38DE8A}.Debug|Win32.ActiveCfg = Debug|Win32 + {56D453A7-787B-49E3-84BC-465EED38DE8A}.Debug|Win32.Build.0 = Debug|Win32 + {56D453A7-787B-49E3-84BC-465EED38DE8A}.Debug|Win64.ActiveCfg = Debug|x64 + {56D453A7-787B-49E3-84BC-465EED38DE8A}.Debug|Win64.Build.0 = Debug|x64 + {56D453A7-787B-49E3-84BC-465EED38DE8A}.Release|Win32.ActiveCfg = Release|Win32 + {56D453A7-787B-49E3-84BC-465EED38DE8A}.Release|Win32.Build.0 = Release|Win32 + {56D453A7-787B-49E3-84BC-465EED38DE8A}.Release|Win64.ActiveCfg = Release|x64 + {56D453A7-787B-49E3-84BC-465EED38DE8A}.Release|Win64.Build.0 = Release|x64 + {5EF24499-4F74-4AF6-8048-650BE7BD7808}.Debug|Win32.ActiveCfg = Debug|Win32 + {5EF24499-4F74-4AF6-8048-650BE7BD7808}.Debug|Win32.Build.0 = Debug|Win32 + {5EF24499-4F74-4AF6-8048-650BE7BD7808}.Debug|Win64.ActiveCfg = Debug|x64 + {5EF24499-4F74-4AF6-8048-650BE7BD7808}.Debug|Win64.Build.0 = Debug|x64 + {5EF24499-4F74-4AF6-8048-650BE7BD7808}.Release|Win32.ActiveCfg = Release|Win32 + {5EF24499-4F74-4AF6-8048-650BE7BD7808}.Release|Win32.Build.0 = Release|Win32 + {5EF24499-4F74-4AF6-8048-650BE7BD7808}.Release|Win64.ActiveCfg = Release|x64 + {5EF24499-4F74-4AF6-8048-650BE7BD7808}.Release|Win64.Build.0 = Release|x64 + {863B8043-533C-499D-B48E-F5EC91A6CB9B}.Debug|Win32.ActiveCfg = Debug|Win32 + {863B8043-533C-499D-B48E-F5EC91A6CB9B}.Debug|Win32.Build.0 = Debug|Win32 + {863B8043-533C-499D-B48E-F5EC91A6CB9B}.Debug|Win64.ActiveCfg = Debug|x64 + {863B8043-533C-499D-B48E-F5EC91A6CB9B}.Debug|Win64.Build.0 = Debug|x64 + {863B8043-533C-499D-B48E-F5EC91A6CB9B}.Release|Win32.ActiveCfg = Release|Win32 + {863B8043-533C-499D-B48E-F5EC91A6CB9B}.Release|Win32.Build.0 = Release|Win32 + {863B8043-533C-499D-B48E-F5EC91A6CB9B}.Release|Win64.ActiveCfg = Release|x64 + {863B8043-533C-499D-B48E-F5EC91A6CB9B}.Release|Win64.Build.0 = Release|x64 + {04C620E9-BE02-498E-A713-B9E470D75F15}.Debug|Win32.ActiveCfg = Debug|Win32 + {04C620E9-BE02-498E-A713-B9E470D75F15}.Debug|Win32.Build.0 = Debug|Win32 + {04C620E9-BE02-498E-A713-B9E470D75F15}.Debug|Win64.ActiveCfg = Debug|x64 + {04C620E9-BE02-498E-A713-B9E470D75F15}.Debug|Win64.Build.0 = Debug|x64 + {04C620E9-BE02-498E-A713-B9E470D75F15}.Release|Win32.ActiveCfg = Release|Win32 + {04C620E9-BE02-498E-A713-B9E470D75F15}.Release|Win32.Build.0 = Release|Win32 + {04C620E9-BE02-498E-A713-B9E470D75F15}.Release|Win64.ActiveCfg = Release|x64 + {04C620E9-BE02-498E-A713-B9E470D75F15}.Release|Win64.Build.0 = Release|x64 + {89DA21A1-625B-4482-B386-D7C0AF97E477}.Debug|Win32.ActiveCfg = Debug|Win32 + {89DA21A1-625B-4482-B386-D7C0AF97E477}.Debug|Win32.Build.0 = Debug|Win32 + {89DA21A1-625B-4482-B386-D7C0AF97E477}.Debug|Win64.ActiveCfg = Debug|x64 + {89DA21A1-625B-4482-B386-D7C0AF97E477}.Debug|Win64.Build.0 = Debug|x64 + {89DA21A1-625B-4482-B386-D7C0AF97E477}.Release|Win32.ActiveCfg = Release|Win32 + {89DA21A1-625B-4482-B386-D7C0AF97E477}.Release|Win32.Build.0 = Release|Win32 + {89DA21A1-625B-4482-B386-D7C0AF97E477}.Release|Win64.ActiveCfg = Release|x64 + {89DA21A1-625B-4482-B386-D7C0AF97E477}.Release|Win64.Build.0 = Release|x64 + {1BAD0518-F2BB-42F5-83E7-9E1804FAD7C1}.Debug|Win32.ActiveCfg = Debug|Win32 + {1BAD0518-F2BB-42F5-83E7-9E1804FAD7C1}.Debug|Win32.Build.0 = Debug|Win32 + {1BAD0518-F2BB-42F5-83E7-9E1804FAD7C1}.Debug|Win64.ActiveCfg = Debug|x64 + {1BAD0518-F2BB-42F5-83E7-9E1804FAD7C1}.Debug|Win64.Build.0 = Debug|x64 + {1BAD0518-F2BB-42F5-83E7-9E1804FAD7C1}.Release|Win32.ActiveCfg = Release|Win32 + {1BAD0518-F2BB-42F5-83E7-9E1804FAD7C1}.Release|Win32.Build.0 = Release|Win32 + {1BAD0518-F2BB-42F5-83E7-9E1804FAD7C1}.Release|Win64.ActiveCfg = Release|x64 + {1BAD0518-F2BB-42F5-83E7-9E1804FAD7C1}.Release|Win64.Build.0 = Release|x64 + {FE88370E-80F7-49BD-A8D1-E7A2CA699E44}.Debug|Win32.ActiveCfg = Debug|Win32 + {FE88370E-80F7-49BD-A8D1-E7A2CA699E44}.Debug|Win32.Build.0 = Debug|Win32 + {FE88370E-80F7-49BD-A8D1-E7A2CA699E44}.Debug|Win64.ActiveCfg = Debug|x64 + {FE88370E-80F7-49BD-A8D1-E7A2CA699E44}.Debug|Win64.Build.0 = Debug|x64 + {FE88370E-80F7-49BD-A8D1-E7A2CA699E44}.Release|Win32.ActiveCfg = Release|Win32 + {FE88370E-80F7-49BD-A8D1-E7A2CA699E44}.Release|Win32.Build.0 = Release|Win32 + {FE88370E-80F7-49BD-A8D1-E7A2CA699E44}.Release|Win64.ActiveCfg = Release|x64 + {FE88370E-80F7-49BD-A8D1-E7A2CA699E44}.Release|Win64.Build.0 = Release|x64 + {9EA2E328-E6BE-4DF0-A8DC-05C08A0B822B}.Debug|Win32.ActiveCfg = Debug|Win32 + {9EA2E328-E6BE-4DF0-A8DC-05C08A0B822B}.Debug|Win32.Build.0 = Debug|Win32 + {9EA2E328-E6BE-4DF0-A8DC-05C08A0B822B}.Debug|Win64.ActiveCfg = Debug|x64 + {9EA2E328-E6BE-4DF0-A8DC-05C08A0B822B}.Debug|Win64.Build.0 = Debug|x64 + {9EA2E328-E6BE-4DF0-A8DC-05C08A0B822B}.Release|Win32.ActiveCfg = Release|Win32 + {9EA2E328-E6BE-4DF0-A8DC-05C08A0B822B}.Release|Win32.Build.0 = Release|Win32 + {9EA2E328-E6BE-4DF0-A8DC-05C08A0B822B}.Release|Win64.ActiveCfg = Release|x64 + {9EA2E328-E6BE-4DF0-A8DC-05C08A0B822B}.Release|Win64.Build.0 = Release|x64 + {69C18C83-5753-4F36-BF5A-6EE217304D2D}.Debug|Win32.ActiveCfg = Debug|Win32 + {69C18C83-5753-4F36-BF5A-6EE217304D2D}.Debug|Win32.Build.0 = Debug|Win32 + {69C18C83-5753-4F36-BF5A-6EE217304D2D}.Debug|Win64.ActiveCfg = Debug|x64 + {69C18C83-5753-4F36-BF5A-6EE217304D2D}.Debug|Win64.Build.0 = Debug|x64 + {69C18C83-5753-4F36-BF5A-6EE217304D2D}.Release|Win32.ActiveCfg = Release|Win32 + {69C18C83-5753-4F36-BF5A-6EE217304D2D}.Release|Win32.Build.0 = Release|Win32 + {69C18C83-5753-4F36-BF5A-6EE217304D2D}.Release|Win64.ActiveCfg = Release|x64 + {69C18C83-5753-4F36-BF5A-6EE217304D2D}.Release|Win64.Build.0 = Release|x64 + {4AF743A4-8C2F-4424-BB4F-A5B0B5C128BD}.Debug|Win32.ActiveCfg = Debug|Win32 + {4AF743A4-8C2F-4424-BB4F-A5B0B5C128BD}.Debug|Win32.Build.0 = Debug|Win32 + {4AF743A4-8C2F-4424-BB4F-A5B0B5C128BD}.Debug|Win64.ActiveCfg = Debug|x64 + {4AF743A4-8C2F-4424-BB4F-A5B0B5C128BD}.Debug|Win64.Build.0 = Debug|x64 + {4AF743A4-8C2F-4424-BB4F-A5B0B5C128BD}.Release|Win32.ActiveCfg = Release|Win32 + {4AF743A4-8C2F-4424-BB4F-A5B0B5C128BD}.Release|Win32.Build.0 = Release|Win32 + {4AF743A4-8C2F-4424-BB4F-A5B0B5C128BD}.Release|Win64.ActiveCfg = Release|x64 + {4AF743A4-8C2F-4424-BB4F-A5B0B5C128BD}.Release|Win64.Build.0 = Release|x64 + {E17527D0-CCA7-4063-924B-C09A1BD43EC3}.Debug|Win32.ActiveCfg = Debug|Win32 + {E17527D0-CCA7-4063-924B-C09A1BD43EC3}.Debug|Win32.Build.0 = Debug|Win32 + {E17527D0-CCA7-4063-924B-C09A1BD43EC3}.Debug|Win64.ActiveCfg = Debug|x64 + {E17527D0-CCA7-4063-924B-C09A1BD43EC3}.Debug|Win64.Build.0 = Debug|x64 + {E17527D0-CCA7-4063-924B-C09A1BD43EC3}.Release|Win32.ActiveCfg = Release|Win32 + {E17527D0-CCA7-4063-924B-C09A1BD43EC3}.Release|Win32.Build.0 = Release|Win32 + {E17527D0-CCA7-4063-924B-C09A1BD43EC3}.Release|Win64.ActiveCfg = Release|x64 + {E17527D0-CCA7-4063-924B-C09A1BD43EC3}.Release|Win64.Build.0 = Release|x64 + {896616D2-A5D2-4548-859E-D59905F46140}.Debug|Win32.ActiveCfg = Debug|Win32 + {896616D2-A5D2-4548-859E-D59905F46140}.Debug|Win32.Build.0 = Debug|Win32 + {896616D2-A5D2-4548-859E-D59905F46140}.Debug|Win64.ActiveCfg = Debug|x64 + {896616D2-A5D2-4548-859E-D59905F46140}.Debug|Win64.Build.0 = Debug|x64 + {896616D2-A5D2-4548-859E-D59905F46140}.Release|Win32.ActiveCfg = Release|Win32 + {896616D2-A5D2-4548-859E-D59905F46140}.Release|Win32.Build.0 = Release|Win32 + {896616D2-A5D2-4548-859E-D59905F46140}.Release|Win64.ActiveCfg = Release|x64 + {896616D2-A5D2-4548-859E-D59905F46140}.Release|Win64.Build.0 = Release|x64 + {78EE8B08-4E07-4C07-9592-BC4D09503155}.Debug|Win32.ActiveCfg = Debug|Win32 + {78EE8B08-4E07-4C07-9592-BC4D09503155}.Debug|Win32.Build.0 = Debug|Win32 + {78EE8B08-4E07-4C07-9592-BC4D09503155}.Debug|Win64.ActiveCfg = Debug|x64 + {78EE8B08-4E07-4C07-9592-BC4D09503155}.Debug|Win64.Build.0 = Debug|x64 + {78EE8B08-4E07-4C07-9592-BC4D09503155}.Release|Win32.ActiveCfg = Release|Win32 + {78EE8B08-4E07-4C07-9592-BC4D09503155}.Release|Win32.Build.0 = Release|Win32 + {78EE8B08-4E07-4C07-9592-BC4D09503155}.Release|Win64.ActiveCfg = Release|x64 + {78EE8B08-4E07-4C07-9592-BC4D09503155}.Release|Win64.Build.0 = Release|x64 + {3F8B2057-492F-4332-99DE-3E12F20ED489}.Debug|Win32.ActiveCfg = Debug|Win32 + {3F8B2057-492F-4332-99DE-3E12F20ED489}.Debug|Win32.Build.0 = Debug|Win32 + {3F8B2057-492F-4332-99DE-3E12F20ED489}.Debug|Win64.ActiveCfg = Debug|x64 + {3F8B2057-492F-4332-99DE-3E12F20ED489}.Debug|Win64.Build.0 = Debug|x64 + {3F8B2057-492F-4332-99DE-3E12F20ED489}.Release|Win32.ActiveCfg = Release|Win32 + {3F8B2057-492F-4332-99DE-3E12F20ED489}.Release|Win32.Build.0 = Release|Win32 + {3F8B2057-492F-4332-99DE-3E12F20ED489}.Release|Win64.ActiveCfg = Release|x64 + {3F8B2057-492F-4332-99DE-3E12F20ED489}.Release|Win64.Build.0 = Release|x64 + {6C92BA78-CFAA-4524-A636-F044B0280AB0}.Debug|Win32.ActiveCfg = Debug|Win32 + {6C92BA78-CFAA-4524-A636-F044B0280AB0}.Debug|Win32.Build.0 = Debug|Win32 + {6C92BA78-CFAA-4524-A636-F044B0280AB0}.Debug|Win64.ActiveCfg = Debug|x64 + {6C92BA78-CFAA-4524-A636-F044B0280AB0}.Debug|Win64.Build.0 = Debug|x64 + {6C92BA78-CFAA-4524-A636-F044B0280AB0}.Release|Win32.ActiveCfg = Release|Win32 + {6C92BA78-CFAA-4524-A636-F044B0280AB0}.Release|Win32.Build.0 = Release|Win32 + {6C92BA78-CFAA-4524-A636-F044B0280AB0}.Release|Win64.ActiveCfg = Release|x64 + {6C92BA78-CFAA-4524-A636-F044B0280AB0}.Release|Win64.Build.0 = Release|x64 + {24CB1A43-C6E1-442E-AF99-A91D26BA8FA0}.Debug|Win32.ActiveCfg = Debug|Win32 + {24CB1A43-C6E1-442E-AF99-A91D26BA8FA0}.Debug|Win32.Build.0 = Debug|Win32 + {24CB1A43-C6E1-442E-AF99-A91D26BA8FA0}.Debug|Win64.ActiveCfg = Debug|x64 + {24CB1A43-C6E1-442E-AF99-A91D26BA8FA0}.Debug|Win64.Build.0 = Debug|x64 + {24CB1A43-C6E1-442E-AF99-A91D26BA8FA0}.Release|Win32.ActiveCfg = Release|Win32 + {24CB1A43-C6E1-442E-AF99-A91D26BA8FA0}.Release|Win32.Build.0 = Release|Win32 + {24CB1A43-C6E1-442E-AF99-A91D26BA8FA0}.Release|Win64.ActiveCfg = Release|x64 + {24CB1A43-C6E1-442E-AF99-A91D26BA8FA0}.Release|Win64.Build.0 = Release|x64 + {E999D336-32D1-48CC-A71F-FC552BD5093B}.Debug|Win32.ActiveCfg = Debug|Win32 + {E999D336-32D1-48CC-A71F-FC552BD5093B}.Debug|Win32.Build.0 = Debug|Win32 + {E999D336-32D1-48CC-A71F-FC552BD5093B}.Debug|Win64.ActiveCfg = Debug|x64 + {E999D336-32D1-48CC-A71F-FC552BD5093B}.Debug|Win64.Build.0 = Debug|x64 + {E999D336-32D1-48CC-A71F-FC552BD5093B}.Release|Win32.ActiveCfg = Release|Win32 + {E999D336-32D1-48CC-A71F-FC552BD5093B}.Release|Win32.Build.0 = Release|Win32 + {E999D336-32D1-48CC-A71F-FC552BD5093B}.Release|Win64.ActiveCfg = Release|x64 + {E999D336-32D1-48CC-A71F-FC552BD5093B}.Release|Win64.Build.0 = Release|x64 + {7817D13F-1FAA-47D8-8C9F-B790FE2D6B0A}.Debug|Win32.ActiveCfg = Debug|Win32 + {7817D13F-1FAA-47D8-8C9F-B790FE2D6B0A}.Debug|Win32.Build.0 = Debug|Win32 + {7817D13F-1FAA-47D8-8C9F-B790FE2D6B0A}.Debug|Win64.ActiveCfg = Debug|x64 + {7817D13F-1FAA-47D8-8C9F-B790FE2D6B0A}.Debug|Win64.Build.0 = Debug|x64 + {7817D13F-1FAA-47D8-8C9F-B790FE2D6B0A}.Release|Win32.ActiveCfg = Release|Win32 + {7817D13F-1FAA-47D8-8C9F-B790FE2D6B0A}.Release|Win32.Build.0 = Release|Win32 + {7817D13F-1FAA-47D8-8C9F-B790FE2D6B0A}.Release|Win64.ActiveCfg = Release|x64 + {7817D13F-1FAA-47D8-8C9F-B790FE2D6B0A}.Release|Win64.Build.0 = Release|x64 + {45A08C1B-D249-42C4-932C-615ED3924CBB}.Debug|Win32.ActiveCfg = Debug|Win32 + {45A08C1B-D249-42C4-932C-615ED3924CBB}.Debug|Win32.Build.0 = Debug|Win32 + {45A08C1B-D249-42C4-932C-615ED3924CBB}.Debug|Win64.ActiveCfg = Debug|x64 + {45A08C1B-D249-42C4-932C-615ED3924CBB}.Debug|Win64.Build.0 = Debug|x64 + {45A08C1B-D249-42C4-932C-615ED3924CBB}.Release|Win32.ActiveCfg = Release|Win32 + {45A08C1B-D249-42C4-932C-615ED3924CBB}.Release|Win32.Build.0 = Release|Win32 + {45A08C1B-D249-42C4-932C-615ED3924CBB}.Release|Win64.ActiveCfg = Release|x64 + {45A08C1B-D249-42C4-932C-615ED3924CBB}.Release|Win64.Build.0 = Release|x64 + {9085A36E-CAF8-4E86-93AF-373D2554026B}.Debug|Win32.ActiveCfg = Debug|Win32 + {9085A36E-CAF8-4E86-93AF-373D2554026B}.Debug|Win32.Build.0 = Debug|Win32 + {9085A36E-CAF8-4E86-93AF-373D2554026B}.Debug|Win64.ActiveCfg = Debug|x64 + {9085A36E-CAF8-4E86-93AF-373D2554026B}.Debug|Win64.Build.0 = Debug|x64 + {9085A36E-CAF8-4E86-93AF-373D2554026B}.Release|Win32.ActiveCfg = Release|Win32 + {9085A36E-CAF8-4E86-93AF-373D2554026B}.Release|Win32.Build.0 = Release|Win32 + {9085A36E-CAF8-4E86-93AF-373D2554026B}.Release|Win64.ActiveCfg = Release|x64 + {9085A36E-CAF8-4E86-93AF-373D2554026B}.Release|Win64.Build.0 = Release|x64 + {70ABE75C-FBC2-4C82-9103-B952FA5CD168}.Debug|Win32.ActiveCfg = Debug|Win32 + {70ABE75C-FBC2-4C82-9103-B952FA5CD168}.Debug|Win32.Build.0 = Debug|Win32 + {70ABE75C-FBC2-4C82-9103-B952FA5CD168}.Debug|Win64.ActiveCfg = Debug|x64 + {70ABE75C-FBC2-4C82-9103-B952FA5CD168}.Debug|Win64.Build.0 = Debug|x64 + {70ABE75C-FBC2-4C82-9103-B952FA5CD168}.Release|Win32.ActiveCfg = Release|Win32 + {70ABE75C-FBC2-4C82-9103-B952FA5CD168}.Release|Win32.Build.0 = Release|Win32 + {70ABE75C-FBC2-4C82-9103-B952FA5CD168}.Release|Win64.ActiveCfg = Release|x64 + {70ABE75C-FBC2-4C82-9103-B952FA5CD168}.Release|Win64.Build.0 = Release|x64 + {24265BD3-3CF3-41A9-B137-53DBEC0F53BF}.Debug|Win32.ActiveCfg = Debug|Win32 + {24265BD3-3CF3-41A9-B137-53DBEC0F53BF}.Debug|Win32.Build.0 = Debug|Win32 + {24265BD3-3CF3-41A9-B137-53DBEC0F53BF}.Debug|Win64.ActiveCfg = Debug|x64 + {24265BD3-3CF3-41A9-B137-53DBEC0F53BF}.Debug|Win64.Build.0 = Debug|x64 + {24265BD3-3CF3-41A9-B137-53DBEC0F53BF}.Release|Win32.ActiveCfg = Release|Win32 + {24265BD3-3CF3-41A9-B137-53DBEC0F53BF}.Release|Win32.Build.0 = Release|Win32 + {24265BD3-3CF3-41A9-B137-53DBEC0F53BF}.Release|Win64.ActiveCfg = Release|x64 + {24265BD3-3CF3-41A9-B137-53DBEC0F53BF}.Release|Win64.Build.0 = Release|x64 + {C6C836E6-2899-458A-90BA-184192342D30}.Debug|Win32.ActiveCfg = Debug|Win32 + {C6C836E6-2899-458A-90BA-184192342D30}.Debug|Win32.Build.0 = Debug|Win32 + {C6C836E6-2899-458A-90BA-184192342D30}.Debug|Win64.ActiveCfg = Debug|x64 + {C6C836E6-2899-458A-90BA-184192342D30}.Debug|Win64.Build.0 = Debug|x64 + {C6C836E6-2899-458A-90BA-184192342D30}.Release|Win32.ActiveCfg = Release|Win32 + {C6C836E6-2899-458A-90BA-184192342D30}.Release|Win32.Build.0 = Release|Win32 + {C6C836E6-2899-458A-90BA-184192342D30}.Release|Win64.ActiveCfg = Release|x64 + {C6C836E6-2899-458A-90BA-184192342D30}.Release|Win64.Build.0 = Release|x64 + {7CDBBFFF-3B84-413E-BA1A-47CB8D454A46}.Debug|Win32.ActiveCfg = Debug|Win32 + {7CDBBFFF-3B84-413E-BA1A-47CB8D454A46}.Debug|Win32.Build.0 = Debug|Win32 + {7CDBBFFF-3B84-413E-BA1A-47CB8D454A46}.Debug|Win64.ActiveCfg = Debug|x64 + {7CDBBFFF-3B84-413E-BA1A-47CB8D454A46}.Debug|Win64.Build.0 = Debug|x64 + {7CDBBFFF-3B84-413E-BA1A-47CB8D454A46}.Release|Win32.ActiveCfg = Release|Win32 + {7CDBBFFF-3B84-413E-BA1A-47CB8D454A46}.Release|Win32.Build.0 = Release|Win32 + {7CDBBFFF-3B84-413E-BA1A-47CB8D454A46}.Release|Win64.ActiveCfg = Release|x64 + {7CDBBFFF-3B84-413E-BA1A-47CB8D454A46}.Release|Win64.Build.0 = Release|x64 + {561FA9F7-14D0-4D94-822C-AAE9B08AACEF}.Debug|Win32.ActiveCfg = Debug|Win32 + {561FA9F7-14D0-4D94-822C-AAE9B08AACEF}.Debug|Win32.Build.0 = Debug|Win32 + {561FA9F7-14D0-4D94-822C-AAE9B08AACEF}.Debug|Win64.ActiveCfg = Debug|x64 + {561FA9F7-14D0-4D94-822C-AAE9B08AACEF}.Debug|Win64.Build.0 = Debug|x64 + {561FA9F7-14D0-4D94-822C-AAE9B08AACEF}.Release|Win32.ActiveCfg = Release|Win32 + {561FA9F7-14D0-4D94-822C-AAE9B08AACEF}.Release|Win32.Build.0 = Release|Win32 + {561FA9F7-14D0-4D94-822C-AAE9B08AACEF}.Release|Win64.ActiveCfg = Release|x64 + {561FA9F7-14D0-4D94-822C-AAE9B08AACEF}.Release|Win64.Build.0 = Release|x64 + {42FC1F89-7136-45E8-94E3-594C173C2937}.Debug|Win32.ActiveCfg = Debug|Win32 + {42FC1F89-7136-45E8-94E3-594C173C2937}.Debug|Win32.Build.0 = Debug|Win32 + {42FC1F89-7136-45E8-94E3-594C173C2937}.Debug|Win64.ActiveCfg = Debug|x64 + {42FC1F89-7136-45E8-94E3-594C173C2937}.Debug|Win64.Build.0 = Debug|x64 + {42FC1F89-7136-45E8-94E3-594C173C2937}.Release|Win32.ActiveCfg = Release|Win32 + {42FC1F89-7136-45E8-94E3-594C173C2937}.Release|Win32.Build.0 = Release|Win32 + {42FC1F89-7136-45E8-94E3-594C173C2937}.Release|Win64.ActiveCfg = Release|x64 + {42FC1F89-7136-45E8-94E3-594C173C2937}.Release|Win64.Build.0 = Release|x64 + {30042FAB-AD01-BF78-3C79-F1F6316BC947}.Debug|Win32.ActiveCfg = Debug|Win32 + {30042FAB-AD01-BF78-3C79-F1F6316BC947}.Debug|Win32.Build.0 = Debug|Win32 + {30042FAB-AD01-BF78-3C79-F1F6316BC947}.Debug|Win64.ActiveCfg = Debug|x64 + {30042FAB-AD01-BF78-3C79-F1F6316BC947}.Debug|Win64.Build.0 = Debug|x64 + {30042FAB-AD01-BF78-3C79-F1F6316BC947}.Release|Win32.ActiveCfg = Release|Win32 + {30042FAB-AD01-BF78-3C79-F1F6316BC947}.Release|Win32.Build.0 = Release|Win32 + {30042FAB-AD01-BF78-3C79-F1F6316BC947}.Release|Win64.ActiveCfg = Release|x64 + {30042FAB-AD01-BF78-3C79-F1F6316BC947}.Release|Win64.Build.0 = Release|x64 + {A04109A7-46C9-42F9-AB29-8E3D84450172}.Debug|Win32.ActiveCfg = Debug|Win32 + {A04109A7-46C9-42F9-AB29-8E3D84450172}.Debug|Win32.Build.0 = Debug|Win32 + {A04109A7-46C9-42F9-AB29-8E3D84450172}.Debug|Win64.ActiveCfg = Debug|x64 + {A04109A7-46C9-42F9-AB29-8E3D84450172}.Debug|Win64.Build.0 = Debug|x64 + {A04109A7-46C9-42F9-AB29-8E3D84450172}.Release|Win32.ActiveCfg = Release|Win32 + {A04109A7-46C9-42F9-AB29-8E3D84450172}.Release|Win32.Build.0 = Release|Win32 + {A04109A7-46C9-42F9-AB29-8E3D84450172}.Release|Win64.ActiveCfg = Release|x64 + {A04109A7-46C9-42F9-AB29-8E3D84450172}.Release|Win64.Build.0 = Release|x64 + {13E85053-54B3-487B-8DDB-3430B1C1B3BF}.Debug|Win32.ActiveCfg = Debug|Win32 + {13E85053-54B3-487B-8DDB-3430B1C1B3BF}.Debug|Win32.Build.0 = Debug|Win32 + {13E85053-54B3-487B-8DDB-3430B1C1B3BF}.Debug|Win64.ActiveCfg = Debug|x64 + {13E85053-54B3-487B-8DDB-3430B1C1B3BF}.Debug|Win64.Build.0 = Debug|x64 + {13E85053-54B3-487B-8DDB-3430B1C1B3BF}.Release|Win32.ActiveCfg = Release|Win32 + {13E85053-54B3-487B-8DDB-3430B1C1B3BF}.Release|Win32.Build.0 = Release|Win32 + {13E85053-54B3-487B-8DDB-3430B1C1B3BF}.Release|Win64.ActiveCfg = Release|x64 + {13E85053-54B3-487B-8DDB-3430B1C1B3BF}.Release|Win64.Build.0 = Release|x64 + {70E890EA-9496-4F7C-B903-5FCF87A73AC4}.Debug|Win32.ActiveCfg = Debug|Win32 + {70E890EA-9496-4F7C-B903-5FCF87A73AC4}.Debug|Win32.Build.0 = Debug|Win32 + {70E890EA-9496-4F7C-B903-5FCF87A73AC4}.Debug|Win64.ActiveCfg = Debug|x64 + {70E890EA-9496-4F7C-B903-5FCF87A73AC4}.Debug|Win64.Build.0 = Debug|x64 + {70E890EA-9496-4F7C-B903-5FCF87A73AC4}.Release|Win32.ActiveCfg = Release|Win32 + {70E890EA-9496-4F7C-B903-5FCF87A73AC4}.Release|Win32.Build.0 = Release|Win32 + {70E890EA-9496-4F7C-B903-5FCF87A73AC4}.Release|Win64.ActiveCfg = Release|x64 + {70E890EA-9496-4F7C-B903-5FCF87A73AC4}.Release|Win64.Build.0 = Release|x64 + {938D17CB-3F34-46A4-AF64-2099B7FF6902}.Debug|Win32.ActiveCfg = Debug|Win32 + {938D17CB-3F34-46A4-AF64-2099B7FF6902}.Debug|Win32.Build.0 = Debug|Win32 + {938D17CB-3F34-46A4-AF64-2099B7FF6902}.Debug|Win64.ActiveCfg = Debug|x64 + {938D17CB-3F34-46A4-AF64-2099B7FF6902}.Debug|Win64.Build.0 = Debug|x64 + {938D17CB-3F34-46A4-AF64-2099B7FF6902}.Release|Win32.ActiveCfg = Release|Win32 + {938D17CB-3F34-46A4-AF64-2099B7FF6902}.Release|Win32.Build.0 = Release|Win32 + {938D17CB-3F34-46A4-AF64-2099B7FF6902}.Release|Win64.ActiveCfg = Release|x64 + {938D17CB-3F34-46A4-AF64-2099B7FF6902}.Release|Win64.Build.0 = Release|x64 + {13E95053-54B3-497B-9DDB-3430B1C1B6BF}.Debug|Win32.ActiveCfg = Debug|Win32 + {13E95053-54B3-497B-9DDB-3430B1C1B6BF}.Debug|Win32.Build.0 = Debug|Win32 + {13E95053-54B3-497B-9DDB-3430B1C1B6BF}.Debug|Win64.ActiveCfg = Debug|x64 + {13E95053-54B3-497B-9DDB-3430B1C1B6BF}.Debug|Win64.Build.0 = Debug|x64 + {13E95053-54B3-497B-9DDB-3430B1C1B6BF}.Release|Win32.ActiveCfg = Release|Win32 + {13E95053-54B3-497B-9DDB-3430B1C1B6BF}.Release|Win32.Build.0 = Release|Win32 + {13E95053-54B3-497B-9DDB-3430B1C1B6BF}.Release|Win64.ActiveCfg = Release|x64 + {13E95053-54B3-497B-9DDB-3430B1C1B6BF}.Release|Win64.Build.0 = Release|x64 + {DA6F56B4-06A4-441D-AD70-AC5A7D51FADB}.Debug|Win32.ActiveCfg = LIB Debug - LIB OpenSSL|Win32 + {DA6F56B4-06A4-441D-AD70-AC5A7D51FADB}.Debug|Win32.Build.0 = LIB Debug - LIB OpenSSL|Win32 + {DA6F56B4-06A4-441D-AD70-AC5A7D51FADB}.Debug|Win64.ActiveCfg = LIB Debug - LIB OpenSSL|x64 + {DA6F56B4-06A4-441D-AD70-AC5A7D51FADB}.Debug|Win64.Build.0 = LIB Debug - LIB OpenSSL|x64 + {DA6F56B4-06A4-441D-AD70-AC5A7D51FADB}.Release|Win32.ActiveCfg = LIB Release - LIB OpenSSL|Win32 + {DA6F56B4-06A4-441D-AD70-AC5A7D51FADB}.Release|Win32.Build.0 = LIB Release - LIB OpenSSL|Win32 + {DA6F56B4-06A4-441D-AD70-AC5A7D51FADB}.Release|Win64.ActiveCfg = LIB Release - LIB OpenSSL|x64 + {DA6F56B4-06A4-441D-AD70-AC5A7D51FADB}.Release|Win64.Build.0 = LIB Release - LIB OpenSSL|x64 + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Debug|Win32.ActiveCfg = Debug|Win32 + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Debug|Win32.Build.0 = Debug|Win32 + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Debug|Win64.ActiveCfg = Debug|x64 + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Debug|Win64.Build.0 = Debug|x64 + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Release|Win32.ActiveCfg = Release|Win32 + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Release|Win32.Build.0 = Release|Win32 + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Release|Win64.ActiveCfg = Release|x64 + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26}.Release|Win64.Build.0 = Release|x64 + {86937F53-C189-40EF-8CE8-8759D8E7D480}.Debug|Win32.ActiveCfg = Debug|Win32 + {86937F53-C189-40EF-8CE8-8759D8E7D480}.Debug|Win32.Build.0 = Debug|Win32 + {86937F53-C189-40EF-8CE8-8759D8E7D480}.Debug|Win64.ActiveCfg = Debug|x64 + {86937F53-C189-40EF-8CE8-8759D8E7D480}.Debug|Win64.Build.0 = Debug|x64 + {86937F53-C189-40EF-8CE8-8759D8E7D480}.Release|Win32.ActiveCfg = Release|Win32 + {86937F53-C189-40EF-8CE8-8759D8E7D480}.Release|Win32.Build.0 = Release|Win32 + {86937F53-C189-40EF-8CE8-8759D8E7D480}.Release|Win64.ActiveCfg = Release|x64 + {86937F53-C189-40EF-8CE8-8759D8E7D480}.Release|Win64.Build.0 = Release|x64 + {18CAE28C-B454-46C1-87A0-493D91D97F03}.Debug|Win32.ActiveCfg = Debug|Win32 + {18CAE28C-B454-46C1-87A0-493D91D97F03}.Debug|Win32.Build.0 = Debug|Win32 + {18CAE28C-B454-46C1-87A0-493D91D97F03}.Debug|Win64.ActiveCfg = Debug|x64 + {18CAE28C-B454-46C1-87A0-493D91D97F03}.Debug|Win64.Build.0 = Debug|x64 + {18CAE28C-B454-46C1-87A0-493D91D97F03}.Release|Win32.ActiveCfg = Release|Win32 + {18CAE28C-B454-46C1-87A0-493D91D97F03}.Release|Win32.Build.0 = Release|Win32 + {18CAE28C-B454-46C1-87A0-493D91D97F03}.Release|Win64.ActiveCfg = Release|x64 + {18CAE28C-B454-46C1-87A0-493D91D97F03}.Release|Win64.Build.0 = Release|x64 + {0E9791DB-593A-465F-98BC-681011311618}.Debug|Win32.ActiveCfg = Debug|Win32 + {0E9791DB-593A-465F-98BC-681011311618}.Debug|Win32.Build.0 = Debug|Win32 + {0E9791DB-593A-465F-98BC-681011311618}.Debug|Win64.ActiveCfg = Debug|x64 + {0E9791DB-593A-465F-98BC-681011311618}.Debug|Win64.Build.0 = Debug|x64 + {0E9791DB-593A-465F-98BC-681011311618}.Release|Win32.ActiveCfg = Release|Win32 + {0E9791DB-593A-465F-98BC-681011311618}.Release|Win32.Build.0 = Release|Win32 + {0E9791DB-593A-465F-98BC-681011311618}.Release|Win64.ActiveCfg = Release|x64 + {0E9791DB-593A-465F-98BC-681011311618}.Release|Win64.Build.0 = Release|x64 + {17E1E049-C309-4D79-843F-AE483C264AEA}.Debug|Win32.ActiveCfg = Debug|Win32 + {17E1E049-C309-4D79-843F-AE483C264AEA}.Debug|Win32.Build.0 = Debug|Win32 + {17E1E049-C309-4D79-843F-AE483C264AEA}.Debug|Win64.ActiveCfg = Debug|x64 + {17E1E049-C309-4D79-843F-AE483C264AEA}.Debug|Win64.Build.0 = Debug|x64 + {17E1E049-C309-4D79-843F-AE483C264AEA}.Release|Win32.ActiveCfg = Release|Win32 + {17E1E049-C309-4D79-843F-AE483C264AEA}.Release|Win32.Build.0 = Release|Win32 + {17E1E049-C309-4D79-843F-AE483C264AEA}.Release|Win64.ActiveCfg = Release|x64 + {17E1E049-C309-4D79-843F-AE483C264AEA}.Release|Win64.Build.0 = Release|x64 + {D06B6426-4762-44CC-8BAD-D79052507F2F}.Debug|Win32.ActiveCfg = Debug|Win32 + {D06B6426-4762-44CC-8BAD-D79052507F2F}.Debug|Win32.Build.0 = Debug|Win32 + {D06B6426-4762-44CC-8BAD-D79052507F2F}.Debug|Win64.ActiveCfg = Debug|x64 + {D06B6426-4762-44CC-8BAD-D79052507F2F}.Debug|Win64.Build.0 = Debug|x64 + {D06B6426-4762-44CC-8BAD-D79052507F2F}.Release|Win32.ActiveCfg = Release|Win32 + {D06B6426-4762-44CC-8BAD-D79052507F2F}.Release|Win32.Build.0 = Release|Win32 + {D06B6426-4762-44CC-8BAD-D79052507F2F}.Release|Win64.ActiveCfg = Release|x64 + {D06B6426-4762-44CC-8BAD-D79052507F2F}.Release|Win64.Build.0 = Release|x64 + {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Debug|Win32.ActiveCfg = Debug|Win32 + {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Debug|Win32.Build.0 = Debug|Win32 + {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Debug|Win64.ActiveCfg = Debug|x64 + {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Debug|Win64.Build.0 = Debug|x64 + {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Release|Win32.ActiveCfg = Release|Win32 + {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Release|Win32.Build.0 = Release|Win32 + {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Release|Win64.ActiveCfg = Release|x64 + {ECC7CEAC-A5E5-458E-BB9E-2413CC847881}.Release|Win64.Build.0 = Release|x64 + {C6E20F84-3247-4AD6-B051-B073268F73BA}.Debug|Win32.ActiveCfg = Debug|Win32 + {C6E20F84-3247-4AD6-B051-B073268F73BA}.Debug|Win32.Build.0 = Debug|Win32 + {C6E20F84-3247-4AD6-B051-B073268F73BA}.Debug|Win64.ActiveCfg = Debug|x64 + {C6E20F84-3247-4AD6-B051-B073268F73BA}.Debug|Win64.Build.0 = Debug|x64 + {C6E20F84-3247-4AD6-B051-B073268F73BA}.Release|Win32.ActiveCfg = Release|Win32 + {C6E20F84-3247-4AD6-B051-B073268F73BA}.Release|Win32.Build.0 = Release|Win32 + {C6E20F84-3247-4AD6-B051-B073268F73BA}.Release|Win64.ActiveCfg = Release|x64 + {C6E20F84-3247-4AD6-B051-B073268F73BA}.Release|Win64.Build.0 = Release|x64 + {447F05A8-F581-4CAC-A466-5AC7936E207E}.Debug|Win32.ActiveCfg = Debug|Win32 + {447F05A8-F581-4CAC-A466-5AC7936E207E}.Debug|Win32.Build.0 = Debug|Win32 + {447F05A8-F581-4CAC-A466-5AC7936E207E}.Debug|Win64.ActiveCfg = Debug|x64 + {447F05A8-F581-4CAC-A466-5AC7936E207E}.Debug|Win64.Build.0 = Debug|x64 + {447F05A8-F581-4CAC-A466-5AC7936E207E}.Release|Win32.ActiveCfg = Release|Win32 + {447F05A8-F581-4CAC-A466-5AC7936E207E}.Release|Win32.Build.0 = Release|Win32 + {447F05A8-F581-4CAC-A466-5AC7936E207E}.Release|Win64.ActiveCfg = Release|x64 + {447F05A8-F581-4CAC-A466-5AC7936E207E}.Release|Win64.Build.0 = Release|x64 + {745DEC58-EBB3-47A9-A9B8-4C6627C01BF8}.Debug|Win32.ActiveCfg = Debug|Win32 + {745DEC58-EBB3-47A9-A9B8-4C6627C01BF8}.Debug|Win32.Build.0 = Debug|Win32 + {745DEC58-EBB3-47A9-A9B8-4C6627C01BF8}.Debug|Win64.ActiveCfg = Debug|x64 + {745DEC58-EBB3-47A9-A9B8-4C6627C01BF8}.Debug|Win64.Build.0 = Debug|x64 + {745DEC58-EBB3-47A9-A9B8-4C6627C01BF8}.Release|Win32.ActiveCfg = Release|Win32 + {745DEC58-EBB3-47A9-A9B8-4C6627C01BF8}.Release|Win32.Build.0 = Release|Win32 + {745DEC58-EBB3-47A9-A9B8-4C6627C01BF8}.Release|Win64.ActiveCfg = Release|x64 + {745DEC58-EBB3-47A9-A9B8-4C6627C01BF8}.Release|Win64.Build.0 = Release|x64 + {384C224A-7474-476E-A01B-750EA7DE918C}.Debug|Win32.ActiveCfg = Debug|Win32 + {384C224A-7474-476E-A01B-750EA7DE918C}.Debug|Win32.Build.0 = Debug|Win32 + {384C224A-7474-476E-A01B-750EA7DE918C}.Debug|Win64.ActiveCfg = Debug|x64 + {384C224A-7474-476E-A01B-750EA7DE918C}.Debug|Win64.Build.0 = Debug|x64 + {384C224A-7474-476E-A01B-750EA7DE918C}.Release|Win32.ActiveCfg = Release|Win32 + {384C224A-7474-476E-A01B-750EA7DE918C}.Release|Win32.Build.0 = Release|Win32 + {384C224A-7474-476E-A01B-750EA7DE918C}.Release|Win64.ActiveCfg = Release|x64 + {384C224A-7474-476E-A01B-750EA7DE918C}.Release|Win64.Build.0 = Release|x64 + {0E9791DB-593A-465F-98BC-681011311617}.Debug|Win32.ActiveCfg = Debug|Win32 + {0E9791DB-593A-465F-98BC-681011311617}.Debug|Win32.Build.0 = Debug|Win32 + {0E9791DB-593A-465F-98BC-681011311617}.Debug|Win64.ActiveCfg = Debug|x64 + {0E9791DB-593A-465F-98BC-681011311617}.Debug|Win64.Build.0 = Debug|x64 + {0E9791DB-593A-465F-98BC-681011311617}.Release|Win32.ActiveCfg = Release|Win32 + {0E9791DB-593A-465F-98BC-681011311617}.Release|Win32.Build.0 = Release|Win32 + {0E9791DB-593A-465F-98BC-681011311617}.Release|Win64.ActiveCfg = Release|x64 + {0E9791DB-593A-465F-98BC-681011311617}.Release|Win64.Build.0 = Release|x64 + {9E48B300-37D1-11DD-8C41-005056C00008}.Debug|Win32.ActiveCfg = Debug|Win32 + {9E48B300-37D1-11DD-8C41-005056C00008}.Debug|Win32.Build.0 = Debug|Win32 + {9E48B300-37D1-11DD-8C41-005056C00008}.Debug|Win64.ActiveCfg = Debug|x64 + {9E48B300-37D1-11DD-8C41-005056C00008}.Debug|Win64.Build.0 = Debug|x64 + {9E48B300-37D1-11DD-8C41-005056C00008}.Release|Win32.ActiveCfg = Release|Win32 + {9E48B300-37D1-11DD-8C41-005056C00008}.Release|Win32.Build.0 = Release|Win32 + {9E48B300-37D1-11DD-8C41-005056C00008}.Release|Win64.ActiveCfg = Release|x64 + {9E48B300-37D1-11DD-8C41-005056C00008}.Release|Win64.Build.0 = Release|x64 + {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Debug|Win32.ActiveCfg = Debug|Win32 + {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Debug|Win32.Build.0 = Debug|Win32 + {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Debug|Win64.ActiveCfg = Debug|x64 + {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Debug|Win64.Build.0 = Debug|x64 + {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Release|Win32.ActiveCfg = Release|Win32 + {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Release|Win32.Build.0 = Release|Win32 + {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Release|Win64.ActiveCfg = Release|x64 + {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}.Release|Win64.Build.0 = Release|x64 + {78D80A15-BD8C-44E2-B49E-1F05B0A0A687}.Debug|Win32.ActiveCfg = Debug|Win32 + {78D80A15-BD8C-44E2-B49E-1F05B0A0A687}.Debug|Win32.Build.0 = Debug|Win32 + {78D80A15-BD8C-44E2-B49E-1F05B0A0A687}.Debug|Win64.ActiveCfg = Debug|x64 + {78D80A15-BD8C-44E2-B49E-1F05B0A0A687}.Debug|Win64.Build.0 = Debug|x64 + {78D80A15-BD8C-44E2-B49E-1F05B0A0A687}.Release|Win32.ActiveCfg = Release|Win32 + {78D80A15-BD8C-44E2-B49E-1F05B0A0A687}.Release|Win32.Build.0 = Release|Win32 + {78D80A15-BD8C-44E2-B49E-1F05B0A0A687}.Release|Win64.ActiveCfg = Release|x64 + {78D80A15-BD8C-44E2-B49E-1F05B0A0A687}.Release|Win64.Build.0 = Release|x64 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {44FA54E8-0A60-4A6C-A449-118872E75502} = {C57D8946-CC54-4360-89FA-DAC04E93423A} + {0E032FA8-BB7B-40F6-8CB1-15F204113B24} = {C57D8946-CC54-4360-89FA-DAC04E93423A} + {F67B2C56-D1B1-4EA7-B16E-EF8E7F1B6C5F} = {C57D8946-CC54-4360-89FA-DAC04E93423A} + {7A411D5D-DA3B-474D-B08B-3DD787A1F375} = {C57D8946-CC54-4360-89FA-DAC04E93423A} + {2992ABDB-0853-4C43-B9E2-98D4211192EE} = {C57D8946-CC54-4360-89FA-DAC04E93423A} + {56D453A7-787B-49E3-84BC-465EED38DE8A} = {C57D8946-CC54-4360-89FA-DAC04E93423A} + {5EF24499-4F74-4AF6-8048-650BE7BD7808} = {C57D8946-CC54-4360-89FA-DAC04E93423A} + {863B8043-533C-499D-B48E-F5EC91A6CB9B} = {C57D8946-CC54-4360-89FA-DAC04E93423A} + {04C620E9-BE02-498E-A713-B9E470D75F15} = {89F7F2F7-EB4D-4810-AA06-A96D592678E6} + {89DA21A1-625B-4482-B386-D7C0AF97E477} = {89F7F2F7-EB4D-4810-AA06-A96D592678E6} + {1BAD0518-F2BB-42F5-83E7-9E1804FAD7C1} = {89F7F2F7-EB4D-4810-AA06-A96D592678E6} + {FE88370E-80F7-49BD-A8D1-E7A2CA699E44} = {89F7F2F7-EB4D-4810-AA06-A96D592678E6} + {9EA2E328-E6BE-4DF0-A8DC-05C08A0B822B} = {89F7F2F7-EB4D-4810-AA06-A96D592678E6} + {69C18C83-5753-4F36-BF5A-6EE217304D2D} = {89F7F2F7-EB4D-4810-AA06-A96D592678E6} + {4AF743A4-8C2F-4424-BB4F-A5B0B5C128BD} = {89F7F2F7-EB4D-4810-AA06-A96D592678E6} + {E17527D0-CCA7-4063-924B-C09A1BD43EC3} = {54612744-437F-4400-AE6C-BE4CD68C8507} + {896616D2-A5D2-4548-859E-D59905F46140} = {C57D8946-CC54-4360-89FA-DAC04E93423A} + {78EE8B08-4E07-4C07-9592-BC4D09503155} = {54612744-437F-4400-AE6C-BE4CD68C8507} + {3F8B2057-492F-4332-99DE-3E12F20ED489} = {C57D8946-CC54-4360-89FA-DAC04E93423A} + {6C92BA78-CFAA-4524-A636-F044B0280AB0} = {C57D8946-CC54-4360-89FA-DAC04E93423A} + {24CB1A43-C6E1-442E-AF99-A91D26BA8FA0} = {C57D8946-CC54-4360-89FA-DAC04E93423A} + {E999D336-32D1-48CC-A71F-FC552BD5093B} = {54612744-437F-4400-AE6C-BE4CD68C8507} + {7817D13F-1FAA-47D8-8C9F-B790FE2D6B0A} = {54612744-437F-4400-AE6C-BE4CD68C8507} + {45A08C1B-D249-42C4-932C-615ED3924CBB} = {8BF90F4F-766B-4B4D-B7F3-BF4189A29F4E} + {9085A36E-CAF8-4E86-93AF-373D2554026B} = {C57D8946-CC54-4360-89FA-DAC04E93423A} + {70ABE75C-FBC2-4C82-9103-B952FA5CD168} = {B82CBA7A-6640-47BD-94C9-9EEE37CECBE0} + {24265BD3-3CF3-41A9-B137-53DBEC0F53BF} = {B82CBA7A-6640-47BD-94C9-9EEE37CECBE0} + {C6C836E6-2899-458A-90BA-184192342D30} = {67707698-A79D-4C72-8F99-3B96BA970D0D} + {7CDBBFFF-3B84-413E-BA1A-47CB8D454A46} = {67707698-A79D-4C72-8F99-3B96BA970D0D} + {561FA9F7-14D0-4D94-822C-AAE9B08AACEF} = {67707698-A79D-4C72-8F99-3B96BA970D0D} + {42FC1F89-7136-45E8-94E3-594C173C2937} = {8BF90F4F-766B-4B4D-B7F3-BF4189A29F4E} + {67707698-A79D-4C72-8F99-3B96BA970D0D} = {8BF90F4F-766B-4B4D-B7F3-BF4189A29F4E} + {B82CBA7A-6640-47BD-94C9-9EEE37CECBE0} = {8BF90F4F-766B-4B4D-B7F3-BF4189A29F4E} + {30042FAB-AD01-BF78-3C79-F1F6316BC947} = {67707698-A79D-4C72-8F99-3B96BA970D0D} + {A04109A7-46C9-42F9-AB29-8E3D84450172} = {C57D8946-CC54-4360-89FA-DAC04E93423A} + {0E68FE27-FE42-4AA2-BE7A-C267B8F5FD2E} = {8BF90F4F-766B-4B4D-B7F3-BF4189A29F4E} + {13E85053-54B3-487B-8DDB-3430B1C1B3BF} = {F4FD40AD-244F-4162-88D7-FFD4722FAFAE} + {F4FD40AD-244F-4162-88D7-FFD4722FAFAE} = {8BF90F4F-766B-4B4D-B7F3-BF4189A29F4E} + {70E890EA-9496-4F7C-B903-5FCF87A73AC4} = {C57D8946-CC54-4360-89FA-DAC04E93423A} + {938D17CB-3F34-46A4-AF64-2099B7FF6902} = {54612744-437F-4400-AE6C-BE4CD68C8507} + {13E95053-54B3-497B-9DDB-3430B1C1B6BF} = {8BF90F4F-766B-4B4D-B7F3-BF4189A29F4E} + {DA6F56B4-06A4-441D-AD70-AC5A7D51FADB} = {8BF90F4F-766B-4B4D-B7F3-BF4189A29F4E} + {CF7AC3D1-E2DF-41D2-BEA6-1E2556CDEA26} = {0E68FE27-FE42-4AA2-BE7A-C267B8F5FD2E} + {04274E83-D752-4CC9-BFDE-A717D4E6ABAA} = {0E68FE27-FE42-4AA2-BE7A-C267B8F5FD2E} + {86937F53-C189-40EF-8CE8-8759D8E7D480} = {04274E83-D752-4CC9-BFDE-A717D4E6ABAA} + {18CAE28C-B454-46C1-87A0-493D91D97F03} = {04274E83-D752-4CC9-BFDE-A717D4E6ABAA} + {0E9791DB-593A-465F-98BC-681011311618} = {04274E83-D752-4CC9-BFDE-A717D4E6ABAA} + {17E1E049-C309-4D79-843F-AE483C264AEA} = {04274E83-D752-4CC9-BFDE-A717D4E6ABAA} + {D06B6426-4762-44CC-8BAD-D79052507F2F} = {04274E83-D752-4CC9-BFDE-A717D4E6ABAA} + {ECC7CEAC-A5E5-458E-BB9E-2413CC847881} = {04274E83-D752-4CC9-BFDE-A717D4E6ABAA} + {C6E20F84-3247-4AD6-B051-B073268F73BA} = {04274E83-D752-4CC9-BFDE-A717D4E6ABAA} + {447F05A8-F581-4CAC-A466-5AC7936E207E} = {04274E83-D752-4CC9-BFDE-A717D4E6ABAA} + {745DEC58-EBB3-47A9-A9B8-4C6627C01BF8} = {8BF90F4F-766B-4B4D-B7F3-BF4189A29F4E} + {384C224A-7474-476E-A01B-750EA7DE918C} = {04274E83-D752-4CC9-BFDE-A717D4E6ABAA} + {0E9791DB-593A-465F-98BC-681011311617} = {04274E83-D752-4CC9-BFDE-A717D4E6ABAA} + {9E48B300-37D1-11DD-8C41-005056C00008} = {04274E83-D752-4CC9-BFDE-A717D4E6ABAA} + {EB6E69DD-04BF-4543-9B92-49FAABCEAC2E} = {04274E83-D752-4CC9-BFDE-A717D4E6ABAA} + {78D80A15-BD8C-44E2-B49E-1F05B0A0A687} = {04274E83-D752-4CC9-BFDE-A717D4E6ABAA} + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {98BC2259-260D-4CB1-B4A6-FAAA2DC3E91B} + EndGlobalSection +EndGlobal diff --git a/kbe/src/lib/dependencies/jemalloc/.cirrus.yml b/kbe/src/lib/dependencies/jemalloc/.cirrus.yml new file mode 100644 index 0000000000..019d2c38cd --- /dev/null +++ b/kbe/src/lib/dependencies/jemalloc/.cirrus.yml @@ -0,0 +1,21 @@ +env: + CIRRUS_CLONE_DEPTH: 1 + ARCH: amd64 + +task: + freebsd_instance: + matrix: + image: freebsd-12-0-release-amd64 + image: freebsd-11-2-release-amd64 + install_script: + - sed -i.bak -e 's,pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly,pkg+http://pkg.FreeBSD.org/\${ABI}/latest,' /etc/pkg/FreeBSD.conf + - pkg upgrade -y + - pkg install -y autoconf gmake + script: + - autoconf + #- ./configure ${COMPILER_FLAGS:+ CC="$CC $COMPILER_FLAGS" CXX="$CXX $COMPILER_FLAGS" } $CONFIGURE_FLAGS + - ./configure + - export JFLAG=`sysctl -n kern.smp.cpus` + - gmake -j${JFLAG} + - gmake -j${JFLAG} tests + - gmake check diff --git a/kbe/src/lib/dependencies/jemalloc/.gitignore b/kbe/src/lib/dependencies/jemalloc/.gitignore index a25aaf7e9f..5ca0ad1daa 100755 --- a/kbe/src/lib/dependencies/jemalloc/.gitignore +++ b/kbe/src/lib/dependencies/jemalloc/.gitignore @@ -30,7 +30,6 @@ /include/jemalloc/internal/public_namespace.h /include/jemalloc/internal/public_symbols.txt /include/jemalloc/internal/public_unnamespace.h -/include/jemalloc/internal/size_classes.h /include/jemalloc/jemalloc.h /include/jemalloc/jemalloc_defs.h /include/jemalloc/jemalloc_macros.h @@ -77,12 +76,14 @@ test/include/test/jemalloc_test_defs.h *.pdb *.sdf *.opendb +*.VC.db *.opensdf *.cachefile *.suo *.user *.sln.docstates *.tmp +.vs/ /msvc/Win32/ /msvc/x64/ /msvc/projects/*/*/Debug*/ diff --git a/kbe/src/lib/dependencies/jemalloc/.travis.yml b/kbe/src/lib/dependencies/jemalloc/.travis.yml index 418fc6fd6d..40b2eb5f44 100755 --- a/kbe/src/lib/dependencies/jemalloc/.travis.yml +++ b/kbe/src/lib/dependencies/jemalloc/.travis.yml @@ -1,4 +1,5 @@ language: generic +dist: precise matrix: include: @@ -10,7 +11,7 @@ matrix: env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - addons: + addons: &gcc_multilib apt: packages: - gcc-multilib @@ -20,6 +21,8 @@ matrix: env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + - os: linux + env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-libdl" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux @@ -36,20 +39,21 @@ matrix: env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: osx env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + - os: osx + env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-libdl" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: osx env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=clang CXX=clang++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - addons: - apt: - packages: - - gcc-multilib + addons: *gcc_multilib - os: linux env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + - os: linux + env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-libdl" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux @@ -60,50 +64,34 @@ matrix: env: CC=clang CXX=clang++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=background_thread:true" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--enable-debug" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - addons: - apt: - packages: - - gcc-multilib + addons: *gcc_multilib - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--enable-prof" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - addons: - apt: - packages: - - gcc-multilib + addons: *gcc_multilib - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--disable-stats" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - addons: - apt: - packages: - - gcc-multilib + addons: *gcc_multilib + - os: linux + env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--disable-libdl" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + addons: *gcc_multilib - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - addons: - apt: - packages: - - gcc-multilib + addons: *gcc_multilib - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--with-malloc-conf=dss:primary" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - addons: - apt: - packages: - - gcc-multilib + addons: *gcc_multilib - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--with-malloc-conf=percpu_arena:percpu" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - addons: - apt: - packages: - - gcc-multilib + addons: *gcc_multilib - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="-m32" CONFIGURE_FLAGS="--with-malloc-conf=background_thread:true" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - addons: - apt: - packages: - - gcc-multilib + addons: *gcc_multilib - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --enable-prof" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --disable-stats" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + - os: linux + env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --disable-libdl" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux @@ -114,6 +102,8 @@ matrix: env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --with-malloc-conf=background_thread:true" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof --disable-stats" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + - os: linux + env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof --disable-libdl" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof --with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux @@ -122,6 +112,8 @@ matrix: env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof --with-malloc-conf=percpu_arena:percpu" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-prof --with-malloc-conf=background_thread:true" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + - os: linux + env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats --disable-libdl" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats --with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux @@ -130,6 +122,14 @@ matrix: env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats --with-malloc-conf=percpu_arena:percpu" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-stats --with-malloc-conf=background_thread:true" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + - os: linux + env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-libdl --with-malloc-conf=tcache:false" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + - os: linux + env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-libdl --with-malloc-conf=dss:primary" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + - os: linux + env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-libdl --with-malloc-conf=percpu_arena:percpu" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + - os: linux + env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--disable-libdl --with-malloc-conf=background_thread:true" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=tcache:false,dss:primary" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux @@ -142,10 +142,25 @@ matrix: env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=dss:primary,background_thread:true" EXTRA_CFLAGS="-Werror -Wno-array-bounds" - os: linux env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--with-malloc-conf=percpu_arena:percpu,background_thread:true" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + # Development build + - os: linux + env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --disable-cache-oblivious --enable-stats --enable-log --enable-prof" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + # --enable-expermental-smallocx: + - os: linux + env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --enable-experimental-smallocx --enable-stats --enable-prof" EXTRA_CFLAGS="-Werror -Wno-array-bounds" + + # Valgrind + - os: linux + env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="" EXTRA_CFLAGS="-Werror -Wno-array-bounds" JEMALLOC_TEST_PREFIX="valgrind" + addons: + apt: + packages: + - valgrind before_script: - autoconf + - scripts/gen_travis.py > travis_script && diff .travis.yml travis_script - ./configure ${COMPILER_FLAGS:+ CC="$CC $COMPILER_FLAGS" CXX="$CXX $COMPILER_FLAGS" } $CONFIGURE_FLAGS - make -j3 - make -j3 tests diff --git a/kbe/src/lib/dependencies/jemalloc/COPYING b/kbe/src/lib/dependencies/jemalloc/COPYING index e308632a81..3b7fd3585d 100755 --- a/kbe/src/lib/dependencies/jemalloc/COPYING +++ b/kbe/src/lib/dependencies/jemalloc/COPYING @@ -1,10 +1,10 @@ Unless otherwise specified, files in the jemalloc source distribution are subject to the following license: -------------------------------------------------------------------------------- -Copyright (C) 2002-2017 Jason Evans . +Copyright (C) 2002-present Jason Evans . All rights reserved. Copyright (C) 2007-2012 Mozilla Foundation. All rights reserved. -Copyright (C) 2009-2017 Facebook, Inc. All rights reserved. +Copyright (C) 2009-present Facebook, Inc. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/kbe/src/lib/dependencies/jemalloc/ChangeLog b/kbe/src/lib/dependencies/jemalloc/ChangeLog index ee1b7ead39..7c73a8f237 100755 --- a/kbe/src/lib/dependencies/jemalloc/ChangeLog +++ b/kbe/src/lib/dependencies/jemalloc/ChangeLog @@ -4,6 +4,226 @@ brevity. Much more detail can be found in the git revision history: https://github.com/jemalloc/jemalloc +* 5.2.0 (April 2, 2019) + + This release includes a few notable improvements, which are summarized below: + 1) improved fast-path performance from the optimizations by @djwatson; 2) + reduced virtual memory fragmentation and metadata usage; and 3) bug fixes on + setting the number of background threads. In addition, peak / spike memory + usage is improved with certain allocation patterns. As usual, the release and + prior dev versions have gone through large-scale production testing. + + New features: + - Implement oversize_threshold, which uses a dedicated arena for allocations + crossing the specified threshold to reduce fragmentation. (@interwq) + - Add extents usage information to stats. (@tyleretzel) + - Log time information for sampled allocations. (@tyleretzel) + - Support 0 size in sdallocx. (@djwatson) + - Output rate for certain counters in malloc_stats. (@zinoale) + - Add configure option --enable-readlinkat, which allows the use of readlinkat + over readlink. (@davidtgoldblatt) + - Add configure options --{enable,disable}-{static,shared} to allow not + building unwanted libraries. (@Ericson2314) + - Add configure option --disable-libdl to enable fully static builds. + (@interwq) + - Add mallctl interfaces: + + opt.oversize_threshold (@interwq) + + stats.arenas..extent_avail (@tyleretzel) + + stats.arenas..extents..n{dirty,muzzy,retained} (@tyleretzel) + + stats.arenas..extents..{dirty,muzzy,retained}_bytes + (@tyleretzel) + + Portability improvements: + - Update MSVC builds. (@maksqwe, @rustyx) + - Workaround a compiler optimizer bug on s390x. (@rkmisra) + - Make use of pthread_set_name_np(3) on FreeBSD. (@trasz) + - Implement malloc_getcpu() to enable percpu_arena for windows. (@santagada) + - Link against -pthread instead of -lpthread. (@paravoid) + - Make background_thread not dependent on libdl. (@interwq) + - Add stringify to fix a linker directive issue on MSVC. (@daverigby) + - Detect and fall back when 8-bit atomics are unavailable. (@interwq) + - Fall back to the default pthread_create if dlsym(3) fails. (@interwq) + + Optimizations and refactors: + - Refactor the TSD module. (@davidtgoldblatt) + - Avoid taking extents_muzzy mutex when muzzy is disabled. (@interwq) + - Avoid taking large_mtx for auto arenas on the tcache flush path. (@interwq) + - Optimize ixalloc by avoiding a size lookup. (@interwq) + - Implement opt.oversize_threshold which uses a dedicated arena for requests + crossing the threshold, also eagerly purges the oversize extents. Default + the threshold to 8 MiB. (@interwq) + - Clean compilation with -Wextra. (@gnzlbg, @jasone) + - Refactor the size class module. (@davidtgoldblatt) + - Refactor the stats emitter. (@tyleretzel) + - Optimize pow2_ceil. (@rkmisra) + - Avoid runtime detection of lazy purging on FreeBSD. (@trasz) + - Optimize mmap(2) alignment handling on FreeBSD. (@trasz) + - Improve error handling for THP state initialization. (@jsteemann) + - Rework the malloc() fast path. (@djwatson) + - Rework the free() fast path. (@djwatson) + - Refactor and optimize the tcache fill / flush paths. (@djwatson) + - Optimize sync / lwsync on PowerPC. (@chmeeedalf) + - Bypass extent_dalloc() when retain is enabled. (@interwq) + - Optimize the locking on large deallocation. (@interwq) + - Reduce the number of pages committed from sanity checking in debug build. + (@trasz, @interwq) + - Deprecate OSSpinLock. (@interwq) + - Lower the default number of background threads to 4 (when the feature + is enabled). (@interwq) + - Optimize the trylock spin wait. (@djwatson) + - Use arena index for arena-matching checks. (@interwq) + - Avoid forced decay on thread termination when using background threads. + (@interwq) + - Disable muzzy decay by default. (@djwatson, @interwq) + - Only initialize libgcc unwinder when profiling is enabled. (@paravoid, + @interwq) + + Bug fixes (all only relevant to jemalloc 5.x): + - Fix background thread index issues with max_background_threads. (@djwatson, + @interwq) + - Fix stats output for opt.lg_extent_max_active_fit. (@interwq) + - Fix opt.prof_prefix initialization. (@davidtgoldblatt) + - Properly trigger decay on tcache destroy. (@interwq, @amosbird) + - Fix tcache.flush. (@interwq) + - Detect whether explicit extent zero out is necessary with huge pages or + custom extent hooks, which may change the purge semantics. (@interwq) + - Fix a side effect caused by extent_max_active_fit combined with decay-based + purging, where freed extents can accumulate and not be reused for an + extended period of time. (@interwq, @mpghf) + - Fix a missing unlock on extent register error handling. (@zoulasc) + + Testing: + - Simplify the Travis script output. (@gnzlbg) + - Update the test scripts for FreeBSD. (@devnexen) + - Add unit tests for the producer-consumer pattern. (@interwq) + - Add Cirrus-CI config for FreeBSD builds. (@jasone) + - Add size-matching sanity checks on tcache flush. (@davidtgoldblatt, + @interwq) + + Incompatible changes: + - Remove --with-lg-page-sizes. (@davidtgoldblatt) + + Documentation: + - Attempt to build docs by default, however skip doc building when xsltproc + is missing. (@interwq, @cmuellner) + +* 5.1.0 (May 4, 2018) + + This release is primarily about fine-tuning, ranging from several new features + to numerous notable performance and portability enhancements. The release and + prior dev versions have been running in multiple large scale applications for + months, and the cumulative improvements are substantial in many cases. + + Given the long and successful production runs, this release is likely a good + candidate for applications to upgrade, from both jemalloc 5.0 and before. For + performance-critical applications, the newly added TUNING.md provides + guidelines on jemalloc tuning. + + New features: + - Implement transparent huge page support for internal metadata. (@interwq) + - Add opt.thp to allow enabling / disabling transparent huge pages for all + mappings. (@interwq) + - Add maximum background thread count option. (@djwatson) + - Allow prof_active to control opt.lg_prof_interval and prof.gdump. + (@interwq) + - Allow arena index lookup based on allocation addresses via mallctl. + (@lionkov) + - Allow disabling initial-exec TLS model. (@davidtgoldblatt, @KenMacD) + - Add opt.lg_extent_max_active_fit to set the max ratio between the size of + the active extent selected (to split off from) and the size of the requested + allocation. (@interwq, @davidtgoldblatt) + - Add retain_grow_limit to set the max size when growing virtual address + space. (@interwq) + - Add mallctl interfaces: + + arena..retain_grow_limit (@interwq) + + arenas.lookup (@lionkov) + + max_background_threads (@djwatson) + + opt.lg_extent_max_active_fit (@interwq) + + opt.max_background_threads (@djwatson) + + opt.metadata_thp (@interwq) + + opt.thp (@interwq) + + stats.metadata_thp (@interwq) + + Portability improvements: + - Support GNU/kFreeBSD configuration. (@paravoid) + - Support m68k, nios2 and SH3 architectures. (@paravoid) + - Fall back to FD_CLOEXEC when O_CLOEXEC is unavailable. (@zonyitoo) + - Fix symbol listing for cross-compiling. (@tamird) + - Fix high bits computation on ARM. (@davidtgoldblatt, @paravoid) + - Disable the CPU_SPINWAIT macro for Power. (@davidtgoldblatt, @marxin) + - Fix MSVC 2015 & 2017 builds. (@rustyx) + - Improve RISC-V support. (@EdSchouten) + - Set name mangling script in strict mode. (@nicolov) + - Avoid MADV_HUGEPAGE on ARM. (@marxin) + - Modify configure to determine return value of strerror_r. + (@davidtgoldblatt, @cferris1000) + - Make sure CXXFLAGS is tested with CPP compiler. (@nehaljwani) + - Fix 32-bit build on MSVC. (@rustyx) + - Fix external symbol on MSVC. (@maksqwe) + - Avoid a printf format specifier warning. (@jasone) + - Add configure option --disable-initial-exec-tls which can allow jemalloc to + be dynamically loaded after program startup. (@davidtgoldblatt, @KenMacD) + - AArch64: Add ILP32 support. (@cmuellner) + - Add --with-lg-vaddr configure option to support cross compiling. + (@cmuellner, @davidtgoldblatt) + + Optimizations and refactors: + - Improve active extent fit with extent_max_active_fit. This considerably + reduces fragmentation over time and improves virtual memory and metadata + usage. (@davidtgoldblatt, @interwq) + - Eagerly coalesce large extents to reduce fragmentation. (@interwq) + - sdallocx: only read size info when page aligned (i.e. possibly sampled), + which speeds up the sized deallocation path significantly. (@interwq) + - Avoid attempting new mappings for in place expansion with retain, since + it rarely succeeds in practice and causes high overhead. (@interwq) + - Refactor OOM handling in newImpl. (@wqfish) + - Add internal fine-grained logging functionality for debugging use. + (@davidtgoldblatt) + - Refactor arena / tcache interactions. (@davidtgoldblatt) + - Refactor extent management with dumpable flag. (@davidtgoldblatt) + - Add runtime detection of lazy purging. (@interwq) + - Use pairing heap instead of red-black tree for extents_avail. (@djwatson) + - Use sysctl on startup in FreeBSD. (@trasz) + - Use thread local prng state instead of atomic. (@djwatson) + - Make decay to always purge one more extent than before, because in + practice large extents are usually the ones that cross the decay threshold. + Purging the additional extent helps save memory as well as reduce VM + fragmentation. (@interwq) + - Fast division by dynamic values. (@davidtgoldblatt) + - Improve the fit for aligned allocation. (@interwq, @edwinsmith) + - Refactor extent_t bitpacking. (@rkmisra) + - Optimize the generated assembly for ticker operations. (@davidtgoldblatt) + - Convert stats printing to use a structured text emitter. (@davidtgoldblatt) + - Remove preserve_lru feature for extents management. (@djwatson) + - Consolidate two memory loads into one on the fast deallocation path. + (@davidtgoldblatt, @interwq) + + Bug fixes (most of the issues are only relevant to jemalloc 5.0): + - Fix deadlock with multithreaded fork in OS X. (@davidtgoldblatt) + - Validate returned file descriptor before use. (@zonyitoo) + - Fix a few background thread initialization and shutdown issues. (@interwq) + - Fix an extent coalesce + decay race by taking both coalescing extents off + the LRU list. (@interwq) + - Fix potentially unbound increase during decay, caused by one thread keep + stashing memory to purge while other threads generating new pages. The + number of pages to purge is checked to prevent this. (@interwq) + - Fix a FreeBSD bootstrap assertion. (@strejda, @interwq) + - Handle 32 bit mutex counters. (@rkmisra) + - Fix a indexing bug when creating background threads. (@davidtgoldblatt, + @binliu19) + - Fix arguments passed to extent_init. (@yuleniwo, @interwq) + - Fix addresses used for ordering mutexes. (@rkmisra) + - Fix abort_conf processing during bootstrap. (@interwq) + - Fix include path order for out-of-tree builds. (@cmuellner) + + Incompatible changes: + - Remove --disable-thp. (@interwq) + - Remove mallctl interfaces: + + config.thp (@interwq) + + Documentation: + - Add TUNING.md. (@interwq, @davidtgoldblatt, @djwatson) + * 5.0.1 (July 1, 2017) This bugfix release fixes several issues, most of which are obscure enough @@ -22,7 +242,7 @@ brevity. Much more detail can be found in the git revision history: unlikely to be an issue with other libc implementations. (@interwq) - Mask signals during background thread creation. This prevents signals from being inadvertently delivered to background threads. (@jasone, - @davidgoldblatt, @interwq) + @davidtgoldblatt, @interwq) - Avoid inactivity checks within background threads, in order to prevent recursive mutex acquisition. (@interwq) - Fix extent_grow_retained() to use the specified hooks when the @@ -515,7 +735,7 @@ brevity. Much more detail can be found in the git revision history: these fixes, xallocx() now tries harder to partially fulfill requests for optional extra space. Note that a couple of minor heap profiling optimizations are included, but these are better thought of as performance - fixes that were integral to disovering most of the other bugs. + fixes that were integral to discovering most of the other bugs. Optimizations: - Avoid a chunk metadata read in arena_prof_tctx_set(), since it is in the diff --git a/kbe/src/lib/dependencies/jemalloc/INSTALL.md b/kbe/src/lib/dependencies/jemalloc/INSTALL.md index dff7cebbbb..b8f729b0d7 100755 --- a/kbe/src/lib/dependencies/jemalloc/INSTALL.md +++ b/kbe/src/lib/dependencies/jemalloc/INSTALL.md @@ -157,11 +157,6 @@ any of the following arguments (not a definitive list) to 'configure': Statically link against the specified libunwind.a rather than dynamically linking with -lunwind. -* `--disable-thp` - - Disable transparent huge page (THP) integration. This option can be useful - when cross compiling. - * `--disable-fill` Disable support for junk/zero filling of memory. See the "opt.junk" and @@ -226,13 +221,6 @@ any of the following arguments (not a definitive list) to 'configure': system page size may change between configuration and execution, e.g. when cross compiling. -* `--with-lg-page-sizes=` - - Specify the comma-separated base 2 logs of the page sizes to support. This - option may be useful when cross compiling in combination with - `--with-lg-page`, but its primary use case is for integration with FreeBSD's - libc, wherein jemalloc is embedded. - * `--with-lg-hugepage=` Specify the base 2 log of the system huge page size. This option is useful @@ -265,6 +253,27 @@ any of the following arguments (not a definitive list) to 'configure': configuration, jemalloc will provide additional size classes that are not 16-byte-aligned (24, 40, and 56). +* `--with-lg-vaddr=` + + Specify the number of significant virtual address bits. By default, the + configure script attempts to detect virtual address size on those platforms + where it knows how, and picks a default otherwise. This option may be + useful when cross-compiling. + +* `--disable-initial-exec-tls` + + Disable the initial-exec TLS model for jemalloc's internal thread-local + storage (on those platforms that support explicit settings). This can allow + jemalloc to be dynamically loaded after program startup (e.g. using dlopen). + Note that in this case, there will be two malloc implementations operating + in the same process, which will almost certainly result in confusing runtime + crashes if pointers leak from one implementation to the other. + +* `--disable-libdl` + + Disable the usage of libdl, namely dlsym(3) which is required by the lazy + lock option. This can allow building static binaries. + The following environment variables (not a definitive list) impact configure's behavior: @@ -329,6 +338,7 @@ To install only parts of jemalloc, use the following targets: install_include install_lib_shared install_lib_static + install_lib_pc install_lib install_doc_html install_doc_man diff --git a/kbe/src/lib/dependencies/jemalloc/Makefile.in b/kbe/src/lib/dependencies/jemalloc/Makefile.in index fec1397a55..0777f6a8ab 100755 --- a/kbe/src/lib/dependencies/jemalloc/Makefile.in +++ b/kbe/src/lib/dependencies/jemalloc/Makefile.in @@ -24,7 +24,7 @@ abs_srcroot := @abs_srcroot@ abs_objroot := @abs_objroot@ # Build parameters. -CPPFLAGS := @CPPFLAGS@ -I$(srcroot)include -I$(objroot)include +CPPFLAGS := @CPPFLAGS@ -I$(objroot)include -I$(srcroot)include CONFIGURE_CFLAGS := @CONFIGURE_CFLAGS@ SPECIFIED_CFLAGS := @SPECIFIED_CFLAGS@ EXTRA_CFLAGS := @EXTRA_CFLAGS@ @@ -47,6 +47,7 @@ REV := @rev@ install_suffix := @install_suffix@ ABI := @abi@ XSLTPROC := @XSLTPROC@ +XSLROOT := @XSLROOT@ AUTOCONF := @AUTOCONF@ _RPATH = @RPATH@ RPATH = $(if $(1),$(call _RPATH,$(1))) @@ -55,8 +56,11 @@ cfghdrs_out := @cfghdrs_out@ cfgoutputs_in := $(addprefix $(srcroot),@cfgoutputs_in@) cfgoutputs_out := @cfgoutputs_out@ enable_autogen := @enable_autogen@ +enable_shared := @enable_shared@ +enable_static := @enable_static@ enable_prof := @enable_prof@ enable_zone_allocator := @enable_zone_allocator@ +enable_experimental_smallocx := @enable_experimental_smallocx@ MALLOC_CONF := @JEMALLOC_CPREFIX@MALLOC_CONF link_whole_archive := @link_whole_archive@ DSO_LDFLAGS = @DSO_LDFLAGS@ @@ -93,15 +97,18 @@ C_SRCS := $(srcroot)src/jemalloc.c \ $(srcroot)src/arena.c \ $(srcroot)src/background_thread.c \ $(srcroot)src/base.c \ + $(srcroot)src/bin.c \ $(srcroot)src/bitmap.c \ $(srcroot)src/ckh.c \ $(srcroot)src/ctl.c \ + $(srcroot)src/div.c \ $(srcroot)src/extent.c \ $(srcroot)src/extent_dss.c \ $(srcroot)src/extent_mmap.c \ $(srcroot)src/hash.c \ - $(srcroot)src/hooks.c \ + $(srcroot)src/hook.c \ $(srcroot)src/large.c \ + $(srcroot)src/log.c \ $(srcroot)src/malloc_io.c \ $(srcroot)src/mutex.c \ $(srcroot)src/mutex_pool.c \ @@ -111,9 +118,10 @@ C_SRCS := $(srcroot)src/jemalloc.c \ $(srcroot)src/prof.c \ $(srcroot)src/rtree.c \ $(srcroot)src/stats.c \ - $(srcroot)src/spin.c \ + $(srcroot)src/sc.c \ $(srcroot)src/sz.c \ $(srcroot)src/tcache.c \ + $(srcroot)src/test_hooks.c \ $(srcroot)src/ticker.c \ $(srcroot)src/tsd.c \ $(srcroot)src/witness.c @@ -160,17 +168,24 @@ TESTS_UNIT := \ $(srcroot)test/unit/arena_reset.c \ $(srcroot)test/unit/atomic.c \ $(srcroot)test/unit/background_thread.c \ + $(srcroot)test/unit/background_thread_enable.c \ $(srcroot)test/unit/base.c \ $(srcroot)test/unit/bitmap.c \ + $(srcroot)test/unit/bit_util.c \ + $(srcroot)test/unit/binshard.c \ $(srcroot)test/unit/ckh.c \ $(srcroot)test/unit/decay.c \ + $(srcroot)test/unit/div.c \ + $(srcroot)test/unit/emitter.c \ $(srcroot)test/unit/extent_quantize.c \ $(srcroot)test/unit/fork.c \ $(srcroot)test/unit/hash.c \ - $(srcroot)test/unit/hooks.c \ + $(srcroot)test/unit/hook.c \ + $(srcroot)test/unit/huge.c \ $(srcroot)test/unit/junk.c \ $(srcroot)test/unit/junk_alloc.c \ $(srcroot)test/unit/junk_free.c \ + $(srcroot)test/unit/log.c \ $(srcroot)test/unit/mallctl.c \ $(srcroot)test/unit/malloc_io.c \ $(srcroot)test/unit/math.c \ @@ -184,6 +199,7 @@ TESTS_UNIT := \ $(srcroot)test/unit/prof_active.c \ $(srcroot)test/unit/prof_gdump.c \ $(srcroot)test/unit/prof_idump.c \ + $(srcroot)test/unit/prof_log.c \ $(srcroot)test/unit/prof_reset.c \ $(srcroot)test/unit/prof_tctx.c \ $(srcroot)test/unit/prof_thread_name.c \ @@ -192,13 +208,16 @@ TESTS_UNIT := \ $(srcroot)test/unit/rb.c \ $(srcroot)test/unit/retained.c \ $(srcroot)test/unit/rtree.c \ + $(srcroot)test/unit/seq.c \ $(srcroot)test/unit/SFMT.c \ + $(srcroot)test/unit/sc.c \ $(srcroot)test/unit/size_classes.c \ $(srcroot)test/unit/slab.c \ $(srcroot)test/unit/smoothstep.c \ $(srcroot)test/unit/spin.c \ $(srcroot)test/unit/stats.c \ $(srcroot)test/unit/stats_print.c \ + $(srcroot)test/unit/test_hooks.c \ $(srcroot)test/unit/ticker.c \ $(srcroot)test/unit/nstime.c \ $(srcroot)test/unit/tsd.c \ @@ -211,15 +230,21 @@ endif TESTS_INTEGRATION := $(srcroot)test/integration/aligned_alloc.c \ $(srcroot)test/integration/allocated.c \ $(srcroot)test/integration/extent.c \ + $(srcroot)test/integration/malloc.c \ $(srcroot)test/integration/mallocx.c \ $(srcroot)test/integration/MALLOCX_ARENA.c \ $(srcroot)test/integration/overflow.c \ $(srcroot)test/integration/posix_memalign.c \ $(srcroot)test/integration/rallocx.c \ $(srcroot)test/integration/sdallocx.c \ + $(srcroot)test/integration/slab_sizes.c \ $(srcroot)test/integration/thread_arena.c \ $(srcroot)test/integration/thread_tcache_enabled.c \ $(srcroot)test/integration/xallocx.c +ifeq (@enable_experimental_smallocx@, 1) +TESTS_INTEGRATION += \ + $(srcroot)test/integration/smallocx.c +endif ifeq (@enable_cxx@, 1) CPP_SRCS := $(srcroot)src/jemalloc_cpp.cpp TESTS_INTEGRATION_CPP := $(srcroot)test/integration/cpp/basic.cpp @@ -227,7 +252,9 @@ else CPP_SRCS := TESTS_INTEGRATION_CPP := endif -TESTS_STRESS := $(srcroot)test/stress/microbench.c +TESTS_STRESS := $(srcroot)test/stress/microbench.c \ + $(srcroot)test/stress/hookbench.c + TESTS := $(TESTS_UNIT) $(TESTS_INTEGRATION) $(TESTS_INTEGRATION_CPP) $(TESTS_STRESS) @@ -268,10 +295,24 @@ all: build_lib dist: build_doc $(objroot)doc/%.html : $(objroot)doc/%.xml $(srcroot)doc/stylesheet.xsl $(objroot)doc/html.xsl +ifneq ($(XSLROOT),) $(XSLTPROC) -o $@ $(objroot)doc/html.xsl $< +else +ifeq ($(wildcard $(DOCS_HTML)),) + @echo "

Missing xsltproc. Doc not built.

" > $@ +endif + @echo "Missing xsltproc. "$@" not (re)built." +endif $(objroot)doc/%.3 : $(objroot)doc/%.xml $(srcroot)doc/stylesheet.xsl $(objroot)doc/manpages.xsl +ifneq ($(XSLROOT),) $(XSLTPROC) -o $@ $(objroot)doc/manpages.xsl $< +else +ifeq ($(wildcard $(DOCS_MAN3)),) + @echo "Missing xsltproc. Doc not built." > $@ +endif + @echo "Missing xsltproc. "$@" not (re)built." +endif build_doc_html: $(DOCS_HTML) build_doc_man: $(DOCS_MAN3) @@ -394,7 +435,7 @@ $(objroot)test/unit/%$(EXE): $(objroot)test/unit/%.$(O) $(C_JET_OBJS) $(C_TESTLI $(objroot)test/integration/%$(EXE): $(objroot)test/integration/%.$(O) $(C_TESTLIB_INTEGRATION_OBJS) $(C_UTIL_INTEGRATION_OBJS) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB) @mkdir -p $(@D) - $(CC) $(TEST_LD_MODE) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(LJEMALLOC) $(LDFLAGS) $(filter-out -lm,$(filter -lrt -lpthread -lstdc++,$(LIBS))) $(LM) $(EXTRA_LDFLAGS) + $(CC) $(TEST_LD_MODE) $(LDTARGET) $(filter %.$(O),$^) $(call RPATH,$(objroot)lib) $(LJEMALLOC) $(LDFLAGS) $(filter-out -lm,$(filter -lrt -pthread -lstdc++,$(LIBS))) $(LM) $(EXTRA_LDFLAGS) $(objroot)test/integration/cpp/%$(EXE): $(objroot)test/integration/cpp/%.$(O) $(C_TESTLIB_INTEGRATION_OBJS) $(C_UTIL_INTEGRATION_OBJS) $(objroot)lib/$(LIBJEMALLOC).$(IMPORTLIB) @mkdir -p $(@D) @@ -406,7 +447,12 @@ $(objroot)test/stress/%$(EXE): $(objroot)test/stress/%.$(O) $(C_JET_OBJS) $(C_TE build_lib_shared: $(DSOS) build_lib_static: $(STATIC_LIBS) -build_lib: build_lib_shared build_lib_static +ifeq ($(enable_shared), 1) +build_lib: build_lib_shared +endif +ifeq ($(enable_static), 1) +build_lib: build_lib_static +endif install_bin: $(INSTALL) -d $(BINDIR) @@ -443,7 +489,13 @@ install_lib_pc: $(PC) $(INSTALL) -m 644 $$l $(LIBDIR)/pkgconfig; \ done -install_lib: install_lib_shared install_lib_static install_lib_pc +ifeq ($(enable_shared), 1) +install_lib: install_lib_shared +endif +ifeq ($(enable_static), 1) +install_lib: install_lib_static +endif +install_lib: install_lib_pc install_doc_html: $(INSTALL) -d $(DATADIR)/doc/jemalloc$(install_suffix) @@ -459,7 +511,7 @@ install_doc_man: $(INSTALL) -m 644 $$d $(MANDIR)/man3; \ done -install_doc: install_doc_html install_doc_man +install_doc: build_doc install_doc_html install_doc_man install: install_bin install_include install_lib install_doc diff --git a/kbe/src/lib/dependencies/jemalloc/TUNING.md b/kbe/src/lib/dependencies/jemalloc/TUNING.md new file mode 100644 index 0000000000..34fca05b43 --- /dev/null +++ b/kbe/src/lib/dependencies/jemalloc/TUNING.md @@ -0,0 +1,129 @@ +This document summarizes the common approaches for performance fine tuning with +jemalloc (as of 5.1.0). The default configuration of jemalloc tends to work +reasonably well in practice, and most applications should not have to tune any +options. However, in order to cover a wide range of applications and avoid +pathological cases, the default setting is sometimes kept conservative and +suboptimal, even for many common workloads. When jemalloc is properly tuned for +a specific application / workload, it is common to improve system level metrics +by a few percent, or make favorable trade-offs. + + +## Notable runtime options for performance tuning + +Runtime options can be set via +[malloc_conf](http://jemalloc.net/jemalloc.3.html#tuning). + +* [background_thread](http://jemalloc.net/jemalloc.3.html#background_thread) + + Enabling jemalloc background threads generally improves the tail latency for + application threads, since unused memory purging is shifted to the dedicated + background threads. In addition, unintended purging delay caused by + application inactivity is avoided with background threads. + + Suggested: `background_thread:true` when jemalloc managed threads can be + allowed. + +* [metadata_thp](http://jemalloc.net/jemalloc.3.html#opt.metadata_thp) + + Allowing jemalloc to utilize transparent huge pages for its internal + metadata usually reduces TLB misses significantly, especially for programs + with large memory footprint and frequent allocation / deallocation + activities. Metadata memory usage may increase due to the use of huge + pages. + + Suggested for allocation intensive programs: `metadata_thp:auto` or + `metadata_thp:always`, which is expected to improve CPU utilization at a + small memory cost. + +* [dirty_decay_ms](http://jemalloc.net/jemalloc.3.html#opt.dirty_decay_ms) and + [muzzy_decay_ms](http://jemalloc.net/jemalloc.3.html#opt.muzzy_decay_ms) + + Decay time determines how fast jemalloc returns unused pages back to the + operating system, and therefore provides a fairly straightforward trade-off + between CPU and memory usage. Shorter decay time purges unused pages faster + to reduces memory usage (usually at the cost of more CPU cycles spent on + purging), and vice versa. + + Suggested: tune the values based on the desired trade-offs. + +* [narenas](http://jemalloc.net/jemalloc.3.html#opt.narenas) + + By default jemalloc uses multiple arenas to reduce internal lock contention. + However high arena count may also increase overall memory fragmentation, + since arenas manage memory independently. When high degree of parallelism + is not expected at the allocator level, lower number of arenas often + improves memory usage. + + Suggested: if low parallelism is expected, try lower arena count while + monitoring CPU and memory usage. + +* [percpu_arena](http://jemalloc.net/jemalloc.3.html#opt.percpu_arena) + + Enable dynamic thread to arena association based on running CPU. This has + the potential to improve locality, e.g. when thread to CPU affinity is + present. + + Suggested: try `percpu_arena:percpu` or `percpu_arena:phycpu` if + thread migration between processors is expected to be infrequent. + +Examples: + +* High resource consumption application, prioritizing CPU utilization: + + `background_thread:true,metadata_thp:auto` combined with relaxed decay time + (increased `dirty_decay_ms` and / or `muzzy_decay_ms`, + e.g. `dirty_decay_ms:30000,muzzy_decay_ms:30000`). + +* High resource consumption application, prioritizing memory usage: + + `background_thread:true` combined with shorter decay time (decreased + `dirty_decay_ms` and / or `muzzy_decay_ms`, + e.g. `dirty_decay_ms:5000,muzzy_decay_ms:5000`), and lower arena count + (e.g. number of CPUs). + +* Low resource consumption application: + + `narenas:1,lg_tcache_max:13` combined with shorter decay time (decreased + `dirty_decay_ms` and / or `muzzy_decay_ms`,e.g. + `dirty_decay_ms:1000,muzzy_decay_ms:0`). + +* Extremely conservative -- minimize memory usage at all costs, only suitable when +allocation activity is very rare: + + `narenas:1,tcache:false,dirty_decay_ms:0,muzzy_decay_ms:0` + +Note that it is recommended to combine the options with `abort_conf:true` which +aborts immediately on illegal options. + +## Beyond runtime options + +In addition to the runtime options, there are a number of programmatic ways to +improve application performance with jemalloc. + +* [Explicit arenas](http://jemalloc.net/jemalloc.3.html#arenas.create) + + Manually created arenas can help performance in various ways, e.g. by + managing locality and contention for specific usages. For example, + applications can explicitly allocate frequently accessed objects from a + dedicated arena with + [mallocx()](http://jemalloc.net/jemalloc.3.html#MALLOCX_ARENA) to improve + locality. In addition, explicit arenas often benefit from individually + tuned options, e.g. relaxed [decay + time](http://jemalloc.net/jemalloc.3.html#arena.i.dirty_decay_ms) if + frequent reuse is expected. + +* [Extent hooks](http://jemalloc.net/jemalloc.3.html#arena.i.extent_hooks) + + Extent hooks allow customization for managing underlying memory. One use + case for performance purpose is to utilize huge pages -- for example, + [HHVM](https://github.com/facebook/hhvm/blob/master/hphp/util/alloc.cpp) + uses explicit arenas with customized extent hooks to manage 1GB huge pages + for frequently accessed data, which reduces TLB misses significantly. + +* [Explicit thread-to-arena + binding](http://jemalloc.net/jemalloc.3.html#thread.arena) + + It is common for some threads in an application to have different memory + access / allocation patterns. Threads with heavy workloads often benefit + from explicit binding, e.g. binding very active threads to dedicated arenas + may reduce contention at the allocator level. diff --git a/kbe/src/lib/dependencies/jemalloc/bin/jeprof.in b/kbe/src/lib/dependencies/jemalloc/bin/jeprof.in index e6f4af4b71..588c6b4381 100755 --- a/kbe/src/lib/dependencies/jemalloc/bin/jeprof.in +++ b/kbe/src/lib/dependencies/jemalloc/bin/jeprof.in @@ -2895,6 +2895,8 @@ sub RemoveUninterestingFrames { foreach my $name ('@JEMALLOC_PREFIX@calloc', 'cfree', '@JEMALLOC_PREFIX@malloc', + 'newImpl', + 'void* newImpl', '@JEMALLOC_PREFIX@free', '@JEMALLOC_PREFIX@memalign', '@JEMALLOC_PREFIX@posix_memalign', diff --git a/kbe/src/lib/dependencies/jemalloc/configure.ac b/kbe/src/lib/dependencies/jemalloc/configure.ac index 1551ded8ca..96f76d3553 100755 --- a/kbe/src/lib/dependencies/jemalloc/configure.ac +++ b/kbe/src/lib/dependencies/jemalloc/configure.ac @@ -8,9 +8,9 @@ dnl ============================================================================ dnl Custom macro definitions. dnl JE_CONCAT_VVV(r, a, b) -dnl +dnl dnl Set $r to the concatenation of $a and $b, with a space separating them iff -dnl both $a and $b are non-emty. +dnl both $a and $b are non-empty. AC_DEFUN([JE_CONCAT_VVV], if test "x[$]{$2}" = "x" -o "x[$]{$3}" = "x" ; then $1="[$]{$2}[$]{$3}" @@ -20,7 +20,7 @@ fi ) dnl JE_APPEND_VS(a, b) -dnl +dnl dnl Set $a to the concatenation of $a and b, with a space separating them iff dnl both $a and b are non-empty. AC_DEFUN([JE_APPEND_VS], @@ -31,7 +31,7 @@ AC_DEFUN([JE_APPEND_VS], CONFIGURE_CFLAGS= SPECIFIED_CFLAGS="${CFLAGS}" dnl JE_CFLAGS_ADD(cflag) -dnl +dnl dnl CFLAGS is the concatenation of CONFIGURE_CFLAGS and SPECIFIED_CFLAGS dnl (ignoring EXTRA_CFLAGS, which does not impact configure tests. This macro dnl appends to CONFIGURE_CFLAGS and regenerates CFLAGS. @@ -57,7 +57,7 @@ JE_CONCAT_VVV(CFLAGS, CONFIGURE_CFLAGS, SPECIFIED_CFLAGS) dnl JE_CFLAGS_SAVE() dnl JE_CFLAGS_RESTORE() -dnl +dnl dnl Save/restore CFLAGS. Nesting is not supported. AC_DEFUN([JE_CFLAGS_SAVE], SAVED_CONFIGURE_CFLAGS="${CONFIGURE_CFLAGS}" @@ -76,6 +76,7 @@ AC_MSG_CHECKING([whether compiler supports $1]) T_CONFIGURE_CXXFLAGS="${CONFIGURE_CXXFLAGS}" JE_APPEND_VS(CONFIGURE_CXXFLAGS, $1) JE_CONCAT_VVV(CXXFLAGS, CONFIGURE_CXXFLAGS, SPECIFIED_CXXFLAGS) +AC_LANG_PUSH([C++]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM( [[ ]], [[ @@ -87,11 +88,12 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM( AC_MSG_RESULT([no]) [CONFIGURE_CXXFLAGS="${T_CONFIGURE_CXXFLAGS}"] ) +AC_LANG_POP([C++]) JE_CONCAT_VVV(CXXFLAGS, CONFIGURE_CXXFLAGS, SPECIFIED_CXXFLAGS) ]) dnl JE_COMPILABLE(label, hcode, mcode, rvar) -dnl +dnl dnl Use AC_LINK_IFELSE() rather than AC_COMPILE_IFELSE() so that linker errors dnl cause failure. AC_DEFUN([JE_COMPILABLE], @@ -173,6 +175,9 @@ fi ], XSLROOT="${DEFAULT_XSLROOT}" ) +if test "x$XSLTPROC" = "xfalse" ; then + XSLROOT="" +fi AC_SUBST([XSLROOT]) dnl If CFLAGS isn't defined, set CFLAGS to something reasonable. Otherwise, @@ -240,9 +245,11 @@ if test "x$GCC" = "xyes" ; then fi fi JE_CFLAGS_ADD([-Wall]) + JE_CFLAGS_ADD([-Wextra]) JE_CFLAGS_ADD([-Wshorten-64-to-32]) JE_CFLAGS_ADD([-Wsign-compare]) JE_CFLAGS_ADD([-Wundef]) + JE_CFLAGS_ADD([-Wno-format-zero-length]) JE_CFLAGS_ADD([-pipe]) JE_CFLAGS_ADD([-g3]) elif test "x$je_cv_msvc" = "xyes" ; then @@ -286,6 +293,7 @@ if test "x$enable_cxx" = "x1" ; then AX_CXX_COMPILE_STDCXX([14], [noext], [optional]) if test "x${HAVE_CXX14}" = "x1" ; then JE_CXXFLAGS_ADD([-Wall]) + JE_CXXFLAGS_ADD([-Wextra]) JE_CXXFLAGS_ADD([-g3]) SAVED_LIBS="${LIBS}" @@ -380,6 +388,7 @@ dnl CPU-specific settings. CPU_SPINWAIT="" case "${host_cpu}" in i686|x86_64) + HAVE_CPU_SPINWAIT=1 if test "x${je_cv_msvc}" = "xyes" ; then AC_CACHE_VAL([je_cv_pause_msvc], [JE_COMPILABLE([pause instruction MSVC], [], @@ -398,25 +407,36 @@ case "${host_cpu}" in fi fi ;; - powerpc*) - AC_DEFINE_UNQUOTED([HAVE_ALTIVEC], [ ]) - CPU_SPINWAIT='__asm__ volatile("or 31,31,31")' - ;; *) + HAVE_CPU_SPINWAIT=0 ;; esac +AC_DEFINE_UNQUOTED([HAVE_CPU_SPINWAIT], [$HAVE_CPU_SPINWAIT]) AC_DEFINE_UNQUOTED([CPU_SPINWAIT], [$CPU_SPINWAIT]) +AC_ARG_WITH([lg_vaddr], + [AS_HELP_STRING([--with-lg-vaddr=], [Number of significant virtual address bits])], + [LG_VADDR="$with_lg_vaddr"], [LG_VADDR="detect"]) + case "${host_cpu}" in aarch64) - AC_MSG_CHECKING([number of significant virtual address bits]) - LG_VADDR=48 - AC_MSG_RESULT([$LG_VADDR]) + if test "x$LG_VADDR" = "xdetect"; then + AC_MSG_CHECKING([number of significant virtual address bits]) + if test "x${LG_SIZEOF_PTR}" = "x2" ; then + #aarch64 ILP32 + LG_VADDR=32 + else + #aarch64 LP64 + LG_VADDR=48 + fi + AC_MSG_RESULT([$LG_VADDR]) + fi ;; x86_64) - AC_CACHE_CHECK([number of significant virtual address bits], - [je_cv_lg_vaddr], - AC_RUN_IFELSE([AC_LANG_PROGRAM( + if test "x$LG_VADDR" = "xdetect"; then + AC_CACHE_CHECK([number of significant virtual address bits], + [je_cv_lg_vaddr], + AC_RUN_IFELSE([AC_LANG_PROGRAM( [[ #include #ifdef _WIN32 @@ -453,27 +473,30 @@ typedef unsigned __int32 uint32_t; [je_cv_lg_vaddr=`cat conftest.out`], [je_cv_lg_vaddr=error], [je_cv_lg_vaddr=57])) - if test "x${je_cv_lg_vaddr}" != "x" ; then - LG_VADDR="${je_cv_lg_vaddr}" - fi - if test "x${LG_VADDR}" != "xerror" ; then - AC_DEFINE_UNQUOTED([LG_VADDR], [$LG_VADDR]) - else - AC_MSG_ERROR([cannot determine number of significant virtual address bits]) + if test "x${je_cv_lg_vaddr}" != "x" ; then + LG_VADDR="${je_cv_lg_vaddr}" + fi + if test "x${LG_VADDR}" != "xerror" ; then + AC_DEFINE_UNQUOTED([LG_VADDR], [$LG_VADDR]) + else + AC_MSG_ERROR([cannot determine number of significant virtual address bits]) + fi fi ;; *) - AC_MSG_CHECKING([number of significant virtual address bits]) - if test "x${LG_SIZEOF_PTR}" = "x3" ; then - LG_VADDR=64 - elif test "x${LG_SIZEOF_PTR}" = "x2" ; then - LG_VADDR=32 - elif test "x${LG_SIZEOF_PTR}" = "xLG_SIZEOF_PTR_WIN" ; then - LG_VADDR="(1U << (LG_SIZEOF_PTR_WIN+3))" - else - AC_MSG_ERROR([Unsupported lg(pointer size): ${LG_SIZEOF_PTR}]) + if test "x$LG_VADDR" = "xdetect"; then + AC_MSG_CHECKING([number of significant virtual address bits]) + if test "x${LG_SIZEOF_PTR}" = "x3" ; then + LG_VADDR=64 + elif test "x${LG_SIZEOF_PTR}" = "x2" ; then + LG_VADDR=32 + elif test "x${LG_SIZEOF_PTR}" = "xLG_SIZEOF_PTR_WIN" ; then + LG_VADDR="(1U << (LG_SIZEOF_PTR_WIN+3))" + else + AC_MSG_ERROR([Unsupported lg(pointer size): ${LG_SIZEOF_PTR}]) + fi + AC_MSG_RESULT([$LG_VADDR]) fi - AC_MSG_RESULT([$LG_VADDR]) ;; esac AC_DEFINE_UNQUOTED([LG_VADDR], [$LG_VADDR]) @@ -511,18 +534,83 @@ AN_PROGRAM([ar], [AC_PROG_AR]) AC_DEFUN([AC_PROG_AR], [AC_CHECK_TOOL(AR, ar, :)]) AC_PROG_AR +AN_MAKEVAR([NM], [AC_PROG_NM]) +AN_PROGRAM([nm], [AC_PROG_NM]) +AC_DEFUN([AC_PROG_NM], [AC_CHECK_TOOL(NM, nm, :)]) +AC_PROG_NM + AC_PROG_AWK +dnl ============================================================================ +dnl jemalloc version. +dnl + +AC_ARG_WITH([version], + [AS_HELP_STRING([--with-version=..--g], + [Version string])], + [ + echo "${with_version}" | grep ['^[0-9]\+\.[0-9]\+\.[0-9]\+-[0-9]\+-g[0-9a-f]\+$'] 2>&1 1>/dev/null + if test $? -eq 0 ; then + echo "$with_version" > "${objroot}VERSION" + else + echo "${with_version}" | grep ['^VERSION$'] 2>&1 1>/dev/null + if test $? -ne 0 ; then + AC_MSG_ERROR([${with_version} does not match ..--g or VERSION]) + fi + fi + ], [ + dnl Set VERSION if source directory is inside a git repository. + if test "x`test ! \"${srcroot}\" && cd \"${srcroot}\"; git rev-parse --is-inside-work-tree 2>/dev/null`" = "xtrue" ; then + dnl Pattern globs aren't powerful enough to match both single- and + dnl double-digit version numbers, so iterate over patterns to support up + dnl to version 99.99.99 without any accidental matches. + for pattern in ['[0-9].[0-9].[0-9]' '[0-9].[0-9].[0-9][0-9]' \ + '[0-9].[0-9][0-9].[0-9]' '[0-9].[0-9][0-9].[0-9][0-9]' \ + '[0-9][0-9].[0-9].[0-9]' '[0-9][0-9].[0-9].[0-9][0-9]' \ + '[0-9][0-9].[0-9][0-9].[0-9]' \ + '[0-9][0-9].[0-9][0-9].[0-9][0-9]']; do + (test ! "${srcroot}" && cd "${srcroot}"; git describe --long --abbrev=40 --match="${pattern}") > "${objroot}VERSION.tmp" 2>/dev/null + if test $? -eq 0 ; then + mv "${objroot}VERSION.tmp" "${objroot}VERSION" + break + fi + done + fi + rm -f "${objroot}VERSION.tmp" + ]) + +if test ! -e "${objroot}VERSION" ; then + if test ! -e "${srcroot}VERSION" ; then + AC_MSG_RESULT( + [Missing VERSION file, and unable to generate it; creating bogus VERSION]) + echo "0.0.0-0-g0000000000000000000000000000000000000000" > "${objroot}VERSION" + else + cp ${srcroot}VERSION ${objroot}VERSION + fi +fi +jemalloc_version=`cat "${objroot}VERSION"` +jemalloc_version_major=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]1}'` +jemalloc_version_minor=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]2}'` +jemalloc_version_bugfix=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]3}'` +jemalloc_version_nrev=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]4}'` +jemalloc_version_gid=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]5}'` +AC_SUBST([jemalloc_version]) +AC_SUBST([jemalloc_version_major]) +AC_SUBST([jemalloc_version_minor]) +AC_SUBST([jemalloc_version_bugfix]) +AC_SUBST([jemalloc_version_nrev]) +AC_SUBST([jemalloc_version_gid]) + dnl Platform-specific settings. abi and RPATH can probably be determined dnl programmatically, but doing so is error-prone, which makes it generally dnl not worth the trouble. -dnl +dnl dnl Define cpp macros in CPPFLAGS, rather than doing AC_DEFINE(macro), since the dnl definitions need to be seen before any headers are included, which is a pain dnl to make happen otherwise. default_retain="0" maps_coalesce="1" -DUMP_SYMS="nm -a" +DUMP_SYMS="${NM} -a" SYM_PREFIX="" case "${host}" in *-*-darwin* | *-*-ios*) @@ -556,7 +644,7 @@ case "${host}" in dnl syscall(2) and secure_getenv(3) are exposed by _GNU_SOURCE. JE_APPEND_VS(CPPFLAGS, -D_GNU_SOURCE) abi="elf" - AC_DEFINE([JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS]) + AC_DEFINE([JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS], [ ]) AC_DEFINE([JEMALLOC_HAS_ALLOCA_H]) AC_DEFINE([JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY], [ ]) AC_DEFINE([JEMALLOC_THREADED_INIT], [ ]) @@ -566,11 +654,11 @@ case "${host}" in default_retain="1" fi ;; - *-*-linux* | *-*-kfreebsd*) + *-*-linux*) dnl syscall(2) and secure_getenv(3) are exposed by _GNU_SOURCE. JE_APPEND_VS(CPPFLAGS, -D_GNU_SOURCE) abi="elf" - AC_DEFINE([JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS]) + AC_DEFINE([JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS], [ ]) AC_DEFINE([JEMALLOC_HAS_ALLOCA_H]) AC_DEFINE([JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY], [ ]) AC_DEFINE([JEMALLOC_THREADED_INIT], [ ]) @@ -579,6 +667,15 @@ case "${host}" in default_retain="1" fi ;; + *-*-kfreebsd*) + dnl syscall(2) and secure_getenv(3) are exposed by _GNU_SOURCE. + JE_APPEND_VS(CPPFLAGS, -D_GNU_SOURCE) + abi="elf" + AC_DEFINE([JEMALLOC_HAS_ALLOCA_H]) + AC_DEFINE([JEMALLOC_SYSCTL_VM_OVERCOMMIT], [ ]) + AC_DEFINE([JEMALLOC_THREADED_INIT], [ ]) + AC_DEFINE([JEMALLOC_USE_CXX_THROW], [ ]) + ;; *-*-netbsd*) AC_MSG_CHECKING([ABI]) AC_COMPILE_IFELSE([AC_LANG_PROGRAM( @@ -630,7 +727,13 @@ case "${host}" in DSO_LDFLAGS="-shared" link_whole_archive="1" fi - DUMP_SYMS="dumpbin /SYMBOLS" + case "${host}" in + *-*-cygwin*) + DUMP_SYMS="dumpbin /SYMBOLS" + ;; + *) + ;; + esac a="lib" libprefix="" SOREV="${so}" @@ -711,12 +814,9 @@ JE_COMPILABLE([tls_model attribute], [], foo = 0;], [je_cv_tls_model]) JE_CFLAGS_RESTORE() -if test "x${je_cv_tls_model}" = "xyes" ; then - AC_DEFINE([JEMALLOC_TLS_MODEL], - [__attribute__((tls_model("initial-exec")))]) -else - AC_DEFINE([JEMALLOC_TLS_MODEL], [ ]) -fi +dnl (Setting of JEMALLOC_TLS_MODEL is done later, after we've checked for +dnl --disable-initial-exec-tls) + dnl Check for alloc_size attribute support. JE_CFLAGS_SAVE() JE_CFLAGS_ADD([-Werror]) @@ -781,6 +881,36 @@ AC_PROG_RANLIB AC_PATH_PROG([LD], [ld], [false], [$PATH]) AC_PATH_PROG([AUTOCONF], [autoconf], [false], [$PATH]) +dnl Enable shared libs +AC_ARG_ENABLE([shared], + [AS_HELP_STRING([--enable-shared], [Build shared libaries])], +if test "x$enable_shared" = "xno" ; then + enable_shared="0" +else + enable_shared="1" +fi +, +enable_shared="1" +) +AC_SUBST([enable_shared]) + +dnl Enable static libs +AC_ARG_ENABLE([static], + [AS_HELP_STRING([--enable-static], [Build static libaries])], +if test "x$enable_static" = "xno" ; then + enable_static="0" +else + enable_static="1" +fi +, +enable_static="1" +) +AC_SUBST([enable_static]) + +if test "$enable_shared$enable_static" = "00" ; then + AC_MSG_ERROR([Please enable one of shared or static builds]) +fi + dnl Perform no name mangling by default. AC_ARG_WITH([mangling], [AS_HELP_STRING([--with-mangling=], [Mangle symbols in ])], @@ -813,7 +943,7 @@ AC_ARG_WITH([export], fi] ) -public_syms="aligned_alloc calloc dallocx free mallctl mallctlbymib mallctlnametomib malloc malloc_conf malloc_message malloc_stats_print malloc_usable_size mallocx nallocx posix_memalign rallocx realloc sallocx sdallocx xallocx" +public_syms="aligned_alloc calloc dallocx free mallctl mallctlbymib mallctlnametomib malloc malloc_conf malloc_message malloc_stats_print malloc_usable_size mallocx smallocx_${jemalloc_version_gid} nallocx posix_memalign rallocx realloc sallocx sdallocx xallocx" dnl Check for additional platform-specific public API functions. AC_CHECK_FUNC([memalign], [AC_DEFINE([JEMALLOC_OVERRIDE_MEMALIGN], [ ]) @@ -931,7 +1061,6 @@ cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/private_symbols.sh" cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/private_namespace.sh" cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/public_namespace.sh" cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/public_unnamespace.sh" -cfghdrs_in="${cfghdrs_in} include/jemalloc/internal/size_classes.sh" cfghdrs_in="${cfghdrs_in} include/jemalloc/jemalloc_rename.sh" cfghdrs_in="${cfghdrs_in} include/jemalloc/jemalloc_mangle.sh" cfghdrs_in="${cfghdrs_in} include/jemalloc/jemalloc.sh" @@ -944,7 +1073,6 @@ cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/private_symbols_jet.awk" cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/public_symbols.txt" cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/public_namespace.h" cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/public_unnamespace.h" -cfghdrs_out="${cfghdrs_out} include/jemalloc/internal/size_classes.h" cfghdrs_out="${cfghdrs_out} include/jemalloc/jemalloc_protos_jet.h" cfghdrs_out="${cfghdrs_out} include/jemalloc/jemalloc_rename.h" cfghdrs_out="${cfghdrs_out} include/jemalloc/jemalloc_mangle.h" @@ -956,6 +1084,10 @@ cfghdrs_tup="include/jemalloc/jemalloc_defs.h:include/jemalloc/jemalloc_defs.h.i cfghdrs_tup="${cfghdrs_tup} include/jemalloc/internal/jemalloc_internal_defs.h:include/jemalloc/internal/jemalloc_internal_defs.h.in" cfghdrs_tup="${cfghdrs_tup} test/include/test/jemalloc_test_defs.h:test/include/test/jemalloc_test_defs.h.in" +dnl ============================================================================ +dnl jemalloc build options. +dnl + dnl Do not compile with debugging by default. AC_ARG_ENABLE([debug], [AS_HELP_STRING([--enable-debug], @@ -1008,6 +1140,22 @@ if test "x$enable_stats" = "x1" ; then fi AC_SUBST([enable_stats]) +dnl Do not enable smallocx by default. +AC_ARG_ENABLE([experimental_smallocx], + [AS_HELP_STRING([--enable-experimental-smallocx], [Enable experimental smallocx API])], +[if test "x$enable_experimental_smallocx" = "xno" ; then +enable_experimental_smallocx="0" +else +enable_experimental_smallocx="1" +fi +], +[enable_experimental_smallocx="0"] +) +if test "x$enable_experimental_smallocx" = "x1" ; then + AC_DEFINE([JEMALLOC_EXPERIMENTAL_SMALLOCX_API]) +fi +AC_SUBST([enable_experimental_smallocx]) + dnl Do not enable profiling by default. AC_ARG_ENABLE([prof], [AS_HELP_STRING([--enable-prof], [Enable allocation profiling])], @@ -1226,7 +1374,54 @@ if test "x$enable_cache_oblivious" = "x1" ; then fi AC_SUBST([enable_cache_oblivious]) +dnl Do not log by default. +AC_ARG_ENABLE([log], + [AS_HELP_STRING([--enable-log], [Support debug logging])], +[if test "x$enable_log" = "xno" ; then + enable_log="0" +else + enable_log="1" +fi +], +[enable_log="0"] +) +if test "x$enable_log" = "x1" ; then + AC_DEFINE([JEMALLOC_LOG], [ ]) +fi +AC_SUBST([enable_log]) + +dnl Do not use readlinkat by default +AC_ARG_ENABLE([readlinkat], + [AS_HELP_STRING([--enable-readlinkat], [Use readlinkat over readlink])], +[if test "x$enable_readlinkat" = "xno" ; then + enable_readlinkat="0" +else + enable_readlinkat="1" +fi +], +[enable_readlinkat="0"] +) +if test "x$enable_readlinkat" = "x1" ; then + AC_DEFINE([JEMALLOC_READLINKAT], [ ]) +fi +AC_SUBST([enable_readlinkat]) +dnl Avoid the extra size checking by default +AC_ARG_ENABLE([extra-size-check], + [AS_HELP_STRING([--enable-extra-size-check], + [Perform additonal size related sanity checks])], +[if test "x$enable_extra_size_check" = "xno" ; then + enable_extra_size_check="0" +else + enable_extra_size_check="1" +fi +], +[enable_extra_size_check="0"] +) +if test "x$enable_extra_size_check" = "x1" ; then + AC_DEFINE([JEMALLOC_EXTRA_SIZE_CHECK], [ ]) +fi +AC_SUBST([enable_extra_size_check]) JE_COMPILABLE([a program using __builtin_unreachable], [ void foo (void) { @@ -1283,6 +1478,21 @@ else fi fi +JE_COMPILABLE([a program using __builtin_popcountl], [ +#include +#include +#include +], [ + { + int rv = __builtin_popcountl(0x08); + printf("%d\n", rv); + } +], [je_cv_gcc_builtin_popcountl]) +if test "x${je_cv_gcc_builtin_popcountl}" = "xyes" ; then + AC_DEFINE([JEMALLOC_INTERNAL_POPCOUNT], [__builtin_popcount]) + AC_DEFINE([JEMALLOC_INTERNAL_POPCOUNTL], [__builtin_popcountl]) +fi + AC_ARG_WITH([lg_quantum], [AS_HELP_STRING([--with-lg-quantum=], [Base 2 log of minimum allocation alignment])], @@ -1380,70 +1590,20 @@ if test "x${LG_PAGE}" != "xundefined" -a \ fi AC_DEFINE_UNQUOTED([LG_HUGEPAGE], [${je_cv_lg_hugepage}]) -AC_ARG_WITH([lg_page_sizes], - [AS_HELP_STRING([--with-lg-page-sizes=], - [Base 2 logs of system page sizes to support])], - [LG_PAGE_SIZES="$with_lg_page_sizes"], [LG_PAGE_SIZES="$LG_PAGE"]) - dnl ============================================================================ -dnl jemalloc configuration. -dnl - -AC_ARG_WITH([version], - [AS_HELP_STRING([--with-version=..--g], - [Version string])], - [ - echo "${with_version}" | grep ['^[0-9]\+\.[0-9]\+\.[0-9]\+-[0-9]\+-g[0-9a-f]\+$'] 2>&1 1>/dev/null - if test $? -eq 0 ; then - echo "$with_version" > "${objroot}VERSION" - else - echo "${with_version}" | grep ['^VERSION$'] 2>&1 1>/dev/null - if test $? -ne 0 ; then - AC_MSG_ERROR([${with_version} does not match ..--g or VERSION]) - fi - fi - ], [ - dnl Set VERSION if source directory is inside a git repository. - if test "x`test ! \"${srcroot}\" && cd \"${srcroot}\"; git rev-parse --is-inside-work-tree 2>/dev/null`" = "xtrue" ; then - dnl Pattern globs aren't powerful enough to match both single- and - dnl double-digit version numbers, so iterate over patterns to support up - dnl to version 99.99.99 without any accidental matches. - for pattern in ['[0-9].[0-9].[0-9]' '[0-9].[0-9].[0-9][0-9]' \ - '[0-9].[0-9][0-9].[0-9]' '[0-9].[0-9][0-9].[0-9][0-9]' \ - '[0-9][0-9].[0-9].[0-9]' '[0-9][0-9].[0-9].[0-9][0-9]' \ - '[0-9][0-9].[0-9][0-9].[0-9]' \ - '[0-9][0-9].[0-9][0-9].[0-9][0-9]']; do - (test ! "${srcroot}" && cd "${srcroot}"; git describe --long --abbrev=40 --match="${pattern}") > "${objroot}VERSION.tmp" 2>/dev/null - if test $? -eq 0 ; then - mv "${objroot}VERSION.tmp" "${objroot}VERSION" - break - fi - done - fi - rm -f "${objroot}VERSION.tmp" - ]) - -if test ! -e "${objroot}VERSION" ; then - if test ! -e "${srcroot}VERSION" ; then - AC_MSG_RESULT( - [Missing VERSION file, and unable to generate it; creating bogus VERSION]) - echo "0.0.0-0-g0000000000000000000000000000000000000000" > "${objroot}VERSION" - else - cp ${srcroot}VERSION ${objroot}VERSION - fi +dnl Enable libdl by default. +AC_ARG_ENABLE([libdl], + [AS_HELP_STRING([--disable-libdl], + [Do not use libdl])], +[if test "x$enable_libdl" = "xno" ; then + enable_libdl="0" +else + enable_libdl="1" fi -jemalloc_version=`cat "${objroot}VERSION"` -jemalloc_version_major=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]1}'` -jemalloc_version_minor=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]2}'` -jemalloc_version_bugfix=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]3}'` -jemalloc_version_nrev=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]4}'` -jemalloc_version_gid=`echo ${jemalloc_version} | tr ".g-" " " | awk '{print [$]5}'` -AC_SUBST([jemalloc_version]) -AC_SUBST([jemalloc_version_major]) -AC_SUBST([jemalloc_version_minor]) -AC_SUBST([jemalloc_version_bugfix]) -AC_SUBST([jemalloc_version_nrev]) -AC_SUBST([jemalloc_version_gid]) +], +[enable_libdl="1"] +) +AC_SUBST([libdl]) dnl ============================================================================ dnl Configure pthreads. @@ -1453,20 +1613,26 @@ if test "x$abi" != "xpecoff" ; then AC_CHECK_HEADERS([pthread.h], , [AC_MSG_ERROR([pthread.h is missing])]) dnl Some systems may embed pthreads functionality in libc; check for libpthread dnl first, but try libc too before failing. - AC_CHECK_LIB([pthread], [pthread_create], [JE_APPEND_VS(LIBS, -lpthread)], + AC_CHECK_LIB([pthread], [pthread_create], [JE_APPEND_VS(LIBS, -pthread)], [AC_SEARCH_LIBS([pthread_create], , , AC_MSG_ERROR([libpthread is missing]))]) wrap_syms="${wrap_syms} pthread_create" have_pthread="1" - dnl Check if we have dlsym support. - have_dlsym="1" - AC_CHECK_HEADERS([dlfcn.h], - AC_CHECK_FUNC([dlsym], [], - [AC_CHECK_LIB([dl], [dlsym], [LIBS="$LIBS -ldl"], [have_dlsym="0"])]), - [have_dlsym="0"]) - if test "x$have_dlsym" = "x1" ; then - AC_DEFINE([JEMALLOC_HAVE_DLSYM], [ ]) + +dnl Check if we have dlsym support. + if test "x$enable_libdl" = "x1" ; then + have_dlsym="1" + AC_CHECK_HEADERS([dlfcn.h], + AC_CHECK_FUNC([dlsym], [], + [AC_CHECK_LIB([dl], [dlsym], [LIBS="$LIBS -ldl"], [have_dlsym="0"])]), + [have_dlsym="0"]) + if test "x$have_dlsym" = "x1" ; then + AC_DEFINE([JEMALLOC_HAVE_DLSYM], [ ]) + fi + else + have_dlsym="0" fi + JE_COMPILABLE([pthread_atfork(3)], [ #include ], [ @@ -1730,6 +1896,19 @@ JE_COMPILABLE([GCC __atomic atomics], [ ], [je_cv_gcc_atomic_atomics]) if test "x${je_cv_gcc_atomic_atomics}" = "xyes" ; then AC_DEFINE([JEMALLOC_GCC_ATOMIC_ATOMICS]) + + dnl check for 8-bit atomic support + JE_COMPILABLE([GCC 8-bit __atomic atomics], [ + ], [ + unsigned char x = 0; + int val = 1; + int y = __atomic_fetch_add(&x, val, __ATOMIC_RELAXED); + int after_add = (int)x; + return after_add == 1; + ], [je_cv_gcc_u8_atomic_atomics]) + if test "x${je_cv_gcc_u8_atomic_atomics}" = "xyes" ; then + AC_DEFINE([JEMALLOC_GCC_U8_ATOMIC_ATOMICS]) + fi fi dnl ============================================================================ @@ -1744,12 +1923,24 @@ JE_COMPILABLE([GCC __sync atomics], [ ], [je_cv_gcc_sync_atomics]) if test "x${je_cv_gcc_sync_atomics}" = "xyes" ; then AC_DEFINE([JEMALLOC_GCC_SYNC_ATOMICS]) + + dnl check for 8-bit atomic support + JE_COMPILABLE([GCC 8-bit __sync atomics], [ + ], [ + unsigned char x = 0; + int before_add = __sync_fetch_and_add(&x, 1); + int after_add = (int)x; + return (before_add == 0) && (after_add == 1); + ], [je_cv_gcc_u8_sync_atomics]) + if test "x${je_cv_gcc_u8_sync_atomics}" = "xyes" ; then + AC_DEFINE([JEMALLOC_GCC_U8_SYNC_ATOMICS]) + fi fi dnl ============================================================================ dnl Check for atomic(3) operations as provided on Darwin. dnl We need this not for the atomic operations (which are provided above), but -dnl rather for the OSSpinLock type it exposes. +dnl rather for the OS_unfair_lock type it exposes. JE_COMPILABLE([Darwin OSAtomic*()], [ #include @@ -1789,6 +1980,15 @@ if test "x${je_cv_madvise}" = "xyes" ; then ], [je_cv_madv_free]) if test "x${je_cv_madv_free}" = "xyes" ; then AC_DEFINE([JEMALLOC_PURGE_MADVISE_FREE], [ ]) + elif test "x${je_cv_madvise}" = "xyes" ; then + case "${host_cpu}" in i686|x86_64) + case "${host}" in *-*-linux*) + AC_DEFINE([JEMALLOC_PURGE_MADVISE_FREE], [ ]) + AC_DEFINE([JEMALLOC_DEFINE_MADVISE_FREE], [ ]) + ;; + esac + ;; + esac fi dnl Check for madvise(..., MADV_DONTNEED). @@ -1801,6 +2001,17 @@ if test "x${je_cv_madvise}" = "xyes" ; then AC_DEFINE([JEMALLOC_PURGE_MADVISE_DONTNEED], [ ]) fi + dnl Check for madvise(..., MADV_DO[NT]DUMP). + JE_COMPILABLE([madvise(..., MADV_DO[[NT]]DUMP)], [ +#include +], [ + madvise((void *)0, 0, MADV_DONTDUMP); + madvise((void *)0, 0, MADV_DODUMP); +], [je_cv_madv_dontdump]) + if test "x${je_cv_madv_dontdump}" = "xyes" ; then + AC_DEFINE([JEMALLOC_MADVISE_DONTDUMP], [ ]) + fi + dnl Check for madvise(..., MADV_[NO]HUGEPAGE). JE_COMPILABLE([madvise(..., MADV_[[NO]]HUGEPAGE)], [ #include @@ -1808,61 +2019,15 @@ if test "x${je_cv_madvise}" = "xyes" ; then madvise((void *)0, 0, MADV_HUGEPAGE); madvise((void *)0, 0, MADV_NOHUGEPAGE); ], [je_cv_thp]) -fi - -dnl Enable transparent huge page support by default. -AC_ARG_ENABLE([thp], - [AS_HELP_STRING([--disable-thp], - [Disable transparent huge page support])], -[if test "x$enable_thp" = "xno" -o "x${je_cv_thp}" != "xyes" ; then - enable_thp="0" -else - enable_thp="1" -fi -], -[if test "x${je_cv_thp}" = "xyes" ; then - enable_thp="1" -else - enable_thp="0" -fi -]) -if test "x$enable_thp" = "x1" ; then - AC_DEFINE([JEMALLOC_THP], [ ]) -fi -AC_SUBST([enable_thp]) - -dnl ============================================================================ -dnl Check whether __sync_{add,sub}_and_fetch() are available despite -dnl __GCC_HAVE_SYNC_COMPARE_AND_SWAP_n macros being undefined. - -AC_DEFUN([JE_SYNC_COMPARE_AND_SWAP_CHECK],[ - AC_CACHE_CHECK([whether to force $1-bit __sync_{add,sub}_and_fetch()], - [je_cv_sync_compare_and_swap_$2], - [AC_LINK_IFELSE([AC_LANG_PROGRAM([ - #include - ], - [ - #ifndef __GCC_HAVE_SYNC_COMPARE_AND_SWAP_$2 - { - uint$1_t x$1 = 0; - __sync_add_and_fetch(&x$1, 42); - __sync_sub_and_fetch(&x$1, 1); - } - #else - #error __GCC_HAVE_SYNC_COMPARE_AND_SWAP_$2 is defined, no need to force - #endif - ])], - [je_cv_sync_compare_and_swap_$2=yes], - [je_cv_sync_compare_and_swap_$2=no])]) - - if test "x${je_cv_sync_compare_and_swap_$2}" = "xyes" ; then - AC_DEFINE([JE_FORCE_SYNC_COMPARE_AND_SWAP_$2], [ ]) +case "${host_cpu}" in + arm*) + ;; + *) + if test "x${je_cv_thp}" = "xyes" ; then + AC_DEFINE([JEMALLOC_HAVE_MADVISE_HUGE], [ ]) fi -]) - -if test "x${je_cv_atomic9}" != "xyes" -a "x${je_cv_osatomic}" != "xyes" ; then - JE_SYNC_COMPARE_AND_SWAP_CHECK(32, 4) - JE_SYNC_COMPARE_AND_SWAP_CHECK(64, 8) + ;; +esac fi dnl ============================================================================ @@ -1907,21 +2072,6 @@ if test "x${je_cv_os_unfair_lock}" = "xyes" ; then AC_DEFINE([JEMALLOC_OS_UNFAIR_LOCK], [ ]) fi -dnl ============================================================================ -dnl Check for spinlock(3) operations as provided on Darwin. - -JE_COMPILABLE([Darwin OSSpin*()], [ -#include -#include -], [ - OSSpinLock lock = 0; - OSSpinLockLock(&lock); - OSSpinLockUnlock(&lock); -], [je_cv_osspin]) -if test "x${je_cv_osspin}" = "xyes" ; then - AC_DEFINE([JEMALLOC_OSSPIN], [ ]) -fi - dnl ============================================================================ dnl Darwin-related configuration. @@ -1948,12 +2098,33 @@ if test "x${enable_zone_allocator}" = "x1" ; then AC_DEFINE([JEMALLOC_ZONE], [ ]) fi +dnl ============================================================================ +dnl Use initial-exec TLS by default. +AC_ARG_ENABLE([initial-exec-tls], + [AS_HELP_STRING([--disable-initial-exec-tls], + [Disable the initial-exec tls model])], +[if test "x$enable_initial_exec_tls" = "xno" ; then + enable_initial_exec_tls="0" +else + enable_initial_exec_tls="1" +fi +], +[enable_initial_exec_tls="1"] +) +AC_SUBST([enable_initial_exec_tls]) + +if test "x${je_cv_tls_model}" = "xyes" -a \ + "x${enable_initial_exec_tls}" = "x1" ; then + AC_DEFINE([JEMALLOC_TLS_MODEL], + [__attribute__((tls_model("initial-exec")))]) +else + AC_DEFINE([JEMALLOC_TLS_MODEL], [ ]) +fi + dnl ============================================================================ dnl Enable background threads if possible. -if test "x${have_pthread}" = "x1" -a "x${have_dlsym}" = "x1" \ - -a "x${je_cv_os_unfair_lock}" != "xyes" \ - -a "x${je_cv_osspin}" != "xyes" ; then +if test "x${have_pthread}" = "x1" -a "x${je_cv_os_unfair_lock}" != "xyes" ; then AC_DEFINE([JEMALLOC_BACKGROUND_THREAD]) fi @@ -2006,6 +2177,25 @@ if test "x${je_cv_pthread_mutex_adaptive_np}" = "xyes" ; then AC_DEFINE([JEMALLOC_HAVE_PTHREAD_MUTEX_ADAPTIVE_NP], [ ]) fi +JE_CFLAGS_SAVE() +JE_CFLAGS_ADD([-D_GNU_SOURCE]) +JE_CFLAGS_ADD([-Werror]) +JE_CFLAGS_ADD([-herror_on_warning]) +JE_COMPILABLE([strerror_r returns char with gnu source], [ +#include +#include +#include +#include +], [ + char *buffer = (char *) malloc(100); + char *error = strerror_r(EINVAL, buffer, 100); + printf("%s\n", error); +], [je_cv_strerror_r_returns_char_with_gnu_source]) +JE_CFLAGS_RESTORE() +if test "x${je_cv_strerror_r_returns_char_with_gnu_source}" = "xyes" ; then + AC_DEFINE([JEMALLOC_STRERROR_R_RETURNS_CHAR_WITH_GNU_SOURCE], [ ]) +fi + dnl ============================================================================ dnl Check for typedefs, structures, and compiler characteristics. AC_HEADER_STDBOOL @@ -2075,16 +2265,6 @@ AC_CONFIG_COMMANDS([include/jemalloc/internal/public_unnamespace.h], [ srcdir="${srcdir}" objroot="${objroot}" ]) -AC_CONFIG_COMMANDS([include/jemalloc/internal/size_classes.h], [ - mkdir -p "${objroot}include/jemalloc/internal" - "${SHELL}" "${srcdir}/include/jemalloc/internal/size_classes.sh" "${LG_QUANTA}" 3 "${LG_PAGE_SIZES}" 2 > "${objroot}include/jemalloc/internal/size_classes.h" -], [ - SHELL="${SHELL}" - srcdir="${srcdir}" - objroot="${objroot}" - LG_QUANTA="${LG_QUANTA}" - LG_PAGE_SIZES="${LG_PAGE_SIZES}" -]) AC_CONFIG_COMMANDS([include/jemalloc/jemalloc_protos_jet.h], [ mkdir -p "${objroot}include/jemalloc" cat "${srcdir}/include/jemalloc/jemalloc_protos.h.in" | sed -e 's/@je_@/jet_/g' > "${objroot}include/jemalloc/jemalloc_protos_jet.h" @@ -2177,17 +2357,20 @@ AC_MSG_RESULT([JEMALLOC_PRIVATE_NAMESPACE]) AC_MSG_RESULT([ : ${JEMALLOC_PRIVATE_NAMESPACE}]) AC_MSG_RESULT([install_suffix : ${install_suffix}]) AC_MSG_RESULT([malloc_conf : ${config_malloc_conf}]) +AC_MSG_RESULT([shared libs : ${enable_shared}]) +AC_MSG_RESULT([static libs : ${enable_static}]) AC_MSG_RESULT([autogen : ${enable_autogen}]) AC_MSG_RESULT([debug : ${enable_debug}]) AC_MSG_RESULT([stats : ${enable_stats}]) +AC_MSG_RESULT([experimetal_smallocx : ${enable_experimental_smallocx}]) AC_MSG_RESULT([prof : ${enable_prof}]) AC_MSG_RESULT([prof-libunwind : ${enable_prof_libunwind}]) AC_MSG_RESULT([prof-libgcc : ${enable_prof_libgcc}]) AC_MSG_RESULT([prof-gcc : ${enable_prof_gcc}]) -AC_MSG_RESULT([thp : ${enable_thp}]) AC_MSG_RESULT([fill : ${enable_fill}]) AC_MSG_RESULT([utrace : ${enable_utrace}]) AC_MSG_RESULT([xmalloc : ${enable_xmalloc}]) +AC_MSG_RESULT([log : ${enable_log}]) AC_MSG_RESULT([lazy_lock : ${enable_lazy_lock}]) AC_MSG_RESULT([cache-oblivious : ${enable_cache_oblivious}]) AC_MSG_RESULT([cxx : ${enable_cxx}]) diff --git a/kbe/src/lib/dependencies/jemalloc/doc/jemalloc.xml.in b/kbe/src/lib/dependencies/jemalloc/doc/jemalloc.xml.in index 21e401acb4..fd0edb30b0 100755 --- a/kbe/src/lib/dependencies/jemalloc/doc/jemalloc.xml.in +++ b/kbe/src/lib/dependencies/jemalloc/doc/jemalloc.xml.in @@ -433,10 +433,11 @@ for (i = 0; i < nbins; i++) { arena statistics, respectively; b and l can be specified to omit per size class statistics for bins and large objects, respectively; x can be specified to omit all mutex - statistics. Unrecognized characters are silently ignored. Note that - thread caching may prevent some statistics from being completely up to - date, since extra locking would be required to merge counters that track - thread cache operations. + statistics; e can be used to omit extent statistics. + Unrecognized characters are silently ignored. Note that thread caching + may prevent some statistics from being completely up to date, since extra + locking would be required to merge counters that track thread cache + operations. The malloc_usable_size() function returns the usable size of the allocation pointed to by @@ -761,6 +762,18 @@ mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".decay", selected pthread-based platforms. + + + max_background_threads + (size_t) + rw + + Maximum number of background worker threads that will + be created. This value is capped at opt.max_background_threads at + startup. + + config.cache_oblivious @@ -852,16 +865,6 @@ mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".decay", build configuration. - - - config.thp - (bool) - r- - - was not specified - during build configuration, and the system supports transparent huge - page manipulation. - @@ -916,6 +919,20 @@ mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".decay", + + + opt.metadata_thp + (const char *) + r- + + Controls whether to allow jemalloc to use transparent + huge page (THP) for internal metadata (see stats.metadata). always + allows such usage. auto uses no THP initially, but may + begin to do so when metadata usage reaches certain level. The default + is disabled. + + opt.retain @@ -927,6 +944,9 @@ mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".decay", munmap 2 or equivalent (see stats.retained for related details). + It also makes jemalloc use + mmap2 + in a more greedy way, mapping larger chunks in one go. This option is disabled by default unless discarding virtual memory is known to trigger platform-specific performance problems, e.g. for [64-bit] Linux, which @@ -972,6 +992,24 @@ mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".decay", number of CPUs, or one if there is a single CPU. + + + opt.oversize_threshold + (size_t) + r- + + The threshold in bytes of which requests are considered + oversize. Allocation requests with greater sizes are fulfilled from a + dedicated arena (automatically managed, however not within + narenas), in order to reduce fragmentation by not + mixing huge allocations with small ones. In addition, the decay API + guarantees on the extents greater than the specified threshold may be + overridden. Note that requests with arena index specified via + MALLOCX_ARENA, or threads associated with explicit + arenas will not be considered. The default threshold is 8MiB. Values + not within large size classes disables this feature. + + opt.percpu_arena @@ -993,15 +1031,29 @@ mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".decay", opt.background_thread - (const bool) + (bool) r- - Internal background worker threads enabled/disabled. See - background_thread for dynamic - control options and details. This option is disabled by + Internal background worker threads enabled/disabled. + Because of potential circular dependencies, enabling background thread + using this option may cause crash or deadlock during initialization. For + a reliable way to use this feature, see background_thread for dynamic control + options and details. This option is disabled by default. + + + opt.max_background_threads + (size_t) + r- + + Maximum number of background threads that will be created + if background_thread is set. + Defaults to number of cpus. + + opt.dirty_decay_ms @@ -1022,10 +1074,14 @@ mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".decay", The default decay time is 10 seconds. See arenas.dirty_decay_ms and arena.<i>.muzzy_decay_ms + linkend="arena.i.dirty_decay_ms">arena.<i>.dirty_decay_ms for related dynamic control options. See opt.muzzy_decay_ms - for a description of muzzy pages. + for a description of muzzy pages.for a description of muzzy pages. Note + that when the oversize_threshold + feature is enabled, the arenas reserved for oversize requests may have + its own default decay settings. @@ -1052,6 +1108,22 @@ mallctl("arena." STRINGIFY(MALLCTL_ARENAS_ALL) ".decay", for related dynamic control options. + + + opt.lg_extent_max_active_fit + (size_t) + r- + + When reusing dirty extents, this determines the (log + base 2 of the) maximum ratio between the size of the active extent + selected (to split off from) and the size of the requested allocation. + This prevents the splitting of large active extents for smaller + allocations, which can reduce fragmentation over the long run + (especially for non-active extents). Lower value may reduce + fragmentation, at the cost of extra active extents. The default value + is 6, which gives a maximum ratio of 64 (2^6). + + opt.stats_print @@ -1194,6 +1266,28 @@ malloc_conf = "xmalloc:true";]]> default maximum is 32 KiB (2^15). + + + opt.thp + (const char *) + r- + + Transparent hugepage (THP) mode. Settings "always", + "never" and "default" are available if THP is supported by the operating + system. The "always" setting enables transparent hugepage for all user + memory mappings with + MADV_HUGEPAGE; "never" + ensures no transparent hugepage with + MADV_NOHUGEPAGE; the default + setting "default" makes no changes. Note that: this option does not + affect THP for jemalloc internal metadata (see opt.metadata_thp); + in addition, for arenas with customized extent_hooks, + this option is bypassed as it is implemented as part of the default + extent hooks. + + opt.prof @@ -1666,6 +1760,22 @@ malloc_conf = "xmalloc:true";]]> for additional information. + + + arena.<i>.retain_grow_limit + (size_t) + rw + + Maximum size to grow retained region (only relevant when + opt.retain is + enabled). This controls the maximum increment to expand virtual memory, + or allocation through arena.<i>extent_hooks. + In particular, if customized extent hooks reserve physical memory + (e.g. 1G huge pages), this is useful to control the allocation hook's + input size. The default is no limit. + + arena.<i>.extent_hooks @@ -1679,10 +1789,11 @@ malloc_conf = "xmalloc:true";]]> to control allocation for arenas explicitly created via arenas.create such that all extents originate from an application-supplied extent allocator - (by specifying the custom extent hook functions during arena creation), - but the automatically created arenas will have already created extents - prior to the application having an opportunity to take over extent - allocation. + (by specifying the custom extent hook functions during arena creation). + However, the API guarantees for the automatically created arenas may be + relaxed -- hooks set there may be called in a "best effort" fashion; in + addition there may be extents created prior to the application having an + opportunity to take over extent allocation. NULL, or selectively opted out - of by returning failure. + of by returning failure. Note that once the extent hook is set, the + structure is accessed directly by the associated arenas, so it must + remain valid for the entire lifetime of the arenas. typedef void *(extent_alloc_t) @@ -2044,6 +2157,15 @@ struct extent_hooks_s { and return the new arena index. + + + arenas.lookup + (unsigned, void*) + rw + + Index of the arena to which an allocation belongs to. + + prof.thread_active_init @@ -2187,7 +2309,24 @@ struct extent_hooks_s { metadata structures (see stats.arenas.<i>.base) and internal allocations (see stats.arenas.<i>.internal). + linkend="stats.arenas.i.internal">stats.arenas.<i>.internal). + Transparent huge page (enabled with opt.metadata_thp) usage is not + considered. + + + + + stats.metadata_thp + (size_t) + r- + [] + + Number of transparent huge pages (THP) used for + metadata. See stats.metadata and + opt.metadata_thp) for + details. @@ -2481,6 +2620,17 @@ struct extent_hooks_s { details. + + + stats.arenas.<i>.extent_avail + (size_t) + r- + [] + + Number of allocated (but unused) extent structs in this + arena. + + stats.arenas.<i>.base @@ -2506,6 +2656,18 @@ struct extent_hooks_s { profiles. + + + stats.arenas.<i>.metadata_thp + (size_t) + r- + [] + + Number of transparent huge pages (THP) used for + metadata. See opt.metadata_thp + for details. + + stats.arenas.<i>.resident @@ -2798,6 +2960,30 @@ struct extent_hooks_s { counters. + + + stats.arenas.<i>.extents.<j>.n{extent_type} + (size_t) + r- + [] + + Number of extents of the given type in this arena in + the bucket corresponding to page size index <j>. The extent type + is one of dirty, muzzy, or retained. + + + + + stats.arenas.<i>.extents.<j>.{extent_type}_bytes + (size_t) + r- + [] + + Sum of the bytes managed by extents of the given type + in this arena in the bucket corresponding to page size index <j>. + The extent type is one of dirty, muzzy, or retained. + + stats.arenas.<i>.lextents.<j>.nmalloc diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/arena_externs.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/arena_externs.h index af16d15885..2bdddb77a0 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/arena_externs.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/arena_externs.h @@ -1,33 +1,32 @@ #ifndef JEMALLOC_INTERNAL_ARENA_EXTERNS_H #define JEMALLOC_INTERNAL_ARENA_EXTERNS_H +#include "jemalloc/internal/bin.h" #include "jemalloc/internal/extent_dss.h" +#include "jemalloc/internal/hook.h" #include "jemalloc/internal/pages.h" -#include "jemalloc/internal/size_classes.h" #include "jemalloc/internal/stats.h" extern ssize_t opt_dirty_decay_ms; extern ssize_t opt_muzzy_decay_ms; -extern const arena_bin_info_t arena_bin_info[NBINS]; - extern percpu_arena_mode_t opt_percpu_arena; extern const char *percpu_arena_mode_names[]; extern const uint64_t h_steps[SMOOTHSTEP_NSTEPS]; extern malloc_mutex_t arenas_lock; -void arena_stats_large_nrequests_add(tsdn_t *tsdn, arena_stats_t *arena_stats, - szind_t szind, uint64_t nrequests); -void arena_stats_mapped_add(tsdn_t *tsdn, arena_stats_t *arena_stats, - size_t size); +extern size_t opt_oversize_threshold; +extern size_t oversize_threshold; + void arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms, size_t *nactive, size_t *ndirty, size_t *nmuzzy); void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms, size_t *nactive, size_t *ndirty, size_t *nmuzzy, arena_stats_t *astats, - malloc_bin_stats_t *bstats, malloc_large_stats_t *lstats); + bin_stats_t *bstats, arena_stats_large_t *lstats, + arena_stats_extents_t *estats); void arena_extents_dirty_dalloc(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *extent); #ifdef JEMALLOC_JET @@ -50,11 +49,11 @@ void arena_decay(tsdn_t *tsdn, arena_t *arena, bool is_background_thread, void arena_reset(tsd_t *tsd, arena_t *arena); void arena_destroy(tsd_t *tsd, arena_t *arena); void arena_tcache_fill_small(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache, - tcache_bin_t *tbin, szind_t binind, uint64_t prof_accumbytes); -void arena_alloc_junk_small(void *ptr, const arena_bin_info_t *bin_info, + cache_bin_t *tbin, szind_t binind, uint64_t prof_accumbytes); +void arena_alloc_junk_small(void *ptr, const bin_info_t *bin_info, bool zero); -typedef void (arena_dalloc_junk_small_t)(void *, const arena_bin_info_t *); +typedef void (arena_dalloc_junk_small_t)(void *, const bin_info_t *); extern arena_dalloc_junk_small_t *JET_MUTABLE arena_dalloc_junk_small; void *arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, @@ -64,25 +63,33 @@ void *arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, void arena_prof_promote(tsdn_t *tsdn, const void *ptr, size_t usize); void arena_dalloc_promoted(tsdn_t *tsdn, void *ptr, tcache_t *tcache, bool slow_path); -void arena_dalloc_bin_junked_locked(tsdn_t *tsdn, arena_t *arena, - extent_t *extent, void *ptr); +void arena_dalloc_bin_junked_locked(tsdn_t *tsdn, arena_t *arena, bin_t *bin, + szind_t binind, extent_t *extent, void *ptr); void arena_dalloc_small(tsdn_t *tsdn, void *ptr); bool arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, - size_t extra, bool zero); + size_t extra, bool zero, size_t *newsize); void *arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize, - size_t size, size_t alignment, bool zero, tcache_t *tcache); + size_t size, size_t alignment, bool zero, tcache_t *tcache, + hook_ralloc_args_t *hook_args); dss_prec_t arena_dss_prec_get(arena_t *arena); bool arena_dss_prec_set(arena_t *arena, dss_prec_t dss_prec); ssize_t arena_dirty_decay_ms_default_get(void); bool arena_dirty_decay_ms_default_set(ssize_t decay_ms); ssize_t arena_muzzy_decay_ms_default_get(void); bool arena_muzzy_decay_ms_default_set(ssize_t decay_ms); +bool arena_retain_grow_limit_get_set(tsd_t *tsd, arena_t *arena, + size_t *old_limit, size_t *new_limit); unsigned arena_nthreads_get(arena_t *arena, bool internal); void arena_nthreads_inc(arena_t *arena, bool internal); void arena_nthreads_dec(arena_t *arena, bool internal); size_t arena_extent_sn_next(arena_t *arena); arena_t *arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks); -void arena_boot(void); +bool arena_init_huge(void); +bool arena_is_huge(unsigned arena_ind); +arena_t *arena_choose_huge(tsd_t *tsd); +bin_t *arena_bin_choose_lock(tsdn_t *tsdn, arena_t *arena, szind_t binind, + unsigned *binshard); +void arena_boot(sc_data_t *sc_data); void arena_prefork0(tsdn_t *tsdn, arena_t *arena); void arena_prefork1(tsdn_t *tsdn, arena_t *arena); void arena_prefork2(tsdn_t *tsdn, arena_t *arena); diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/arena_inlines_a.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/arena_inlines_a.h index da5877060a..9abf7f6ac7 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/arena_inlines_a.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/arena_inlines_a.h @@ -25,7 +25,7 @@ static inline bool arena_prof_accum(tsdn_t *tsdn, arena_t *arena, uint64_t accumbytes) { cassert(config_prof); - if (likely(prof_interval == 0)) { + if (likely(prof_interval == 0 || !prof_active_get_unlocked())) { return false; } diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/arena_inlines_b.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/arena_inlines_b.h index 003abe116f..614deddd20 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/arena_inlines_b.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/arena_inlines_b.h @@ -4,15 +4,34 @@ #include "jemalloc/internal/jemalloc_internal_types.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/rtree.h" -#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/sc.h" #include "jemalloc/internal/sz.h" #include "jemalloc/internal/ticker.h" -static inline szind_t -arena_bin_index(arena_t *arena, arena_bin_t *bin) { - szind_t binind = (szind_t)(bin - arena->bins); - assert(binind < NBINS); - return binind; +JEMALLOC_ALWAYS_INLINE bool +arena_has_default_hooks(arena_t *arena) { + return (extent_hooks_get(arena) == &extent_hooks_default); +} + +JEMALLOC_ALWAYS_INLINE arena_t * +arena_choose_maybe_huge(tsd_t *tsd, arena_t *arena, size_t size) { + if (arena != NULL) { + return arena; + } + + /* + * For huge allocations, use the dedicated huge arena if both are true: + * 1) is using auto arena selection (i.e. arena == NULL), and 2) the + * thread is not assigned to a manual arena. + */ + if (unlikely(size >= oversize_threshold)) { + arena_t *tsd_arena = tsd_arena_get(tsd); + if (tsd_arena == NULL || arena_is_auto(tsd_arena)) { + return arena_choose_huge(tsd); + } + } + + return arena_choose(tsd, NULL); } JEMALLOC_ALWAYS_INLINE prof_tctx_t * @@ -64,6 +83,32 @@ arena_prof_tctx_reset(tsdn_t *tsdn, const void *ptr, prof_tctx_t *tctx) { large_prof_tctx_reset(tsdn, extent); } +JEMALLOC_ALWAYS_INLINE nstime_t +arena_prof_alloc_time_get(tsdn_t *tsdn, const void *ptr, + alloc_ctx_t *alloc_ctx) { + cassert(config_prof); + assert(ptr != NULL); + + extent_t *extent = iealloc(tsdn, ptr); + /* + * Unlike arena_prof_prof_tctx_{get, set}, we only call this once we're + * sure we have a sampled allocation. + */ + assert(!extent_slab_get(extent)); + return large_prof_alloc_time_get(extent); +} + +JEMALLOC_ALWAYS_INLINE void +arena_prof_alloc_time_set(tsdn_t *tsdn, const void *ptr, alloc_ctx_t *alloc_ctx, + nstime_t t) { + cassert(config_prof); + assert(ptr != NULL); + + extent_t *extent = iealloc(tsdn, ptr); + assert(!extent_slab_get(extent)); + large_prof_alloc_time_set(extent, t); +} + JEMALLOC_ALWAYS_INLINE void arena_decay_ticks(tsdn_t *tsdn, arena_t *arena, unsigned nticks) { tsd_t *tsd; @@ -90,14 +135,33 @@ arena_decay_tick(tsdn_t *tsdn, arena_t *arena) { arena_decay_ticks(tsdn, arena, 1); } +/* Purge a single extent to retained / unmapped directly. */ +JEMALLOC_ALWAYS_INLINE void +arena_decay_extent(tsdn_t *tsdn,arena_t *arena, extent_hooks_t **r_extent_hooks, + extent_t *extent) { + size_t extent_size = extent_size_get(extent); + extent_dalloc_wrapper(tsdn, arena, + r_extent_hooks, extent); + if (config_stats) { + /* Update stats accordingly. */ + arena_stats_lock(tsdn, &arena->stats); + arena_stats_add_u64(tsdn, &arena->stats, + &arena->decay_dirty.stats->nmadvise, 1); + arena_stats_add_u64(tsdn, &arena->stats, + &arena->decay_dirty.stats->purged, extent_size >> LG_PAGE); + arena_stats_sub_zu(tsdn, &arena->stats, &arena->stats.mapped, + extent_size); + arena_stats_unlock(tsdn, &arena->stats); + } +} + JEMALLOC_ALWAYS_INLINE void * arena_malloc(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, bool zero, tcache_t *tcache, bool slow_path) { assert(!tsdn_null(tsdn) || tcache == NULL); - assert(size != 0); if (likely(tcache != NULL)) { - if (likely(size <= SMALL_MAXCLASS)) { + if (likely(size <= SC_SMALL_MAXCLASS)) { return tcache_alloc_small(tsdn_tsd(tsdn), arena, tcache, size, ind, zero, slow_path); } @@ -126,7 +190,7 @@ arena_salloc(tsdn_t *tsdn, const void *ptr) { szind_t szind = rtree_szind_read(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)ptr, true); - assert(szind != NSIZES); + assert(szind != SC_NSIZES); return sz_index2size(szind); } @@ -159,7 +223,7 @@ arena_vsalloc(tsdn_t *tsdn, const void *ptr) { /* Only slab members should be looked up via interior pointers. */ assert(extent_addr_get(extent) == ptr || extent_slab_get(extent)); - assert(szind != NSIZES); + assert(szind != SC_NSIZES); return sz_index2size(szind); } @@ -180,7 +244,7 @@ arena_dalloc_no_tcache(tsdn_t *tsdn, void *ptr) { extent_t *extent = rtree_extent_read(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)ptr, true); assert(szind == extent_szind_get(extent)); - assert(szind < NSIZES); + assert(szind < SC_NSIZES); assert(slab == extent_slab_get(extent)); } @@ -210,7 +274,7 @@ arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache, if (alloc_ctx != NULL) { szind = alloc_ctx->szind; slab = alloc_ctx->slab; - assert(szind != NSIZES); + assert(szind != SC_NSIZES); } else { rtree_ctx = tsd_rtree_ctx(tsdn_tsd(tsdn)); rtree_szind_slab_read(tsdn, &extents_rtree, rtree_ctx, @@ -222,7 +286,7 @@ arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache, extent_t *extent = rtree_extent_read(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)ptr, true); assert(szind == extent_szind_get(extent)); - assert(szind < NSIZES); + assert(szind < SC_NSIZES); assert(slab == extent_slab_get(extent)); } @@ -232,7 +296,7 @@ arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache, slow_path); } else { if (szind < nhbins) { - if (config_prof && unlikely(szind < NBINS)) { + if (config_prof && unlikely(szind < SC_NBINS)) { arena_dalloc_promoted(tsdn, ptr, tcache, slow_path); } else { @@ -249,7 +313,7 @@ arena_dalloc(tsdn_t *tsdn, void *ptr, tcache_t *tcache, static inline void arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) { assert(ptr != NULL); - assert(size <= LARGE_MAXCLASS); + assert(size <= SC_LARGE_MAXCLASS); szind_t szind; bool slab; @@ -259,7 +323,7 @@ arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) { * object, so base szind and slab on the given size. */ szind = sz_size2index(size); - slab = (szind < NBINS); + slab = (szind < SC_NBINS); } if ((config_prof && opt_prof) || config_debug) { @@ -271,7 +335,7 @@ arena_sdalloc_no_tcache(tsdn_t *tsdn, void *ptr, size_t size) { (uintptr_t)ptr, true, &szind, &slab); assert(szind == sz_size2index(size)); - assert((config_prof && opt_prof) || slab == (szind < NBINS)); + assert((config_prof && opt_prof) || slab == (szind < SC_NBINS)); if (config_debug) { extent_t *extent = rtree_extent_read(tsdn, @@ -295,7 +359,7 @@ arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, alloc_ctx_t *alloc_ctx, bool slow_path) { assert(!tsdn_null(tsdn) || tcache == NULL); assert(ptr != NULL); - assert(size <= LARGE_MAXCLASS); + assert(size <= SC_LARGE_MAXCLASS); if (unlikely(tcache == NULL)) { arena_sdalloc_no_tcache(tsdn, ptr, size); @@ -304,7 +368,7 @@ arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, szind_t szind; bool slab; - UNUSED alloc_ctx_t local_ctx; + alloc_ctx_t local_ctx; if (config_prof && opt_prof) { if (alloc_ctx == NULL) { /* Uncommon case and should be a static check. */ @@ -325,7 +389,7 @@ arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, * object, so base szind and slab on the given size. */ szind = sz_size2index(size); - slab = (szind < NBINS); + slab = (szind < SC_NBINS); } if (config_debug) { @@ -344,7 +408,7 @@ arena_sdalloc(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, slow_path); } else { if (szind < nhbins) { - if (config_prof && unlikely(szind < NBINS)) { + if (config_prof && unlikely(szind < SC_NBINS)) { arena_dalloc_promoted(tsdn, ptr, tcache, slow_path); } else { diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/arena_stats.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/arena_stats.h new file mode 100644 index 0000000000..ef1e25b31f --- /dev/null +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/arena_stats.h @@ -0,0 +1,258 @@ +#ifndef JEMALLOC_INTERNAL_ARENA_STATS_H +#define JEMALLOC_INTERNAL_ARENA_STATS_H + +#include "jemalloc/internal/atomic.h" +#include "jemalloc/internal/mutex.h" +#include "jemalloc/internal/mutex_prof.h" +#include "jemalloc/internal/sc.h" + +JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS + +/* + * In those architectures that support 64-bit atomics, we use atomic updates for + * our 64-bit values. Otherwise, we use a plain uint64_t and synchronize + * externally. + */ +#ifdef JEMALLOC_ATOMIC_U64 +typedef atomic_u64_t arena_stats_u64_t; +#else +/* Must hold the arena stats mutex while reading atomically. */ +typedef uint64_t arena_stats_u64_t; +#endif + +typedef struct arena_stats_large_s arena_stats_large_t; +struct arena_stats_large_s { + /* + * Total number of allocation/deallocation requests served directly by + * the arena. + */ + arena_stats_u64_t nmalloc; + arena_stats_u64_t ndalloc; + + /* + * Number of allocation requests that correspond to this size class. + * This includes requests served by tcache, though tcache only + * periodically merges into this counter. + */ + arena_stats_u64_t nrequests; /* Partially derived. */ + + /* Current number of allocations of this size class. */ + size_t curlextents; /* Derived. */ +}; + +typedef struct arena_stats_decay_s arena_stats_decay_t; +struct arena_stats_decay_s { + /* Total number of purge sweeps. */ + arena_stats_u64_t npurge; + /* Total number of madvise calls made. */ + arena_stats_u64_t nmadvise; + /* Total number of pages purged. */ + arena_stats_u64_t purged; +}; + +typedef struct arena_stats_extents_s arena_stats_extents_t; +struct arena_stats_extents_s { + /* + * Stats for a given index in the range [0, SC_NPSIZES] in an extents_t. + * We track both bytes and # of extents: two extents in the same bucket + * may have different sizes if adjacent size classes differ by more than + * a page, so bytes cannot always be derived from # of extents. + */ + atomic_zu_t ndirty; + atomic_zu_t dirty_bytes; + atomic_zu_t nmuzzy; + atomic_zu_t muzzy_bytes; + atomic_zu_t nretained; + atomic_zu_t retained_bytes; +}; + +/* + * Arena stats. Note that fields marked "derived" are not directly maintained + * within the arena code; rather their values are derived during stats merge + * requests. + */ +typedef struct arena_stats_s arena_stats_t; +struct arena_stats_s { +#ifndef JEMALLOC_ATOMIC_U64 + malloc_mutex_t mtx; +#endif + + /* Number of bytes currently mapped, excluding retained memory. */ + atomic_zu_t mapped; /* Partially derived. */ + + /* + * Number of unused virtual memory bytes currently retained. Retained + * bytes are technically mapped (though always decommitted or purged), + * but they are excluded from the mapped statistic (above). + */ + atomic_zu_t retained; /* Derived. */ + + /* Number of extent_t structs allocated by base, but not being used. */ + atomic_zu_t extent_avail; + + arena_stats_decay_t decay_dirty; + arena_stats_decay_t decay_muzzy; + + atomic_zu_t base; /* Derived. */ + atomic_zu_t internal; + atomic_zu_t resident; /* Derived. */ + atomic_zu_t metadata_thp; + + atomic_zu_t allocated_large; /* Derived. */ + arena_stats_u64_t nmalloc_large; /* Derived. */ + arena_stats_u64_t ndalloc_large; /* Derived. */ + arena_stats_u64_t nrequests_large; /* Derived. */ + + /* Number of bytes cached in tcache associated with this arena. */ + atomic_zu_t tcache_bytes; /* Derived. */ + + mutex_prof_data_t mutex_prof_data[mutex_prof_num_arena_mutexes]; + + /* One element for each large size class. */ + arena_stats_large_t lstats[SC_NSIZES - SC_NBINS]; + + /* Arena uptime. */ + nstime_t uptime; +}; + +static inline bool +arena_stats_init(tsdn_t *tsdn, arena_stats_t *arena_stats) { + if (config_debug) { + for (size_t i = 0; i < sizeof(arena_stats_t); i++) { + assert(((char *)arena_stats)[i] == 0); + } + } +#ifndef JEMALLOC_ATOMIC_U64 + if (malloc_mutex_init(&arena_stats->mtx, "arena_stats", + WITNESS_RANK_ARENA_STATS, malloc_mutex_rank_exclusive)) { + return true; + } +#endif + /* Memory is zeroed, so there is no need to clear stats. */ + return false; +} + +static inline void +arena_stats_lock(tsdn_t *tsdn, arena_stats_t *arena_stats) { +#ifndef JEMALLOC_ATOMIC_U64 + malloc_mutex_lock(tsdn, &arena_stats->mtx); +#endif +} + +static inline void +arena_stats_unlock(tsdn_t *tsdn, arena_stats_t *arena_stats) { +#ifndef JEMALLOC_ATOMIC_U64 + malloc_mutex_unlock(tsdn, &arena_stats->mtx); +#endif +} + +static inline uint64_t +arena_stats_read_u64(tsdn_t *tsdn, arena_stats_t *arena_stats, + arena_stats_u64_t *p) { +#ifdef JEMALLOC_ATOMIC_U64 + return atomic_load_u64(p, ATOMIC_RELAXED); +#else + malloc_mutex_assert_owner(tsdn, &arena_stats->mtx); + return *p; +#endif +} + +static inline void +arena_stats_add_u64(tsdn_t *tsdn, arena_stats_t *arena_stats, + arena_stats_u64_t *p, uint64_t x) { +#ifdef JEMALLOC_ATOMIC_U64 + atomic_fetch_add_u64(p, x, ATOMIC_RELAXED); +#else + malloc_mutex_assert_owner(tsdn, &arena_stats->mtx); + *p += x; +#endif +} + +static inline void +arena_stats_sub_u64(tsdn_t *tsdn, arena_stats_t *arena_stats, + arena_stats_u64_t *p, uint64_t x) { +#ifdef JEMALLOC_ATOMIC_U64 + uint64_t r = atomic_fetch_sub_u64(p, x, ATOMIC_RELAXED); + assert(r - x <= r); +#else + malloc_mutex_assert_owner(tsdn, &arena_stats->mtx); + *p -= x; + assert(*p + x >= *p); +#endif +} + +/* + * Non-atomically sets *dst += src. *dst needs external synchronization. + * This lets us avoid the cost of a fetch_add when its unnecessary (note that + * the types here are atomic). + */ +static inline void +arena_stats_accum_u64(arena_stats_u64_t *dst, uint64_t src) { +#ifdef JEMALLOC_ATOMIC_U64 + uint64_t cur_dst = atomic_load_u64(dst, ATOMIC_RELAXED); + atomic_store_u64(dst, src + cur_dst, ATOMIC_RELAXED); +#else + *dst += src; +#endif +} + +static inline size_t +arena_stats_read_zu(tsdn_t *tsdn, arena_stats_t *arena_stats, + atomic_zu_t *p) { +#ifdef JEMALLOC_ATOMIC_U64 + return atomic_load_zu(p, ATOMIC_RELAXED); +#else + malloc_mutex_assert_owner(tsdn, &arena_stats->mtx); + return atomic_load_zu(p, ATOMIC_RELAXED); +#endif +} + +static inline void +arena_stats_add_zu(tsdn_t *tsdn, arena_stats_t *arena_stats, + atomic_zu_t *p, size_t x) { +#ifdef JEMALLOC_ATOMIC_U64 + atomic_fetch_add_zu(p, x, ATOMIC_RELAXED); +#else + malloc_mutex_assert_owner(tsdn, &arena_stats->mtx); + size_t cur = atomic_load_zu(p, ATOMIC_RELAXED); + atomic_store_zu(p, cur + x, ATOMIC_RELAXED); +#endif +} + +static inline void +arena_stats_sub_zu(tsdn_t *tsdn, arena_stats_t *arena_stats, + atomic_zu_t *p, size_t x) { +#ifdef JEMALLOC_ATOMIC_U64 + size_t r = atomic_fetch_sub_zu(p, x, ATOMIC_RELAXED); + assert(r - x <= r); +#else + malloc_mutex_assert_owner(tsdn, &arena_stats->mtx); + size_t cur = atomic_load_zu(p, ATOMIC_RELAXED); + atomic_store_zu(p, cur - x, ATOMIC_RELAXED); +#endif +} + +/* Like the _u64 variant, needs an externally synchronized *dst. */ +static inline void +arena_stats_accum_zu(atomic_zu_t *dst, size_t src) { + size_t cur_dst = atomic_load_zu(dst, ATOMIC_RELAXED); + atomic_store_zu(dst, src + cur_dst, ATOMIC_RELAXED); +} + +static inline void +arena_stats_large_nrequests_add(tsdn_t *tsdn, arena_stats_t *arena_stats, + szind_t szind, uint64_t nrequests) { + arena_stats_lock(tsdn, arena_stats); + arena_stats_add_u64(tsdn, arena_stats, &arena_stats->lstats[szind - + SC_NBINS].nrequests, nrequests); + arena_stats_unlock(tsdn, arena_stats); +} + +static inline void +arena_stats_mapped_add(tsdn_t *tsdn, arena_stats_t *arena_stats, size_t size) { + arena_stats_lock(tsdn, arena_stats); + arena_stats_add_zu(tsdn, arena_stats, &arena_stats->mapped, size); + arena_stats_unlock(tsdn, arena_stats); +} + +#endif /* JEMALLOC_INTERNAL_ARENA_STATS_H */ diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/arena_structs_b.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/arena_structs_b.h index d1fffec193..950bd13c27 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/arena_structs_b.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/arena_structs_b.h @@ -1,54 +1,19 @@ #ifndef JEMALLOC_INTERNAL_ARENA_STRUCTS_B_H #define JEMALLOC_INTERNAL_ARENA_STRUCTS_B_H +#include "jemalloc/internal/arena_stats.h" #include "jemalloc/internal/atomic.h" +#include "jemalloc/internal/bin.h" #include "jemalloc/internal/bitmap.h" #include "jemalloc/internal/extent_dss.h" #include "jemalloc/internal/jemalloc_internal_types.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/nstime.h" #include "jemalloc/internal/ql.h" -#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/sc.h" #include "jemalloc/internal/smoothstep.h" -#include "jemalloc/internal/stats.h" #include "jemalloc/internal/ticker.h" -/* - * Read-only information associated with each element of arena_t's bins array - * is stored separately, partly to reduce memory usage (only one copy, rather - * than one per arena), but mainly to avoid false cacheline sharing. - * - * Each slab has the following layout: - * - * /--------------------\ - * | region 0 | - * |--------------------| - * | region 1 | - * |--------------------| - * | ... | - * | ... | - * | ... | - * |--------------------| - * | region nregs-1 | - * \--------------------/ - */ -struct arena_bin_info_s { - /* Size of regions in a slab for this bin's size class. */ - size_t reg_size; - - /* Total size of a slab for this bin's size class. */ - size_t slab_size; - - /* Total number of regions in a slab for this bin's size class. */ - uint32_t nregs; - - /* - * Metadata used to manipulate bitmaps for slabs associated with this - * bin. - */ - bitmap_info_t bitmap_info; -}; - struct arena_decay_s { /* Synchronizes all non-atomic fields. */ malloc_mutex_t mtx; @@ -104,37 +69,11 @@ struct arena_decay_s { * arena and ctl code. * * Synchronization: Same as associated arena's stats field. */ - decay_stats_t *stats; + arena_stats_decay_t *stats; /* Peak number of pages in associated extents. Used for debug only. */ uint64_t ceil_npages; }; -struct arena_bin_s { - /* All operations on arena_bin_t fields require lock ownership. */ - malloc_mutex_t lock; - - /* - * Current slab being used to service allocations of this bin's size - * class. slabcur is independent of slabs_{nonfull,full}; whenever - * slabcur is reassigned, the previous slab must be deallocated or - * inserted into slabs_{nonfull,full}. - */ - extent_t *slabcur; - - /* - * Heap of non-full slabs. This heap is used to assure that new - * allocations come from the non-full slab that is oldest/lowest in - * memory. - */ - extent_heap_t slabs_nonfull; - - /* List used to track full slabs. */ - extent_list_t slabs_full; - - /* Bin statistics. */ - malloc_bin_stats_t stats; -}; - struct arena_s { /* * Number of threads currently assigned to this arena. Each thread has @@ -151,6 +90,9 @@ struct arena_s { */ atomic_u_t nthreads[2]; + /* Next bin shard for binding new threads. Synchronization: atomic. */ + atomic_u_t binshard_next; + /* * When percpu_arena is enabled, to amortize the cost of reading / * updating the current CPU id, track the most recent thread accessing @@ -162,14 +104,15 @@ struct arena_s { arena_stats_t stats; /* - * List of tcaches for extant threads associated with this arena. - * Stats from these are merged incrementally, and at exit if - * opt_stats_print is enabled. + * Lists of tcaches and cache_bin_array_descriptors for extant threads + * associated with this arena. Stats from these are merged + * incrementally, and at exit if opt_stats_print is enabled. * * Synchronization: tcache_ql_mtx. */ - ql_head(tcache_t) tcache_ql; - malloc_mutex_t tcache_ql_mtx; + ql_head(tcache_t) tcache_ql; + ql_head(cache_bin_array_descriptor_t) cache_bin_array_descriptor_ql; + malloc_mutex_t tcache_ql_mtx; /* Synchronization: internal. */ prof_accum_t prof_accum; @@ -239,9 +182,14 @@ struct arena_s { * be effective even if multiple arenas' extent allocation requests are * highly interleaved. * + * retain_grow_limit is the max allowed size ind to expand (unless the + * required size is greater). Default is no limit, and controlled + * through mallctl only. + * * Synchronization: extent_grow_mtx */ pszind_t extent_grow_next; + pszind_t retain_grow_limit; malloc_mutex_t extent_grow_mtx; /* @@ -251,6 +199,7 @@ struct arena_s { * Synchronization: extent_avail_mtx. */ extent_tree_t extent_avail; + atomic_zu_t extent_avail_cnt; malloc_mutex_t extent_avail_mtx; /* @@ -258,7 +207,7 @@ struct arena_s { * * Synchronization: internal. */ - arena_bin_t bins[NBINS]; + bins_t bins[SC_NBINS]; /* * Base allocator, from which arena metadata are allocated. diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/arena_types.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/arena_types.h index a691bd811e..624937e4f5 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/arena_types.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/arena_types.h @@ -1,20 +1,20 @@ #ifndef JEMALLOC_INTERNAL_ARENA_TYPES_H #define JEMALLOC_INTERNAL_ARENA_TYPES_H +#include "jemalloc/internal/sc.h" + /* Maximum number of regions in one slab. */ -#define LG_SLAB_MAXREGS (LG_PAGE - LG_TINY_MIN) +#define LG_SLAB_MAXREGS (LG_PAGE - SC_LG_TINY_MIN) #define SLAB_MAXREGS (1U << LG_SLAB_MAXREGS) /* Default decay times in milliseconds. */ #define DIRTY_DECAY_MS_DEFAULT ZD(10 * 1000) -#define MUZZY_DECAY_MS_DEFAULT ZD(10 * 1000) +#define MUZZY_DECAY_MS_DEFAULT (0) /* Number of event ticks between time checks. */ #define DECAY_NTICKS_PER_UPDATE 1000 typedef struct arena_slab_data_s arena_slab_data_t; -typedef struct arena_bin_info_s arena_bin_info_t; typedef struct arena_decay_s arena_decay_t; -typedef struct arena_bin_s arena_bin_t; typedef struct arena_s arena_t; typedef struct arena_tdata_s arena_tdata_t; typedef struct alloc_ctx_s alloc_ctx_t; @@ -42,4 +42,10 @@ typedef enum { #define PERCPU_ARENA_ENABLED(m) ((m) >= percpu_arena_mode_enabled_base) #define PERCPU_ARENA_DEFAULT percpu_arena_disabled +/* + * When allocation_size >= oversize_threshold, use the dedicated huge arena + * (unless have explicitly spicified arena index). 0 disables the feature. + */ +#define OVERSIZE_THRESHOLD_DEFAULT (8 << 20) + #endif /* JEMALLOC_INTERNAL_ARENA_TYPES_H */ diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/atomic.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/atomic.h index adadb1a3ac..a76f54cee3 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/atomic.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/atomic.h @@ -1,12 +1,19 @@ #ifndef JEMALLOC_INTERNAL_ATOMIC_H #define JEMALLOC_INTERNAL_ATOMIC_H -#define ATOMIC_INLINE static inline +#define ATOMIC_INLINE JEMALLOC_ALWAYS_INLINE +#define JEMALLOC_U8_ATOMICS #if defined(JEMALLOC_GCC_ATOMIC_ATOMICS) # include "jemalloc/internal/atomic_gcc_atomic.h" +# if !defined(JEMALLOC_GCC_U8_ATOMIC_ATOMICS) +# undef JEMALLOC_U8_ATOMICS +# endif #elif defined(JEMALLOC_GCC_SYNC_ATOMICS) # include "jemalloc/internal/atomic_gcc_sync.h" +# if !defined(JEMALLOC_GCC_U8_SYNC_ATOMICS) +# undef JEMALLOC_U8_ATOMICS +# endif #elif defined(_MSC_VER) # include "jemalloc/internal/atomic_msvc.h" #elif defined(JEMALLOC_C11_ATOMICS) @@ -66,6 +73,8 @@ JEMALLOC_GENERATE_INT_ATOMICS(size_t, zu, LG_SIZEOF_PTR) JEMALLOC_GENERATE_INT_ATOMICS(ssize_t, zd, LG_SIZEOF_PTR) +JEMALLOC_GENERATE_INT_ATOMICS(uint8_t, u8, 0) + JEMALLOC_GENERATE_INT_ATOMICS(uint32_t, u32, 2) #ifdef JEMALLOC_ATOMIC_U64 diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/atomic_gcc_sync.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/atomic_gcc_sync.h index 30846e4d27..e02b7cbe3c 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/atomic_gcc_sync.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/atomic_gcc_sync.h @@ -27,8 +27,10 @@ atomic_fence(atomic_memory_order_t mo) { asm volatile("" ::: "memory"); # if defined(__i386__) || defined(__x86_64__) /* This is implicit on x86. */ -# elif defined(__ppc__) +# elif defined(__ppc64__) asm volatile("lwsync"); +# elif defined(__ppc__) + asm volatile("sync"); # elif defined(__sparc__) && defined(__arch64__) if (mo == atomic_memory_order_acquire) { asm volatile("membar #LoadLoad | #LoadStore"); @@ -113,8 +115,8 @@ atomic_store_##short_type(atomic_##short_type##_t *a, \ } \ \ ATOMIC_INLINE type \ -atomic_exchange_##short_type(atomic_##short_type##_t *a, type val, \ - atomic_memory_order_t mo) { \ +atomic_exchange_##short_type(atomic_##short_type##_t *a, type val, \ + atomic_memory_order_t mo) { \ /* \ * Because of FreeBSD, we care about gcc 4.2, which doesn't have\ * an atomic exchange builtin. We fake it with a CAS loop. \ @@ -129,8 +131,9 @@ atomic_exchange_##short_type(atomic_##short_type##_t *a, type val, \ \ ATOMIC_INLINE bool \ atomic_compare_exchange_weak_##short_type(atomic_##short_type##_t *a, \ - type *expected, type desired, atomic_memory_order_t success_mo, \ - atomic_memory_order_t failure_mo) { \ + type *expected, type desired, \ + atomic_memory_order_t success_mo, \ + atomic_memory_order_t failure_mo) { \ type prev = __sync_val_compare_and_swap(&a->repr, *expected, \ desired); \ if (prev == *expected) { \ @@ -142,8 +145,9 @@ atomic_compare_exchange_weak_##short_type(atomic_##short_type##_t *a, \ } \ ATOMIC_INLINE bool \ atomic_compare_exchange_strong_##short_type(atomic_##short_type##_t *a, \ - type *expected, type desired, atomic_memory_order_t success_mo, \ - atomic_memory_order_t failure_mo) { \ + type *expected, type desired, \ + atomic_memory_order_t success_mo, \ + atomic_memory_order_t failure_mo) { \ type prev = __sync_val_compare_and_swap(&a->repr, *expected, \ desired); \ if (prev == *expected) { \ diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/background_thread_externs.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/background_thread_externs.h index 8b4b8471a9..0f997e18be 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/background_thread_externs.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/background_thread_externs.h @@ -2,11 +2,12 @@ #define JEMALLOC_INTERNAL_BACKGROUND_THREAD_EXTERNS_H extern bool opt_background_thread; +extern size_t opt_max_background_threads; extern malloc_mutex_t background_thread_lock; extern atomic_b_t background_thread_enabled_state; extern size_t n_background_threads; +extern size_t max_background_threads; extern background_thread_info_t *background_thread_info; -extern bool can_enable_background_thread; bool background_thread_create(tsd_t *tsd, unsigned arena_ind); bool background_threads_enable(tsd_t *tsd); diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/background_thread_inlines.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/background_thread_inlines.h index ef50231e8d..f85e86fa37 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/background_thread_inlines.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/background_thread_inlines.h @@ -15,7 +15,12 @@ background_thread_enabled_set(tsdn_t *tsdn, bool state) { JEMALLOC_ALWAYS_INLINE background_thread_info_t * arena_background_thread_info_get(arena_t *arena) { unsigned arena_ind = arena_ind_get(arena); - return &background_thread_info[arena_ind % ncpus]; + return &background_thread_info[arena_ind % max_background_threads]; +} + +JEMALLOC_ALWAYS_INLINE background_thread_info_t * +background_thread_info_get(size_t ind) { + return &background_thread_info[ind % max_background_threads]; } JEMALLOC_ALWAYS_INLINE uint64_t diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/background_thread_structs.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/background_thread_structs.h index e69a7d022b..c02aa434c7 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/background_thread_structs.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/background_thread_structs.h @@ -8,6 +8,8 @@ #endif #define BACKGROUND_THREAD_INDEFINITE_SLEEP UINT64_MAX +#define MAX_BACKGROUND_THREAD_LIMIT MALLOCX_ARENA_LIMIT +#define DEFAULT_NUM_BACKGROUND_THREAD 4 typedef enum { background_thread_stopped, diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/base_externs.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/base_externs.h index a4fd5ac7d9..7b705c9b4d 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/base_externs.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/base_externs.h @@ -1,6 +1,9 @@ #ifndef JEMALLOC_INTERNAL_BASE_EXTERNS_H #define JEMALLOC_INTERNAL_BASE_EXTERNS_H +extern metadata_thp_mode_t opt_metadata_thp; +extern const char *metadata_thp_mode_names[]; + base_t *b0get(void); base_t *base_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks); void base_delete(tsdn_t *tsdn, base_t *base); @@ -10,7 +13,7 @@ extent_hooks_t *base_extent_hooks_set(base_t *base, void *base_alloc(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment); extent_t *base_alloc_extent(tsdn_t *tsdn, base_t *base); void base_stats_get(tsdn_t *tsdn, base_t *base, size_t *allocated, - size_t *resident, size_t *mapped); + size_t *resident, size_t *mapped, size_t *n_thp); void base_prefork(tsdn_t *tsdn, base_t *base); void base_postfork_parent(tsdn_t *tsdn, base_t *base); void base_postfork_child(tsdn_t *tsdn, base_t *base); diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/base_inlines.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/base_inlines.h index 931560bfae..aec0e2e1e1 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/base_inlines.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/base_inlines.h @@ -6,4 +6,8 @@ base_ind_get(const base_t *base) { return base->ind; } +static inline bool +metadata_thp_enabled(void) { + return (opt_metadata_thp != metadata_thp_disabled); +} #endif /* JEMALLOC_INTERNAL_BASE_INLINES_H */ diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/base_structs.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/base_structs.h index 18e227bd5a..07f214eb2f 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/base_structs.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/base_structs.h @@ -3,7 +3,7 @@ #include "jemalloc/internal/jemalloc_internal_types.h" #include "jemalloc/internal/mutex.h" -#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/sc.h" /* Embedded at the beginning of every block of base-managed virtual memory. */ struct base_block_s { @@ -30,6 +30,8 @@ struct base_s { /* Protects base_alloc() and base_stats_get() operations. */ malloc_mutex_t mtx; + /* Using THP when true (metadata_thp auto mode). */ + bool auto_thp_switched; /* * Most recent size class in the series of increasingly large base * extents. Logarithmic spacing between subsequent allocations ensures @@ -44,12 +46,14 @@ struct base_s { base_block_t *blocks; /* Heap of extents that track unused trailing space within blocks. */ - extent_heap_t avail[NSIZES]; + extent_heap_t avail[SC_NSIZES]; /* Stats, only maintained if config_stats. */ size_t allocated; size_t resident; size_t mapped; + /* Number of THP regions touched. */ + size_t n_thp; }; #endif /* JEMALLOC_INTERNAL_BASE_STRUCTS_H */ diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/base_types.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/base_types.h index be7ee82589..b6db77df7c 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/base_types.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/base_types.h @@ -4,4 +4,30 @@ typedef struct base_block_s base_block_t; typedef struct base_s base_t; +#define METADATA_THP_DEFAULT metadata_thp_disabled + +/* + * In auto mode, arenas switch to huge pages for the base allocator on the + * second base block. a0 switches to thp on the 5th block (after 20 megabytes + * of metadata), since more metadata (e.g. rtree nodes) come from a0's base. + */ + +#define BASE_AUTO_THP_THRESHOLD 2 +#define BASE_AUTO_THP_THRESHOLD_A0 5 + +typedef enum { + metadata_thp_disabled = 0, + /* + * Lazily enable hugepage for metadata. To avoid high RSS caused by THP + * + low usage arena (i.e. THP becomes a significant percentage), the + * "auto" option only starts using THP after a base allocator used up + * the first THP region. Starting from the second hugepage (in a single + * arena), "auto" behaves the same as "always", i.e. madvise hugepage + * right away. + */ + metadata_thp_auto = 1, + metadata_thp_always = 2, + metadata_thp_mode_limit = 3 +} metadata_thp_mode_t; + #endif /* JEMALLOC_INTERNAL_BASE_TYPES_H */ diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/bin.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/bin.h new file mode 100644 index 0000000000..f542c88298 --- /dev/null +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/bin.h @@ -0,0 +1,122 @@ +#ifndef JEMALLOC_INTERNAL_BIN_H +#define JEMALLOC_INTERNAL_BIN_H + +#include "jemalloc/internal/bin_stats.h" +#include "jemalloc/internal/bin_types.h" +#include "jemalloc/internal/extent_types.h" +#include "jemalloc/internal/extent_structs.h" +#include "jemalloc/internal/mutex.h" +#include "jemalloc/internal/sc.h" + +/* + * A bin contains a set of extents that are currently being used for slab + * allocations. + */ + +/* + * Read-only information associated with each element of arena_t's bins array + * is stored separately, partly to reduce memory usage (only one copy, rather + * than one per arena), but mainly to avoid false cacheline sharing. + * + * Each slab has the following layout: + * + * /--------------------\ + * | region 0 | + * |--------------------| + * | region 1 | + * |--------------------| + * | ... | + * | ... | + * | ... | + * |--------------------| + * | region nregs-1 | + * \--------------------/ + */ +typedef struct bin_info_s bin_info_t; +struct bin_info_s { + /* Size of regions in a slab for this bin's size class. */ + size_t reg_size; + + /* Total size of a slab for this bin's size class. */ + size_t slab_size; + + /* Total number of regions in a slab for this bin's size class. */ + uint32_t nregs; + + /* Number of sharded bins in each arena for this size class. */ + uint32_t n_shards; + + /* + * Metadata used to manipulate bitmaps for slabs associated with this + * bin. + */ + bitmap_info_t bitmap_info; +}; + +extern bin_info_t bin_infos[SC_NBINS]; + +typedef struct bin_s bin_t; +struct bin_s { + /* All operations on bin_t fields require lock ownership. */ + malloc_mutex_t lock; + + /* + * Current slab being used to service allocations of this bin's size + * class. slabcur is independent of slabs_{nonfull,full}; whenever + * slabcur is reassigned, the previous slab must be deallocated or + * inserted into slabs_{nonfull,full}. + */ + extent_t *slabcur; + + /* + * Heap of non-full slabs. This heap is used to assure that new + * allocations come from the non-full slab that is oldest/lowest in + * memory. + */ + extent_heap_t slabs_nonfull; + + /* List used to track full slabs. */ + extent_list_t slabs_full; + + /* Bin statistics. */ + bin_stats_t stats; +}; + +/* A set of sharded bins of the same size class. */ +typedef struct bins_s bins_t; +struct bins_s { + /* Sharded bins. Dynamically sized. */ + bin_t *bin_shards; +}; + +void bin_shard_sizes_boot(unsigned bin_shards[SC_NBINS]); +bool bin_update_shard_size(unsigned bin_shards[SC_NBINS], size_t start_size, + size_t end_size, size_t nshards); +void bin_boot(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS]); + +/* Initializes a bin to empty. Returns true on error. */ +bool bin_init(bin_t *bin); + +/* Forking. */ +void bin_prefork(tsdn_t *tsdn, bin_t *bin); +void bin_postfork_parent(tsdn_t *tsdn, bin_t *bin); +void bin_postfork_child(tsdn_t *tsdn, bin_t *bin); + +/* Stats. */ +static inline void +bin_stats_merge(tsdn_t *tsdn, bin_stats_t *dst_bin_stats, bin_t *bin) { + malloc_mutex_lock(tsdn, &bin->lock); + malloc_mutex_prof_accum(tsdn, &dst_bin_stats->mutex_data, &bin->lock); + dst_bin_stats->nmalloc += bin->stats.nmalloc; + dst_bin_stats->ndalloc += bin->stats.ndalloc; + dst_bin_stats->nrequests += bin->stats.nrequests; + dst_bin_stats->curregs += bin->stats.curregs; + dst_bin_stats->nfills += bin->stats.nfills; + dst_bin_stats->nflushes += bin->stats.nflushes; + dst_bin_stats->nslabs += bin->stats.nslabs; + dst_bin_stats->reslabs += bin->stats.reslabs; + dst_bin_stats->curslabs += bin->stats.curslabs; + malloc_mutex_unlock(tsdn, &bin->lock); +} + +#endif /* JEMALLOC_INTERNAL_BIN_H */ diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/bin_stats.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/bin_stats.h new file mode 100644 index 0000000000..86e673ec44 --- /dev/null +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/bin_stats.h @@ -0,0 +1,51 @@ +#ifndef JEMALLOC_INTERNAL_BIN_STATS_H +#define JEMALLOC_INTERNAL_BIN_STATS_H + +#include "jemalloc/internal/mutex_prof.h" + +typedef struct bin_stats_s bin_stats_t; +struct bin_stats_s { + /* + * Total number of allocation/deallocation requests served directly by + * the bin. Note that tcache may allocate an object, then recycle it + * many times, resulting many increments to nrequests, but only one + * each to nmalloc and ndalloc. + */ + uint64_t nmalloc; + uint64_t ndalloc; + + /* + * Number of allocation requests that correspond to the size of this + * bin. This includes requests served by tcache, though tcache only + * periodically merges into this counter. + */ + uint64_t nrequests; + + /* + * Current number of regions of this size class, including regions + * currently cached by tcache. + */ + size_t curregs; + + /* Number of tcache fills from this bin. */ + uint64_t nfills; + + /* Number of tcache flushes to this bin. */ + uint64_t nflushes; + + /* Total number of slabs created for this bin's size class. */ + uint64_t nslabs; + + /* + * Total number of slabs reused by extracting them from the slabs heap + * for this bin's size class. + */ + uint64_t reslabs; + + /* Current number of slabs in this bin. */ + size_t curslabs; + + mutex_prof_data_t mutex_data; +}; + +#endif /* JEMALLOC_INTERNAL_BIN_STATS_H */ diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/bin_types.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/bin_types.h new file mode 100644 index 0000000000..3533606b90 --- /dev/null +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/bin_types.h @@ -0,0 +1,17 @@ +#ifndef JEMALLOC_INTERNAL_BIN_TYPES_H +#define JEMALLOC_INTERNAL_BIN_TYPES_H + +#include "jemalloc/internal/sc.h" + +#define BIN_SHARDS_MAX (1 << EXTENT_BITS_BINSHARD_WIDTH) +#define N_BIN_SHARDS_DEFAULT 1 + +/* Used in TSD static initializer only. Real init in arena_bind(). */ +#define TSD_BINSHARDS_ZERO_INITIALIZER {{UINT8_MAX}} + +typedef struct tsd_binshards_s tsd_binshards_t; +struct tsd_binshards_s { + uint8_t binshard[SC_NBINS]; +}; + +#endif /* JEMALLOC_INTERNAL_BIN_TYPES_H */ diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/bit_util.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/bit_util.h index 8d078a8a35..c045eb8687 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/bit_util.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/bit_util.h @@ -27,6 +27,25 @@ ffs_u(unsigned bitmap) { return JEMALLOC_INTERNAL_FFS(bitmap); } +#ifdef JEMALLOC_INTERNAL_POPCOUNTL +BIT_UTIL_INLINE unsigned +popcount_lu(unsigned long bitmap) { + return JEMALLOC_INTERNAL_POPCOUNTL(bitmap); +} +#endif + +/* + * Clears first unset bit in bitmap, and returns + * place of bit. bitmap *must not* be 0. + */ + +BIT_UTIL_INLINE size_t +cfs_lu(unsigned long* bitmap) { + size_t bit = ffs_lu(*bitmap) - 1; + *bitmap ^= ZU(1) << bit; + return bit; +} + BIT_UTIL_INLINE unsigned ffs_zu(size_t bitmap) { #if LG_SIZEOF_PTR == LG_SIZEOF_INT @@ -63,6 +82,22 @@ ffs_u32(uint32_t bitmap) { BIT_UTIL_INLINE uint64_t pow2_ceil_u64(uint64_t x) { +#if (defined(__amd64__) || defined(__x86_64__) || defined(JEMALLOC_HAVE_BUILTIN_CLZ)) + if(unlikely(x <= 1)) { + return x; + } + size_t msb_on_index; +#if (defined(__amd64__) || defined(__x86_64__)) + asm ("bsrq %1, %0" + : "=r"(msb_on_index) // Outputs. + : "r"(x-1) // Inputs. + ); +#elif (defined(JEMALLOC_HAVE_BUILTIN_CLZ)) + msb_on_index = (63 ^ __builtin_clzll(x - 1)); +#endif + assert(msb_on_index < 63); + return 1ULL << (msb_on_index + 1); +#else x--; x |= x >> 1; x |= x >> 2; @@ -72,10 +107,27 @@ pow2_ceil_u64(uint64_t x) { x |= x >> 32; x++; return x; +#endif } BIT_UTIL_INLINE uint32_t pow2_ceil_u32(uint32_t x) { +#if ((defined(__i386__) || defined(JEMALLOC_HAVE_BUILTIN_CLZ)) && (!defined(__s390__))) + if(unlikely(x <= 1)) { + return x; + } + size_t msb_on_index; +#if (defined(__i386__)) + asm ("bsr %1, %0" + : "=r"(msb_on_index) // Outputs. + : "r"(x-1) // Inputs. + ); +#elif (defined(JEMALLOC_HAVE_BUILTIN_CLZ)) + msb_on_index = (31 ^ __builtin_clz(x - 1)); +#endif + assert(msb_on_index < 31); + return 1U << (msb_on_index + 1); +#else x--; x |= x >> 1; x |= x >> 2; @@ -84,6 +136,7 @@ pow2_ceil_u32(uint32_t x) { x |= x >> 16; x++; return x; +#endif } /* Compute the smallest power of 2 that is >= x. */ @@ -160,6 +213,27 @@ lg_floor(size_t x) { } #endif +BIT_UTIL_INLINE unsigned +lg_ceil(size_t x) { + return lg_floor(x) + ((x & (x - 1)) == 0 ? 0 : 1); +} + #undef BIT_UTIL_INLINE +/* A compile-time version of lg_floor and lg_ceil. */ +#define LG_FLOOR_1(x) 0 +#define LG_FLOOR_2(x) (x < (1ULL << 1) ? LG_FLOOR_1(x) : 1 + LG_FLOOR_1(x >> 1)) +#define LG_FLOOR_4(x) (x < (1ULL << 2) ? LG_FLOOR_2(x) : 2 + LG_FLOOR_2(x >> 2)) +#define LG_FLOOR_8(x) (x < (1ULL << 4) ? LG_FLOOR_4(x) : 4 + LG_FLOOR_4(x >> 4)) +#define LG_FLOOR_16(x) (x < (1ULL << 8) ? LG_FLOOR_8(x) : 8 + LG_FLOOR_8(x >> 8)) +#define LG_FLOOR_32(x) (x < (1ULL << 16) ? LG_FLOOR_16(x) : 16 + LG_FLOOR_16(x >> 16)) +#define LG_FLOOR_64(x) (x < (1ULL << 32) ? LG_FLOOR_32(x) : 32 + LG_FLOOR_32(x >> 32)) +#if LG_SIZEOF_PTR == 2 +# define LG_FLOOR(x) LG_FLOOR_32((x)) +#else +# define LG_FLOOR(x) LG_FLOOR_64((x)) +#endif + +#define LG_CEIL(x) (LG_FLOOR(x) + (((x) & ((x) - 1)) == 0 ? 0 : 1)) + #endif /* JEMALLOC_INTERNAL_BIT_UTIL_H */ diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/bitmap.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/bitmap.h index ac990290a5..c3f9cb490f 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/bitmap.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/bitmap.h @@ -3,18 +3,18 @@ #include "jemalloc/internal/arena_types.h" #include "jemalloc/internal/bit_util.h" -#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/sc.h" typedef unsigned long bitmap_t; #define LG_SIZEOF_BITMAP LG_SIZEOF_LONG /* Maximum bitmap bit count is 2^LG_BITMAP_MAXBITS. */ -#if LG_SLAB_MAXREGS > LG_CEIL_NSIZES +#if LG_SLAB_MAXREGS > LG_CEIL(SC_NSIZES) /* Maximum bitmap bit count is determined by maximum regions per slab. */ # define LG_BITMAP_MAXBITS LG_SLAB_MAXREGS #else /* Maximum bitmap bit count is determined by number of extent size classes. */ -# define LG_BITMAP_MAXBITS LG_CEIL_NSIZES +# define LG_BITMAP_MAXBITS LG_CEIL(SC_NSIZES) #endif #define BITMAP_MAXBITS (ZU(1) << LG_BITMAP_MAXBITS) diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/cache_bin.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/cache_bin.h new file mode 100644 index 0000000000..d14556a3da --- /dev/null +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/cache_bin.h @@ -0,0 +1,131 @@ +#ifndef JEMALLOC_INTERNAL_CACHE_BIN_H +#define JEMALLOC_INTERNAL_CACHE_BIN_H + +#include "jemalloc/internal/ql.h" + +/* + * The cache_bins are the mechanism that the tcache and the arena use to + * communicate. The tcache fills from and flushes to the arena by passing a + * cache_bin_t to fill/flush. When the arena needs to pull stats from the + * tcaches associated with it, it does so by iterating over its + * cache_bin_array_descriptor_t objects and reading out per-bin stats it + * contains. This makes it so that the arena need not know about the existence + * of the tcache at all. + */ + + +/* + * The count of the number of cached allocations in a bin. We make this signed + * so that negative numbers can encode "invalid" states (e.g. a low water mark + * of -1 for a cache that has been depleted). + */ +typedef int32_t cache_bin_sz_t; + +typedef struct cache_bin_stats_s cache_bin_stats_t; +struct cache_bin_stats_s { + /* + * Number of allocation requests that corresponded to the size of this + * bin. + */ + uint64_t nrequests; +}; + +/* + * Read-only information associated with each element of tcache_t's tbins array + * is stored separately, mainly to reduce memory usage. + */ +typedef struct cache_bin_info_s cache_bin_info_t; +struct cache_bin_info_s { + /* Upper limit on ncached. */ + cache_bin_sz_t ncached_max; +}; + +typedef struct cache_bin_s cache_bin_t; +struct cache_bin_s { + /* Min # cached since last GC. */ + cache_bin_sz_t low_water; + /* # of cached objects. */ + cache_bin_sz_t ncached; + /* + * ncached and stats are both modified frequently. Let's keep them + * close so that they have a higher chance of being on the same + * cacheline, thus less write-backs. + */ + cache_bin_stats_t tstats; + /* + * Stack of available objects. + * + * To make use of adjacent cacheline prefetch, the items in the avail + * stack goes to higher address for newer allocations. avail points + * just above the available space, which means that + * avail[-ncached, ... -1] are available items and the lowest item will + * be allocated first. + */ + void **avail; +}; + +typedef struct cache_bin_array_descriptor_s cache_bin_array_descriptor_t; +struct cache_bin_array_descriptor_s { + /* + * The arena keeps a list of the cache bins associated with it, for + * stats collection. + */ + ql_elm(cache_bin_array_descriptor_t) link; + /* Pointers to the tcache bins. */ + cache_bin_t *bins_small; + cache_bin_t *bins_large; +}; + +static inline void +cache_bin_array_descriptor_init(cache_bin_array_descriptor_t *descriptor, + cache_bin_t *bins_small, cache_bin_t *bins_large) { + ql_elm_new(descriptor, link); + descriptor->bins_small = bins_small; + descriptor->bins_large = bins_large; +} + +JEMALLOC_ALWAYS_INLINE void * +cache_bin_alloc_easy(cache_bin_t *bin, bool *success) { + void *ret; + + bin->ncached--; + + /* + * Check for both bin->ncached == 0 and ncached < low_water + * in a single branch. + */ + if (unlikely(bin->ncached <= bin->low_water)) { + bin->low_water = bin->ncached; + if (bin->ncached == -1) { + bin->ncached = 0; + *success = false; + return NULL; + } + } + + /* + * success (instead of ret) should be checked upon the return of this + * function. We avoid checking (ret == NULL) because there is never a + * null stored on the avail stack (which is unknown to the compiler), + * and eagerly checking ret would cause pipeline stall (waiting for the + * cacheline). + */ + *success = true; + ret = *(bin->avail - (bin->ncached + 1)); + + return ret; +} + +JEMALLOC_ALWAYS_INLINE bool +cache_bin_dalloc_easy(cache_bin_t *bin, cache_bin_info_t *bin_info, void *ptr) { + if (unlikely(bin->ncached == bin_info->ncached_max)) { + return false; + } + assert(bin->ncached < bin_info->ncached_max); + bin->ncached++; + *(bin->avail - bin->ncached) = ptr; + + return true; +} + +#endif /* JEMALLOC_INTERNAL_CACHE_BIN_H */ diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/ctl.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/ctl.h index a91c4cf556..775fdec04e 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/ctl.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/ctl.h @@ -5,7 +5,7 @@ #include "jemalloc/internal/malloc_io.h" #include "jemalloc/internal/mutex_prof.h" #include "jemalloc/internal/ql.h" -#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/sc.h" #include "jemalloc/internal/stats.h" /* Maximum ctl tree depth. */ @@ -40,14 +40,16 @@ typedef struct ctl_arena_stats_s { uint64_t ndalloc_small; uint64_t nrequests_small; - malloc_bin_stats_t bstats[NBINS]; - malloc_large_stats_t lstats[NSIZES - NBINS]; + bin_stats_t bstats[SC_NBINS]; + arena_stats_large_t lstats[SC_NSIZES - SC_NBINS]; + arena_stats_extents_t estats[SC_NPSIZES]; } ctl_arena_stats_t; typedef struct ctl_stats_s { size_t allocated; size_t active; size_t metadata; + size_t metadata_thp; size_t resident; size_t mapped; size_t retained; diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/div.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/div.h new file mode 100644 index 0000000000..aebae9398c --- /dev/null +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/div.h @@ -0,0 +1,41 @@ +#ifndef JEMALLOC_INTERNAL_DIV_H +#define JEMALLOC_INTERNAL_DIV_H + +#include "jemalloc/internal/assert.h" + +/* + * This module does the division that computes the index of a region in a slab, + * given its offset relative to the base. + * That is, given a divisor d, an n = i * d (all integers), we'll return i. + * We do some pre-computation to do this more quickly than a CPU division + * instruction. + * We bound n < 2^32, and don't support dividing by one. + */ + +typedef struct div_info_s div_info_t; +struct div_info_s { + uint32_t magic; +#ifdef JEMALLOC_DEBUG + size_t d; +#endif +}; + +void div_init(div_info_t *div_info, size_t divisor); + +static inline size_t +div_compute(div_info_t *div_info, size_t n) { + assert(n <= (uint32_t)-1); + /* + * This generates, e.g. mov; imul; shr on x86-64. On a 32-bit machine, + * the compilers I tried were all smart enough to turn this into the + * appropriate "get the high 32 bits of the result of a multiply" (e.g. + * mul; mov edx eax; on x86, umull on arm, etc.). + */ + size_t i = ((uint64_t)n * (uint64_t)div_info->magic) >> 32; +#ifdef JEMALLOC_DEBUG + assert(i * div_info->d == n); +#endif + return i; +} + +#endif /* JEMALLOC_INTERNAL_DIV_H */ diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/emitter.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/emitter.h new file mode 100644 index 0000000000..0a8bc2c06b --- /dev/null +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/emitter.h @@ -0,0 +1,483 @@ +#ifndef JEMALLOC_INTERNAL_EMITTER_H +#define JEMALLOC_INTERNAL_EMITTER_H + +#include "jemalloc/internal/ql.h" + +typedef enum emitter_output_e emitter_output_t; +enum emitter_output_e { + emitter_output_json, + emitter_output_table +}; + +typedef enum emitter_justify_e emitter_justify_t; +enum emitter_justify_e { + emitter_justify_left, + emitter_justify_right, + /* Not for users; just to pass to internal functions. */ + emitter_justify_none +}; + +typedef enum emitter_type_e emitter_type_t; +enum emitter_type_e { + emitter_type_bool, + emitter_type_int, + emitter_type_unsigned, + emitter_type_uint32, + emitter_type_uint64, + emitter_type_size, + emitter_type_ssize, + emitter_type_string, + /* + * A title is a column title in a table; it's just a string, but it's + * not quoted. + */ + emitter_type_title, +}; + +typedef struct emitter_col_s emitter_col_t; +struct emitter_col_s { + /* Filled in by the user. */ + emitter_justify_t justify; + int width; + emitter_type_t type; + union { + bool bool_val; + int int_val; + unsigned unsigned_val; + uint32_t uint32_val; + uint32_t uint32_t_val; + uint64_t uint64_val; + uint64_t uint64_t_val; + size_t size_val; + ssize_t ssize_val; + const char *str_val; + }; + + /* Filled in by initialization. */ + ql_elm(emitter_col_t) link; +}; + +typedef struct emitter_row_s emitter_row_t; +struct emitter_row_s { + ql_head(emitter_col_t) cols; +}; + +typedef struct emitter_s emitter_t; +struct emitter_s { + emitter_output_t output; + /* The output information. */ + void (*write_cb)(void *, const char *); + void *cbopaque; + int nesting_depth; + /* True if we've already emitted a value at the given depth. */ + bool item_at_depth; + /* True if we emitted a key and will emit corresponding value next. */ + bool emitted_key; +}; + +/* Internal convenience function. Write to the emitter the given string. */ +JEMALLOC_FORMAT_PRINTF(2, 3) +static inline void +emitter_printf(emitter_t *emitter, const char *format, ...) { + va_list ap; + + va_start(ap, format); + malloc_vcprintf(emitter->write_cb, emitter->cbopaque, format, ap); + va_end(ap); +} + +static inline void +emitter_gen_fmt(char *out_fmt, size_t out_size, const char *fmt_specifier, + emitter_justify_t justify, int width) { + size_t written; + if (justify == emitter_justify_none) { + written = malloc_snprintf(out_fmt, out_size, + "%%%s", fmt_specifier); + } else if (justify == emitter_justify_left) { + written = malloc_snprintf(out_fmt, out_size, + "%%-%d%s", width, fmt_specifier); + } else { + written = malloc_snprintf(out_fmt, out_size, + "%%%d%s", width, fmt_specifier); + } + /* Only happens in case of bad format string, which *we* choose. */ + assert(written < out_size); +} + +/* + * Internal. Emit the given value type in the relevant encoding (so that the + * bool true gets mapped to json "true", but the string "true" gets mapped to + * json "\"true\"", for instance. + * + * Width is ignored if justify is emitter_justify_none. + */ +static inline void +emitter_print_value(emitter_t *emitter, emitter_justify_t justify, int width, + emitter_type_t value_type, const void *value) { + size_t str_written; +#define BUF_SIZE 256 +#define FMT_SIZE 10 + /* + * We dynamically generate a format string to emit, to let us use the + * snprintf machinery. This is kinda hacky, but gets the job done + * quickly without having to think about the various snprintf edge + * cases. + */ + char fmt[FMT_SIZE]; + char buf[BUF_SIZE]; + +#define EMIT_SIMPLE(type, format) \ + emitter_gen_fmt(fmt, FMT_SIZE, format, justify, width); \ + emitter_printf(emitter, fmt, *(const type *)value); \ + + switch (value_type) { + case emitter_type_bool: + emitter_gen_fmt(fmt, FMT_SIZE, "s", justify, width); + emitter_printf(emitter, fmt, *(const bool *)value ? + "true" : "false"); + break; + case emitter_type_int: + EMIT_SIMPLE(int, "d") + break; + case emitter_type_unsigned: + EMIT_SIMPLE(unsigned, "u") + break; + case emitter_type_ssize: + EMIT_SIMPLE(ssize_t, "zd") + break; + case emitter_type_size: + EMIT_SIMPLE(size_t, "zu") + break; + case emitter_type_string: + str_written = malloc_snprintf(buf, BUF_SIZE, "\"%s\"", + *(const char *const *)value); + /* + * We control the strings we output; we shouldn't get anything + * anywhere near the fmt size. + */ + assert(str_written < BUF_SIZE); + emitter_gen_fmt(fmt, FMT_SIZE, "s", justify, width); + emitter_printf(emitter, fmt, buf); + break; + case emitter_type_uint32: + EMIT_SIMPLE(uint32_t, FMTu32) + break; + case emitter_type_uint64: + EMIT_SIMPLE(uint64_t, FMTu64) + break; + case emitter_type_title: + EMIT_SIMPLE(char *const, "s"); + break; + default: + unreachable(); + } +#undef BUF_SIZE +#undef FMT_SIZE +} + + +/* Internal functions. In json mode, tracks nesting state. */ +static inline void +emitter_nest_inc(emitter_t *emitter) { + emitter->nesting_depth++; + emitter->item_at_depth = false; +} + +static inline void +emitter_nest_dec(emitter_t *emitter) { + emitter->nesting_depth--; + emitter->item_at_depth = true; +} + +static inline void +emitter_indent(emitter_t *emitter) { + int amount = emitter->nesting_depth; + const char *indent_str; + if (emitter->output == emitter_output_json) { + indent_str = "\t"; + } else { + amount *= 2; + indent_str = " "; + } + for (int i = 0; i < amount; i++) { + emitter_printf(emitter, "%s", indent_str); + } +} + +static inline void +emitter_json_key_prefix(emitter_t *emitter) { + if (emitter->emitted_key) { + emitter->emitted_key = false; + return; + } + emitter_printf(emitter, "%s\n", emitter->item_at_depth ? "," : ""); + emitter_indent(emitter); +} + +/******************************************************************************/ +/* Public functions for emitter_t. */ + +static inline void +emitter_init(emitter_t *emitter, emitter_output_t emitter_output, + void (*write_cb)(void *, const char *), void *cbopaque) { + emitter->output = emitter_output; + emitter->write_cb = write_cb; + emitter->cbopaque = cbopaque; + emitter->item_at_depth = false; + emitter->emitted_key = false; + emitter->nesting_depth = 0; +} + +/******************************************************************************/ +/* JSON public API. */ + +/* + * Emits a key (e.g. as appears in an object). The next json entity emitted will + * be the corresponding value. + */ +static inline void +emitter_json_key(emitter_t *emitter, const char *json_key) { + if (emitter->output == emitter_output_json) { + emitter_json_key_prefix(emitter); + emitter_printf(emitter, "\"%s\": ", json_key); + emitter->emitted_key = true; + } +} + +static inline void +emitter_json_value(emitter_t *emitter, emitter_type_t value_type, + const void *value) { + if (emitter->output == emitter_output_json) { + emitter_json_key_prefix(emitter); + emitter_print_value(emitter, emitter_justify_none, -1, + value_type, value); + emitter->item_at_depth = true; + } +} + +/* Shorthand for calling emitter_json_key and then emitter_json_value. */ +static inline void +emitter_json_kv(emitter_t *emitter, const char *json_key, + emitter_type_t value_type, const void *value) { + emitter_json_key(emitter, json_key); + emitter_json_value(emitter, value_type, value); +} + +static inline void +emitter_json_array_begin(emitter_t *emitter) { + if (emitter->output == emitter_output_json) { + emitter_json_key_prefix(emitter); + emitter_printf(emitter, "["); + emitter_nest_inc(emitter); + } +} + +/* Shorthand for calling emitter_json_key and then emitter_json_array_begin. */ +static inline void +emitter_json_array_kv_begin(emitter_t *emitter, const char *json_key) { + emitter_json_key(emitter, json_key); + emitter_json_array_begin(emitter); +} + +static inline void +emitter_json_array_end(emitter_t *emitter) { + if (emitter->output == emitter_output_json) { + assert(emitter->nesting_depth > 0); + emitter_nest_dec(emitter); + emitter_printf(emitter, "\n"); + emitter_indent(emitter); + emitter_printf(emitter, "]"); + } +} + +static inline void +emitter_json_object_begin(emitter_t *emitter) { + if (emitter->output == emitter_output_json) { + emitter_json_key_prefix(emitter); + emitter_printf(emitter, "{"); + emitter_nest_inc(emitter); + } +} + +/* Shorthand for calling emitter_json_key and then emitter_json_object_begin. */ +static inline void +emitter_json_object_kv_begin(emitter_t *emitter, const char *json_key) { + emitter_json_key(emitter, json_key); + emitter_json_object_begin(emitter); +} + +static inline void +emitter_json_object_end(emitter_t *emitter) { + if (emitter->output == emitter_output_json) { + assert(emitter->nesting_depth > 0); + emitter_nest_dec(emitter); + emitter_printf(emitter, "\n"); + emitter_indent(emitter); + emitter_printf(emitter, "}"); + } +} + + +/******************************************************************************/ +/* Table public API. */ + +static inline void +emitter_table_dict_begin(emitter_t *emitter, const char *table_key) { + if (emitter->output == emitter_output_table) { + emitter_indent(emitter); + emitter_printf(emitter, "%s\n", table_key); + emitter_nest_inc(emitter); + } +} + +static inline void +emitter_table_dict_end(emitter_t *emitter) { + if (emitter->output == emitter_output_table) { + emitter_nest_dec(emitter); + } +} + +static inline void +emitter_table_kv_note(emitter_t *emitter, const char *table_key, + emitter_type_t value_type, const void *value, + const char *table_note_key, emitter_type_t table_note_value_type, + const void *table_note_value) { + if (emitter->output == emitter_output_table) { + emitter_indent(emitter); + emitter_printf(emitter, "%s: ", table_key); + emitter_print_value(emitter, emitter_justify_none, -1, + value_type, value); + if (table_note_key != NULL) { + emitter_printf(emitter, " (%s: ", table_note_key); + emitter_print_value(emitter, emitter_justify_none, -1, + table_note_value_type, table_note_value); + emitter_printf(emitter, ")"); + } + emitter_printf(emitter, "\n"); + } + emitter->item_at_depth = true; +} + +static inline void +emitter_table_kv(emitter_t *emitter, const char *table_key, + emitter_type_t value_type, const void *value) { + emitter_table_kv_note(emitter, table_key, value_type, value, NULL, + emitter_type_bool, NULL); +} + + +/* Write to the emitter the given string, but only in table mode. */ +JEMALLOC_FORMAT_PRINTF(2, 3) +static inline void +emitter_table_printf(emitter_t *emitter, const char *format, ...) { + if (emitter->output == emitter_output_table) { + va_list ap; + va_start(ap, format); + malloc_vcprintf(emitter->write_cb, emitter->cbopaque, format, ap); + va_end(ap); + } +} + +static inline void +emitter_table_row(emitter_t *emitter, emitter_row_t *row) { + if (emitter->output != emitter_output_table) { + return; + } + emitter_col_t *col; + ql_foreach(col, &row->cols, link) { + emitter_print_value(emitter, col->justify, col->width, + col->type, (const void *)&col->bool_val); + } + emitter_table_printf(emitter, "\n"); +} + +static inline void +emitter_row_init(emitter_row_t *row) { + ql_new(&row->cols); +} + +static inline void +emitter_col_init(emitter_col_t *col, emitter_row_t *row) { + ql_elm_new(col, link); + ql_tail_insert(&row->cols, col, link); +} + + +/******************************************************************************/ +/* + * Generalized public API. Emits using either JSON or table, according to + * settings in the emitter_t. */ + +/* + * Note emits a different kv pair as well, but only in table mode. Omits the + * note if table_note_key is NULL. + */ +static inline void +emitter_kv_note(emitter_t *emitter, const char *json_key, const char *table_key, + emitter_type_t value_type, const void *value, + const char *table_note_key, emitter_type_t table_note_value_type, + const void *table_note_value) { + if (emitter->output == emitter_output_json) { + emitter_json_key(emitter, json_key); + emitter_json_value(emitter, value_type, value); + } else { + emitter_table_kv_note(emitter, table_key, value_type, value, + table_note_key, table_note_value_type, table_note_value); + } + emitter->item_at_depth = true; +} + +static inline void +emitter_kv(emitter_t *emitter, const char *json_key, const char *table_key, + emitter_type_t value_type, const void *value) { + emitter_kv_note(emitter, json_key, table_key, value_type, value, NULL, + emitter_type_bool, NULL); +} + +static inline void +emitter_dict_begin(emitter_t *emitter, const char *json_key, + const char *table_header) { + if (emitter->output == emitter_output_json) { + emitter_json_key(emitter, json_key); + emitter_json_object_begin(emitter); + } else { + emitter_table_dict_begin(emitter, table_header); + } +} + +static inline void +emitter_dict_end(emitter_t *emitter) { + if (emitter->output == emitter_output_json) { + emitter_json_object_end(emitter); + } else { + emitter_table_dict_end(emitter); + } +} + +static inline void +emitter_begin(emitter_t *emitter) { + if (emitter->output == emitter_output_json) { + assert(emitter->nesting_depth == 0); + emitter_printf(emitter, "{"); + emitter_nest_inc(emitter); + } else { + /* + * This guarantees that we always call write_cb at least once. + * This is useful if some invariant is established by each call + * to write_cb, but doesn't hold initially: e.g., some buffer + * holds a null-terminated string. + */ + emitter_printf(emitter, "%s", ""); + } +} + +static inline void +emitter_end(emitter_t *emitter) { + if (emitter->output == emitter_output_json) { + assert(emitter->nesting_depth == 1); + emitter_nest_dec(emitter); + emitter_printf(emitter, "\n}\n"); + } +} + +#endif /* JEMALLOC_INTERNAL_EMITTER_H */ diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/extent_externs.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/extent_externs.h index 489a813c80..8680251aba 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/extent_externs.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/extent_externs.h @@ -4,12 +4,13 @@ #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/mutex_pool.h" #include "jemalloc/internal/ph.h" -#include "jemalloc/internal/rb.h" #include "jemalloc/internal/rtree.h" -extern rtree_t extents_rtree; -extern const extent_hooks_t extent_hooks_default; -extern mutex_pool_t extent_mutex_pool; +extern size_t opt_lg_extent_max_active_fit; + +extern rtree_t extents_rtree; +extern const extent_hooks_t extent_hooks_default; +extern mutex_pool_t extent_mutex_pool; extent_t *extent_alloc(tsdn_t *tsdn, arena_t *arena); void extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent); @@ -30,6 +31,10 @@ bool extents_init(tsdn_t *tsdn, extents_t *extents, extent_state_t state, bool delay_coalesce); extent_state_t extents_state_get(const extents_t *extents); size_t extents_npages_get(extents_t *extents); +/* Get the number of extents in the given page size index. */ +size_t extents_nextents_get(extents_t *extents, pszind_t ind); +/* Get the sum total bytes of the extents in the given page size index. */ +size_t extents_nbytes_get(extents_t *extents, pszind_t ind); extent_t *extents_alloc(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extents_t *extents, void *new_addr, size_t size, size_t pad, size_t alignment, bool slab, szind_t szind, diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/extent_inlines.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/extent_inlines.h index bb2bd699ed..63b710dcf0 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/extent_inlines.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/extent_inlines.h @@ -6,6 +6,7 @@ #include "jemalloc/internal/pages.h" #include "jemalloc/internal/prng.h" #include "jemalloc/internal/ql.h" +#include "jemalloc/internal/sc.h" #include "jemalloc/internal/sz.h" static inline void @@ -34,18 +35,19 @@ extent_unlock2(tsdn_t *tsdn, extent_t *extent1, extent_t *extent2) { (uintptr_t)extent2); } -static inline arena_t * -extent_arena_get(const extent_t *extent) { +static inline unsigned +extent_arena_ind_get(const extent_t *extent) { unsigned arena_ind = (unsigned)((extent->e_bits & EXTENT_BITS_ARENA_MASK) >> EXTENT_BITS_ARENA_SHIFT); - /* - * The following check is omitted because we should never actually read - * a NULL arena pointer. - */ - if (false && arena_ind >= MALLOCX_ARENA_LIMIT) { - return NULL; - } assert(arena_ind < MALLOCX_ARENA_LIMIT); + + return arena_ind; +} + +static inline arena_t * +extent_arena_get(const extent_t *extent) { + unsigned arena_ind = extent_arena_ind_get(extent); + return (arena_t *)atomic_load_p(&arenas[arena_ind], ATOMIC_ACQUIRE); } @@ -53,14 +55,14 @@ static inline szind_t extent_szind_get_maybe_invalid(const extent_t *extent) { szind_t szind = (szind_t)((extent->e_bits & EXTENT_BITS_SZIND_MASK) >> EXTENT_BITS_SZIND_SHIFT); - assert(szind <= NSIZES); + assert(szind <= SC_NSIZES); return szind; } static inline szind_t extent_szind_get(const extent_t *extent) { szind_t szind = extent_szind_get_maybe_invalid(extent); - assert(szind < NSIZES); /* Never call when "invalid". */ + assert(szind < SC_NSIZES); /* Never call when "invalid". */ return szind; } @@ -69,6 +71,14 @@ extent_usize_get(const extent_t *extent) { return sz_index2size(extent_szind_get(extent)); } +static inline unsigned +extent_binshard_get(const extent_t *extent) { + unsigned binshard = (unsigned)((extent->e_bits & + EXTENT_BITS_BINSHARD_MASK) >> EXTENT_BITS_BINSHARD_SHIFT); + assert(binshard < bin_infos[extent_szind_get(extent)].n_shards); + return binshard; +} + static inline size_t extent_sn_get(const extent_t *extent) { return (size_t)((extent->e_bits & EXTENT_BITS_SN_MASK) >> @@ -93,6 +103,12 @@ extent_committed_get(const extent_t *extent) { EXTENT_BITS_COMMITTED_SHIFT); } +static inline bool +extent_dumpable_get(const extent_t *extent) { + return (bool)((extent->e_bits & EXTENT_BITS_DUMPABLE_MASK) >> + EXTENT_BITS_DUMPABLE_SHIFT); +} + static inline bool extent_slab_get(const extent_t *extent) { return (bool)((extent->e_bits & EXTENT_BITS_SLAB_MASK) >> @@ -170,6 +186,11 @@ extent_prof_tctx_get(const extent_t *extent) { ATOMIC_ACQUIRE); } +static inline nstime_t +extent_prof_alloc_time_get(const extent_t *extent) { + return extent->e_alloc_time; +} + static inline void extent_arena_set(extent_t *extent, arena_t *arena) { unsigned arena_ind = (arena != NULL) ? arena_ind_get(arena) : ((1U << @@ -178,6 +199,14 @@ extent_arena_set(extent_t *extent, arena_t *arena) { ((uint64_t)arena_ind << EXTENT_BITS_ARENA_SHIFT); } +static inline void +extent_binshard_set(extent_t *extent, unsigned binshard) { + /* The assertion assumes szind is set already. */ + assert(binshard < bin_infos[extent_szind_get(extent)].n_shards); + extent->e_bits = (extent->e_bits & ~EXTENT_BITS_BINSHARD_MASK) | + ((uint64_t)binshard << EXTENT_BITS_BINSHARD_SHIFT); +} + static inline void extent_addr_set(extent_t *extent, void *addr) { extent->e_addr = addr; @@ -190,9 +219,16 @@ extent_addr_randomize(tsdn_t *tsdn, extent_t *extent, size_t alignment) { if (alignment < PAGE) { unsigned lg_range = LG_PAGE - lg_floor(CACHELINE_CEILING(alignment)); - size_t r = - prng_lg_range_zu(&extent_arena_get(extent)->offset_state, - lg_range, true); + size_t r; + if (!tsdn_null(tsdn)) { + tsd_t *tsd = tsdn_tsd(tsdn); + r = (size_t)prng_lg_range_u64( + tsd_offset_statep_get(tsd), lg_range); + } else { + r = prng_lg_range_zu( + &extent_arena_get(extent)->offset_state, + lg_range, true); + } uintptr_t random_offset = ((uintptr_t)r) << (LG_PAGE - lg_range); extent->e_addr = (void *)((uintptr_t)extent->e_addr + @@ -221,7 +257,7 @@ extent_bsize_set(extent_t *extent, size_t bsize) { static inline void extent_szind_set(extent_t *extent, szind_t szind) { - assert(szind <= NSIZES); /* NSIZES means "invalid". */ + assert(szind <= SC_NSIZES); /* SC_NSIZES means "invalid". */ extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SZIND_MASK) | ((uint64_t)szind << EXTENT_BITS_SZIND_SHIFT); } @@ -233,6 +269,16 @@ extent_nfree_set(extent_t *extent, unsigned nfree) { ((uint64_t)nfree << EXTENT_BITS_NFREE_SHIFT); } +static inline void +extent_nfree_binshard_set(extent_t *extent, unsigned nfree, unsigned binshard) { + /* The assertion assumes szind is set already. */ + assert(binshard < bin_infos[extent_szind_get(extent)].n_shards); + extent->e_bits = (extent->e_bits & + (~EXTENT_BITS_NFREE_MASK & ~EXTENT_BITS_BINSHARD_MASK)) | + ((uint64_t)binshard << EXTENT_BITS_BINSHARD_SHIFT) | + ((uint64_t)nfree << EXTENT_BITS_NFREE_SHIFT); +} + static inline void extent_nfree_inc(extent_t *extent) { assert(extent_slab_get(extent)); @@ -245,6 +291,12 @@ extent_nfree_dec(extent_t *extent) { extent->e_bits -= ((uint64_t)1U << EXTENT_BITS_NFREE_SHIFT); } +static inline void +extent_nfree_sub(extent_t *extent, uint64_t n) { + assert(extent_slab_get(extent)); + extent->e_bits -= (n << EXTENT_BITS_NFREE_SHIFT); +} + static inline void extent_sn_set(extent_t *extent, size_t sn) { extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SN_MASK) | @@ -269,6 +321,12 @@ extent_committed_set(extent_t *extent, bool committed) { ((uint64_t)committed << EXTENT_BITS_COMMITTED_SHIFT); } +static inline void +extent_dumpable_set(extent_t *extent, bool dumpable) { + extent->e_bits = (extent->e_bits & ~EXTENT_BITS_DUMPABLE_MASK) | + ((uint64_t)dumpable << EXTENT_BITS_DUMPABLE_SHIFT); +} + static inline void extent_slab_set(extent_t *extent, bool slab) { extent->e_bits = (extent->e_bits & ~EXTENT_BITS_SLAB_MASK) | @@ -280,10 +338,15 @@ extent_prof_tctx_set(extent_t *extent, prof_tctx_t *tctx) { atomic_store_p(&extent->e_prof_tctx, tctx, ATOMIC_RELEASE); } +static inline void +extent_prof_alloc_time_set(extent_t *extent, nstime_t t) { + nstime_copy(&extent->e_alloc_time, &t); +} + static inline void extent_init(extent_t *extent, arena_t *arena, void *addr, size_t size, bool slab, szind_t szind, size_t sn, extent_state_t state, bool zeroed, - bool committed) { + bool committed, bool dumpable) { assert(addr == PAGE_ADDR2BASE(addr) || !slab); extent_arena_set(extent, arena); @@ -295,6 +358,7 @@ extent_init(extent_t *extent, arena_t *arena, void *addr, size_t size, extent_state_set(extent, state); extent_zeroed_set(extent, zeroed); extent_committed_set(extent, committed); + extent_dumpable_set(extent, dumpable); ql_elm_new(extent, ql_link); if (config_prof) { extent_prof_tctx_set(extent, NULL); @@ -307,11 +371,12 @@ extent_binit(extent_t *extent, void *addr, size_t bsize, size_t sn) { extent_addr_set(extent, addr); extent_bsize_set(extent, bsize); extent_slab_set(extent, false); - extent_szind_set(extent, NSIZES); + extent_szind_set(extent, SC_NSIZES); extent_sn_set(extent, sn); extent_state_set(extent, extent_state_active); extent_zeroed_set(extent, true); extent_committed_set(extent, true); + extent_dumpable_set(extent, true); } static inline void @@ -334,6 +399,11 @@ extent_list_append(extent_list_t *list, extent_t *extent) { ql_tail_insert(list, extent, ql_link); } +static inline void +extent_list_prepend(extent_list_t *list, extent_t *extent) { + ql_head_insert(list, extent, ql_link); +} + static inline void extent_list_replace(extent_list_t *list, extent_t *to_remove, extent_t *to_insert) { diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/extent_structs.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/extent_structs.h index d297950345..ceb18979f1 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/extent_structs.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/extent_structs.h @@ -2,12 +2,12 @@ #define JEMALLOC_INTERNAL_EXTENT_STRUCTS_H #include "jemalloc/internal/atomic.h" +#include "jemalloc/internal/bit_util.h" #include "jemalloc/internal/bitmap.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/ql.h" -#include "jemalloc/internal/rb.h" #include "jemalloc/internal/ph.h" -#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/sc.h" typedef enum { extent_state_active = 0, @@ -24,13 +24,15 @@ struct extent_s { * a: arena_ind * b: slab * c: committed + * d: dumpable * z: zeroed * t: state * i: szind * f: nfree + * s: bin_shard * n: sn * - * nnnnnnnn ... nnnnnfff fffffffi iiiiiiit tzcbaaaa aaaaaaaa + * nnnnnnnn ... nnnnnnss ssssffff ffffffii iiiiiitt zdcbaaaa aaaaaaaa * * arena_ind: Arena from which this extent came, or all 1 bits if * unassociated. @@ -45,6 +47,23 @@ struct extent_s { * as on a system that overcommits and satisfies physical * memory needs on demand via soft page faults. * + * dumpable: The dumpable flag indicates whether or not we've set the + * memory in question to be dumpable. Note that this + * interacts somewhat subtly with user-specified extent hooks, + * since we don't know if *they* are fiddling with + * dumpability (in which case, we don't want to undo whatever + * they're doing). To deal with this scenario, we: + * - Make dumpable false only for memory allocated with the + * default hooks. + * - Only allow memory to go from non-dumpable to dumpable, + * and only once. + * - Never make the OS call to allow dumping when the + * dumpable bit is already set. + * These three constraints mean that we will never + * accidentally dump user memory that the user meant to set + * nondumpable with their extent hooks. + * + * * zeroed: The zeroed flag is used by extent recycling code to track * whether memory is zero-filled. * @@ -58,6 +77,8 @@ struct extent_s { * * nfree: Number of free regions in slab. * + * bin_shard: the shard of the bin from which this extent came. + * * sn: Serial number (potentially non-unique). * * Serial numbers may wrap around if !opt_retain, but as long as @@ -69,38 +90,46 @@ struct extent_s { * serial number to both resulting adjacent extents. */ uint64_t e_bits; -#define EXTENT_BITS_ARENA_SHIFT 0 -#define EXTENT_BITS_ARENA_MASK \ - (((uint64_t)(1U << MALLOCX_ARENA_BITS) - 1) << EXTENT_BITS_ARENA_SHIFT) +#define MASK(CURRENT_FIELD_WIDTH, CURRENT_FIELD_SHIFT) ((((((uint64_t)0x1U) << (CURRENT_FIELD_WIDTH)) - 1)) << (CURRENT_FIELD_SHIFT)) + +#define EXTENT_BITS_ARENA_WIDTH MALLOCX_ARENA_BITS +#define EXTENT_BITS_ARENA_SHIFT 0 +#define EXTENT_BITS_ARENA_MASK MASK(EXTENT_BITS_ARENA_WIDTH, EXTENT_BITS_ARENA_SHIFT) -#define EXTENT_BITS_SLAB_SHIFT MALLOCX_ARENA_BITS -#define EXTENT_BITS_SLAB_MASK \ - ((uint64_t)0x1U << EXTENT_BITS_SLAB_SHIFT) +#define EXTENT_BITS_SLAB_WIDTH 1 +#define EXTENT_BITS_SLAB_SHIFT (EXTENT_BITS_ARENA_WIDTH + EXTENT_BITS_ARENA_SHIFT) +#define EXTENT_BITS_SLAB_MASK MASK(EXTENT_BITS_SLAB_WIDTH, EXTENT_BITS_SLAB_SHIFT) -#define EXTENT_BITS_COMMITTED_SHIFT (MALLOCX_ARENA_BITS + 1) -#define EXTENT_BITS_COMMITTED_MASK \ - ((uint64_t)0x1U << EXTENT_BITS_COMMITTED_SHIFT) +#define EXTENT_BITS_COMMITTED_WIDTH 1 +#define EXTENT_BITS_COMMITTED_SHIFT (EXTENT_BITS_SLAB_WIDTH + EXTENT_BITS_SLAB_SHIFT) +#define EXTENT_BITS_COMMITTED_MASK MASK(EXTENT_BITS_COMMITTED_WIDTH, EXTENT_BITS_COMMITTED_SHIFT) -#define EXTENT_BITS_ZEROED_SHIFT (MALLOCX_ARENA_BITS + 2) -#define EXTENT_BITS_ZEROED_MASK \ - ((uint64_t)0x1U << EXTENT_BITS_ZEROED_SHIFT) +#define EXTENT_BITS_DUMPABLE_WIDTH 1 +#define EXTENT_BITS_DUMPABLE_SHIFT (EXTENT_BITS_COMMITTED_WIDTH + EXTENT_BITS_COMMITTED_SHIFT) +#define EXTENT_BITS_DUMPABLE_MASK MASK(EXTENT_BITS_DUMPABLE_WIDTH, EXTENT_BITS_DUMPABLE_SHIFT) -#define EXTENT_BITS_STATE_SHIFT (MALLOCX_ARENA_BITS + 3) -#define EXTENT_BITS_STATE_MASK \ - ((uint64_t)0x3U << EXTENT_BITS_STATE_SHIFT) +#define EXTENT_BITS_ZEROED_WIDTH 1 +#define EXTENT_BITS_ZEROED_SHIFT (EXTENT_BITS_DUMPABLE_WIDTH + EXTENT_BITS_DUMPABLE_SHIFT) +#define EXTENT_BITS_ZEROED_MASK MASK(EXTENT_BITS_ZEROED_WIDTH, EXTENT_BITS_ZEROED_SHIFT) -#define EXTENT_BITS_SZIND_SHIFT (MALLOCX_ARENA_BITS + 5) -#define EXTENT_BITS_SZIND_MASK \ - (((uint64_t)(1U << LG_CEIL_NSIZES) - 1) << EXTENT_BITS_SZIND_SHIFT) +#define EXTENT_BITS_STATE_WIDTH 2 +#define EXTENT_BITS_STATE_SHIFT (EXTENT_BITS_ZEROED_WIDTH + EXTENT_BITS_ZEROED_SHIFT) +#define EXTENT_BITS_STATE_MASK MASK(EXTENT_BITS_STATE_WIDTH, EXTENT_BITS_STATE_SHIFT) -#define EXTENT_BITS_NFREE_SHIFT \ - (MALLOCX_ARENA_BITS + 5 + LG_CEIL_NSIZES) -#define EXTENT_BITS_NFREE_MASK \ - ((uint64_t)((1U << (LG_SLAB_MAXREGS + 1)) - 1) << EXTENT_BITS_NFREE_SHIFT) +#define EXTENT_BITS_SZIND_WIDTH LG_CEIL(SC_NSIZES) +#define EXTENT_BITS_SZIND_SHIFT (EXTENT_BITS_STATE_WIDTH + EXTENT_BITS_STATE_SHIFT) +#define EXTENT_BITS_SZIND_MASK MASK(EXTENT_BITS_SZIND_WIDTH, EXTENT_BITS_SZIND_SHIFT) -#define EXTENT_BITS_SN_SHIFT \ - (MALLOCX_ARENA_BITS + 5 + LG_CEIL_NSIZES + (LG_SLAB_MAXREGS + 1)) -#define EXTENT_BITS_SN_MASK (UINT64_MAX << EXTENT_BITS_SN_SHIFT) +#define EXTENT_BITS_NFREE_WIDTH (LG_SLAB_MAXREGS + 1) +#define EXTENT_BITS_NFREE_SHIFT (EXTENT_BITS_SZIND_WIDTH + EXTENT_BITS_SZIND_SHIFT) +#define EXTENT_BITS_NFREE_MASK MASK(EXTENT_BITS_NFREE_WIDTH, EXTENT_BITS_NFREE_SHIFT) + +#define EXTENT_BITS_BINSHARD_WIDTH 6 +#define EXTENT_BITS_BINSHARD_SHIFT (EXTENT_BITS_NFREE_WIDTH + EXTENT_BITS_NFREE_SHIFT) +#define EXTENT_BITS_BINSHARD_MASK MASK(EXTENT_BITS_BINSHARD_WIDTH, EXTENT_BITS_BINSHARD_SHIFT) + +#define EXTENT_BITS_SN_SHIFT (EXTENT_BITS_BINSHARD_WIDTH + EXTENT_BITS_BINSHARD_SHIFT) +#define EXTENT_BITS_SN_MASK (UINT64_MAX << EXTENT_BITS_SN_SHIFT) /* Pointer to the extent that this structure is responsible for. */ void *e_addr; @@ -120,35 +149,36 @@ struct extent_s { size_t e_bsize; }; - union { - /* - * List linkage, used by a variety of lists: - * - arena_bin_t's slabs_full - * - extents_t's LRU - * - stashed dirty extents - * - arena's large allocations - */ - ql_elm(extent_t) ql_link; - /* Red-black tree linkage, used by arena's extent_avail. */ - rb_node(extent_t) rb_link; - }; + /* + * List linkage, used by a variety of lists: + * - bin_t's slabs_full + * - extents_t's LRU + * - stashed dirty extents + * - arena's large allocations + */ + ql_elm(extent_t) ql_link; - /* Linkage for per size class sn/address-ordered heaps. */ + /* + * Linkage for per size class sn/address-ordered heaps, and + * for extent_avail + */ phn(extent_t) ph_link; union { /* Small region slab metadata. */ arena_slab_data_t e_slab_data; - /* - * Profile counters, used for large objects. Points to a - * prof_tctx_t. - */ - atomic_p_t e_prof_tctx; + /* Profiling data, used for large objects. */ + struct { + /* Time when this was allocated. */ + nstime_t e_alloc_time; + /* Points to a prof_tctx_t. */ + atomic_p_t e_prof_tctx; + }; }; }; typedef ql_head(extent_t) extent_list_t; -typedef rb_tree(extent_t) extent_tree_t; +typedef ph(extent_t) extent_tree_t; typedef ph(extent_t) extent_heap_t; /* Quantized collection of extents, with built-in LRU queue. */ @@ -160,14 +190,16 @@ struct extents_s { * * Synchronization: mtx. */ - extent_heap_t heaps[NPSIZES+1]; + extent_heap_t heaps[SC_NPSIZES + 1]; + atomic_zu_t nextents[SC_NPSIZES + 1]; + atomic_zu_t nbytes[SC_NPSIZES + 1]; /* * Bitmap for which set bits correspond to non-empty heaps. * * Synchronization: mtx. */ - bitmap_t bitmap[BITMAP_GROUPS(NPSIZES+1)]; + bitmap_t bitmap[BITMAP_GROUPS(SC_NPSIZES + 1)]; /* * LRU of all extents in heaps. diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/extent_types.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/extent_types.h index b6905ce105..acbcf27b51 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/extent_types.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/extent_types.h @@ -6,4 +6,10 @@ typedef struct extents_s extents_t; #define EXTENT_HOOKS_INITIALIZER NULL +/* + * When reuse (and split) an active extent, (1U << opt_lg_extent_max_active_fit) + * is the max ratio between the size of the active extent and the new extent. + */ +#define LG_EXTENT_MAX_ACTIVE_FIT_DEFAULT 6 + #endif /* JEMALLOC_INTERNAL_EXTENT_TYPES_H */ diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/hash.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/hash.h index 188296cf0e..0270034e87 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/hash.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/hash.h @@ -104,8 +104,8 @@ hash_x86_32(const void *key, int len, uint32_t seed) { uint32_t k1 = 0; switch (len & 3) { - case 3: k1 ^= tail[2] << 16; - case 2: k1 ^= tail[1] << 8; + case 3: k1 ^= tail[2] << 16; JEMALLOC_FALLTHROUGH + case 2: k1 ^= tail[1] << 8; JEMALLOC_FALLTHROUGH case 1: k1 ^= tail[0]; k1 *= c1; k1 = hash_rotl_32(k1, 15); k1 *= c2; h1 ^= k1; } @@ -119,7 +119,7 @@ hash_x86_32(const void *key, int len, uint32_t seed) { return h1; } -UNUSED static inline void +static inline void hash_x86_128(const void *key, const int len, uint32_t seed, uint64_t r_out[2]) { const uint8_t * data = (const uint8_t *) key; @@ -177,28 +177,29 @@ hash_x86_128(const void *key, const int len, uint32_t seed, uint32_t k4 = 0; switch (len & 15) { - case 15: k4 ^= tail[14] << 16; - case 14: k4 ^= tail[13] << 8; + case 15: k4 ^= tail[14] << 16; JEMALLOC_FALLTHROUGH + case 14: k4 ^= tail[13] << 8; JEMALLOC_FALLTHROUGH case 13: k4 ^= tail[12] << 0; k4 *= c4; k4 = hash_rotl_32(k4, 18); k4 *= c1; h4 ^= k4; - - case 12: k3 ^= tail[11] << 24; - case 11: k3 ^= tail[10] << 16; - case 10: k3 ^= tail[ 9] << 8; + JEMALLOC_FALLTHROUGH + case 12: k3 ^= tail[11] << 24; JEMALLOC_FALLTHROUGH + case 11: k3 ^= tail[10] << 16; JEMALLOC_FALLTHROUGH + case 10: k3 ^= tail[ 9] << 8; JEMALLOC_FALLTHROUGH case 9: k3 ^= tail[ 8] << 0; k3 *= c3; k3 = hash_rotl_32(k3, 17); k3 *= c4; h3 ^= k3; - - case 8: k2 ^= tail[ 7] << 24; - case 7: k2 ^= tail[ 6] << 16; - case 6: k2 ^= tail[ 5] << 8; + JEMALLOC_FALLTHROUGH + case 8: k2 ^= tail[ 7] << 24; JEMALLOC_FALLTHROUGH + case 7: k2 ^= tail[ 6] << 16; JEMALLOC_FALLTHROUGH + case 6: k2 ^= tail[ 5] << 8; JEMALLOC_FALLTHROUGH case 5: k2 ^= tail[ 4] << 0; k2 *= c2; k2 = hash_rotl_32(k2, 16); k2 *= c3; h2 ^= k2; - - case 4: k1 ^= tail[ 3] << 24; - case 3: k1 ^= tail[ 2] << 16; - case 2: k1 ^= tail[ 1] << 8; + JEMALLOC_FALLTHROUGH + case 4: k1 ^= tail[ 3] << 24; JEMALLOC_FALLTHROUGH + case 3: k1 ^= tail[ 2] << 16; JEMALLOC_FALLTHROUGH + case 2: k1 ^= tail[ 1] << 8; JEMALLOC_FALLTHROUGH case 1: k1 ^= tail[ 0] << 0; k1 *= c1; k1 = hash_rotl_32(k1, 15); k1 *= c2; h1 ^= k1; + JEMALLOC_FALLTHROUGH } } @@ -220,7 +221,7 @@ hash_x86_128(const void *key, const int len, uint32_t seed, r_out[1] = (((uint64_t) h4) << 32) | h3; } -UNUSED static inline void +static inline void hash_x64_128(const void *key, const int len, const uint32_t seed, uint64_t r_out[2]) { const uint8_t *data = (const uint8_t *) key; @@ -260,22 +261,22 @@ hash_x64_128(const void *key, const int len, const uint32_t seed, uint64_t k2 = 0; switch (len & 15) { - case 15: k2 ^= ((uint64_t)(tail[14])) << 48; - case 14: k2 ^= ((uint64_t)(tail[13])) << 40; - case 13: k2 ^= ((uint64_t)(tail[12])) << 32; - case 12: k2 ^= ((uint64_t)(tail[11])) << 24; - case 11: k2 ^= ((uint64_t)(tail[10])) << 16; - case 10: k2 ^= ((uint64_t)(tail[ 9])) << 8; + case 15: k2 ^= ((uint64_t)(tail[14])) << 48; JEMALLOC_FALLTHROUGH + case 14: k2 ^= ((uint64_t)(tail[13])) << 40; JEMALLOC_FALLTHROUGH + case 13: k2 ^= ((uint64_t)(tail[12])) << 32; JEMALLOC_FALLTHROUGH + case 12: k2 ^= ((uint64_t)(tail[11])) << 24; JEMALLOC_FALLTHROUGH + case 11: k2 ^= ((uint64_t)(tail[10])) << 16; JEMALLOC_FALLTHROUGH + case 10: k2 ^= ((uint64_t)(tail[ 9])) << 8; JEMALLOC_FALLTHROUGH case 9: k2 ^= ((uint64_t)(tail[ 8])) << 0; k2 *= c2; k2 = hash_rotl_64(k2, 33); k2 *= c1; h2 ^= k2; - - case 8: k1 ^= ((uint64_t)(tail[ 7])) << 56; - case 7: k1 ^= ((uint64_t)(tail[ 6])) << 48; - case 6: k1 ^= ((uint64_t)(tail[ 5])) << 40; - case 5: k1 ^= ((uint64_t)(tail[ 4])) << 32; - case 4: k1 ^= ((uint64_t)(tail[ 3])) << 24; - case 3: k1 ^= ((uint64_t)(tail[ 2])) << 16; - case 2: k1 ^= ((uint64_t)(tail[ 1])) << 8; + JEMALLOC_FALLTHROUGH + case 8: k1 ^= ((uint64_t)(tail[ 7])) << 56; JEMALLOC_FALLTHROUGH + case 7: k1 ^= ((uint64_t)(tail[ 6])) << 48; JEMALLOC_FALLTHROUGH + case 6: k1 ^= ((uint64_t)(tail[ 5])) << 40; JEMALLOC_FALLTHROUGH + case 5: k1 ^= ((uint64_t)(tail[ 4])) << 32; JEMALLOC_FALLTHROUGH + case 4: k1 ^= ((uint64_t)(tail[ 3])) << 24; JEMALLOC_FALLTHROUGH + case 3: k1 ^= ((uint64_t)(tail[ 2])) << 16; JEMALLOC_FALLTHROUGH + case 2: k1 ^= ((uint64_t)(tail[ 1])) << 8; JEMALLOC_FALLTHROUGH case 1: k1 ^= ((uint64_t)(tail[ 0])) << 0; k1 *= c1; k1 = hash_rotl_64(k1, 31); k1 *= c2; h1 ^= k1; } diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/hook.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/hook.h new file mode 100644 index 0000000000..ee246b1e0b --- /dev/null +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/hook.h @@ -0,0 +1,163 @@ +#ifndef JEMALLOC_INTERNAL_HOOK_H +#define JEMALLOC_INTERNAL_HOOK_H + +#include "jemalloc/internal/tsd.h" + +/* + * This API is *extremely* experimental, and may get ripped out, changed in API- + * and ABI-incompatible ways, be insufficiently or incorrectly documented, etc. + * + * It allows hooking the stateful parts of the API to see changes as they + * happen. + * + * Allocation hooks are called after the allocation is done, free hooks are + * called before the free is done, and expand hooks are called after the + * allocation is expanded. + * + * For realloc and rallocx, if the expansion happens in place, the expansion + * hook is called. If it is moved, then the alloc hook is called on the new + * location, and then the free hook is called on the old location (i.e. both + * hooks are invoked in between the alloc and the dalloc). + * + * If we return NULL from OOM, then usize might not be trustworthy. Calling + * realloc(NULL, size) only calls the alloc hook, and calling realloc(ptr, 0) + * only calls the free hook. (Calling realloc(NULL, 0) is treated as malloc(0), + * and only calls the alloc hook). + * + * Reentrancy: + * Reentrancy is guarded against from within the hook implementation. If you + * call allocator functions from within a hook, the hooks will not be invoked + * again. + * Threading: + * The installation of a hook synchronizes with all its uses. If you can + * prove the installation of a hook happens-before a jemalloc entry point, + * then the hook will get invoked (unless there's a racing removal). + * + * Hook insertion appears to be atomic at a per-thread level (i.e. if a thread + * allocates and has the alloc hook invoked, then a subsequent free on the + * same thread will also have the free hook invoked). + * + * The *removal* of a hook does *not* block until all threads are done with + * the hook. Hook authors have to be resilient to this, and need some + * out-of-band mechanism for cleaning up any dynamically allocated memory + * associated with their hook. + * Ordering: + * Order of hook execution is unspecified, and may be different than insertion + * order. + */ + +#define HOOK_MAX 4 + +enum hook_alloc_e { + hook_alloc_malloc, + hook_alloc_posix_memalign, + hook_alloc_aligned_alloc, + hook_alloc_calloc, + hook_alloc_memalign, + hook_alloc_valloc, + hook_alloc_mallocx, + + /* The reallocating functions have both alloc and dalloc variants */ + hook_alloc_realloc, + hook_alloc_rallocx, +}; +/* + * We put the enum typedef after the enum, since this file may get included by + * jemalloc_cpp.cpp, and C++ disallows enum forward declarations. + */ +typedef enum hook_alloc_e hook_alloc_t; + +enum hook_dalloc_e { + hook_dalloc_free, + hook_dalloc_dallocx, + hook_dalloc_sdallocx, + + /* + * The dalloc halves of reallocation (not called if in-place expansion + * happens). + */ + hook_dalloc_realloc, + hook_dalloc_rallocx, +}; +typedef enum hook_dalloc_e hook_dalloc_t; + + +enum hook_expand_e { + hook_expand_realloc, + hook_expand_rallocx, + hook_expand_xallocx, +}; +typedef enum hook_expand_e hook_expand_t; + +typedef void (*hook_alloc)( + void *extra, hook_alloc_t type, void *result, uintptr_t result_raw, + uintptr_t args_raw[3]); + +typedef void (*hook_dalloc)( + void *extra, hook_dalloc_t type, void *address, uintptr_t args_raw[3]); + +typedef void (*hook_expand)( + void *extra, hook_expand_t type, void *address, size_t old_usize, + size_t new_usize, uintptr_t result_raw, uintptr_t args_raw[4]); + +typedef struct hooks_s hooks_t; +struct hooks_s { + hook_alloc alloc_hook; + hook_dalloc dalloc_hook; + hook_expand expand_hook; + void *extra; +}; + +/* + * Begin implementation details; everything above this point might one day live + * in a public API. Everything below this point never will. + */ + +/* + * The realloc pathways haven't gotten any refactoring love in a while, and it's + * fairly difficult to pass information from the entry point to the hooks. We + * put the informaiton the hooks will need into a struct to encapsulate + * everything. + * + * Much of these pathways are force-inlined, so that the compiler can avoid + * materializing this struct until we hit an extern arena function. For fairly + * goofy reasons, *many* of the realloc paths hit an extern arena function. + * These paths are cold enough that it doesn't matter; eventually, we should + * rewrite the realloc code to make the expand-in-place and the + * free-then-realloc paths more orthogonal, at which point we don't need to + * spread the hook logic all over the place. + */ +typedef struct hook_ralloc_args_s hook_ralloc_args_t; +struct hook_ralloc_args_s { + /* I.e. as opposed to rallocx. */ + bool is_realloc; + /* + * The expand hook takes 4 arguments, even if only 3 are actually used; + * we add an extra one in case the user decides to memcpy without + * looking too closely at the hooked function. + */ + uintptr_t args[4]; +}; + +/* + * Returns an opaque handle to be used when removing the hook. NULL means that + * we couldn't install the hook. + */ +bool hook_boot(); + +void *hook_install(tsdn_t *tsdn, hooks_t *hooks); +/* Uninstalls the hook with the handle previously returned from hook_install. */ +void hook_remove(tsdn_t *tsdn, void *opaque); + +/* Hooks */ + +void hook_invoke_alloc(hook_alloc_t type, void *result, uintptr_t result_raw, + uintptr_t args_raw[3]); + +void hook_invoke_dalloc(hook_dalloc_t type, void *address, + uintptr_t args_raw[3]); + +void hook_invoke_expand(hook_expand_t type, void *address, size_t old_usize, + size_t new_usize, uintptr_t result_raw, uintptr_t args_raw[4]); + +#endif /* JEMALLOC_INTERNAL_HOOK_H */ diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/hooks.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/hooks.h deleted file mode 100755 index cd49afcb09..0000000000 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/hooks.h +++ /dev/null @@ -1,19 +0,0 @@ -#ifndef JEMALLOC_INTERNAL_HOOKS_H -#define JEMALLOC_INTERNAL_HOOKS_H - -extern JEMALLOC_EXPORT void (*hooks_arena_new_hook)(); -extern JEMALLOC_EXPORT void (*hooks_libc_hook)(); - -#define JEMALLOC_HOOK(fn, hook) ((void)(hook != NULL && (hook(), 0)), fn) - -#define open JEMALLOC_HOOK(open, hooks_libc_hook) -#define read JEMALLOC_HOOK(read, hooks_libc_hook) -#define write JEMALLOC_HOOK(write, hooks_libc_hook) -#define readlink JEMALLOC_HOOK(readlink, hooks_libc_hook) -#define close JEMALLOC_HOOK(close, hooks_libc_hook) -#define creat JEMALLOC_HOOK(creat, hooks_libc_hook) -#define secure_getenv JEMALLOC_HOOK(secure_getenv, hooks_libc_hook) -/* Note that this is undef'd and re-define'd in src/prof.c. */ -#define _Unwind_Backtrace JEMALLOC_HOOK(_Unwind_Backtrace, hooks_libc_hook) - -#endif /* JEMALLOC_INTERNAL_HOOKS_H */ diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/jemalloc_internal_decls.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/jemalloc_internal_decls.h index 8ae5ef48cd..7d6053e219 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/jemalloc_internal_decls.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/jemalloc_internal_decls.h @@ -5,7 +5,16 @@ #ifdef _WIN32 # include # include "msvc_compat/windows_extra.h" - +# ifdef _WIN64 +# if LG_VADDR <= 32 +# error Generate the headers using x64 vcargs +# endif +# else +# if LG_VADDR > 32 +# undef LG_VADDR +# define LG_VADDR 32 +# endif +# endif #else # include # include @@ -22,6 +31,9 @@ # include # endif # include +# ifdef __FreeBSD__ +# include +# endif # include # ifdef JEMALLOC_OS_UNFAIR_LOCK # include diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/jemalloc_internal_defs.h.in b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/jemalloc_internal_defs.h.in index c0f834f2a4..21b6514784 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/jemalloc_internal_defs.h.in @@ -33,6 +33,8 @@ * order to yield to another virtual CPU. */ #undef CPU_SPINWAIT +/* 1 if CPU_SPINWAIT is defined, 0 otherwise. */ +#undef HAVE_CPU_SPINWAIT /* * Number of significant bits in virtual addresses. This may be less than the @@ -46,25 +48,13 @@ /* Defined if GCC __atomic atomics are available. */ #undef JEMALLOC_GCC_ATOMIC_ATOMICS +/* and the 8-bit variant support. */ +#undef JEMALLOC_GCC_U8_ATOMIC_ATOMICS /* Defined if GCC __sync atomics are available. */ #undef JEMALLOC_GCC_SYNC_ATOMICS - -/* - * Defined if __sync_add_and_fetch(uint32_t *, uint32_t) and - * __sync_sub_and_fetch(uint32_t *, uint32_t) are available, despite - * __GCC_HAVE_SYNC_COMPARE_AND_SWAP_4 not being defined (which means the - * functions are defined in libgcc instead of being inlines). - */ -#undef JE_FORCE_SYNC_COMPARE_AND_SWAP_4 - -/* - * Defined if __sync_add_and_fetch(uint64_t *, uint64_t) and - * __sync_sub_and_fetch(uint64_t *, uint64_t) are available, despite - * __GCC_HAVE_SYNC_COMPARE_AND_SWAP_8 not being defined (which means the - * functions are defined in libgcc instead of being inlines). - */ -#undef JE_FORCE_SYNC_COMPARE_AND_SWAP_8 +/* and the 8-bit variant support. */ +#undef JEMALLOC_GCC_U8_SYNC_ATOMICS /* * Defined if __builtin_clz() and __builtin_clzl() are available. @@ -76,12 +66,6 @@ */ #undef JEMALLOC_OS_UNFAIR_LOCK -/* - * Defined if OSSpin*() functions are available, as provided by Darwin, and - * documented in the spinlock(3) manual page. - */ -#undef JEMALLOC_OSSPIN - /* Defined if syscall(2) is usable. */ #undef JEMALLOC_USE_SYSCALL @@ -151,6 +135,9 @@ /* JEMALLOC_STATS enables statistics calculation. */ #undef JEMALLOC_STATS +/* JEMALLOC_EXPERIMENTAL_SMALLOCX_API enables experimental smallocx API. */ +#undef JEMALLOC_EXPERIMENTAL_SMALLOCX_API + /* JEMALLOC_PROF enables allocation profiling. */ #undef JEMALLOC_PROF @@ -231,12 +218,30 @@ #undef JEMALLOC_INTERNAL_FFSL #undef JEMALLOC_INTERNAL_FFS +/* + * popcount*() functions to use for bitmapping. + */ +#undef JEMALLOC_INTERNAL_POPCOUNTL +#undef JEMALLOC_INTERNAL_POPCOUNT + /* * If defined, explicitly attempt to more uniformly distribute large allocation * pointer alignments across all cache indices. */ #undef JEMALLOC_CACHE_OBLIVIOUS +/* + * If defined, enable logging facilities. We make this a configure option to + * avoid taking extra branches everywhere. + */ +#undef JEMALLOC_LOG + +/* + * If defined, use readlinkat() (instead of readlink()) to follow + * /etc/malloc_conf. + */ +#undef JEMALLOC_READLINKAT + /* * Darwin (OS X) uses zones to work around Mach-O symbol override shortcomings. */ @@ -254,6 +259,12 @@ /* Defined if madvise(2) is available. */ #undef JEMALLOC_HAVE_MADVISE +/* + * Defined if transparent huge pages are supported via the MADV_[NO]HUGEPAGE + * arguments to madvise(2). + */ +#undef JEMALLOC_HAVE_MADVISE_HUGE + /* * Methods for purging unused pages differ between operating systems. * @@ -271,6 +282,14 @@ #undef JEMALLOC_PURGE_MADVISE_DONTNEED #undef JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS +/* Defined if madvise(2) is available but MADV_FREE is not (x86 Linux only). */ +#undef JEMALLOC_DEFINE_MADVISE_FREE + +/* + * Defined if MADV_DO[NT]DUMP is supported as an argument to madvise. + */ +#undef JEMALLOC_MADVISE_DONTDUMP + /* * Defined if transparent huge pages (THPs) are supported via the * MADV_[NO]HUGEPAGE arguments to madvise(2), and THP support is enabled. @@ -336,4 +355,12 @@ /* If defined, jemalloc takes the malloc/free/etc. symbol names. */ #undef JEMALLOC_IS_MALLOC +/* + * Defined if strerror_r returns char * if _GNU_SOURCE is defined. + */ +#undef JEMALLOC_STRERROR_R_RETURNS_CHAR_WITH_GNU_SOURCE + +/* Performs additional size-matching sanity checks when defined. */ +#undef JEMALLOC_EXTRA_SIZE_CHECK + #endif /* JEMALLOC_INTERNAL_DEFS_H_ */ diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/jemalloc_internal_externs.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/jemalloc_internal_externs.h index e10fb275d4..b784362338 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/jemalloc_internal_externs.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/jemalloc_internal_externs.h @@ -2,7 +2,6 @@ #define JEMALLOC_INTERNAL_EXTERNS_H #include "jemalloc/internal/atomic.h" -#include "jemalloc/internal/size_classes.h" #include "jemalloc/internal/tsd_types.h" /* TSD checks this to set thread local slow state accordingly. */ @@ -25,6 +24,9 @@ extern unsigned ncpus; /* Number of arenas used for automatic multiplexing of threads and arenas. */ extern unsigned narenas_auto; +/* Base index for manual arenas. */ +extern unsigned manual_arena_base; + /* * Arenas that are used to service external requests. Not all elements of the * arenas array are necessarily used; arenas are created lazily as needed. diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_a.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_a.h index 24ea416297..ddde9b4e63 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_a.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_a.h @@ -4,13 +4,15 @@ #include "jemalloc/internal/atomic.h" #include "jemalloc/internal/bit_util.h" #include "jemalloc/internal/jemalloc_internal_types.h" -#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/sc.h" #include "jemalloc/internal/ticker.h" JEMALLOC_ALWAYS_INLINE malloc_cpuid_t malloc_getcpu(void) { assert(have_percpu_arena); -#if defined(JEMALLOC_HAVE_SCHED_GETCPU) +#if defined(_WIN32) + return GetCurrentProcessorNumber(); +#elif defined(JEMALLOC_HAVE_SCHED_GETCPU) return (malloc_cpuid_t)sched_getcpu(); #else not_reached(); @@ -106,16 +108,16 @@ decay_ticker_get(tsd_t *tsd, unsigned ind) { return &tdata->decay_ticker; } -JEMALLOC_ALWAYS_INLINE tcache_bin_t * +JEMALLOC_ALWAYS_INLINE cache_bin_t * tcache_small_bin_get(tcache_t *tcache, szind_t binind) { - assert(binind < NBINS); - return &tcache->tbins_small[binind]; + assert(binind < SC_NBINS); + return &tcache->bins_small[binind]; } -JEMALLOC_ALWAYS_INLINE tcache_bin_t * +JEMALLOC_ALWAYS_INLINE cache_bin_t * tcache_large_bin_get(tcache_t *tcache, szind_t binind) { - assert(binind >= NBINS &&binind < nhbins); - return &tcache->tbins_large[binind - NBINS]; + assert(binind >= SC_NBINS &&binind < nhbins); + return &tcache->bins_large[binind - SC_NBINS]; } JEMALLOC_ALWAYS_INLINE bool @@ -151,11 +153,12 @@ pre_reentrancy(tsd_t *tsd, arena_t *arena) { assert(arena != arena_get(tsd_tsdn(tsd), 0, false)); bool fast = tsd_fast(tsd); + assert(tsd_reentrancy_level_get(tsd) < INT8_MAX); ++*tsd_reentrancy_levelp_get(tsd); if (fast) { /* Prepare slow path for reentrancy. */ tsd_slow_update(tsd); - assert(tsd->state == tsd_state_nominal_slow); + assert(tsd_state_get(tsd) == tsd_state_nominal_slow); } } diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_b.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_b.h index 2e76e5d8f7..70d6e57885 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_b.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_b.h @@ -71,7 +71,8 @@ arena_ichoose(tsd_t *tsd, arena_t *arena) { static inline bool arena_is_auto(arena_t *arena) { assert(narenas_auto > 0); - return (arena_ind_get(arena) < narenas_auto); + + return (arena_ind_get(arena) < manual_arena_base); } JEMALLOC_ALWAYS_INLINE extent_t * diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_c.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_c.h index 7ffce6fb03..cdb10eb21f 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_c.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/jemalloc_internal_inlines_c.h @@ -1,10 +1,29 @@ #ifndef JEMALLOC_INTERNAL_INLINES_C_H #define JEMALLOC_INTERNAL_INLINES_C_H +#include "jemalloc/internal/hook.h" #include "jemalloc/internal/jemalloc_internal_types.h" #include "jemalloc/internal/sz.h" #include "jemalloc/internal/witness.h" +/* + * Translating the names of the 'i' functions: + * Abbreviations used in the first part of the function name (before + * alloc/dalloc) describe what that function accomplishes: + * a: arena (query) + * s: size (query, or sized deallocation) + * e: extent (query) + * p: aligned (allocates) + * vs: size (query, without knowing that the pointer is into the heap) + * r: rallocx implementation + * x: xallocx implementation + * Abbreviations used in the second part of the function name (after + * alloc/dalloc) describe the arguments it takes + * z: whether to return zeroed memory + * t: accepts a tcache_t * parameter + * m: accepts an arena_t * parameter + */ + JEMALLOC_ALWAYS_INLINE arena_t * iaalloc(tsdn_t *tsdn, const void *ptr) { assert(ptr != NULL); @@ -24,11 +43,12 @@ iallocztm(tsdn_t *tsdn, size_t size, szind_t ind, bool zero, tcache_t *tcache, bool is_internal, arena_t *arena, bool slow_path) { void *ret; - assert(size != 0); assert(!is_internal || tcache == NULL); assert(!is_internal || arena == NULL || arena_is_auto(arena)); - witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), - WITNESS_RANK_CORE, 0); + if (!tsdn_null(tsdn) && tsd_reentrancy_level_get(tsdn_tsd(tsdn)) == 0) { + witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), + WITNESS_RANK_CORE, 0); + } ret = arena_malloc(tsdn, arena, size, ind, zero, tcache, slow_path); if (config_stats && is_internal && likely(ret != NULL)) { @@ -91,7 +111,8 @@ idalloctm(tsdn_t *tsdn, void *ptr, tcache_t *tcache, alloc_ctx_t *alloc_ctx, if (config_stats && is_internal) { arena_internal_sub(iaalloc(tsdn, ptr), isalloc(tsdn, ptr)); } - if (!is_internal && tsd_reentrancy_level_get(tsdn_tsd(tsdn)) != 0) { + if (!is_internal && !tsdn_null(tsdn) && + tsd_reentrancy_level_get(tsdn_tsd(tsdn)) != 0) { assert(tcache == NULL); } arena_dalloc(tsdn, ptr, tcache, alloc_ctx, slow_path); @@ -112,31 +133,20 @@ isdalloct(tsdn_t *tsdn, void *ptr, size_t size, tcache_t *tcache, JEMALLOC_ALWAYS_INLINE void * iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, - size_t extra, size_t alignment, bool zero, tcache_t *tcache, - arena_t *arena) { + size_t alignment, bool zero, tcache_t *tcache, arena_t *arena, + hook_ralloc_args_t *hook_args) { witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); void *p; size_t usize, copysize; - usize = sz_sa2u(size + extra, alignment); - if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) { + usize = sz_sa2u(size, alignment); + if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) { return NULL; } p = ipalloct(tsdn, usize, alignment, zero, tcache, arena); if (p == NULL) { - if (extra == 0) { - return NULL; - } - /* Try again, without extra this time. */ - usize = sz_sa2u(size, alignment); - if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) { - return NULL; - } - p = ipalloct(tsdn, usize, alignment, zero, tcache, arena); - if (p == NULL) { - return NULL; - } + return NULL; } /* * Copy at most size bytes (not size+extra), since the caller has no @@ -144,13 +154,26 @@ iralloct_realign(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, */ copysize = (size < oldsize) ? size : oldsize; memcpy(p, ptr, copysize); + hook_invoke_alloc(hook_args->is_realloc + ? hook_alloc_realloc : hook_alloc_rallocx, p, (uintptr_t)p, + hook_args->args); + hook_invoke_dalloc(hook_args->is_realloc + ? hook_dalloc_realloc : hook_dalloc_rallocx, ptr, hook_args->args); isdalloct(tsdn, ptr, oldsize, tcache, NULL, true); return p; } +/* + * is_realloc threads through the knowledge of whether or not this call comes + * from je_realloc (as opposed to je_rallocx); this ensures that we pass the + * correct entry point into any hooks. + * Note that these functions are all force-inlined, so no actual bool gets + * passed-around anywhere. + */ JEMALLOC_ALWAYS_INLINE void * iralloct(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t alignment, - bool zero, tcache_t *tcache, arena_t *arena) { + bool zero, tcache_t *tcache, arena_t *arena, hook_ralloc_args_t *hook_args) +{ assert(ptr != NULL); assert(size != 0); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), @@ -162,24 +185,24 @@ iralloct(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t alignment, * Existing object alignment is inadequate; allocate new space * and copy. */ - return iralloct_realign(tsdn, ptr, oldsize, size, 0, alignment, - zero, tcache, arena); + return iralloct_realign(tsdn, ptr, oldsize, size, alignment, + zero, tcache, arena, hook_args); } return arena_ralloc(tsdn, arena, ptr, oldsize, size, alignment, zero, - tcache); + tcache, hook_args); } JEMALLOC_ALWAYS_INLINE void * iralloc(tsd_t *tsd, void *ptr, size_t oldsize, size_t size, size_t alignment, - bool zero) { + bool zero, hook_ralloc_args_t *hook_args) { return iralloct(tsd_tsdn(tsd), ptr, oldsize, size, alignment, zero, - tcache_get(tsd), NULL); + tcache_get(tsd), NULL, hook_args); } JEMALLOC_ALWAYS_INLINE bool ixalloc(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t extra, - size_t alignment, bool zero) { + size_t alignment, bool zero, size_t *newsize) { assert(ptr != NULL); assert(size != 0); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), @@ -188,10 +211,12 @@ ixalloc(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, size_t extra, if (alignment != 0 && ((uintptr_t)ptr & ((uintptr_t)alignment-1)) != 0) { /* Existing object alignment is inadequate. */ + *newsize = oldsize; return true; } - return arena_ralloc_no_move(tsdn, ptr, oldsize, size, extra, zero); + return arena_ralloc_no_move(tsdn, ptr, oldsize, size, extra, zero, + newsize); } #endif /* JEMALLOC_INTERNAL_INLINES_C_H */ diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/jemalloc_internal_macros.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/jemalloc_internal_macros.h index 4571895ec3..d8ea06f6d0 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/jemalloc_internal_macros.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/jemalloc_internal_macros.h @@ -30,11 +30,85 @@ # define restrict #endif -/* Various function pointers are statick and immutable except during testing. */ +/* Various function pointers are static and immutable except during testing. */ #ifdef JEMALLOC_JET # define JET_MUTABLE #else # define JET_MUTABLE const #endif +#define JEMALLOC_VA_ARGS_HEAD(head, ...) head +#define JEMALLOC_VA_ARGS_TAIL(head, ...) __VA_ARGS__ + +#if (defined(__GNUC__) || defined(__GNUG__)) && !defined(__clang__) \ + && defined(JEMALLOC_HAVE_ATTR) && (__GNUC__ >= 7) +#define JEMALLOC_FALLTHROUGH JEMALLOC_ATTR(fallthrough); +#else +#define JEMALLOC_FALLTHROUGH /* falls through */ +#endif + +/* Diagnostic suppression macros */ +#if defined(_MSC_VER) && !defined(__clang__) +# define JEMALLOC_DIAGNOSTIC_PUSH __pragma(warning(push)) +# define JEMALLOC_DIAGNOSTIC_POP __pragma(warning(pop)) +# define JEMALLOC_DIAGNOSTIC_IGNORE(W) __pragma(warning(disable:W)) +# define JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS +# define JEMALLOC_DIAGNOSTIC_IGNORE_TYPE_LIMITS +# define JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN +# define JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS +/* #pragma GCC diagnostic first appeared in gcc 4.6. */ +#elif (defined(__GNUC__) && ((__GNUC__ > 4) || ((__GNUC__ == 4) && \ + (__GNUC_MINOR__ > 5)))) || defined(__clang__) +/* + * The JEMALLOC_PRAGMA__ macro is an implementation detail of the GCC and Clang + * diagnostic suppression macros and should not be used anywhere else. + */ +# define JEMALLOC_PRAGMA__(X) _Pragma(#X) +# define JEMALLOC_DIAGNOSTIC_PUSH JEMALLOC_PRAGMA__(GCC diagnostic push) +# define JEMALLOC_DIAGNOSTIC_POP JEMALLOC_PRAGMA__(GCC diagnostic pop) +# define JEMALLOC_DIAGNOSTIC_IGNORE(W) \ + JEMALLOC_PRAGMA__(GCC diagnostic ignored W) + +/* + * The -Wmissing-field-initializers warning is buggy in GCC versions < 5.1 and + * all clang versions up to version 7 (currently trunk, unreleased). This macro + * suppresses the warning for the affected compiler versions only. + */ +# if ((defined(__GNUC__) && !defined(__clang__)) && (__GNUC__ < 5)) || \ + defined(__clang__) +# define JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS \ + JEMALLOC_DIAGNOSTIC_IGNORE("-Wmissing-field-initializers") +# else +# define JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS +# endif + +# define JEMALLOC_DIAGNOSTIC_IGNORE_TYPE_LIMITS \ + JEMALLOC_DIAGNOSTIC_IGNORE("-Wtype-limits") +# define JEMALLOC_DIAGNOSTIC_IGNORE_UNUSED_PARAMETER \ + JEMALLOC_DIAGNOSTIC_IGNORE("-Wunused-parameter") +# if defined(__GNUC__) && !defined(__clang__) && (__GNUC__ >= 7) +# define JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN \ + JEMALLOC_DIAGNOSTIC_IGNORE("-Walloc-size-larger-than=") +# else +# define JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN +# endif +# define JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS \ + JEMALLOC_DIAGNOSTIC_PUSH \ + JEMALLOC_DIAGNOSTIC_IGNORE_UNUSED_PARAMETER +#else +# define JEMALLOC_DIAGNOSTIC_PUSH +# define JEMALLOC_DIAGNOSTIC_POP +# define JEMALLOC_DIAGNOSTIC_IGNORE(W) +# define JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS +# define JEMALLOC_DIAGNOSTIC_IGNORE_TYPE_LIMITS +# define JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN +# define JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS +#endif + +/* + * Disables spurious diagnostics for all headers. Since these headers are not + * included by users directly, it does not affect their diagnostic settings. + */ +JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS + #endif /* JEMALLOC_INTERNAL_MACROS_H */ diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/jemalloc_internal_types.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/jemalloc_internal_types.h index 50f9d001d5..e296c5a7e8 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/jemalloc_internal_types.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/jemalloc_internal_types.h @@ -1,6 +1,8 @@ #ifndef JEMALLOC_INTERNAL_TYPES_H #define JEMALLOC_INTERNAL_TYPES_H +#include "jemalloc/internal/quantum.h" + /* Page size index type. */ typedef unsigned pszind_t; @@ -50,72 +52,6 @@ typedef int malloc_cpuid_t; /* Smallest size class to support. */ #define TINY_MIN (1U << LG_TINY_MIN) -/* - * Minimum allocation alignment is 2^LG_QUANTUM bytes (ignoring tiny size - * classes). - */ -#ifndef LG_QUANTUM -# if (defined(__i386__) || defined(_M_IX86)) -# define LG_QUANTUM 4 -# endif -# ifdef __ia64__ -# define LG_QUANTUM 4 -# endif -# ifdef __alpha__ -# define LG_QUANTUM 4 -# endif -# if (defined(__sparc64__) || defined(__sparcv9) || defined(__sparc_v9__)) -# define LG_QUANTUM 4 -# endif -# if (defined(__amd64__) || defined(__x86_64__) || defined(_M_X64)) -# define LG_QUANTUM 4 -# endif -# ifdef __arm__ -# define LG_QUANTUM 3 -# endif -# ifdef __aarch64__ -# define LG_QUANTUM 4 -# endif -# ifdef __hppa__ -# define LG_QUANTUM 4 -# endif -# ifdef __mips__ -# define LG_QUANTUM 3 -# endif -# ifdef __or1k__ -# define LG_QUANTUM 3 -# endif -# ifdef __powerpc__ -# define LG_QUANTUM 4 -# endif -# ifdef __riscv__ -# define LG_QUANTUM 4 -# endif -# ifdef __s390__ -# define LG_QUANTUM 4 -# endif -# ifdef __SH4__ -# define LG_QUANTUM 4 -# endif -# ifdef __tile__ -# define LG_QUANTUM 4 -# endif -# ifdef __le32__ -# define LG_QUANTUM 4 -# endif -# ifndef LG_QUANTUM -# error "Unknown minimum alignment for architecture; specify via " - "--with-lg-quantum" -# endif -#endif - -#define QUANTUM ((size_t)(1U << LG_QUANTUM)) -#define QUANTUM_MASK (QUANTUM - 1) - -/* Return the smallest quantum multiple that is >= a. */ -#define QUANTUM_CEILING(a) \ - (((a) + QUANTUM_MASK) & ~QUANTUM_MASK) - #define LONG ((size_t)(1U << LG_SIZEOF_LONG)) #define LONG_MASK (LONG - 1) diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/jemalloc_preamble.h.in b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/jemalloc_preamble.h.in index 18539a0988..4bfdb32cb4 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/jemalloc_preamble.h.in +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/jemalloc_preamble.h.in @@ -21,7 +21,7 @@ # include "../jemalloc@install_suffix@.h" #endif -#if (defined(JEMALLOC_OSATOMIC) || defined(JEMALLOC_OSSPIN)) +#if defined(JEMALLOC_OSATOMIC) #include #endif @@ -45,7 +45,11 @@ # include "jemalloc/internal/private_namespace_jet.h" # endif #endif -#include "jemalloc/internal/hooks.h" +#include "jemalloc/internal/test_hooks.h" + +#ifdef JEMALLOC_DEFINE_MADVISE_FREE +# define JEMALLOC_MADV_FREE 8 +#endif static const bool config_debug = #ifdef JEMALLOC_DEBUG @@ -61,6 +65,13 @@ static const bool have_dss = false #endif ; +static const bool have_madvise_huge = +#ifdef JEMALLOC_HAVE_MADVISE_HUGE + true +#else + false +#endif + ; static const bool config_fill = #ifdef JEMALLOC_FILL true @@ -111,13 +122,6 @@ static const bool config_stats = false #endif ; -static const bool config_thp = -#ifdef JEMALLOC_THP - true -#else - false -#endif - ; static const bool config_tls = #ifdef JEMALLOC_TLS true @@ -146,7 +150,18 @@ static const bool config_cache_oblivious = false #endif ; -#ifdef JEMALLOC_HAVE_SCHED_GETCPU +/* + * Undocumented, for jemalloc development use only at the moment. See the note + * in jemalloc/internal/log.h. + */ +static const bool config_log = +#ifdef JEMALLOC_LOG + true +#else + false +#endif + ; +#if defined(_WIN32) || defined(JEMALLOC_HAVE_SCHED_GETCPU) /* Currently percpu_arena depends on sched_getcpu. */ #define JEMALLOC_PERCPU_ARENA #endif diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/large_externs.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/large_externs.h index 3f36282cd4..a05019e8a5 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/large_externs.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/large_externs.h @@ -1,13 +1,16 @@ #ifndef JEMALLOC_INTERNAL_LARGE_EXTERNS_H #define JEMALLOC_INTERNAL_LARGE_EXTERNS_H +#include "jemalloc/internal/hook.h" + void *large_malloc(tsdn_t *tsdn, arena_t *arena, size_t usize, bool zero); void *large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, bool zero); bool large_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, size_t usize_min, size_t usize_max, bool zero); -void *large_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, size_t usize, - size_t alignment, bool zero, tcache_t *tcache); +void *large_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t usize, + size_t alignment, bool zero, tcache_t *tcache, + hook_ralloc_args_t *hook_args); typedef void (large_dalloc_junk_t)(void *, size_t); extern large_dalloc_junk_t *JET_MUTABLE large_dalloc_junk; @@ -23,4 +26,7 @@ prof_tctx_t *large_prof_tctx_get(tsdn_t *tsdn, const extent_t *extent); void large_prof_tctx_set(tsdn_t *tsdn, extent_t *extent, prof_tctx_t *tctx); void large_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent); +nstime_t large_prof_alloc_time_get(const extent_t *extent); +void large_prof_alloc_time_set(extent_t *extent, nstime_t time); + #endif /* JEMALLOC_INTERNAL_LARGE_EXTERNS_H */ diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/log.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/log.h new file mode 100644 index 0000000000..6420858635 --- /dev/null +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/log.h @@ -0,0 +1,115 @@ +#ifndef JEMALLOC_INTERNAL_LOG_H +#define JEMALLOC_INTERNAL_LOG_H + +#include "jemalloc/internal/atomic.h" +#include "jemalloc/internal/malloc_io.h" +#include "jemalloc/internal/mutex.h" + +#ifdef JEMALLOC_LOG +# define JEMALLOC_LOG_VAR_BUFSIZE 1000 +#else +# define JEMALLOC_LOG_VAR_BUFSIZE 1 +#endif + +#define JEMALLOC_LOG_BUFSIZE 4096 + +/* + * The log malloc_conf option is a '|'-delimited list of log_var name segments + * which should be logged. The names are themselves hierarchical, with '.' as + * the delimiter (a "segment" is just a prefix in the log namespace). So, if + * you have: + * + * log("arena", "log msg for arena"); // 1 + * log("arena.a", "log msg for arena.a"); // 2 + * log("arena.b", "log msg for arena.b"); // 3 + * log("arena.a.a", "log msg for arena.a.a"); // 4 + * log("extent.a", "log msg for extent.a"); // 5 + * log("extent.b", "log msg for extent.b"); // 6 + * + * And your malloc_conf option is "log=arena.a|extent", then lines 2, 4, 5, and + * 6 will print at runtime. You can enable logging from all log vars by + * writing "log=.". + * + * None of this should be regarded as a stable API for right now. It's intended + * as a debugging interface, to let us keep around some of our printf-debugging + * statements. + */ + +extern char log_var_names[JEMALLOC_LOG_VAR_BUFSIZE]; +extern atomic_b_t log_init_done; + +typedef struct log_var_s log_var_t; +struct log_var_s { + /* + * Lowest bit is "inited", second lowest is "enabled". Putting them in + * a single word lets us avoid any fences on weak architectures. + */ + atomic_u_t state; + const char *name; +}; + +#define LOG_NOT_INITIALIZED 0U +#define LOG_INITIALIZED_NOT_ENABLED 1U +#define LOG_ENABLED 2U + +#define LOG_VAR_INIT(name_str) {ATOMIC_INIT(LOG_NOT_INITIALIZED), name_str} + +/* + * Returns the value we should assume for state (which is not necessarily + * accurate; if logging is done before logging has finished initializing, then + * we default to doing the safe thing by logging everything). + */ +unsigned log_var_update_state(log_var_t *log_var); + +/* We factor out the metadata management to allow us to test more easily. */ +#define log_do_begin(log_var) \ +if (config_log) { \ + unsigned log_state = atomic_load_u(&(log_var).state, \ + ATOMIC_RELAXED); \ + if (unlikely(log_state == LOG_NOT_INITIALIZED)) { \ + log_state = log_var_update_state(&(log_var)); \ + assert(log_state != LOG_NOT_INITIALIZED); \ + } \ + if (log_state == LOG_ENABLED) { \ + { + /* User code executes here. */ +#define log_do_end(log_var) \ + } \ + } \ +} + +/* + * MSVC has some preprocessor bugs in its expansion of __VA_ARGS__ during + * preprocessing. To work around this, we take all potential extra arguments in + * a var-args functions. Since a varargs macro needs at least one argument in + * the "...", we accept the format string there, and require that the first + * argument in this "..." is a const char *. + */ +static inline void +log_impl_varargs(const char *name, ...) { + char buf[JEMALLOC_LOG_BUFSIZE]; + va_list ap; + + va_start(ap, name); + const char *format = va_arg(ap, const char *); + size_t dst_offset = 0; + dst_offset += malloc_snprintf(buf, JEMALLOC_LOG_BUFSIZE, "%s: ", name); + dst_offset += malloc_vsnprintf(buf + dst_offset, + JEMALLOC_LOG_BUFSIZE - dst_offset, format, ap); + dst_offset += malloc_snprintf(buf + dst_offset, + JEMALLOC_LOG_BUFSIZE - dst_offset, "\n"); + va_end(ap); + + malloc_write(buf); +} + +/* Call as log("log.var.str", "format_string %d", arg_for_format_string); */ +#define LOG(log_var_str, ...) \ +do { \ + static log_var_t log_var = LOG_VAR_INIT(log_var_str); \ + log_do_begin(log_var) \ + log_impl_varargs((log_var).name, __VA_ARGS__); \ + log_do_end(log_var) \ +} while (0) + +#endif /* JEMALLOC_INTERNAL_LOG_H */ diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/malloc_io.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/malloc_io.h index 47ae58ec35..bfe556b523 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/malloc_io.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/malloc_io.h @@ -53,10 +53,50 @@ size_t malloc_vsnprintf(char *str, size_t size, const char *format, va_list ap); size_t malloc_snprintf(char *str, size_t size, const char *format, ...) JEMALLOC_FORMAT_PRINTF(3, 4); +/* + * The caller can set write_cb and cbopaque to null to choose to print with the + * je_malloc_message hook. + */ void malloc_vcprintf(void (*write_cb)(void *, const char *), void *cbopaque, const char *format, va_list ap); void malloc_cprintf(void (*write_cb)(void *, const char *), void *cbopaque, const char *format, ...) JEMALLOC_FORMAT_PRINTF(3, 4); void malloc_printf(const char *format, ...) JEMALLOC_FORMAT_PRINTF(1, 2); +static inline ssize_t +malloc_write_fd(int fd, const void *buf, size_t count) { +#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_write) + /* + * Use syscall(2) rather than write(2) when possible in order to avoid + * the possibility of memory allocation within libc. This is necessary + * on FreeBSD; most operating systems do not have this problem though. + * + * syscall() returns long or int, depending on platform, so capture the + * result in the widest plausible type to avoid compiler warnings. + */ + long result = syscall(SYS_write, fd, buf, count); +#else + ssize_t result = (ssize_t)write(fd, buf, +#ifdef _WIN32 + (unsigned int) +#endif + count); +#endif + return (ssize_t)result; +} + +static inline ssize_t +malloc_read_fd(int fd, void *buf, size_t count) { +#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_read) + long result = syscall(SYS_read, fd, buf, count); +#else + ssize_t result = read(fd, buf, +#ifdef _WIN32 + (unsigned int) +#endif + count); +#endif + return (ssize_t)result; +} + #endif /* JEMALLOC_INTERNAL_MALLOC_IO_H */ diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/mutex.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/mutex.h index 6520c2512d..7c24f0725e 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/mutex.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/mutex.h @@ -37,14 +37,17 @@ struct malloc_mutex_s { # endif #elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) os_unfair_lock lock; -#elif (defined(JEMALLOC_OSSPIN)) - OSSpinLock lock; #elif (defined(JEMALLOC_MUTEX_INIT_CB)) pthread_mutex_t lock; malloc_mutex_t *postponed_next; #else pthread_mutex_t lock; #endif + /* + * Hint flag to avoid exclusive cache line contention + * during spin waiting + */ + atomic_b_t locked; }; /* * We only touch witness when configured w/ debug. However we @@ -84,10 +87,6 @@ struct malloc_mutex_s { # define MALLOC_MUTEX_LOCK(m) os_unfair_lock_lock(&(m)->lock) # define MALLOC_MUTEX_UNLOCK(m) os_unfair_lock_unlock(&(m)->lock) # define MALLOC_MUTEX_TRYLOCK(m) (!os_unfair_lock_trylock(&(m)->lock)) -#elif (defined(JEMALLOC_OSSPIN)) -# define MALLOC_MUTEX_LOCK(m) OSSpinLockLock(&(m)->lock) -# define MALLOC_MUTEX_UNLOCK(m) OSSpinLockUnlock(&(m)->lock) -# define MALLOC_MUTEX_TRYLOCK(m) (!OSSpinLockTry(&(m)->lock)) #else # define MALLOC_MUTEX_LOCK(m) pthread_mutex_lock(&(m)->lock) # define MALLOC_MUTEX_UNLOCK(m) pthread_mutex_unlock(&(m)->lock) @@ -101,22 +100,37 @@ struct malloc_mutex_s { #ifdef _WIN32 # define MALLOC_MUTEX_INITIALIZER #elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) -# define MALLOC_MUTEX_INITIALIZER \ - {{{LOCK_PROF_DATA_INITIALIZER, OS_UNFAIR_LOCK_INIT}}, \ - WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)} -#elif (defined(JEMALLOC_OSSPIN)) -# define MALLOC_MUTEX_INITIALIZER \ - {{{LOCK_PROF_DATA_INITIALIZER, 0}}, \ +# if defined(JEMALLOC_DEBUG) +# define MALLOC_MUTEX_INITIALIZER \ + {{{LOCK_PROF_DATA_INITIALIZER, OS_UNFAIR_LOCK_INIT, ATOMIC_INIT(false)}}, \ + WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0} +# else +# define MALLOC_MUTEX_INITIALIZER \ + {{{LOCK_PROF_DATA_INITIALIZER, OS_UNFAIR_LOCK_INIT, ATOMIC_INIT(false)}}, \ WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)} +# endif #elif (defined(JEMALLOC_MUTEX_INIT_CB)) -# define MALLOC_MUTEX_INITIALIZER \ - {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, NULL}}, \ - WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)} +# if (defined(JEMALLOC_DEBUG)) +# define MALLOC_MUTEX_INITIALIZER \ + {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, NULL, ATOMIC_INIT(false)}}, \ + WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0} +# else +# define MALLOC_MUTEX_INITIALIZER \ + {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, NULL, ATOMIC_INIT(false)}}, \ + WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)} +# endif + #else # define MALLOC_MUTEX_TYPE PTHREAD_MUTEX_DEFAULT +# if defined(JEMALLOC_DEBUG) # define MALLOC_MUTEX_INITIALIZER \ - {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER}}, \ - WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)} + {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, ATOMIC_INIT(false)}}, \ + WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0} +# else +# define MALLOC_MUTEX_INITIALIZER \ + {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, ATOMIC_INIT(false)}}, \ + WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)} +# endif #endif #ifdef JEMALLOC_LAZY_LOCK @@ -139,6 +153,7 @@ void malloc_mutex_lock_slow(malloc_mutex_t *mutex); static inline void malloc_mutex_lock_final(malloc_mutex_t *mutex) { MALLOC_MUTEX_LOCK(mutex); + atomic_store_b(&mutex->locked, true, ATOMIC_RELAXED); } static inline bool @@ -164,6 +179,7 @@ malloc_mutex_trylock(tsdn_t *tsdn, malloc_mutex_t *mutex) { witness_assert_not_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness); if (isthreaded) { if (malloc_mutex_trylock_final(mutex)) { + atomic_store_b(&mutex->locked, true, ATOMIC_RELAXED); return true; } mutex_owner_stats_update(tsdn, mutex); @@ -203,6 +219,7 @@ malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex) { if (isthreaded) { if (malloc_mutex_trylock_final(mutex)) { malloc_mutex_lock_slow(mutex); + atomic_store_b(&mutex->locked, true, ATOMIC_RELAXED); } mutex_owner_stats_update(tsdn, mutex); } @@ -211,6 +228,7 @@ malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex) { static inline void malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex) { + atomic_store_b(&mutex->locked, false, ATOMIC_RELAXED); witness_unlock(tsdn_witness_tsdp_get(tsdn), &mutex->witness); if (isthreaded) { MALLOC_MUTEX_UNLOCK(mutex); @@ -245,4 +263,26 @@ malloc_mutex_prof_read(tsdn_t *tsdn, mutex_prof_data_t *data, atomic_store_u32(&data->n_waiting_thds, 0, ATOMIC_RELAXED); } +static inline void +malloc_mutex_prof_accum(tsdn_t *tsdn, mutex_prof_data_t *data, + malloc_mutex_t *mutex) { + mutex_prof_data_t *source = &mutex->prof_data; + /* Can only read holding the mutex. */ + malloc_mutex_assert_owner(tsdn, mutex); + + nstime_add(&data->tot_wait_time, &source->tot_wait_time); + if (nstime_compare(&source->max_wait_time, &data->max_wait_time) > 0) { + nstime_copy(&data->max_wait_time, &source->max_wait_time); + } + data->n_wait_times += source->n_wait_times; + data->n_spin_acquired += source->n_spin_acquired; + if (data->max_n_thds < source->max_n_thds) { + data->max_n_thds = source->max_n_thds; + } + /* n_wait_thds is not reported. */ + atomic_store_u32(&data->n_waiting_thds, 0, ATOMIC_RELAXED); + data->n_owner_switches += source->n_owner_switches; + data->n_lock_ops += source->n_lock_ops; +} + #endif /* JEMALLOC_INTERNAL_MUTEX_H */ diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/mutex_prof.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/mutex_prof.h index 3358bcf535..2cb8fb0cbf 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/mutex_prof.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/mutex_prof.h @@ -35,21 +35,43 @@ typedef enum { mutex_prof_num_arena_mutexes } mutex_prof_arena_ind_t; +/* + * The forth parameter is a boolean value that is true for derived rate counters + * and false for real ones. + */ +#define MUTEX_PROF_UINT64_COUNTERS \ + OP(num_ops, uint64_t, "n_lock_ops", false, num_ops) \ + OP(num_ops_ps, uint64_t, "(#/sec)", true, num_ops) \ + OP(num_wait, uint64_t, "n_waiting", false, num_wait) \ + OP(num_wait_ps, uint64_t, "(#/sec)", true, num_wait) \ + OP(num_spin_acq, uint64_t, "n_spin_acq", false, num_spin_acq) \ + OP(num_spin_acq_ps, uint64_t, "(#/sec)", true, num_spin_acq) \ + OP(num_owner_switch, uint64_t, "n_owner_switch", false, num_owner_switch) \ + OP(num_owner_switch_ps, uint64_t, "(#/sec)", true, num_owner_switch) \ + OP(total_wait_time, uint64_t, "total_wait_ns", false, total_wait_time) \ + OP(total_wait_time_ps, uint64_t, "(#/sec)", true, total_wait_time) \ + OP(max_wait_time, uint64_t, "max_wait_ns", false, max_wait_time) + +#define MUTEX_PROF_UINT32_COUNTERS \ + OP(max_num_thds, uint32_t, "max_n_thds", false, max_num_thds) + #define MUTEX_PROF_COUNTERS \ - OP(num_ops, uint64_t) \ - OP(num_wait, uint64_t) \ - OP(num_spin_acq, uint64_t) \ - OP(num_owner_switch, uint64_t) \ - OP(total_wait_time, uint64_t) \ - OP(max_wait_time, uint64_t) \ - OP(max_num_thds, uint32_t) + MUTEX_PROF_UINT64_COUNTERS \ + MUTEX_PROF_UINT32_COUNTERS -typedef enum { -#define OP(counter, type) mutex_counter_##counter, - MUTEX_PROF_COUNTERS +#define OP(counter, type, human, derived, base_counter) mutex_counter_##counter, + +#define COUNTER_ENUM(counter_list, t) \ + typedef enum { \ + counter_list \ + mutex_prof_num_##t##_counters \ + } mutex_prof_##t##_counter_ind_t; + +COUNTER_ENUM(MUTEX_PROF_UINT64_COUNTERS, uint64_t) +COUNTER_ENUM(MUTEX_PROF_UINT32_COUNTERS, uint32_t) + +#undef COUNTER_ENUM #undef OP - mutex_prof_num_counters -} mutex_prof_counter_ind_t; typedef struct { /* diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/pages.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/pages.h index 28383b7f97..7dae633afe 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/pages.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/pages.h @@ -58,6 +58,20 @@ static const bool pages_can_purge_forced = #endif ; +typedef enum { + thp_mode_default = 0, /* Do not change hugepage settings. */ + thp_mode_always = 1, /* Always set MADV_HUGEPAGE. */ + thp_mode_never = 2, /* Always set MADV_NOHUGEPAGE. */ + + thp_mode_names_limit = 3, /* Used for option processing. */ + thp_mode_not_supported = 3 /* No THP support detected. */ +} thp_mode_t; + +#define THP_MODE_DEFAULT thp_mode_default +extern thp_mode_t opt_thp; +extern thp_mode_t init_system_thp_mode; /* Initial system wide state. */ +extern const char *thp_mode_names[]; + void *pages_map(void *addr, size_t size, size_t alignment, bool *commit); void pages_unmap(void *addr, size_t size); bool pages_commit(void *addr, size_t size); @@ -66,6 +80,9 @@ bool pages_purge_lazy(void *addr, size_t size); bool pages_purge_forced(void *addr, size_t size); bool pages_huge(void *addr, size_t size); bool pages_nohuge(void *addr, size_t size); +bool pages_dontdump(void *addr, size_t size); +bool pages_dodump(void *addr, size_t size); bool pages_boot(void); +void pages_set_thp_state (void *ptr, size_t size); #endif /* JEMALLOC_INTERNAL_PAGES_EXTERNS_H */ diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/prof_externs.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/prof_externs.h index 04348696f5..094f3e170a 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/prof_externs.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/prof_externs.h @@ -14,6 +14,7 @@ extern bool opt_prof_gdump; /* High-water memory dumping. */ extern bool opt_prof_final; /* Final profile dumping. */ extern bool opt_prof_leak; /* Dump leak summary at exit. */ extern bool opt_prof_accum; /* Report cumulative bytes. */ +extern bool opt_prof_log; /* Turn logging on at boot. */ extern char opt_prof_prefix[ /* Minimize memory bloat for non-prof builds. */ #ifdef JEMALLOC_PROF @@ -45,7 +46,8 @@ extern size_t lg_prof_sample; void prof_alloc_rollback(tsd_t *tsd, prof_tctx_t *tctx, bool updated); void prof_malloc_sample_object(tsdn_t *tsdn, const void *ptr, size_t usize, prof_tctx_t *tctx); -void prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_tctx_t *tctx); +void prof_free_sampled_object(tsd_t *tsd, const void *ptr, size_t usize, + prof_tctx_t *tctx); void bt_init(prof_bt_t *bt, void **vec); void prof_backtrace(prof_bt_t *bt); prof_tctx_t *prof_lookup(tsd_t *tsd, prof_bt_t *bt); @@ -89,4 +91,15 @@ void prof_postfork_parent(tsdn_t *tsdn); void prof_postfork_child(tsdn_t *tsdn); void prof_sample_threshold_update(prof_tdata_t *tdata); +bool prof_log_start(tsdn_t *tsdn, const char *filename); +bool prof_log_stop(tsdn_t *tsdn); +#ifdef JEMALLOC_JET +size_t prof_log_bt_count(void); +size_t prof_log_alloc_count(void); +size_t prof_log_thr_count(void); +bool prof_log_is_logging(void); +bool prof_log_rep_check(void); +void prof_log_dummy_set(bool new_value); +#endif + #endif /* JEMALLOC_INTERNAL_PROF_EXTERNS_H */ diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/prof_inlines_a.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/prof_inlines_a.h index eda6839ade..471d9853cf 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/prof_inlines_a.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/prof_inlines_a.h @@ -4,7 +4,8 @@ #include "jemalloc/internal/mutex.h" static inline bool -prof_accum_add(tsdn_t *tsdn, prof_accum_t *prof_accum, uint64_t accumbytes) { +prof_accum_add(tsdn_t *tsdn, prof_accum_t *prof_accum, + uint64_t accumbytes) { cassert(config_prof); bool overflow; @@ -42,7 +43,8 @@ prof_accum_add(tsdn_t *tsdn, prof_accum_t *prof_accum, uint64_t accumbytes) { } static inline void -prof_accum_cancel(tsdn_t *tsdn, prof_accum_t *prof_accum, size_t usize) { +prof_accum_cancel(tsdn_t *tsdn, prof_accum_t *prof_accum, + size_t usize) { cassert(config_prof); /* @@ -55,18 +57,29 @@ prof_accum_cancel(tsdn_t *tsdn, prof_accum_t *prof_accum, size_t usize) { #ifdef JEMALLOC_ATOMIC_U64 a0 = atomic_load_u64(&prof_accum->accumbytes, ATOMIC_RELAXED); do { - a1 = (a0 >= LARGE_MINCLASS - usize) ? a0 - (LARGE_MINCLASS - - usize) : 0; + a1 = (a0 >= SC_LARGE_MINCLASS - usize) + ? a0 - (SC_LARGE_MINCLASS - usize) : 0; } while (!atomic_compare_exchange_weak_u64(&prof_accum->accumbytes, &a0, a1, ATOMIC_RELAXED, ATOMIC_RELAXED)); #else malloc_mutex_lock(tsdn, &prof_accum->mtx); a0 = prof_accum->accumbytes; - a1 = (a0 >= LARGE_MINCLASS - usize) ? a0 - (LARGE_MINCLASS - usize) : - 0; + a1 = (a0 >= SC_LARGE_MINCLASS - usize) + ? a0 - (SC_LARGE_MINCLASS - usize) : 0; prof_accum->accumbytes = a1; malloc_mutex_unlock(tsdn, &prof_accum->mtx); #endif } +JEMALLOC_ALWAYS_INLINE bool +prof_active_get_unlocked(void) { + /* + * Even if opt_prof is true, sampling can be temporarily disabled by + * setting prof_active to false. No locking is used when reading + * prof_active in the fast path, so there are no guarantees regarding + * how long it will take for all threads to notice state changes. + */ + return prof_active; +} + #endif /* JEMALLOC_INTERNAL_PROF_INLINES_A_H */ diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/prof_inlines_b.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/prof_inlines_b.h index d670cb7b8f..8358bffb5e 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/prof_inlines_b.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/prof_inlines_b.h @@ -3,17 +3,6 @@ #include "jemalloc/internal/sz.h" -JEMALLOC_ALWAYS_INLINE bool -prof_active_get_unlocked(void) { - /* - * Even if opt_prof is true, sampling can be temporarily disabled by - * setting prof_active to false. No locking is used when reading - * prof_active in the fast path, so there are no guarantees regarding - * how long it will take for all threads to notice state changes. - */ - return prof_active; -} - JEMALLOC_ALWAYS_INLINE bool prof_gdump_get_unlocked(void) { /* @@ -72,13 +61,54 @@ prof_tctx_reset(tsdn_t *tsdn, const void *ptr, prof_tctx_t *tctx) { arena_prof_tctx_reset(tsdn, ptr, tctx); } +JEMALLOC_ALWAYS_INLINE nstime_t +prof_alloc_time_get(tsdn_t *tsdn, const void *ptr, alloc_ctx_t *alloc_ctx) { + cassert(config_prof); + assert(ptr != NULL); + + return arena_prof_alloc_time_get(tsdn, ptr, alloc_ctx); +} + +JEMALLOC_ALWAYS_INLINE void +prof_alloc_time_set(tsdn_t *tsdn, const void *ptr, alloc_ctx_t *alloc_ctx, + nstime_t t) { + cassert(config_prof); + assert(ptr != NULL); + + arena_prof_alloc_time_set(tsdn, ptr, alloc_ctx, t); +} + +JEMALLOC_ALWAYS_INLINE bool +prof_sample_check(tsd_t *tsd, size_t usize, bool update) { + ssize_t check = update ? 0 : usize; + + int64_t bytes_until_sample = tsd_bytes_until_sample_get(tsd); + if (update) { + bytes_until_sample -= usize; + if (tsd_nominal(tsd)) { + tsd_bytes_until_sample_set(tsd, bytes_until_sample); + } + } + if (likely(bytes_until_sample >= check)) { + return true; + } + + return false; +} + JEMALLOC_ALWAYS_INLINE bool prof_sample_accum_update(tsd_t *tsd, size_t usize, bool update, - prof_tdata_t **tdata_out) { + prof_tdata_t **tdata_out) { prof_tdata_t *tdata; cassert(config_prof); + /* Fastpath: no need to load tdata */ + if (likely(prof_sample_check(tsd, usize, update))) { + return true; + } + + bool booted = tsd_prof_tdata_get(tsd); tdata = prof_tdata_get(tsd, true); if (unlikely((uintptr_t)tdata <= (uintptr_t)PROF_TDATA_STATE_MAX)) { tdata = NULL; @@ -92,21 +122,23 @@ prof_sample_accum_update(tsd_t *tsd, size_t usize, bool update, return true; } - if (likely(tdata->bytes_until_sample >= usize)) { - if (update) { - tdata->bytes_until_sample -= usize; - } + /* + * If this was the first creation of tdata, then + * prof_tdata_get() reset bytes_until_sample, so decrement and + * check it again + */ + if (!booted && prof_sample_check(tsd, usize, update)) { return true; - } else { - if (tsd_reentrancy_level_get(tsd) > 0) { - return true; - } - /* Compute new sample threshold. */ - if (update) { - prof_sample_threshold_update(tdata); - } - return !tdata->active; } + + if (tsd_reentrancy_level_get(tsd) > 0) { + return true; + } + /* Compute new sample threshold. */ + if (update) { + prof_sample_threshold_update(tdata); + } + return !tdata->active; } JEMALLOC_ALWAYS_INLINE prof_tctx_t * @@ -198,7 +230,7 @@ prof_realloc(tsd_t *tsd, const void *ptr, size_t usize, prof_tctx_t *tctx, * counters. */ if (unlikely(old_sampled)) { - prof_free_sampled_object(tsd, old_usize, old_tctx); + prof_free_sampled_object(tsd, ptr, old_usize, old_tctx); } } @@ -210,7 +242,7 @@ prof_free(tsd_t *tsd, const void *ptr, size_t usize, alloc_ctx_t *alloc_ctx) { assert(usize == isalloc(tsd_tsdn(tsd), ptr)); if (unlikely((uintptr_t)tctx > (uintptr_t)1U)) { - prof_free_sampled_object(tsd, usize, tctx); + prof_free_sampled_object(tsd, ptr, usize, tctx); } } diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/prof_structs.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/prof_structs.h index 0d58ae1005..34ed4822b6 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/prof_structs.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/prof_structs.h @@ -169,7 +169,6 @@ struct prof_tdata_s { /* Sampling state. */ uint64_t prng_state; - uint64_t bytes_until_sample; /* State used to avoid dumping while operating on prof internals. */ bool enq; diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/quantum.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/quantum.h new file mode 100644 index 0000000000..821086e992 --- /dev/null +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/quantum.h @@ -0,0 +1,77 @@ +#ifndef JEMALLOC_INTERNAL_QUANTUM_H +#define JEMALLOC_INTERNAL_QUANTUM_H + +/* + * Minimum allocation alignment is 2^LG_QUANTUM bytes (ignoring tiny size + * classes). + */ +#ifndef LG_QUANTUM +# if (defined(__i386__) || defined(_M_IX86)) +# define LG_QUANTUM 4 +# endif +# ifdef __ia64__ +# define LG_QUANTUM 4 +# endif +# ifdef __alpha__ +# define LG_QUANTUM 4 +# endif +# if (defined(__sparc64__) || defined(__sparcv9) || defined(__sparc_v9__)) +# define LG_QUANTUM 4 +# endif +# if (defined(__amd64__) || defined(__x86_64__) || defined(_M_X64)) +# define LG_QUANTUM 4 +# endif +# ifdef __arm__ +# define LG_QUANTUM 3 +# endif +# ifdef __aarch64__ +# define LG_QUANTUM 4 +# endif +# ifdef __hppa__ +# define LG_QUANTUM 4 +# endif +# ifdef __m68k__ +# define LG_QUANTUM 3 +# endif +# ifdef __mips__ +# define LG_QUANTUM 3 +# endif +# ifdef __nios2__ +# define LG_QUANTUM 3 +# endif +# ifdef __or1k__ +# define LG_QUANTUM 3 +# endif +# ifdef __powerpc__ +# define LG_QUANTUM 4 +# endif +# if defined(__riscv) || defined(__riscv__) +# define LG_QUANTUM 4 +# endif +# ifdef __s390__ +# define LG_QUANTUM 4 +# endif +# if (defined (__SH3E__) || defined(__SH4_SINGLE__) || defined(__SH4__) || \ + defined(__SH4_SINGLE_ONLY__)) +# define LG_QUANTUM 4 +# endif +# ifdef __tile__ +# define LG_QUANTUM 4 +# endif +# ifdef __le32__ +# define LG_QUANTUM 4 +# endif +# ifndef LG_QUANTUM +# error "Unknown minimum alignment for architecture; specify via " + "--with-lg-quantum" +# endif +#endif + +#define QUANTUM ((size_t)(1U << LG_QUANTUM)) +#define QUANTUM_MASK (QUANTUM - 1) + +/* Return the smallest quantum multiple that is >= a. */ +#define QUANTUM_CEILING(a) \ + (((a) + QUANTUM_MASK) & ~QUANTUM_MASK) + +#endif /* JEMALLOC_INTERNAL_QUANTUM_H */ diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/rtree.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/rtree.h index b5d4db3988..16ccbebee7 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/rtree.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/rtree.h @@ -4,7 +4,7 @@ #include "jemalloc/internal/atomic.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/rtree_tsd.h" -#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/sc.h" #include "jemalloc/internal/tsd.h" /* @@ -31,7 +31,7 @@ # error Unsupported number of significant virtual address bits #endif /* Use compact leaf representation if virtual address encoding allows. */ -#if RTREE_NHIB >= LG_CEIL_NSIZES +#if RTREE_NHIB >= LG_CEIL(SC_NSIZES) # define RTREE_LEAF_COMPACT #endif @@ -170,17 +170,29 @@ rtree_subkey(uintptr_t key, unsigned level) { */ # ifdef RTREE_LEAF_COMPACT JEMALLOC_ALWAYS_INLINE uintptr_t -rtree_leaf_elm_bits_read(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, - bool dependent) { +rtree_leaf_elm_bits_read(tsdn_t *tsdn, rtree_t *rtree, + rtree_leaf_elm_t *elm, bool dependent) { return (uintptr_t)atomic_load_p(&elm->le_bits, dependent ? ATOMIC_RELAXED : ATOMIC_ACQUIRE); } JEMALLOC_ALWAYS_INLINE extent_t * rtree_leaf_elm_bits_extent_get(uintptr_t bits) { +# ifdef __aarch64__ + /* + * aarch64 doesn't sign extend the highest virtual address bit to set + * the higher ones. Instead, the high bits gets zeroed. + */ + uintptr_t high_bit_mask = ((uintptr_t)1 << LG_VADDR) - 1; + /* Mask off the slab bit. */ + uintptr_t low_bit_mask = ~(uintptr_t)1; + uintptr_t mask = high_bit_mask & low_bit_mask; + return (extent_t *)(bits & mask); +# else /* Restore sign-extended high bits, mask slab bit. */ return (extent_t *)((uintptr_t)((intptr_t)(bits << RTREE_NHIB) >> RTREE_NHIB) & ~((uintptr_t)0x1)); +# endif } JEMALLOC_ALWAYS_INLINE szind_t @@ -196,8 +208,8 @@ rtree_leaf_elm_bits_slab_get(uintptr_t bits) { # endif JEMALLOC_ALWAYS_INLINE extent_t * -rtree_leaf_elm_extent_read(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, - bool dependent) { +rtree_leaf_elm_extent_read(tsdn_t *tsdn, rtree_t *rtree, + rtree_leaf_elm_t *elm, bool dependent) { #ifdef RTREE_LEAF_COMPACT uintptr_t bits = rtree_leaf_elm_bits_read(tsdn, rtree, elm, dependent); return rtree_leaf_elm_bits_extent_get(bits); @@ -209,8 +221,8 @@ rtree_leaf_elm_extent_read(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, } JEMALLOC_ALWAYS_INLINE szind_t -rtree_leaf_elm_szind_read(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, - bool dependent) { +rtree_leaf_elm_szind_read(tsdn_t *tsdn, rtree_t *rtree, + rtree_leaf_elm_t *elm, bool dependent) { #ifdef RTREE_LEAF_COMPACT uintptr_t bits = rtree_leaf_elm_bits_read(tsdn, rtree, elm, dependent); return rtree_leaf_elm_bits_szind_get(bits); @@ -221,8 +233,8 @@ rtree_leaf_elm_szind_read(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, } JEMALLOC_ALWAYS_INLINE bool -rtree_leaf_elm_slab_read(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, - bool dependent) { +rtree_leaf_elm_slab_read(tsdn_t *tsdn, rtree_t *rtree, + rtree_leaf_elm_t *elm, bool dependent) { #ifdef RTREE_LEAF_COMPACT uintptr_t bits = rtree_leaf_elm_bits_read(tsdn, rtree, elm, dependent); return rtree_leaf_elm_bits_slab_get(bits); @@ -233,8 +245,8 @@ rtree_leaf_elm_slab_read(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, } static inline void -rtree_leaf_elm_extent_write(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, - extent_t *extent) { +rtree_leaf_elm_extent_write(tsdn_t *tsdn, rtree_t *rtree, + rtree_leaf_elm_t *elm, extent_t *extent) { #ifdef RTREE_LEAF_COMPACT uintptr_t old_bits = rtree_leaf_elm_bits_read(tsdn, rtree, elm, true); uintptr_t bits = ((uintptr_t)rtree_leaf_elm_bits_szind_get(old_bits) << @@ -247,9 +259,9 @@ rtree_leaf_elm_extent_write(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, } static inline void -rtree_leaf_elm_szind_write(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, - szind_t szind) { - assert(szind <= NSIZES); +rtree_leaf_elm_szind_write(tsdn_t *tsdn, rtree_t *rtree, + rtree_leaf_elm_t *elm, szind_t szind) { + assert(szind <= SC_NSIZES); #ifdef RTREE_LEAF_COMPACT uintptr_t old_bits = rtree_leaf_elm_bits_read(tsdn, rtree, elm, @@ -265,8 +277,8 @@ rtree_leaf_elm_szind_write(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, } static inline void -rtree_leaf_elm_slab_write(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, - bool slab) { +rtree_leaf_elm_slab_write(tsdn_t *tsdn, rtree_t *rtree, + rtree_leaf_elm_t *elm, bool slab) { #ifdef RTREE_LEAF_COMPACT uintptr_t old_bits = rtree_leaf_elm_bits_read(tsdn, rtree, elm, true); @@ -280,8 +292,8 @@ rtree_leaf_elm_slab_write(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, } static inline void -rtree_leaf_elm_write(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, - extent_t *extent, szind_t szind, bool slab) { +rtree_leaf_elm_write(tsdn_t *tsdn, rtree_t *rtree, + rtree_leaf_elm_t *elm, extent_t *extent, szind_t szind, bool slab) { #ifdef RTREE_LEAF_COMPACT uintptr_t bits = ((uintptr_t)szind << LG_VADDR) | ((uintptr_t)extent & (((uintptr_t)0x1 << LG_VADDR) - 1)) | @@ -301,7 +313,7 @@ rtree_leaf_elm_write(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, static inline void rtree_leaf_elm_szind_slab_update(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *elm, szind_t szind, bool slab) { - assert(!slab || szind < NBINS); + assert(!slab || szind < SC_NBINS); /* * The caller implicitly assures that it is the only writer to the szind @@ -417,7 +429,7 @@ rtree_szind_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, rtree_leaf_elm_t *elm = rtree_read(tsdn, rtree, rtree_ctx, key, dependent); if (!dependent && elm == NULL) { - return NSIZES; + return SC_NSIZES; } return rtree_leaf_elm_szind_read(tsdn, rtree, elm, dependent); } @@ -440,6 +452,42 @@ rtree_extent_szind_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, return false; } +/* + * Try to read szind_slab from the L1 cache. Returns true on a hit, + * and fills in r_szind and r_slab. Otherwise returns false. + * + * Key is allowed to be NULL in order to save an extra branch on the + * fastpath. returns false in this case. + */ +JEMALLOC_ALWAYS_INLINE bool +rtree_szind_slab_read_fast(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, + uintptr_t key, szind_t *r_szind, bool *r_slab) { + rtree_leaf_elm_t *elm; + + size_t slot = rtree_cache_direct_map(key); + uintptr_t leafkey = rtree_leafkey(key); + assert(leafkey != RTREE_LEAFKEY_INVALID); + + if (likely(rtree_ctx->cache[slot].leafkey == leafkey)) { + rtree_leaf_elm_t *leaf = rtree_ctx->cache[slot].leaf; + assert(leaf != NULL); + uintptr_t subkey = rtree_subkey(key, RTREE_HEIGHT-1); + elm = &leaf[subkey]; + +#ifdef RTREE_LEAF_COMPACT + uintptr_t bits = rtree_leaf_elm_bits_read(tsdn, rtree, + elm, true); + *r_szind = rtree_leaf_elm_bits_szind_get(bits); + *r_slab = rtree_leaf_elm_bits_slab_get(bits); +#else + *r_szind = rtree_leaf_elm_szind_read(tsdn, rtree, elm, true); + *r_slab = rtree_leaf_elm_slab_read(tsdn, rtree, elm, true); +#endif + return true; + } else { + return false; + } +} JEMALLOC_ALWAYS_INLINE bool rtree_szind_slab_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, uintptr_t key, bool dependent, szind_t *r_szind, bool *r_slab) { @@ -448,15 +496,21 @@ rtree_szind_slab_read(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, if (!dependent && elm == NULL) { return true; } +#ifdef RTREE_LEAF_COMPACT + uintptr_t bits = rtree_leaf_elm_bits_read(tsdn, rtree, elm, dependent); + *r_szind = rtree_leaf_elm_bits_szind_get(bits); + *r_slab = rtree_leaf_elm_bits_slab_get(bits); +#else *r_szind = rtree_leaf_elm_szind_read(tsdn, rtree, elm, dependent); *r_slab = rtree_leaf_elm_slab_read(tsdn, rtree, elm, dependent); +#endif return false; } static inline void rtree_szind_slab_update(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, uintptr_t key, szind_t szind, bool slab) { - assert(!slab || szind < NBINS); + assert(!slab || szind < SC_NBINS); rtree_leaf_elm_t *elm = rtree_read(tsdn, rtree, rtree_ctx, key, true); rtree_leaf_elm_szind_slab_update(tsdn, rtree, elm, szind, slab); @@ -468,7 +522,7 @@ rtree_clear(tsdn_t *tsdn, rtree_t *rtree, rtree_ctx_t *rtree_ctx, rtree_leaf_elm_t *elm = rtree_read(tsdn, rtree, rtree_ctx, key, true); assert(rtree_leaf_elm_extent_read(tsdn, rtree, elm, false) != NULL); - rtree_leaf_elm_write(tsdn, rtree, elm, NULL, NSIZES, false); + rtree_leaf_elm_write(tsdn, rtree, elm, NULL, SC_NSIZES, false); } #endif /* JEMALLOC_INTERNAL_RTREE_H */ diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/rtree_tsd.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/rtree_tsd.h index 3cdc862548..562e29297a 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/rtree_tsd.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/rtree_tsd.h @@ -26,7 +26,7 @@ * Zero initializer required for tsd initialization only. Proper initialization * done via rtree_ctx_data_init(). */ -#define RTREE_CTX_ZERO_INITIALIZER {{{0}}} +#define RTREE_CTX_ZERO_INITIALIZER {{{0, 0}}, {{0, 0}}} typedef struct rtree_leaf_elm_s rtree_leaf_elm_t; diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/sc.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/sc.h new file mode 100644 index 0000000000..ef0a451237 --- /dev/null +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/sc.h @@ -0,0 +1,320 @@ +#ifndef JEMALLOC_INTERNAL_SC_H +#define JEMALLOC_INTERNAL_SC_H + +#include "jemalloc/internal/jemalloc_internal_types.h" + +/* + * Size class computations: + * + * These are a little tricky; we'll first start by describing how things + * generally work, and then describe some of the details. + * + * Ignore the first few size classes for a moment. We can then split all the + * remaining size classes into groups. The size classes in a group are spaced + * such that they cover allocation request sizes in a power-of-2 range. The + * power of two is called the base of the group, and the size classes in it + * satisfy allocations in the half-open range (base, base * 2]. There are + * SC_NGROUP size classes in each group, equally spaced in the range, so that + * each one covers allocations for base / SC_NGROUP possible allocation sizes. + * We call that value (base / SC_NGROUP) the delta of the group. Each size class + * is delta larger than the one before it (including the initial size class in a + * group, which is delta large than 2**base, the largest size class in the + * previous group). + * To make the math all work out nicely, we require that SC_NGROUP is a power of + * two, and define it in terms of SC_LG_NGROUP. We'll often talk in terms of + * lg_base and lg_delta. For each of these groups then, we have that + * lg_delta == lg_base - SC_LG_NGROUP. + * The size classes in a group with a given lg_base and lg_delta (which, recall, + * can be computed from lg_base for these groups) are therefore: + * base + 1 * delta + * which covers allocations in (base, base + 1 * delta] + * base + 2 * delta + * which covers allocations in (base + 1 * delta, base + 2 * delta]. + * base + 3 * delta + * which covers allocations in (base + 2 * delta, base + 3 * delta]. + * ... + * base + SC_NGROUP * delta ( == 2 * base) + * which covers allocations in (base + (SC_NGROUP - 1) * delta, 2 * base]. + * (Note that currently SC_NGROUP is always 4, so the "..." is empty in + * practice.) + * Note that the last size class in the group is the next power of two (after + * base), so that we've set up the induction correctly for the next group's + * selection of delta. + * + * Now, let's start considering the first few size classes. Two extra constants + * come into play here: LG_QUANTUM and SC_LG_TINY_MIN. LG_QUANTUM ensures + * correct platform alignment; all objects of size (1 << LG_QUANTUM) or larger + * are at least (1 << LG_QUANTUM) aligned; this can be used to ensure that we + * never return improperly aligned memory, by making (1 << LG_QUANTUM) equal the + * highest required alignment of a platform. For allocation sizes smaller than + * (1 << LG_QUANTUM) though, we can be more relaxed (since we don't support + * platforms with types with alignment larger than their size). To allow such + * allocations (without wasting space unnecessarily), we introduce tiny size + * classes; one per power of two, up until we hit the quantum size. There are + * therefore LG_QUANTUM - SC_LG_TINY_MIN such size classes. + * + * Next, we have a size class of size LG_QUANTUM. This can't be the start of a + * group in the sense we described above (covering a power of two range) since, + * if we divided into it to pick a value of delta, we'd get a delta smaller than + * (1 << LG_QUANTUM) for sizes >= (1 << LG_QUANTUM), which is against the rules. + * + * The first base we can divide by SC_NGROUP while still being at least + * (1 << LG_QUANTUM) is SC_NGROUP * (1 << LG_QUANTUM). We can get there by + * having SC_NGROUP size classes, spaced (1 << LG_QUANTUM) apart. These size + * classes are: + * 1 * (1 << LG_QUANTUM) + * 2 * (1 << LG_QUANTUM) + * 3 * (1 << LG_QUANTUM) + * ... (although, as above, this "..." is empty in practice) + * SC_NGROUP * (1 << LG_QUANTUM). + * + * There are SC_NGROUP of these size classes, so we can regard it as a sort of + * pseudo-group, even though it spans multiple powers of 2, is divided + * differently, and both starts and ends on a power of 2 (as opposed to just + * ending). SC_NGROUP is itself a power of two, so the first group after the + * pseudo-group has the power-of-two base SC_NGROUP * (1 << LG_QUANTUM), for a + * lg_base of LG_QUANTUM + SC_LG_NGROUP. We can divide this base into SC_NGROUP + * sizes without violating our LG_QUANTUM requirements, so we can safely set + * lg_delta = lg_base - SC_LG_GROUP (== LG_QUANTUM). + * + * So, in order, the size classes are: + * + * Tiny size classes: + * - Count: LG_QUANTUM - SC_LG_TINY_MIN. + * - Sizes: + * 1 << SC_LG_TINY_MIN + * 1 << (SC_LG_TINY_MIN + 1) + * 1 << (SC_LG_TINY_MIN + 2) + * ... + * 1 << (LG_QUANTUM - 1) + * + * Initial pseudo-group: + * - Count: SC_NGROUP + * - Sizes: + * 1 * (1 << LG_QUANTUM) + * 2 * (1 << LG_QUANTUM) + * 3 * (1 << LG_QUANTUM) + * ... + * SC_NGROUP * (1 << LG_QUANTUM) + * + * Regular group 0: + * - Count: SC_NGROUP + * - Sizes: + * (relative to lg_base of LG_QUANTUM + SC_LG_NGROUP and lg_delta of + * lg_base - SC_LG_NGROUP) + * (1 << lg_base) + 1 * (1 << lg_delta) + * (1 << lg_base) + 2 * (1 << lg_delta) + * (1 << lg_base) + 3 * (1 << lg_delta) + * ... + * (1 << lg_base) + SC_NGROUP * (1 << lg_delta) [ == (1 << (lg_base + 1)) ] + * + * Regular group 1: + * - Count: SC_NGROUP + * - Sizes: + * (relative to lg_base of LG_QUANTUM + SC_LG_NGROUP + 1 and lg_delta of + * lg_base - SC_LG_NGROUP) + * (1 << lg_base) + 1 * (1 << lg_delta) + * (1 << lg_base) + 2 * (1 << lg_delta) + * (1 << lg_base) + 3 * (1 << lg_delta) + * ... + * (1 << lg_base) + SC_NGROUP * (1 << lg_delta) [ == (1 << (lg_base + 1)) ] + * + * ... + * + * Regular group N: + * - Count: SC_NGROUP + * - Sizes: + * (relative to lg_base of LG_QUANTUM + SC_LG_NGROUP + N and lg_delta of + * lg_base - SC_LG_NGROUP) + * (1 << lg_base) + 1 * (1 << lg_delta) + * (1 << lg_base) + 2 * (1 << lg_delta) + * (1 << lg_base) + 3 * (1 << lg_delta) + * ... + * (1 << lg_base) + SC_NGROUP * (1 << lg_delta) [ == (1 << (lg_base + 1)) ] + * + * + * Representation of metadata: + * To make the math easy, we'll mostly work in lg quantities. We record lg_base, + * lg_delta, and ndelta (i.e. number of deltas above the base) on a + * per-size-class basis, and maintain the invariant that, across all size + * classes, size == (1 << lg_base) + ndelta * (1 << lg_delta). + * + * For regular groups (i.e. those with lg_base >= LG_QUANTUM + SC_LG_NGROUP), + * lg_delta is lg_base - SC_LG_NGROUP, and ndelta goes from 1 to SC_NGROUP. + * + * For the initial tiny size classes (if any), lg_base is lg(size class size). + * lg_delta is lg_base for the first size class, and lg_base - 1 for all + * subsequent ones. ndelta is always 0. + * + * For the pseudo-group, if there are no tiny size classes, then we set + * lg_base == LG_QUANTUM, lg_delta == LG_QUANTUM, and have ndelta range from 0 + * to SC_NGROUP - 1. (Note that delta == base, so base + (SC_NGROUP - 1) * delta + * is just SC_NGROUP * base, or (1 << (SC_LG_NGROUP + LG_QUANTUM)), so we do + * indeed get a power of two that way). If there *are* tiny size classes, then + * the first size class needs to have lg_delta relative to the largest tiny size + * class. We therefore set lg_base == LG_QUANTUM - 1, + * lg_delta == LG_QUANTUM - 1, and ndelta == 1, keeping the rest of the + * pseudo-group the same. + * + * + * Other terminology: + * "Small" size classes mean those that are allocated out of bins, which is the + * same as those that are slab allocated. + * "Large" size classes are those that are not small. The cutoff for counting as + * large is page size * group size. + */ + +/* + * Size class N + (1 << SC_LG_NGROUP) twice the size of size class N. + */ +#define SC_LG_NGROUP 2 +#define SC_LG_TINY_MIN 3 + +#if SC_LG_TINY_MIN == 0 +/* The div module doesn't support division by 1, which this would require. */ +#error "Unsupported LG_TINY_MIN" +#endif + +/* + * The definitions below are all determined by the above settings and system + * characteristics. + */ +#define SC_NGROUP (1ULL << SC_LG_NGROUP) +#define SC_PTR_BITS ((1ULL << LG_SIZEOF_PTR) * 8) +#define SC_NTINY (LG_QUANTUM - SC_LG_TINY_MIN) +#define SC_LG_TINY_MAXCLASS (LG_QUANTUM > SC_LG_TINY_MIN ? LG_QUANTUM - 1 : -1) +#define SC_NPSEUDO SC_NGROUP +#define SC_LG_FIRST_REGULAR_BASE (LG_QUANTUM + SC_LG_NGROUP) +/* + * We cap allocations to be less than 2 ** (ptr_bits - 1), so the highest base + * we need is 2 ** (ptr_bits - 2). (This also means that the last group is 1 + * size class shorter than the others). + * We could probably save some space in arenas by capping this at LG_VADDR size. + */ +#define SC_LG_BASE_MAX (SC_PTR_BITS - 2) +#define SC_NREGULAR (SC_NGROUP * \ + (SC_LG_BASE_MAX - SC_LG_FIRST_REGULAR_BASE + 1) - 1) +#define SC_NSIZES (SC_NTINY + SC_NPSEUDO + SC_NREGULAR) + + /* The number of size classes that are a multiple of the page size. */ +#define SC_NPSIZES ( \ + /* Start with all the size classes. */ \ + SC_NSIZES \ + /* Subtract out those groups with too small a base. */ \ + - (LG_PAGE - 1 - SC_LG_FIRST_REGULAR_BASE) * SC_NGROUP \ + /* And the pseudo-group. */ \ + - SC_NPSEUDO \ + /* And the tiny group. */ \ + - SC_NTINY \ + /* Groups where ndelta*delta is not a multiple of the page size. */ \ + - (2 * (SC_NGROUP))) + +/* + * We declare a size class is binnable if size < page size * group. Or, in other + * words, lg(size) < lg(page size) + lg(group size). + */ +#define SC_NBINS ( \ + /* Sub-regular size classes. */ \ + SC_NTINY + SC_NPSEUDO \ + /* Groups with lg_regular_min_base <= lg_base <= lg_base_max */ \ + + SC_NGROUP * (LG_PAGE + SC_LG_NGROUP - SC_LG_FIRST_REGULAR_BASE) \ + /* Last SC of the last group hits the bound exactly; exclude it. */ \ + - 1) + +/* + * The size2index_tab lookup table uses uint8_t to encode each bin index, so we + * cannot support more than 256 small size classes. + */ +#if (SC_NBINS > 256) +# error "Too many small size classes" +#endif + +/* The largest size class in the lookup table. */ +#define SC_LOOKUP_MAXCLASS ((size_t)1 << 12) + +/* Internal, only used for the definition of SC_SMALL_MAXCLASS. */ +#define SC_SMALL_MAX_BASE ((size_t)1 << (LG_PAGE + SC_LG_NGROUP - 1)) +#define SC_SMALL_MAX_DELTA ((size_t)1 << (LG_PAGE - 1)) + +/* The largest size class allocated out of a slab. */ +#define SC_SMALL_MAXCLASS (SC_SMALL_MAX_BASE \ + + (SC_NGROUP - 1) * SC_SMALL_MAX_DELTA) + +/* The smallest size class not allocated out of a slab. */ +#define SC_LARGE_MINCLASS ((size_t)1ULL << (LG_PAGE + SC_LG_NGROUP)) +#define SC_LG_LARGE_MINCLASS (LG_PAGE + SC_LG_NGROUP) + +/* Internal; only used for the definition of SC_LARGE_MAXCLASS. */ +#define SC_MAX_BASE ((size_t)1 << (SC_PTR_BITS - 2)) +#define SC_MAX_DELTA ((size_t)1 << (SC_PTR_BITS - 2 - SC_LG_NGROUP)) + +/* The largest size class supported. */ +#define SC_LARGE_MAXCLASS (SC_MAX_BASE + (SC_NGROUP - 1) * SC_MAX_DELTA) + +typedef struct sc_s sc_t; +struct sc_s { + /* Size class index, or -1 if not a valid size class. */ + int index; + /* Lg group base size (no deltas added). */ + int lg_base; + /* Lg delta to previous size class. */ + int lg_delta; + /* Delta multiplier. size == 1<data) / sizeof(size_t)]; \ + buf[sizeof(buf) / sizeof(size_t) - 1] = 0; \ + memcpy(buf, src, sizeof(type)); \ + size_t old_seq = atomic_load_zu(&dst->seq, ATOMIC_RELAXED); \ + atomic_store_zu(&dst->seq, old_seq + 1, ATOMIC_RELAXED); \ + atomic_fence(ATOMIC_RELEASE); \ + for (size_t i = 0; i < sizeof(buf) / sizeof(size_t); i++) { \ + atomic_store_zu(&dst->data[i], buf[i], ATOMIC_RELAXED); \ + } \ + atomic_store_zu(&dst->seq, old_seq + 2, ATOMIC_RELEASE); \ +} \ + \ +/* Returns whether or not the read was consistent. */ \ +static inline bool \ +seq_try_load_##short_type(type *dst, seq_##short_type##_t *src) { \ + size_t buf[sizeof(src->data) / sizeof(size_t)]; \ + size_t seq1 = atomic_load_zu(&src->seq, ATOMIC_ACQUIRE); \ + if (seq1 % 2 != 0) { \ + return false; \ + } \ + for (size_t i = 0; i < sizeof(buf) / sizeof(size_t); i++) { \ + buf[i] = atomic_load_zu(&src->data[i], ATOMIC_RELAXED); \ + } \ + atomic_fence(ATOMIC_ACQUIRE); \ + size_t seq2 = atomic_load_zu(&src->seq, ATOMIC_RELAXED); \ + if (seq1 != seq2) { \ + return false; \ + } \ + memcpy(dst, buf, sizeof(type)); \ + return true; \ +} + +#endif /* JEMALLOC_INTERNAL_SEQ_H */ diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/size_classes.sh b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/size_classes.sh deleted file mode 100755 index 998994d09e..0000000000 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/size_classes.sh +++ /dev/null @@ -1,361 +0,0 @@ -#!/bin/sh -# -# Usage: size_classes.sh - -# The following limits are chosen such that they cover all supported platforms. - -# Pointer sizes. -lg_zarr="2 3" - -# Quanta. -lg_qarr=$1 - -# The range of tiny size classes is [2^lg_tmin..2^(lg_q-1)]. -lg_tmin=$2 - -# Maximum lookup size. -lg_kmax=12 - -# Page sizes. -lg_parr=`echo $3 | tr ',' ' '` - -# Size class group size (number of size classes for each size doubling). -lg_g=$4 - -pow2() { - e=$1 - pow2_result=1 - while [ ${e} -gt 0 ] ; do - pow2_result=$((${pow2_result} + ${pow2_result})) - e=$((${e} - 1)) - done -} - -lg() { - x=$1 - lg_result=0 - while [ ${x} -gt 1 ] ; do - lg_result=$((${lg_result} + 1)) - x=$((${x} / 2)) - done -} - -lg_ceil() { - y=$1 - lg ${y}; lg_floor=${lg_result} - pow2 ${lg_floor}; pow2_floor=${pow2_result} - if [ ${pow2_floor} -lt ${y} ] ; then - lg_ceil_result=$((${lg_floor} + 1)) - else - lg_ceil_result=${lg_floor} - fi -} - -reg_size_compute() { - lg_grp=$1 - lg_delta=$2 - ndelta=$3 - - pow2 ${lg_grp}; grp=${pow2_result} - pow2 ${lg_delta}; delta=${pow2_result} - reg_size=$((${grp} + ${delta}*${ndelta})) -} - -slab_size() { - lg_p=$1 - lg_grp=$2 - lg_delta=$3 - ndelta=$4 - - pow2 ${lg_p}; p=${pow2_result} - reg_size_compute ${lg_grp} ${lg_delta} ${ndelta} - - # Compute smallest slab size that is an integer multiple of reg_size. - try_slab_size=${p} - try_nregs=$((${try_slab_size} / ${reg_size})) - perfect=0 - while [ ${perfect} -eq 0 ] ; do - perfect_slab_size=${try_slab_size} - perfect_nregs=${try_nregs} - - try_slab_size=$((${try_slab_size} + ${p})) - try_nregs=$((${try_slab_size} / ${reg_size})) - if [ ${perfect_slab_size} -eq $((${perfect_nregs} * ${reg_size})) ] ; then - perfect=1 - fi - done - - slab_size_pgs=$((${perfect_slab_size} / ${p})) -} - -size_class() { - index=$1 - lg_grp=$2 - lg_delta=$3 - ndelta=$4 - lg_p=$5 - lg_kmax=$6 - - if [ ${lg_delta} -ge ${lg_p} ] ; then - psz="yes" - else - pow2 ${lg_p}; p=${pow2_result} - pow2 ${lg_grp}; grp=${pow2_result} - pow2 ${lg_delta}; delta=${pow2_result} - sz=$((${grp} + ${delta} * ${ndelta})) - npgs=$((${sz} / ${p})) - if [ ${sz} -eq $((${npgs} * ${p})) ] ; then - psz="yes" - else - psz="no" - fi - fi - - lg ${ndelta}; lg_ndelta=${lg_result}; pow2 ${lg_ndelta} - if [ ${pow2_result} -lt ${ndelta} ] ; then - rem="yes" - else - rem="no" - fi - - lg_size=${lg_grp} - if [ $((${lg_delta} + ${lg_ndelta})) -eq ${lg_grp} ] ; then - lg_size=$((${lg_grp} + 1)) - else - lg_size=${lg_grp} - rem="yes" - fi - - if [ ${lg_size} -lt $((${lg_p} + ${lg_g})) ] ; then - bin="yes" - slab_size ${lg_p} ${lg_grp} ${lg_delta} ${ndelta}; pgs=${slab_size_pgs} - else - bin="no" - pgs=0 - fi - if [ ${lg_size} -lt ${lg_kmax} \ - -o ${lg_size} -eq ${lg_kmax} -a ${rem} = "no" ] ; then - lg_delta_lookup=${lg_delta} - else - lg_delta_lookup="no" - fi - printf ' SC(%3d, %6d, %8d, %6d, %3s, %3s, %3d, %2s) \\\n' ${index} ${lg_grp} ${lg_delta} ${ndelta} ${psz} ${bin} ${pgs} ${lg_delta_lookup} - # Defined upon return: - # - psz ("yes" or "no") - # - bin ("yes" or "no") - # - pgs - # - lg_delta_lookup (${lg_delta} or "no") -} - -sep_line() { - echo " \\" -} - -size_classes() { - lg_z=$1 - lg_q=$2 - lg_t=$3 - lg_p=$4 - lg_g=$5 - - pow2 $((${lg_z} + 3)); ptr_bits=${pow2_result} - pow2 ${lg_g}; g=${pow2_result} - - echo "#define SIZE_CLASSES \\" - echo " /* index, lg_grp, lg_delta, ndelta, psz, bin, pgs, lg_delta_lookup */ \\" - - ntbins=0 - nlbins=0 - lg_tiny_maxclass='"NA"' - nbins=0 - npsizes=0 - - # Tiny size classes. - ndelta=0 - index=0 - lg_grp=${lg_t} - lg_delta=${lg_grp} - while [ ${lg_grp} -lt ${lg_q} ] ; do - size_class ${index} ${lg_grp} ${lg_delta} ${ndelta} ${lg_p} ${lg_kmax} - if [ ${lg_delta_lookup} != "no" ] ; then - nlbins=$((${index} + 1)) - fi - if [ ${psz} = "yes" ] ; then - npsizes=$((${npsizes} + 1)) - fi - if [ ${bin} != "no" ] ; then - nbins=$((${index} + 1)) - fi - ntbins=$((${ntbins} + 1)) - lg_tiny_maxclass=${lg_grp} # Final written value is correct. - index=$((${index} + 1)) - lg_delta=${lg_grp} - lg_grp=$((${lg_grp} + 1)) - done - - # First non-tiny group. - if [ ${ntbins} -gt 0 ] ; then - sep_line - # The first size class has an unusual encoding, because the size has to be - # split between grp and delta*ndelta. - lg_grp=$((${lg_grp} - 1)) - ndelta=1 - size_class ${index} ${lg_grp} ${lg_delta} ${ndelta} ${lg_p} ${lg_kmax} - index=$((${index} + 1)) - lg_grp=$((${lg_grp} + 1)) - lg_delta=$((${lg_delta} + 1)) - if [ ${psz} = "yes" ] ; then - npsizes=$((${npsizes} + 1)) - fi - fi - while [ ${ndelta} -lt ${g} ] ; do - size_class ${index} ${lg_grp} ${lg_delta} ${ndelta} ${lg_p} ${lg_kmax} - index=$((${index} + 1)) - ndelta=$((${ndelta} + 1)) - if [ ${psz} = "yes" ] ; then - npsizes=$((${npsizes} + 1)) - fi - done - - # All remaining groups. - lg_grp=$((${lg_grp} + ${lg_g})) - while [ ${lg_grp} -lt $((${ptr_bits} - 1)) ] ; do - sep_line - ndelta=1 - if [ ${lg_grp} -eq $((${ptr_bits} - 2)) ] ; then - ndelta_limit=$((${g} - 1)) - else - ndelta_limit=${g} - fi - while [ ${ndelta} -le ${ndelta_limit} ] ; do - size_class ${index} ${lg_grp} ${lg_delta} ${ndelta} ${lg_p} ${lg_kmax} - if [ ${lg_delta_lookup} != "no" ] ; then - nlbins=$((${index} + 1)) - # Final written value is correct: - lookup_maxclass="((((size_t)1) << ${lg_grp}) + (((size_t)${ndelta}) << ${lg_delta}))" - fi - if [ ${psz} = "yes" ] ; then - npsizes=$((${npsizes} + 1)) - fi - if [ ${bin} != "no" ] ; then - nbins=$((${index} + 1)) - # Final written value is correct: - small_maxclass="((((size_t)1) << ${lg_grp}) + (((size_t)${ndelta}) << ${lg_delta}))" - if [ ${lg_g} -gt 0 ] ; then - lg_large_minclass=$((${lg_grp} + 1)) - else - lg_large_minclass=$((${lg_grp} + 2)) - fi - fi - # Final written value is correct: - large_maxclass="((((size_t)1) << ${lg_grp}) + (((size_t)${ndelta}) << ${lg_delta}))" - index=$((${index} + 1)) - ndelta=$((${ndelta} + 1)) - done - lg_grp=$((${lg_grp} + 1)) - lg_delta=$((${lg_delta} + 1)) - done - echo - nsizes=${index} - lg_ceil ${nsizes}; lg_ceil_nsizes=${lg_ceil_result} - - # Defined upon completion: - # - ntbins - # - nlbins - # - nbins - # - nsizes - # - lg_ceil_nsizes - # - npsizes - # - lg_tiny_maxclass - # - lookup_maxclass - # - small_maxclass - # - lg_large_minclass - # - large_maxclass -} - -cat < 256) -# error "Too many small size classes" -#endif - -#endif /* JEMALLOC_INTERNAL_SIZE_CLASSES_H */ -EOF diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/spin.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/spin.h index e2afc98cfd..22804c687f 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/spin.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/spin.h @@ -1,25 +1,29 @@ #ifndef JEMALLOC_INTERNAL_SPIN_H #define JEMALLOC_INTERNAL_SPIN_H -#ifdef JEMALLOC_SPIN_C_ -# define SPIN_INLINE extern inline -#else -# define SPIN_INLINE inline -#endif - #define SPIN_INITIALIZER {0U} typedef struct { unsigned iteration; } spin_t; -SPIN_INLINE void +static inline void +spin_cpu_spinwait() { +# if HAVE_CPU_SPINWAIT + CPU_SPINWAIT; +# else + volatile int x = 0; + x = x; +# endif +} + +static inline void spin_adaptive(spin_t *spin) { volatile uint32_t i; if (spin->iteration < 5) { for (i = 0; i < (1U << spin->iteration); i++) { - CPU_SPINWAIT; + spin_cpu_spinwait(); } spin->iteration++; } else { diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/stats.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/stats.h index 1198779ab9..3b9e0eac12 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/stats.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/stats.h @@ -1,12 +1,6 @@ #ifndef JEMALLOC_INTERNAL_STATS_H #define JEMALLOC_INTERNAL_STATS_H -#include "jemalloc/internal/atomic.h" -#include "jemalloc/internal/mutex_prof.h" -#include "jemalloc/internal/mutex.h" -#include "jemalloc/internal/size_classes.h" -#include "jemalloc/internal/stats_tsd.h" - /* OPTION(opt, var_name, default, set_value_to) */ #define STATS_PRINT_OPTIONS \ OPTION('J', json, false, true) \ @@ -16,7 +10,8 @@ OPTION('a', unmerged, config_stats, false) \ OPTION('b', bins, true, false) \ OPTION('l', large, true, false) \ - OPTION('x', mutex, true, false) + OPTION('x', mutex, true, false) \ + OPTION('e', extents, true, false) enum { #define OPTION(o, v, d, s) stats_print_option_num_##v, @@ -33,132 +28,4 @@ extern char opt_stats_print_opts[stats_print_tot_num_options+1]; void stats_print(void (*write_cb)(void *, const char *), void *cbopaque, const char *opts); -/* - * In those architectures that support 64-bit atomics, we use atomic updates for - * our 64-bit values. Otherwise, we use a plain uint64_t and synchronize - * externally. - */ -#ifdef JEMALLOC_ATOMIC_U64 -typedef atomic_u64_t arena_stats_u64_t; -#else -/* Must hold the arena stats mutex while reading atomically. */ -typedef uint64_t arena_stats_u64_t; -#endif - -typedef struct malloc_bin_stats_s { - /* - * Total number of allocation/deallocation requests served directly by - * the bin. Note that tcache may allocate an object, then recycle it - * many times, resulting many increments to nrequests, but only one - * each to nmalloc and ndalloc. - */ - uint64_t nmalloc; - uint64_t ndalloc; - - /* - * Number of allocation requests that correspond to the size of this - * bin. This includes requests served by tcache, though tcache only - * periodically merges into this counter. - */ - uint64_t nrequests; - - /* - * Current number of regions of this size class, including regions - * currently cached by tcache. - */ - size_t curregs; - - /* Number of tcache fills from this bin. */ - uint64_t nfills; - - /* Number of tcache flushes to this bin. */ - uint64_t nflushes; - - /* Total number of slabs created for this bin's size class. */ - uint64_t nslabs; - - /* - * Total number of slabs reused by extracting them from the slabs heap - * for this bin's size class. - */ - uint64_t reslabs; - - /* Current number of slabs in this bin. */ - size_t curslabs; - - mutex_prof_data_t mutex_data; -} malloc_bin_stats_t; - -typedef struct malloc_large_stats_s { - /* - * Total number of allocation/deallocation requests served directly by - * the arena. - */ - arena_stats_u64_t nmalloc; - arena_stats_u64_t ndalloc; - - /* - * Number of allocation requests that correspond to this size class. - * This includes requests served by tcache, though tcache only - * periodically merges into this counter. - */ - arena_stats_u64_t nrequests; /* Partially derived. */ - - /* Current number of allocations of this size class. */ - size_t curlextents; /* Derived. */ -} malloc_large_stats_t; - -typedef struct decay_stats_s { - /* Total number of purge sweeps. */ - arena_stats_u64_t npurge; - /* Total number of madvise calls made. */ - arena_stats_u64_t nmadvise; - /* Total number of pages purged. */ - arena_stats_u64_t purged; -} decay_stats_t; - -/* - * Arena stats. Note that fields marked "derived" are not directly maintained - * within the arena code; rather their values are derived during stats merge - * requests. - */ -typedef struct arena_stats_s { -#ifndef JEMALLOC_ATOMIC_U64 - malloc_mutex_t mtx; -#endif - - /* Number of bytes currently mapped, excluding retained memory. */ - atomic_zu_t mapped; /* Partially derived. */ - - /* - * Number of unused virtual memory bytes currently retained. Retained - * bytes are technically mapped (though always decommitted or purged), - * but they are excluded from the mapped statistic (above). - */ - atomic_zu_t retained; /* Derived. */ - - decay_stats_t decay_dirty; - decay_stats_t decay_muzzy; - - atomic_zu_t base; /* Derived. */ - atomic_zu_t internal; - atomic_zu_t resident; /* Derived. */ - - atomic_zu_t allocated_large; /* Derived. */ - arena_stats_u64_t nmalloc_large; /* Derived. */ - arena_stats_u64_t ndalloc_large; /* Derived. */ - arena_stats_u64_t nrequests_large; /* Derived. */ - - /* Number of bytes cached in tcache associated with this arena. */ - atomic_zu_t tcache_bytes; /* Derived. */ - - mutex_prof_data_t mutex_prof_data[mutex_prof_num_arena_mutexes]; - - /* One element for each large size class. */ - malloc_large_stats_t lstats[NSIZES - NBINS]; - - /* Arena uptime. */ - nstime_t uptime; -} arena_stats_t; - #endif /* JEMALLOC_INTERNAL_STATS_H */ diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/stats_tsd.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/stats_tsd.h deleted file mode 100755 index d0c3bbe494..0000000000 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/stats_tsd.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef JEMALLOC_INTERNAL_STATS_TSD_H -#define JEMALLOC_INTERNAL_STATS_TSD_H - -typedef struct tcache_bin_stats_s { - /* - * Number of allocation requests that corresponded to the size of this - * bin. - */ - uint64_t nrequests; -} tcache_bin_stats_t; - -#endif /* JEMALLOC_INTERNAL_STATS_TSD_H */ diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/sz.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/sz.h index 7f640d55ad..68e558abfe 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/sz.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/sz.h @@ -3,7 +3,7 @@ #include "jemalloc/internal/bit_util.h" #include "jemalloc/internal/pages.h" -#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/sc.h" #include "jemalloc/internal/util.h" /* @@ -26,18 +26,18 @@ * sz_pind2sz_tab encodes the same information as could be computed by * sz_pind2sz_compute(). */ -extern size_t const sz_pind2sz_tab[NPSIZES+1]; +extern size_t sz_pind2sz_tab[SC_NPSIZES + 1]; /* * sz_index2size_tab encodes the same information as could be computed (at * unacceptable cost in some code paths) by sz_index2size_compute(). */ -extern size_t const sz_index2size_tab[NSIZES]; +extern size_t sz_index2size_tab[SC_NSIZES]; /* * sz_size2index_tab is a compact lookup table that rounds request sizes up to * size classes. In order to reduce cache footprint, the table is compressed, * and all accesses are via sz_size2index(). */ -extern uint8_t const sz_size2index_tab[]; +extern uint8_t sz_size2index_tab[]; static const size_t sz_large_pad = #ifdef JEMALLOC_CACHE_OBLIVIOUS @@ -47,49 +47,47 @@ static const size_t sz_large_pad = #endif ; +extern void sz_boot(const sc_data_t *sc_data); + JEMALLOC_ALWAYS_INLINE pszind_t sz_psz2ind(size_t psz) { - if (unlikely(psz > LARGE_MAXCLASS)) { - return NPSIZES; + if (unlikely(psz > SC_LARGE_MAXCLASS)) { + return SC_NPSIZES; } - { - pszind_t x = lg_floor((psz<<1)-1); - pszind_t shift = (x < LG_SIZE_CLASS_GROUP + LG_PAGE) ? 0 : x - - (LG_SIZE_CLASS_GROUP + LG_PAGE); - pszind_t grp = shift << LG_SIZE_CLASS_GROUP; + pszind_t x = lg_floor((psz<<1)-1); + pszind_t shift = (x < SC_LG_NGROUP + LG_PAGE) ? + 0 : x - (SC_LG_NGROUP + LG_PAGE); + pszind_t grp = shift << SC_LG_NGROUP; - pszind_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_PAGE + 1) ? - LG_PAGE : x - LG_SIZE_CLASS_GROUP - 1; + pszind_t lg_delta = (x < SC_LG_NGROUP + LG_PAGE + 1) ? + LG_PAGE : x - SC_LG_NGROUP - 1; - size_t delta_inverse_mask = ZD(-1) << lg_delta; - pszind_t mod = ((((psz-1) & delta_inverse_mask) >> lg_delta)) & - ((ZU(1) << LG_SIZE_CLASS_GROUP) - 1); + size_t delta_inverse_mask = ZU(-1) << lg_delta; + pszind_t mod = ((((psz-1) & delta_inverse_mask) >> lg_delta)) & + ((ZU(1) << SC_LG_NGROUP) - 1); - pszind_t ind = grp + mod; - return ind; - } + pszind_t ind = grp + mod; + return ind; } static inline size_t sz_pind2sz_compute(pszind_t pind) { - if (unlikely(pind == NPSIZES)) { - return LARGE_MAXCLASS + PAGE; + if (unlikely(pind == SC_NPSIZES)) { + return SC_LARGE_MAXCLASS + PAGE; } - { - size_t grp = pind >> LG_SIZE_CLASS_GROUP; - size_t mod = pind & ((ZU(1) << LG_SIZE_CLASS_GROUP) - 1); + size_t grp = pind >> SC_LG_NGROUP; + size_t mod = pind & ((ZU(1) << SC_LG_NGROUP) - 1); - size_t grp_size_mask = ~((!!grp)-1); - size_t grp_size = ((ZU(1) << (LG_PAGE + - (LG_SIZE_CLASS_GROUP-1))) << grp) & grp_size_mask; + size_t grp_size_mask = ~((!!grp)-1); + size_t grp_size = ((ZU(1) << (LG_PAGE + (SC_LG_NGROUP-1))) << grp) + & grp_size_mask; - size_t shift = (grp == 0) ? 1 : grp; - size_t lg_delta = shift + (LG_PAGE-1); - size_t mod_size = (mod+1) << lg_delta; + size_t shift = (grp == 0) ? 1 : grp; + size_t lg_delta = shift + (LG_PAGE-1); + size_t mod_size = (mod+1) << lg_delta; - size_t sz = grp_size + mod_size; - return sz; - } + size_t sz = grp_size + mod_size; + return sz; } static inline size_t @@ -101,70 +99,70 @@ sz_pind2sz_lookup(pszind_t pind) { static inline size_t sz_pind2sz(pszind_t pind) { - assert(pind < NPSIZES+1); + assert(pind < SC_NPSIZES + 1); return sz_pind2sz_lookup(pind); } static inline size_t sz_psz2u(size_t psz) { - if (unlikely(psz > LARGE_MAXCLASS)) { - return LARGE_MAXCLASS + PAGE; - } - { - size_t x = lg_floor((psz<<1)-1); - size_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_PAGE + 1) ? - LG_PAGE : x - LG_SIZE_CLASS_GROUP - 1; - size_t delta = ZU(1) << lg_delta; - size_t delta_mask = delta - 1; - size_t usize = (psz + delta_mask) & ~delta_mask; - return usize; + if (unlikely(psz > SC_LARGE_MAXCLASS)) { + return SC_LARGE_MAXCLASS + PAGE; } + size_t x = lg_floor((psz<<1)-1); + size_t lg_delta = (x < SC_LG_NGROUP + LG_PAGE + 1) ? + LG_PAGE : x - SC_LG_NGROUP - 1; + size_t delta = ZU(1) << lg_delta; + size_t delta_mask = delta - 1; + size_t usize = (psz + delta_mask) & ~delta_mask; + return usize; } static inline szind_t sz_size2index_compute(size_t size) { - if (unlikely(size > LARGE_MAXCLASS)) { - return NSIZES; + if (unlikely(size > SC_LARGE_MAXCLASS)) { + return SC_NSIZES; + } + + if (size == 0) { + return 0; } -#if (NTBINS != 0) - if (size <= (ZU(1) << LG_TINY_MAXCLASS)) { - szind_t lg_tmin = LG_TINY_MAXCLASS - NTBINS + 1; +#if (SC_NTINY != 0) + if (size <= (ZU(1) << SC_LG_TINY_MAXCLASS)) { + szind_t lg_tmin = SC_LG_TINY_MAXCLASS - SC_NTINY + 1; szind_t lg_ceil = lg_floor(pow2_ceil_zu(size)); return (lg_ceil < lg_tmin ? 0 : lg_ceil - lg_tmin); } #endif { szind_t x = lg_floor((size<<1)-1); - szind_t shift = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM) ? 0 : - x - (LG_SIZE_CLASS_GROUP + LG_QUANTUM); - szind_t grp = shift << LG_SIZE_CLASS_GROUP; + szind_t shift = (x < SC_LG_NGROUP + LG_QUANTUM) ? 0 : + x - (SC_LG_NGROUP + LG_QUANTUM); + szind_t grp = shift << SC_LG_NGROUP; - szind_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM + 1) - ? LG_QUANTUM : x - LG_SIZE_CLASS_GROUP - 1; + szind_t lg_delta = (x < SC_LG_NGROUP + LG_QUANTUM + 1) + ? LG_QUANTUM : x - SC_LG_NGROUP - 1; - size_t delta_inverse_mask = ZD(-1) << lg_delta; + size_t delta_inverse_mask = ZU(-1) << lg_delta; szind_t mod = ((((size-1) & delta_inverse_mask) >> lg_delta)) & - ((ZU(1) << LG_SIZE_CLASS_GROUP) - 1); + ((ZU(1) << SC_LG_NGROUP) - 1); - szind_t index = NTBINS + grp + mod; + szind_t index = SC_NTINY + grp + mod; return index; } } JEMALLOC_ALWAYS_INLINE szind_t sz_size2index_lookup(size_t size) { - assert(size <= LOOKUP_MAXCLASS); - { - szind_t ret = (sz_size2index_tab[(size-1) >> LG_TINY_MIN]); - assert(ret == sz_size2index_compute(size)); - return ret; - } + assert(size <= SC_LOOKUP_MAXCLASS); + szind_t ret = (sz_size2index_tab[(size + (ZU(1) << SC_LG_TINY_MIN) - 1) + >> SC_LG_TINY_MIN]); + assert(ret == sz_size2index_compute(size)); + return ret; } JEMALLOC_ALWAYS_INLINE szind_t sz_size2index(size_t size) { - assert(size > 0); - if (likely(size <= LOOKUP_MAXCLASS)) { + if (likely(size <= SC_LOOKUP_MAXCLASS)) { return sz_size2index_lookup(size); } return sz_size2index_compute(size); @@ -172,20 +170,20 @@ sz_size2index(size_t size) { static inline size_t sz_index2size_compute(szind_t index) { -#if (NTBINS > 0) - if (index < NTBINS) { - return (ZU(1) << (LG_TINY_MAXCLASS - NTBINS + 1 + index)); +#if (SC_NTINY > 0) + if (index < SC_NTINY) { + return (ZU(1) << (SC_LG_TINY_MAXCLASS - SC_NTINY + 1 + index)); } #endif { - size_t reduced_index = index - NTBINS; - size_t grp = reduced_index >> LG_SIZE_CLASS_GROUP; - size_t mod = reduced_index & ((ZU(1) << LG_SIZE_CLASS_GROUP) - + size_t reduced_index = index - SC_NTINY; + size_t grp = reduced_index >> SC_LG_NGROUP; + size_t mod = reduced_index & ((ZU(1) << SC_LG_NGROUP) - 1); size_t grp_size_mask = ~((!!grp)-1); size_t grp_size = ((ZU(1) << (LG_QUANTUM + - (LG_SIZE_CLASS_GROUP-1))) << grp) & grp_size_mask; + (SC_LG_NGROUP-1))) << grp) & grp_size_mask; size_t shift = (grp == 0) ? 1 : grp; size_t lg_delta = shift + (LG_QUANTUM-1); @@ -205,18 +203,22 @@ sz_index2size_lookup(szind_t index) { JEMALLOC_ALWAYS_INLINE size_t sz_index2size(szind_t index) { - assert(index < NSIZES); + assert(index < SC_NSIZES); return sz_index2size_lookup(index); } JEMALLOC_ALWAYS_INLINE size_t sz_s2u_compute(size_t size) { - if (unlikely(size > LARGE_MAXCLASS)) { + if (unlikely(size > SC_LARGE_MAXCLASS)) { return 0; } -#if (NTBINS > 0) - if (size <= (ZU(1) << LG_TINY_MAXCLASS)) { - size_t lg_tmin = LG_TINY_MAXCLASS - NTBINS + 1; + + if (size == 0) { + size++; + } +#if (SC_NTINY > 0) + if (size <= (ZU(1) << SC_LG_TINY_MAXCLASS)) { + size_t lg_tmin = SC_LG_TINY_MAXCLASS - SC_NTINY + 1; size_t lg_ceil = lg_floor(pow2_ceil_zu(size)); return (lg_ceil < lg_tmin ? (ZU(1) << lg_tmin) : (ZU(1) << lg_ceil)); @@ -224,8 +226,8 @@ sz_s2u_compute(size_t size) { #endif { size_t x = lg_floor((size<<1)-1); - size_t lg_delta = (x < LG_SIZE_CLASS_GROUP + LG_QUANTUM + 1) - ? LG_QUANTUM : x - LG_SIZE_CLASS_GROUP - 1; + size_t lg_delta = (x < SC_LG_NGROUP + LG_QUANTUM + 1) + ? LG_QUANTUM : x - SC_LG_NGROUP - 1; size_t delta = ZU(1) << lg_delta; size_t delta_mask = delta - 1; size_t usize = (size + delta_mask) & ~delta_mask; @@ -247,8 +249,7 @@ sz_s2u_lookup(size_t size) { */ JEMALLOC_ALWAYS_INLINE size_t sz_s2u(size_t size) { - assert(size > 0); - if (likely(size <= LOOKUP_MAXCLASS)) { + if (likely(size <= SC_LOOKUP_MAXCLASS)) { return sz_s2u_lookup(size); } return sz_s2u_compute(size); @@ -265,7 +266,7 @@ sz_sa2u(size_t size, size_t alignment) { assert(alignment != 0 && ((alignment - 1) & alignment) == 0); /* Try for a small size class. */ - if (size <= SMALL_MAXCLASS && alignment < PAGE) { + if (size <= SC_SMALL_MAXCLASS && alignment < PAGE) { /* * Round size up to the nearest multiple of alignment. * @@ -281,20 +282,20 @@ sz_sa2u(size_t size, size_t alignment) { * 192 | 11000000 | 64 */ usize = sz_s2u(ALIGNMENT_CEILING(size, alignment)); - if (usize < LARGE_MINCLASS) { + if (usize < SC_LARGE_MINCLASS) { return usize; } } /* Large size class. Beware of overflow. */ - if (unlikely(alignment > LARGE_MAXCLASS)) { + if (unlikely(alignment > SC_LARGE_MAXCLASS)) { return 0; } /* Make sure result is a large size class. */ - if (size <= LARGE_MINCLASS) { - usize = LARGE_MINCLASS; + if (size <= SC_LARGE_MINCLASS) { + usize = SC_LARGE_MINCLASS; } else { usize = sz_s2u(size); if (usize < size) { diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/tcache_externs.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/tcache_externs.h index db3e9c7d5d..d63eafde8c 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/tcache_externs.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/tcache_externs.h @@ -1,15 +1,13 @@ #ifndef JEMALLOC_INTERNAL_TCACHE_EXTERNS_H #define JEMALLOC_INTERNAL_TCACHE_EXTERNS_H -#include "jemalloc/internal/size_classes.h" - extern bool opt_tcache; extern ssize_t opt_lg_tcache_max; -extern tcache_bin_info_t *tcache_bin_info; +extern cache_bin_info_t *tcache_bin_info; /* - * Number of tcache bins. There are NBINS small-object bins, plus 0 or more + * Number of tcache bins. There are SC_NBINS small-object bins, plus 0 or more * large-object bins. */ extern unsigned nhbins; @@ -30,10 +28,10 @@ extern tcaches_t *tcaches; size_t tcache_salloc(tsdn_t *tsdn, const void *ptr); void tcache_event_hard(tsd_t *tsd, tcache_t *tcache); void *tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache, - tcache_bin_t *tbin, szind_t binind, bool *tcache_success); -void tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin, + cache_bin_t *tbin, szind_t binind, bool *tcache_success); +void tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, cache_bin_t *tbin, szind_t binind, unsigned rem); -void tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind, +void tcache_bin_flush_large(tsd_t *tsd, cache_bin_t *tbin, szind_t binind, unsigned rem, tcache_t *tcache); void tcache_arena_reassociate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena); diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/tcache_inlines.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/tcache_inlines.h index c55bcd2723..5eca20e893 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/tcache_inlines.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/tcache_inlines.h @@ -1,8 +1,9 @@ #ifndef JEMALLOC_INTERNAL_TCACHE_INLINES_H #define JEMALLOC_INTERNAL_TCACHE_INLINES_H +#include "jemalloc/internal/bin.h" #include "jemalloc/internal/jemalloc_internal_types.h" -#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/sc.h" #include "jemalloc/internal/sz.h" #include "jemalloc/internal/ticker.h" #include "jemalloc/internal/util.h" @@ -38,43 +39,16 @@ tcache_event(tsd_t *tsd, tcache_t *tcache) { } JEMALLOC_ALWAYS_INLINE void * -tcache_alloc_easy(tcache_bin_t *tbin, bool *tcache_success) { +tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache, + size_t size, szind_t binind, bool zero, bool slow_path) { void *ret; - - if (unlikely(tbin->ncached == 0)) { - tbin->low_water = -1; - *tcache_success = false; - return NULL; - } - /* - * tcache_success (instead of ret) should be checked upon the return of - * this function. We avoid checking (ret == NULL) because there is - * never a null stored on the avail stack (which is unknown to the - * compiler), and eagerly checking ret would cause pipeline stall - * (waiting for the cacheline). - */ - *tcache_success = true; - ret = *(tbin->avail - tbin->ncached); - tbin->ncached--; - - if (unlikely((low_water_t)tbin->ncached < tbin->low_water)) { - tbin->low_water = tbin->ncached; - } - - return ret; -} - -JEMALLOC_ALWAYS_INLINE void * -tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size, - szind_t binind, bool zero, bool slow_path) { - void *ret; - tcache_bin_t *tbin; + cache_bin_t *bin; bool tcache_success; size_t usize JEMALLOC_CC_SILENCE_INIT(0); - assert(binind < NBINS); - tbin = tcache_small_bin_get(tcache, binind); - ret = tcache_alloc_easy(tbin, &tcache_success); + assert(binind < SC_NBINS); + bin = tcache_small_bin_get(tcache, binind); + ret = cache_bin_alloc_easy(bin, &tcache_success); assert(tcache_success == (ret != NULL)); if (unlikely(!tcache_success)) { bool tcache_hard_success; @@ -84,7 +58,7 @@ tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size, } ret = tcache_alloc_small_hard(tsd_tsdn(tsd), arena, tcache, - tbin, binind, &tcache_hard_success); + bin, binind, &tcache_hard_success); if (tcache_hard_success == false) { return NULL; } @@ -103,22 +77,21 @@ tcache_alloc_small(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size, if (likely(!zero)) { if (slow_path && config_fill) { if (unlikely(opt_junk_alloc)) { - arena_alloc_junk_small(ret, - &arena_bin_info[binind], false); + arena_alloc_junk_small(ret, &bin_infos[binind], + false); } else if (unlikely(opt_zero)) { memset(ret, 0, usize); } } } else { if (slow_path && config_fill && unlikely(opt_junk_alloc)) { - arena_alloc_junk_small(ret, &arena_bin_info[binind], - true); + arena_alloc_junk_small(ret, &bin_infos[binind], true); } memset(ret, 0, usize); } if (config_stats) { - tbin->tstats.nrequests++; + bin->tstats.nrequests++; } if (config_prof) { tcache->prof_accumbytes += usize; @@ -131,12 +104,12 @@ JEMALLOC_ALWAYS_INLINE void * tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size, szind_t binind, bool zero, bool slow_path) { void *ret; - tcache_bin_t *tbin; + cache_bin_t *bin; bool tcache_success; - assert(binind >= NBINS &&binind < nhbins); - tbin = tcache_large_bin_get(tcache, binind); - ret = tcache_alloc_easy(tbin, &tcache_success); + assert(binind >= SC_NBINS &&binind < nhbins); + bin = tcache_large_bin_get(tcache, binind); + ret = cache_bin_alloc_easy(bin, &tcache_success); assert(tcache_success == (ret != NULL)); if (unlikely(!tcache_success)) { /* @@ -176,7 +149,7 @@ tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size, } if (config_stats) { - tbin->tstats.nrequests++; + bin->tstats.nrequests++; } if (config_prof) { tcache->prof_accumbytes += usize; @@ -190,24 +163,24 @@ tcache_alloc_large(tsd_t *tsd, arena_t *arena, tcache_t *tcache, size_t size, JEMALLOC_ALWAYS_INLINE void tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind, bool slow_path) { - tcache_bin_t *tbin; - tcache_bin_info_t *tbin_info; + cache_bin_t *bin; + cache_bin_info_t *bin_info; - assert(tcache_salloc(tsd_tsdn(tsd), ptr) <= SMALL_MAXCLASS); + assert(tcache_salloc(tsd_tsdn(tsd), ptr) + <= SC_SMALL_MAXCLASS); if (slow_path && config_fill && unlikely(opt_junk_free)) { - arena_dalloc_junk_small(ptr, &arena_bin_info[binind]); + arena_dalloc_junk_small(ptr, &bin_infos[binind]); } - tbin = tcache_small_bin_get(tcache, binind); - tbin_info = &tcache_bin_info[binind]; - if (unlikely(tbin->ncached == tbin_info->ncached_max)) { - tcache_bin_flush_small(tsd, tcache, tbin, binind, - (tbin_info->ncached_max >> 1)); + bin = tcache_small_bin_get(tcache, binind); + bin_info = &tcache_bin_info[binind]; + if (unlikely(!cache_bin_dalloc_easy(bin, bin_info, ptr))) { + tcache_bin_flush_small(tsd, tcache, bin, binind, + (bin_info->ncached_max >> 1)); + bool ret = cache_bin_dalloc_easy(bin, bin_info, ptr); + assert(ret); } - assert(tbin->ncached < tbin_info->ncached_max); - tbin->ncached++; - *(tbin->avail - tbin->ncached) = ptr; tcache_event(tsd, tcache); } @@ -215,25 +188,26 @@ tcache_dalloc_small(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind, JEMALLOC_ALWAYS_INLINE void tcache_dalloc_large(tsd_t *tsd, tcache_t *tcache, void *ptr, szind_t binind, bool slow_path) { - tcache_bin_t *tbin; - tcache_bin_info_t *tbin_info; + cache_bin_t *bin; + cache_bin_info_t *bin_info; - assert(tcache_salloc(tsd_tsdn(tsd), ptr) > SMALL_MAXCLASS); + assert(tcache_salloc(tsd_tsdn(tsd), ptr) + > SC_SMALL_MAXCLASS); assert(tcache_salloc(tsd_tsdn(tsd), ptr) <= tcache_maxclass); if (slow_path && config_fill && unlikely(opt_junk_free)) { large_dalloc_junk(ptr, sz_index2size(binind)); } - tbin = tcache_large_bin_get(tcache, binind); - tbin_info = &tcache_bin_info[binind]; - if (unlikely(tbin->ncached == tbin_info->ncached_max)) { - tcache_bin_flush_large(tsd, tbin, binind, - (tbin_info->ncached_max >> 1), tcache); + bin = tcache_large_bin_get(tcache, binind); + bin_info = &tcache_bin_info[binind]; + if (unlikely(bin->ncached == bin_info->ncached_max)) { + tcache_bin_flush_large(tsd, bin, binind, + (bin_info->ncached_max >> 1), tcache); } - assert(tbin->ncached < tbin_info->ncached_max); - tbin->ncached++; - *(tbin->avail - tbin->ncached) = ptr; + assert(bin->ncached < bin_info->ncached_max); + bin->ncached++; + *(bin->avail - bin->ncached) = ptr; tcache_event(tsd, tcache); } @@ -242,6 +216,9 @@ JEMALLOC_ALWAYS_INLINE tcache_t * tcaches_get(tsd_t *tsd, unsigned ind) { tcaches_t *elm = &tcaches[ind]; if (unlikely(elm->tcache == NULL)) { + malloc_printf(": invalid tcache id (%u).\n", ind); + abort(); + } else if (unlikely(elm->tcache == TCACHES_ELM_NEED_REINIT)) { elm->tcache = tcache_create_explicit(tsd); } return elm->tcache; diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/tcache_structs.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/tcache_structs.h index 7eb516fb6b..172ef9040c 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/tcache_structs.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/tcache_structs.h @@ -1,56 +1,62 @@ #ifndef JEMALLOC_INTERNAL_TCACHE_STRUCTS_H #define JEMALLOC_INTERNAL_TCACHE_STRUCTS_H +#include "jemalloc/internal/cache_bin.h" #include "jemalloc/internal/ql.h" -#include "jemalloc/internal/size_classes.h" -#include "jemalloc/internal/stats_tsd.h" +#include "jemalloc/internal/sc.h" #include "jemalloc/internal/ticker.h" +#include "jemalloc/internal/tsd_types.h" -/* - * Read-only information associated with each element of tcache_t's tbins array - * is stored separately, mainly to reduce memory usage. - */ -struct tcache_bin_info_s { - unsigned ncached_max; /* Upper limit on ncached. */ -}; +/* Various uses of this struct need it to be a named type. */ +typedef ql_elm(tsd_t) tsd_link_t; -struct tcache_bin_s { - low_water_t low_water; /* Min # cached since last GC. */ - uint32_t ncached; /* # of cached objects. */ +struct tcache_s { /* - * ncached and stats are both modified frequently. Let's keep them - * close so that they have a higher chance of being on the same - * cacheline, thus less write-backs. + * To minimize our cache-footprint, we put the frequently accessed data + * together at the start of this struct. */ - tcache_bin_stats_t tstats; + + /* Cleared after arena_prof_accum(). */ + uint64_t prof_accumbytes; + /* Drives incremental GC. */ + ticker_t gc_ticker; /* - * To make use of adjacent cacheline prefetch, the items in the avail - * stack goes to higher address for newer allocations. avail points - * just above the available space, which means that - * avail[-ncached, ... -1] are available items and the lowest item will - * be allocated first. + * The pointer stacks associated with bins follow as a contiguous array. + * During tcache initialization, the avail pointer in each element of + * tbins is initialized to point to the proper offset within this array. */ - void **avail; /* Stack of available objects. */ -}; + cache_bin_t bins_small[SC_NBINS]; + + /* + * This data is less hot; we can be a little less careful with our + * footprint here. + */ + /* Lets us track all the tcaches in an arena. */ + ql_elm(tcache_t) link; + + /* Logically scoped to tsd, but put here for cache layout reasons. */ + ql_elm(tsd_t) tsd_link; + bool in_hook; -struct tcache_s { - /* Data accessed frequently first: prof, ticker and small bins. */ - uint64_t prof_accumbytes;/* Cleared after arena_prof_accum(). */ - ticker_t gc_ticker; /* Drives incremental GC. */ /* - * The pointer stacks associated with tbins follow as a contiguous - * array. During tcache initialization, the avail pointer in each - * element of tbins is initialized to point to the proper offset within - * this array. + * The descriptor lets the arena find our cache bins without seeing the + * tcache definition. This enables arenas to aggregate stats across + * tcaches without having a tcache dependency. */ - tcache_bin_t tbins_small[NBINS]; - /* Data accessed less often below. */ - ql_elm(tcache_t) link; /* Used for aggregating stats. */ - arena_t *arena; /* Associated arena. */ - szind_t next_gc_bin; /* Next bin to GC. */ + cache_bin_array_descriptor_t cache_bin_array_descriptor; + + /* The arena this tcache is associated with. */ + arena_t *arena; + /* Next bin to GC. */ + szind_t next_gc_bin; /* For small bins, fill (ncached_max >> lg_fill_div). */ - uint8_t lg_fill_div[NBINS]; - tcache_bin_t tbins_large[NSIZES-NBINS]; + uint8_t lg_fill_div[SC_NBINS]; + /* + * We put the cache bins for large size classes at the end of the + * struct, since some of them might not get used. This might end up + * letting us avoid touching an extra page if we don't have to. + */ + cache_bin_t bins_large[SC_NSIZES-SC_NBINS]; }; /* Linkage for list of available (previously used) explicit tcache IDs. */ diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/tcache_types.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/tcache_types.h index 1155d62cb4..dce69382eb 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/tcache_types.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/tcache_types.h @@ -1,16 +1,11 @@ #ifndef JEMALLOC_INTERNAL_TCACHE_TYPES_H #define JEMALLOC_INTERNAL_TCACHE_TYPES_H -#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/sc.h" -typedef struct tcache_bin_info_s tcache_bin_info_t; -typedef struct tcache_bin_s tcache_bin_t; typedef struct tcache_s tcache_t; typedef struct tcaches_s tcaches_t; -/* ncached is cast to this type for comparison. */ -typedef int32_t low_water_t; - /* * tcache pointers close to NULL are used to encode state information that is * used for two purposes: preventing thread caching on a per thread basis and @@ -50,7 +45,7 @@ typedef int32_t low_water_t; /* Number of tcache allocation/deallocation events between incremental GCs. */ #define TCACHE_GC_INCR \ - ((TCACHE_GC_SWEEP / NBINS) + ((TCACHE_GC_SWEEP / NBINS == 0) ? 0 : 1)) + ((TCACHE_GC_SWEEP / SC_NBINS) + ((TCACHE_GC_SWEEP / SC_NBINS == 0) ? 0 : 1)) /* Used in TSD static initializer only. Real init in tcache_data_init(). */ #define TCACHE_ZERO_INITIALIZER {0} @@ -58,4 +53,7 @@ typedef int32_t low_water_t; /* Used in TSD static initializer only. Will be initialized to opt_tcache. */ #define TCACHE_ENABLED_ZERO_INITIALIZER false +/* Used for explicit tcache only. Means flushed but not destroyed. */ +#define TCACHES_ELM_NEED_REINIT ((tcache_t *)(uintptr_t)1) + #endif /* JEMALLOC_INTERNAL_TCACHE_TYPES_H */ diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/test_hooks.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/test_hooks.h new file mode 100644 index 0000000000..a6351e59af --- /dev/null +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/test_hooks.h @@ -0,0 +1,19 @@ +#ifndef JEMALLOC_INTERNAL_TEST_HOOKS_H +#define JEMALLOC_INTERNAL_TEST_HOOKS_H + +extern JEMALLOC_EXPORT void (*test_hooks_arena_new_hook)(); +extern JEMALLOC_EXPORT void (*test_hooks_libc_hook)(); + +#define JEMALLOC_HOOK(fn, hook) ((void)(hook != NULL && (hook(), 0)), fn) + +#define open JEMALLOC_HOOK(open, test_hooks_libc_hook) +#define read JEMALLOC_HOOK(read, test_hooks_libc_hook) +#define write JEMALLOC_HOOK(write, test_hooks_libc_hook) +#define readlink JEMALLOC_HOOK(readlink, test_hooks_libc_hook) +#define close JEMALLOC_HOOK(close, test_hooks_libc_hook) +#define creat JEMALLOC_HOOK(creat, test_hooks_libc_hook) +#define secure_getenv JEMALLOC_HOOK(secure_getenv, test_hooks_libc_hook) +/* Note that this is undef'd and re-define'd in src/prof.c. */ +#define _Unwind_Backtrace JEMALLOC_HOOK(_Unwind_Backtrace, test_hooks_libc_hook) + +#endif /* JEMALLOC_INTERNAL_TEST_HOOKS_H */ diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/ticker.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/ticker.h index 572b96459c..52d0db4c89 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/ticker.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/ticker.h @@ -32,14 +32,42 @@ ticker_read(const ticker_t *ticker) { return ticker->tick; } +/* + * Not intended to be a public API. Unfortunately, on x86, neither gcc nor + * clang seems smart enough to turn + * ticker->tick -= nticks; + * if (unlikely(ticker->tick < 0)) { + * fixup ticker + * return true; + * } + * return false; + * into + * subq %nticks_reg, (%ticker_reg) + * js fixup ticker + * + * unless we force "fixup ticker" out of line. In that case, gcc gets it right, + * but clang now does worse than before. So, on x86 with gcc, we force it out + * of line, but otherwise let the inlining occur. Ordinarily this wouldn't be + * worth the hassle, but this is on the fast path of both malloc and free (via + * tcache_event). + */ +#if defined(__GNUC__) && !defined(__clang__) \ + && (defined(__x86_64__) || defined(__i386__)) +JEMALLOC_NOINLINE +#endif +static bool +ticker_fixup(ticker_t *ticker) { + ticker->tick = ticker->nticks; + return true; +} + static inline bool ticker_ticks(ticker_t *ticker, int32_t nticks) { - if (unlikely(ticker->tick < nticks)) { - ticker->tick = ticker->nticks; - return true; - } ticker->tick -= nticks; - return(false); + if (unlikely(ticker->tick < 0)) { + return ticker_fixup(ticker); + } + return false; } static inline bool @@ -47,4 +75,17 @@ ticker_tick(ticker_t *ticker) { return ticker_ticks(ticker, 1); } +/* + * Try to tick. If ticker would fire, return true, but rely on + * slowpath to reset ticker. + */ +static inline bool +ticker_trytick(ticker_t *ticker) { + --ticker->tick; + if (unlikely(ticker->tick < 0)) { + return true; + } + return false; +} + #endif /* JEMALLOC_INTERNAL_TICKER_H */ diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/tsd.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/tsd.h index 155a2ec6c4..9ba2600453 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/tsd.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/tsd.h @@ -3,6 +3,7 @@ #include "jemalloc/internal/arena_types.h" #include "jemalloc/internal/assert.h" +#include "jemalloc/internal/bin_types.h" #include "jemalloc/internal/jemalloc_internal_externs.h" #include "jemalloc/internal/prof_types.h" #include "jemalloc/internal/ql.h" @@ -65,48 +66,120 @@ typedef void (*test_callback_t)(int *); O(arenas_tdata_bypass, bool, bool) \ O(reentrancy_level, int8_t, int8_t) \ O(narenas_tdata, uint32_t, uint32_t) \ + O(offset_state, uint64_t, uint64_t) \ O(thread_allocated, uint64_t, uint64_t) \ O(thread_deallocated, uint64_t, uint64_t) \ + O(bytes_until_sample, int64_t, int64_t) \ O(prof_tdata, prof_tdata_t *, prof_tdata_t *) \ O(rtree_ctx, rtree_ctx_t, rtree_ctx_t) \ O(iarena, arena_t *, arena_t *) \ O(arena, arena_t *, arena_t *) \ O(arenas_tdata, arena_tdata_t *, arena_tdata_t *)\ + O(binshards, tsd_binshards_t, tsd_binshards_t)\ O(tcache, tcache_t, tcache_t) \ O(witness_tsd, witness_tsd_t, witness_tsdn_t) \ MALLOC_TEST_TSD #define TSD_INITIALIZER { \ - tsd_state_uninitialized, \ + ATOMIC_INIT(tsd_state_uninitialized), \ TCACHE_ENABLED_ZERO_INITIALIZER, \ false, \ 0, \ 0, \ 0, \ 0, \ + 0, \ + 0, \ NULL, \ RTREE_CTX_ZERO_INITIALIZER, \ NULL, \ NULL, \ NULL, \ + TSD_BINSHARDS_ZERO_INITIALIZER, \ TCACHE_ZERO_INITIALIZER, \ WITNESS_TSD_INITIALIZER \ MALLOC_TEST_TSD_INITIALIZER \ } +void *malloc_tsd_malloc(size_t size); +void malloc_tsd_dalloc(void *wrapper); +void malloc_tsd_cleanup_register(bool (*f)(void)); +tsd_t *malloc_tsd_boot0(void); +void malloc_tsd_boot1(void); +void tsd_cleanup(void *arg); +tsd_t *tsd_fetch_slow(tsd_t *tsd, bool internal); +void tsd_state_set(tsd_t *tsd, uint8_t new_state); +void tsd_slow_update(tsd_t *tsd); +void tsd_prefork(tsd_t *tsd); +void tsd_postfork_parent(tsd_t *tsd); +void tsd_postfork_child(tsd_t *tsd); + +/* + * Call ..._inc when your module wants to take all threads down the slow paths, + * and ..._dec when it no longer needs to. + */ +void tsd_global_slow_inc(tsdn_t *tsdn); +void tsd_global_slow_dec(tsdn_t *tsdn); +bool tsd_global_slow(); + enum { - tsd_state_nominal = 0, /* Common case --> jnz. */ - tsd_state_nominal_slow = 1, /* Initialized but on slow path. */ - /* the above 2 nominal states should be lower values. */ - tsd_state_nominal_max = 1, /* used for comparison only. */ - tsd_state_minimal_initialized = 2, - tsd_state_purgatory = 3, - tsd_state_reincarnated = 4, - tsd_state_uninitialized = 5 + /* Common case --> jnz. */ + tsd_state_nominal = 0, + /* Initialized but on slow path. */ + tsd_state_nominal_slow = 1, + /* + * Some thread has changed global state in such a way that all nominal + * threads need to recompute their fast / slow status the next time they + * get a chance. + * + * Any thread can change another thread's status *to* recompute, but + * threads are the only ones who can change their status *from* + * recompute. + */ + tsd_state_nominal_recompute = 2, + /* + * The above nominal states should be lower values. We use + * tsd_nominal_max to separate nominal states from threads in the + * process of being born / dying. + */ + tsd_state_nominal_max = 2, + + /* + * A thread might free() during its death as its only allocator action; + * in such scenarios, we need tsd, but set up in such a way that no + * cleanup is necessary. + */ + tsd_state_minimal_initialized = 3, + /* States during which we know we're in thread death. */ + tsd_state_purgatory = 4, + tsd_state_reincarnated = 5, + /* + * What it says on the tin; tsd that hasn't been initialized. Note + * that even when the tsd struct lives in TLS, when need to keep track + * of stuff like whether or not our pthread destructors have been + * scheduled, so this really truly is different than the nominal state. + */ + tsd_state_uninitialized = 6 }; -/* Manually limit tsd_state_t to a single byte. */ -typedef uint8_t tsd_state_t; +/* + * Some TSD accesses can only be done in a nominal state. To enforce this, we + * wrap TSD member access in a function that asserts on TSD state, and mangle + * field names to prevent touching them accidentally. + */ +#define TSD_MANGLE(n) cant_access_tsd_items_directly_use_a_getter_or_setter_##n + +#ifdef JEMALLOC_U8_ATOMICS +# define tsd_state_t atomic_u8_t +# define tsd_atomic_load atomic_load_u8 +# define tsd_atomic_store atomic_store_u8 +# define tsd_atomic_exchange atomic_exchange_u8 +#else +# define tsd_state_t atomic_u32_t +# define tsd_atomic_load atomic_load_u32 +# define tsd_atomic_store atomic_store_u32 +# define tsd_atomic_exchange atomic_exchange_u32 +#endif /* The actual tsd. */ struct tsd_s { @@ -115,13 +188,29 @@ struct tsd_s { * module. Access any thread-local state through the getters and * setters below. */ - tsd_state_t state; + + /* + * We manually limit the state to just a single byte. Unless the 8-bit + * atomics are unavailable (which is rare). + */ + tsd_state_t state; #define O(n, t, nt) \ - t use_a_getter_or_setter_instead_##n; + t TSD_MANGLE(n); MALLOC_TSD #undef O }; +JEMALLOC_ALWAYS_INLINE uint8_t +tsd_state_get(tsd_t *tsd) { + /* + * This should be atomic. Unfortunately, compilers right now can't tell + * that this can be done as a memory comparison, and forces a load into + * a register that hurts fast-path performance. + */ + /* return atomic_load_u8(&tsd->state, ATOMIC_RELAXED); */ + return *(uint8_t *)&tsd->state; +} + /* * Wrapper around tsd_t that makes it possible to avoid implicit conversion * between tsd_t and tsdn_t, where tsdn_t is "nullable" and has to be @@ -148,15 +237,6 @@ tsdn_tsd(tsdn_t *tsdn) { return &tsdn->tsd; } -void *malloc_tsd_malloc(size_t size); -void malloc_tsd_dalloc(void *wrapper); -void malloc_tsd_cleanup_register(bool (*f)(void)); -tsd_t *malloc_tsd_boot0(void); -void malloc_tsd_boot1(void); -void tsd_cleanup(void *arg); -tsd_t *tsd_fetch_slow(tsd_t *tsd, bool internal); -void tsd_slow_update(tsd_t *tsd); - /* * We put the platform-specific data declarations and inlines into their own * header files to avoid cluttering this file. They define tsd_boot0, @@ -180,7 +260,7 @@ void tsd_slow_update(tsd_t *tsd); #define O(n, t, nt) \ JEMALLOC_ALWAYS_INLINE t * \ tsd_##n##p_get_unsafe(tsd_t *tsd) { \ - return &tsd->use_a_getter_or_setter_instead_##n; \ + return &tsd->TSD_MANGLE(n); \ } MALLOC_TSD #undef O @@ -189,10 +269,16 @@ MALLOC_TSD #define O(n, t, nt) \ JEMALLOC_ALWAYS_INLINE t * \ tsd_##n##p_get(tsd_t *tsd) { \ - assert(tsd->state == tsd_state_nominal || \ - tsd->state == tsd_state_nominal_slow || \ - tsd->state == tsd_state_reincarnated || \ - tsd->state == tsd_state_minimal_initialized); \ + /* \ + * Because the state might change asynchronously if it's \ + * nominal, we need to make sure that we only read it once. \ + */ \ + uint8_t state = tsd_state_get(tsd); \ + assert(state == tsd_state_nominal || \ + state == tsd_state_nominal_slow || \ + state == tsd_state_nominal_recompute || \ + state == tsd_state_reincarnated || \ + state == tsd_state_minimal_initialized); \ return tsd_##n##p_get_unsafe(tsd); \ } MALLOC_TSD @@ -227,8 +313,8 @@ MALLOC_TSD #define O(n, t, nt) \ JEMALLOC_ALWAYS_INLINE void \ tsd_##n##_set(tsd_t *tsd, t val) { \ - assert(tsd->state != tsd_state_reincarnated && \ - tsd->state != tsd_state_minimal_initialized); \ + assert(tsd_state_get(tsd) != tsd_state_reincarnated && \ + tsd_state_get(tsd) != tsd_state_minimal_initialized); \ *tsd_##n##p_get(tsd) = val; \ } MALLOC_TSD @@ -236,13 +322,18 @@ MALLOC_TSD JEMALLOC_ALWAYS_INLINE void tsd_assert_fast(tsd_t *tsd) { + /* + * Note that our fastness assertion does *not* include global slowness + * counters; it's not in general possible to ensure that they won't + * change asynchronously from underneath us. + */ assert(!malloc_slow && tsd_tcache_enabled_get(tsd) && tsd_reentrancy_level_get(tsd) == 0); } JEMALLOC_ALWAYS_INLINE bool tsd_fast(tsd_t *tsd) { - bool fast = (tsd->state == tsd_state_nominal); + bool fast = (tsd_state_get(tsd) == tsd_state_nominal); if (fast) { tsd_assert_fast(tsd); } @@ -259,7 +350,7 @@ tsd_fetch_impl(bool init, bool minimal) { } assert(tsd != NULL); - if (unlikely(tsd->state != tsd_state_nominal)) { + if (unlikely(tsd_state_get(tsd) != tsd_state_nominal)) { return tsd_fetch_slow(tsd, minimal); } assert(tsd_fast(tsd)); @@ -279,7 +370,7 @@ JEMALLOC_ALWAYS_INLINE tsd_t * tsd_internal_fetch(void) { tsd_t *tsd = tsd_fetch_min(); /* Use reincarnated state to prevent full initialization. */ - tsd->state = tsd_state_reincarnated; + tsd_state_set(tsd, tsd_state_reincarnated); return tsd; } @@ -291,7 +382,7 @@ tsd_fetch(void) { static inline bool tsd_nominal(tsd_t *tsd) { - return (tsd->state <= tsd_state_nominal_max); + return (tsd_state_get(tsd) <= tsd_state_nominal_max); } JEMALLOC_ALWAYS_INLINE tsdn_t * diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/tsd_generic.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/tsd_generic.h index 1e52ef767f..cf73c0c715 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/tsd_generic.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/tsd_generic.h @@ -77,7 +77,10 @@ tsd_wrapper_get(bool init) { abort(); } else { wrapper->initialized = false; + JEMALLOC_DIAGNOSTIC_PUSH + JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS tsd_t initializer = TSD_INITIALIZER; + JEMALLOC_DIAGNOSTIC_POP wrapper->val = initializer; } tsd_wrapper_set(wrapper); @@ -107,7 +110,10 @@ tsd_boot1(void) { tsd_boot_wrapper.initialized = false; tsd_cleanup(&tsd_boot_wrapper.val); wrapper->initialized = false; + JEMALLOC_DIAGNOSTIC_PUSH + JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS tsd_t initializer = TSD_INITIALIZER; + JEMALLOC_DIAGNOSTIC_POP wrapper->val = initializer; tsd_wrapper_set(wrapper); } diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/tsd_malloc_thread_cleanup.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/tsd_malloc_thread_cleanup.h index beb467a67e..bf8801effe 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/tsd_malloc_thread_cleanup.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/tsd_malloc_thread_cleanup.h @@ -47,7 +47,6 @@ tsd_get_allocates(void) { /* Get/set. */ JEMALLOC_ALWAYS_INLINE tsd_t * tsd_get(bool init) { - assert(tsd_booted); return &tsd_tls; } JEMALLOC_ALWAYS_INLINE void diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/tsd_tls.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/tsd_tls.h index 757aaa0eef..f4f165c7f5 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/tsd_tls.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/tsd_tls.h @@ -40,7 +40,6 @@ tsd_get_allocates(void) { /* Get/set. */ JEMALLOC_ALWAYS_INLINE tsd_t * tsd_get(bool init) { - assert(tsd_booted); return &tsd_tls; } diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/witness.h b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/witness.h index 33be666107..fff9e98cb6 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/witness.h +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/internal/witness.h @@ -27,9 +27,9 @@ #define WITNESS_RANK_PROF_BT2GCTX 6U #define WITNESS_RANK_PROF_TDATAS 7U #define WITNESS_RANK_PROF_TDATA 8U -#define WITNESS_RANK_PROF_GCTX 9U - -#define WITNESS_RANK_BACKGROUND_THREAD 10U +#define WITNESS_RANK_PROF_LOG 9U +#define WITNESS_RANK_PROF_GCTX 10U +#define WITNESS_RANK_BACKGROUND_THREAD 11U /* * Used as an argument to witness_assert_depth_to_rank() in order to validate @@ -37,21 +37,22 @@ * witness_assert_depth_to_rank() is inclusive rather than exclusive, this * definition can have the same value as the minimally ranked core lock. */ -#define WITNESS_RANK_CORE 11U +#define WITNESS_RANK_CORE 12U -#define WITNESS_RANK_DECAY 11U -#define WITNESS_RANK_TCACHE_QL 12U -#define WITNESS_RANK_EXTENT_GROW 13U -#define WITNESS_RANK_EXTENTS 14U -#define WITNESS_RANK_EXTENT_AVAIL 15U +#define WITNESS_RANK_DECAY 12U +#define WITNESS_RANK_TCACHE_QL 13U +#define WITNESS_RANK_EXTENT_GROW 14U +#define WITNESS_RANK_EXTENTS 15U +#define WITNESS_RANK_EXTENT_AVAIL 16U -#define WITNESS_RANK_EXTENT_POOL 16U -#define WITNESS_RANK_RTREE 17U -#define WITNESS_RANK_BASE 18U -#define WITNESS_RANK_ARENA_LARGE 19U +#define WITNESS_RANK_EXTENT_POOL 17U +#define WITNESS_RANK_RTREE 18U +#define WITNESS_RANK_BASE 19U +#define WITNESS_RANK_ARENA_LARGE 20U +#define WITNESS_RANK_HOOK 21U #define WITNESS_RANK_LEAF 0xffffffffU -#define WITNESS_RANK_ARENA_BIN WITNESS_RANK_LEAF +#define WITNESS_RANK_BIN WITNESS_RANK_LEAF #define WITNESS_RANK_ARENA_STATS WITNESS_RANK_LEAF #define WITNESS_RANK_DSS WITNESS_RANK_LEAF #define WITNESS_RANK_PROF_ACTIVE WITNESS_RANK_LEAF diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/jemalloc_macros.h.in b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/jemalloc_macros.h.in index aee55438c9..a00ce11a79 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/jemalloc_macros.h.in +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/jemalloc_macros.h.in @@ -10,6 +10,7 @@ #define JEMALLOC_VERSION_BUGFIX @jemalloc_version_bugfix@ #define JEMALLOC_VERSION_NREV @jemalloc_version_nrev@ #define JEMALLOC_VERSION_GID "@jemalloc_version_gid@" +#define JEMALLOC_VERSION_GID_IDENT @jemalloc_version_gid@ #define MALLOCX_LG_ALIGN(la) ((int)(la)) #if LG_SIZEOF_PTR == 2 diff --git a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/jemalloc_mangle.sh b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/jemalloc_mangle.sh index df328b78da..c675bb469f 100755 --- a/kbe/src/lib/dependencies/jemalloc/include/jemalloc/jemalloc_mangle.sh +++ b/kbe/src/lib/dependencies/jemalloc/include/jemalloc/jemalloc_mangle.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -eu public_symbols_txt=$1 symbol_prefix=$2 diff --git a/kbe/src/lib/dependencies/jemalloc/jemalloc.pc.in b/kbe/src/lib/dependencies/jemalloc/jemalloc.pc.in index a318e8dd3f..c428a86dc3 100755 --- a/kbe/src/lib/dependencies/jemalloc/jemalloc.pc.in +++ b/kbe/src/lib/dependencies/jemalloc/jemalloc.pc.in @@ -7,6 +7,6 @@ install_suffix=@install_suffix@ Name: jemalloc Description: A general purpose malloc(3) implementation that emphasizes fragmentation avoidance and scalable concurrency support. URL: http://jemalloc.net/ -Version: @jemalloc_version@ +Version: @jemalloc_version_major@.@jemalloc_version_minor@.@jemalloc_version_bugfix@_@jemalloc_version_nrev@ Cflags: -I${includedir} Libs: -L${libdir} -ljemalloc${install_suffix} diff --git a/kbe/src/lib/dependencies/jemalloc/msvc/ReadMe.txt b/kbe/src/lib/dependencies/jemalloc/msvc/ReadMe.txt index 77d567da0f..633a7d49f4 100755 --- a/kbe/src/lib/dependencies/jemalloc/msvc/ReadMe.txt +++ b/kbe/src/lib/dependencies/jemalloc/msvc/ReadMe.txt @@ -9,16 +9,15 @@ How to build jemalloc for Windows * grep * sed -2. Install Visual Studio 2015 with Visual C++ +2. Install Visual Studio 2015 or 2017 with Visual C++ 3. Add Cygwin\bin to the PATH environment variable -4. Open "VS2015 x86 Native Tools Command Prompt" +4. Open "x64 Native Tools Command Prompt for VS 2017" (note: x86/x64 doesn't matter at this point) 5. Generate header files: sh -c "CC=cl ./autogen.sh" 6. Now the project can be opened and built in Visual Studio: - msvc\jemalloc_vc2015.sln - + msvc\jemalloc_vc2017.sln diff --git a/kbe/src/lib/dependencies/jemalloc/msvc/jemalloc_vc2017.sln b/kbe/src/lib/dependencies/jemalloc/msvc/jemalloc_vc2017.sln new file mode 100644 index 0000000000..c22fcb437b --- /dev/null +++ b/kbe/src/lib/dependencies/jemalloc/msvc/jemalloc_vc2017.sln @@ -0,0 +1,63 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 14 +VisualStudioVersion = 14.0.24720.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{70A99006-6DE9-472B-8F83-4CEE6C616DF3}" + ProjectSection(SolutionItems) = preProject + ReadMe.txt = ReadMe.txt + EndProjectSection +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "jemalloc", "projects\vc2017\jemalloc\jemalloc.vcxproj", "{8D6BB292-9E1C-413D-9F98-4864BDC1514A}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "test_threads", "projects\vc2017\test_threads\test_threads.vcxproj", "{09028CFD-4EB7-491D-869C-0708DB97ED44}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Debug-static|x64 = Debug-static|x64 + Debug-static|x86 = Debug-static|x86 + Release|x64 = Release|x64 + Release|x86 = Release|x86 + Release-static|x64 = Release-static|x64 + Release-static|x86 = Release-static|x86 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {8D6BB292-9E1C-413D-9F98-4864BDC1514A}.Debug|x64.ActiveCfg = Debug|x64 + {8D6BB292-9E1C-413D-9F98-4864BDC1514A}.Debug|x64.Build.0 = Debug|x64 + {8D6BB292-9E1C-413D-9F98-4864BDC1514A}.Debug|x86.ActiveCfg = Debug|Win32 + {8D6BB292-9E1C-413D-9F98-4864BDC1514A}.Debug|x86.Build.0 = Debug|Win32 + {8D6BB292-9E1C-413D-9F98-4864BDC1514A}.Debug-static|x64.ActiveCfg = Debug-static|x64 + {8D6BB292-9E1C-413D-9F98-4864BDC1514A}.Debug-static|x64.Build.0 = Debug-static|x64 + {8D6BB292-9E1C-413D-9F98-4864BDC1514A}.Debug-static|x86.ActiveCfg = Debug-static|Win32 + {8D6BB292-9E1C-413D-9F98-4864BDC1514A}.Debug-static|x86.Build.0 = Debug-static|Win32 + {8D6BB292-9E1C-413D-9F98-4864BDC1514A}.Release|x64.ActiveCfg = Release|x64 + {8D6BB292-9E1C-413D-9F98-4864BDC1514A}.Release|x64.Build.0 = Release|x64 + {8D6BB292-9E1C-413D-9F98-4864BDC1514A}.Release|x86.ActiveCfg = Release|Win32 + {8D6BB292-9E1C-413D-9F98-4864BDC1514A}.Release|x86.Build.0 = Release|Win32 + {8D6BB292-9E1C-413D-9F98-4864BDC1514A}.Release-static|x64.ActiveCfg = Release-static|x64 + {8D6BB292-9E1C-413D-9F98-4864BDC1514A}.Release-static|x64.Build.0 = Release-static|x64 + {8D6BB292-9E1C-413D-9F98-4864BDC1514A}.Release-static|x86.ActiveCfg = Release-static|Win32 + {8D6BB292-9E1C-413D-9F98-4864BDC1514A}.Release-static|x86.Build.0 = Release-static|Win32 + {09028CFD-4EB7-491D-869C-0708DB97ED44}.Debug|x64.ActiveCfg = Debug|x64 + {09028CFD-4EB7-491D-869C-0708DB97ED44}.Debug|x64.Build.0 = Debug|x64 + {09028CFD-4EB7-491D-869C-0708DB97ED44}.Debug|x86.ActiveCfg = Debug|Win32 + {09028CFD-4EB7-491D-869C-0708DB97ED44}.Debug|x86.Build.0 = Debug|Win32 + {09028CFD-4EB7-491D-869C-0708DB97ED44}.Debug-static|x64.ActiveCfg = Debug-static|x64 + {09028CFD-4EB7-491D-869C-0708DB97ED44}.Debug-static|x64.Build.0 = Debug-static|x64 + {09028CFD-4EB7-491D-869C-0708DB97ED44}.Debug-static|x86.ActiveCfg = Debug-static|Win32 + {09028CFD-4EB7-491D-869C-0708DB97ED44}.Debug-static|x86.Build.0 = Debug-static|Win32 + {09028CFD-4EB7-491D-869C-0708DB97ED44}.Release|x64.ActiveCfg = Release|x64 + {09028CFD-4EB7-491D-869C-0708DB97ED44}.Release|x64.Build.0 = Release|x64 + {09028CFD-4EB7-491D-869C-0708DB97ED44}.Release|x86.ActiveCfg = Release|Win32 + {09028CFD-4EB7-491D-869C-0708DB97ED44}.Release|x86.Build.0 = Release|Win32 + {09028CFD-4EB7-491D-869C-0708DB97ED44}.Release-static|x64.ActiveCfg = Release-static|x64 + {09028CFD-4EB7-491D-869C-0708DB97ED44}.Release-static|x64.Build.0 = Release-static|x64 + {09028CFD-4EB7-491D-869C-0708DB97ED44}.Release-static|x86.ActiveCfg = Release-static|Win32 + {09028CFD-4EB7-491D-869C-0708DB97ED44}.Release-static|x86.Build.0 = Release-static|Win32 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/kbe/src/lib/dependencies/jemalloc/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj b/kbe/src/lib/dependencies/jemalloc/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj index 2addd295d7..ddc6781ca8 100755 --- a/kbe/src/lib/dependencies/jemalloc/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj +++ b/kbe/src/lib/dependencies/jemalloc/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj @@ -38,16 +38,19 @@ + + - + + @@ -56,7 +59,7 @@ - + @@ -197,7 +200,7 @@ Level3 Disabled - _REENTRANT;_WINDLL;DLLEXPORT;JEMALLOC_DEBUG;_DEBUG;%(PreprocessorDefinitions) + JEMALLOC_NO_PRIVATE_NAMESPACE;_REENTRANT;_WINDLL;DLLEXPORT;JEMALLOC_DEBUG;_DEBUG;%(PreprocessorDefinitions) ..\..\..\..\include;..\..\..\..\include\msvc_compat;%(AdditionalIncludeDirectories) 4090;4146;4267;4334 $(OutputPath)$(TargetName).pdb @@ -213,7 +216,7 @@ Level3 Disabled - JEMALLOC_DEBUG;_REENTRANT;JEMALLOC_EXPORT=;_DEBUG;_LIB;%(PreprocessorDefinitions) + JEMALLOC_NO_PRIVATE_NAMESPACE;JEMALLOC_DEBUG;_REENTRANT;JEMALLOC_EXPORT=;_DEBUG;_LIB;%(PreprocessorDefinitions) ..\..\..\..\include;..\..\..\..\include\msvc_compat;%(AdditionalIncludeDirectories) MultiThreadedDebug 4090;4146;4267;4334 @@ -266,7 +269,7 @@ MaxSpeed true true - _REENTRANT;_WINDLL;DLLEXPORT;NDEBUG;%(PreprocessorDefinitions) + JEMALLOC_NO_PRIVATE_NAMESPACE;_REENTRANT;_WINDLL;DLLEXPORT;NDEBUG;%(PreprocessorDefinitions) ..\..\..\..\include;..\..\..\..\include\msvc_compat;%(AdditionalIncludeDirectories) 4090;4146;4267;4334 $(OutputPath)$(TargetName).pdb @@ -286,7 +289,7 @@ MaxSpeed true true - _REENTRANT;JEMALLOC_EXPORT=;NDEBUG;_LIB;%(PreprocessorDefinitions) + JEMALLOC_NO_PRIVATE_NAMESPACE;_REENTRANT;JEMALLOC_EXPORT=;NDEBUG;_LIB;%(PreprocessorDefinitions) ..\..\..\..\include;..\..\..\..\include\msvc_compat;%(AdditionalIncludeDirectories) MultiThreaded 4090;4146;4267;4334 diff --git a/kbe/src/lib/dependencies/jemalloc/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters b/kbe/src/lib/dependencies/jemalloc/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters index 4edf09b449..1dcf4ed542 100755 --- a/kbe/src/lib/dependencies/jemalloc/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters +++ b/kbe/src/lib/dependencies/jemalloc/msvc/projects/vc2015/jemalloc/jemalloc.vcxproj.filters @@ -37,7 +37,7 @@ Source Files - + Source Files @@ -70,7 +70,7 @@ Source Files - + Source Files @@ -91,5 +91,14 @@ Source Files + + Source Files + + + Source Files + + + Source Files + \ No newline at end of file diff --git a/kbe/src/lib/dependencies/jemalloc/msvc/projects/vc2015/test_threads/test_threads.vcxproj b/kbe/src/lib/dependencies/jemalloc/msvc/projects/vc2015/test_threads/test_threads.vcxproj index f5e9898f29..325876d6e8 100755 --- a/kbe/src/lib/dependencies/jemalloc/msvc/projects/vc2015/test_threads/test_threads.vcxproj +++ b/kbe/src/lib/dependencies/jemalloc/msvc/projects/vc2015/test_threads/test_threads.vcxproj @@ -310,8 +310,8 @@ - - + + @@ -319,7 +319,7 @@ - + diff --git a/kbe/src/lib/dependencies/jemalloc/msvc/projects/vc2015/test_threads/test_threads.vcxproj.filters b/kbe/src/lib/dependencies/jemalloc/msvc/projects/vc2015/test_threads/test_threads.vcxproj.filters index 4c23340738..fa4588fd87 100755 --- a/kbe/src/lib/dependencies/jemalloc/msvc/projects/vc2015/test_threads/test_threads.vcxproj.filters +++ b/kbe/src/lib/dependencies/jemalloc/msvc/projects/vc2015/test_threads/test_threads.vcxproj.filters @@ -11,15 +11,15 @@ - + Source Files - + Source Files - + Header Files diff --git a/kbe/src/lib/dependencies/jemalloc/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj b/kbe/src/lib/dependencies/jemalloc/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj new file mode 100644 index 0000000000..21481d5ee3 --- /dev/null +++ b/kbe/src/lib/dependencies/jemalloc/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj @@ -0,0 +1,349 @@ + + + + + Debug-static + Win32 + + + Debug-static + x64 + + + Debug + Win32 + + + Release-static + Win32 + + + Release-static + x64 + + + Release + Win32 + + + Debug + x64 + + + Release + x64 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + {8D6BB292-9E1C-413D-9F98-4864BDC1514A} + Win32Proj + jemalloc + + + + DynamicLibrary + true + v141 + MultiByte + + + StaticLibrary + true + v141 + MultiByte + + + DynamicLibrary + false + v141 + true + MultiByte + + + StaticLibrary + false + v141 + true + MultiByte + + + DynamicLibrary + true + v141 + MultiByte + + + StaticLibrary + true + v141 + MultiByte + + + DynamicLibrary + false + v141 + true + MultiByte + + + StaticLibrary + false + v141 + true + MultiByte + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $(SolutionDir)$(Platform)\$(Configuration)\ + $(Platform)\$(Configuration)\ + $(ProjectName)d + + + $(SolutionDir)$(Platform)\$(Configuration)\ + $(Platform)\$(Configuration)\ + $(ProjectName)-$(PlatformToolset)-$(Configuration) + + + $(SolutionDir)$(Platform)\$(Configuration)\ + $(Platform)\$(Configuration)\ + + + $(SolutionDir)$(Platform)\$(Configuration)\ + $(Platform)\$(Configuration)\ + $(ProjectName)-$(PlatformToolset)-$(Configuration) + + + $(SolutionDir)$(Platform)\$(Configuration)\ + $(Platform)\$(Configuration)\ + $(ProjectName)d + + + $(SolutionDir)$(Platform)\$(Configuration)\ + $(Platform)\$(Configuration)\ + $(ProjectName)-vc$(PlatformToolsetVersion)-$(Configuration) + + + $(SolutionDir)$(Platform)\$(Configuration)\ + $(Platform)\$(Configuration)\ + + + $(SolutionDir)$(Platform)\$(Configuration)\ + $(Platform)\$(Configuration)\ + $(ProjectName)-vc$(PlatformToolsetVersion)-$(Configuration) + + + + + + Level3 + Disabled + _REENTRANT;_WINDLL;DLLEXPORT;JEMALLOC_DEBUG;_DEBUG;%(PreprocessorDefinitions) + ..\..\..\..\include;..\..\..\..\include\msvc_compat;%(AdditionalIncludeDirectories) + 4090;4146;4267;4334 + $(OutputPath)$(TargetName).pdb + + + Windows + true + + + + + + + Level3 + Disabled + JEMALLOC_DEBUG;_REENTRANT;JEMALLOC_EXPORT=;_DEBUG;_LIB;%(PreprocessorDefinitions) + ..\..\..\..\include;..\..\..\..\include\msvc_compat;%(AdditionalIncludeDirectories) + MultiThreadedDebug + 4090;4146;4267;4334 + $(OutputPath)$(TargetName).pdb + + + Windows + true + + + + + + + Level3 + Disabled + JEMALLOC_NO_PRIVATE_NAMESPACE;_REENTRANT;_WINDLL;DLLEXPORT;JEMALLOC_DEBUG;_DEBUG;%(PreprocessorDefinitions) + ..\..\..\..\include;..\..\..\..\include\msvc_compat;%(AdditionalIncludeDirectories) + 4090;4146;4267;4334 + $(OutputPath)$(TargetName).pdb + + + Windows + true + + + + + + + Level3 + Disabled + JEMALLOC_NO_PRIVATE_NAMESPACE;JEMALLOC_DEBUG;_REENTRANT;JEMALLOC_EXPORT=;_DEBUG;_LIB;%(PreprocessorDefinitions) + ..\..\..\..\include;..\..\..\..\include\msvc_compat;%(AdditionalIncludeDirectories) + MultiThreadedDebug + 4090;4146;4267;4334 + OldStyle + false + + + Windows + true + + + + + Level3 + + + MaxSpeed + true + true + _REENTRANT;_WINDLL;DLLEXPORT;NDEBUG;%(PreprocessorDefinitions) + ..\..\..\..\include;..\..\..\..\include\msvc_compat;%(AdditionalIncludeDirectories) + 4090;4146;4267;4334 + $(OutputPath)$(TargetName).pdb + + + Windows + true + true + true + + + + + Level3 + + + MaxSpeed + true + true + _REENTRANT;JEMALLOC_EXPORT=;NDEBUG;_LIB;%(PreprocessorDefinitions) + ..\..\..\..\include;..\..\..\..\include\msvc_compat;%(AdditionalIncludeDirectories) + MultiThreaded + 4090;4146;4267;4334 + $(OutputPath)$(TargetName).pdb + + + Windows + true + true + true + + + + + Level3 + + + MaxSpeed + true + true + ..\..\..\..\include;..\..\..\..\include\msvc_compat;%(AdditionalIncludeDirectories) + JEMALLOC_NO_PRIVATE_NAMESPACE;_REENTRANT;_WINDLL;DLLEXPORT;NDEBUG;%(PreprocessorDefinitions) + 4090;4146;4267;4334 + $(OutputPath)$(TargetName).pdb + + + Windows + true + true + true + + + + + Level3 + + + MaxSpeed + true + true + JEMALLOC_NO_PRIVATE_NAMESPACE;_REENTRANT;JEMALLOC_EXPORT=;NDEBUG;_LIB;%(PreprocessorDefinitions) + ..\..\..\..\include;..\..\..\..\include\msvc_compat;%(AdditionalIncludeDirectories) + MultiThreaded + 4090;4146;4267;4334 + OldStyle + + + Windows + true + true + true + + + + + + \ No newline at end of file diff --git a/kbe/src/lib/dependencies/jemalloc/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters b/kbe/src/lib/dependencies/jemalloc/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters new file mode 100644 index 0000000000..466dc63fa3 --- /dev/null +++ b/kbe/src/lib/dependencies/jemalloc/msvc/projects/vc2017/jemalloc/jemalloc.vcxproj.filters @@ -0,0 +1,107 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + Source Files + + + \ No newline at end of file diff --git a/kbe/src/lib/dependencies/jemalloc/msvc/projects/vc2017/test_threads/test_threads.vcxproj b/kbe/src/lib/dependencies/jemalloc/msvc/projects/vc2017/test_threads/test_threads.vcxproj new file mode 100644 index 0000000000..c35b0f5aab --- /dev/null +++ b/kbe/src/lib/dependencies/jemalloc/msvc/projects/vc2017/test_threads/test_threads.vcxproj @@ -0,0 +1,326 @@ + + + + + Debug-static + Win32 + + + Debug-static + x64 + + + Debug + Win32 + + + Release-static + Win32 + + + Release-static + x64 + + + Release + Win32 + + + Debug + x64 + + + Release + x64 + + + + {09028CFD-4EB7-491D-869C-0708DB97ED44} + Win32Proj + test_threads + + + + Application + true + v141 + MultiByte + + + Application + true + v141 + MultiByte + + + Application + false + v141 + true + MultiByte + + + Application + false + v141 + true + MultiByte + + + Application + true + v141 + MultiByte + + + Application + true + v141 + MultiByte + + + Application + false + v141 + true + MultiByte + + + Application + false + v141 + true + MultiByte + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $(SolutionDir)$(Platform)\$(Configuration)\ + $(Platform)\$(Configuration)\ + true + + + $(SolutionDir)$(Platform)\$(Configuration)\ + $(Platform)\$(Configuration)\ + true + + + true + $(SolutionDir)$(Platform)\$(Configuration)\ + + + true + $(SolutionDir)$(Platform)\$(Configuration)\ + + + $(SolutionDir)$(Platform)\$(Configuration)\ + $(Platform)\$(Configuration)\ + false + + + $(SolutionDir)$(Platform)\$(Configuration)\ + $(Platform)\$(Configuration)\ + false + + + $(SolutionDir)$(Platform)\$(Configuration)\ + $(Platform)\$(Configuration)\ + false + + + $(SolutionDir)$(Platform)\$(Configuration)\ + $(Platform)\$(Configuration)\ + false + + + + + + Level3 + Disabled + WIN32;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + ..\..\..\..\test\include;..\..\..\..\include;..\..\..\..\include\msvc_compat;%(AdditionalIncludeDirectories) + + + Console + true + $(SolutionDir)$(Platform)\$(Configuration) + jemallocd.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies) + + + + + + + Level3 + Disabled + JEMALLOC_EXPORT=;JEMALLOC_STATIC;_DEBUG;_CONSOLE;%(PreprocessorDefinitions) + ..\..\..\..\test\include;..\..\..\..\include;..\..\..\..\include\msvc_compat;%(AdditionalIncludeDirectories) + MultiThreadedDebug + + + Console + true + $(SolutionDir)$(Platform)\$(Configuration) + jemalloc-$(PlatformToolset)-$(Configuration).lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies) + + + + + + + Level3 + Disabled + _DEBUG;%(PreprocessorDefinitions) + ..\..\..\..\test\include;..\..\..\..\include;..\..\..\..\include\msvc_compat;%(AdditionalIncludeDirectories) + + + Console + true + jemallocd.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies) + $(SolutionDir)$(Platform)\$(Configuration) + + + + + + + Level3 + Disabled + JEMALLOC_EXPORT=;JEMALLOC_STATIC;_DEBUG;%(PreprocessorDefinitions) + ..\..\..\..\test\include;..\..\..\..\include;..\..\..\..\include\msvc_compat;%(AdditionalIncludeDirectories) + MultiThreadedDebug + + + Console + true + jemalloc-vc$(PlatformToolsetVersion)-$(Configuration).lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies) + $(SolutionDir)$(Platform)\$(Configuration) + + + + + Level3 + + + MaxSpeed + true + true + WIN32;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + ..\..\..\..\test\include;..\..\..\..\include;..\..\..\..\include\msvc_compat;%(AdditionalIncludeDirectories) + + + Console + true + true + true + $(SolutionDir)$(Platform)\$(Configuration) + jemalloc.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies) + + + + + Level3 + + + MaxSpeed + true + true + JEMALLOC_EXPORT=;JEMALLOC_STATIC;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + ..\..\..\..\test\include;..\..\..\..\include;..\..\..\..\include\msvc_compat;%(AdditionalIncludeDirectories) + MultiThreaded + + + Console + true + true + true + $(SolutionDir)$(Platform)\$(Configuration) + jemalloc-$(PlatformToolset)-$(Configuration).lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies) + + + + + Level3 + + + MaxSpeed + true + true + NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + ..\..\..\..\test\include;..\..\..\..\include;..\..\..\..\include\msvc_compat;%(AdditionalIncludeDirectories) + + + Console + true + true + true + $(SolutionDir)$(Platform)\$(Configuration) + jemalloc.lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies) + + + + + Level3 + + + MaxSpeed + true + true + JEMALLOC_EXPORT=;JEMALLOC_STATIC;NDEBUG;_CONSOLE;%(PreprocessorDefinitions) + ..\..\..\..\test\include;..\..\..\..\include;..\..\..\..\include\msvc_compat;%(AdditionalIncludeDirectories) + MultiThreaded + + + Console + true + true + true + $(SolutionDir)$(Platform)\$(Configuration) + jemalloc-vc$(PlatformToolsetVersion)-$(Configuration).lib;kernel32.lib;user32.lib;gdi32.lib;winspool.lib;comdlg32.lib;advapi32.lib;shell32.lib;ole32.lib;oleaut32.lib;uuid.lib;odbc32.lib;odbccp32.lib;%(AdditionalDependencies) + + + + + + + + + {8d6bb292-9e1c-413d-9f98-4864bdc1514a} + + + + + + + + + \ No newline at end of file diff --git a/kbe/src/lib/dependencies/jemalloc/msvc/projects/vc2017/test_threads/test_threads.vcxproj.filters b/kbe/src/lib/dependencies/jemalloc/msvc/projects/vc2017/test_threads/test_threads.vcxproj.filters new file mode 100644 index 0000000000..fa4588fd87 --- /dev/null +++ b/kbe/src/lib/dependencies/jemalloc/msvc/projects/vc2017/test_threads/test_threads.vcxproj.filters @@ -0,0 +1,26 @@ + + + + + {4FC737F1-C7A5-4376-A066-2A32D752A2FF} + cpp;c;cc;cxx;def;odl;idl;hpj;bat;asm;asmx + + + {93995380-89BD-4b04-88EB-625FBE52EBFB} + h;hh;hpp;hxx;hm;inl;inc;xsd + + + + + Source Files + + + Source Files + + + + + Header Files + + + \ No newline at end of file diff --git a/kbe/src/lib/dependencies/jemalloc/msvc/projects/vc2015/test_threads/test_threads.cpp b/kbe/src/lib/dependencies/jemalloc/msvc/test_threads/test_threads.cpp old mode 100755 new mode 100644 similarity index 100% rename from kbe/src/lib/dependencies/jemalloc/msvc/projects/vc2015/test_threads/test_threads.cpp rename to kbe/src/lib/dependencies/jemalloc/msvc/test_threads/test_threads.cpp diff --git a/kbe/src/lib/dependencies/jemalloc/msvc/projects/vc2015/test_threads/test_threads.h b/kbe/src/lib/dependencies/jemalloc/msvc/test_threads/test_threads.h old mode 100755 new mode 100644 similarity index 100% rename from kbe/src/lib/dependencies/jemalloc/msvc/projects/vc2015/test_threads/test_threads.h rename to kbe/src/lib/dependencies/jemalloc/msvc/test_threads/test_threads.h diff --git a/kbe/src/lib/dependencies/jemalloc/msvc/projects/vc2015/test_threads/test_threads_main.cpp b/kbe/src/lib/dependencies/jemalloc/msvc/test_threads/test_threads_main.cpp old mode 100755 new mode 100644 similarity index 100% rename from kbe/src/lib/dependencies/jemalloc/msvc/projects/vc2015/test_threads/test_threads_main.cpp rename to kbe/src/lib/dependencies/jemalloc/msvc/test_threads/test_threads_main.cpp diff --git a/kbe/src/lib/dependencies/jemalloc/scripts/gen_run_tests.py b/kbe/src/lib/dependencies/jemalloc/scripts/gen_run_tests.py index ddf215335b..5052b3e0d3 100755 --- a/kbe/src/lib/dependencies/jemalloc/scripts/gen_run_tests.py +++ b/kbe/src/lib/dependencies/jemalloc/scripts/gen_run_tests.py @@ -1,20 +1,38 @@ #!/usr/bin/env python +import sys from itertools import combinations from os import uname from multiprocessing import cpu_count +from subprocess import call + +# Later, we want to test extended vaddr support. Apparently, the "real" way of +# checking this is flaky on OS X. +bits_64 = sys.maxsize > 2**32 nparallel = cpu_count() * 2 uname = uname()[0] +if "BSD" in uname: + make_cmd = 'gmake' +else: + make_cmd = 'make' + def powerset(items): result = [] for i in xrange(len(items) + 1): result += combinations(items, i) return result -possible_compilers = [('gcc', 'g++'), ('clang', 'clang++')] +possible_compilers = [] +for cc, cxx in (['gcc', 'g++'], ['clang', 'clang++']): + try: + cmd_ret = call([cc, "-v"]) + if cmd_ret == 0: + possible_compilers.append((cc, cxx)) + except: + pass possible_compiler_opts = [ '-m32', ] @@ -22,8 +40,10 @@ def powerset(items): '--enable-debug', '--enable-prof', '--disable-stats', - '--with-malloc-conf=tcache:false', ] +if bits_64: + possible_config_opts.append('--with-lg-vaddr=56') + possible_malloc_conf_opts = [ 'tcache:false', 'dss:primary', @@ -32,7 +52,7 @@ def powerset(items): ] print 'set -e' -print 'if [ -f Makefile ] ; then make relclean ; fi' +print 'if [ -f Makefile ] ; then %(make_cmd)s relclean ; fi' % {'make_cmd': make_cmd} print 'autoconf' print 'rm -rf run_tests.out' print 'mkdir run_tests.out' @@ -57,6 +77,11 @@ def powerset(items): else '') ) + # We don't want to test large vaddr spaces in 32-bit mode. + if ('-m32' in compiler_opts and '--with-lg-vaddr=56' in + config_opts): + continue + # Per CPU arenas are only supported on Linux. linux_supported = ('percpu_arena:percpu' in malloc_conf_opts \ or 'background_thread:true' in malloc_conf_opts) @@ -90,11 +115,11 @@ def powerset(items): echo "==> %(config_line)s" >> run_test.log %(config_line)s >> run_test.log 2>&1 || abort -run_cmd make all tests -run_cmd make check -run_cmd make distclean +run_cmd %(make_cmd)s all tests +run_cmd %(make_cmd)s check +run_cmd %(make_cmd)s distclean EOF -chmod 755 run_test_%(ind)d.sh""" % {'ind': ind, 'config_line': config_line} +chmod 755 run_test_%(ind)d.sh""" % {'ind': ind, 'config_line': config_line, 'make_cmd': make_cmd} ind += 1 print 'for i in `seq 0 %(last_ind)d` ; do echo run_test_${i}.sh ; done | xargs -P %(nparallel)d -n 1 sh' % {'last_ind': ind-1, 'nparallel': nparallel} diff --git a/kbe/src/lib/dependencies/jemalloc/scripts/gen_travis.py b/kbe/src/lib/dependencies/jemalloc/scripts/gen_travis.py index 6dd39290c9..65b0b67c7d 100755 --- a/kbe/src/lib/dependencies/jemalloc/scripts/gen_travis.py +++ b/kbe/src/lib/dependencies/jemalloc/scripts/gen_travis.py @@ -4,6 +4,7 @@ travis_template = """\ language: generic +dist: precise matrix: include: @@ -11,6 +12,7 @@ before_script: - autoconf + - scripts/gen_travis.py > travis_script && diff .travis.yml travis_script - ./configure ${COMPILER_FLAGS:+ \ CC="$CC $COMPILER_FLAGS" \ CXX="$CXX $COMPILER_FLAGS" } \ @@ -43,6 +45,7 @@ '--enable-debug', '--enable-prof', '--disable-stats', + '--disable-libdl', ] malloc_conf_unusuals = [ @@ -61,47 +64,85 @@ for i in xrange(MAX_UNUSUAL_OPTIONS + 1): unusual_combinations_to_test += combinations(all_unusuals, i) -include_rows = "" -for unusual_combination in unusual_combinations_to_test: - os = os_default - if os_unusual in unusual_combination: - os = os_unusual - - compilers = compilers_default - if compilers_unusual in unusual_combination: - compilers = compilers_unusual +gcc_multilib_set = False +# Formats a job from a combination of flags +def format_job(combination): + global gcc_multilib_set - compiler_flags = [ - x for x in unusual_combination if x in compiler_flag_unusuals] + os = os_unusual if os_unusual in combination else os_default + compilers = compilers_unusual if compilers_unusual in combination else compilers_default - configure_flags = [ - x for x in unusual_combination if x in configure_flag_unusuals] + compiler_flags = [x for x in combination if x in compiler_flag_unusuals] + configure_flags = [x for x in combination if x in configure_flag_unusuals] + malloc_conf = [x for x in combination if x in malloc_conf_unusuals] - malloc_conf = [ - x for x in unusual_combination if x in malloc_conf_unusuals] # Filter out unsupported configurations on OS X. if os == 'osx' and ('dss:primary' in malloc_conf or \ 'percpu_arena:percpu' in malloc_conf or 'background_thread:true' \ in malloc_conf): - continue + return "" if len(malloc_conf) > 0: configure_flags.append('--with-malloc-conf=' + ",".join(malloc_conf)) # Filter out an unsupported configuration - heap profiling on OS X. if os == 'osx' and '--enable-prof' in configure_flags: - continue + return "" # We get some spurious errors when -Warray-bounds is enabled. env_string = ('{} COMPILER_FLAGS="{}" CONFIGURE_FLAGS="{}" ' 'EXTRA_CFLAGS="-Werror -Wno-array-bounds"').format( compilers, " ".join(compiler_flags), " ".join(configure_flags)) - include_rows += ' - os: %s\n' % os - include_rows += ' env: %s\n' % env_string - if '-m32' in unusual_combination and os == 'linux': - include_rows += ' addons:\n' - include_rows += ' apt:\n' - include_rows += ' packages:\n' - include_rows += ' - gcc-multilib\n' + job = "" + job += ' - os: %s\n' % os + job += ' env: %s\n' % env_string + if '-m32' in combination and os == 'linux': + job += ' addons:' + if gcc_multilib_set: + job += ' *gcc_multilib\n' + else: + job += ' &gcc_multilib\n' + job += ' apt:\n' + job += ' packages:\n' + job += ' - gcc-multilib\n' + gcc_multilib_set = True + return job + +include_rows = "" +for combination in unusual_combinations_to_test: + include_rows += format_job(combination) + +# Development build +include_rows += '''\ + # Development build + - os: linux + env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --disable-cache-oblivious --enable-stats --enable-log --enable-prof" EXTRA_CFLAGS="-Werror -Wno-array-bounds" +''' + +# Enable-expermental-smallocx +include_rows += '''\ + # --enable-expermental-smallocx: + - os: linux + env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="--enable-debug --enable-experimental-smallocx --enable-stats --enable-prof" EXTRA_CFLAGS="-Werror -Wno-array-bounds" +''' + +# Valgrind build bots +include_rows += ''' + # Valgrind + - os: linux + env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="" EXTRA_CFLAGS="-Werror -Wno-array-bounds" JEMALLOC_TEST_PREFIX="valgrind" + addons: + apt: + packages: + - valgrind +''' + +# To enable valgrind on macosx add: +# +# - os: osx +# env: CC=gcc CXX=g++ COMPILER_FLAGS="" CONFIGURE_FLAGS="" EXTRA_CFLAGS="-Werror -Wno-array-bounds" JEMALLOC_TEST_PREFIX="valgrind" +# install: brew install valgrind +# +# It currently fails due to: https://github.com/jemalloc/jemalloc/issues/1274 print travis_template % include_rows diff --git a/kbe/src/lib/dependencies/jemalloc/src/arena.c b/kbe/src/lib/dependencies/jemalloc/src/arena.c index 632fce5233..60eac232a9 100755 --- a/kbe/src/lib/dependencies/jemalloc/src/arena.c +++ b/kbe/src/lib/dependencies/jemalloc/src/arena.c @@ -3,13 +3,15 @@ #include "jemalloc/internal/jemalloc_internal_includes.h" #include "jemalloc/internal/assert.h" +#include "jemalloc/internal/div.h" #include "jemalloc/internal/extent_dss.h" #include "jemalloc/internal/extent_mmap.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/rtree.h" -#include "jemalloc/internal/size_classes.h" #include "jemalloc/internal/util.h" +JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS + /******************************************************************************/ /* Data. */ @@ -32,21 +34,6 @@ ssize_t opt_muzzy_decay_ms = MUZZY_DECAY_MS_DEFAULT; static atomic_zd_t dirty_decay_ms_default; static atomic_zd_t muzzy_decay_ms_default; -const arena_bin_info_t arena_bin_info[NBINS] = { -#define BIN_INFO_bin_yes(reg_size, slab_size, nregs) \ - {reg_size, slab_size, nregs, BITMAP_INFO_INITIALIZER(nregs)}, -#define BIN_INFO_bin_no(reg_size, slab_size, nregs) -#define SC(index, lg_grp, lg_delta, ndelta, psz, bin, pgs, \ - lg_delta_lookup) \ - BIN_INFO_bin_##bin((1U<mtx, "arena_stats", - WITNESS_RANK_ARENA_STATS, malloc_mutex_rank_exclusive)) { - return true; - } -#endif - /* Memory is zeroed, so there is no need to clear stats. */ - return false; -} - -static void -arena_stats_lock(tsdn_t *tsdn, arena_stats_t *arena_stats) { -#ifndef JEMALLOC_ATOMIC_U64 - malloc_mutex_lock(tsdn, &arena_stats->mtx); -#endif -} - -static void -arena_stats_unlock(tsdn_t *tsdn, arena_stats_t *arena_stats) { -#ifndef JEMALLOC_ATOMIC_U64 - malloc_mutex_unlock(tsdn, &arena_stats->mtx); -#endif -} - -static uint64_t -arena_stats_read_u64(tsdn_t *tsdn, arena_stats_t *arena_stats, - arena_stats_u64_t *p) { -#ifdef JEMALLOC_ATOMIC_U64 - return atomic_load_u64(p, ATOMIC_RELAXED); -#else - malloc_mutex_assert_owner(tsdn, &arena_stats->mtx); - return *p; -#endif -} - -static void -arena_stats_add_u64(tsdn_t *tsdn, arena_stats_t *arena_stats, - arena_stats_u64_t *p, uint64_t x) { -#ifdef JEMALLOC_ATOMIC_U64 - atomic_fetch_add_u64(p, x, ATOMIC_RELAXED); -#else - malloc_mutex_assert_owner(tsdn, &arena_stats->mtx); - *p += x; -#endif -} - -UNUSED static void -arena_stats_sub_u64(tsdn_t *tsdn, arena_stats_t *arena_stats, - arena_stats_u64_t *p, uint64_t x) { -#ifdef JEMALLOC_ATOMIC_U64 - UNUSED uint64_t r = atomic_fetch_sub_u64(p, x, ATOMIC_RELAXED); - assert(r - x <= r); -#else - malloc_mutex_assert_owner(tsdn, &arena_stats->mtx); - *p -= x; - assert(*p + x >= *p); -#endif -} - -/* - * Non-atomically sets *dst += src. *dst needs external synchronization. - * This lets us avoid the cost of a fetch_add when its unnecessary (note that - * the types here are atomic). - */ -static void -arena_stats_accum_u64(arena_stats_u64_t *dst, uint64_t src) { -#ifdef JEMALLOC_ATOMIC_U64 - uint64_t cur_dst = atomic_load_u64(dst, ATOMIC_RELAXED); - atomic_store_u64(dst, src + cur_dst, ATOMIC_RELAXED); -#else - *dst += src; -#endif -} - -static size_t -arena_stats_read_zu(tsdn_t *tsdn, arena_stats_t *arena_stats, atomic_zu_t *p) { -#ifdef JEMALLOC_ATOMIC_U64 - return atomic_load_zu(p, ATOMIC_RELAXED); -#else - malloc_mutex_assert_owner(tsdn, &arena_stats->mtx); - return atomic_load_zu(p, ATOMIC_RELAXED); -#endif -} - -static void -arena_stats_add_zu(tsdn_t *tsdn, arena_stats_t *arena_stats, atomic_zu_t *p, - size_t x) { -#ifdef JEMALLOC_ATOMIC_U64 - atomic_fetch_add_zu(p, x, ATOMIC_RELAXED); -#else - malloc_mutex_assert_owner(tsdn, &arena_stats->mtx); - size_t cur = atomic_load_zu(p, ATOMIC_RELAXED); - atomic_store_zu(p, cur + x, ATOMIC_RELAXED); -#endif -} - -static void -arena_stats_sub_zu(tsdn_t *tsdn, arena_stats_t *arena_stats, atomic_zu_t *p, - size_t x) { -#ifdef JEMALLOC_ATOMIC_U64 - UNUSED size_t r = atomic_fetch_sub_zu(p, x, ATOMIC_RELAXED); - assert(r - x <= r); -#else - malloc_mutex_assert_owner(tsdn, &arena_stats->mtx); - size_t cur = atomic_load_zu(p, ATOMIC_RELAXED); - atomic_store_zu(p, cur - x, ATOMIC_RELAXED); -#endif -} - -/* Like the _u64 variant, needs an externally synchronized *dst. */ -static void -arena_stats_accum_zu(atomic_zu_t *dst, size_t src) { - size_t cur_dst = atomic_load_zu(dst, ATOMIC_RELAXED); - atomic_store_zu(dst, src + cur_dst, ATOMIC_RELAXED); -} - -void -arena_stats_large_nrequests_add(tsdn_t *tsdn, arena_stats_t *arena_stats, - szind_t szind, uint64_t nrequests) { - arena_stats_lock(tsdn, arena_stats); - arena_stats_add_u64(tsdn, arena_stats, &arena_stats->lstats[szind - - NBINS].nrequests, nrequests); - arena_stats_unlock(tsdn, arena_stats); -} - -void -arena_stats_mapped_add(tsdn_t *tsdn, arena_stats_t *arena_stats, size_t size) { - arena_stats_lock(tsdn, arena_stats); - arena_stats_add_zu(tsdn, arena_stats, &arena_stats->mapped, size); - arena_stats_unlock(tsdn, arena_stats); -} - void arena_basic_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms, @@ -228,15 +82,16 @@ void arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, const char **dss, ssize_t *dirty_decay_ms, ssize_t *muzzy_decay_ms, size_t *nactive, size_t *ndirty, size_t *nmuzzy, arena_stats_t *astats, - malloc_bin_stats_t *bstats, malloc_large_stats_t *lstats) { + bin_stats_t *bstats, arena_stats_large_t *lstats, + arena_stats_extents_t *estats) { cassert(config_stats); arena_basic_stats_merge(tsdn, arena, nthreads, dss, dirty_decay_ms, muzzy_decay_ms, nactive, ndirty, nmuzzy); - size_t base_allocated, base_resident, base_mapped; + size_t base_allocated, base_resident, base_mapped, metadata_thp; base_stats_get(tsdn, arena->base, &base_allocated, &base_resident, - &base_mapped); + &base_mapped, &metadata_thp); arena_stats_lock(tsdn, &arena->stats); @@ -245,6 +100,10 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, arena_stats_accum_zu(&astats->retained, extents_npages_get(&arena->extents_retained) << LG_PAGE); + atomic_store_zu(&astats->extent_avail, + atomic_load_zu(&arena->extent_avail_cnt, ATOMIC_RELAXED), + ATOMIC_RELAXED); + arena_stats_accum_u64(&astats->decay_dirty.npurge, arena_stats_read_u64(tsdn, &arena->stats, &arena->stats.decay_dirty.npurge)); @@ -267,12 +126,13 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, arena_stats_accum_zu(&astats->base, base_allocated); arena_stats_accum_zu(&astats->internal, arena_internal_get(arena)); + arena_stats_accum_zu(&astats->metadata_thp, metadata_thp); arena_stats_accum_zu(&astats->resident, base_resident + (((atomic_load_zu(&arena->nactive, ATOMIC_RELAXED) + extents_npages_get(&arena->extents_dirty) + extents_npages_get(&arena->extents_muzzy)) << LG_PAGE))); - for (szind_t i = 0; i < NSIZES - NBINS; i++) { + for (szind_t i = 0; i < SC_NSIZES - SC_NBINS; i++) { uint64_t nmalloc = arena_stats_read_u64(tsdn, &arena->stats, &arena->stats.lstats[i].nmalloc); arena_stats_accum_u64(&lstats[i].nmalloc, nmalloc); @@ -295,7 +155,29 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, size_t curlextents = (size_t)(nmalloc - ndalloc); lstats[i].curlextents += curlextents; arena_stats_accum_zu(&astats->allocated_large, - curlextents * sz_index2size(NBINS + i)); + curlextents * sz_index2size(SC_NBINS + i)); + } + + for (pszind_t i = 0; i < SC_NPSIZES; i++) { + size_t dirty, muzzy, retained, dirty_bytes, muzzy_bytes, + retained_bytes; + dirty = extents_nextents_get(&arena->extents_dirty, i); + muzzy = extents_nextents_get(&arena->extents_muzzy, i); + retained = extents_nextents_get(&arena->extents_retained, i); + dirty_bytes = extents_nbytes_get(&arena->extents_dirty, i); + muzzy_bytes = extents_nbytes_get(&arena->extents_muzzy, i); + retained_bytes = + extents_nbytes_get(&arena->extents_retained, i); + + atomic_store_zu(&estats[i].ndirty, dirty, ATOMIC_RELAXED); + atomic_store_zu(&estats[i].nmuzzy, muzzy, ATOMIC_RELAXED); + atomic_store_zu(&estats[i].nretained, retained, ATOMIC_RELAXED); + atomic_store_zu(&estats[i].dirty_bytes, dirty_bytes, + ATOMIC_RELAXED); + atomic_store_zu(&estats[i].muzzy_bytes, muzzy_bytes, + ATOMIC_RELAXED); + atomic_store_zu(&estats[i].retained_bytes, retained_bytes, + ATOMIC_RELAXED); } arena_stats_unlock(tsdn, &arena->stats); @@ -303,16 +185,16 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, /* tcache_bytes counts currently cached bytes. */ atomic_store_zu(&astats->tcache_bytes, 0, ATOMIC_RELAXED); malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx); - tcache_t *tcache; - ql_foreach(tcache, &arena->tcache_ql, link) { + cache_bin_array_descriptor_t *descriptor; + ql_foreach(descriptor, &arena->cache_bin_array_descriptor_ql, link) { szind_t i = 0; - for (; i < NBINS; i++) { - tcache_bin_t *tbin = tcache_small_bin_get(tcache, i); + for (; i < SC_NBINS; i++) { + cache_bin_t *tbin = &descriptor->bins_small[i]; arena_stats_accum_zu(&astats->tcache_bytes, tbin->ncached * sz_index2size(i)); } for (; i < nhbins; i++) { - tcache_bin_t *tbin = tcache_large_bin_get(tcache, i); + cache_bin_t *tbin = &descriptor->bins_large[i]; arena_stats_accum_zu(&astats->tcache_bytes, tbin->ncached * sz_index2size(i)); } @@ -350,21 +232,11 @@ arena_stats_merge(tsdn_t *tsdn, arena_t *arena, unsigned *nthreads, nstime_update(&astats->uptime); nstime_subtract(&astats->uptime, &arena->create_time); - for (szind_t i = 0; i < NBINS; i++) { - arena_bin_t *bin = &arena->bins[i]; - - malloc_mutex_lock(tsdn, &bin->lock); - malloc_mutex_prof_read(tsdn, &bstats[i].mutex_data, &bin->lock); - bstats[i].nmalloc += bin->stats.nmalloc; - bstats[i].ndalloc += bin->stats.ndalloc; - bstats[i].nrequests += bin->stats.nrequests; - bstats[i].curregs += bin->stats.curregs; - bstats[i].nfills += bin->stats.nfills; - bstats[i].nflushes += bin->stats.nflushes; - bstats[i].nslabs += bin->stats.nslabs; - bstats[i].reslabs += bin->stats.reslabs; - bstats[i].curslabs += bin->stats.curslabs; - malloc_mutex_unlock(tsdn, &bin->lock); + for (szind_t i = 0; i < SC_NBINS; i++) { + for (unsigned j = 0; j < bin_infos[i].n_shards; j++) { + bin_stats_merge(tsdn, &bstats[i], + &arena->bins[i].bin_shards[j]); + } } } @@ -384,8 +256,7 @@ arena_extents_dirty_dalloc(tsdn_t *tsdn, arena_t *arena, } static void * -arena_slab_reg_alloc(tsdn_t *tsdn, extent_t *slab, - const arena_bin_info_t *bin_info) { +arena_slab_reg_alloc(extent_t *slab, const bin_info_t *bin_info) { void *ret; arena_slab_data_t *slab_data = extent_slab_data_get(slab); size_t regind; @@ -400,6 +271,54 @@ arena_slab_reg_alloc(tsdn_t *tsdn, extent_t *slab, return ret; } +static void +arena_slab_reg_alloc_batch(extent_t *slab, const bin_info_t *bin_info, + unsigned cnt, void** ptrs) { + arena_slab_data_t *slab_data = extent_slab_data_get(slab); + + assert(extent_nfree_get(slab) >= cnt); + assert(!bitmap_full(slab_data->bitmap, &bin_info->bitmap_info)); + +#if (! defined JEMALLOC_INTERNAL_POPCOUNTL) || (defined BITMAP_USE_TREE) + for (unsigned i = 0; i < cnt; i++) { + size_t regind = bitmap_sfu(slab_data->bitmap, + &bin_info->bitmap_info); + *(ptrs + i) = (void *)((uintptr_t)extent_addr_get(slab) + + (uintptr_t)(bin_info->reg_size * regind)); + } +#else + unsigned group = 0; + bitmap_t g = slab_data->bitmap[group]; + unsigned i = 0; + while (i < cnt) { + while (g == 0) { + g = slab_data->bitmap[++group]; + } + size_t shift = group << LG_BITMAP_GROUP_NBITS; + size_t pop = popcount_lu(g); + if (pop > (cnt - i)) { + pop = cnt - i; + } + + /* + * Load from memory locations only once, outside the + * hot loop below. + */ + uintptr_t base = (uintptr_t)extent_addr_get(slab); + uintptr_t regsize = (uintptr_t)bin_info->reg_size; + while (pop--) { + size_t bit = cfs_lu(&g); + size_t regind = shift + bit; + *(ptrs + i) = (void *)(base + regsize * regind); + + i++; + } + slab_data->bitmap[group] = g; + } +#endif + extent_nfree_sub(slab, cnt); +} + #ifndef JEMALLOC_JET static #endif @@ -412,37 +331,22 @@ arena_slab_regind(extent_t *slab, szind_t binind, const void *ptr) { assert((uintptr_t)ptr < (uintptr_t)extent_past_get(slab)); /* Freeing an interior pointer can cause assertion failure. */ assert(((uintptr_t)ptr - (uintptr_t)extent_addr_get(slab)) % - (uintptr_t)arena_bin_info[binind].reg_size == 0); + (uintptr_t)bin_infos[binind].reg_size == 0); - /* Avoid doing division with a variable divisor. */ diff = (size_t)((uintptr_t)ptr - (uintptr_t)extent_addr_get(slab)); - switch (binind) { -#define REGIND_bin_yes(index, reg_size) \ - case index: \ - regind = diff / (reg_size); \ - assert(diff == regind * (reg_size)); \ - break; -#define REGIND_bin_no(index, reg_size) -#define SC(index, lg_grp, lg_delta, ndelta, psz, bin, pgs, \ - lg_delta_lookup) \ - REGIND_bin_##bin(index, (1U<nregs); @@ -470,11 +374,11 @@ arena_large_malloc_stats_update(tsdn_t *tsdn, arena_t *arena, size_t usize) { cassert(config_stats); - if (usize < LARGE_MINCLASS) { - usize = LARGE_MINCLASS; + if (usize < SC_LARGE_MINCLASS) { + usize = SC_LARGE_MINCLASS; } index = sz_size2index(usize); - hindex = (index >= NBINS) ? index - NBINS : 0; + hindex = (index >= SC_NBINS) ? index - SC_NBINS : 0; arena_stats_add_u64(tsdn, &arena->stats, &arena->stats.lstats[hindex].nmalloc, 1); @@ -486,11 +390,11 @@ arena_large_dalloc_stats_update(tsdn_t *tsdn, arena_t *arena, size_t usize) { cassert(config_stats); - if (usize < LARGE_MINCLASS) { - usize = LARGE_MINCLASS; + if (usize < SC_LARGE_MINCLASS) { + usize = SC_LARGE_MINCLASS; } index = sz_size2index(usize); - hindex = (index >= NBINS) ? index - NBINS : 0; + hindex = (index >= SC_NBINS) ? index - SC_NBINS : 0; arena_stats_add_u64(tsdn, &arena->stats, &arena->stats.lstats[hindex].ndalloc, 1); @@ -503,6 +407,11 @@ arena_large_ralloc_stats_update(tsdn_t *tsdn, arena_t *arena, size_t oldusize, arena_large_malloc_stats_update(tsdn, arena, usize); } +static bool +arena_may_have_muzzy(arena_t *arena) { + return (pages_can_purge_lazy && (arena_muzzy_decay_ms_get(arena) != 0)); +} + extent_t * arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, bool *zero) { @@ -517,7 +426,7 @@ arena_extent_alloc_large(tsdn_t *tsdn, arena_t *arena, size_t usize, extent_t *extent = extents_alloc(tsdn, arena, &extent_hooks, &arena->extents_dirty, NULL, usize, sz_large_pad, alignment, false, szind, zero, &commit); - if (extent == NULL) { + if (extent == NULL && arena_may_have_muzzy(arena)) { extent = extents_alloc(tsdn, arena, &extent_hooks, &arena->extents_muzzy, NULL, usize, sz_large_pad, alignment, false, szind, zero, &commit); @@ -692,7 +601,8 @@ arena_decay_try_purge(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, bool is_background_thread) { if (current_npages > npages_limit) { arena_decay_to_limit(tsdn, arena, decay, extents, false, - npages_limit, is_background_thread); + npages_limit, current_npages - npages_limit, + is_background_thread); } } @@ -738,7 +648,7 @@ arena_decay_epoch_advance(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, } static void -arena_decay_reinit(arena_decay_t *decay, extents_t *extents, ssize_t decay_ms) { +arena_decay_reinit(arena_decay_t *decay, ssize_t decay_ms) { arena_decay_ms_write(decay, decay_ms); if (decay_ms > 0) { nstime_init(&decay->interval, (uint64_t)decay_ms * @@ -755,8 +665,8 @@ arena_decay_reinit(arena_decay_t *decay, extents_t *extents, ssize_t decay_ms) { } static bool -arena_decay_init(arena_decay_t *decay, extents_t *extents, ssize_t decay_ms, - decay_stats_t *stats) { +arena_decay_init(arena_decay_t *decay, ssize_t decay_ms, + arena_stats_decay_t *stats) { if (config_debug) { for (size_t i = 0; i < sizeof(arena_decay_t); i++) { assert(((char *)decay)[i] == 0); @@ -768,7 +678,7 @@ arena_decay_init(arena_decay_t *decay, extents_t *extents, ssize_t decay_ms, return true; } decay->purging = false; - arena_decay_reinit(decay, extents, decay_ms); + arena_decay_reinit(decay, decay_ms); /* Memory is zeroed, so there is no need to clear stats. */ if (config_stats) { decay->stats = stats; @@ -798,7 +708,8 @@ arena_maybe_decay(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, if (decay_ms <= 0) { if (decay_ms == 0) { arena_decay_to_limit(tsdn, arena, decay, extents, false, - 0, is_background_thread); + 0, extents_npages_get(extents), + is_background_thread); } return false; } @@ -876,7 +787,7 @@ arena_decay_ms_set(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, * infrequent, either between the {-1, 0, >0} states, or a one-time * arbitrary change during initial arena configuration. */ - arena_decay_reinit(decay, extents, decay_ms); + arena_decay_reinit(decay, decay_ms); arena_maybe_decay(tsdn, arena, decay, extents, false); malloc_mutex_unlock(tsdn, &decay->mtx); @@ -900,14 +811,15 @@ arena_muzzy_decay_ms_set(tsdn_t *tsdn, arena_t *arena, static size_t arena_stash_decayed(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extents_t *extents, size_t npages_limit, - extent_list_t *decay_extents) { + size_t npages_decay_max, extent_list_t *decay_extents) { witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); /* Stash extents according to npages_limit. */ size_t nstashed = 0; extent_t *extent; - while ((extent = extents_evict(tsdn, arena, r_extent_hooks, extents, + while (nstashed < npages_decay_max && + (extent = extents_evict(tsdn, arena, r_extent_hooks, extents, npages_limit)) != NULL) { extent_list_append(decay_extents, extent); nstashed += extent_size_get(extent) >> LG_PAGE; @@ -919,7 +831,7 @@ static size_t arena_decay_stashed(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, arena_decay_t *decay, extents_t *extents, bool all, extent_list_t *decay_extents, bool is_background_thread) { - UNUSED size_t nmadvise, nunmapped; + size_t nmadvise, nunmapped; size_t npurged; if (config_stats) { @@ -982,12 +894,15 @@ arena_decay_stashed(tsdn_t *tsdn, arena_t *arena, } /* - * npages_limit: Decay as many dirty extents as possible without violating the - * invariant: (extents_npages_get(extents) >= npages_limit) + * npages_limit: Decay at most npages_decay_max pages without violating the + * invariant: (extents_npages_get(extents) >= npages_limit). We need an upper + * bound on number of pages in order to prevent unbounded growth (namely in + * stashed), otherwise unbounded new pages could be added to extents during the + * current decay run, so that the purging thread never finishes. */ static void arena_decay_to_limit(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, - extents_t *extents, bool all, size_t npages_limit, + extents_t *extents, bool all, size_t npages_limit, size_t npages_decay_max, bool is_background_thread) { witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 1); @@ -1005,9 +920,9 @@ arena_decay_to_limit(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, extent_list_init(&decay_extents); size_t npurge = arena_stash_decayed(tsdn, arena, &extent_hooks, extents, - npages_limit, &decay_extents); + npages_limit, npages_decay_max, &decay_extents); if (npurge != 0) { - UNUSED size_t npurged = arena_decay_stashed(tsdn, arena, + size_t npurged = arena_decay_stashed(tsdn, arena, &extent_hooks, decay, extents, all, &decay_extents, is_background_thread); assert(npurged == npurge); @@ -1023,7 +938,7 @@ arena_decay_impl(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, if (all) { malloc_mutex_lock(tsdn, &decay->mtx); arena_decay_to_limit(tsdn, arena, decay, extents, all, 0, - is_background_thread); + extents_npages_get(extents), is_background_thread); malloc_mutex_unlock(tsdn, &decay->mtx); return false; @@ -1045,7 +960,8 @@ arena_decay_impl(tsdn_t *tsdn, arena_t *arena, arena_decay_t *decay, if (have_background_thread && background_thread_enabled() && epoch_advanced && !is_background_thread) { - background_thread_interval_check(tsdn, arena, decay, npages_new); + background_thread_interval_check(tsdn, arena, decay, + npages_new); } return false; @@ -1082,18 +998,18 @@ arena_slab_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *slab) { } static void -arena_bin_slabs_nonfull_insert(arena_bin_t *bin, extent_t *slab) { +arena_bin_slabs_nonfull_insert(bin_t *bin, extent_t *slab) { assert(extent_nfree_get(slab) > 0); extent_heap_insert(&bin->slabs_nonfull, slab); } static void -arena_bin_slabs_nonfull_remove(arena_bin_t *bin, extent_t *slab) { +arena_bin_slabs_nonfull_remove(bin_t *bin, extent_t *slab) { extent_heap_remove(&bin->slabs_nonfull, slab); } static extent_t * -arena_bin_slabs_nonfull_tryget(arena_bin_t *bin) { +arena_bin_slabs_nonfull_tryget(bin_t *bin) { extent_t *slab = extent_heap_remove_first(&bin->slabs_nonfull); if (slab == NULL) { return NULL; @@ -1105,7 +1021,7 @@ arena_bin_slabs_nonfull_tryget(arena_bin_t *bin) { } static void -arena_bin_slabs_full_insert(arena_t *arena, arena_bin_t *bin, extent_t *slab) { +arena_bin_slabs_full_insert(arena_t *arena, bin_t *bin, extent_t *slab) { assert(extent_nfree_get(slab) == 0); /* * Tracking extents is required by arena_reset, which is not allowed @@ -1119,13 +1035,44 @@ arena_bin_slabs_full_insert(arena_t *arena, arena_bin_t *bin, extent_t *slab) { } static void -arena_bin_slabs_full_remove(arena_t *arena, arena_bin_t *bin, extent_t *slab) { +arena_bin_slabs_full_remove(arena_t *arena, bin_t *bin, extent_t *slab) { if (arena_is_auto(arena)) { return; } extent_list_remove(&bin->slabs_full, slab); } +static void +arena_bin_reset(tsd_t *tsd, arena_t *arena, bin_t *bin) { + extent_t *slab; + + malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); + if (bin->slabcur != NULL) { + slab = bin->slabcur; + bin->slabcur = NULL; + malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); + arena_slab_dalloc(tsd_tsdn(tsd), arena, slab); + malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); + } + while ((slab = extent_heap_remove_first(&bin->slabs_nonfull)) != NULL) { + malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); + arena_slab_dalloc(tsd_tsdn(tsd), arena, slab); + malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); + } + for (slab = extent_list_first(&bin->slabs_full); slab != NULL; + slab = extent_list_first(&bin->slabs_full)) { + arena_bin_slabs_full_remove(arena, bin, slab); + malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); + arena_slab_dalloc(tsd_tsdn(tsd), arena, slab); + malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); + } + if (config_stats) { + bin->stats.curregs = 0; + bin->stats.curslabs = 0; + } + malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); +} + void arena_reset(tsd_t *tsd, arena_t *arena) { /* @@ -1155,7 +1102,7 @@ arena_reset(tsd_t *tsd, arena_t *arena) { rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx, (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab); - assert(alloc_ctx.szind != NSIZES); + assert(alloc_ctx.szind != SC_NSIZES); if (config_stats || (config_prof && opt_prof)) { usize = sz_index2size(alloc_ctx.szind); @@ -1171,35 +1118,11 @@ arena_reset(tsd_t *tsd, arena_t *arena) { malloc_mutex_unlock(tsd_tsdn(tsd), &arena->large_mtx); /* Bins. */ - for (unsigned i = 0; i < NBINS; i++) { - extent_t *slab; - arena_bin_t *bin = &arena->bins[i]; - malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); - if (bin->slabcur != NULL) { - slab = bin->slabcur; - bin->slabcur = NULL; - malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); - arena_slab_dalloc(tsd_tsdn(tsd), arena, slab); - malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); - } - while ((slab = extent_heap_remove_first(&bin->slabs_nonfull)) != - NULL) { - malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); - arena_slab_dalloc(tsd_tsdn(tsd), arena, slab); - malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); - } - for (slab = extent_list_first(&bin->slabs_full); slab != NULL; - slab = extent_list_first(&bin->slabs_full)) { - arena_bin_slabs_full_remove(arena, bin, slab); - malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); - arena_slab_dalloc(tsd_tsdn(tsd), arena, slab); - malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); + for (unsigned i = 0; i < SC_NBINS; i++) { + for (unsigned j = 0; j < bin_infos[i].n_shards; j++) { + arena_bin_reset(tsd, arena, + &arena->bins[i].bin_shards[j]); } - if (config_stats) { - bin->stats.curregs = 0; - bin->stats.curslabs = 0; - } - malloc_mutex_unlock(tsd_tsdn(tsd), &bin->lock); } atomic_store_zu(&arena->nactive, 0, ATOMIC_RELAXED); @@ -1262,7 +1185,7 @@ arena_destroy(tsd_t *tsd, arena_t *arena) { static extent_t * arena_slab_alloc_hard(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t **r_extent_hooks, const arena_bin_info_t *bin_info, + extent_hooks_t **r_extent_hooks, const bin_info_t *bin_info, szind_t szind) { extent_t *slab; bool zero, commit; @@ -1284,8 +1207,8 @@ arena_slab_alloc_hard(tsdn_t *tsdn, arena_t *arena, } static extent_t * -arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, - const arena_bin_info_t *bin_info) { +arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, unsigned binshard, + const bin_info_t *bin_info) { witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); @@ -1296,7 +1219,7 @@ arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, extent_t *slab = extents_alloc(tsdn, arena, &extent_hooks, &arena->extents_dirty, NULL, bin_info->slab_size, 0, PAGE, true, binind, &zero, &commit); - if (slab == NULL) { + if (slab == NULL && arena_may_have_muzzy(arena)) { slab = extents_alloc(tsdn, arena, &extent_hooks, &arena->extents_muzzy, NULL, bin_info->slab_size, 0, PAGE, true, binind, &zero, &commit); @@ -1312,7 +1235,7 @@ arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, /* Initialize slab internals. */ arena_slab_data_t *slab_data = extent_slab_data_get(slab); - extent_nfree_set(slab, bin_info->nregs); + extent_nfree_binshard_set(slab, bin_info->nregs, binshard); bitmap_init(slab_data->bitmap, &bin_info->bitmap_info, false); arena_nactive_add(arena, extent_size_get(slab) >> LG_PAGE); @@ -1321,10 +1244,10 @@ arena_slab_alloc(tsdn_t *tsdn, arena_t *arena, szind_t binind, } static extent_t * -arena_bin_nonfull_slab_get(tsdn_t *tsdn, arena_t *arena, arena_bin_t *bin, - szind_t binind) { +arena_bin_nonfull_slab_get(tsdn_t *tsdn, arena_t *arena, bin_t *bin, + szind_t binind, unsigned binshard) { extent_t *slab; - const arena_bin_info_t *bin_info; + const bin_info_t *bin_info; /* Look for a usable slab. */ slab = arena_bin_slabs_nonfull_tryget(bin); @@ -1333,12 +1256,12 @@ arena_bin_nonfull_slab_get(tsdn_t *tsdn, arena_t *arena, arena_bin_t *bin, } /* No existing slabs have any space available. */ - bin_info = &arena_bin_info[binind]; + bin_info = &bin_infos[binind]; /* Allocate a new slab. */ malloc_mutex_unlock(tsdn, &bin->lock); /******************************/ - slab = arena_slab_alloc(tsdn, arena, binind, bin_info); + slab = arena_slab_alloc(tsdn, arena, binind, binshard, bin_info); /********************************/ malloc_mutex_lock(tsdn, &bin->lock); if (slab != NULL) { @@ -1364,24 +1287,24 @@ arena_bin_nonfull_slab_get(tsdn_t *tsdn, arena_t *arena, arena_bin_t *bin, /* Re-fill bin->slabcur, then call arena_slab_reg_alloc(). */ static void * -arena_bin_malloc_hard(tsdn_t *tsdn, arena_t *arena, arena_bin_t *bin, - szind_t binind) { - const arena_bin_info_t *bin_info; +arena_bin_malloc_hard(tsdn_t *tsdn, arena_t *arena, bin_t *bin, + szind_t binind, unsigned binshard) { + const bin_info_t *bin_info; extent_t *slab; - bin_info = &arena_bin_info[binind]; + bin_info = &bin_infos[binind]; if (!arena_is_auto(arena) && bin->slabcur != NULL) { arena_bin_slabs_full_insert(arena, bin, bin->slabcur); bin->slabcur = NULL; } - slab = arena_bin_nonfull_slab_get(tsdn, arena, bin, binind); + slab = arena_bin_nonfull_slab_get(tsdn, arena, bin, binind, binshard); if (bin->slabcur != NULL) { /* * Another thread updated slabcur while this one ran without the * bin lock in arena_bin_nonfull_slab_get(). */ if (extent_nfree_get(bin->slabcur) > 0) { - void *ret = arena_slab_reg_alloc(tsdn, bin->slabcur, + void *ret = arena_slab_reg_alloc(bin->slabcur, bin_info); if (slab != NULL) { /* @@ -1415,51 +1338,78 @@ arena_bin_malloc_hard(tsdn_t *tsdn, arena_t *arena, arena_bin_t *bin, assert(extent_nfree_get(bin->slabcur) > 0); - return arena_slab_reg_alloc(tsdn, slab, bin_info); + return arena_slab_reg_alloc(slab, bin_info); +} + +/* Choose a bin shard and return the locked bin. */ +bin_t * +arena_bin_choose_lock(tsdn_t *tsdn, arena_t *arena, szind_t binind, + unsigned *binshard) { + bin_t *bin; + if (tsdn_null(tsdn) || tsd_arena_get(tsdn_tsd(tsdn)) == NULL) { + *binshard = 0; + } else { + *binshard = tsd_binshardsp_get(tsdn_tsd(tsdn))->binshard[binind]; + } + assert(*binshard < bin_infos[binind].n_shards); + bin = &arena->bins[binind].bin_shards[*binshard]; + malloc_mutex_lock(tsdn, &bin->lock); + + return bin; } void arena_tcache_fill_small(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache, - tcache_bin_t *tbin, szind_t binind, uint64_t prof_accumbytes) { - unsigned i, nfill; - arena_bin_t *bin; + cache_bin_t *tbin, szind_t binind, uint64_t prof_accumbytes) { + unsigned i, nfill, cnt; assert(tbin->ncached == 0); if (config_prof && arena_prof_accum(tsdn, arena, prof_accumbytes)) { prof_idump(tsdn); } - bin = &arena->bins[binind]; - malloc_mutex_lock(tsdn, &bin->lock); + + unsigned binshard; + bin_t *bin = arena_bin_choose_lock(tsdn, arena, binind, &binshard); + for (i = 0, nfill = (tcache_bin_info[binind].ncached_max >> - tcache->lg_fill_div[binind]); i < nfill; i++) { + tcache->lg_fill_div[binind]); i < nfill; i += cnt) { extent_t *slab; - void *ptr; if ((slab = bin->slabcur) != NULL && extent_nfree_get(slab) > 0) { - ptr = arena_slab_reg_alloc(tsdn, slab, - &arena_bin_info[binind]); + unsigned tofill = nfill - i; + cnt = tofill < extent_nfree_get(slab) ? + tofill : extent_nfree_get(slab); + arena_slab_reg_alloc_batch( + slab, &bin_infos[binind], cnt, + tbin->avail - nfill + i); } else { - ptr = arena_bin_malloc_hard(tsdn, arena, bin, binind); - } - if (ptr == NULL) { + cnt = 1; + void *ptr = arena_bin_malloc_hard(tsdn, arena, bin, + binind, binshard); /* * OOM. tbin->avail isn't yet filled down to its first * element, so the successful allocations (if any) must * be moved just before tbin->avail before bailing out. */ - if (i > 0) { - memmove(tbin->avail - i, tbin->avail - nfill, - i * sizeof(void *)); + if (ptr == NULL) { + if (i > 0) { + memmove(tbin->avail - i, + tbin->avail - nfill, + i * sizeof(void *)); + } + break; } - break; + /* Insert such that low regions get used first. */ + *(tbin->avail - nfill + i) = ptr; } if (config_fill && unlikely(opt_junk_alloc)) { - arena_alloc_junk_small(ptr, &arena_bin_info[binind], - true); + for (unsigned j = 0; j < cnt; j++) { + void* ptr = *(tbin->avail - nfill + i + j); + arena_alloc_junk_small(ptr, &bin_infos[binind], + true); + } } - /* Insert such that low regions get used first. */ - *(tbin->avail - nfill + i) = ptr; } if (config_stats) { bin->stats.nmalloc += i; @@ -1474,14 +1424,14 @@ arena_tcache_fill_small(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache, } void -arena_alloc_junk_small(void *ptr, const arena_bin_info_t *bin_info, bool zero) { +arena_alloc_junk_small(void *ptr, const bin_info_t *bin_info, bool zero) { if (!zero) { memset(ptr, JEMALLOC_ALLOC_JUNK, bin_info->reg_size); } } static void -arena_dalloc_junk_small_impl(void *ptr, const arena_bin_info_t *bin_info) { +arena_dalloc_junk_small_impl(void *ptr, const bin_info_t *bin_info) { memset(ptr, JEMALLOC_FREE_JUNK, bin_info->reg_size); } arena_dalloc_junk_small_t *JET_MUTABLE arena_dalloc_junk_small = @@ -1490,19 +1440,19 @@ arena_dalloc_junk_small_t *JET_MUTABLE arena_dalloc_junk_small = static void * arena_malloc_small(tsdn_t *tsdn, arena_t *arena, szind_t binind, bool zero) { void *ret; - arena_bin_t *bin; + bin_t *bin; size_t usize; extent_t *slab; - assert(binind < NBINS); - bin = &arena->bins[binind]; + assert(binind < SC_NBINS); usize = sz_index2size(binind); + unsigned binshard; + bin = arena_bin_choose_lock(tsdn, arena, binind, &binshard); - malloc_mutex_lock(tsdn, &bin->lock); if ((slab = bin->slabcur) != NULL && extent_nfree_get(slab) > 0) { - ret = arena_slab_reg_alloc(tsdn, slab, &arena_bin_info[binind]); + ret = arena_slab_reg_alloc(slab, &bin_infos[binind]); } else { - ret = arena_bin_malloc_hard(tsdn, arena, bin, binind); + ret = arena_bin_malloc_hard(tsdn, arena, bin, binind, binshard); } if (ret == NULL) { @@ -1524,14 +1474,14 @@ arena_malloc_small(tsdn_t *tsdn, arena_t *arena, szind_t binind, bool zero) { if (config_fill) { if (unlikely(opt_junk_alloc)) { arena_alloc_junk_small(ret, - &arena_bin_info[binind], false); + &bin_infos[binind], false); } else if (unlikely(opt_zero)) { memset(ret, 0, usize); } } } else { if (config_fill && unlikely(opt_junk_alloc)) { - arena_alloc_junk_small(ret, &arena_bin_info[binind], + arena_alloc_junk_small(ret, &bin_infos[binind], true); } memset(ret, 0, usize); @@ -1547,13 +1497,13 @@ arena_malloc_hard(tsdn_t *tsdn, arena_t *arena, size_t size, szind_t ind, assert(!tsdn_null(tsdn) || arena != NULL); if (likely(!tsdn_null(tsdn))) { - arena = arena_choose(tsdn_tsd(tsdn), arena); + arena = arena_choose_maybe_huge(tsdn_tsd(tsdn), arena, size); } if (unlikely(arena == NULL)) { return NULL; } - if (likely(size <= SMALL_MAXCLASS)) { + if (likely(size <= SC_SMALL_MAXCLASS)) { return arena_malloc_small(tsdn, arena, ind, zero); } return large_malloc(tsdn, arena, sz_index2size(ind), zero); @@ -1564,8 +1514,9 @@ arena_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, bool zero, tcache_t *tcache) { void *ret; - if (usize <= SMALL_MAXCLASS && (alignment < PAGE || (alignment == PAGE - && (usize & PAGE_MASK) == 0))) { + if (usize <= SC_SMALL_MAXCLASS + && (alignment < PAGE + || (alignment == PAGE && (usize & PAGE_MASK) == 0))) { /* Small; alignment doesn't require special slab placement. */ ret = arena_malloc(tsdn, arena, usize, sz_size2index(usize), zero, tcache, true); @@ -1583,8 +1534,8 @@ void arena_prof_promote(tsdn_t *tsdn, const void *ptr, size_t usize) { cassert(config_prof); assert(ptr != NULL); - assert(isalloc(tsdn, ptr) == LARGE_MINCLASS); - assert(usize <= SMALL_MAXCLASS); + assert(isalloc(tsdn, ptr) == SC_LARGE_MINCLASS); + assert(usize <= SC_SMALL_MAXCLASS); rtree_ctx_t rtree_ctx_fallback; rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); @@ -1608,15 +1559,15 @@ arena_prof_demote(tsdn_t *tsdn, extent_t *extent, const void *ptr) { cassert(config_prof); assert(ptr != NULL); - extent_szind_set(extent, NBINS); + extent_szind_set(extent, SC_NBINS); rtree_ctx_t rtree_ctx_fallback; rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); rtree_szind_slab_update(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)ptr, - NBINS, false); + SC_NBINS, false); - assert(isalloc(tsdn, ptr) == LARGE_MINCLASS); + assert(isalloc(tsdn, ptr) == SC_LARGE_MINCLASS); - return LARGE_MINCLASS; + return SC_LARGE_MINCLASS; } void @@ -1636,13 +1587,13 @@ arena_dalloc_promoted(tsdn_t *tsdn, void *ptr, tcache_t *tcache, } static void -arena_dissociate_bin_slab(arena_t *arena, extent_t *slab, arena_bin_t *bin) { +arena_dissociate_bin_slab(arena_t *arena, extent_t *slab, bin_t *bin) { /* Dissociate slab from bin. */ if (slab == bin->slabcur) { bin->slabcur = NULL; } else { szind_t binind = extent_szind_get(slab); - const arena_bin_info_t *bin_info = &arena_bin_info[binind]; + const bin_info_t *bin_info = &bin_infos[binind]; /* * The following block's conditional is necessary because if the @@ -1659,7 +1610,7 @@ arena_dissociate_bin_slab(arena_t *arena, extent_t *slab, arena_bin_t *bin) { static void arena_dalloc_bin_slab(tsdn_t *tsdn, arena_t *arena, extent_t *slab, - arena_bin_t *bin) { + bin_t *bin) { assert(slab != bin->slabcur); malloc_mutex_unlock(tsdn, &bin->lock); @@ -1674,7 +1625,7 @@ arena_dalloc_bin_slab(tsdn_t *tsdn, arena_t *arena, extent_t *slab, static void arena_bin_lower_slab(tsdn_t *tsdn, arena_t *arena, extent_t *slab, - arena_bin_t *bin) { + bin_t *bin) { assert(extent_nfree_get(slab) > 0); /* @@ -1700,18 +1651,16 @@ arena_bin_lower_slab(tsdn_t *tsdn, arena_t *arena, extent_t *slab, } static void -arena_dalloc_bin_locked_impl(tsdn_t *tsdn, arena_t *arena, extent_t *slab, - void *ptr, bool junked) { +arena_dalloc_bin_locked_impl(tsdn_t *tsdn, arena_t *arena, bin_t *bin, + szind_t binind, extent_t *slab, void *ptr, bool junked) { arena_slab_data_t *slab_data = extent_slab_data_get(slab); - szind_t binind = extent_szind_get(slab); - arena_bin_t *bin = &arena->bins[binind]; - const arena_bin_info_t *bin_info = &arena_bin_info[binind]; + const bin_info_t *bin_info = &bin_infos[binind]; if (!junked && config_fill && unlikely(opt_junk_free)) { arena_dalloc_junk_small(ptr, bin_info); } - arena_slab_reg_dalloc(tsdn, slab, slab_data, ptr); + arena_slab_reg_dalloc(slab, slab_data, ptr); unsigned nfree = extent_nfree_get(slab); if (nfree == bin_info->nregs) { arena_dissociate_bin_slab(arena, slab, bin); @@ -1728,18 +1677,21 @@ arena_dalloc_bin_locked_impl(tsdn_t *tsdn, arena_t *arena, extent_t *slab, } void -arena_dalloc_bin_junked_locked(tsdn_t *tsdn, arena_t *arena, extent_t *extent, - void *ptr) { - arena_dalloc_bin_locked_impl(tsdn, arena, extent, ptr, true); +arena_dalloc_bin_junked_locked(tsdn_t *tsdn, arena_t *arena, bin_t *bin, + szind_t binind, extent_t *extent, void *ptr) { + arena_dalloc_bin_locked_impl(tsdn, arena, bin, binind, extent, ptr, + true); } static void arena_dalloc_bin(tsdn_t *tsdn, arena_t *arena, extent_t *extent, void *ptr) { szind_t binind = extent_szind_get(extent); - arena_bin_t *bin = &arena->bins[binind]; + unsigned binshard = extent_binshard_get(extent); + bin_t *bin = &arena->bins[binind].bin_shards[binshard]; malloc_mutex_lock(tsdn, &bin->lock); - arena_dalloc_bin_locked_impl(tsdn, arena, extent, ptr, false); + arena_dalloc_bin_locked_impl(tsdn, arena, bin, binind, extent, ptr, + false); malloc_mutex_unlock(tsdn, &bin->lock); } @@ -1754,38 +1706,48 @@ arena_dalloc_small(tsdn_t *tsdn, void *ptr) { bool arena_ralloc_no_move(tsdn_t *tsdn, void *ptr, size_t oldsize, size_t size, - size_t extra, bool zero) { + size_t extra, bool zero, size_t *newsize) { + bool ret; /* Calls with non-zero extra had to clamp extra. */ - assert(extra == 0 || size + extra <= LARGE_MAXCLASS); + assert(extra == 0 || size + extra <= SC_LARGE_MAXCLASS); - if (unlikely(size > LARGE_MAXCLASS)) { - return true; + extent_t *extent = iealloc(tsdn, ptr); + if (unlikely(size > SC_LARGE_MAXCLASS)) { + ret = true; + goto done; } - extent_t *extent = iealloc(tsdn, ptr); size_t usize_min = sz_s2u(size); size_t usize_max = sz_s2u(size + extra); - if (likely(oldsize <= SMALL_MAXCLASS && usize_min <= SMALL_MAXCLASS)) { + if (likely(oldsize <= SC_SMALL_MAXCLASS && usize_min + <= SC_SMALL_MAXCLASS)) { /* * Avoid moving the allocation if the size class can be left the * same. */ - assert(arena_bin_info[sz_size2index(oldsize)].reg_size == + assert(bin_infos[sz_size2index(oldsize)].reg_size == oldsize); - if ((usize_max > SMALL_MAXCLASS || sz_size2index(usize_max) != - sz_size2index(oldsize)) && (size > oldsize || usize_max < - oldsize)) { - return true; + if ((usize_max > SC_SMALL_MAXCLASS + || sz_size2index(usize_max) != sz_size2index(oldsize)) + && (size > oldsize || usize_max < oldsize)) { + ret = true; + goto done; } arena_decay_tick(tsdn, extent_arena_get(extent)); - return false; - } else if (oldsize >= LARGE_MINCLASS && usize_max >= LARGE_MINCLASS) { - return large_ralloc_no_move(tsdn, extent, usize_min, usize_max, + ret = false; + } else if (oldsize >= SC_LARGE_MINCLASS + && usize_max >= SC_LARGE_MINCLASS) { + ret = large_ralloc_no_move(tsdn, extent, usize_min, usize_max, zero); + } else { + ret = true; } +done: + assert(extent == iealloc(tsdn, ptr)); + *newsize = extent_usize_get(extent); - return true; + return ret; } static void * @@ -1796,7 +1758,7 @@ arena_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize, zero, tcache, true); } usize = sz_sa2u(usize, alignment); - if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) { + if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) { return NULL; } return ipalloct(tsdn, usize, alignment, zero, tcache, arena); @@ -1804,22 +1766,30 @@ arena_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize, void * arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize, - size_t size, size_t alignment, bool zero, tcache_t *tcache) { + size_t size, size_t alignment, bool zero, tcache_t *tcache, + hook_ralloc_args_t *hook_args) { size_t usize = sz_s2u(size); - if (unlikely(usize == 0 || size > LARGE_MAXCLASS)) { + if (unlikely(usize == 0 || size > SC_LARGE_MAXCLASS)) { return NULL; } - if (likely(usize <= SMALL_MAXCLASS)) { + if (likely(usize <= SC_SMALL_MAXCLASS)) { /* Try to avoid moving the allocation. */ - if (!arena_ralloc_no_move(tsdn, ptr, oldsize, usize, 0, zero)) { + UNUSED size_t newsize; + if (!arena_ralloc_no_move(tsdn, ptr, oldsize, usize, 0, zero, + &newsize)) { + hook_invoke_expand(hook_args->is_realloc + ? hook_expand_realloc : hook_expand_rallocx, + ptr, oldsize, usize, (uintptr_t)ptr, + hook_args->args); return ptr; } } - if (oldsize >= LARGE_MINCLASS && usize >= LARGE_MINCLASS) { - return large_ralloc(tsdn, arena, iealloc(tsdn, ptr), usize, - alignment, zero, tcache); + if (oldsize >= SC_LARGE_MINCLASS + && usize >= SC_LARGE_MINCLASS) { + return large_ralloc(tsdn, arena, ptr, usize, + alignment, zero, tcache, hook_args); } /* @@ -1832,11 +1802,16 @@ arena_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t oldsize, return NULL; } + hook_invoke_alloc(hook_args->is_realloc + ? hook_alloc_realloc : hook_alloc_rallocx, ret, (uintptr_t)ret, + hook_args->args); + hook_invoke_dalloc(hook_args->is_realloc + ? hook_dalloc_realloc : hook_dalloc_rallocx, ptr, hook_args->args); + /* * Junk/zero-filling were already done by * ipalloc()/arena_malloc(). */ - size_t copysize = (usize < oldsize) ? usize : oldsize; memcpy(ret, ptr, copysize); isdalloct(tsdn, ptr, oldsize, tcache, NULL, true); @@ -1885,6 +1860,32 @@ arena_muzzy_decay_ms_default_set(ssize_t decay_ms) { return false; } +bool +arena_retain_grow_limit_get_set(tsd_t *tsd, arena_t *arena, size_t *old_limit, + size_t *new_limit) { + assert(opt_retain); + + pszind_t new_ind JEMALLOC_CC_SILENCE_INIT(0); + if (new_limit != NULL) { + size_t limit = *new_limit; + /* Grow no more than the new limit. */ + if ((new_ind = sz_psz2ind(limit + 1) - 1) >= SC_NPSIZES) { + return true; + } + } + + malloc_mutex_lock(tsd_tsdn(tsd), &arena->extent_grow_mtx); + if (old_limit != NULL) { + *old_limit = sz_pind2sz(arena->retain_grow_limit); + } + if (new_limit != NULL) { + arena->retain_grow_limit = new_ind; + } + malloc_mutex_unlock(tsd_tsdn(tsd), &arena->extent_grow_mtx); + + return false; +} + unsigned arena_nthreads_get(arena_t *arena, bool internal) { return atomic_load_u(&arena->nthreads[internal], ATOMIC_RELAXED); @@ -1920,7 +1921,12 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { } } - arena = (arena_t *)base_alloc(tsdn, base, sizeof(arena_t), CACHELINE); + unsigned nbins_total = 0; + for (i = 0; i < SC_NBINS; i++) { + nbins_total += bin_infos[i].n_shards; + } + size_t arena_size = sizeof(arena_t) + sizeof(bin_t) * nbins_total; + arena = (arena_t *)base_alloc(tsdn, base, arena_size, CACHELINE); if (arena == NULL) { goto label_error; } @@ -1935,6 +1941,7 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { } ql_new(&arena->tcache_ql); + ql_new(&arena->cache_bin_array_descriptor_ql); if (malloc_mutex_init(&arena->tcache_ql_mtx, "tcache_ql", WITNESS_RANK_TCACHE_QL, malloc_mutex_rank_exclusive)) { goto label_error; @@ -2001,16 +2008,17 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { goto label_error; } - if (arena_decay_init(&arena->decay_dirty, &arena->extents_dirty, + if (arena_decay_init(&arena->decay_dirty, arena_dirty_decay_ms_default_get(), &arena->stats.decay_dirty)) { goto label_error; } - if (arena_decay_init(&arena->decay_muzzy, &arena->extents_muzzy, + if (arena_decay_init(&arena->decay_muzzy, arena_muzzy_decay_ms_default_get(), &arena->stats.decay_muzzy)) { goto label_error; } arena->extent_grow_next = sz_psz2ind(HUGEPAGE); + arena->retain_grow_limit = sz_psz2ind(SC_LARGE_MAXCLASS); if (malloc_mutex_init(&arena->extent_grow_mtx, "extent_grow", WITNESS_RANK_EXTENT_GROW, malloc_mutex_rank_exclusive)) { goto label_error; @@ -2023,19 +2031,20 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { } /* Initialize bins. */ - for (i = 0; i < NBINS; i++) { - arena_bin_t *bin = &arena->bins[i]; - if (malloc_mutex_init(&bin->lock, "arena_bin", - WITNESS_RANK_ARENA_BIN, malloc_mutex_rank_exclusive)) { - goto label_error; - } - bin->slabcur = NULL; - extent_heap_new(&bin->slabs_nonfull); - extent_list_init(&bin->slabs_full); - if (config_stats) { - memset(&bin->stats, 0, sizeof(malloc_bin_stats_t)); + uintptr_t bin_addr = (uintptr_t)arena + sizeof(arena_t); + atomic_store_u(&arena->binshard_next, 0, ATOMIC_RELEASE); + for (i = 0; i < SC_NBINS; i++) { + unsigned nshards = bin_infos[i].n_shards; + arena->bins[i].bin_shards = (bin_t *)bin_addr; + bin_addr += nshards * sizeof(bin_t); + for (unsigned j = 0; j < nshards; j++) { + bool err = bin_init(&arena->bins[i].bin_shards[j]); + if (err) { + goto label_error; + } } } + assert(bin_addr == (uintptr_t)arena + arena_size); arena->base = base; /* Set arena before creating background threads. */ @@ -2052,8 +2061,8 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { */ assert(!tsdn_null(tsdn)); pre_reentrancy(tsdn_tsd(tsdn), arena); - if (hooks_arena_new_hook) { - hooks_arena_new_hook(); + if (test_hooks_arena_new_hook) { + test_hooks_arena_new_hook(); } post_reentrancy(tsdn_tsd(tsdn)); } @@ -2066,10 +2075,75 @@ arena_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { return NULL; } +arena_t * +arena_choose_huge(tsd_t *tsd) { + /* huge_arena_ind can be 0 during init (will use a0). */ + if (huge_arena_ind == 0) { + assert(!malloc_initialized()); + } + + arena_t *huge_arena = arena_get(tsd_tsdn(tsd), huge_arena_ind, false); + if (huge_arena == NULL) { + /* Create the huge arena on demand. */ + assert(huge_arena_ind != 0); + huge_arena = arena_get(tsd_tsdn(tsd), huge_arena_ind, true); + if (huge_arena == NULL) { + return NULL; + } + /* + * Purge eagerly for huge allocations, because: 1) number of + * huge allocations is usually small, which means ticker based + * decay is not reliable; and 2) less immediate reuse is + * expected for huge allocations. + */ + if (arena_dirty_decay_ms_default_get() > 0) { + arena_dirty_decay_ms_set(tsd_tsdn(tsd), huge_arena, 0); + } + if (arena_muzzy_decay_ms_default_get() > 0) { + arena_muzzy_decay_ms_set(tsd_tsdn(tsd), huge_arena, 0); + } + } + + return huge_arena; +} + +bool +arena_init_huge(void) { + bool huge_enabled; + + /* The threshold should be large size class. */ + if (opt_oversize_threshold > SC_LARGE_MAXCLASS || + opt_oversize_threshold < SC_LARGE_MINCLASS) { + opt_oversize_threshold = 0; + oversize_threshold = SC_LARGE_MAXCLASS + PAGE; + huge_enabled = false; + } else { + /* Reserve the index for the huge arena. */ + huge_arena_ind = narenas_total_get(); + oversize_threshold = opt_oversize_threshold; + huge_enabled = true; + } + + return huge_enabled; +} + +bool +arena_is_huge(unsigned arena_ind) { + if (huge_arena_ind == 0) { + return false; + } + return (arena_ind == huge_arena_ind); +} + void -arena_boot(void) { +arena_boot(sc_data_t *sc_data) { arena_dirty_decay_ms_default_set(opt_dirty_decay_ms); arena_muzzy_decay_ms_default_set(opt_muzzy_decay_ms); + for (unsigned i = 0; i < SC_NBINS; i++) { + sc_t *sc = &sc_data->sc[i]; + div_init(&arena_binind_div_info[i], + (1U << sc->lg_base) + (sc->ndelta << sc->lg_delta)); + } } void @@ -2114,8 +2188,10 @@ arena_prefork6(tsdn_t *tsdn, arena_t *arena) { void arena_prefork7(tsdn_t *tsdn, arena_t *arena) { - for (unsigned i = 0; i < NBINS; i++) { - malloc_mutex_prefork(tsdn, &arena->bins[i].lock); + for (unsigned i = 0; i < SC_NBINS; i++) { + for (unsigned j = 0; j < bin_infos[i].n_shards; j++) { + bin_prefork(tsdn, &arena->bins[i].bin_shards[j]); + } } } @@ -2123,8 +2199,11 @@ void arena_postfork_parent(tsdn_t *tsdn, arena_t *arena) { unsigned i; - for (i = 0; i < NBINS; i++) { - malloc_mutex_postfork_parent(tsdn, &arena->bins[i].lock); + for (i = 0; i < SC_NBINS; i++) { + for (unsigned j = 0; j < bin_infos[i].n_shards; j++) { + bin_postfork_parent(tsdn, + &arena->bins[i].bin_shards[j]); + } } malloc_mutex_postfork_parent(tsdn, &arena->large_mtx); base_postfork_parent(tsdn, arena->base); @@ -2154,15 +2233,23 @@ arena_postfork_child(tsdn_t *tsdn, arena_t *arena) { } if (config_stats) { ql_new(&arena->tcache_ql); + ql_new(&arena->cache_bin_array_descriptor_ql); tcache_t *tcache = tcache_get(tsdn_tsd(tsdn)); if (tcache != NULL && tcache->arena == arena) { ql_elm_new(tcache, link); ql_tail_insert(&arena->tcache_ql, tcache, link); + cache_bin_array_descriptor_init( + &tcache->cache_bin_array_descriptor, + tcache->bins_small, tcache->bins_large); + ql_tail_insert(&arena->cache_bin_array_descriptor_ql, + &tcache->cache_bin_array_descriptor, link); } } - for (i = 0; i < NBINS; i++) { - malloc_mutex_postfork_child(tsdn, &arena->bins[i].lock); + for (i = 0; i < SC_NBINS; i++) { + for (unsigned j = 0; j < bin_infos[i].n_shards; j++) { + bin_postfork_child(tsdn, &arena->bins[i].bin_shards[j]); + } } malloc_mutex_postfork_child(tsdn, &arena->large_mtx); base_postfork_child(tsdn, arena->base); diff --git a/kbe/src/lib/dependencies/jemalloc/src/background_thread.c b/kbe/src/lib/dependencies/jemalloc/src/background_thread.c index eb30eb5b42..5ed6c1c959 100755 --- a/kbe/src/lib/dependencies/jemalloc/src/background_thread.c +++ b/kbe/src/lib/dependencies/jemalloc/src/background_thread.c @@ -4,6 +4,8 @@ #include "jemalloc/internal/assert.h" +JEMALLOC_DIAGNOSTIC_DISABLE_SPURIOUS + /******************************************************************************/ /* Data. */ @@ -11,38 +13,37 @@ #define BACKGROUND_THREAD_DEFAULT false /* Read-only after initialization. */ bool opt_background_thread = BACKGROUND_THREAD_DEFAULT; +size_t opt_max_background_threads = MAX_BACKGROUND_THREAD_LIMIT + 1; /* Used for thread creation, termination and stats. */ malloc_mutex_t background_thread_lock; /* Indicates global state. Atomic because decay reads this w/o locking. */ atomic_b_t background_thread_enabled_state; size_t n_background_threads; +size_t max_background_threads; /* Thread info per-index. */ background_thread_info_t *background_thread_info; -/* False if no necessary runtime support. */ -bool can_enable_background_thread; - /******************************************************************************/ #ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER -#include static int (*pthread_create_fptr)(pthread_t *__restrict, const pthread_attr_t *, void *(*)(void *), void *__restrict); -static pthread_once_t once_control = PTHREAD_ONCE_INIT; static void -pthread_create_wrapper_once(void) { +pthread_create_wrapper_init(void) { #ifdef JEMALLOC_LAZY_LOCK - isthreaded = true; + if (!isthreaded) { + isthreaded = true; + } #endif } int pthread_create_wrapper(pthread_t *__restrict thread, const pthread_attr_t *attr, void *(*start_routine)(void *), void *__restrict arg) { - pthread_once(&once_control, pthread_create_wrapper_once); + pthread_create_wrapper_init(); return pthread_create_fptr(thread, attr, start_routine, arg); } @@ -78,7 +79,7 @@ background_thread_info_init(tsdn_t *tsdn, background_thread_info_t *info) { } static inline bool -set_current_thread_affinity(UNUSED int cpu) { +set_current_thread_affinity(int cpu) { #if defined(JEMALLOC_HAVE_SCHED_SETAFFINITY) cpu_set_t cpuset; CPU_ZERO(&cpuset); @@ -286,7 +287,7 @@ background_work_sleep_once(tsdn_t *tsdn, background_thread_info_t *info, unsigne uint64_t min_interval = BACKGROUND_THREAD_INDEFINITE_SLEEP; unsigned narenas = narenas_total_get(); - for (unsigned i = ind; i < narenas; i += ncpus) { + for (unsigned i = ind; i < narenas; i += max_background_threads) { arena_t *arena = arena_get(tsdn, i, false); if (!arena) { continue; @@ -379,35 +380,32 @@ background_thread_create_signals_masked(pthread_t *thread, return create_err; } -static void +static bool check_background_thread_creation(tsd_t *tsd, unsigned *n_created, bool *created_threads) { + bool ret = false; if (likely(*n_created == n_background_threads)) { - return; + return ret; } - malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_info[0].mtx); -label_restart: - malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock); - for (unsigned i = 1; i < ncpus; i++) { + tsdn_t *tsdn = tsd_tsdn(tsd); + malloc_mutex_unlock(tsdn, &background_thread_info[0].mtx); + for (unsigned i = 1; i < max_background_threads; i++) { if (created_threads[i]) { continue; } background_thread_info_t *info = &background_thread_info[i]; - malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx); - assert(info->state != background_thread_paused); + malloc_mutex_lock(tsdn, &info->mtx); + /* + * In case of the background_thread_paused state because of + * arena reset, delay the creation. + */ bool create = (info->state == background_thread_started); - malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx); + malloc_mutex_unlock(tsdn, &info->mtx); if (!create) { continue; } - /* - * To avoid deadlock with prefork handlers (which waits for the - * mutex held here), unlock before calling pthread_create(). - */ - malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock); - pre_reentrancy(tsd, NULL); int err = background_thread_create_signals_masked(&info->thread, NULL, background_thread_entry, (void *)(uintptr_t)i); @@ -423,19 +421,21 @@ check_background_thread_creation(tsd_t *tsd, unsigned *n_created, abort(); } } - /* Restart since we unlocked. */ - goto label_restart; + /* Return to restart the loop since we unlocked. */ + ret = true; + break; } - malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_info[0].mtx); - malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock); + malloc_mutex_lock(tsdn, &background_thread_info[0].mtx); + + return ret; } static void background_thread0_work(tsd_t *tsd) { /* Thread0 is also responsible for launching / terminating threads. */ - VARIABLE_ARRAY(bool, created_threads, ncpus); + VARIABLE_ARRAY(bool, created_threads, max_background_threads); unsigned i; - for (i = 1; i < ncpus; i++) { + for (i = 1; i < max_background_threads; i++) { created_threads[i] = false; } /* Start working, and create more threads when asked. */ @@ -445,8 +445,10 @@ background_thread0_work(tsd_t *tsd) { &background_thread_info[0])) { continue; } - check_background_thread_creation(tsd, &n_created, - (bool *)&created_threads); + if (check_background_thread_creation(tsd, &n_created, + (bool *)&created_threads)) { + continue; + } background_work_sleep_once(tsd_tsdn(tsd), &background_thread_info[0], 0); } @@ -456,15 +458,20 @@ background_thread0_work(tsd_t *tsd) { * the global background_thread mutex (and is waiting) for us. */ assert(!background_thread_enabled()); - for (i = 1; i < ncpus; i++) { + for (i = 1; i < max_background_threads; i++) { background_thread_info_t *info = &background_thread_info[i]; assert(info->state != background_thread_paused); if (created_threads[i]) { background_threads_disable_single(tsd, info); } else { malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx); - /* Clear in case the thread wasn't created. */ - info->state = background_thread_stopped; + if (info->state != background_thread_stopped) { + /* The thread was not created. */ + assert(info->state == + background_thread_started); + n_background_threads--; + info->state = background_thread_stopped; + } malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx); } } @@ -498,9 +505,11 @@ background_work(tsd_t *tsd, unsigned ind) { static void * background_thread_entry(void *ind_arg) { unsigned thread_ind = (unsigned)(uintptr_t)ind_arg; - assert(thread_ind < ncpus); + assert(thread_ind < max_background_threads); #ifdef JEMALLOC_HAVE_PTHREAD_SETNAME_NP pthread_setname_np(pthread_self(), "jemalloc_bg_thd"); +#elif defined(__FreeBSD__) + pthread_set_name_np(pthread_self(), "jemalloc_bg_thd"); #endif if (opt_percpu_arena != percpu_arena_disabled) { set_current_thread_affinity((int)thread_ind); @@ -525,14 +534,13 @@ background_thread_init(tsd_t *tsd, background_thread_info_t *info) { n_background_threads++; } -/* Create a new background thread if needed. */ -bool -background_thread_create(tsd_t *tsd, unsigned arena_ind) { +static bool +background_thread_create_locked(tsd_t *tsd, unsigned arena_ind) { assert(have_background_thread); malloc_mutex_assert_owner(tsd_tsdn(tsd), &background_thread_lock); /* We create at most NCPUs threads. */ - size_t thread_ind = arena_ind % ncpus; + size_t thread_ind = arena_ind % max_background_threads; background_thread_info_t *info = &background_thread_info[thread_ind]; bool need_new_thread; @@ -580,37 +588,53 @@ background_thread_create(tsd_t *tsd, unsigned arena_ind) { return false; } +/* Create a new background thread if needed. */ +bool +background_thread_create(tsd_t *tsd, unsigned arena_ind) { + assert(have_background_thread); + + bool ret; + malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock); + ret = background_thread_create_locked(tsd, arena_ind); + malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock); + + return ret; +} + bool background_threads_enable(tsd_t *tsd) { assert(n_background_threads == 0); assert(background_thread_enabled()); malloc_mutex_assert_owner(tsd_tsdn(tsd), &background_thread_lock); - VARIABLE_ARRAY(bool, marked, ncpus); + VARIABLE_ARRAY(bool, marked, max_background_threads); unsigned i, nmarked; - for (i = 0; i < ncpus; i++) { + for (i = 0; i < max_background_threads; i++) { marked[i] = false; } nmarked = 0; + /* Thread 0 is required and created at the end. */ + marked[0] = true; /* Mark the threads we need to create for thread 0. */ unsigned n = narenas_total_get(); for (i = 1; i < n; i++) { - if (marked[i % ncpus] || + if (marked[i % max_background_threads] || arena_get(tsd_tsdn(tsd), i, false) == NULL) { continue; } - background_thread_info_t *info = &background_thread_info[i]; + background_thread_info_t *info = &background_thread_info[ + i % max_background_threads]; malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx); assert(info->state == background_thread_stopped); background_thread_init(tsd, info); malloc_mutex_unlock(tsd_tsdn(tsd), &info->mtx); - marked[i % ncpus] = true; - if (++nmarked == ncpus) { + marked[i % max_background_threads] = true; + if (++nmarked == max_background_threads) { break; } } - return background_thread_create(tsd, 0); + return background_thread_create_locked(tsd, 0); } bool @@ -720,14 +744,14 @@ background_thread_prefork0(tsdn_t *tsdn) { void background_thread_prefork1(tsdn_t *tsdn) { - for (unsigned i = 0; i < ncpus; i++) { + for (unsigned i = 0; i < max_background_threads; i++) { malloc_mutex_prefork(tsdn, &background_thread_info[i].mtx); } } void background_thread_postfork_parent(tsdn_t *tsdn) { - for (unsigned i = 0; i < ncpus; i++) { + for (unsigned i = 0; i < max_background_threads; i++) { malloc_mutex_postfork_parent(tsdn, &background_thread_info[i].mtx); } @@ -736,7 +760,7 @@ background_thread_postfork_parent(tsdn_t *tsdn) { void background_thread_postfork_child(tsdn_t *tsdn) { - for (unsigned i = 0; i < ncpus; i++) { + for (unsigned i = 0; i < max_background_threads; i++) { malloc_mutex_postfork_child(tsdn, &background_thread_info[i].mtx); } @@ -749,7 +773,7 @@ background_thread_postfork_child(tsdn_t *tsdn) { malloc_mutex_lock(tsdn, &background_thread_lock); n_background_threads = 0; background_thread_enabled_set(tsdn, false); - for (unsigned i = 0; i < ncpus; i++) { + for (unsigned i = 0; i < max_background_threads; i++) { background_thread_info_t *info = &background_thread_info[i]; malloc_mutex_lock(tsdn, &info->mtx); info->state = background_thread_stopped; @@ -773,7 +797,7 @@ background_thread_stats_read(tsdn_t *tsdn, background_thread_stats_t *stats) { stats->num_threads = n_background_threads; uint64_t num_runs = 0; nstime_init(&stats->run_interval, 0); - for (unsigned i = 0; i < ncpus; i++) { + for (unsigned i = 0; i < max_background_threads; i++) { background_thread_info_t *info = &background_thread_info[i]; malloc_mutex_lock(tsdn, &info->mtx); if (info->state != background_thread_stopped) { @@ -795,6 +819,39 @@ background_thread_stats_read(tsdn_t *tsdn, background_thread_stats_t *stats) { #undef BILLION #undef BACKGROUND_THREAD_MIN_INTERVAL_NS +#ifdef JEMALLOC_HAVE_DLSYM +#include +#endif + +static bool +pthread_create_fptr_init(void) { + if (pthread_create_fptr != NULL) { + return false; + } + /* + * Try the next symbol first, because 1) when use lazy_lock we have a + * wrapper for pthread_create; and 2) application may define its own + * wrapper as well (and can call malloc within the wrapper). + */ +#ifdef JEMALLOC_HAVE_DLSYM + pthread_create_fptr = dlsym(RTLD_NEXT, "pthread_create"); +#else + pthread_create_fptr = NULL; +#endif + if (pthread_create_fptr == NULL) { + if (config_lazy_lock) { + malloc_write(": Error in dlsym(RTLD_NEXT, " + "\"pthread_create\")\n"); + abort(); + } else { + /* Fall back to the default symbol. */ + pthread_create_fptr = pthread_create; + } + } + + return false; +} + /* * When lazy lock is enabled, we need to make sure setting isthreaded before * taking any background_thread locks. This is called early in ctl (instead of @@ -805,7 +862,8 @@ void background_thread_ctl_init(tsdn_t *tsdn) { malloc_mutex_assert_not_owner(tsdn, &background_thread_lock); #ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER - pthread_once(&once_control, pthread_create_wrapper_once); + pthread_create_fptr_init(); + pthread_create_wrapper_init(); #endif } @@ -818,18 +876,10 @@ background_thread_boot0(void) { "supports pthread only\n"); return true; } - #ifdef JEMALLOC_PTHREAD_CREATE_WRAPPER - pthread_create_fptr = dlsym(RTLD_NEXT, "pthread_create"); - if (pthread_create_fptr == NULL) { - can_enable_background_thread = false; - if (config_lazy_lock || opt_background_thread) { - malloc_write(": Error in dlsym(RTLD_NEXT, " - "\"pthread_create\")\n"); - abort(); - } - } else { - can_enable_background_thread = true; + if ((config_lazy_lock || opt_background_thread) && + pthread_create_fptr_init()) { + return true; } #endif return false; @@ -841,6 +891,11 @@ background_thread_boot1(tsdn_t *tsdn) { assert(have_background_thread); assert(narenas_total_get() > 0); + if (opt_max_background_threads > MAX_BACKGROUND_THREAD_LIMIT) { + opt_max_background_threads = DEFAULT_NUM_BACKGROUND_THREAD; + } + max_background_threads = opt_max_background_threads; + background_thread_enabled_set(tsdn, opt_background_thread); if (malloc_mutex_init(&background_thread_lock, "background_thread_global", @@ -848,17 +903,15 @@ background_thread_boot1(tsdn_t *tsdn) { malloc_mutex_rank_exclusive)) { return true; } - if (opt_background_thread) { - background_thread_ctl_init(tsdn); - } background_thread_info = (background_thread_info_t *)base_alloc(tsdn, - b0get(), ncpus * sizeof(background_thread_info_t), CACHELINE); + b0get(), opt_max_background_threads * + sizeof(background_thread_info_t), CACHELINE); if (background_thread_info == NULL) { return true; } - for (unsigned i = 0; i < ncpus; i++) { + for (unsigned i = 0; i < max_background_threads; i++) { background_thread_info_t *info = &background_thread_info[i]; /* Thread mutex is rank_inclusive because of thread0. */ if (malloc_mutex_init(&info->mtx, "background_thread", diff --git a/kbe/src/lib/dependencies/jemalloc/src/base.c b/kbe/src/lib/dependencies/jemalloc/src/base.c index 97078b134d..f3c61661a2 100755 --- a/kbe/src/lib/dependencies/jemalloc/src/base.c +++ b/kbe/src/lib/dependencies/jemalloc/src/base.c @@ -10,25 +10,40 @@ /******************************************************************************/ /* Data. */ -static base_t *b0; +static base_t *b0; + +metadata_thp_mode_t opt_metadata_thp = METADATA_THP_DEFAULT; + +const char *metadata_thp_mode_names[] = { + "disabled", + "auto", + "always" +}; /******************************************************************************/ +static inline bool +metadata_thp_madvise(void) { + return (metadata_thp_enabled() && + (init_system_thp_mode == thp_mode_default)); +} + static void * base_map(tsdn_t *tsdn, extent_hooks_t *extent_hooks, unsigned ind, size_t size) { void *addr; bool zero = true; bool commit = true; + /* Use huge page sizes and alignment regardless of opt_metadata_thp. */ assert(size == HUGEPAGE_CEILING(size)); - + size_t alignment = HUGEPAGE; if (extent_hooks == &extent_hooks_default) { - addr = extent_alloc_mmap(NULL, size, PAGE, &zero, &commit); + addr = extent_alloc_mmap(NULL, size, alignment, &zero, &commit); } else { /* No arena context as we are creating new arenas. */ tsd_t *tsd = tsdn_null(tsdn) ? tsd_fetch() : tsdn_tsd(tsdn); pre_reentrancy(tsd, NULL); - addr = extent_hooks->alloc(extent_hooks, NULL, size, PAGE, + addr = extent_hooks->alloc(extent_hooks, NULL, size, alignment, &zero, &commit, ind); post_reentrancy(tsd); } @@ -51,16 +66,16 @@ base_unmap(tsdn_t *tsdn, extent_hooks_t *extent_hooks, unsigned ind, void *addr, */ if (extent_hooks == &extent_hooks_default) { if (!extent_dalloc_mmap(addr, size)) { - return; + goto label_done; } if (!pages_decommit(addr, size)) { - return; + goto label_done; } if (!pages_purge_forced(addr, size)) { - return; + goto label_done; } if (!pages_purge_lazy(addr, size)) { - return; + goto label_done; } /* Nothing worked. This should never happen. */ not_reached(); @@ -70,27 +85,33 @@ base_unmap(tsdn_t *tsdn, extent_hooks_t *extent_hooks, unsigned ind, void *addr, if (extent_hooks->dalloc != NULL && !extent_hooks->dalloc(extent_hooks, addr, size, true, ind)) { - goto label_done; + goto label_post_reentrancy; } if (extent_hooks->decommit != NULL && !extent_hooks->decommit(extent_hooks, addr, size, 0, size, ind)) { - goto label_done; + goto label_post_reentrancy; } if (extent_hooks->purge_forced != NULL && !extent_hooks->purge_forced(extent_hooks, addr, size, 0, size, ind)) { - goto label_done; + goto label_post_reentrancy; } if (extent_hooks->purge_lazy != NULL && !extent_hooks->purge_lazy(extent_hooks, addr, size, 0, size, ind)) { - goto label_done; + goto label_post_reentrancy; } /* Nothing worked. That's the application's problem. */ - label_done: + label_post_reentrancy: post_reentrancy(tsd); - return; + } +label_done: + if (metadata_thp_madvise()) { + /* Set NOHUGEPAGE after unmap to avoid kernel defrag. */ + assert(((uintptr_t)addr & HUGEPAGE_MASK) == 0 && + (size & HUGEPAGE_MASK) == 0); + pages_nohuge(addr, size); } } @@ -105,6 +126,56 @@ base_extent_init(size_t *extent_sn_next, extent_t *extent, void *addr, extent_binit(extent, addr, size, sn); } +static size_t +base_get_num_blocks(base_t *base, bool with_new_block) { + base_block_t *b = base->blocks; + assert(b != NULL); + + size_t n_blocks = with_new_block ? 2 : 1; + while (b->next != NULL) { + n_blocks++; + b = b->next; + } + + return n_blocks; +} + +static void +base_auto_thp_switch(tsdn_t *tsdn, base_t *base) { + assert(opt_metadata_thp == metadata_thp_auto); + malloc_mutex_assert_owner(tsdn, &base->mtx); + if (base->auto_thp_switched) { + return; + } + /* Called when adding a new block. */ + bool should_switch; + if (base_ind_get(base) != 0) { + should_switch = (base_get_num_blocks(base, true) == + BASE_AUTO_THP_THRESHOLD); + } else { + should_switch = (base_get_num_blocks(base, true) == + BASE_AUTO_THP_THRESHOLD_A0); + } + if (!should_switch) { + return; + } + + base->auto_thp_switched = true; + assert(!config_stats || base->n_thp == 0); + /* Make the initial blocks THP lazily. */ + base_block_t *block = base->blocks; + while (block != NULL) { + assert((block->size & HUGEPAGE_MASK) == 0); + pages_huge(block, block->size); + if (config_stats) { + base->n_thp += HUGEPAGE_CEILING(block->size - + extent_bsize_get(&block->extent)) >> LG_HUGEPAGE; + } + block = block->next; + assert(block == NULL || (base_ind_get(base) == 0)); + } +} + static void * base_extent_bump_alloc_helper(extent_t *extent, size_t *gap_size, size_t size, size_t alignment) { @@ -124,8 +195,8 @@ base_extent_bump_alloc_helper(extent_t *extent, size_t *gap_size, size_t size, } static void -base_extent_bump_alloc_post(tsdn_t *tsdn, base_t *base, extent_t *extent, - size_t gap_size, void *addr, size_t size) { +base_extent_bump_alloc_post(base_t *base, extent_t *extent, size_t gap_size, + void *addr, size_t size) { if (extent_bsize_get(extent) > 0) { /* * Compute the index for the largest size class that does not @@ -140,23 +211,31 @@ base_extent_bump_alloc_post(tsdn_t *tsdn, base_t *base, extent_t *extent, base->allocated += size; /* * Add one PAGE to base_resident for every page boundary that is - * crossed by the new allocation. + * crossed by the new allocation. Adjust n_thp similarly when + * metadata_thp is enabled. */ base->resident += PAGE_CEILING((uintptr_t)addr + size) - PAGE_CEILING((uintptr_t)addr - gap_size); assert(base->allocated <= base->resident); assert(base->resident <= base->mapped); + if (metadata_thp_madvise() && (opt_metadata_thp == + metadata_thp_always || base->auto_thp_switched)) { + base->n_thp += (HUGEPAGE_CEILING((uintptr_t)addr + size) + - HUGEPAGE_CEILING((uintptr_t)addr - gap_size)) >> + LG_HUGEPAGE; + assert(base->mapped >= base->n_thp << LG_HUGEPAGE); + } } } static void * -base_extent_bump_alloc(tsdn_t *tsdn, base_t *base, extent_t *extent, - size_t size, size_t alignment) { +base_extent_bump_alloc(base_t *base, extent_t *extent, size_t size, + size_t alignment) { void *ret; size_t gap_size; ret = base_extent_bump_alloc_helper(extent, &gap_size, size, alignment); - base_extent_bump_alloc_post(tsdn, base, extent, gap_size, ret, size); + base_extent_bump_alloc_post(base, extent, gap_size, ret, size); return ret; } @@ -166,8 +245,8 @@ base_extent_bump_alloc(tsdn_t *tsdn, base_t *base, extent_t *extent, * On success a pointer to the initialized base_block_t header is returned. */ static base_block_t * -base_block_alloc(tsdn_t *tsdn, extent_hooks_t *extent_hooks, unsigned ind, - pszind_t *pind_last, size_t *extent_sn_next, size_t size, +base_block_alloc(tsdn_t *tsdn, base_t *base, extent_hooks_t *extent_hooks, + unsigned ind, pszind_t *pind_last, size_t *extent_sn_next, size_t size, size_t alignment) { alignment = ALIGNMENT_CEILING(alignment, QUANTUM); size_t usize = ALIGNMENT_CEILING(size, alignment); @@ -183,8 +262,8 @@ base_block_alloc(tsdn_t *tsdn, extent_hooks_t *extent_hooks, unsigned ind, */ size_t min_block_size = HUGEPAGE_CEILING(sz_psz2u(header_size + gap_size + usize)); - pszind_t pind_next = (*pind_last + 1 < NPSIZES) ? *pind_last + 1 : - *pind_last; + pszind_t pind_next = (*pind_last + 1 < sz_psz2ind(SC_LARGE_MAXCLASS)) ? + *pind_last + 1 : *pind_last; size_t next_block_size = HUGEPAGE_CEILING(sz_pind2sz(pind_next)); size_t block_size = (min_block_size > next_block_size) ? min_block_size : next_block_size; @@ -193,6 +272,25 @@ base_block_alloc(tsdn_t *tsdn, extent_hooks_t *extent_hooks, unsigned ind, if (block == NULL) { return NULL; } + + if (metadata_thp_madvise()) { + void *addr = (void *)block; + assert(((uintptr_t)addr & HUGEPAGE_MASK) == 0 && + (block_size & HUGEPAGE_MASK) == 0); + if (opt_metadata_thp == metadata_thp_always) { + pages_huge(addr, block_size); + } else if (opt_metadata_thp == metadata_thp_auto && + base != NULL) { + /* base != NULL indicates this is not a new base. */ + malloc_mutex_lock(tsdn, &base->mtx); + base_auto_thp_switch(tsdn, base); + if (base->auto_thp_switched) { + pages_huge(addr, block_size); + } + malloc_mutex_unlock(tsdn, &base->mtx); + } + } + *pind_last = sz_psz2ind(block_size); block->size = block_size; block->next = NULL; @@ -216,7 +314,7 @@ base_extent_alloc(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment) { * called. */ malloc_mutex_unlock(tsdn, &base->mtx); - base_block_t *block = base_block_alloc(tsdn, extent_hooks, + base_block_t *block = base_block_alloc(tsdn, base, extent_hooks, base_ind_get(base), &base->pind_last, &base->extent_sn_next, size, alignment); malloc_mutex_lock(tsdn, &base->mtx); @@ -229,8 +327,16 @@ base_extent_alloc(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment) { base->allocated += sizeof(base_block_t); base->resident += PAGE_CEILING(sizeof(base_block_t)); base->mapped += block->size; + if (metadata_thp_madvise() && + !(opt_metadata_thp == metadata_thp_auto + && !base->auto_thp_switched)) { + assert(base->n_thp > 0); + base->n_thp += HUGEPAGE_CEILING(sizeof(base_block_t)) >> + LG_HUGEPAGE; + } assert(base->allocated <= base->resident); assert(base->resident <= base->mapped); + assert(base->n_thp << LG_HUGEPAGE <= base->mapped); } return &block->extent; } @@ -244,7 +350,7 @@ base_t * base_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { pszind_t pind_last = 0; size_t extent_sn_next = 0; - base_block_t *block = base_block_alloc(tsdn, extent_hooks, ind, + base_block_t *block = base_block_alloc(tsdn, NULL, extent_hooks, ind, &pind_last, &extent_sn_next, sizeof(base_t), QUANTUM); if (block == NULL) { return NULL; @@ -265,17 +371,22 @@ base_new(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { base->pind_last = pind_last; base->extent_sn_next = extent_sn_next; base->blocks = block; - for (szind_t i = 0; i < NSIZES; i++) { + base->auto_thp_switched = false; + for (szind_t i = 0; i < SC_NSIZES; i++) { extent_heap_new(&base->avail[i]); } if (config_stats) { base->allocated = sizeof(base_block_t); base->resident = PAGE_CEILING(sizeof(base_block_t)); base->mapped = block->size; + base->n_thp = (opt_metadata_thp == metadata_thp_always) && + metadata_thp_madvise() ? HUGEPAGE_CEILING(sizeof(base_block_t)) + >> LG_HUGEPAGE : 0; assert(base->allocated <= base->resident); assert(base->resident <= base->mapped); + assert(base->n_thp << LG_HUGEPAGE <= base->mapped); } - base_extent_bump_alloc_post(tsdn, base, &block->extent, gap_size, base, + base_extent_bump_alloc_post(base, &block->extent, gap_size, base, base_size); return base; @@ -315,7 +426,7 @@ base_alloc_impl(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment, extent_t *extent = NULL; malloc_mutex_lock(tsdn, &base->mtx); - for (szind_t i = sz_size2index(asize); i < NSIZES; i++) { + for (szind_t i = sz_size2index(asize); i < SC_NSIZES; i++) { extent = extent_heap_remove_first(&base->avail[i]); if (extent != NULL) { /* Use existing space. */ @@ -332,7 +443,7 @@ base_alloc_impl(tsdn_t *tsdn, base_t *base, size_t size, size_t alignment, goto label_return; } - ret = base_extent_bump_alloc(tsdn, base, extent, usize, alignment); + ret = base_extent_bump_alloc(base, extent, usize, alignment); if (esn != NULL) { *esn = extent_sn_get(extent); } @@ -368,7 +479,7 @@ base_alloc_extent(tsdn_t *tsdn, base_t *base) { void base_stats_get(tsdn_t *tsdn, base_t *base, size_t *allocated, size_t *resident, - size_t *mapped) { + size_t *mapped, size_t *n_thp) { cassert(config_stats); malloc_mutex_lock(tsdn, &base->mtx); @@ -377,6 +488,7 @@ base_stats_get(tsdn_t *tsdn, base_t *base, size_t *allocated, size_t *resident, *allocated = base->allocated; *resident = base->resident; *mapped = base->mapped; + *n_thp = base->n_thp; malloc_mutex_unlock(tsdn, &base->mtx); } diff --git a/kbe/src/lib/dependencies/jemalloc/src/bin.c b/kbe/src/lib/dependencies/jemalloc/src/bin.c new file mode 100644 index 0000000000..bca6b12c35 --- /dev/null +++ b/kbe/src/lib/dependencies/jemalloc/src/bin.c @@ -0,0 +1,95 @@ +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/jemalloc_internal_includes.h" + +#include "jemalloc/internal/assert.h" +#include "jemalloc/internal/bin.h" +#include "jemalloc/internal/sc.h" +#include "jemalloc/internal/witness.h" + +bin_info_t bin_infos[SC_NBINS]; + +static void +bin_infos_init(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS], + bin_info_t bin_infos[SC_NBINS]) { + for (unsigned i = 0; i < SC_NBINS; i++) { + bin_info_t *bin_info = &bin_infos[i]; + sc_t *sc = &sc_data->sc[i]; + bin_info->reg_size = ((size_t)1U << sc->lg_base) + + ((size_t)sc->ndelta << sc->lg_delta); + bin_info->slab_size = (sc->pgs << LG_PAGE); + bin_info->nregs = + (uint32_t)(bin_info->slab_size / bin_info->reg_size); + bin_info->n_shards = bin_shard_sizes[i]; + bitmap_info_t bitmap_info = BITMAP_INFO_INITIALIZER( + bin_info->nregs); + bin_info->bitmap_info = bitmap_info; + } +} + +bool +bin_update_shard_size(unsigned bin_shard_sizes[SC_NBINS], size_t start_size, + size_t end_size, size_t nshards) { + if (nshards > BIN_SHARDS_MAX || nshards == 0) { + return true; + } + + if (start_size > SC_SMALL_MAXCLASS) { + return false; + } + if (end_size > SC_SMALL_MAXCLASS) { + end_size = SC_SMALL_MAXCLASS; + } + + /* Compute the index since this may happen before sz init. */ + szind_t ind1 = sz_size2index_compute(start_size); + szind_t ind2 = sz_size2index_compute(end_size); + for (unsigned i = ind1; i <= ind2; i++) { + bin_shard_sizes[i] = (unsigned)nshards; + } + + return false; +} + +void +bin_shard_sizes_boot(unsigned bin_shard_sizes[SC_NBINS]) { + /* Load the default number of shards. */ + for (unsigned i = 0; i < SC_NBINS; i++) { + bin_shard_sizes[i] = N_BIN_SHARDS_DEFAULT; + } +} + +void +bin_boot(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS]) { + assert(sc_data->initialized); + bin_infos_init(sc_data, bin_shard_sizes, bin_infos); +} + +bool +bin_init(bin_t *bin) { + if (malloc_mutex_init(&bin->lock, "bin", WITNESS_RANK_BIN, + malloc_mutex_rank_exclusive)) { + return true; + } + bin->slabcur = NULL; + extent_heap_new(&bin->slabs_nonfull); + extent_list_init(&bin->slabs_full); + if (config_stats) { + memset(&bin->stats, 0, sizeof(bin_stats_t)); + } + return false; +} + +void +bin_prefork(tsdn_t *tsdn, bin_t *bin) { + malloc_mutex_prefork(tsdn, &bin->lock); +} + +void +bin_postfork_parent(tsdn_t *tsdn, bin_t *bin) { + malloc_mutex_postfork_parent(tsdn, &bin->lock); +} + +void +bin_postfork_child(tsdn_t *tsdn, bin_t *bin) { + malloc_mutex_postfork_child(tsdn, &bin->lock); +} diff --git a/kbe/src/lib/dependencies/jemalloc/src/ckh.c b/kbe/src/lib/dependencies/jemalloc/src/ckh.c index e95e0a3ed5..1bf6df5a11 100755 --- a/kbe/src/lib/dependencies/jemalloc/src/ckh.c +++ b/kbe/src/lib/dependencies/jemalloc/src/ckh.c @@ -275,7 +275,8 @@ ckh_grow(tsd_t *tsd, ckh_t *ckh) { lg_curcells++; usize = sz_sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE); - if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) { + if (unlikely(usize == 0 + || usize > SC_LARGE_MAXCLASS)) { ret = true; goto label_return; } @@ -320,7 +321,7 @@ ckh_shrink(tsd_t *tsd, ckh_t *ckh) { lg_prevbuckets = ckh->lg_curbuckets; lg_curcells = ckh->lg_curbuckets + LG_CKH_BUCKET_CELLS - 1; usize = sz_sa2u(sizeof(ckhc_t) << lg_curcells, CACHELINE); - if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) { + if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) { return; } tab = (ckhc_t *)ipallocztm(tsd_tsdn(tsd), usize, CACHELINE, true, NULL, @@ -396,7 +397,7 @@ ckh_new(tsd_t *tsd, ckh_t *ckh, size_t minitems, ckh_hash_t *hash, ckh->keycomp = keycomp; usize = sz_sa2u(sizeof(ckhc_t) << lg_mincells, CACHELINE); - if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) { + if (unlikely(usize == 0 || usize > SC_LARGE_MAXCLASS)) { ret = true; goto label_return; } diff --git a/kbe/src/lib/dependencies/jemalloc/src/ctl.c b/kbe/src/lib/dependencies/jemalloc/src/ctl.c index 36bc8fb5b7..09310a9d19 100755 --- a/kbe/src/lib/dependencies/jemalloc/src/ctl.c +++ b/kbe/src/lib/dependencies/jemalloc/src/ctl.c @@ -8,7 +8,7 @@ #include "jemalloc/internal/extent_mmap.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/nstime.h" -#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/sc.h" #include "jemalloc/internal/util.h" /******************************************************************************/ @@ -57,6 +57,7 @@ static const ctl_named_node_t *n##_index(tsdn_t *tsdn, \ CTL_PROTO(version) CTL_PROTO(epoch) CTL_PROTO(background_thread) +CTL_PROTO(max_background_threads) CTL_PROTO(thread_tcache_enabled) CTL_PROTO(thread_tcache_flush) CTL_PROTO(thread_prof_name) @@ -75,16 +76,18 @@ CTL_PROTO(config_prof) CTL_PROTO(config_prof_libgcc) CTL_PROTO(config_prof_libunwind) CTL_PROTO(config_stats) -CTL_PROTO(config_thp) CTL_PROTO(config_utrace) CTL_PROTO(config_xmalloc) CTL_PROTO(opt_abort) CTL_PROTO(opt_abort_conf) +CTL_PROTO(opt_metadata_thp) CTL_PROTO(opt_retain) CTL_PROTO(opt_dss) CTL_PROTO(opt_narenas) CTL_PROTO(opt_percpu_arena) +CTL_PROTO(opt_oversize_threshold) CTL_PROTO(opt_background_thread) +CTL_PROTO(opt_max_background_threads) CTL_PROTO(opt_dirty_decay_ms) CTL_PROTO(opt_muzzy_decay_ms) CTL_PROTO(opt_stats_print) @@ -94,6 +97,8 @@ CTL_PROTO(opt_zero) CTL_PROTO(opt_utrace) CTL_PROTO(opt_xmalloc) CTL_PROTO(opt_tcache) +CTL_PROTO(opt_thp) +CTL_PROTO(opt_lg_extent_max_active_fit) CTL_PROTO(opt_lg_tcache_max) CTL_PROTO(opt_prof) CTL_PROTO(opt_prof_prefix) @@ -117,10 +122,12 @@ CTL_PROTO(arena_i_dss) CTL_PROTO(arena_i_dirty_decay_ms) CTL_PROTO(arena_i_muzzy_decay_ms) CTL_PROTO(arena_i_extent_hooks) +CTL_PROTO(arena_i_retain_grow_limit) INDEX_PROTO(arena_i) CTL_PROTO(arenas_bin_i_size) CTL_PROTO(arenas_bin_i_nregs) CTL_PROTO(arenas_bin_i_slab_size) +CTL_PROTO(arenas_bin_i_nshards) INDEX_PROTO(arenas_bin_i) CTL_PROTO(arenas_lextent_i_size) INDEX_PROTO(arenas_lextent_i) @@ -134,6 +141,7 @@ CTL_PROTO(arenas_nbins) CTL_PROTO(arenas_nhbins) CTL_PROTO(arenas_nlextents) CTL_PROTO(arenas_create) +CTL_PROTO(arenas_lookup) CTL_PROTO(prof_thread_active_init) CTL_PROTO(prof_active) CTL_PROTO(prof_dump) @@ -141,6 +149,8 @@ CTL_PROTO(prof_gdump) CTL_PROTO(prof_reset) CTL_PROTO(prof_interval) CTL_PROTO(lg_prof_sample) +CTL_PROTO(prof_log_start) +CTL_PROTO(prof_log_stop) CTL_PROTO(stats_arenas_i_small_allocated) CTL_PROTO(stats_arenas_i_small_nmalloc) CTL_PROTO(stats_arenas_i_small_ndalloc) @@ -164,6 +174,13 @@ CTL_PROTO(stats_arenas_i_lextents_j_ndalloc) CTL_PROTO(stats_arenas_i_lextents_j_nrequests) CTL_PROTO(stats_arenas_i_lextents_j_curlextents) INDEX_PROTO(stats_arenas_i_lextents_j) +CTL_PROTO(stats_arenas_i_extents_j_ndirty) +CTL_PROTO(stats_arenas_i_extents_j_nmuzzy) +CTL_PROTO(stats_arenas_i_extents_j_nretained) +CTL_PROTO(stats_arenas_i_extents_j_dirty_bytes) +CTL_PROTO(stats_arenas_i_extents_j_muzzy_bytes) +CTL_PROTO(stats_arenas_i_extents_j_retained_bytes) +INDEX_PROTO(stats_arenas_i_extents_j) CTL_PROTO(stats_arenas_i_nthreads) CTL_PROTO(stats_arenas_i_uptime) CTL_PROTO(stats_arenas_i_dss) @@ -174,6 +191,7 @@ CTL_PROTO(stats_arenas_i_pdirty) CTL_PROTO(stats_arenas_i_pmuzzy) CTL_PROTO(stats_arenas_i_mapped) CTL_PROTO(stats_arenas_i_retained) +CTL_PROTO(stats_arenas_i_extent_avail) CTL_PROTO(stats_arenas_i_dirty_npurge) CTL_PROTO(stats_arenas_i_dirty_nmadvise) CTL_PROTO(stats_arenas_i_dirty_purged) @@ -182,6 +200,7 @@ CTL_PROTO(stats_arenas_i_muzzy_nmadvise) CTL_PROTO(stats_arenas_i_muzzy_purged) CTL_PROTO(stats_arenas_i_base) CTL_PROTO(stats_arenas_i_internal) +CTL_PROTO(stats_arenas_i_metadata_thp) CTL_PROTO(stats_arenas_i_tcache_bytes) CTL_PROTO(stats_arenas_i_resident) INDEX_PROTO(stats_arenas_i) @@ -191,9 +210,12 @@ CTL_PROTO(stats_background_thread_num_threads) CTL_PROTO(stats_background_thread_num_runs) CTL_PROTO(stats_background_thread_run_interval) CTL_PROTO(stats_metadata) +CTL_PROTO(stats_metadata_thp) CTL_PROTO(stats_resident) CTL_PROTO(stats_mapped) CTL_PROTO(stats_retained) +CTL_PROTO(experimental_hooks_install) +CTL_PROTO(experimental_hooks_remove) #define MUTEX_STATS_CTL_PROTO_GEN(n) \ CTL_PROTO(stats_##n##_num_ops) \ @@ -266,7 +288,6 @@ static const ctl_named_node_t config_node[] = { {NAME("prof_libgcc"), CTL(config_prof_libgcc)}, {NAME("prof_libunwind"), CTL(config_prof_libunwind)}, {NAME("stats"), CTL(config_stats)}, - {NAME("thp"), CTL(config_thp)}, {NAME("utrace"), CTL(config_utrace)}, {NAME("xmalloc"), CTL(config_xmalloc)} }; @@ -274,11 +295,14 @@ static const ctl_named_node_t config_node[] = { static const ctl_named_node_t opt_node[] = { {NAME("abort"), CTL(opt_abort)}, {NAME("abort_conf"), CTL(opt_abort_conf)}, + {NAME("metadata_thp"), CTL(opt_metadata_thp)}, {NAME("retain"), CTL(opt_retain)}, {NAME("dss"), CTL(opt_dss)}, {NAME("narenas"), CTL(opt_narenas)}, {NAME("percpu_arena"), CTL(opt_percpu_arena)}, + {NAME("oversize_threshold"), CTL(opt_oversize_threshold)}, {NAME("background_thread"), CTL(opt_background_thread)}, + {NAME("max_background_threads"), CTL(opt_max_background_threads)}, {NAME("dirty_decay_ms"), CTL(opt_dirty_decay_ms)}, {NAME("muzzy_decay_ms"), CTL(opt_muzzy_decay_ms)}, {NAME("stats_print"), CTL(opt_stats_print)}, @@ -288,6 +312,8 @@ static const ctl_named_node_t opt_node[] = { {NAME("utrace"), CTL(opt_utrace)}, {NAME("xmalloc"), CTL(opt_xmalloc)}, {NAME("tcache"), CTL(opt_tcache)}, + {NAME("thp"), CTL(opt_thp)}, + {NAME("lg_extent_max_active_fit"), CTL(opt_lg_extent_max_active_fit)}, {NAME("lg_tcache_max"), CTL(opt_lg_tcache_max)}, {NAME("prof"), CTL(opt_prof)}, {NAME("prof_prefix"), CTL(opt_prof_prefix)}, @@ -316,7 +342,8 @@ static const ctl_named_node_t arena_i_node[] = { {NAME("dss"), CTL(arena_i_dss)}, {NAME("dirty_decay_ms"), CTL(arena_i_dirty_decay_ms)}, {NAME("muzzy_decay_ms"), CTL(arena_i_muzzy_decay_ms)}, - {NAME("extent_hooks"), CTL(arena_i_extent_hooks)} + {NAME("extent_hooks"), CTL(arena_i_extent_hooks)}, + {NAME("retain_grow_limit"), CTL(arena_i_retain_grow_limit)} }; static const ctl_named_node_t super_arena_i_node[] = { {NAME(""), CHILD(named, arena_i)} @@ -329,7 +356,8 @@ static const ctl_indexed_node_t arena_node[] = { static const ctl_named_node_t arenas_bin_i_node[] = { {NAME("size"), CTL(arenas_bin_i_size)}, {NAME("nregs"), CTL(arenas_bin_i_nregs)}, - {NAME("slab_size"), CTL(arenas_bin_i_slab_size)} + {NAME("slab_size"), CTL(arenas_bin_i_slab_size)}, + {NAME("nshards"), CTL(arenas_bin_i_nshards)} }; static const ctl_named_node_t super_arenas_bin_i_node[] = { {NAME(""), CHILD(named, arenas_bin_i)} @@ -362,7 +390,8 @@ static const ctl_named_node_t arenas_node[] = { {NAME("bin"), CHILD(indexed, arenas_bin)}, {NAME("nlextents"), CTL(arenas_nlextents)}, {NAME("lextent"), CHILD(indexed, arenas_lextent)}, - {NAME("create"), CTL(arenas_create)} + {NAME("create"), CTL(arenas_create)}, + {NAME("lookup"), CTL(arenas_lookup)} }; static const ctl_named_node_t prof_node[] = { @@ -372,9 +401,10 @@ static const ctl_named_node_t prof_node[] = { {NAME("gdump"), CTL(prof_gdump)}, {NAME("reset"), CTL(prof_reset)}, {NAME("interval"), CTL(prof_interval)}, - {NAME("lg_sample"), CTL(lg_prof_sample)} + {NAME("lg_sample"), CTL(lg_prof_sample)}, + {NAME("log_start"), CTL(prof_log_start)}, + {NAME("log_stop"), CTL(prof_log_stop)} }; - static const ctl_named_node_t stats_arenas_i_small_node[] = { {NAME("allocated"), CTL(stats_arenas_i_small_allocated)}, {NAME("nmalloc"), CTL(stats_arenas_i_small_nmalloc)}, @@ -445,6 +475,23 @@ static const ctl_indexed_node_t stats_arenas_i_lextents_node[] = { {INDEX(stats_arenas_i_lextents_j)} }; +static const ctl_named_node_t stats_arenas_i_extents_j_node[] = { + {NAME("ndirty"), CTL(stats_arenas_i_extents_j_ndirty)}, + {NAME("nmuzzy"), CTL(stats_arenas_i_extents_j_nmuzzy)}, + {NAME("nretained"), CTL(stats_arenas_i_extents_j_nretained)}, + {NAME("dirty_bytes"), CTL(stats_arenas_i_extents_j_dirty_bytes)}, + {NAME("muzzy_bytes"), CTL(stats_arenas_i_extents_j_muzzy_bytes)}, + {NAME("retained_bytes"), CTL(stats_arenas_i_extents_j_retained_bytes)} +}; + +static const ctl_named_node_t super_stats_arenas_i_extents_j_node[] = { + {NAME(""), CHILD(named, stats_arenas_i_extents_j)} +}; + +static const ctl_indexed_node_t stats_arenas_i_extents_node[] = { + {INDEX(stats_arenas_i_extents_j)} +}; + #define OP(mtx) MUTEX_PROF_DATA_NODE(arenas_i_mutexes_##mtx) MUTEX_PROF_ARENA_MUTEXES #undef OP @@ -466,6 +513,7 @@ static const ctl_named_node_t stats_arenas_i_node[] = { {NAME("pmuzzy"), CTL(stats_arenas_i_pmuzzy)}, {NAME("mapped"), CTL(stats_arenas_i_mapped)}, {NAME("retained"), CTL(stats_arenas_i_retained)}, + {NAME("extent_avail"), CTL(stats_arenas_i_extent_avail)}, {NAME("dirty_npurge"), CTL(stats_arenas_i_dirty_npurge)}, {NAME("dirty_nmadvise"), CTL(stats_arenas_i_dirty_nmadvise)}, {NAME("dirty_purged"), CTL(stats_arenas_i_dirty_purged)}, @@ -474,12 +522,14 @@ static const ctl_named_node_t stats_arenas_i_node[] = { {NAME("muzzy_purged"), CTL(stats_arenas_i_muzzy_purged)}, {NAME("base"), CTL(stats_arenas_i_base)}, {NAME("internal"), CTL(stats_arenas_i_internal)}, + {NAME("metadata_thp"), CTL(stats_arenas_i_metadata_thp)}, {NAME("tcache_bytes"), CTL(stats_arenas_i_tcache_bytes)}, {NAME("resident"), CTL(stats_arenas_i_resident)}, {NAME("small"), CHILD(named, stats_arenas_i_small)}, {NAME("large"), CHILD(named, stats_arenas_i_large)}, {NAME("bins"), CHILD(indexed, stats_arenas_i_bins)}, {NAME("lextents"), CHILD(indexed, stats_arenas_i_lextents)}, + {NAME("extents"), CHILD(indexed, stats_arenas_i_extents)}, {NAME("mutexes"), CHILD(named, stats_arenas_i_mutexes)} }; static const ctl_named_node_t super_stats_arenas_i_node[] = { @@ -512,6 +562,7 @@ static const ctl_named_node_t stats_node[] = { {NAME("allocated"), CTL(stats_allocated)}, {NAME("active"), CTL(stats_active)}, {NAME("metadata"), CTL(stats_metadata)}, + {NAME("metadata_thp"), CTL(stats_metadata_thp)}, {NAME("resident"), CTL(stats_resident)}, {NAME("mapped"), CTL(stats_mapped)}, {NAME("retained"), CTL(stats_retained)}, @@ -521,10 +572,20 @@ static const ctl_named_node_t stats_node[] = { {NAME("arenas"), CHILD(indexed, stats_arenas)} }; +static const ctl_named_node_t hooks_node[] = { + {NAME("install"), CTL(experimental_hooks_install)}, + {NAME("remove"), CTL(experimental_hooks_remove)}, +}; + +static const ctl_named_node_t experimental_node[] = { + {NAME("hooks"), CHILD(named, hooks)} +}; + static const ctl_named_node_t root_node[] = { {NAME("version"), CTL(version)}, {NAME("epoch"), CTL(epoch)}, {NAME("background_thread"), CTL(background_thread)}, + {NAME("max_background_threads"), CTL(max_background_threads)}, {NAME("thread"), CHILD(named, thread)}, {NAME("config"), CHILD(named, config)}, {NAME("opt"), CHILD(named, opt)}, @@ -532,7 +593,8 @@ static const ctl_named_node_t root_node[] = { {NAME("arena"), CHILD(indexed, arena)}, {NAME("arenas"), CHILD(named, arenas)}, {NAME("prof"), CHILD(named, prof)}, - {NAME("stats"), CHILD(named, stats)} + {NAME("stats"), CHILD(named, stats)}, + {NAME("experimental"), CHILD(named, experimental)} }; static const ctl_named_node_t super_root_node[] = { {NAME(""), CHILD(named, root)} @@ -550,7 +612,7 @@ static const ctl_named_node_t super_root_node[] = { * synchronized by the ctl mutex. */ static void -accum_arena_stats_u64(arena_stats_u64_t *dst, arena_stats_u64_t *src) { +ctl_accum_arena_stats_u64(arena_stats_u64_t *dst, arena_stats_u64_t *src) { #ifdef JEMALLOC_ATOMIC_U64 uint64_t cur_dst = atomic_load_u64(dst, ATOMIC_RELAXED); uint64_t cur_src = atomic_load_u64(src, ATOMIC_RELAXED); @@ -562,7 +624,7 @@ accum_arena_stats_u64(arena_stats_u64_t *dst, arena_stats_u64_t *src) { /* Likewise: with ctl mutex synchronization, reading is simple. */ static uint64_t -arena_stats_read_u64(arena_stats_u64_t *p) { +ctl_arena_stats_read_u64(arena_stats_u64_t *p) { #ifdef JEMALLOC_ATOMIC_U64 return atomic_load_u64(p, ATOMIC_RELAXED); #else @@ -570,7 +632,8 @@ arena_stats_read_u64(arena_stats_u64_t *p) { #endif } -static void accum_atomic_zu(atomic_zu_t *dst, atomic_zu_t *src) { +static void +accum_atomic_zu(atomic_zu_t *dst, atomic_zu_t *src) { size_t cur_dst = atomic_load_zu(dst, ATOMIC_RELAXED); size_t cur_src = atomic_load_zu(src, ATOMIC_RELAXED); atomic_store_zu(dst, cur_dst + cur_src, ATOMIC_RELAXED); @@ -679,10 +742,12 @@ ctl_arena_clear(ctl_arena_t *ctl_arena) { ctl_arena->astats->nmalloc_small = 0; ctl_arena->astats->ndalloc_small = 0; ctl_arena->astats->nrequests_small = 0; - memset(ctl_arena->astats->bstats, 0, NBINS * - sizeof(malloc_bin_stats_t)); - memset(ctl_arena->astats->lstats, 0, (NSIZES - NBINS) * - sizeof(malloc_large_stats_t)); + memset(ctl_arena->astats->bstats, 0, SC_NBINS * + sizeof(bin_stats_t)); + memset(ctl_arena->astats->lstats, 0, (SC_NSIZES - SC_NBINS) * + sizeof(arena_stats_large_t)); + memset(ctl_arena->astats->estats, 0, SC_NPSIZES * + sizeof(arena_stats_extents_t)); } } @@ -696,9 +761,9 @@ ctl_arena_stats_amerge(tsdn_t *tsdn, ctl_arena_t *ctl_arena, arena_t *arena) { &ctl_arena->muzzy_decay_ms, &ctl_arena->pactive, &ctl_arena->pdirty, &ctl_arena->pmuzzy, &ctl_arena->astats->astats, ctl_arena->astats->bstats, - ctl_arena->astats->lstats); + ctl_arena->astats->lstats, ctl_arena->astats->estats); - for (i = 0; i < NBINS; i++) { + for (i = 0; i < SC_NBINS; i++) { ctl_arena->astats->allocated_small += ctl_arena->astats->bstats[i].curregs * sz_index2size(i); @@ -743,20 +808,22 @@ ctl_arena_stats_sdmerge(ctl_arena_t *ctl_sdarena, ctl_arena_t *ctl_arena, &astats->astats.mapped); accum_atomic_zu(&sdstats->astats.retained, &astats->astats.retained); + accum_atomic_zu(&sdstats->astats.extent_avail, + &astats->astats.extent_avail); } - accum_arena_stats_u64(&sdstats->astats.decay_dirty.npurge, + ctl_accum_arena_stats_u64(&sdstats->astats.decay_dirty.npurge, &astats->astats.decay_dirty.npurge); - accum_arena_stats_u64(&sdstats->astats.decay_dirty.nmadvise, + ctl_accum_arena_stats_u64(&sdstats->astats.decay_dirty.nmadvise, &astats->astats.decay_dirty.nmadvise); - accum_arena_stats_u64(&sdstats->astats.decay_dirty.purged, + ctl_accum_arena_stats_u64(&sdstats->astats.decay_dirty.purged, &astats->astats.decay_dirty.purged); - accum_arena_stats_u64(&sdstats->astats.decay_muzzy.npurge, + ctl_accum_arena_stats_u64(&sdstats->astats.decay_muzzy.npurge, &astats->astats.decay_muzzy.npurge); - accum_arena_stats_u64(&sdstats->astats.decay_muzzy.nmadvise, + ctl_accum_arena_stats_u64(&sdstats->astats.decay_muzzy.nmadvise, &astats->astats.decay_muzzy.nmadvise); - accum_arena_stats_u64(&sdstats->astats.decay_muzzy.purged, + ctl_accum_arena_stats_u64(&sdstats->astats.decay_muzzy.purged, &astats->astats.decay_muzzy.purged); #define OP(mtx) malloc_mutex_prof_merge( \ @@ -773,6 +840,8 @@ MUTEX_PROF_ARENA_MUTEXES &astats->astats.internal); accum_atomic_zu(&sdstats->astats.resident, &astats->astats.resident); + accum_atomic_zu(&sdstats->astats.metadata_thp, + &astats->astats.metadata_thp); } else { assert(atomic_load_zu( &astats->astats.internal, ATOMIC_RELAXED) == 0); @@ -794,11 +863,11 @@ MUTEX_PROF_ARENA_MUTEXES assert(atomic_load_zu(&astats->astats.allocated_large, ATOMIC_RELAXED) == 0); } - accum_arena_stats_u64(&sdstats->astats.nmalloc_large, + ctl_accum_arena_stats_u64(&sdstats->astats.nmalloc_large, &astats->astats.nmalloc_large); - accum_arena_stats_u64(&sdstats->astats.ndalloc_large, + ctl_accum_arena_stats_u64(&sdstats->astats.ndalloc_large, &astats->astats.ndalloc_large); - accum_arena_stats_u64(&sdstats->astats.nrequests_large, + ctl_accum_arena_stats_u64(&sdstats->astats.nrequests_large, &astats->astats.nrequests_large); accum_atomic_zu(&sdstats->astats.tcache_bytes, @@ -808,7 +877,8 @@ MUTEX_PROF_ARENA_MUTEXES sdstats->astats.uptime = astats->astats.uptime; } - for (i = 0; i < NBINS; i++) { + /* Merge bin stats. */ + for (i = 0; i < SC_NBINS; i++) { sdstats->bstats[i].nmalloc += astats->bstats[i].nmalloc; sdstats->bstats[i].ndalloc += astats->bstats[i].ndalloc; sdstats->bstats[i].nrequests += @@ -834,12 +904,13 @@ MUTEX_PROF_ARENA_MUTEXES &astats->bstats[i].mutex_data); } - for (i = 0; i < NSIZES - NBINS; i++) { - accum_arena_stats_u64(&sdstats->lstats[i].nmalloc, + /* Merge stats for large allocations. */ + for (i = 0; i < SC_NSIZES - SC_NBINS; i++) { + ctl_accum_arena_stats_u64(&sdstats->lstats[i].nmalloc, &astats->lstats[i].nmalloc); - accum_arena_stats_u64(&sdstats->lstats[i].ndalloc, + ctl_accum_arena_stats_u64(&sdstats->lstats[i].ndalloc, &astats->lstats[i].ndalloc); - accum_arena_stats_u64(&sdstats->lstats[i].nrequests, + ctl_accum_arena_stats_u64(&sdstats->lstats[i].nrequests, &astats->lstats[i].nrequests); if (!destroyed) { sdstats->lstats[i].curlextents += @@ -848,6 +919,22 @@ MUTEX_PROF_ARENA_MUTEXES assert(astats->lstats[i].curlextents == 0); } } + + /* Merge extents stats. */ + for (i = 0; i < SC_NPSIZES; i++) { + accum_atomic_zu(&sdstats->estats[i].ndirty, + &astats->estats[i].ndirty); + accum_atomic_zu(&sdstats->estats[i].nmuzzy, + &astats->estats[i].nmuzzy); + accum_atomic_zu(&sdstats->estats[i].nretained, + &astats->estats[i].nretained); + accum_atomic_zu(&sdstats->estats[i].dirty_bytes, + &astats->estats[i].dirty_bytes); + accum_atomic_zu(&sdstats->estats[i].muzzy_bytes, + &astats->estats[i].muzzy_bytes); + accum_atomic_zu(&sdstats->estats[i].retained_bytes, + &astats->estats[i].retained_bytes); + } } } @@ -938,6 +1025,8 @@ ctl_refresh(tsdn_t *tsdn) { &ctl_sarena->astats->astats.base, ATOMIC_RELAXED) + atomic_load_zu(&ctl_sarena->astats->astats.internal, ATOMIC_RELAXED); + ctl_stats->metadata_thp = atomic_load_zu( + &ctl_sarena->astats->astats.metadata_thp, ATOMIC_RELAXED); ctl_stats->resident = atomic_load_zu( &ctl_sarena->astats->astats.resident, ATOMIC_RELAXED); ctl_stats->mapped = atomic_load_zu( @@ -1357,8 +1446,8 @@ label_return: \ #define CTL_RO_CGEN(c, n, v, t) \ static int \ -n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \ - size_t *oldlenp, void *newp, size_t newlen) { \ +n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \ + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { \ int ret; \ t oldval; \ \ @@ -1400,8 +1489,8 @@ label_return: \ */ #define CTL_RO_NL_CGEN(c, n, v, t) \ static int \ -n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \ - size_t *oldlenp, void *newp, size_t newlen) { \ +n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \ + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { \ int ret; \ t oldval; \ \ @@ -1419,8 +1508,8 @@ label_return: \ #define CTL_RO_NL_GEN(n, v, t) \ static int \ -n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \ - size_t *oldlenp, void *newp, size_t newlen) { \ +n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \ + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { \ int ret; \ t oldval; \ \ @@ -1454,8 +1543,8 @@ label_return: \ #define CTL_RO_CONFIG_GEN(n, t) \ static int \ -n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, \ - size_t *oldlenp, void *newp, size_t newlen) { \ +n##_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, \ + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { \ int ret; \ t oldval; \ \ @@ -1473,8 +1562,8 @@ label_return: \ CTL_RO_NL_GEN(version, JEMALLOC_VERSION, const char *) static int -epoch_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) { +epoch_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; UNUSED uint64_t newval; @@ -1492,8 +1581,9 @@ epoch_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, } static int -background_thread_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { +background_thread_ctl(tsd_t *tsd, const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, + void *newp, size_t newlen) { int ret; bool oldval; @@ -1523,22 +1613,74 @@ background_thread_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, background_thread_enabled_set(tsd_tsdn(tsd), newval); if (newval) { - if (!can_enable_background_thread) { - malloc_printf(": Error in dlsym(" - "RTLD_NEXT, \"pthread_create\"). Cannot " - "enable background_thread\n"); + if (background_threads_enable(tsd)) { ret = EFAULT; goto label_return; } - if (background_threads_enable(tsd)) { + } else { + if (background_threads_disable(tsd)) { ret = EFAULT; goto label_return; } - } else { + } + } + ret = 0; +label_return: + malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock); + malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); + + return ret; +} + +static int +max_background_threads_ctl(tsd_t *tsd, const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, void *newp, + size_t newlen) { + int ret; + size_t oldval; + + if (!have_background_thread) { + return ENOENT; + } + background_thread_ctl_init(tsd_tsdn(tsd)); + + malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); + malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock); + if (newp == NULL) { + oldval = max_background_threads; + READ(oldval, size_t); + } else { + if (newlen != sizeof(size_t)) { + ret = EINVAL; + goto label_return; + } + oldval = max_background_threads; + READ(oldval, size_t); + + size_t newval = *(size_t *)newp; + if (newval == oldval) { + ret = 0; + goto label_return; + } + if (newval > opt_max_background_threads) { + ret = EINVAL; + goto label_return; + } + + if (background_thread_enabled()) { + background_thread_enabled_set(tsd_tsdn(tsd), false); if (background_threads_disable(tsd)) { ret = EFAULT; goto label_return; } + max_background_threads = newval; + background_thread_enabled_set(tsd_tsdn(tsd), true); + if (background_threads_enable(tsd)) { + ret = EFAULT; + goto label_return; + } + } else { + max_background_threads = newval; } } ret = 0; @@ -1560,7 +1702,6 @@ CTL_RO_CONFIG_GEN(config_prof, bool) CTL_RO_CONFIG_GEN(config_prof_libgcc, bool) CTL_RO_CONFIG_GEN(config_prof_libunwind, bool) CTL_RO_CONFIG_GEN(config_stats, bool) -CTL_RO_CONFIG_GEN(config_thp, bool) CTL_RO_CONFIG_GEN(config_utrace, bool) CTL_RO_CONFIG_GEN(config_xmalloc, bool) @@ -1568,12 +1709,16 @@ CTL_RO_CONFIG_GEN(config_xmalloc, bool) CTL_RO_NL_GEN(opt_abort, opt_abort, bool) CTL_RO_NL_GEN(opt_abort_conf, opt_abort_conf, bool) +CTL_RO_NL_GEN(opt_metadata_thp, metadata_thp_mode_names[opt_metadata_thp], + const char *) CTL_RO_NL_GEN(opt_retain, opt_retain, bool) CTL_RO_NL_GEN(opt_dss, opt_dss, const char *) CTL_RO_NL_GEN(opt_narenas, opt_narenas, unsigned) CTL_RO_NL_GEN(opt_percpu_arena, percpu_arena_mode_names[opt_percpu_arena], const char *) +CTL_RO_NL_GEN(opt_oversize_threshold, opt_oversize_threshold, size_t) CTL_RO_NL_GEN(opt_background_thread, opt_background_thread, bool) +CTL_RO_NL_GEN(opt_max_background_threads, opt_max_background_threads, size_t) CTL_RO_NL_GEN(opt_dirty_decay_ms, opt_dirty_decay_ms, ssize_t) CTL_RO_NL_GEN(opt_muzzy_decay_ms, opt_muzzy_decay_ms, ssize_t) CTL_RO_NL_GEN(opt_stats_print, opt_stats_print, bool) @@ -1583,6 +1728,9 @@ CTL_RO_NL_CGEN(config_fill, opt_zero, opt_zero, bool) CTL_RO_NL_CGEN(config_utrace, opt_utrace, opt_utrace, bool) CTL_RO_NL_CGEN(config_xmalloc, opt_xmalloc, opt_xmalloc, bool) CTL_RO_NL_GEN(opt_tcache, opt_tcache, bool) +CTL_RO_NL_GEN(opt_thp, thp_mode_names[opt_thp], const char *) +CTL_RO_NL_GEN(opt_lg_extent_max_active_fit, opt_lg_extent_max_active_fit, + size_t) CTL_RO_NL_GEN(opt_lg_tcache_max, opt_lg_tcache_max, ssize_t) CTL_RO_NL_CGEN(config_prof, opt_prof, opt_prof, bool) CTL_RO_NL_CGEN(config_prof, opt_prof_prefix, opt_prof_prefix, const char *) @@ -1599,8 +1747,8 @@ CTL_RO_NL_CGEN(config_prof, opt_prof_leak, opt_prof_leak, bool) /******************************************************************************/ static int -thread_arena_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) { +thread_arena_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; arena_t *oldarena; unsigned newind, oldind; @@ -1664,8 +1812,9 @@ CTL_TSD_RO_NL_CGEN(config_stats, thread_deallocatedp, tsd_thread_deallocatedp_get, uint64_t *) static int -thread_tcache_enabled_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { +thread_tcache_enabled_ctl(tsd_t *tsd, const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, void *newp, + size_t newlen) { int ret; bool oldval; @@ -1685,8 +1834,9 @@ thread_tcache_enabled_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, } static int -thread_tcache_flush_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { +thread_tcache_flush_ctl(tsd_t *tsd, const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, void *newp, + size_t newlen) { int ret; if (!tcache_available(tsd)) { @@ -1705,8 +1855,9 @@ thread_tcache_flush_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, } static int -thread_prof_name_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) { +thread_prof_name_ctl(tsd_t *tsd, const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, void *newp, + size_t newlen) { int ret; if (!config_prof) { @@ -1736,8 +1887,9 @@ thread_prof_name_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, } static int -thread_prof_active_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) { +thread_prof_active_ctl(tsd_t *tsd, const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, void *newp, + size_t newlen) { int ret; bool oldval; @@ -1766,8 +1918,8 @@ thread_prof_active_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, /******************************************************************************/ static int -tcache_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) { +tcache_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; unsigned tcache_ind; @@ -1784,8 +1936,8 @@ tcache_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, } static int -tcache_flush_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) { +tcache_flush_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; unsigned tcache_ind; @@ -1804,8 +1956,8 @@ tcache_flush_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, } static int -tcache_destroy_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) { +tcache_destroy_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; unsigned tcache_ind; @@ -1953,9 +2105,8 @@ arena_reset_prepare_background_thread(tsd_t *tsd, unsigned arena_ind) { if (have_background_thread) { malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock); if (background_thread_enabled()) { - unsigned ind = arena_ind % ncpus; background_thread_info_t *info = - &background_thread_info[ind]; + background_thread_info_get(arena_ind); assert(info->state == background_thread_started); malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx); info->state = background_thread_paused; @@ -1968,9 +2119,8 @@ static void arena_reset_finish_background_thread(tsd_t *tsd, unsigned arena_ind) { if (have_background_thread) { if (background_thread_enabled()) { - unsigned ind = arena_ind % ncpus; background_thread_info_t *info = - &background_thread_info[ind]; + background_thread_info_get(arena_ind); assert(info->state == background_thread_paused); malloc_mutex_lock(tsd_tsdn(tsd), &info->mtx); info->state = background_thread_started; @@ -2126,6 +2276,17 @@ arena_i_decay_ms_ctl_impl(tsd_t *tsd, const size_t *mib, size_t miblen, ret = EINVAL; goto label_return; } + if (arena_is_huge(arena_ind) && *(ssize_t *)newp > 0) { + /* + * By default the huge arena purges eagerly. If it is + * set to non-zero decay time afterwards, background + * thread might be needed. + */ + if (background_thread_create(tsd, arena_ind)) { + ret = EFAULT; + goto label_return; + } + } if (dirty ? arena_dirty_decay_ms_set(tsd_tsdn(tsd), arena, *(ssize_t *)newp) : arena_muzzy_decay_ms_set(tsd_tsdn(tsd), arena, *(ssize_t *)newp)) { @@ -2162,20 +2323,41 @@ arena_i_extent_hooks_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); MIB_UNSIGNED(arena_ind, 1); - if (arena_ind < narenas_total_get() && (arena = - arena_get(tsd_tsdn(tsd), arena_ind, false)) != NULL) { - if (newp != NULL) { - extent_hooks_t *old_extent_hooks; - extent_hooks_t *new_extent_hooks - JEMALLOC_CC_SILENCE_INIT(NULL); - WRITE(new_extent_hooks, extent_hooks_t *); - old_extent_hooks = extent_hooks_set(tsd, arena, - new_extent_hooks); + if (arena_ind < narenas_total_get()) { + extent_hooks_t *old_extent_hooks; + arena = arena_get(tsd_tsdn(tsd), arena_ind, false); + if (arena == NULL) { + if (arena_ind >= narenas_auto) { + ret = EFAULT; + goto label_return; + } + old_extent_hooks = + (extent_hooks_t *)&extent_hooks_default; READ(old_extent_hooks, extent_hooks_t *); + if (newp != NULL) { + /* Initialize a new arena as a side effect. */ + extent_hooks_t *new_extent_hooks + JEMALLOC_CC_SILENCE_INIT(NULL); + WRITE(new_extent_hooks, extent_hooks_t *); + arena = arena_init(tsd_tsdn(tsd), arena_ind, + new_extent_hooks); + if (arena == NULL) { + ret = EFAULT; + goto label_return; + } + } } else { - extent_hooks_t *old_extent_hooks = - extent_hooks_get(arena); - READ(old_extent_hooks, extent_hooks_t *); + if (newp != NULL) { + extent_hooks_t *new_extent_hooks + JEMALLOC_CC_SILENCE_INIT(NULL); + WRITE(new_extent_hooks, extent_hooks_t *); + old_extent_hooks = extent_hooks_set(tsd, arena, + new_extent_hooks); + READ(old_extent_hooks, extent_hooks_t *); + } else { + old_extent_hooks = extent_hooks_get(arena); + READ(old_extent_hooks, extent_hooks_t *); + } } } else { ret = EFAULT; @@ -2187,8 +2369,46 @@ arena_i_extent_hooks_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, return ret; } +static int +arena_i_retain_grow_limit_ctl(tsd_t *tsd, const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, void *newp, + size_t newlen) { + int ret; + unsigned arena_ind; + arena_t *arena; + + if (!opt_retain) { + /* Only relevant when retain is enabled. */ + return ENOENT; + } + + malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); + MIB_UNSIGNED(arena_ind, 1); + if (arena_ind < narenas_total_get() && (arena = + arena_get(tsd_tsdn(tsd), arena_ind, false)) != NULL) { + size_t old_limit, new_limit; + if (newp != NULL) { + WRITE(new_limit, size_t); + } + bool err = arena_retain_grow_limit_get_set(tsd, arena, + &old_limit, newp != NULL ? &new_limit : NULL); + if (!err) { + READ(old_limit, size_t); + ret = 0; + } else { + ret = EFAULT; + } + } else { + ret = EFAULT; + } +label_return: + malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); + return ret; +} + static const ctl_named_node_t * -arena_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i) { +arena_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, + size_t i) { const ctl_named_node_t *ret; malloc_mutex_lock(tsdn, &ctl_mtx); @@ -2213,8 +2433,8 @@ arena_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i) { /******************************************************************************/ static int -arenas_narenas_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) { +arenas_narenas_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; unsigned narenas; @@ -2234,8 +2454,9 @@ arenas_narenas_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, } static int -arenas_decay_ms_ctl_impl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen, bool dirty) { +arenas_decay_ms_ctl_impl(tsd_t *tsd, const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, void *newp, + size_t newlen, bool dirty) { int ret; if (oldp != NULL && oldlenp != NULL) { @@ -2248,7 +2469,7 @@ arenas_decay_ms_ctl_impl(tsd_t *tsd, const size_t *mib, size_t miblen, ret = EINVAL; goto label_return; } - if (dirty ? arena_dirty_decay_ms_default_set(*(ssize_t *)newp) + if (dirty ? arena_dirty_decay_ms_default_set(*(ssize_t *)newp) : arena_muzzy_decay_ms_default_set(*(ssize_t *)newp)) { ret = EFAULT; goto label_return; @@ -2277,34 +2498,36 @@ arenas_muzzy_decay_ms_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, CTL_RO_NL_GEN(arenas_quantum, QUANTUM, size_t) CTL_RO_NL_GEN(arenas_page, PAGE, size_t) CTL_RO_NL_GEN(arenas_tcache_max, tcache_maxclass, size_t) -CTL_RO_NL_GEN(arenas_nbins, NBINS, unsigned) +CTL_RO_NL_GEN(arenas_nbins, SC_NBINS, unsigned) CTL_RO_NL_GEN(arenas_nhbins, nhbins, unsigned) -CTL_RO_NL_GEN(arenas_bin_i_size, arena_bin_info[mib[2]].reg_size, size_t) -CTL_RO_NL_GEN(arenas_bin_i_nregs, arena_bin_info[mib[2]].nregs, uint32_t) -CTL_RO_NL_GEN(arenas_bin_i_slab_size, arena_bin_info[mib[2]].slab_size, size_t) +CTL_RO_NL_GEN(arenas_bin_i_size, bin_infos[mib[2]].reg_size, size_t) +CTL_RO_NL_GEN(arenas_bin_i_nregs, bin_infos[mib[2]].nregs, uint32_t) +CTL_RO_NL_GEN(arenas_bin_i_slab_size, bin_infos[mib[2]].slab_size, size_t) +CTL_RO_NL_GEN(arenas_bin_i_nshards, bin_infos[mib[2]].n_shards, uint32_t) static const ctl_named_node_t * -arenas_bin_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i) { - if (i > NBINS) { +arenas_bin_i_index(tsdn_t *tsdn, const size_t *mib, + size_t miblen, size_t i) { + if (i > SC_NBINS) { return NULL; } return super_arenas_bin_i_node; } -CTL_RO_NL_GEN(arenas_nlextents, NSIZES - NBINS, unsigned) -CTL_RO_NL_GEN(arenas_lextent_i_size, sz_index2size(NBINS+(szind_t)mib[2]), +CTL_RO_NL_GEN(arenas_nlextents, SC_NSIZES - SC_NBINS, unsigned) +CTL_RO_NL_GEN(arenas_lextent_i_size, sz_index2size(SC_NBINS+(szind_t)mib[2]), size_t) static const ctl_named_node_t * -arenas_lextent_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, - size_t i) { - if (i > NSIZES - NBINS) { +arenas_lextent_i_index(tsdn_t *tsdn, const size_t *mib, + size_t miblen, size_t i) { + if (i > SC_NSIZES - SC_NBINS) { return NULL; } return super_arenas_lextent_i_node; } static int -arenas_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) { +arenas_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; extent_hooks_t *extent_hooks; unsigned arena_ind; @@ -2325,11 +2548,43 @@ arenas_create_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, return ret; } +static int +arenas_lookup_ctl(tsd_t *tsd, const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, void *newp, + size_t newlen) { + int ret; + unsigned arena_ind; + void *ptr; + extent_t *extent; + arena_t *arena; + + ptr = NULL; + ret = EINVAL; + malloc_mutex_lock(tsd_tsdn(tsd), &ctl_mtx); + WRITE(ptr, void *); + extent = iealloc(tsd_tsdn(tsd), ptr); + if (extent == NULL) + goto label_return; + + arena = extent_arena_get(extent); + if (arena == NULL) + goto label_return; + + arena_ind = arena_ind_get(arena); + READ(arena_ind, unsigned); + + ret = 0; +label_return: + malloc_mutex_unlock(tsd_tsdn(tsd), &ctl_mtx); + return ret; +} + /******************************************************************************/ static int -prof_thread_active_init_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { +prof_thread_active_init_ctl(tsd_t *tsd, const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, void *newp, + size_t newlen) { int ret; bool oldval; @@ -2355,8 +2610,8 @@ prof_thread_active_init_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, } static int -prof_active_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) { +prof_active_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; bool oldval; @@ -2381,8 +2636,8 @@ prof_active_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, } static int -prof_dump_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) { +prof_dump_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; const char *filename = NULL; @@ -2404,8 +2659,8 @@ prof_dump_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, } static int -prof_gdump_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) { +prof_gdump_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; bool oldval; @@ -2430,8 +2685,8 @@ prof_gdump_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, } static int -prof_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, - size_t *oldlenp, void *newp, size_t newlen) { +prof_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { int ret; size_t lg_sample = lg_prof_sample; @@ -2455,11 +2710,50 @@ prof_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, CTL_RO_NL_CGEN(config_prof, prof_interval, prof_interval, uint64_t) CTL_RO_NL_CGEN(config_prof, lg_prof_sample, lg_prof_sample, size_t) +static int +prof_log_start_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) { + int ret; + + const char *filename = NULL; + + if (!config_prof) { + return ENOENT; + } + + WRITEONLY(); + WRITE(filename, const char *); + + if (prof_log_start(tsd_tsdn(tsd), filename)) { + ret = EFAULT; + goto label_return; + } + + ret = 0; +label_return: + return ret; +} + +static int +prof_log_stop_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, void *oldp, + size_t *oldlenp, void *newp, size_t newlen) { + if (!config_prof) { + return ENOENT; + } + + if (prof_log_stop(tsd_tsdn(tsd))) { + return EFAULT; + } + + return 0; +} + /******************************************************************************/ CTL_RO_CGEN(config_stats, stats_allocated, ctl_stats->allocated, size_t) CTL_RO_CGEN(config_stats, stats_active, ctl_stats->active, size_t) CTL_RO_CGEN(config_stats, stats_metadata, ctl_stats->metadata, size_t) +CTL_RO_CGEN(config_stats, stats_metadata_thp, ctl_stats->metadata_thp, size_t) CTL_RO_CGEN(config_stats, stats_resident, ctl_stats->resident, size_t) CTL_RO_CGEN(config_stats, stats_mapped, ctl_stats->mapped, size_t) CTL_RO_CGEN(config_stats, stats_retained, ctl_stats->retained, size_t) @@ -2488,26 +2782,30 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_mapped, CTL_RO_CGEN(config_stats, stats_arenas_i_retained, atomic_load_zu(&arenas_i(mib[2])->astats->astats.retained, ATOMIC_RELAXED), size_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_extent_avail, + atomic_load_zu(&arenas_i(mib[2])->astats->astats.extent_avail, + ATOMIC_RELAXED), + size_t) CTL_RO_CGEN(config_stats, stats_arenas_i_dirty_npurge, - arena_stats_read_u64(&arenas_i(mib[2])->astats->astats.decay_dirty.npurge), - uint64_t) + ctl_arena_stats_read_u64( + &arenas_i(mib[2])->astats->astats.decay_dirty.npurge), uint64_t) CTL_RO_CGEN(config_stats, stats_arenas_i_dirty_nmadvise, - arena_stats_read_u64( + ctl_arena_stats_read_u64( &arenas_i(mib[2])->astats->astats.decay_dirty.nmadvise), uint64_t) CTL_RO_CGEN(config_stats, stats_arenas_i_dirty_purged, - arena_stats_read_u64(&arenas_i(mib[2])->astats->astats.decay_dirty.purged), - uint64_t) + ctl_arena_stats_read_u64( + &arenas_i(mib[2])->astats->astats.decay_dirty.purged), uint64_t) CTL_RO_CGEN(config_stats, stats_arenas_i_muzzy_npurge, - arena_stats_read_u64(&arenas_i(mib[2])->astats->astats.decay_muzzy.npurge), - uint64_t) + ctl_arena_stats_read_u64( + &arenas_i(mib[2])->astats->astats.decay_muzzy.npurge), uint64_t) CTL_RO_CGEN(config_stats, stats_arenas_i_muzzy_nmadvise, - arena_stats_read_u64( + ctl_arena_stats_read_u64( &arenas_i(mib[2])->astats->astats.decay_muzzy.nmadvise), uint64_t) CTL_RO_CGEN(config_stats, stats_arenas_i_muzzy_purged, - arena_stats_read_u64(&arenas_i(mib[2])->astats->astats.decay_muzzy.purged), - uint64_t) + ctl_arena_stats_read_u64( + &arenas_i(mib[2])->astats->astats.decay_muzzy.purged), uint64_t) CTL_RO_CGEN(config_stats, stats_arenas_i_base, atomic_load_zu(&arenas_i(mib[2])->astats->astats.base, ATOMIC_RELAXED), @@ -2515,6 +2813,9 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_base, CTL_RO_CGEN(config_stats, stats_arenas_i_internal, atomic_load_zu(&arenas_i(mib[2])->astats->astats.internal, ATOMIC_RELAXED), size_t) +CTL_RO_CGEN(config_stats, stats_arenas_i_metadata_thp, + atomic_load_zu(&arenas_i(mib[2])->astats->astats.metadata_thp, + ATOMIC_RELAXED), size_t) CTL_RO_CGEN(config_stats, stats_arenas_i_tcache_bytes, atomic_load_zu(&arenas_i(mib[2])->astats->astats.tcache_bytes, ATOMIC_RELAXED), size_t) @@ -2534,14 +2835,17 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_large_allocated, atomic_load_zu(&arenas_i(mib[2])->astats->astats.allocated_large, ATOMIC_RELAXED), size_t) CTL_RO_CGEN(config_stats, stats_arenas_i_large_nmalloc, - arena_stats_read_u64(&arenas_i(mib[2])->astats->astats.nmalloc_large), - uint64_t) + ctl_arena_stats_read_u64( + &arenas_i(mib[2])->astats->astats.nmalloc_large), uint64_t) CTL_RO_CGEN(config_stats, stats_arenas_i_large_ndalloc, - arena_stats_read_u64(&arenas_i(mib[2])->astats->astats.ndalloc_large), - uint64_t) + ctl_arena_stats_read_u64( + &arenas_i(mib[2])->astats->astats.ndalloc_large), uint64_t) +/* + * Note: "nmalloc" here instead of "nrequests" in the read. This is intentional. + */ CTL_RO_CGEN(config_stats, stats_arenas_i_large_nrequests, - arena_stats_read_u64(&arenas_i(mib[2])->astats->astats.nmalloc_large), - uint64_t) /* Intentional. */ + ctl_arena_stats_read_u64( + &arenas_i(mib[2])->astats->astats.nmalloc_large), uint64_t) /* Intentional. */ /* Lock profiling related APIs below. */ #define RO_MUTEX_CTL_GEN(n, l) \ @@ -2580,8 +2884,9 @@ RO_MUTEX_CTL_GEN(arenas_i_bins_j_mutex, /* Resets all mutex stats, including global, arena and bin mutexes. */ static int -stats_mutexes_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, - void *oldp, size_t *oldlenp, void *newp, size_t newlen) { +stats_mutexes_reset_ctl(tsd_t *tsd, const size_t *mib, + size_t miblen, void *oldp, size_t *oldlenp, + void *newp, size_t newlen) { if (!config_stats) { return ENOENT; } @@ -2621,9 +2926,11 @@ stats_mutexes_reset_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, MUTEX_PROF_RESET(arena->tcache_ql_mtx); MUTEX_PROF_RESET(arena->base->mtx); - for (szind_t i = 0; i < NBINS; i++) { - arena_bin_t *bin = &arena->bins[i]; - MUTEX_PROF_RESET(bin->lock); + for (szind_t i = 0; i < SC_NBINS; i++) { + for (unsigned j = 0; j < bin_infos[i].n_shards; j++) { + bin_t *bin = &arena->bins[i].bin_shards[j]; + MUTEX_PROF_RESET(bin->lock); + } } } #undef MUTEX_PROF_RESET @@ -2650,37 +2957,72 @@ CTL_RO_CGEN(config_stats, stats_arenas_i_bins_j_curslabs, arenas_i(mib[2])->astats->bstats[mib[4]].curslabs, size_t) static const ctl_named_node_t * -stats_arenas_i_bins_j_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, - size_t j) { - if (j > NBINS) { +stats_arenas_i_bins_j_index(tsdn_t *tsdn, const size_t *mib, + size_t miblen, size_t j) { + if (j > SC_NBINS) { return NULL; } return super_stats_arenas_i_bins_j_node; } CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_nmalloc, - arena_stats_read_u64(&arenas_i(mib[2])->astats->lstats[mib[4]].nmalloc), - uint64_t) + ctl_arena_stats_read_u64( + &arenas_i(mib[2])->astats->lstats[mib[4]].nmalloc), uint64_t) CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_ndalloc, - arena_stats_read_u64(&arenas_i(mib[2])->astats->lstats[mib[4]].ndalloc), - uint64_t) + ctl_arena_stats_read_u64( + &arenas_i(mib[2])->astats->lstats[mib[4]].ndalloc), uint64_t) CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_nrequests, - arena_stats_read_u64(&arenas_i(mib[2])->astats->lstats[mib[4]].nrequests), - uint64_t) + ctl_arena_stats_read_u64( + &arenas_i(mib[2])->astats->lstats[mib[4]].nrequests), uint64_t) CTL_RO_CGEN(config_stats, stats_arenas_i_lextents_j_curlextents, arenas_i(mib[2])->astats->lstats[mib[4]].curlextents, size_t) static const ctl_named_node_t * -stats_arenas_i_lextents_j_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, - size_t j) { - if (j > NSIZES - NBINS) { +stats_arenas_i_lextents_j_index(tsdn_t *tsdn, const size_t *mib, + size_t miblen, size_t j) { + if (j > SC_NSIZES - SC_NBINS) { return NULL; } return super_stats_arenas_i_lextents_j_node; } +CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_ndirty, + atomic_load_zu( + &arenas_i(mib[2])->astats->estats[mib[4]].ndirty, + ATOMIC_RELAXED), size_t); +CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_nmuzzy, + atomic_load_zu( + &arenas_i(mib[2])->astats->estats[mib[4]].nmuzzy, + ATOMIC_RELAXED), size_t); +CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_nretained, + atomic_load_zu( + &arenas_i(mib[2])->astats->estats[mib[4]].nretained, + ATOMIC_RELAXED), size_t); +CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_dirty_bytes, + atomic_load_zu( + &arenas_i(mib[2])->astats->estats[mib[4]].dirty_bytes, + ATOMIC_RELAXED), size_t); +CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_muzzy_bytes, + atomic_load_zu( + &arenas_i(mib[2])->astats->estats[mib[4]].muzzy_bytes, + ATOMIC_RELAXED), size_t); +CTL_RO_CGEN(config_stats, stats_arenas_i_extents_j_retained_bytes, + atomic_load_zu( + &arenas_i(mib[2])->astats->estats[mib[4]].retained_bytes, + ATOMIC_RELAXED), size_t); + static const ctl_named_node_t * -stats_arenas_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i) { +stats_arenas_i_extents_j_index(tsdn_t *tsdn, const size_t *mib, + size_t miblen, size_t j) { + if (j >= SC_NPSIZES) { + return NULL; + } + return super_stats_arenas_i_extents_j_node; +} + +static const ctl_named_node_t * +stats_arenas_i_index(tsdn_t *tsdn, const size_t *mib, + size_t miblen, size_t i) { const ctl_named_node_t *ret; size_t a; @@ -2696,3 +3038,48 @@ stats_arenas_i_index(tsdn_t *tsdn, const size_t *mib, size_t miblen, size_t i) { malloc_mutex_unlock(tsdn, &ctl_mtx); return ret; } + +static int +experimental_hooks_install_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { + int ret; + if (oldp == NULL || oldlenp == NULL|| newp == NULL) { + ret = EINVAL; + goto label_return; + } + /* + * Note: this is a *private* struct. This is an experimental interface; + * forcing the user to know the jemalloc internals well enough to + * extract the ABI hopefully ensures nobody gets too comfortable with + * this API, which can change at a moment's notice. + */ + hooks_t hooks; + WRITE(hooks, hooks_t); + void *handle = hook_install(tsd_tsdn(tsd), &hooks); + if (handle == NULL) { + ret = EAGAIN; + goto label_return; + } + READ(handle, void *); + + ret = 0; +label_return: + return ret; +} + +static int +experimental_hooks_remove_ctl(tsd_t *tsd, const size_t *mib, size_t miblen, + void *oldp, size_t *oldlenp, void *newp, size_t newlen) { + int ret; + WRITEONLY(); + void *handle = NULL; + WRITE(handle, void *); + if (handle == NULL) { + ret = EINVAL; + goto label_return; + } + hook_remove(tsd_tsdn(tsd), handle); + ret = 0; +label_return: + return ret; +} diff --git a/kbe/src/lib/dependencies/jemalloc/src/div.c b/kbe/src/lib/dependencies/jemalloc/src/div.c new file mode 100644 index 0000000000..808892a133 --- /dev/null +++ b/kbe/src/lib/dependencies/jemalloc/src/div.c @@ -0,0 +1,55 @@ +#include "jemalloc/internal/jemalloc_preamble.h" + +#include "jemalloc/internal/div.h" + +#include "jemalloc/internal/assert.h" + +/* + * Suppose we have n = q * d, all integers. We know n and d, and want q = n / d. + * + * For any k, we have (here, all division is exact; not C-style rounding): + * floor(ceil(2^k / d) * n / 2^k) = floor((2^k + r) / d * n / 2^k), where + * r = (-2^k) mod d. + * + * Expanding this out: + * ... = floor(2^k / d * n / 2^k + r / d * n / 2^k) + * = floor(n / d + (r / d) * (n / 2^k)). + * + * The fractional part of n / d is 0 (because of the assumption that d divides n + * exactly), so we have: + * ... = n / d + floor((r / d) * (n / 2^k)) + * + * So that our initial expression is equal to the quantity we seek, so long as + * (r / d) * (n / 2^k) < 1. + * + * r is a remainder mod d, so r < d and r / d < 1 always. We can make + * n / 2 ^ k < 1 by setting k = 32. This gets us a value of magic that works. + */ + +void +div_init(div_info_t *div_info, size_t d) { + /* Nonsensical. */ + assert(d != 0); + /* + * This would make the value of magic too high to fit into a uint32_t + * (we would want magic = 2^32 exactly). This would mess with code gen + * on 32-bit machines. + */ + assert(d != 1); + + uint64_t two_to_k = ((uint64_t)1 << 32); + uint32_t magic = (uint32_t)(two_to_k / d); + + /* + * We want magic = ceil(2^k / d), but C gives us floor. We have to + * increment it unless the result was exact (i.e. unless d is a power of + * two). + */ + if (two_to_k % d != 0) { + magic++; + } + div_info->magic = magic; +#ifdef JEMALLOC_DEBUG + div_info->d = d; +#endif +} diff --git a/kbe/src/lib/dependencies/jemalloc/src/extent.c b/kbe/src/lib/dependencies/jemalloc/src/extent.c index fa45c84d34..62086c7d7b 100755 --- a/kbe/src/lib/dependencies/jemalloc/src/extent.c +++ b/kbe/src/lib/dependencies/jemalloc/src/extent.c @@ -17,8 +17,10 @@ rtree_t extents_rtree; /* Keyed by the address of the extent_t being protected. */ mutex_pool_t extent_mutex_pool; +size_t opt_lg_extent_max_active_fit = LG_EXTENT_MAX_ACTIVE_FIT_DEFAULT; + static const bitmap_info_t extents_bitmap_info = - BITMAP_INFO_INITIALIZER(NPSIZES+1); + BITMAP_INFO_INITIALIZER(SC_NPSIZES+1); static void *extent_alloc_default(extent_hooks_t *extent_hooks, void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit, @@ -117,9 +119,13 @@ static void extent_record(tsdn_t *tsdn, arena_t *arena, /******************************************************************************/ -rb_gen(UNUSED, extent_avail_, extent_tree_t, extent_t, rb_link, +#define ATTR_NONE /* does nothing */ + +ph_gen(ATTR_NONE, extent_avail_, extent_tree_t, extent_t, ph_link, extent_esnead_comp) +#undef ATTR_NONE + typedef enum { lock_result_success, lock_result_failure, @@ -128,13 +134,16 @@ typedef enum { static lock_result_t extent_rtree_leaf_elm_try_lock(tsdn_t *tsdn, rtree_leaf_elm_t *elm, - extent_t **result) { + extent_t **result, bool inactive_only) { extent_t *extent1 = rtree_leaf_elm_extent_read(tsdn, &extents_rtree, elm, true); - if (extent1 == NULL) { + /* Slab implies active extents and should be skipped. */ + if (extent1 == NULL || (inactive_only && rtree_leaf_elm_slab_read(tsdn, + &extents_rtree, elm, true))) { return lock_result_no_extent; } + /* * It's possible that the extent changed out from under us, and with it * the leaf->extent mapping. We have to recheck while holding the lock. @@ -157,7 +166,8 @@ extent_rtree_leaf_elm_try_lock(tsdn_t *tsdn, rtree_leaf_elm_t *elm, * address, and NULL otherwise. */ static extent_t * -extent_lock_from_addr(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx, void *addr) { +extent_lock_from_addr(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx, void *addr, + bool inactive_only) { extent_t *ret = NULL; rtree_leaf_elm_t *elm = rtree_leaf_elm_lookup(tsdn, &extents_rtree, rtree_ctx, (uintptr_t)addr, false, false); @@ -166,7 +176,8 @@ extent_lock_from_addr(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx, void *addr) { } lock_result_t lock_result; do { - lock_result = extent_rtree_leaf_elm_try_lock(tsdn, elm, &ret); + lock_result = extent_rtree_leaf_elm_try_lock(tsdn, elm, &ret, + inactive_only); } while (lock_result == lock_result_failure); return ret; } @@ -180,6 +191,7 @@ extent_alloc(tsdn_t *tsdn, arena_t *arena) { return base_alloc_extent(tsdn, arena->base); } extent_avail_remove(&arena->extent_avail, extent); + atomic_fetch_sub_zu(&arena->extent_avail_cnt, 1, ATOMIC_RELAXED); malloc_mutex_unlock(tsdn, &arena->extent_avail_mtx); return extent; } @@ -188,6 +200,7 @@ void extent_dalloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent) { malloc_mutex_lock(tsdn, &arena->extent_avail_mtx); extent_avail_insert(&arena->extent_avail, extent); + atomic_fetch_add_zu(&arena->extent_avail_cnt, 1, ATOMIC_RELAXED); malloc_mutex_unlock(tsdn, &arena->extent_avail_mtx); } @@ -253,7 +266,7 @@ extent_size_quantize_ceil(size_t size) { size_t ret; assert(size > 0); - assert(size - sz_large_pad <= LARGE_MAXCLASS); + assert(size - sz_large_pad <= SC_LARGE_MAXCLASS); assert((size & PAGE_MASK) == 0); ret = extent_size_quantize_floor(size); @@ -282,7 +295,7 @@ extents_init(tsdn_t *tsdn, extents_t *extents, extent_state_t state, malloc_mutex_rank_exclusive)) { return true; } - for (unsigned i = 0; i < NPSIZES+1; i++) { + for (unsigned i = 0; i < SC_NPSIZES + 1; i++) { extent_heap_new(&extents->heaps[i]); } bitmap_init(extents->bitmap, &extents_bitmap_info, true); @@ -303,9 +316,34 @@ extents_npages_get(extents_t *extents) { return atomic_load_zu(&extents->npages, ATOMIC_RELAXED); } +size_t +extents_nextents_get(extents_t *extents, pszind_t pind) { + return atomic_load_zu(&extents->nextents[pind], ATOMIC_RELAXED); +} + +size_t +extents_nbytes_get(extents_t *extents, pszind_t pind) { + return atomic_load_zu(&extents->nbytes[pind], ATOMIC_RELAXED); +} + +static void +extents_stats_add(extents_t *extent, pszind_t pind, size_t sz) { + size_t cur = atomic_load_zu(&extent->nextents[pind], ATOMIC_RELAXED); + atomic_store_zu(&extent->nextents[pind], cur + 1, ATOMIC_RELAXED); + cur = atomic_load_zu(&extent->nbytes[pind], ATOMIC_RELAXED); + atomic_store_zu(&extent->nbytes[pind], cur + sz, ATOMIC_RELAXED); +} + +static void +extents_stats_sub(extents_t *extent, pszind_t pind, size_t sz) { + size_t cur = atomic_load_zu(&extent->nextents[pind], ATOMIC_RELAXED); + atomic_store_zu(&extent->nextents[pind], cur - 1, ATOMIC_RELAXED); + cur = atomic_load_zu(&extent->nbytes[pind], ATOMIC_RELAXED); + atomic_store_zu(&extent->nbytes[pind], cur - sz, ATOMIC_RELAXED); +} + static void -extents_insert_locked(tsdn_t *tsdn, extents_t *extents, extent_t *extent, - bool preserve_lru) { +extents_insert_locked(tsdn_t *tsdn, extents_t *extents, extent_t *extent) { malloc_mutex_assert_owner(tsdn, &extents->mtx); assert(extent_state_get(extent) == extents->state); @@ -317,9 +355,12 @@ extents_insert_locked(tsdn_t *tsdn, extents_t *extents, extent_t *extent, (size_t)pind); } extent_heap_insert(&extents->heaps[pind], extent); - if (!preserve_lru) { - extent_list_append(&extents->lru, extent); + + if (config_stats) { + extents_stats_add(extents, pind, size); } + + extent_list_append(&extents->lru, extent); size_t npages = size >> LG_PAGE; /* * All modifications to npages hold the mutex (as asserted above), so we @@ -333,8 +374,7 @@ extents_insert_locked(tsdn_t *tsdn, extents_t *extents, extent_t *extent, } static void -extents_remove_locked(tsdn_t *tsdn, extents_t *extents, extent_t *extent, - bool preserve_lru) { +extents_remove_locked(tsdn_t *tsdn, extents_t *extents, extent_t *extent) { malloc_mutex_assert_owner(tsdn, &extents->mtx); assert(extent_state_get(extent) == extents->state); @@ -342,13 +382,16 @@ extents_remove_locked(tsdn_t *tsdn, extents_t *extents, extent_t *extent, size_t psz = extent_size_quantize_floor(size); pszind_t pind = sz_psz2ind(psz); extent_heap_remove(&extents->heaps[pind], extent); + + if (config_stats) { + extents_stats_sub(extents, pind, size); + } + if (extent_heap_empty(&extents->heaps[pind])) { bitmap_set(extents->bitmap, &extents_bitmap_info, (size_t)pind); } - if (!preserve_lru) { - extent_list_remove(&extents->lru, extent); - } + extent_list_remove(&extents->lru, extent); size_t npages = size >> LG_PAGE; /* * As in extents_insert_locked, we hold extents->mtx and so don't need @@ -361,6 +404,43 @@ extents_remove_locked(tsdn_t *tsdn, extents_t *extents, extent_t *extent, cur_extents_npages - (size >> LG_PAGE), ATOMIC_RELAXED); } +/* + * Find an extent with size [min_size, max_size) to satisfy the alignment + * requirement. For each size, try only the first extent in the heap. + */ +static extent_t * +extents_fit_alignment(extents_t *extents, size_t min_size, size_t max_size, + size_t alignment) { + pszind_t pind = sz_psz2ind(extent_size_quantize_ceil(min_size)); + pszind_t pind_max = sz_psz2ind(extent_size_quantize_ceil(max_size)); + + for (pszind_t i = (pszind_t)bitmap_ffu(extents->bitmap, + &extents_bitmap_info, (size_t)pind); i < pind_max; i = + (pszind_t)bitmap_ffu(extents->bitmap, &extents_bitmap_info, + (size_t)i+1)) { + assert(i < SC_NPSIZES); + assert(!extent_heap_empty(&extents->heaps[i])); + extent_t *extent = extent_heap_first(&extents->heaps[i]); + uintptr_t base = (uintptr_t)extent_base_get(extent); + size_t candidate_size = extent_size_get(extent); + assert(candidate_size >= min_size); + + uintptr_t next_align = ALIGNMENT_CEILING((uintptr_t)base, + PAGE_CEILING(alignment)); + if (base > next_align || base + candidate_size <= next_align) { + /* Overflow or not crossing the next alignment. */ + continue; + } + + size_t leadsize = next_align - base; + if (candidate_size - leadsize >= min_size) { + return extent; + } + } + + return NULL; +} + /* Do any-best-fit extent selection, i.e. select any extent that best fits. */ static extent_t * extents_best_fit_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents, @@ -368,9 +448,16 @@ extents_best_fit_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents, pszind_t pind = sz_psz2ind(extent_size_quantize_ceil(size)); pszind_t i = (pszind_t)bitmap_ffu(extents->bitmap, &extents_bitmap_info, (size_t)pind); - if (i < NPSIZES+1) { + if (i < SC_NPSIZES + 1) { + /* + * In order to reduce fragmentation, avoid reusing and splitting + * large extents for much smaller sizes. + */ + if ((sz_pind2sz(i) >> opt_lg_extent_max_active_fit) > size) { + return NULL; + } assert(!extent_heap_empty(&extents->heaps[i])); - extent_t *extent = extent_heap_any(&extents->heaps[i]); + extent_t *extent = extent_heap_first(&extents->heaps[i]); assert(extent_size_get(extent) >= size); return extent; } @@ -389,8 +476,9 @@ extents_first_fit_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents, pszind_t pind = sz_psz2ind(extent_size_quantize_ceil(size)); for (pszind_t i = (pszind_t)bitmap_ffu(extents->bitmap, - &extents_bitmap_info, (size_t)pind); i < NPSIZES+1; i = - (pszind_t)bitmap_ffu(extents->bitmap, &extents_bitmap_info, + &extents_bitmap_info, (size_t)pind); + i < SC_NPSIZES + 1; + i = (pszind_t)bitmap_ffu(extents->bitmap, &extents_bitmap_info, (size_t)i+1)) { assert(!extent_heap_empty(&extents->heaps[i])); extent_t *extent = extent_heap_first(&extents->heaps[i]); @@ -398,10 +486,10 @@ extents_first_fit_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents, if (ret == NULL || extent_snad_comp(extent, ret) < 0) { ret = extent; } - if (i == NPSIZES) { + if (i == SC_NPSIZES) { break; } - assert(i < NPSIZES); + assert(i < SC_NPSIZES); } return ret; @@ -415,12 +503,30 @@ extents_first_fit_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents, */ static extent_t * extents_fit_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents, - size_t size) { + size_t esize, size_t alignment) { malloc_mutex_assert_owner(tsdn, &extents->mtx); - return extents->delay_coalesce ? extents_best_fit_locked(tsdn, arena, - extents, size) : extents_first_fit_locked(tsdn, arena, extents, - size); + size_t max_size = esize + PAGE_CEILING(alignment) - PAGE; + /* Beware size_t wrap-around. */ + if (max_size < esize) { + return NULL; + } + + extent_t *extent = extents->delay_coalesce ? + extents_best_fit_locked(tsdn, arena, extents, max_size) : + extents_first_fit_locked(tsdn, arena, extents, max_size); + + if (alignment > PAGE && extent == NULL) { + /* + * max_size guarantees the alignment requirement but is rather + * pessimistic. Next we try to satisfy the aligned allocation + * with sizes in [esize, max_size). + */ + extent = extents_fit_alignment(extents, esize, max_size, + alignment); + } + + return extent; } static bool @@ -436,7 +542,7 @@ extent_try_delayed_coalesce(tsdn_t *tsdn, arena_t *arena, if (!coalesced) { return true; } - extents_insert_locked(tsdn, extents, extent, true); + extents_insert_locked(tsdn, extents, extent); return false; } @@ -449,8 +555,10 @@ extents_alloc(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); - return extent_recycle(tsdn, arena, r_extent_hooks, extents, new_addr, - size, pad, alignment, slab, szind, zero, commit, false); + extent_t *extent = extent_recycle(tsdn, arena, r_extent_hooks, extents, + new_addr, size, pad, alignment, slab, szind, zero, commit, false); + assert(extent == NULL || extent_dumpable_get(extent)); + return extent; } void @@ -458,6 +566,7 @@ extents_dalloc(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extents_t *extents, extent_t *extent) { assert(extent_base_get(extent) != NULL); assert(extent_size_get(extent) != 0); + assert(extent_dumpable_get(extent)); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); @@ -487,14 +596,13 @@ extents_evict(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, goto label_return; } /* Check the eviction limit. */ - size_t npages = extent_size_get(extent) >> LG_PAGE; size_t extents_npages = atomic_load_zu(&extents->npages, ATOMIC_RELAXED); - if (extents_npages - npages < npages_min) { + if (extents_npages <= npages_min) { extent = NULL; goto label_return; } - extents_remove_locked(tsdn, extents, extent, false); + extents_remove_locked(tsdn, extents, extent); if (!extents->delay_coalesce) { break; } @@ -567,29 +675,29 @@ extents_postfork_child(tsdn_t *tsdn, extents_t *extents) { static void extent_deactivate_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents, - extent_t *extent, bool preserve_lru) { + extent_t *extent) { assert(extent_arena_get(extent) == arena); assert(extent_state_get(extent) == extent_state_active); extent_state_set(extent, extents_state_get(extents)); - extents_insert_locked(tsdn, extents, extent, preserve_lru); + extents_insert_locked(tsdn, extents, extent); } static void extent_deactivate(tsdn_t *tsdn, arena_t *arena, extents_t *extents, - extent_t *extent, bool preserve_lru) { + extent_t *extent) { malloc_mutex_lock(tsdn, &extents->mtx); - extent_deactivate_locked(tsdn, arena, extents, extent, preserve_lru); + extent_deactivate_locked(tsdn, arena, extents, extent); malloc_mutex_unlock(tsdn, &extents->mtx); } static void extent_activate_locked(tsdn_t *tsdn, arena_t *arena, extents_t *extents, - extent_t *extent, bool preserve_lru) { + extent_t *extent) { assert(extent_arena_get(extent) == arena); assert(extent_state_get(extent) == extents_state_get(extents)); - extents_remove_locked(tsdn, extents, extent, preserve_lru); + extents_remove_locked(tsdn, extents, extent); extent_state_set(extent, extent_state_active); } @@ -688,6 +796,7 @@ extent_register_impl(tsdn_t *tsdn, extent_t *extent, bool gdump_add) { if (extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, extent, false, true, &elm_a, &elm_b)) { + extent_unlock(tsdn, extent); return true; } @@ -723,6 +832,13 @@ extent_reregister(tsdn_t *tsdn, extent_t *extent) { assert(!err); } +/* + * Removes all pointers to the given extent from the global rtree indices for + * its interior. This is relevant for slab extents, for which we need to do + * metadata lookups at places other than the head of the extent. We deregister + * on the interior, then, when an extent moves from being an active slab to an + * inactive state. + */ static void extent_interior_deregister(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx, extent_t *extent) { @@ -737,8 +853,11 @@ extent_interior_deregister(tsdn_t *tsdn, rtree_ctx_t *rtree_ctx, } } +/* + * Removes all pointers to the given extent from the global rtree. + */ static void -extent_deregister(tsdn_t *tsdn, extent_t *extent) { +extent_deregister_impl(tsdn_t *tsdn, extent_t *extent, bool gdump) { rtree_ctx_t rtree_ctx_fallback; rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); rtree_leaf_elm_t *elm_a, *elm_b; @@ -747,7 +866,7 @@ extent_deregister(tsdn_t *tsdn, extent_t *extent) { extent_lock(tsdn, extent); - extent_rtree_write_acquired(tsdn, elm_a, elm_b, NULL, NSIZES, false); + extent_rtree_write_acquired(tsdn, elm_a, elm_b, NULL, SC_NSIZES, false); if (extent_slab_get(extent)) { extent_interior_deregister(tsdn, rtree_ctx, extent); extent_slab_set(extent, false); @@ -755,16 +874,30 @@ extent_deregister(tsdn_t *tsdn, extent_t *extent) { extent_unlock(tsdn, extent); - if (config_prof) { + if (config_prof && gdump) { extent_gdump_sub(tsdn, extent); } } +static void +extent_deregister(tsdn_t *tsdn, extent_t *extent) { + extent_deregister_impl(tsdn, extent, true); +} + +static void +extent_deregister_no_gdump_sub(tsdn_t *tsdn, extent_t *extent) { + extent_deregister_impl(tsdn, extent, false); +} + +/* + * Tries to find and remove an extent from extents that can be used for the + * given allocation request. + */ static extent_t * extent_recycle_extract(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, rtree_ctx_t *rtree_ctx, extents_t *extents, void *new_addr, size_t size, size_t pad, size_t alignment, bool slab, - bool *zero, bool *commit, bool growing_retained) { + bool growing_retained) { witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, growing_retained ? 1 : 0); assert(alignment > 0); @@ -786,16 +919,12 @@ extent_recycle_extract(tsdn_t *tsdn, arena_t *arena, } size_t esize = size + pad; - size_t alloc_size = esize + PAGE_CEILING(alignment) - PAGE; - /* Beware size_t wrap-around. */ - if (alloc_size < esize) { - return NULL; - } malloc_mutex_lock(tsdn, &extents->mtx); extent_hooks_assure_initialized(arena, r_extent_hooks); extent_t *extent; if (new_addr != NULL) { - extent = extent_lock_from_addr(tsdn, rtree_ctx, new_addr); + extent = extent_lock_from_addr(tsdn, rtree_ctx, new_addr, + false); if (extent != NULL) { /* * We might null-out extent to report an error, but we @@ -812,86 +941,183 @@ extent_recycle_extract(tsdn_t *tsdn, arena_t *arena, extent_unlock(tsdn, unlock_extent); } } else { - extent = extents_fit_locked(tsdn, arena, extents, alloc_size); + extent = extents_fit_locked(tsdn, arena, extents, esize, + alignment); } if (extent == NULL) { malloc_mutex_unlock(tsdn, &extents->mtx); return NULL; } - extent_activate_locked(tsdn, arena, extents, extent, false); + extent_activate_locked(tsdn, arena, extents, extent); malloc_mutex_unlock(tsdn, &extents->mtx); - if (extent_zeroed_get(extent)) { - *zero = true; - } - if (extent_committed_get(extent)) { - *commit = true; - } - return extent; } -static extent_t * -extent_recycle_split(tsdn_t *tsdn, arena_t *arena, - extent_hooks_t **r_extent_hooks, rtree_ctx_t *rtree_ctx, extents_t *extents, +/* + * Given an allocation request and an extent guaranteed to be able to satisfy + * it, this splits off lead and trail extents, leaving extent pointing to an + * extent satisfying the allocation. + * This function doesn't put lead or trail into any extents_t; it's the caller's + * job to ensure that they can be reused. + */ +typedef enum { + /* + * Split successfully. lead, extent, and trail, are modified to extents + * describing the ranges before, in, and after the given allocation. + */ + extent_split_interior_ok, + /* + * The extent can't satisfy the given allocation request. None of the + * input extent_t *s are touched. + */ + extent_split_interior_cant_alloc, + /* + * In a potentially invalid state. Must leak (if *to_leak is non-NULL), + * and salvage what's still salvageable (if *to_salvage is non-NULL). + * None of lead, extent, or trail are valid. + */ + extent_split_interior_error +} extent_split_interior_result_t; + +static extent_split_interior_result_t +extent_split_interior(tsdn_t *tsdn, arena_t *arena, + extent_hooks_t **r_extent_hooks, rtree_ctx_t *rtree_ctx, + /* The result of splitting, in case of success. */ + extent_t **extent, extent_t **lead, extent_t **trail, + /* The mess to clean up, in case of error. */ + extent_t **to_leak, extent_t **to_salvage, void *new_addr, size_t size, size_t pad, size_t alignment, bool slab, - szind_t szind, extent_t *extent, bool growing_retained) { + szind_t szind, bool growing_retained) { size_t esize = size + pad; - size_t leadsize = ALIGNMENT_CEILING((uintptr_t)extent_base_get(extent), - PAGE_CEILING(alignment)) - (uintptr_t)extent_base_get(extent); + size_t leadsize = ALIGNMENT_CEILING((uintptr_t)extent_base_get(*extent), + PAGE_CEILING(alignment)) - (uintptr_t)extent_base_get(*extent); assert(new_addr == NULL || leadsize == 0); - assert(extent_size_get(extent) >= leadsize + esize); - size_t trailsize = extent_size_get(extent) - leadsize - esize; + if (extent_size_get(*extent) < leadsize + esize) { + return extent_split_interior_cant_alloc; + } + size_t trailsize = extent_size_get(*extent) - leadsize - esize; + + *lead = NULL; + *trail = NULL; + *to_leak = NULL; + *to_salvage = NULL; /* Split the lead. */ if (leadsize != 0) { - extent_t *lead = extent; - extent = extent_split_impl(tsdn, arena, r_extent_hooks, - lead, leadsize, NSIZES, false, esize + trailsize, szind, + *lead = *extent; + *extent = extent_split_impl(tsdn, arena, r_extent_hooks, + *lead, leadsize, SC_NSIZES, false, esize + trailsize, szind, slab, growing_retained); - if (extent == NULL) { - extent_deregister(tsdn, lead); - extents_leak(tsdn, arena, r_extent_hooks, extents, - lead, growing_retained); - return NULL; + if (*extent == NULL) { + *to_leak = *lead; + *lead = NULL; + return extent_split_interior_error; } - extent_deactivate(tsdn, arena, extents, lead, false); } /* Split the trail. */ if (trailsize != 0) { - extent_t *trail = extent_split_impl(tsdn, arena, - r_extent_hooks, extent, esize, szind, slab, trailsize, - NSIZES, false, growing_retained); - if (trail == NULL) { - extent_deregister(tsdn, extent); - extents_leak(tsdn, arena, r_extent_hooks, extents, - extent, growing_retained); - return NULL; + *trail = extent_split_impl(tsdn, arena, r_extent_hooks, *extent, + esize, szind, slab, trailsize, SC_NSIZES, false, + growing_retained); + if (*trail == NULL) { + *to_leak = *extent; + *to_salvage = *lead; + *lead = NULL; + *extent = NULL; + return extent_split_interior_error; } - extent_deactivate(tsdn, arena, extents, trail, false); - } else if (leadsize == 0) { + } + + if (leadsize == 0 && trailsize == 0) { /* * Splitting causes szind to be set as a side effect, but no * splitting occurred. */ - extent_szind_set(extent, szind); - if (szind != NSIZES) { + extent_szind_set(*extent, szind); + if (szind != SC_NSIZES) { rtree_szind_slab_update(tsdn, &extents_rtree, rtree_ctx, - (uintptr_t)extent_addr_get(extent), szind, slab); - if (slab && extent_size_get(extent) > PAGE) { + (uintptr_t)extent_addr_get(*extent), szind, slab); + if (slab && extent_size_get(*extent) > PAGE) { rtree_szind_slab_update(tsdn, &extents_rtree, rtree_ctx, - (uintptr_t)extent_past_get(extent) - + (uintptr_t)extent_past_get(*extent) - (uintptr_t)PAGE, szind, slab); } } } - return extent; + return extent_split_interior_ok; } +/* + * This fulfills the indicated allocation request out of the given extent (which + * the caller should have ensured was big enough). If there's any unused space + * before or after the resulting allocation, that space is given its own extent + * and put back into extents. + */ +static extent_t * +extent_recycle_split(tsdn_t *tsdn, arena_t *arena, + extent_hooks_t **r_extent_hooks, rtree_ctx_t *rtree_ctx, extents_t *extents, + void *new_addr, size_t size, size_t pad, size_t alignment, bool slab, + szind_t szind, extent_t *extent, bool growing_retained) { + extent_t *lead; + extent_t *trail; + extent_t *to_leak; + extent_t *to_salvage; + + extent_split_interior_result_t result = extent_split_interior( + tsdn, arena, r_extent_hooks, rtree_ctx, &extent, &lead, &trail, + &to_leak, &to_salvage, new_addr, size, pad, alignment, slab, szind, + growing_retained); + + if (result == extent_split_interior_ok) { + if (lead != NULL) { + extent_deactivate(tsdn, arena, extents, lead); + } + if (trail != NULL) { + extent_deactivate(tsdn, arena, extents, trail); + } + return extent; + } else { + /* + * We should have picked an extent that was large enough to + * fulfill our allocation request. + */ + assert(result == extent_split_interior_error); + if (to_salvage != NULL) { + extent_deregister(tsdn, to_salvage); + } + if (to_leak != NULL) { + void *leak = extent_base_get(to_leak); + extent_deregister_no_gdump_sub(tsdn, to_leak); + extents_leak(tsdn, arena, r_extent_hooks, extents, + to_leak, growing_retained); + assert(extent_lock_from_addr(tsdn, rtree_ctx, leak, + false) == NULL); + } + return NULL; + } + unreachable(); +} + +static bool +extent_need_manual_zero(arena_t *arena) { + /* + * Need to manually zero the extent on repopulating if either; 1) non + * default extent hooks installed (in which case the purge semantics may + * change); or 2) transparent huge pages enabled. + */ + return (!arena_has_default_hooks(arena) || + (opt_thp == thp_mode_always)); +} + +/* + * Tries to satisfy the given allocation request by reusing one of the extents + * in the given extents_t. + */ static extent_t * extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extents_t *extents, void *new_addr, size_t size, size_t pad, @@ -906,16 +1132,12 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, rtree_ctx_t rtree_ctx_fallback; rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); - bool committed = false; extent_t *extent = extent_recycle_extract(tsdn, arena, r_extent_hooks, - rtree_ctx, extents, new_addr, size, pad, alignment, slab, zero, - &committed, growing_retained); + rtree_ctx, extents, new_addr, size, pad, alignment, slab, + growing_retained); if (extent == NULL) { return NULL; } - if (committed) { - *commit = true; - } extent = extent_recycle_split(tsdn, arena, r_extent_hooks, rtree_ctx, extents, new_addr, size, pad, alignment, slab, szind, extent, @@ -931,7 +1153,16 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent, growing_retained); return NULL; } - extent_zeroed_set(extent, true); + if (!extent_need_manual_zero(arena)) { + extent_zeroed_set(extent, true); + } + } + + if (extent_committed_get(extent)) { + *commit = true; + } + if (extent_zeroed_get(extent)) { + *zero = true; } if (pad != 0) { @@ -945,14 +1176,16 @@ extent_recycle(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, if (*zero) { void *addr = extent_base_get(extent); - size_t size = extent_size_get(extent); if (!extent_zeroed_get(extent)) { - if (pages_purge_forced(addr, size)) { + size_t size = extent_size_get(extent); + if (extent_need_manual_zero(arena) || + pages_purge_forced(addr, size)) { memset(addr, 0, size); } } else if (config_debug) { size_t *p = (size_t *)(uintptr_t)addr; - for (size_t i = 0; i < size / sizeof(size_t); i++) { + /* Check the first page only. */ + for (size_t i = 0; i < PAGE / sizeof(size_t); i++) { assert(p[i] == 0); } } @@ -999,11 +1232,12 @@ extent_alloc_core(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, static void * extent_alloc_default_impl(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, size_t alignment, bool *zero, bool *commit) { - void *ret; - - ret = extent_alloc_core(tsdn, arena, new_addr, size, alignment, zero, + void *ret = extent_alloc_core(tsdn, arena, new_addr, size, alignment, zero, commit, (dss_prec_t)atomic_load_u(&arena->dss_prec, ATOMIC_RELAXED)); + if (have_madvise_huge && ret) { + pages_set_thp_state(ret, size); + } return ret; } @@ -1028,7 +1262,18 @@ extent_alloc_default(extent_hooks_t *extent_hooks, void *new_addr, size_t size, static void extent_hook_pre_reentrancy(tsdn_t *tsdn, arena_t *arena) { tsd_t *tsd = tsdn_null(tsdn) ? tsd_fetch() : tsdn_tsd(tsdn); - pre_reentrancy(tsd, arena); + if (arena == arena_get(tsd_tsdn(tsd), 0, false)) { + /* + * The only legitimate case of customized extent hooks for a0 is + * hooks with no allocation activities. One such example is to + * place metadata on pre-allocated resources such as huge pages. + * In that case, rely on reentrancy_level checks to catch + * infinite recursions. + */ + pre_reentrancy(tsd, NULL); + } else { + pre_reentrancy(tsd, arena); + } } static void @@ -1064,11 +1309,11 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, size_t alloc_size = sz_pind2sz(arena->extent_grow_next + egn_skip); while (alloc_size < alloc_size_min) { egn_skip++; - if (arena->extent_grow_next + egn_skip == NPSIZES) { + if (arena->extent_grow_next + egn_skip >= + sz_psz2ind(SC_LARGE_MAXCLASS)) { /* Outside legal range. */ goto label_err; } - assert(arena->extent_grow_next + egn_skip < NPSIZES); alloc_size = sz_pind2sz(arena->extent_grow_next + egn_skip); } @@ -1081,9 +1326,8 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, void *ptr; if (*r_extent_hooks == &extent_hooks_default) { - ptr = extent_alloc_core(tsdn, arena, NULL, alloc_size, PAGE, - &zeroed, &committed, (dss_prec_t)atomic_load_u( - &arena->dss_prec, ATOMIC_RELAXED)); + ptr = extent_alloc_default_impl(tsdn, arena, NULL, + alloc_size, PAGE, &zeroed, &committed); } else { extent_hook_pre_reentrancy(tsdn, arena); ptr = (*r_extent_hooks)->alloc(*r_extent_hooks, NULL, @@ -1092,23 +1336,20 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, extent_hook_post_reentrancy(tsdn); } - extent_init(extent, arena, ptr, alloc_size, false, NSIZES, + extent_init(extent, arena, ptr, alloc_size, false, SC_NSIZES, arena_extent_sn_next(arena), extent_state_active, zeroed, - committed); + committed, true); if (ptr == NULL) { extent_dalloc(tsdn, arena, extent); goto label_err; } + if (extent_register_no_gdump_add(tsdn, extent)) { extents_leak(tsdn, arena, r_extent_hooks, &arena->extents_retained, extent, true); goto label_err; } - size_t leadsize = ALIGNMENT_CEILING((uintptr_t)ptr, - PAGE_CEILING(alignment)) - (uintptr_t)ptr; - assert(alloc_size >= leadsize + esize); - size_t trailsize = alloc_size - leadsize - esize; if (extent_zeroed_get(extent) && extent_committed_get(extent)) { *zero = true; } @@ -1116,54 +1357,46 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, *commit = true; } - /* Split the lead. */ - if (leadsize != 0) { - extent_t *lead = extent; - extent = extent_split_impl(tsdn, arena, r_extent_hooks, lead, - leadsize, NSIZES, false, esize + trailsize, szind, slab, - true); - if (extent == NULL) { - extent_deregister(tsdn, lead); - extents_leak(tsdn, arena, r_extent_hooks, + rtree_ctx_t rtree_ctx_fallback; + rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); + + extent_t *lead; + extent_t *trail; + extent_t *to_leak; + extent_t *to_salvage; + extent_split_interior_result_t result = extent_split_interior( + tsdn, arena, r_extent_hooks, rtree_ctx, &extent, &lead, &trail, + &to_leak, &to_salvage, NULL, size, pad, alignment, slab, szind, + true); + + if (result == extent_split_interior_ok) { + if (lead != NULL) { + extent_record(tsdn, arena, r_extent_hooks, &arena->extents_retained, lead, true); - goto label_err; } - extent_record(tsdn, arena, r_extent_hooks, - &arena->extents_retained, lead, true); - } - - /* Split the trail. */ - if (trailsize != 0) { - extent_t *trail = extent_split_impl(tsdn, arena, r_extent_hooks, - extent, esize, szind, slab, trailsize, NSIZES, false, true); - if (trail == NULL) { - extent_deregister(tsdn, extent); - extents_leak(tsdn, arena, r_extent_hooks, - &arena->extents_retained, extent, true); - goto label_err; + if (trail != NULL) { + extent_record(tsdn, arena, r_extent_hooks, + &arena->extents_retained, trail, true); } - extent_record(tsdn, arena, r_extent_hooks, - &arena->extents_retained, trail, true); - } else if (leadsize == 0) { + } else { /* - * Splitting causes szind to be set as a side effect, but no - * splitting occurred. + * We should have allocated a sufficiently large extent; the + * cant_alloc case should not occur. */ - rtree_ctx_t rtree_ctx_fallback; - rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, - &rtree_ctx_fallback); - - extent_szind_set(extent, szind); - if (szind != NSIZES) { - rtree_szind_slab_update(tsdn, &extents_rtree, rtree_ctx, - (uintptr_t)extent_addr_get(extent), szind, slab); - if (slab && extent_size_get(extent) > PAGE) { - rtree_szind_slab_update(tsdn, &extents_rtree, - rtree_ctx, - (uintptr_t)extent_past_get(extent) - - (uintptr_t)PAGE, szind, slab); + assert(result == extent_split_interior_error); + if (to_salvage != NULL) { + if (config_prof) { + extent_gdump_add(tsdn, to_salvage); } + extent_record(tsdn, arena, r_extent_hooks, + &arena->extents_retained, to_salvage, true); } + if (to_leak != NULL) { + extent_deregister_no_gdump_sub(tsdn, to_leak); + extents_leak(tsdn, arena, r_extent_hooks, + &arena->extents_retained, to_leak, true); + } + goto label_err; } if (*commit && !extent_committed_get(extent)) { @@ -1173,17 +1406,20 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, &arena->extents_retained, extent, true); goto label_err; } - extent_zeroed_set(extent, true); + if (!extent_need_manual_zero(arena)) { + extent_zeroed_set(extent, true); + } } /* - * Increment extent_grow_next if doing so wouldn't exceed the legal + * Increment extent_grow_next if doing so wouldn't exceed the allowed * range. */ - if (arena->extent_grow_next + egn_skip + 1 < NPSIZES) { + if (arena->extent_grow_next + egn_skip + 1 <= + arena->retain_grow_limit) { arena->extent_grow_next += egn_skip + 1; } else { - arena->extent_grow_next = NPSIZES - 1; + arena->extent_grow_next = arena->retain_grow_limit; } /* All opportunities for failure are past. */ malloc_mutex_unlock(tsdn, &arena->extent_grow_mtx); @@ -1206,7 +1442,8 @@ extent_grow_retained(tsdn_t *tsdn, arena_t *arena, if (*zero && !extent_zeroed_get(extent)) { void *addr = extent_base_get(extent); size_t size = extent_size_get(extent); - if (pages_purge_forced(addr, size)) { + if (extent_need_manual_zero(arena) || + pages_purge_forced(addr, size)) { memset(addr, 0, size); } } @@ -1271,7 +1508,8 @@ extent_alloc_wrapper_hard(tsdn_t *tsdn, arena_t *arena, return NULL; } extent_init(extent, arena, addr, esize, slab, szind, - arena_extent_sn_next(arena), extent_state_active, zero, commit); + arena_extent_sn_next(arena), extent_state_active, *zero, *commit, + true); if (pad != 0) { extent_addr_randomize(tsdn, extent, alignment); } @@ -1296,10 +1534,20 @@ extent_alloc_wrapper(tsdn_t *tsdn, arena_t *arena, extent_t *extent = extent_alloc_retained(tsdn, arena, r_extent_hooks, new_addr, size, pad, alignment, slab, szind, zero, commit); if (extent == NULL) { + if (opt_retain && new_addr != NULL) { + /* + * When retain is enabled and new_addr is set, we do not + * attempt extent_alloc_wrapper_hard which does mmap + * that is very unlikely to succeed (unless it happens + * to be at the end). + */ + return NULL; + } extent = extent_alloc_wrapper_hard(tsdn, arena, r_extent_hooks, new_addr, size, pad, alignment, slab, szind, zero, commit); } + assert(extent == NULL || extent_dumpable_get(extent)); return extent; } @@ -1329,16 +1577,7 @@ extent_coalesce(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, bool growing_retained) { assert(extent_can_coalesce(arena, extents, inner, outer)); - if (forward && extents->delay_coalesce) { - /* - * The extent that remains after coalescing must occupy the - * outer extent's position in the LRU. For forward coalescing, - * swap the inner extent into the LRU. - */ - extent_list_replace(&extents->lru, outer, inner); - } - extent_activate_locked(tsdn, arena, extents, outer, - extents->delay_coalesce); + extent_activate_locked(tsdn, arena, extents, outer); malloc_mutex_unlock(tsdn, &extents->mtx); bool err = extent_merge_impl(tsdn, arena, r_extent_hooks, @@ -1346,20 +1585,22 @@ extent_coalesce(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, malloc_mutex_lock(tsdn, &extents->mtx); if (err) { - if (forward && extents->delay_coalesce) { - extent_list_replace(&extents->lru, inner, outer); - } - extent_deactivate_locked(tsdn, arena, extents, outer, - extents->delay_coalesce); + extent_deactivate_locked(tsdn, arena, extents, outer); } return err; } static extent_t * -extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, +extent_try_coalesce_impl(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, rtree_ctx_t *rtree_ctx, extents_t *extents, - extent_t *extent, bool *coalesced, bool growing_retained) { + extent_t *extent, bool *coalesced, bool growing_retained, + bool inactive_only) { + /* + * We avoid checking / locking inactive neighbors for large size + * classes, since they are eagerly coalesced on deallocation which can + * cause lock contention. + */ /* * Continue attempting to coalesce until failure, to protect against * races with other threads that are thwarted by this one. @@ -1370,7 +1611,7 @@ extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, /* Try to coalesce forward. */ extent_t *next = extent_lock_from_addr(tsdn, rtree_ctx, - extent_past_get(extent)); + extent_past_get(extent), inactive_only); if (next != NULL) { /* * extents->mtx only protects against races for @@ -1396,7 +1637,7 @@ extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, /* Try to coalesce backward. */ extent_t *prev = extent_lock_from_addr(tsdn, rtree_ctx, - extent_before_get(extent)); + extent_before_get(extent), inactive_only); if (prev != NULL) { bool can_coalesce = extent_can_coalesce(arena, extents, extent, prev); @@ -1422,6 +1663,26 @@ extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, return extent; } +static extent_t * +extent_try_coalesce(tsdn_t *tsdn, arena_t *arena, + extent_hooks_t **r_extent_hooks, rtree_ctx_t *rtree_ctx, extents_t *extents, + extent_t *extent, bool *coalesced, bool growing_retained) { + return extent_try_coalesce_impl(tsdn, arena, r_extent_hooks, rtree_ctx, + extents, extent, coalesced, growing_retained, false); +} + +static extent_t * +extent_try_coalesce_large(tsdn_t *tsdn, arena_t *arena, + extent_hooks_t **r_extent_hooks, rtree_ctx_t *rtree_ctx, extents_t *extents, + extent_t *extent, bool *coalesced, bool growing_retained) { + return extent_try_coalesce_impl(tsdn, arena, r_extent_hooks, rtree_ctx, + extents, extent, coalesced, growing_retained, true); +} + +/* + * Does the metadata management portions of putting an unused extent into the + * given extents_t (coalesces, deregisters slab interiors, the heap operations). + */ static void extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extents_t *extents, extent_t *extent, bool growing_retained) { @@ -1435,7 +1696,7 @@ extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, malloc_mutex_lock(tsdn, &extents->mtx); extent_hooks_assure_initialized(arena, r_extent_hooks); - extent_szind_set(extent, NSIZES); + extent_szind_set(extent, SC_NSIZES); if (extent_slab_get(extent)) { extent_interior_deregister(tsdn, rtree_ctx, extent); extent_slab_set(extent, false); @@ -1447,9 +1708,24 @@ extent_record(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, if (!extents->delay_coalesce) { extent = extent_try_coalesce(tsdn, arena, r_extent_hooks, rtree_ctx, extents, extent, NULL, growing_retained); + } else if (extent_size_get(extent) >= SC_LARGE_MINCLASS) { + assert(extents == &arena->extents_dirty); + /* Always coalesce large extents eagerly. */ + bool coalesced; + do { + assert(extent_state_get(extent) == extent_state_active); + extent = extent_try_coalesce_large(tsdn, arena, + r_extent_hooks, rtree_ctx, extents, extent, + &coalesced, growing_retained); + } while (coalesced); + if (extent_size_get(extent) >= oversize_threshold) { + /* Shortcut to purge the oversize extent eagerly. */ + malloc_mutex_unlock(tsdn, &extents->mtx); + arena_decay_extent(tsdn, arena, r_extent_hooks, extent); + return; + } } - - extent_deactivate_locked(tsdn, arena, extents, extent, false); + extent_deactivate_locked(tsdn, arena, extents, extent); malloc_mutex_unlock(tsdn, &extents->mtx); } @@ -1469,6 +1745,12 @@ extent_dalloc_gap(tsdn_t *tsdn, arena_t *arena, extent_t *extent) { extent_dalloc_wrapper(tsdn, arena, &extent_hooks, extent); } +static bool +extent_may_dalloc(void) { + /* With retain enabled, the default dalloc always fails. */ + return !opt_retain; +} + static bool extent_dalloc_default_impl(void *addr, size_t size) { if (!have_dss || !extent_in_dss(addr)) { @@ -1520,19 +1802,24 @@ extent_dalloc_wrapper_try(tsdn_t *tsdn, arena_t *arena, void extent_dalloc_wrapper(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *extent) { + assert(extent_dumpable_get(extent)); witness_assert_depth_to_rank(tsdn_witness_tsdp_get(tsdn), WITNESS_RANK_CORE, 0); - /* - * Deregister first to avoid a race with other allocating threads, and - * reregister if deallocation fails. - */ - extent_deregister(tsdn, extent); - if (!extent_dalloc_wrapper_try(tsdn, arena, r_extent_hooks, extent)) { - return; + /* Avoid calling the default extent_dalloc unless have to. */ + if (*r_extent_hooks != &extent_hooks_default || extent_may_dalloc()) { + /* + * Deregister first to avoid a race with other allocating + * threads, and reregister if deallocation fails. + */ + extent_deregister(tsdn, extent); + if (!extent_dalloc_wrapper_try(tsdn, arena, r_extent_hooks, + extent)) { + return; + } + extent_reregister(tsdn, extent); } - extent_reregister(tsdn, extent); if (*r_extent_hooks != &extent_hooks_default) { extent_hook_pre_reentrancy(tsdn, arena); } @@ -1780,6 +2067,13 @@ extent_split_default(extent_hooks_t *extent_hooks, void *addr, size_t size, } #endif +/* + * Accepts the extent to split, and the characteristics of each side of the + * split. The 'a' parameters go with the 'lead' of the resulting pair of + * extents (the lower addressed portion of the split), and the 'b' parameters go + * with the trail (the higher addressed portion). This makes 'extent' the lead, + * and returns the trail (except in case of error). + */ static extent_t * extent_split_impl(tsdn_t *tsdn, arena_t *arena, extent_hooks_t **r_extent_hooks, extent_t *extent, size_t size_a, @@ -1803,7 +2097,7 @@ extent_split_impl(tsdn_t *tsdn, arena_t *arena, extent_init(trail, arena, (void *)((uintptr_t)extent_base_get(extent) + size_a), size_b, slab_b, szind_b, extent_sn_get(extent), extent_state_get(extent), extent_zeroed_get(extent), - extent_committed_get(extent)); + extent_committed_get(extent), extent_dumpable_get(extent)); rtree_ctx_t rtree_ctx_fallback; rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); @@ -1814,7 +2108,7 @@ extent_split_impl(tsdn_t *tsdn, arena_t *arena, extent_init(&lead, arena, extent_addr_get(extent), size_a, slab_a, szind_a, extent_sn_get(extent), extent_state_get(extent), extent_zeroed_get(extent), - extent_committed_get(extent)); + extent_committed_get(extent), extent_dumpable_get(extent)); extent_rtree_leaf_elms_lookup(tsdn, rtree_ctx, &lead, false, true, &lead_elm_a, &lead_elm_b); @@ -1938,22 +2232,23 @@ extent_merge_impl(tsdn_t *tsdn, arena_t *arena, if (a_elm_b != NULL) { rtree_leaf_elm_write(tsdn, &extents_rtree, a_elm_b, NULL, - NSIZES, false); + SC_NSIZES, false); } if (b_elm_b != NULL) { rtree_leaf_elm_write(tsdn, &extents_rtree, b_elm_a, NULL, - NSIZES, false); + SC_NSIZES, false); } else { b_elm_b = b_elm_a; } extent_size_set(a, extent_size_get(a) + extent_size_get(b)); - extent_szind_set(a, NSIZES); + extent_szind_set(a, SC_NSIZES); extent_sn_set(a, (extent_sn_get(a) < extent_sn_get(b)) ? extent_sn_get(a) : extent_sn_get(b)); extent_zeroed_set(a, extent_zeroed_get(a) && extent_zeroed_get(b)); - extent_rtree_write_acquired(tsdn, a_elm_a, b_elm_b, a, NSIZES, false); + extent_rtree_write_acquired(tsdn, a_elm_a, b_elm_b, a, SC_NSIZES, + false); extent_unlock2(tsdn, a, b); diff --git a/kbe/src/lib/dependencies/jemalloc/src/extent_dss.c b/kbe/src/lib/dependencies/jemalloc/src/extent_dss.c index e72da95870..6c56cf6568 100755 --- a/kbe/src/lib/dependencies/jemalloc/src/extent_dss.c +++ b/kbe/src/lib/dependencies/jemalloc/src/extent_dss.c @@ -154,9 +154,9 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, (uintptr_t)gap_addr_page; if (gap_size_page != 0) { extent_init(gap, arena, gap_addr_page, - gap_size_page, false, NSIZES, + gap_size_page, false, SC_NSIZES, arena_extent_sn_next(arena), - extent_state_active, false, true); + extent_state_active, false, true, true); } /* * Compute the address just past the end of the desired @@ -198,8 +198,9 @@ extent_alloc_dss(tsdn_t *tsdn, arena_t *arena, void *new_addr, size_t size, extent_t extent; extent_init(&extent, arena, ret, size, - size, false, NSIZES, - extent_state_active, false, true); + size, false, SC_NSIZES, + extent_state_active, false, true, + true); if (extent_purge_forced_wrapper(tsdn, arena, &extent_hooks, &extent, 0, size)) { diff --git a/kbe/src/lib/dependencies/jemalloc/src/hook.c b/kbe/src/lib/dependencies/jemalloc/src/hook.c new file mode 100644 index 0000000000..9ac703cf9f --- /dev/null +++ b/kbe/src/lib/dependencies/jemalloc/src/hook.c @@ -0,0 +1,195 @@ +#include "jemalloc/internal/jemalloc_preamble.h" + +#include "jemalloc/internal/hook.h" + +#include "jemalloc/internal/atomic.h" +#include "jemalloc/internal/mutex.h" +#include "jemalloc/internal/seq.h" + +typedef struct hooks_internal_s hooks_internal_t; +struct hooks_internal_s { + hooks_t hooks; + bool in_use; +}; + +seq_define(hooks_internal_t, hooks) + +static atomic_u_t nhooks = ATOMIC_INIT(0); +static seq_hooks_t hooks[HOOK_MAX]; +static malloc_mutex_t hooks_mu; + +bool +hook_boot() { + return malloc_mutex_init(&hooks_mu, "hooks", WITNESS_RANK_HOOK, + malloc_mutex_rank_exclusive); +} + +static void * +hook_install_locked(hooks_t *to_install) { + hooks_internal_t hooks_internal; + for (int i = 0; i < HOOK_MAX; i++) { + bool success = seq_try_load_hooks(&hooks_internal, &hooks[i]); + /* We hold mu; no concurrent access. */ + assert(success); + if (!hooks_internal.in_use) { + hooks_internal.hooks = *to_install; + hooks_internal.in_use = true; + seq_store_hooks(&hooks[i], &hooks_internal); + atomic_store_u(&nhooks, + atomic_load_u(&nhooks, ATOMIC_RELAXED) + 1, + ATOMIC_RELAXED); + return &hooks[i]; + } + } + return NULL; +} + +void * +hook_install(tsdn_t *tsdn, hooks_t *to_install) { + malloc_mutex_lock(tsdn, &hooks_mu); + void *ret = hook_install_locked(to_install); + if (ret != NULL) { + tsd_global_slow_inc(tsdn); + } + malloc_mutex_unlock(tsdn, &hooks_mu); + return ret; +} + +static void +hook_remove_locked(seq_hooks_t *to_remove) { + hooks_internal_t hooks_internal; + bool success = seq_try_load_hooks(&hooks_internal, to_remove); + /* We hold mu; no concurrent access. */ + assert(success); + /* Should only remove hooks that were added. */ + assert(hooks_internal.in_use); + hooks_internal.in_use = false; + seq_store_hooks(to_remove, &hooks_internal); + atomic_store_u(&nhooks, atomic_load_u(&nhooks, ATOMIC_RELAXED) - 1, + ATOMIC_RELAXED); +} + +void +hook_remove(tsdn_t *tsdn, void *opaque) { + if (config_debug) { + char *hooks_begin = (char *)&hooks[0]; + char *hooks_end = (char *)&hooks[HOOK_MAX]; + char *hook = (char *)opaque; + assert(hooks_begin <= hook && hook < hooks_end + && (hook - hooks_begin) % sizeof(seq_hooks_t) == 0); + } + malloc_mutex_lock(tsdn, &hooks_mu); + hook_remove_locked((seq_hooks_t *)opaque); + tsd_global_slow_dec(tsdn); + malloc_mutex_unlock(tsdn, &hooks_mu); +} + +#define FOR_EACH_HOOK_BEGIN(hooks_internal_ptr) \ +for (int for_each_hook_counter = 0; \ + for_each_hook_counter < HOOK_MAX; \ + for_each_hook_counter++) { \ + bool for_each_hook_success = seq_try_load_hooks( \ + (hooks_internal_ptr), &hooks[for_each_hook_counter]); \ + if (!for_each_hook_success) { \ + continue; \ + } \ + if (!(hooks_internal_ptr)->in_use) { \ + continue; \ + } +#define FOR_EACH_HOOK_END \ +} + +static bool * +hook_reentrantp() { + /* + * We prevent user reentrancy within hooks. This is basically just a + * thread-local bool that triggers an early-exit. + * + * We don't fold in_hook into reentrancy. There are two reasons for + * this: + * - Right now, we turn on reentrancy during things like extent hook + * execution. Allocating during extent hooks is not officially + * supported, but we don't want to break it for the time being. These + * sorts of allocations should probably still be hooked, though. + * - If a hook allocates, we may want it to be relatively fast (after + * all, it executes on every allocator operation). Turning on + * reentrancy is a fairly heavyweight mode (disabling tcache, + * redirecting to arena 0, etc.). It's possible we may one day want + * to turn on reentrant mode here, if it proves too difficult to keep + * this working. But that's fairly easy for us to see; OTOH, people + * not using hooks because they're too slow is easy for us to miss. + * + * The tricky part is + * that this code might get invoked even if we don't have access to tsd. + * This function mimics getting a pointer to thread-local data, except + * that it might secretly return a pointer to some global data if we + * know that the caller will take the early-exit path. + * If we return a bool that indicates that we are reentrant, then the + * caller will go down the early exit path, leaving the global + * untouched. + */ + static bool in_hook_global = true; + tsdn_t *tsdn = tsdn_fetch(); + tcache_t *tcache = tsdn_tcachep_get(tsdn); + if (tcache != NULL) { + return &tcache->in_hook; + } + return &in_hook_global; +} + +#define HOOK_PROLOGUE \ + if (likely(atomic_load_u(&nhooks, ATOMIC_RELAXED) == 0)) { \ + return; \ + } \ + bool *in_hook = hook_reentrantp(); \ + if (*in_hook) { \ + return; \ + } \ + *in_hook = true; + +#define HOOK_EPILOGUE \ + *in_hook = false; + +void +hook_invoke_alloc(hook_alloc_t type, void *result, uintptr_t result_raw, + uintptr_t args_raw[3]) { + HOOK_PROLOGUE + + hooks_internal_t hook; + FOR_EACH_HOOK_BEGIN(&hook) + hook_alloc h = hook.hooks.alloc_hook; + if (h != NULL) { + h(hook.hooks.extra, type, result, result_raw, args_raw); + } + FOR_EACH_HOOK_END + + HOOK_EPILOGUE +} + +void +hook_invoke_dalloc(hook_dalloc_t type, void *address, uintptr_t args_raw[3]) { + HOOK_PROLOGUE + hooks_internal_t hook; + FOR_EACH_HOOK_BEGIN(&hook) + hook_dalloc h = hook.hooks.dalloc_hook; + if (h != NULL) { + h(hook.hooks.extra, type, address, args_raw); + } + FOR_EACH_HOOK_END + HOOK_EPILOGUE +} + +void +hook_invoke_expand(hook_expand_t type, void *address, size_t old_usize, + size_t new_usize, uintptr_t result_raw, uintptr_t args_raw[4]) { + HOOK_PROLOGUE + hooks_internal_t hook; + FOR_EACH_HOOK_BEGIN(&hook) + hook_expand h = hook.hooks.expand_hook; + if (h != NULL) { + h(hook.hooks.extra, type, address, old_usize, new_usize, + result_raw, args_raw); + } + FOR_EACH_HOOK_END + HOOK_EPILOGUE +} diff --git a/kbe/src/lib/dependencies/jemalloc/src/jemalloc.c b/kbe/src/lib/dependencies/jemalloc/src/jemalloc.c index 0ee8ad48b9..c8afa9c4d5 100755 --- a/kbe/src/lib/dependencies/jemalloc/src/jemalloc.c +++ b/kbe/src/lib/dependencies/jemalloc/src/jemalloc.c @@ -7,11 +7,13 @@ #include "jemalloc/internal/ctl.h" #include "jemalloc/internal/extent_dss.h" #include "jemalloc/internal/extent_mmap.h" +#include "jemalloc/internal/hook.h" #include "jemalloc/internal/jemalloc_internal_types.h" +#include "jemalloc/internal/log.h" #include "jemalloc/internal/malloc_io.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/rtree.h" -#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/sc.h" #include "jemalloc/internal/spin.h" #include "jemalloc/internal/sz.h" #include "jemalloc/internal/ticker.h" @@ -84,8 +86,10 @@ malloc_mutex_t arenas_lock; JEMALLOC_ALIGNED(CACHELINE) atomic_p_t arenas[MALLOCX_ARENA_LIMIT]; static atomic_u_t narenas_total; /* Use narenas_total_*(). */ -static arena_t *a0; /* arenas[0]; read-only after initialization. */ -unsigned narenas_auto; /* Read-only after initialization. */ +/* Below three are read-only after initialization. */ +static arena_t *a0; /* arenas[0]. */ +unsigned narenas_auto; +unsigned manual_arena_base; typedef enum { malloc_init_uninitialized = 3, @@ -325,7 +329,7 @@ arena_init_locked(tsdn_t *tsdn, unsigned ind, extent_hooks_t *extent_hooks) { */ arena = arena_get(tsdn, ind, false); if (arena != NULL) { - assert(ind < narenas_auto); + assert(arena_is_auto(arena)); return arena; } @@ -340,12 +344,12 @@ arena_new_create_background_thread(tsdn_t *tsdn, unsigned ind) { if (ind == 0) { return; } - if (have_background_thread) { - bool err; - malloc_mutex_lock(tsdn, &background_thread_lock); - err = background_thread_create(tsdn_tsd(tsdn), ind); - malloc_mutex_unlock(tsdn, &background_thread_lock); - if (err) { + /* + * Avoid creating a new background thread just for the huge arena, which + * purges eagerly by default. + */ + if (have_background_thread && !arena_is_huge(ind)) { + if (background_thread_create(tsdn_tsd(tsdn), ind)) { malloc_printf(": error in background thread " "creation for arena %u. Abort.\n", ind); abort(); @@ -375,6 +379,14 @@ arena_bind(tsd_t *tsd, unsigned ind, bool internal) { tsd_iarena_set(tsd, arena); } else { tsd_arena_set(tsd, arena); + unsigned shard = atomic_fetch_add_u(&arena->binshard_next, 1, + ATOMIC_RELAXED); + tsd_binshards_t *bins = tsd_binshardsp_get(tsd); + for (unsigned i = 0; i < SC_NBINS; i++) { + assert(bin_infos[i].n_shards > 0 && + bin_infos[i].n_shards <= BIN_SHARDS_MAX); + bins->binshard[i] = shard % bin_infos[i].n_shards; + } } } @@ -760,6 +772,50 @@ init_opt_stats_print_opts(const char *v, size_t vlen) { assert(opts_len == strlen(opt_stats_print_opts)); } +/* Reads the next size pair in a multi-sized option. */ +static bool +malloc_conf_multi_sizes_next(const char **slab_size_segment_cur, + size_t *vlen_left, size_t *slab_start, size_t *slab_end, size_t *new_size) { + const char *cur = *slab_size_segment_cur; + char *end; + uintmax_t um; + + set_errno(0); + + /* First number, then '-' */ + um = malloc_strtoumax(cur, &end, 0); + if (get_errno() != 0 || *end != '-') { + return true; + } + *slab_start = (size_t)um; + cur = end + 1; + + /* Second number, then ':' */ + um = malloc_strtoumax(cur, &end, 0); + if (get_errno() != 0 || *end != ':') { + return true; + } + *slab_end = (size_t)um; + cur = end + 1; + + /* Last number */ + um = malloc_strtoumax(cur, &end, 0); + if (get_errno() != 0) { + return true; + } + *new_size = (size_t)um; + + /* Consume the separator if there is one. */ + if (*end == '|') { + end++; + } + + *vlen_left -= end - *slab_size_segment_cur; + *slab_size_segment_cur = end; + + return false; +} + static bool malloc_conf_next(char const **opts_p, char const **k_p, size_t *klen_p, char const **v_p, size_t *vlen_p) { @@ -848,10 +904,13 @@ malloc_conf_error(const char *msg, const char *k, size_t klen, const char *v, size_t vlen) { malloc_printf(": %s: %.*s:%.*s\n", msg, (int)klen, k, (int)vlen, v); - had_conf_error = true; - if (opt_abort_conf) { - malloc_abort_invalid_conf(); + /* If abort_conf is set, error out after processing all options. */ + const char *experimental = "experimental_"; + if (strncmp(k, experimental, strlen(experimental)) == 0) { + /* However, tolerate experimental features. */ + return; } + had_conf_error = true; } static void @@ -870,7 +929,7 @@ malloc_slow_flag_init(void) { } static void -malloc_conf_init(void) { +malloc_conf_init(sc_data_t *sc_data, unsigned bin_shard_sizes[SC_NBINS]) { unsigned i; char buf[PATH_MAX + 1]; const char *opts, *k, *v; @@ -911,7 +970,12 @@ malloc_conf_init(void) { * Try to use the contents of the "/etc/malloc.conf" * symbolic link's name. */ +#ifndef JEMALLOC_READLINKAT linklen = readlink(linkname, buf, sizeof(buf) - 1); +#else + linklen = readlinkat(AT_FDCWD, linkname, buf, + sizeof(buf) - 1); +#endif if (linklen == -1) { /* No configuration specified. */ linklen = 0; @@ -968,6 +1032,14 @@ malloc_conf_init(void) { } \ continue; \ } + /* + * One of the CONF_MIN macros below expands, in one of the use points, + * to "unsigned integer < 0", which is always false, triggering the + * GCC -Wtype-limits warning, which we disable here and re-enable below. + */ + JEMALLOC_DIAGNOSTIC_PUSH + JEMALLOC_DIAGNOSTIC_IGNORE_TYPE_LIMITS + #define CONF_MIN_no(um, min) false #define CONF_MIN_yes(um, min) ((um) < (min)) #define CONF_MAX_no(um, max) false @@ -1051,8 +1123,22 @@ malloc_conf_init(void) { CONF_HANDLE_BOOL(opt_abort, "abort") CONF_HANDLE_BOOL(opt_abort_conf, "abort_conf") - if (opt_abort_conf && had_conf_error) { - malloc_abort_invalid_conf(); + if (strncmp("metadata_thp", k, klen) == 0) { + int i; + bool match = false; + for (i = 0; i < metadata_thp_mode_limit; i++) { + if (strncmp(metadata_thp_mode_names[i], + v, vlen) == 0) { + opt_metadata_thp = i; + match = true; + break; + } + } + if (!match) { + malloc_conf_error("Invalid conf value", + k, klen, v, vlen); + } + continue; } CONF_HANDLE_BOOL(opt_retain, "retain") if (strncmp("dss", k, klen) == 0) { @@ -1081,6 +1167,28 @@ malloc_conf_init(void) { } CONF_HANDLE_UNSIGNED(opt_narenas, "narenas", 1, UINT_MAX, yes, no, false) + if (CONF_MATCH("bin_shards")) { + const char *bin_shards_segment_cur = v; + size_t vlen_left = vlen; + do { + size_t size_start; + size_t size_end; + size_t nshards; + bool err = malloc_conf_multi_sizes_next( + &bin_shards_segment_cur, &vlen_left, + &size_start, &size_end, &nshards); + if (err || bin_update_shard_size( + bin_shard_sizes, size_start, + size_end, nshards)) { + malloc_conf_error( + "Invalid settings for " + "bin_shards", k, klen, v, + vlen); + break; + } + } while (vlen_left > 0); + continue; + } CONF_HANDLE_SSIZE_T(opt_dirty_decay_ms, "dirty_decay_ms", -1, NSTIME_SEC_MAX * KQU(1000) < QU(SSIZE_MAX) ? NSTIME_SEC_MAX * KQU(1000) : @@ -1130,10 +1238,25 @@ malloc_conf_init(void) { CONF_HANDLE_BOOL(opt_tcache, "tcache") CONF_HANDLE_SSIZE_T(opt_lg_tcache_max, "lg_tcache_max", -1, (sizeof(size_t) << 3) - 1) + + /* + * The runtime option of oversize_threshold remains + * undocumented. It may be tweaked in the next major + * release (6.0). The default value 8M is rather + * conservative / safe. Tuning it further down may + * improve fragmentation a bit more, but may also cause + * contention on the huge arena. + */ + CONF_HANDLE_SIZE_T(opt_oversize_threshold, + "oversize_threshold", 0, SC_LARGE_MAXCLASS, no, yes, + false) + CONF_HANDLE_SIZE_T(opt_lg_extent_max_active_fit, + "lg_extent_max_active_fit", 0, + (sizeof(size_t) << 3), no, yes, false) + if (strncmp("percpu_arena", k, klen) == 0) { - int i; bool match = false; - for (i = percpu_arena_mode_names_base; i < + for (int i = percpu_arena_mode_names_base; i < percpu_arena_mode_names_limit; i++) { if (strncmp(percpu_arena_mode_names[i], v, vlen) == 0) { @@ -1155,6 +1278,35 @@ malloc_conf_init(void) { } CONF_HANDLE_BOOL(opt_background_thread, "background_thread"); + CONF_HANDLE_SIZE_T(opt_max_background_threads, + "max_background_threads", 1, + opt_max_background_threads, yes, yes, + true); + if (CONF_MATCH("slab_sizes")) { + bool err; + const char *slab_size_segment_cur = v; + size_t vlen_left = vlen; + do { + size_t slab_start; + size_t slab_end; + size_t pgs; + err = malloc_conf_multi_sizes_next( + &slab_size_segment_cur, + &vlen_left, &slab_start, &slab_end, + &pgs); + if (!err) { + sc_data_update_slab_size( + sc_data, slab_start, + slab_end, (int)pgs); + } else { + malloc_conf_error( + "Invalid settings for " + "slab_sizes", k, klen, v, + vlen); + } + } while (!err && vlen_left > 0); + continue; + } if (config_prof) { CONF_HANDLE_BOOL(opt_prof, "prof") CONF_HANDLE_CHAR_P(opt_prof_prefix, @@ -1172,6 +1324,38 @@ malloc_conf_init(void) { CONF_HANDLE_BOOL(opt_prof_gdump, "prof_gdump") CONF_HANDLE_BOOL(opt_prof_final, "prof_final") CONF_HANDLE_BOOL(opt_prof_leak, "prof_leak") + CONF_HANDLE_BOOL(opt_prof_log, "prof_log") + } + if (config_log) { + if (CONF_MATCH("log")) { + size_t cpylen = ( + vlen <= sizeof(log_var_names) ? + vlen : sizeof(log_var_names) - 1); + strncpy(log_var_names, v, cpylen); + log_var_names[cpylen] = '\0'; + continue; + } + } + if (CONF_MATCH("thp")) { + bool match = false; + for (int i = 0; i < thp_mode_names_limit; i++) { + if (strncmp(thp_mode_names[i],v, vlen) + == 0) { + if (!have_madvise_huge) { + malloc_conf_error( + "No THP support", + k, klen, v, vlen); + } + opt_thp = i; + match = true; + break; + } + } + if (!match) { + malloc_conf_error("Invalid conf value", + k, klen, v, vlen); + } + continue; } malloc_conf_error("Invalid conf pair", k, klen, v, vlen); @@ -1187,8 +1371,14 @@ malloc_conf_init(void) { #undef CONF_HANDLE_SIZE_T #undef CONF_HANDLE_SSIZE_T #undef CONF_HANDLE_CHAR_P + /* Re-enable diagnostic "-Wtype-limits" */ + JEMALLOC_DIAGNOSTIC_POP + } + if (opt_abort_conf && had_conf_error) { + malloc_abort_invalid_conf(); } } + atomic_store_b(&log_init_done, true, ATOMIC_RELEASE); } static bool @@ -1221,10 +1411,33 @@ static bool malloc_init_hard_a0_locked() { malloc_initializer = INITIALIZER; + JEMALLOC_DIAGNOSTIC_PUSH + JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS + sc_data_t sc_data = {0}; + JEMALLOC_DIAGNOSTIC_POP + + /* + * Ordering here is somewhat tricky; we need sc_boot() first, since that + * determines what the size classes will be, and then + * malloc_conf_init(), since any slab size tweaking will need to be done + * before sz_boot and bin_boot, which assume that the values they read + * out of sc_data_global are final. + */ + sc_boot(&sc_data); + unsigned bin_shard_sizes[SC_NBINS]; + bin_shard_sizes_boot(bin_shard_sizes); + /* + * prof_boot0 only initializes opt_prof_prefix. We need to do it before + * we parse malloc_conf options, in case malloc_conf parsing overwrites + * it. + */ if (config_prof) { prof_boot0(); } - malloc_conf_init(); + malloc_conf_init(&sc_data, bin_shard_sizes); + sz_boot(&sc_data); + bin_boot(&sc_data, bin_shard_sizes); + if (opt_stats_print) { /* Print statistics at exit. */ if (atexit(stats_print_atexit) != 0) { @@ -1249,7 +1462,7 @@ malloc_init_hard_a0_locked() { if (config_prof) { prof_boot1(); } - arena_boot(); + arena_boot(&sc_data); if (tcache_boot(TSDN_NULL)) { return true; } @@ -1257,11 +1470,13 @@ malloc_init_hard_a0_locked() { malloc_mutex_rank_exclusive)) { return true; } + hook_boot(); /* * Create enough scaffolding to allow recursive allocation in * malloc_ncpus(). */ narenas_auto = 1; + manual_arena_base = narenas_auto + 1; memset(arenas, 0, sizeof(arena_t *) * narenas_auto); /* * Initialize one arena here. The rest are lazily created in @@ -1409,6 +1624,10 @@ malloc_init_narenas(void) { narenas_auto); } narenas_total_set(narenas_auto); + if (arena_init_huge()) { + narenas_total_inc(); + } + manual_arena_base = narenas_total_get(); return false; } @@ -1493,6 +1712,8 @@ malloc_init_hard(void) { post_reentrancy(tsd); malloc_mutex_unlock(tsd_tsdn(tsd), &init_lock); + witness_assert_lockless(witness_tsd_tsdn( + tsd_witness_tsdp_get_unsafe(tsd))); malloc_tsd_boot1(); /* Update TSD after tsd_boot1. */ tsd = tsd_fetch(); @@ -1500,12 +1721,11 @@ malloc_init_hard(void) { assert(have_background_thread); /* * Need to finish init & unlock first before creating background - * threads (pthread_create depends on malloc). + * threads (pthread_create depends on malloc). ctl_init (which + * sets isthreaded) needs to be called without holding any lock. */ - malloc_mutex_lock(tsd_tsdn(tsd), &background_thread_lock); - bool err = background_thread_create(tsd, 0); - malloc_mutex_unlock(tsd_tsdn(tsd), &background_thread_lock); - if (err) { + background_thread_ctl_init(tsd_tsdn(tsd)); + if (background_thread_create(tsd, 0)) { return true; } } @@ -1528,8 +1748,7 @@ typedef struct static_opts_s static_opts_t; struct static_opts_s { /* Whether or not allocation size may overflow. */ bool may_overflow; - /* Whether or not allocations of size 0 should be treated as size 1. */ - bool bump_empty_alloc; + /* * Whether to assert that allocations are not of size 0 (after any * bumping). @@ -1562,12 +1781,15 @@ struct static_opts_s { * initialization) options. */ bool slow; + /* + * Return size. + */ + bool usize; }; JEMALLOC_ALWAYS_INLINE void static_opts_init(static_opts_t *static_opts) { static_opts->may_overflow = false; - static_opts->bump_empty_alloc = false; static_opts->assert_nonempty_alloc = false; static_opts->null_out_result_on_error = false; static_opts->set_errno_on_error = false; @@ -1575,6 +1797,7 @@ static_opts_init(static_opts_t *static_opts) { static_opts->oom_string = ""; static_opts->invalid_alignment_string = ""; static_opts->slow = false; + static_opts->usize = false; } /* @@ -1589,6 +1812,7 @@ static_opts_init(static_opts_t *static_opts) { typedef struct dynamic_opts_s dynamic_opts_t; struct dynamic_opts_s { void **result; + size_t usize; size_t num_items; size_t item_size; size_t alignment; @@ -1600,6 +1824,7 @@ struct dynamic_opts_s { JEMALLOC_ALWAYS_INLINE void dynamic_opts_init(dynamic_opts_t *dynamic_opts) { dynamic_opts->result = NULL; + dynamic_opts->usize = 0; dynamic_opts->num_items = 0; dynamic_opts->item_size = 0; dynamic_opts->alignment = 0; @@ -1663,12 +1888,13 @@ imalloc_sample(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd, szind_t ind_large; size_t bumped_usize = usize; - if (usize <= SMALL_MAXCLASS) { - assert(((dopts->alignment == 0) ? sz_s2u(LARGE_MINCLASS) : - sz_sa2u(LARGE_MINCLASS, dopts->alignment)) - == LARGE_MINCLASS); - ind_large = sz_size2index(LARGE_MINCLASS); - bumped_usize = sz_s2u(LARGE_MINCLASS); + if (usize <= SC_SMALL_MAXCLASS) { + assert(((dopts->alignment == 0) ? + sz_s2u(SC_LARGE_MINCLASS) : + sz_sa2u(SC_LARGE_MINCLASS, dopts->alignment)) + == SC_LARGE_MINCLASS); + ind_large = sz_size2index(SC_LARGE_MINCLASS); + bumped_usize = sz_s2u(SC_LARGE_MINCLASS); ret = imalloc_no_sample(sopts, dopts, tsd, bumped_usize, bumped_usize, ind_large); if (unlikely(ret == NULL)) { @@ -1701,7 +1927,7 @@ compute_size_with_overflow(bool may_overflow, dynamic_opts_t *dopts, } /* A size_t with its high-half bits all set to 1. */ - const static size_t high_bits = SIZE_T_MAX << (sizeof(size_t) * 8 / 2); + static const size_t high_bits = SIZE_T_MAX << (sizeof(size_t) * 8 / 2); *size = dopts->item_size * dopts->num_items; @@ -1752,12 +1978,6 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) { } /* Validate the user input. */ - if (sopts->bump_empty_alloc) { - if (unlikely(size == 0)) { - size = 1; - } - } - if (sopts->assert_nonempty_alloc) { assert (size != 0); } @@ -1771,16 +1991,20 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) { if (dopts->alignment == 0) { ind = sz_size2index(size); - if (unlikely(ind >= NSIZES)) { + if (unlikely(ind >= SC_NSIZES)) { goto label_oom; } - if (config_stats || (config_prof && opt_prof)) { + if (config_stats || (config_prof && opt_prof) || sopts->usize) { usize = sz_index2size(ind); - assert(usize > 0 && usize <= LARGE_MAXCLASS); + dopts->usize = usize; + assert(usize > 0 && usize + <= SC_LARGE_MAXCLASS); } } else { usize = sz_sa2u(size, dopts->alignment); - if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) { + dopts->usize = usize; + if (unlikely(usize == 0 + || usize > SC_LARGE_MAXCLASS)) { goto label_oom; } } @@ -1816,7 +2040,8 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) { alloc_ctx_t alloc_ctx; if (likely((uintptr_t)tctx == (uintptr_t)1U)) { - alloc_ctx.slab = (usize <= SMALL_MAXCLASS); + alloc_ctx.slab = (usize + <= SC_SMALL_MAXCLASS); allocation = imalloc_no_sample( sopts, dopts, tsd, usize, usize, ind); } else if ((uintptr_t)tctx > (uintptr_t)1U) { @@ -1921,9 +2146,8 @@ imalloc_body(static_opts_t *sopts, dynamic_opts_t *dopts, tsd_t *tsd) { return EINVAL; } -/* Returns the errno-style error code of the allocation. */ -JEMALLOC_ALWAYS_INLINE int -imalloc(static_opts_t *sopts, dynamic_opts_t *dopts) { +JEMALLOC_ALWAYS_INLINE bool +imalloc_init_check(static_opts_t *sopts, dynamic_opts_t *dopts) { if (unlikely(!malloc_initialized()) && unlikely(malloc_init())) { if (config_xmalloc && unlikely(opt_xmalloc)) { malloc_write(sopts->oom_string); @@ -1933,6 +2157,16 @@ imalloc(static_opts_t *sopts, dynamic_opts_t *dopts) { set_errno(ENOMEM); *dopts->result = NULL; + return false; + } + + return true; +} + +/* Returns the errno-style error code of the allocation. */ +JEMALLOC_ALWAYS_INLINE int +imalloc(static_opts_t *sopts, dynamic_opts_t *dopts) { + if (tsd_get_allocates() && !imalloc_init_check(sopts, dopts)) { return ENOMEM; } @@ -1945,27 +2179,27 @@ imalloc(static_opts_t *sopts, dynamic_opts_t *dopts) { sopts->slow = false; return imalloc_body(sopts, dopts, tsd); } else { + if (!tsd_get_allocates() && !imalloc_init_check(sopts, dopts)) { + return ENOMEM; + } + sopts->slow = true; return imalloc_body(sopts, dopts, tsd); } } -/******************************************************************************/ -/* - * Begin malloc(3)-compatible functions. - */ -JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN -void JEMALLOC_NOTHROW * -JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1) -je_malloc(size_t size) { +JEMALLOC_NOINLINE +void * +malloc_default(size_t size) { void *ret; static_opts_t sopts; dynamic_opts_t dopts; + LOG("core.malloc.entry", "size: %zu", size); + static_opts_init(&sopts); dynamic_opts_init(&dopts); - sopts.bump_empty_alloc = true; sopts.null_out_result_on_error = true; sopts.set_errno_on_error = true; sopts.oom_string = ": Error in malloc(): out of memory\n"; @@ -1975,10 +2209,107 @@ je_malloc(size_t size) { dopts.item_size = size; imalloc(&sopts, &dopts); + /* + * Note that this branch gets optimized away -- it immediately follows + * the check on tsd_fast that sets sopts.slow. + */ + if (sopts.slow) { + uintptr_t args[3] = {size}; + hook_invoke_alloc(hook_alloc_malloc, ret, (uintptr_t)ret, args); + } + + LOG("core.malloc.exit", "result: %p", ret); return ret; } +/******************************************************************************/ +/* + * Begin malloc(3)-compatible functions. + */ + +/* + * malloc() fastpath. + * + * Fastpath assumes size <= SC_LOOKUP_MAXCLASS, and that we hit + * tcache. If either of these is false, we tail-call to the slowpath, + * malloc_default(). Tail-calling is used to avoid any caller-saved + * registers. + * + * fastpath supports ticker and profiling, both of which will also + * tail-call to the slowpath if they fire. + */ +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN +void JEMALLOC_NOTHROW * +JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1) +je_malloc(size_t size) { + LOG("core.malloc.entry", "size: %zu", size); + + if (tsd_get_allocates() && unlikely(!malloc_initialized())) { + return malloc_default(size); + } + + tsd_t *tsd = tsd_get(false); + if (unlikely(!tsd || !tsd_fast(tsd) || (size > SC_LOOKUP_MAXCLASS))) { + return malloc_default(size); + } + + tcache_t *tcache = tsd_tcachep_get(tsd); + + if (unlikely(ticker_trytick(&tcache->gc_ticker))) { + return malloc_default(size); + } + + szind_t ind = sz_size2index_lookup(size); + size_t usize; + if (config_stats || config_prof) { + usize = sz_index2size(ind); + } + /* Fast path relies on size being a bin. I.e. SC_LOOKUP_MAXCLASS < SC_SMALL_MAXCLASS */ + assert(ind < SC_NBINS); + assert(size <= SC_SMALL_MAXCLASS); + + if (config_prof) { + int64_t bytes_until_sample = tsd_bytes_until_sample_get(tsd); + bytes_until_sample -= usize; + tsd_bytes_until_sample_set(tsd, bytes_until_sample); + + if (unlikely(bytes_until_sample < 0)) { + /* + * Avoid a prof_active check on the fastpath. + * If prof_active is false, set bytes_until_sample to + * a large value. If prof_active is set to true, + * bytes_until_sample will be reset. + */ + if (!prof_active) { + tsd_bytes_until_sample_set(tsd, SSIZE_MAX); + } + return malloc_default(size); + } + } + + cache_bin_t *bin = tcache_small_bin_get(tcache, ind); + bool tcache_success; + void* ret = cache_bin_alloc_easy(bin, &tcache_success); + + if (tcache_success) { + if (config_stats) { + *tsd_thread_allocatedp_get(tsd) += usize; + bin->tstats.nrequests++; + } + if (config_prof) { + tcache->prof_accumbytes += usize; + } + + LOG("core.malloc.exit", "result: %p", ret); + + /* Fastpath success */ + return ret; + } + + return malloc_default(size); +} + JEMALLOC_EXPORT int JEMALLOC_NOTHROW JEMALLOC_ATTR(nonnull(1)) je_posix_memalign(void **memptr, size_t alignment, size_t size) { @@ -1986,10 +2317,12 @@ je_posix_memalign(void **memptr, size_t alignment, size_t size) { static_opts_t sopts; dynamic_opts_t dopts; + LOG("core.posix_memalign.entry", "mem ptr: %p, alignment: %zu, " + "size: %zu", memptr, alignment, size); + static_opts_init(&sopts); dynamic_opts_init(&dopts); - sopts.bump_empty_alloc = true; sopts.min_alignment = sizeof(void *); sopts.oom_string = ": Error allocating aligned memory: out of memory\n"; @@ -2002,6 +2335,16 @@ je_posix_memalign(void **memptr, size_t alignment, size_t size) { dopts.alignment = alignment; ret = imalloc(&sopts, &dopts); + if (sopts.slow) { + uintptr_t args[3] = {(uintptr_t)memptr, (uintptr_t)alignment, + (uintptr_t)size}; + hook_invoke_alloc(hook_alloc_posix_memalign, *memptr, + (uintptr_t)ret, args); + } + + LOG("core.posix_memalign.exit", "result: %d, alloc ptr: %p", ret, + *memptr); + return ret; } @@ -2014,10 +2357,12 @@ je_aligned_alloc(size_t alignment, size_t size) { static_opts_t sopts; dynamic_opts_t dopts; + LOG("core.aligned_alloc.entry", "alignment: %zu, size: %zu\n", + alignment, size); + static_opts_init(&sopts); dynamic_opts_init(&dopts); - sopts.bump_empty_alloc = true; sopts.null_out_result_on_error = true; sopts.set_errno_on_error = true; sopts.min_alignment = 1; @@ -2032,6 +2377,14 @@ je_aligned_alloc(size_t alignment, size_t size) { dopts.alignment = alignment; imalloc(&sopts, &dopts); + if (sopts.slow) { + uintptr_t args[3] = {(uintptr_t)alignment, (uintptr_t)size}; + hook_invoke_alloc(hook_alloc_aligned_alloc, ret, + (uintptr_t)ret, args); + } + + LOG("core.aligned_alloc.exit", "result: %p", ret); + return ret; } @@ -2043,11 +2396,12 @@ je_calloc(size_t num, size_t size) { static_opts_t sopts; dynamic_opts_t dopts; + LOG("core.calloc.entry", "num: %zu, size: %zu\n", num, size); + static_opts_init(&sopts); dynamic_opts_init(&dopts); sopts.may_overflow = true; - sopts.bump_empty_alloc = true; sopts.null_out_result_on_error = true; sopts.set_errno_on_error = true; sopts.oom_string = ": Error in calloc(): out of memory\n"; @@ -2058,26 +2412,34 @@ je_calloc(size_t num, size_t size) { dopts.zero = true; imalloc(&sopts, &dopts); + if (sopts.slow) { + uintptr_t args[3] = {(uintptr_t)num, (uintptr_t)size}; + hook_invoke_alloc(hook_alloc_calloc, ret, (uintptr_t)ret, args); + } + + LOG("core.calloc.exit", "result: %p", ret); return ret; } static void * irealloc_prof_sample(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize, - prof_tctx_t *tctx) { + prof_tctx_t *tctx, hook_ralloc_args_t *hook_args) { void *p; if (tctx == NULL) { return NULL; } - if (usize <= SMALL_MAXCLASS) { - p = iralloc(tsd, old_ptr, old_usize, LARGE_MINCLASS, 0, false); + if (usize <= SC_SMALL_MAXCLASS) { + p = iralloc(tsd, old_ptr, old_usize, + SC_LARGE_MINCLASS, 0, false, hook_args); if (p == NULL) { return NULL; } arena_prof_promote(tsd_tsdn(tsd), p, usize); } else { - p = iralloc(tsd, old_ptr, old_usize, usize, 0, false); + p = iralloc(tsd, old_ptr, old_usize, usize, 0, false, + hook_args); } return p; @@ -2085,7 +2447,7 @@ irealloc_prof_sample(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize, JEMALLOC_ALWAYS_INLINE void * irealloc_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize, - alloc_ctx_t *alloc_ctx) { + alloc_ctx_t *alloc_ctx, hook_ralloc_args_t *hook_args) { void *p; bool prof_active; prof_tctx_t *old_tctx, *tctx; @@ -2094,9 +2456,11 @@ irealloc_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t usize, old_tctx = prof_tctx_get(tsd_tsdn(tsd), old_ptr, alloc_ctx); tctx = prof_alloc_prep(tsd, usize, prof_active, true); if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) { - p = irealloc_prof_sample(tsd, old_ptr, old_usize, usize, tctx); + p = irealloc_prof_sample(tsd, old_ptr, old_usize, usize, tctx, + hook_args); } else { - p = iralloc(tsd, old_ptr, old_usize, usize, 0, false); + p = iralloc(tsd, old_ptr, old_usize, usize, 0, false, + hook_args); } if (unlikely(p == NULL)) { prof_alloc_rollback(tsd, tctx, true); @@ -2125,7 +2489,7 @@ ifree(tsd_t *tsd, void *ptr, tcache_t *tcache, bool slow_path) { rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx, (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab); - assert(alloc_ctx.szind != NSIZES); + assert(alloc_ctx.szind != SC_NSIZES); size_t usize; if (config_prof && opt_prof) { @@ -2161,17 +2525,37 @@ isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path) { assert(malloc_initialized() || IS_INITIALIZER); alloc_ctx_t alloc_ctx, *ctx; - if (config_prof && opt_prof) { + if (!config_cache_oblivious && ((uintptr_t)ptr & PAGE_MASK) != 0) { + /* + * When cache_oblivious is disabled and ptr is not page aligned, + * the allocation was not sampled -- usize can be used to + * determine szind directly. + */ + alloc_ctx.szind = sz_size2index(usize); + alloc_ctx.slab = true; + ctx = &alloc_ctx; + if (config_debug) { + alloc_ctx_t dbg_ctx; + rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); + rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, + rtree_ctx, (uintptr_t)ptr, true, &dbg_ctx.szind, + &dbg_ctx.slab); + assert(dbg_ctx.szind == alloc_ctx.szind); + assert(dbg_ctx.slab == alloc_ctx.slab); + } + } else if (config_prof && opt_prof) { rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx, (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab); assert(alloc_ctx.szind == sz_size2index(usize)); ctx = &alloc_ctx; - prof_free(tsd, ptr, usize, ctx); } else { ctx = NULL; } + if (config_prof && opt_prof) { + prof_free(tsd, ptr, usize, ctx); + } if (config_stats) { *tsd_thread_deallocatedp_get(tsd) += usize; } @@ -2186,11 +2570,14 @@ isfree(tsd_t *tsd, void *ptr, size_t usize, tcache_t *tcache, bool slow_path) { JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN void JEMALLOC_NOTHROW * JEMALLOC_ALLOC_SIZE(2) -je_realloc(void *ptr, size_t size) { +je_realloc(void *ptr, size_t arg_size) { void *ret; tsdn_t *tsdn JEMALLOC_CC_SILENCE_INIT(NULL); size_t usize JEMALLOC_CC_SILENCE_INIT(0); size_t old_usize = 0; + size_t size = arg_size; + + LOG("core.realloc.entry", "ptr: %p, size: %zu\n", ptr, size); if (unlikely(size == 0)) { if (ptr != NULL) { @@ -2203,7 +2590,13 @@ je_realloc(void *ptr, size_t size) { } else { tcache = NULL; } + + uintptr_t args[3] = {(uintptr_t)ptr, size}; + hook_invoke_dalloc(hook_dalloc_realloc, ptr, args); + ifree(tsd, ptr, tcache, true); + + LOG("core.realloc.exit", "result: %p", NULL); return NULL; } size = 1; @@ -2215,28 +2608,59 @@ je_realloc(void *ptr, size_t size) { check_entry_exit_locking(tsd_tsdn(tsd)); + + hook_ralloc_args_t hook_args = {true, {(uintptr_t)ptr, + (uintptr_t)arg_size, 0, 0}}; + alloc_ctx_t alloc_ctx; rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx, (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab); - assert(alloc_ctx.szind != NSIZES); + assert(alloc_ctx.szind != SC_NSIZES); old_usize = sz_index2size(alloc_ctx.szind); assert(old_usize == isalloc(tsd_tsdn(tsd), ptr)); if (config_prof && opt_prof) { usize = sz_s2u(size); - ret = unlikely(usize == 0 || usize > LARGE_MAXCLASS) ? - NULL : irealloc_prof(tsd, ptr, old_usize, usize, - &alloc_ctx); + if (unlikely(usize == 0 + || usize > SC_LARGE_MAXCLASS)) { + ret = NULL; + } else { + ret = irealloc_prof(tsd, ptr, old_usize, usize, + &alloc_ctx, &hook_args); + } } else { if (config_stats) { usize = sz_s2u(size); } - ret = iralloc(tsd, ptr, old_usize, size, 0, false); + ret = iralloc(tsd, ptr, old_usize, size, 0, false, + &hook_args); } tsdn = tsd_tsdn(tsd); } else { /* realloc(NULL, size) is equivalent to malloc(size). */ - return je_malloc(size); + static_opts_t sopts; + dynamic_opts_t dopts; + + static_opts_init(&sopts); + dynamic_opts_init(&dopts); + + sopts.null_out_result_on_error = true; + sopts.set_errno_on_error = true; + sopts.oom_string = + ": Error in realloc(): out of memory\n"; + + dopts.result = &ret; + dopts.num_items = 1; + dopts.item_size = size; + + imalloc(&sopts, &dopts); + if (sopts.slow) { + uintptr_t args[3] = {(uintptr_t)ptr, arg_size}; + hook_invoke_alloc(hook_alloc_realloc, ret, + (uintptr_t)ret, args); + } + + return ret; } if (unlikely(ret == NULL)) { @@ -2257,11 +2681,14 @@ je_realloc(void *ptr, size_t size) { } UTRACE(ptr, size, ret); check_entry_exit_locking(tsdn); + + LOG("core.realloc.exit", "result: %p", ret); return ret; } -JEMALLOC_EXPORT void JEMALLOC_NOTHROW -je_free(void *ptr) { +JEMALLOC_NOINLINE +void +free_default(void *ptr) { UTRACE(ptr, 0, 0); if (likely(ptr != NULL)) { /* @@ -2287,12 +2714,82 @@ je_free(void *ptr) { } else { tcache = NULL; } + uintptr_t args_raw[3] = {(uintptr_t)ptr}; + hook_invoke_dalloc(hook_dalloc_free, ptr, args_raw); ifree(tsd, ptr, tcache, true); } check_entry_exit_locking(tsd_tsdn(tsd)); } } +JEMALLOC_ALWAYS_INLINE +bool free_fastpath(void *ptr, size_t size, bool size_hint) { + tsd_t *tsd = tsd_get(false); + if (unlikely(!tsd || !tsd_fast(tsd))) { + return false; + } + + tcache_t *tcache = tsd_tcachep_get(tsd); + + alloc_ctx_t alloc_ctx; + /* + * If !config_cache_oblivious, we can check PAGE alignment to + * detect sampled objects. Otherwise addresses are + * randomized, and we have to look it up in the rtree anyway. + * See also isfree(). + */ + if (!size_hint || config_cache_oblivious) { + rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); + bool res = rtree_szind_slab_read_fast(tsd_tsdn(tsd), &extents_rtree, + rtree_ctx, (uintptr_t)ptr, + &alloc_ctx.szind, &alloc_ctx.slab); + assert(alloc_ctx.szind != SC_NSIZES); + + /* Note: profiled objects will have alloc_ctx.slab set */ + if (!res || !alloc_ctx.slab) { + return false; + } + } else { + /* + * Check for both sizes that are too large, and for sampled objects. + * Sampled objects are always page-aligned. The sampled object check + * will also check for null ptr. + */ + if (size > SC_LOOKUP_MAXCLASS || (((uintptr_t)ptr & PAGE_MASK) == 0)) { + return false; + } + alloc_ctx.szind = sz_size2index_lookup(size); + } + + if (unlikely(ticker_trytick(&tcache->gc_ticker))) { + return false; + } + + cache_bin_t *bin = tcache_small_bin_get(tcache, alloc_ctx.szind); + cache_bin_info_t *bin_info = &tcache_bin_info[alloc_ctx.szind]; + if (!cache_bin_dalloc_easy(bin, bin_info, ptr)) { + return false; + } + + if (config_stats) { + size_t usize = sz_index2size(alloc_ctx.szind); + *tsd_thread_deallocatedp_get(tsd) += usize; + } + + return true; +} + +JEMALLOC_EXPORT void JEMALLOC_NOTHROW +je_free(void *ptr) { + LOG("core.free.entry", "ptr: %p", ptr); + + if (!free_fastpath(ptr, 0, false)) { + free_default(ptr); + } + + LOG("core.free.exit", ""); +} + /* * End malloc(3)-compatible functions. */ @@ -2310,10 +2807,12 @@ je_memalign(size_t alignment, size_t size) { static_opts_t sopts; dynamic_opts_t dopts; + LOG("core.memalign.entry", "alignment: %zu, size: %zu\n", alignment, + size); + static_opts_init(&sopts); dynamic_opts_init(&dopts); - sopts.bump_empty_alloc = true; sopts.min_alignment = 1; sopts.oom_string = ": Error allocating aligned memory: out of memory\n"; @@ -2327,6 +2826,13 @@ je_memalign(size_t alignment, size_t size) { dopts.alignment = alignment; imalloc(&sopts, &dopts); + if (sopts.slow) { + uintptr_t args[3] = {alignment, size}; + hook_invoke_alloc(hook_alloc_memalign, ret, (uintptr_t)ret, + args); + } + + LOG("core.memalign.exit", "result: %p", ret); return ret; } #endif @@ -2341,10 +2847,11 @@ je_valloc(size_t size) { static_opts_t sopts; dynamic_opts_t dopts; + LOG("core.valloc.entry", "size: %zu\n", size); + static_opts_init(&sopts); dynamic_opts_init(&dopts); - sopts.bump_empty_alloc = true; sopts.null_out_result_on_error = true; sopts.min_alignment = PAGE; sopts.oom_string = @@ -2358,7 +2865,12 @@ je_valloc(size_t size) { dopts.alignment = PAGE; imalloc(&sopts, &dopts); + if (sopts.slow) { + uintptr_t args[3] = {size}; + hook_invoke_alloc(hook_alloc_valloc, ret, (uintptr_t)ret, args); + } + LOG("core.valloc.exit", "result: %p\n", ret); return ret; } #endif @@ -2424,6 +2936,82 @@ int __posix_memalign(void** r, size_t a, size_t s) PREALIAS(je_posix_memalign); * Begin non-standard functions. */ +#ifdef JEMALLOC_EXPERIMENTAL_SMALLOCX_API + +#define JEMALLOC_SMALLOCX_CONCAT_HELPER(x, y) x ## y +#define JEMALLOC_SMALLOCX_CONCAT_HELPER2(x, y) \ + JEMALLOC_SMALLOCX_CONCAT_HELPER(x, y) + +typedef struct { + void *ptr; + size_t size; +} smallocx_return_t; + +JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN +smallocx_return_t JEMALLOC_NOTHROW +/* + * The attribute JEMALLOC_ATTR(malloc) cannot be used due to: + * - https://gcc.gnu.org/bugzilla/show_bug.cgi?id=86488 + */ +JEMALLOC_SMALLOCX_CONCAT_HELPER2(je_smallocx_, JEMALLOC_VERSION_GID_IDENT) + (size_t size, int flags) { + /* + * Note: the attribute JEMALLOC_ALLOC_SIZE(1) cannot be + * used here because it makes writing beyond the `size` + * of the `ptr` undefined behavior, but the objective + * of this function is to allow writing beyond `size` + * up to `smallocx_return_t::size`. + */ + smallocx_return_t ret; + static_opts_t sopts; + dynamic_opts_t dopts; + + LOG("core.smallocx.entry", "size: %zu, flags: %d", size, flags); + + static_opts_init(&sopts); + dynamic_opts_init(&dopts); + + sopts.assert_nonempty_alloc = true; + sopts.null_out_result_on_error = true; + sopts.oom_string = ": Error in mallocx(): out of memory\n"; + sopts.usize = true; + + dopts.result = &ret.ptr; + dopts.num_items = 1; + dopts.item_size = size; + if (unlikely(flags != 0)) { + if ((flags & MALLOCX_LG_ALIGN_MASK) != 0) { + dopts.alignment = MALLOCX_ALIGN_GET_SPECIFIED(flags); + } + + dopts.zero = MALLOCX_ZERO_GET(flags); + + if ((flags & MALLOCX_TCACHE_MASK) != 0) { + if ((flags & MALLOCX_TCACHE_MASK) + == MALLOCX_TCACHE_NONE) { + dopts.tcache_ind = TCACHE_IND_NONE; + } else { + dopts.tcache_ind = MALLOCX_TCACHE_GET(flags); + } + } else { + dopts.tcache_ind = TCACHE_IND_AUTOMATIC; + } + + if ((flags & MALLOCX_ARENA_MASK) != 0) + dopts.arena_ind = MALLOCX_ARENA_GET(flags); + } + + imalloc(&sopts, &dopts); + assert(dopts.usize == je_nallocx(size, flags)); + ret.size = dopts.usize; + + LOG("core.smallocx.exit", "result: %p, size: %zu", ret.ptr, ret.size); + return ret; +} +#undef JEMALLOC_SMALLOCX_CONCAT_HELPER +#undef JEMALLOC_SMALLOCX_CONCAT_HELPER2 +#endif + JEMALLOC_EXPORT JEMALLOC_ALLOCATOR JEMALLOC_RESTRICT_RETURN void JEMALLOC_NOTHROW * JEMALLOC_ATTR(malloc) JEMALLOC_ALLOC_SIZE(1) @@ -2432,6 +3020,8 @@ je_mallocx(size_t size, int flags) { static_opts_t sopts; dynamic_opts_t dopts; + LOG("core.mallocx.entry", "size: %zu, flags: %d", size, flags); + static_opts_init(&sopts); dynamic_opts_init(&dopts); @@ -2465,28 +3055,36 @@ je_mallocx(size_t size, int flags) { } imalloc(&sopts, &dopts); + if (sopts.slow) { + uintptr_t args[3] = {size, flags}; + hook_invoke_alloc(hook_alloc_mallocx, ret, (uintptr_t)ret, + args); + } + + LOG("core.mallocx.exit", "result: %p", ret); return ret; } static void * irallocx_prof_sample(tsdn_t *tsdn, void *old_ptr, size_t old_usize, size_t usize, size_t alignment, bool zero, tcache_t *tcache, arena_t *arena, - prof_tctx_t *tctx) { + prof_tctx_t *tctx, hook_ralloc_args_t *hook_args) { void *p; if (tctx == NULL) { return NULL; } - if (usize <= SMALL_MAXCLASS) { - p = iralloct(tsdn, old_ptr, old_usize, LARGE_MINCLASS, - alignment, zero, tcache, arena); + if (usize <= SC_SMALL_MAXCLASS) { + p = iralloct(tsdn, old_ptr, old_usize, + SC_LARGE_MINCLASS, alignment, zero, tcache, + arena, hook_args); if (p == NULL) { return NULL; } arena_prof_promote(tsdn, p, usize); } else { p = iralloct(tsdn, old_ptr, old_usize, usize, alignment, zero, - tcache, arena); + tcache, arena, hook_args); } return p; @@ -2495,7 +3093,7 @@ irallocx_prof_sample(tsdn_t *tsdn, void *old_ptr, size_t old_usize, JEMALLOC_ALWAYS_INLINE void * irallocx_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t size, size_t alignment, size_t *usize, bool zero, tcache_t *tcache, - arena_t *arena, alloc_ctx_t *alloc_ctx) { + arena_t *arena, alloc_ctx_t *alloc_ctx, hook_ralloc_args_t *hook_args) { void *p; bool prof_active; prof_tctx_t *old_tctx, *tctx; @@ -2505,10 +3103,10 @@ irallocx_prof(tsd_t *tsd, void *old_ptr, size_t old_usize, size_t size, tctx = prof_alloc_prep(tsd, *usize, prof_active, false); if (unlikely((uintptr_t)tctx != (uintptr_t)1U)) { p = irallocx_prof_sample(tsd_tsdn(tsd), old_ptr, old_usize, - *usize, alignment, zero, tcache, arena, tctx); + *usize, alignment, zero, tcache, arena, tctx, hook_args); } else { p = iralloct(tsd_tsdn(tsd), old_ptr, old_usize, size, alignment, - zero, tcache, arena); + zero, tcache, arena, hook_args); } if (unlikely(p == NULL)) { prof_alloc_rollback(tsd, tctx, false); @@ -2545,6 +3143,10 @@ je_rallocx(void *ptr, size_t size, int flags) { arena_t *arena; tcache_t *tcache; + LOG("core.rallocx.entry", "ptr: %p, size: %zu, flags: %d", ptr, + size, flags); + + assert(ptr != NULL); assert(size != 0); assert(malloc_initialized() || IS_INITIALIZER); @@ -2575,23 +3177,27 @@ je_rallocx(void *ptr, size_t size, int flags) { rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx, (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab); - assert(alloc_ctx.szind != NSIZES); + assert(alloc_ctx.szind != SC_NSIZES); old_usize = sz_index2size(alloc_ctx.szind); assert(old_usize == isalloc(tsd_tsdn(tsd), ptr)); + + hook_ralloc_args_t hook_args = {false, {(uintptr_t)ptr, size, flags, + 0}}; if (config_prof && opt_prof) { usize = (alignment == 0) ? sz_s2u(size) : sz_sa2u(size, alignment); - if (unlikely(usize == 0 || usize > LARGE_MAXCLASS)) { + if (unlikely(usize == 0 + || usize > SC_LARGE_MAXCLASS)) { goto label_oom; } p = irallocx_prof(tsd, ptr, old_usize, size, alignment, &usize, - zero, tcache, arena, &alloc_ctx); + zero, tcache, arena, &alloc_ctx, &hook_args); if (unlikely(p == NULL)) { goto label_oom; } } else { p = iralloct(tsd_tsdn(tsd), ptr, old_usize, size, alignment, - zero, tcache, arena); + zero, tcache, arena, &hook_args); if (unlikely(p == NULL)) { goto label_oom; } @@ -2607,6 +3213,8 @@ je_rallocx(void *ptr, size_t size, int flags) { } UTRACE(ptr, size, p); check_entry_exit_locking(tsd_tsdn(tsd)); + + LOG("core.rallocx.exit", "result: %p", p); return p; label_oom: if (config_xmalloc && unlikely(opt_xmalloc)) { @@ -2615,20 +3223,22 @@ je_rallocx(void *ptr, size_t size, int flags) { } UTRACE(ptr, size, 0); check_entry_exit_locking(tsd_tsdn(tsd)); + + LOG("core.rallocx.exit", "result: %p", NULL); return NULL; } JEMALLOC_ALWAYS_INLINE size_t ixallocx_helper(tsdn_t *tsdn, void *ptr, size_t old_usize, size_t size, size_t extra, size_t alignment, bool zero) { - size_t usize; + size_t newsize; - if (ixalloc(tsdn, ptr, old_usize, size, extra, alignment, zero)) { + if (ixalloc(tsdn, ptr, old_usize, size, extra, alignment, zero, + &newsize)) { return old_usize; } - usize = isalloc(tsdn, ptr); - return usize; + return newsize; } static size_t @@ -2662,17 +3272,19 @@ ixallocx_prof(tsd_t *tsd, void *ptr, size_t old_usize, size_t size, */ if (alignment == 0) { usize_max = sz_s2u(size+extra); - assert(usize_max > 0 && usize_max <= LARGE_MAXCLASS); + assert(usize_max > 0 + && usize_max <= SC_LARGE_MAXCLASS); } else { usize_max = sz_sa2u(size+extra, alignment); - if (unlikely(usize_max == 0 || usize_max > LARGE_MAXCLASS)) { + if (unlikely(usize_max == 0 + || usize_max > SC_LARGE_MAXCLASS)) { /* * usize_max is out of range, and chances are that * allocation will fail, but use the maximum possible * value and carry on with prof_alloc_prep(), just in * case allocation succeeds. */ - usize_max = LARGE_MAXCLASS; + usize_max = SC_LARGE_MAXCLASS; } } tctx = prof_alloc_prep(tsd, usize_max, prof_active, false); @@ -2701,6 +3313,9 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) { size_t alignment = MALLOCX_ALIGN_GET(flags); bool zero = flags & MALLOCX_ZERO; + LOG("core.xallocx.entry", "ptr: %p, size: %zu, extra: %zu, " + "flags: %d", ptr, size, extra, flags); + assert(ptr != NULL); assert(size != 0); assert(SIZE_T_MAX - size >= extra); @@ -2712,24 +3327,24 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) { rtree_ctx_t *rtree_ctx = tsd_rtree_ctx(tsd); rtree_szind_slab_read(tsd_tsdn(tsd), &extents_rtree, rtree_ctx, (uintptr_t)ptr, true, &alloc_ctx.szind, &alloc_ctx.slab); - assert(alloc_ctx.szind != NSIZES); + assert(alloc_ctx.szind != SC_NSIZES); old_usize = sz_index2size(alloc_ctx.szind); assert(old_usize == isalloc(tsd_tsdn(tsd), ptr)); /* * The API explicitly absolves itself of protecting against (size + * extra) numerical overflow, but we may need to clamp extra to avoid - * exceeding LARGE_MAXCLASS. + * exceeding SC_LARGE_MAXCLASS. * * Ordinarily, size limit checking is handled deeper down, but here we * have to check as part of (size + extra) clamping, since we need the * clamped value in the above helper functions. */ - if (unlikely(size > LARGE_MAXCLASS)) { + if (unlikely(size > SC_LARGE_MAXCLASS)) { usize = old_usize; goto label_not_resized; } - if (unlikely(LARGE_MAXCLASS - size < extra)) { - extra = LARGE_MAXCLASS - size; + if (unlikely(SC_LARGE_MAXCLASS - size < extra)) { + extra = SC_LARGE_MAXCLASS - size; } if (config_prof && opt_prof) { @@ -2748,8 +3363,16 @@ je_xallocx(void *ptr, size_t size, size_t extra, int flags) { *tsd_thread_deallocatedp_get(tsd) += old_usize; } label_not_resized: + if (unlikely(!tsd_fast(tsd))) { + uintptr_t args[4] = {(uintptr_t)ptr, size, extra, flags}; + hook_invoke_expand(hook_expand_xallocx, ptr, old_usize, + usize, (uintptr_t)usize, args); + } + UTRACE(ptr, size, ptr); check_entry_exit_locking(tsd_tsdn(tsd)); + + LOG("core.xallocx.exit", "result: %zu", usize); return usize; } @@ -2759,6 +3382,8 @@ je_sallocx(const void *ptr, int flags) { size_t usize; tsdn_t *tsdn; + LOG("core.sallocx.entry", "ptr: %p, flags: %d", ptr, flags); + assert(malloc_initialized() || IS_INITIALIZER); assert(ptr != NULL); @@ -2773,11 +3398,15 @@ je_sallocx(const void *ptr, int flags) { } check_entry_exit_locking(tsdn); + + LOG("core.sallocx.exit", "result: %zu", usize); return usize; } JEMALLOC_EXPORT void JEMALLOC_NOTHROW je_dallocx(void *ptr, int flags) { + LOG("core.dallocx.entry", "ptr: %p, flags: %d", ptr, flags); + assert(ptr != NULL); assert(malloc_initialized() || IS_INITIALIZER); @@ -2812,9 +3441,13 @@ je_dallocx(void *ptr, int flags) { tsd_assert_fast(tsd); ifree(tsd, ptr, tcache, false); } else { + uintptr_t args_raw[3] = {(uintptr_t)ptr, flags}; + hook_invoke_dalloc(hook_dalloc_dallocx, ptr, args_raw); ifree(tsd, ptr, tcache, true); } check_entry_exit_locking(tsd_tsdn(tsd)); + + LOG("core.dallocx.exit", ""); } JEMALLOC_ALWAYS_INLINE size_t @@ -2831,8 +3464,8 @@ inallocx(tsdn_t *tsdn, size_t size, int flags) { return usize; } -JEMALLOC_EXPORT void JEMALLOC_NOTHROW -je_sdallocx(void *ptr, size_t size, int flags) { +JEMALLOC_NOINLINE void +sdallocx_default(void *ptr, size_t size, int flags) { assert(ptr != NULL); assert(malloc_initialized() || IS_INITIALIZER); @@ -2869,9 +3502,24 @@ je_sdallocx(void *ptr, size_t size, int flags) { tsd_assert_fast(tsd); isfree(tsd, ptr, usize, tcache, false); } else { + uintptr_t args_raw[3] = {(uintptr_t)ptr, size, flags}; + hook_invoke_dalloc(hook_dalloc_sdallocx, ptr, args_raw); isfree(tsd, ptr, usize, tcache, true); } check_entry_exit_locking(tsd_tsdn(tsd)); + +} + +JEMALLOC_EXPORT void JEMALLOC_NOTHROW +je_sdallocx(void *ptr, size_t size, int flags) { + LOG("core.sdallocx.entry", "ptr: %p, size: %zu, flags: %d", ptr, + size, flags); + + if (flags !=0 || !free_fastpath(ptr, size, true)) { + sdallocx_default(ptr, size, flags); + } + + LOG("core.sdallocx.exit", ""); } JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW @@ -2883,6 +3531,7 @@ je_nallocx(size_t size, int flags) { assert(size != 0); if (unlikely(malloc_init())) { + LOG("core.nallocx.exit", "result: %zu", ZU(0)); return 0; } @@ -2890,11 +3539,13 @@ je_nallocx(size_t size, int flags) { check_entry_exit_locking(tsdn); usize = inallocx(tsdn, size, flags); - if (unlikely(usize > LARGE_MAXCLASS)) { + if (unlikely(usize > SC_LARGE_MAXCLASS)) { + LOG("core.nallocx.exit", "result: %zu", ZU(0)); return 0; } check_entry_exit_locking(tsdn); + LOG("core.nallocx.exit", "result: %zu", usize); return usize; } @@ -2904,7 +3555,10 @@ je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp, int ret; tsd_t *tsd; + LOG("core.mallctl.entry", "name: %s", name); + if (unlikely(malloc_init())) { + LOG("core.mallctl.exit", "result: %d", EAGAIN); return EAGAIN; } @@ -2912,6 +3566,8 @@ je_mallctl(const char *name, void *oldp, size_t *oldlenp, void *newp, check_entry_exit_locking(tsd_tsdn(tsd)); ret = ctl_byname(tsd, name, oldp, oldlenp, newp, newlen); check_entry_exit_locking(tsd_tsdn(tsd)); + + LOG("core.mallctl.exit", "result: %d", ret); return ret; } @@ -2919,7 +3575,10 @@ JEMALLOC_EXPORT int JEMALLOC_NOTHROW je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp) { int ret; + LOG("core.mallctlnametomib.entry", "name: %s", name); + if (unlikely(malloc_init())) { + LOG("core.mallctlnametomib.exit", "result: %d", EAGAIN); return EAGAIN; } @@ -2927,6 +3586,8 @@ je_mallctlnametomib(const char *name, size_t *mibp, size_t *miblenp) { check_entry_exit_locking(tsd_tsdn(tsd)); ret = ctl_nametomib(tsd, name, mibp, miblenp); check_entry_exit_locking(tsd_tsdn(tsd)); + + LOG("core.mallctlnametomib.exit", "result: %d", ret); return ret; } @@ -2936,7 +3597,10 @@ je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, int ret; tsd_t *tsd; + LOG("core.mallctlbymib.entry", ""); + if (unlikely(malloc_init())) { + LOG("core.mallctlbymib.exit", "result: %d", EAGAIN); return EAGAIN; } @@ -2944,6 +3608,7 @@ je_mallctlbymib(const size_t *mib, size_t miblen, void *oldp, size_t *oldlenp, check_entry_exit_locking(tsd_tsdn(tsd)); ret = ctl_bymib(tsd, mib, miblen, oldp, oldlenp, newp, newlen); check_entry_exit_locking(tsd_tsdn(tsd)); + LOG("core.mallctlbymib.exit", "result: %d", ret); return ret; } @@ -2952,10 +3617,13 @@ je_malloc_stats_print(void (*write_cb)(void *, const char *), void *cbopaque, const char *opts) { tsdn_t *tsdn; + LOG("core.malloc_stats_print.entry", ""); + tsdn = tsdn_fetch(); check_entry_exit_locking(tsdn); stats_print(write_cb, cbopaque, opts); check_entry_exit_locking(tsdn); + LOG("core.malloc_stats_print.exit", ""); } JEMALLOC_EXPORT size_t JEMALLOC_NOTHROW @@ -2963,6 +3631,8 @@ je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) { size_t ret; tsdn_t *tsdn; + LOG("core.malloc_usable_size.entry", "ptr: %p", ptr); + assert(malloc_initialized() || IS_INITIALIZER); tsdn = tsdn_fetch(); @@ -2980,6 +3650,7 @@ je_malloc_usable_size(JEMALLOC_USABLE_SIZE_CONST void *ptr) { } check_entry_exit_locking(tsdn); + LOG("core.malloc_usable_size.exit", "result: %zu", ret); return ret; } @@ -3084,6 +3755,7 @@ _malloc_prefork(void) } } prof_prefork1(tsd_tsdn(tsd)); + tsd_prefork(tsd); } #ifndef JEMALLOC_MUTEX_INIT_CB @@ -3106,6 +3778,8 @@ _malloc_postfork(void) tsd = tsd_fetch(); + tsd_postfork_parent(tsd); + witness_postfork_parent(tsd_witness_tsdp_get(tsd)); /* Release all mutexes, now that fork() has completed. */ for (i = 0, narenas = narenas_total_get(); i < narenas; i++) { @@ -3133,6 +3807,8 @@ jemalloc_postfork_child(void) { tsd = tsd_fetch(); + tsd_postfork_child(tsd); + witness_postfork_child(tsd_witness_tsdp_get(tsd)); /* Release all mutexes, now that fork() has completed. */ for (i = 0, narenas = narenas_total_get(); i < narenas; i++) { diff --git a/kbe/src/lib/dependencies/jemalloc/src/jemalloc_cpp.cpp b/kbe/src/lib/dependencies/jemalloc/src/jemalloc_cpp.cpp index 844ab398a7..f0ceddae33 100755 --- a/kbe/src/lib/dependencies/jemalloc/src/jemalloc_cpp.cpp +++ b/kbe/src/lib/dependencies/jemalloc/src/jemalloc_cpp.cpp @@ -39,12 +39,10 @@ void operator delete(void *ptr, std::size_t size) noexcept; void operator delete[](void *ptr, std::size_t size) noexcept; #endif -template -void * -newImpl(std::size_t size) noexcept(IsNoExcept) { - void *ptr = je_malloc(size); - if (likely(ptr != nullptr)) - return ptr; +JEMALLOC_NOINLINE +static void * +handleOOM(std::size_t size, bool nothrow) { + void *ptr = nullptr; while (ptr == nullptr) { std::new_handler handler; @@ -68,11 +66,22 @@ newImpl(std::size_t size) noexcept(IsNoExcept) { ptr = je_malloc(size); } - if (ptr == nullptr && !IsNoExcept) + if (ptr == nullptr && !nothrow) std::__throw_bad_alloc(); return ptr; } +template +JEMALLOC_ALWAYS_INLINE +void * +newImpl(std::size_t size) noexcept(IsNoExcept) { + void *ptr = je_malloc(size); + if (likely(ptr != nullptr)) + return ptr; + + return handleOOM(size, IsNoExcept); +} + void * operator new(std::size_t size) { return newImpl(size); diff --git a/kbe/src/lib/dependencies/jemalloc/src/large.c b/kbe/src/lib/dependencies/jemalloc/src/large.c index 27a2c67987..8e7a781d33 100755 --- a/kbe/src/lib/dependencies/jemalloc/src/large.c +++ b/kbe/src/lib/dependencies/jemalloc/src/large.c @@ -28,7 +28,7 @@ large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, assert(!tsdn_null(tsdn) || arena != NULL); ausize = sz_sa2u(usize, alignment); - if (unlikely(ausize == 0 || ausize > LARGE_MAXCLASS)) { + if (unlikely(ausize == 0 || ausize > SC_LARGE_MAXCLASS)) { return NULL; } @@ -42,7 +42,7 @@ large_palloc(tsdn_t *tsdn, arena_t *arena, size_t usize, size_t alignment, */ is_zeroed = zero; if (likely(!tsdn_null(tsdn))) { - arena = arena_choose(tsdn_tsd(tsdn), arena); + arena = arena_choose_maybe_huge(tsdn_tsd(tsdn), arena, usize); } if (unlikely(arena == NULL) || (extent = arena_extent_alloc_large(tsdn, arena, usize, alignment, &is_zeroed)) == NULL) { @@ -109,7 +109,7 @@ large_ralloc_no_move_shrink(tsdn_t *tsdn, extent_t *extent, size_t usize) { if (diff != 0) { extent_t *trail = extent_split_wrapper(tsdn, arena, &extent_hooks, extent, usize + sz_large_pad, - sz_size2index(usize), false, diff, NSIZES, false); + sz_size2index(usize), false, diff, SC_NSIZES, false); if (trail == NULL) { return true; } @@ -154,17 +154,17 @@ large_ralloc_no_move_expand(tsdn_t *tsdn, extent_t *extent, size_t usize, bool new_mapping; if ((trail = extents_alloc(tsdn, arena, &extent_hooks, &arena->extents_dirty, extent_past_get(extent), trailsize, 0, - CACHELINE, false, NSIZES, &is_zeroed_trail, &commit)) != NULL + CACHELINE, false, SC_NSIZES, &is_zeroed_trail, &commit)) != NULL || (trail = extents_alloc(tsdn, arena, &extent_hooks, &arena->extents_muzzy, extent_past_get(extent), trailsize, 0, - CACHELINE, false, NSIZES, &is_zeroed_trail, &commit)) != NULL) { + CACHELINE, false, SC_NSIZES, &is_zeroed_trail, &commit)) != NULL) { if (config_stats) { new_mapping = false; } } else { if ((trail = extent_alloc_wrapper(tsdn, arena, &extent_hooks, extent_past_get(extent), trailsize, 0, CACHELINE, false, - NSIZES, &is_zeroed_trail, &commit)) == NULL) { + SC_NSIZES, &is_zeroed_trail, &commit)) == NULL) { return true; } if (config_stats) { @@ -221,9 +221,10 @@ large_ralloc_no_move(tsdn_t *tsdn, extent_t *extent, size_t usize_min, size_t oldusize = extent_usize_get(extent); /* The following should have been caught by callers. */ - assert(usize_min > 0 && usize_max <= LARGE_MAXCLASS); + assert(usize_min > 0 && usize_max <= SC_LARGE_MAXCLASS); /* Both allocation sizes must be large to avoid a move. */ - assert(oldusize >= LARGE_MINCLASS && usize_max >= LARGE_MINCLASS); + assert(oldusize >= SC_LARGE_MINCLASS + && usize_max >= SC_LARGE_MINCLASS); if (usize_max > oldusize) { /* Attempt to expand the allocation in-place. */ @@ -270,17 +271,23 @@ large_ralloc_move_helper(tsdn_t *tsdn, arena_t *arena, size_t usize, } void * -large_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, size_t usize, - size_t alignment, bool zero, tcache_t *tcache) { - size_t oldusize = extent_usize_get(extent); +large_ralloc(tsdn_t *tsdn, arena_t *arena, void *ptr, size_t usize, + size_t alignment, bool zero, tcache_t *tcache, + hook_ralloc_args_t *hook_args) { + extent_t *extent = iealloc(tsdn, ptr); + size_t oldusize = extent_usize_get(extent); /* The following should have been caught by callers. */ - assert(usize > 0 && usize <= LARGE_MAXCLASS); + assert(usize > 0 && usize <= SC_LARGE_MAXCLASS); /* Both allocation sizes must be large to avoid a move. */ - assert(oldusize >= LARGE_MINCLASS && usize >= LARGE_MINCLASS); + assert(oldusize >= SC_LARGE_MINCLASS + && usize >= SC_LARGE_MINCLASS); /* Try to avoid moving the allocation. */ if (!large_ralloc_no_move(tsdn, extent, usize, usize, zero)) { + hook_invoke_expand(hook_args->is_realloc + ? hook_expand_realloc : hook_expand_rallocx, ptr, oldusize, + usize, (uintptr_t)ptr, hook_args->args); return extent_addr_get(extent); } @@ -295,6 +302,12 @@ large_ralloc(tsdn_t *tsdn, arena_t *arena, extent_t *extent, size_t usize, return NULL; } + hook_invoke_alloc(hook_args->is_realloc + ? hook_alloc_realloc : hook_alloc_rallocx, ret, (uintptr_t)ret, + hook_args->args); + hook_invoke_dalloc(hook_args->is_realloc + ? hook_dalloc_realloc : hook_dalloc_rallocx, ptr, hook_args->args); + size_t copysize = (usize < oldusize) ? usize : oldusize; memcpy(ret, extent_addr_get(extent), copysize); isdalloct(tsdn, extent_addr_get(extent), oldusize, tcache, NULL, true); @@ -318,8 +331,9 @@ large_dalloc_prep_impl(tsdn_t *tsdn, arena_t *arena, extent_t *extent, large_dalloc_maybe_junk(extent_addr_get(extent), extent_usize_get(extent)); } else { - malloc_mutex_assert_owner(tsdn, &arena->large_mtx); + /* Only hold the large_mtx if necessary. */ if (!arena_is_auto(arena)) { + malloc_mutex_assert_owner(tsdn, &arena->large_mtx); extent_list_remove(&arena->large, extent); } } @@ -369,3 +383,13 @@ void large_prof_tctx_reset(tsdn_t *tsdn, extent_t *extent) { large_prof_tctx_set(tsdn, extent, (prof_tctx_t *)(uintptr_t)1U); } + +nstime_t +large_prof_alloc_time_get(const extent_t *extent) { + return extent_prof_alloc_time_get(extent); +} + +void +large_prof_alloc_time_set(extent_t *extent, nstime_t t) { + extent_prof_alloc_time_set(extent, t); +} diff --git a/kbe/src/lib/dependencies/jemalloc/src/log.c b/kbe/src/lib/dependencies/jemalloc/src/log.c new file mode 100644 index 0000000000..778902fb9b --- /dev/null +++ b/kbe/src/lib/dependencies/jemalloc/src/log.c @@ -0,0 +1,78 @@ +#include "jemalloc/internal/jemalloc_preamble.h" +#include "jemalloc/internal/jemalloc_internal_includes.h" + +#include "jemalloc/internal/log.h" + +char log_var_names[JEMALLOC_LOG_VAR_BUFSIZE]; +atomic_b_t log_init_done = ATOMIC_INIT(false); + +/* + * Returns true if we were able to pick out a segment. Fills in r_segment_end + * with a pointer to the first character after the end of the string. + */ +static const char * +log_var_extract_segment(const char* segment_begin) { + const char *end; + for (end = segment_begin; *end != '\0' && *end != '|'; end++) { + } + return end; +} + +static bool +log_var_matches_segment(const char *segment_begin, const char *segment_end, + const char *log_var_begin, const char *log_var_end) { + assert(segment_begin <= segment_end); + assert(log_var_begin < log_var_end); + + ptrdiff_t segment_len = segment_end - segment_begin; + ptrdiff_t log_var_len = log_var_end - log_var_begin; + /* The special '.' segment matches everything. */ + if (segment_len == 1 && *segment_begin == '.') { + return true; + } + if (segment_len == log_var_len) { + return strncmp(segment_begin, log_var_begin, segment_len) == 0; + } else if (segment_len < log_var_len) { + return strncmp(segment_begin, log_var_begin, segment_len) == 0 + && log_var_begin[segment_len] == '.'; + } else { + return false; + } +} + +unsigned +log_var_update_state(log_var_t *log_var) { + const char *log_var_begin = log_var->name; + const char *log_var_end = log_var->name + strlen(log_var->name); + + /* Pointer to one before the beginning of the current segment. */ + const char *segment_begin = log_var_names; + + /* + * If log_init done is false, we haven't parsed the malloc conf yet. To + * avoid log-spew, we default to not displaying anything. + */ + if (!atomic_load_b(&log_init_done, ATOMIC_ACQUIRE)) { + return LOG_INITIALIZED_NOT_ENABLED; + } + + while (true) { + const char *segment_end = log_var_extract_segment( + segment_begin); + assert(segment_end < log_var_names + JEMALLOC_LOG_VAR_BUFSIZE); + if (log_var_matches_segment(segment_begin, segment_end, + log_var_begin, log_var_end)) { + atomic_store_u(&log_var->state, LOG_ENABLED, + ATOMIC_RELAXED); + return LOG_ENABLED; + } + if (*segment_end == '\0') { + /* Hit the end of the segment string with no match. */ + atomic_store_u(&log_var->state, + LOG_INITIALIZED_NOT_ENABLED, ATOMIC_RELAXED); + return LOG_INITIALIZED_NOT_ENABLED; + } + /* Otherwise, skip the delimiter and continue. */ + segment_begin = segment_end + 1; + } +} diff --git a/kbe/src/lib/dependencies/jemalloc/src/malloc_io.c b/kbe/src/lib/dependencies/jemalloc/src/malloc_io.c index 6b99afcd3f..7bdc13f951 100755 --- a/kbe/src/lib/dependencies/jemalloc/src/malloc_io.c +++ b/kbe/src/lib/dependencies/jemalloc/src/malloc_io.c @@ -70,20 +70,7 @@ static char *x2s(uintmax_t x, bool alt_form, bool uppercase, char *s, /* malloc_message() setup. */ static void wrtmessage(void *cbopaque, const char *s) { -#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_write) - /* - * Use syscall(2) rather than write(2) when possible in order to avoid - * the possibility of memory allocation within libc. This is necessary - * on FreeBSD; most operating systems do not have this problem though. - * - * syscall() returns long or int, depending on platform, so capture the - * unused result in the widest plausible type to avoid compiler - * warnings. - */ - UNUSED long result = syscall(SYS_write, STDERR_FILENO, s, strlen(s)); -#else - UNUSED ssize_t result = write(STDERR_FILENO, s, strlen(s)); -#endif + malloc_write_fd(STDERR_FILENO, s, strlen(s)); } JEMALLOC_EXPORT void (*je_malloc_message)(void *, const char *s); @@ -111,7 +98,7 @@ buferror(int err, char *buf, size_t buflen) { FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM, NULL, err, 0, (LPSTR)buf, (DWORD)buflen, NULL); return 0; -#elif defined(__GLIBC__) && defined(_GNU_SOURCE) +#elif defined(JEMALLOC_STRERROR_R_RETURNS_CHAR_WITH_GNU_SOURCE) && defined(_GNU_SOURCE) char *b = strerror_r(err, buf, buflen); if (b != buf) { strncpy(buf, b, buflen); diff --git a/kbe/src/lib/dependencies/jemalloc/src/mutex.c b/kbe/src/lib/dependencies/jemalloc/src/mutex.c index a528ef0c24..3f920f5b1c 100755 --- a/kbe/src/lib/dependencies/jemalloc/src/mutex.c +++ b/kbe/src/lib/dependencies/jemalloc/src/mutex.c @@ -4,6 +4,7 @@ #include "jemalloc/internal/assert.h" #include "jemalloc/internal/malloc_io.h" +#include "jemalloc/internal/spin.h" #ifndef _CRT_SPINCOUNT #define _CRT_SPINCOUNT 4000 @@ -45,7 +46,7 @@ JEMALLOC_EXPORT int _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex, void malloc_mutex_lock_slow(malloc_mutex_t *mutex) { mutex_prof_data_t *data = &mutex->prof_data; - UNUSED nstime_t before = NSTIME_ZERO_INITIALIZER; + nstime_t before = NSTIME_ZERO_INITIALIZER; if (ncpus == 1) { goto label_spin_done; @@ -53,8 +54,9 @@ malloc_mutex_lock_slow(malloc_mutex_t *mutex) { int cnt = 0, max_cnt = MALLOC_MUTEX_MAX_SPIN; do { - CPU_SPINWAIT; - if (!malloc_mutex_trylock_final(mutex)) { + spin_cpu_spinwait(); + if (!atomic_load_b(&mutex->locked, ATOMIC_RELAXED) + && !malloc_mutex_trylock_final(mutex)) { data->n_spin_acquired++; return; } @@ -143,9 +145,7 @@ malloc_mutex_init(malloc_mutex_t *mutex, const char *name, } # endif #elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) - mutex->lock = OS_UNFAIR_LOCK_INIT; -#elif (defined(JEMALLOC_OSSPIN)) - mutex->lock = 0; + mutex->lock = OS_UNFAIR_LOCK_INIT; #elif (defined(JEMALLOC_MUTEX_INIT_CB)) if (postpone_init) { mutex->postponed_next = postponed_mutexes; @@ -173,7 +173,7 @@ malloc_mutex_init(malloc_mutex_t *mutex, const char *name, mutex->lock_order = lock_order; if (lock_order == malloc_mutex_address_ordered) { witness_init(&mutex->witness, name, rank, - mutex_addr_comp, &mutex); + mutex_addr_comp, mutex); } else { witness_init(&mutex->witness, name, rank, NULL, NULL); } diff --git a/kbe/src/lib/dependencies/jemalloc/src/pages.c b/kbe/src/lib/dependencies/jemalloc/src/pages.c index fec64dd01d..13de27a008 100755 --- a/kbe/src/lib/dependencies/jemalloc/src/pages.c +++ b/kbe/src/lib/dependencies/jemalloc/src/pages.c @@ -10,6 +10,9 @@ #ifdef JEMALLOC_SYSCTL_VM_OVERCOMMIT #include +#ifdef __FreeBSD__ +#include +#endif #endif /******************************************************************************/ @@ -25,6 +28,18 @@ static int mmap_flags; #endif static bool os_overcommits; +const char *thp_mode_names[] = { + "default", + "always", + "never", + "not supported" +}; +thp_mode_t opt_thp = THP_MODE_DEFAULT; +thp_mode_t init_system_thp_mode; + +/* Runtime support for lazy purge. Irrelevant when !pages_can_purge_lazy. */ +static bool pages_can_purge_lazy_runtime = true; + /******************************************************************************/ /* * Function prototypes for static functions that are referenced prior to @@ -165,6 +180,35 @@ pages_map(void *addr, size_t size, size_t alignment, bool *commit) { assert(alignment >= PAGE); assert(ALIGNMENT_ADDR2BASE(addr, alignment) == addr); +#if defined(__FreeBSD__) && defined(MAP_EXCL) + /* + * FreeBSD has mechanisms both to mmap at specific address without + * touching existing mappings, and to mmap with specific alignment. + */ + { + if (os_overcommits) { + *commit = true; + } + + int prot = *commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT; + int flags = mmap_flags; + + if (addr != NULL) { + flags |= MAP_FIXED | MAP_EXCL; + } else { + unsigned alignment_bits = ffs_zu(alignment); + assert(alignment_bits > 1); + flags |= MAP_ALIGNED(alignment_bits - 1); + } + + void *ret = mmap(addr, size, prot, flags, -1, 0); + if (ret == MAP_FAILED) { + ret = NULL; + } + + return ret; + } +#endif /* * Ideally, there would be a way to specify alignment to mmap() (like * NetBSD has), but in the absence of such a feature, we have to work @@ -246,18 +290,31 @@ pages_decommit(void *addr, size_t size) { bool pages_purge_lazy(void *addr, size_t size) { - assert(PAGE_ADDR2BASE(addr) == addr); + assert(ALIGNMENT_ADDR2BASE(addr, os_page) == addr); assert(PAGE_CEILING(size) == size); if (!pages_can_purge_lazy) { return true; } + if (!pages_can_purge_lazy_runtime) { + /* + * Built with lazy purge enabled, but detected it was not + * supported on the current system. + */ + return true; + } #ifdef _WIN32 VirtualAlloc(addr, size, MEM_RESET, PAGE_READWRITE); return false; #elif defined(JEMALLOC_PURGE_MADVISE_FREE) - return (madvise(addr, size, MADV_FREE) != 0); + return (madvise(addr, size, +# ifdef MADV_FREE + MADV_FREE +# else + JEMALLOC_MADV_FREE +# endif + ) != 0); #elif defined(JEMALLOC_PURGE_MADVISE_DONTNEED) && \ !defined(JEMALLOC_PURGE_MADVISE_DONTNEED_ZEROS) return (madvise(addr, size, MADV_DONTNEED) != 0); @@ -286,36 +343,88 @@ pages_purge_forced(void *addr, size_t size) { #endif } +static bool +pages_huge_impl(void *addr, size_t size, bool aligned) { + if (aligned) { + assert(HUGEPAGE_ADDR2BASE(addr) == addr); + assert(HUGEPAGE_CEILING(size) == size); + } +#ifdef JEMALLOC_HAVE_MADVISE_HUGE + return (madvise(addr, size, MADV_HUGEPAGE) != 0); +#else + return true; +#endif +} + bool pages_huge(void *addr, size_t size) { - assert(HUGEPAGE_ADDR2BASE(addr) == addr); - assert(HUGEPAGE_CEILING(size) == size); + return pages_huge_impl(addr, size, true); +} -#ifdef JEMALLOC_THP - return (madvise(addr, size, MADV_HUGEPAGE) != 0); +static bool +pages_huge_unaligned(void *addr, size_t size) { + return pages_huge_impl(addr, size, false); +} + +static bool +pages_nohuge_impl(void *addr, size_t size, bool aligned) { + if (aligned) { + assert(HUGEPAGE_ADDR2BASE(addr) == addr); + assert(HUGEPAGE_CEILING(size) == size); + } + +#ifdef JEMALLOC_HAVE_MADVISE_HUGE + return (madvise(addr, size, MADV_NOHUGEPAGE) != 0); #else - return true; + return false; #endif } bool pages_nohuge(void *addr, size_t size) { - assert(HUGEPAGE_ADDR2BASE(addr) == addr); - assert(HUGEPAGE_CEILING(size) == size); + return pages_nohuge_impl(addr, size, true); +} -#ifdef JEMALLOC_THP - return (madvise(addr, size, MADV_NOHUGEPAGE) != 0); +static bool +pages_nohuge_unaligned(void *addr, size_t size) { + return pages_nohuge_impl(addr, size, false); +} + +bool +pages_dontdump(void *addr, size_t size) { + assert(PAGE_ADDR2BASE(addr) == addr); + assert(PAGE_CEILING(size) == size); +#ifdef JEMALLOC_MADVISE_DONTDUMP + return madvise(addr, size, MADV_DONTDUMP) != 0; #else return false; #endif } +bool +pages_dodump(void *addr, size_t size) { + assert(PAGE_ADDR2BASE(addr) == addr); + assert(PAGE_CEILING(size) == size); +#ifdef JEMALLOC_MADVISE_DONTDUMP + return madvise(addr, size, MADV_DODUMP) != 0; +#else + return false; +#endif +} + + static size_t os_page_detect(void) { #ifdef _WIN32 SYSTEM_INFO si; GetSystemInfo(&si); return si.dwPageSize; +#elif defined(__FreeBSD__) + /* + * This returns the value obtained from + * the auxv vector, avoiding a syscall. + */ + return getpagesize(); #else long result = sysconf(_SC_PAGESIZE); if (result == -1) { @@ -332,9 +441,19 @@ os_overcommits_sysctl(void) { size_t sz; sz = sizeof(vm_overcommit); +#if defined(__FreeBSD__) && defined(VM_OVERCOMMIT) + int mib[2]; + + mib[0] = CTL_VM; + mib[1] = VM_OVERCOMMIT; + if (sysctl(mib, 2, &vm_overcommit, &sz, NULL, 0) != 0) { + return false; /* Error. */ + } +#else if (sysctlbyname("vm.overcommit", &vm_overcommit, &sz, NULL, 0) != 0) { return false; /* Error. */ } +#endif return ((vm_overcommit & 0x3) == 0); } @@ -350,27 +469,44 @@ static bool os_overcommits_proc(void) { int fd; char buf[1]; - ssize_t nread; #if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_open) - fd = (int)syscall(SYS_open, "/proc/sys/vm/overcommit_memory", O_RDONLY | - O_CLOEXEC); + #if defined(O_CLOEXEC) + fd = (int)syscall(SYS_open, "/proc/sys/vm/overcommit_memory", O_RDONLY | + O_CLOEXEC); + #else + fd = (int)syscall(SYS_open, "/proc/sys/vm/overcommit_memory", O_RDONLY); + if (fd != -1) { + fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC); + } + #endif #elif defined(JEMALLOC_USE_SYSCALL) && defined(SYS_openat) - fd = (int)syscall(SYS_openat, - AT_FDCWD, "/proc/sys/vm/overcommit_memory", O_RDONLY | O_CLOEXEC); + #if defined(O_CLOEXEC) + fd = (int)syscall(SYS_openat, + AT_FDCWD, "/proc/sys/vm/overcommit_memory", O_RDONLY | O_CLOEXEC); + #else + fd = (int)syscall(SYS_openat, + AT_FDCWD, "/proc/sys/vm/overcommit_memory", O_RDONLY); + if (fd != -1) { + fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC); + } + #endif #else - fd = open("/proc/sys/vm/overcommit_memory", O_RDONLY | O_CLOEXEC); + #if defined(O_CLOEXEC) + fd = open("/proc/sys/vm/overcommit_memory", O_RDONLY | O_CLOEXEC); + #else + fd = open("/proc/sys/vm/overcommit_memory", O_RDONLY); + if (fd != -1) { + fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | FD_CLOEXEC); + } + #endif #endif + if (fd == -1) { return false; /* Error. */ } -#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_read) - nread = (ssize_t)syscall(SYS_read, fd, &buf, sizeof(buf)); -#else - nread = read(fd, &buf, sizeof(buf)); -#endif - + ssize_t nread = malloc_read_fd(fd, &buf, sizeof(buf)); #if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_close) syscall(SYS_close, fd); #else @@ -390,6 +526,75 @@ os_overcommits_proc(void) { } #endif +void +pages_set_thp_state (void *ptr, size_t size) { + if (opt_thp == thp_mode_default || opt_thp == init_system_thp_mode) { + return; + } + assert(opt_thp != thp_mode_not_supported && + init_system_thp_mode != thp_mode_not_supported); + + if (opt_thp == thp_mode_always + && init_system_thp_mode != thp_mode_never) { + assert(init_system_thp_mode == thp_mode_default); + pages_huge_unaligned(ptr, size); + } else if (opt_thp == thp_mode_never) { + assert(init_system_thp_mode == thp_mode_default || + init_system_thp_mode == thp_mode_always); + pages_nohuge_unaligned(ptr, size); + } +} + +static void +init_thp_state(void) { + if (!have_madvise_huge) { + if (metadata_thp_enabled() && opt_abort) { + malloc_write(": no MADV_HUGEPAGE support\n"); + abort(); + } + goto label_error; + } + + static const char sys_state_madvise[] = "always [madvise] never\n"; + static const char sys_state_always[] = "[always] madvise never\n"; + static const char sys_state_never[] = "always madvise [never]\n"; + char buf[sizeof(sys_state_madvise)]; + +#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_open) + int fd = (int)syscall(SYS_open, + "/sys/kernel/mm/transparent_hugepage/enabled", O_RDONLY); +#else + int fd = open("/sys/kernel/mm/transparent_hugepage/enabled", O_RDONLY); +#endif + if (fd == -1) { + goto label_error; + } + + ssize_t nread = malloc_read_fd(fd, &buf, sizeof(buf)); +#if defined(JEMALLOC_USE_SYSCALL) && defined(SYS_close) + syscall(SYS_close, fd); +#else + close(fd); +#endif + + if (nread < 0) { + goto label_error; + } + + if (strncmp(buf, sys_state_madvise, (size_t)nread) == 0) { + init_system_thp_mode = thp_mode_default; + } else if (strncmp(buf, sys_state_always, (size_t)nread) == 0) { + init_system_thp_mode = thp_mode_always; + } else if (strncmp(buf, sys_state_never, (size_t)nread) == 0) { + init_system_thp_mode = thp_mode_never; + } else { + goto label_error; + } + return; +label_error: + opt_thp = init_system_thp_mode = thp_mode_not_supported; +} + bool pages_boot(void) { os_page = os_page_detect(); @@ -418,5 +623,27 @@ pages_boot(void) { os_overcommits = false; #endif + init_thp_state(); + +#ifdef __FreeBSD__ + /* + * FreeBSD doesn't need the check; madvise(2) is known to work. + */ +#else + /* Detect lazy purge runtime support. */ + if (pages_can_purge_lazy) { + bool committed = false; + void *madv_free_page = os_pages_map(NULL, PAGE, PAGE, &committed); + if (madv_free_page == NULL) { + return true; + } + assert(pages_can_purge_lazy_runtime); + if (pages_purge_lazy(madv_free_page, PAGE)) { + pages_can_purge_lazy_runtime = false; + } + os_pages_unmap(madv_free_page, PAGE); + } +#endif + return false; } diff --git a/kbe/src/lib/dependencies/jemalloc/src/prof.c b/kbe/src/lib/dependencies/jemalloc/src/prof.c index 975722c4c3..4d7d65db40 100755 --- a/kbe/src/lib/dependencies/jemalloc/src/prof.c +++ b/kbe/src/lib/dependencies/jemalloc/src/prof.c @@ -7,6 +7,7 @@ #include "jemalloc/internal/hash.h" #include "jemalloc/internal/malloc_io.h" #include "jemalloc/internal/mutex.h" +#include "jemalloc/internal/emitter.h" /******************************************************************************/ @@ -23,7 +24,7 @@ */ #undef _Unwind_Backtrace #include -#define _Unwind_Backtrace JEMALLOC_HOOK(_Unwind_Backtrace, hooks_libc_hook) +#define _Unwind_Backtrace JEMALLOC_HOOK(_Unwind_Backtrace, test_hooks_libc_hook) #endif /******************************************************************************/ @@ -38,6 +39,7 @@ bool opt_prof_gdump = false; bool opt_prof_final = false; bool opt_prof_leak = false; bool opt_prof_accum = false; +bool opt_prof_log = false; char opt_prof_prefix[ /* Minimize memory bloat for non-prof builds. */ #ifdef JEMALLOC_PROF @@ -70,6 +72,100 @@ uint64_t prof_interval = 0; size_t lg_prof_sample; +typedef enum prof_logging_state_e prof_logging_state_t; +enum prof_logging_state_e { + prof_logging_state_stopped, + prof_logging_state_started, + prof_logging_state_dumping +}; + +/* + * - stopped: log_start never called, or previous log_stop has completed. + * - started: log_start called, log_stop not called yet. Allocations are logged. + * - dumping: log_stop called but not finished; samples are not logged anymore. + */ +prof_logging_state_t prof_logging_state = prof_logging_state_stopped; + +#ifdef JEMALLOC_JET +static bool prof_log_dummy = false; +#endif + +/* Incremented for every log file that is output. */ +static uint64_t log_seq = 0; +static char log_filename[ + /* Minimize memory bloat for non-prof builds. */ +#ifdef JEMALLOC_PROF + PATH_MAX + +#endif + 1]; + +/* Timestamp for most recent call to log_start(). */ +static nstime_t log_start_timestamp = NSTIME_ZERO_INITIALIZER; + +/* Increment these when adding to the log_bt and log_thr linked lists. */ +static size_t log_bt_index = 0; +static size_t log_thr_index = 0; + +/* Linked list node definitions. These are only used in prof.c. */ +typedef struct prof_bt_node_s prof_bt_node_t; + +struct prof_bt_node_s { + prof_bt_node_t *next; + size_t index; + prof_bt_t bt; + /* Variable size backtrace vector pointed to by bt. */ + void *vec[1]; +}; + +typedef struct prof_thr_node_s prof_thr_node_t; + +struct prof_thr_node_s { + prof_thr_node_t *next; + size_t index; + uint64_t thr_uid; + /* Variable size based on thr_name_sz. */ + char name[1]; +}; + +typedef struct prof_alloc_node_s prof_alloc_node_t; + +/* This is output when logging sampled allocations. */ +struct prof_alloc_node_s { + prof_alloc_node_t *next; + /* Indices into an array of thread data. */ + size_t alloc_thr_ind; + size_t free_thr_ind; + + /* Indices into an array of backtraces. */ + size_t alloc_bt_ind; + size_t free_bt_ind; + + uint64_t alloc_time_ns; + uint64_t free_time_ns; + + size_t usize; +}; + +/* + * Created on the first call to prof_log_start and deleted on prof_log_stop. + * These are the backtraces and threads that have already been logged by an + * allocation. + */ +static bool log_tables_initialized = false; +static ckh_t log_bt_node_set; +static ckh_t log_thr_node_set; + +/* Store linked lists for logged data. */ +static prof_bt_node_t *log_bt_first = NULL; +static prof_bt_node_t *log_bt_last = NULL; +static prof_thr_node_t *log_thr_first = NULL; +static prof_thr_node_t *log_thr_last = NULL; +static prof_alloc_node_t *log_alloc_first = NULL; +static prof_alloc_node_t *log_alloc_last = NULL; + +/* Protects the prof_logging_state and any log_{...} variable. */ +static malloc_mutex_t log_mtx; + /* * Table of mutexes that are shared among gctx's. These are leaf locks, so * there is no problem with using them for more than one gctx at the same time. @@ -145,6 +241,12 @@ static void prof_tdata_destroy(tsd_t *tsd, prof_tdata_t *tdata, bool even_if_attached); static char *prof_thread_name_alloc(tsdn_t *tsdn, const char *thread_name); +/* Hashtable functions for log_bt_node_set and log_thr_node_set. */ +static void prof_thr_node_hash(const void *key, size_t r_hash[2]); +static bool prof_thr_node_keycomp(const void *k1, const void *k2); +static void prof_bt_node_hash(const void *key, size_t r_hash[2]); +static bool prof_bt_node_keycomp(const void *k1, const void *k2); + /******************************************************************************/ /* Red-black trees. */ @@ -242,6 +344,12 @@ prof_malloc_sample_object(tsdn_t *tsdn, const void *ptr, size_t usize, prof_tctx_t *tctx) { prof_tctx_set(tsdn, ptr, usize, NULL, tctx); + /* Get the current time and set this in the extent_t. We'll read this + * when free() is called. */ + nstime_t t = NSTIME_ZERO_INITIALIZER; + nstime_update(&t); + prof_alloc_time_set(tsdn, ptr, NULL, t); + malloc_mutex_lock(tsdn, tctx->tdata->lock); tctx->cnts.curobjs++; tctx->cnts.curbytes += usize; @@ -253,14 +361,174 @@ prof_malloc_sample_object(tsdn_t *tsdn, const void *ptr, size_t usize, malloc_mutex_unlock(tsdn, tctx->tdata->lock); } +static size_t +prof_log_bt_index(tsd_t *tsd, prof_bt_t *bt) { + assert(prof_logging_state == prof_logging_state_started); + malloc_mutex_assert_owner(tsd_tsdn(tsd), &log_mtx); + + prof_bt_node_t dummy_node; + dummy_node.bt = *bt; + prof_bt_node_t *node; + + /* See if this backtrace is already cached in the table. */ + if (ckh_search(&log_bt_node_set, (void *)(&dummy_node), + (void **)(&node), NULL)) { + size_t sz = offsetof(prof_bt_node_t, vec) + + (bt->len * sizeof(void *)); + prof_bt_node_t *new_node = (prof_bt_node_t *) + iallocztm(tsd_tsdn(tsd), sz, sz_size2index(sz), false, NULL, + true, arena_get(TSDN_NULL, 0, true), true); + if (log_bt_first == NULL) { + log_bt_first = new_node; + log_bt_last = new_node; + } else { + log_bt_last->next = new_node; + log_bt_last = new_node; + } + + new_node->next = NULL; + new_node->index = log_bt_index; + /* + * Copy the backtrace: bt is inside a tdata or gctx, which + * might die before prof_log_stop is called. + */ + new_node->bt.len = bt->len; + memcpy(new_node->vec, bt->vec, bt->len * sizeof(void *)); + new_node->bt.vec = new_node->vec; + + log_bt_index++; + ckh_insert(tsd, &log_bt_node_set, (void *)new_node, NULL); + return new_node->index; + } else { + return node->index; + } +} +static size_t +prof_log_thr_index(tsd_t *tsd, uint64_t thr_uid, const char *name) { + assert(prof_logging_state == prof_logging_state_started); + malloc_mutex_assert_owner(tsd_tsdn(tsd), &log_mtx); + + prof_thr_node_t dummy_node; + dummy_node.thr_uid = thr_uid; + prof_thr_node_t *node; + + /* See if this thread is already cached in the table. */ + if (ckh_search(&log_thr_node_set, (void *)(&dummy_node), + (void **)(&node), NULL)) { + size_t sz = offsetof(prof_thr_node_t, name) + strlen(name) + 1; + prof_thr_node_t *new_node = (prof_thr_node_t *) + iallocztm(tsd_tsdn(tsd), sz, sz_size2index(sz), false, NULL, + true, arena_get(TSDN_NULL, 0, true), true); + if (log_thr_first == NULL) { + log_thr_first = new_node; + log_thr_last = new_node; + } else { + log_thr_last->next = new_node; + log_thr_last = new_node; + } + + new_node->next = NULL; + new_node->index = log_thr_index; + new_node->thr_uid = thr_uid; + strcpy(new_node->name, name); + + log_thr_index++; + ckh_insert(tsd, &log_thr_node_set, (void *)new_node, NULL); + return new_node->index; + } else { + return node->index; + } +} + +static void +prof_try_log(tsd_t *tsd, const void *ptr, size_t usize, prof_tctx_t *tctx) { + malloc_mutex_assert_owner(tsd_tsdn(tsd), tctx->tdata->lock); + + prof_tdata_t *cons_tdata = prof_tdata_get(tsd, false); + if (cons_tdata == NULL) { + /* + * We decide not to log these allocations. cons_tdata will be + * NULL only when the current thread is in a weird state (e.g. + * it's being destroyed). + */ + return; + } + + malloc_mutex_lock(tsd_tsdn(tsd), &log_mtx); + + if (prof_logging_state != prof_logging_state_started) { + goto label_done; + } + + if (!log_tables_initialized) { + bool err1 = ckh_new(tsd, &log_bt_node_set, PROF_CKH_MINITEMS, + prof_bt_node_hash, prof_bt_node_keycomp); + bool err2 = ckh_new(tsd, &log_thr_node_set, PROF_CKH_MINITEMS, + prof_thr_node_hash, prof_thr_node_keycomp); + if (err1 || err2) { + goto label_done; + } + log_tables_initialized = true; + } + + nstime_t alloc_time = prof_alloc_time_get(tsd_tsdn(tsd), ptr, + (alloc_ctx_t *)NULL); + nstime_t free_time = NSTIME_ZERO_INITIALIZER; + nstime_update(&free_time); + + size_t sz = sizeof(prof_alloc_node_t); + prof_alloc_node_t *new_node = (prof_alloc_node_t *) + iallocztm(tsd_tsdn(tsd), sz, sz_size2index(sz), false, NULL, true, + arena_get(TSDN_NULL, 0, true), true); + + const char *prod_thr_name = (tctx->tdata->thread_name == NULL)? + "" : tctx->tdata->thread_name; + const char *cons_thr_name = prof_thread_name_get(tsd); + + prof_bt_t bt; + /* Initialize the backtrace, using the buffer in tdata to store it. */ + bt_init(&bt, cons_tdata->vec); + prof_backtrace(&bt); + prof_bt_t *cons_bt = &bt; + + /* We haven't destroyed tctx yet, so gctx should be good to read. */ + prof_bt_t *prod_bt = &tctx->gctx->bt; + + new_node->next = NULL; + new_node->alloc_thr_ind = prof_log_thr_index(tsd, tctx->tdata->thr_uid, + prod_thr_name); + new_node->free_thr_ind = prof_log_thr_index(tsd, cons_tdata->thr_uid, + cons_thr_name); + new_node->alloc_bt_ind = prof_log_bt_index(tsd, prod_bt); + new_node->free_bt_ind = prof_log_bt_index(tsd, cons_bt); + new_node->alloc_time_ns = nstime_ns(&alloc_time); + new_node->free_time_ns = nstime_ns(&free_time); + new_node->usize = usize; + + if (log_alloc_first == NULL) { + log_alloc_first = new_node; + log_alloc_last = new_node; + } else { + log_alloc_last->next = new_node; + log_alloc_last = new_node; + } + +label_done: + malloc_mutex_unlock(tsd_tsdn(tsd), &log_mtx); +} + void -prof_free_sampled_object(tsd_t *tsd, size_t usize, prof_tctx_t *tctx) { +prof_free_sampled_object(tsd_t *tsd, const void *ptr, size_t usize, + prof_tctx_t *tctx) { malloc_mutex_lock(tsd_tsdn(tsd), tctx->tdata->lock); + assert(tctx->cnts.curobjs > 0); assert(tctx->cnts.curbytes >= usize); tctx->cnts.curobjs--; tctx->cnts.curbytes -= usize; + prof_try_log(tsd, ptr, usize, tctx); + if (prof_tctx_should_destroy(tsd_tsdn(tsd), tctx)) { prof_tctx_destroy(tsd, tctx); } else { @@ -871,15 +1139,12 @@ prof_lookup(tsd_t *tsd, prof_bt_t *bt) { void prof_sample_threshold_update(prof_tdata_t *tdata) { #ifdef JEMALLOC_PROF - uint64_t r; - double u; - if (!config_prof) { return; } if (lg_prof_sample == 0) { - tdata->bytes_until_sample = 0; + tsd_bytes_until_sample_set(tsd_fetch(), 0); return; } @@ -901,11 +1166,16 @@ prof_sample_threshold_update(prof_tdata_t *tdata) { * pp 500 * (http://luc.devroye.org/rnbookindex.html) */ - r = prng_lg_range_u64(&tdata->prng_state, 53); - u = (double)r * (1.0/9007199254740992.0L); - tdata->bytes_until_sample = (uint64_t)(log(u) / + uint64_t r = prng_lg_range_u64(&tdata->prng_state, 53); + double u = (double)r * (1.0/9007199254740992.0L); + uint64_t bytes_until_sample = (uint64_t)(log(u) / log(1.0 - (1.0 / (double)((uint64_t)1U << lg_prof_sample)))) + (uint64_t)1U; + if (bytes_until_sample > SSIZE_MAX) { + bytes_until_sample = SSIZE_MAX; + } + tsd_bytes_until_sample_set(tsd_fetch(), bytes_until_sample); + #endif } @@ -978,7 +1248,7 @@ prof_dump_flush(bool propagate_err) { cassert(config_prof); - err = write(prof_dump_fd, prof_dump_buf, prof_dump_buf_end); + err = malloc_write_fd(prof_dump_fd, prof_dump_buf, prof_dump_buf_end); if (err == -1) { if (!propagate_err) { malloc_write(": write() failed during heap " @@ -1409,7 +1679,15 @@ prof_open_maps(const char *format, ...) { va_start(ap, format); malloc_vsnprintf(filename, sizeof(filename), format, ap); va_end(ap); + +#if defined(O_CLOEXEC) mfd = open(filename, O_RDONLY | O_CLOEXEC); +#else + mfd = open(filename, O_RDONLY); + if (mfd != -1) { + fcntl(mfd, F_SETFD, fcntl(mfd, F_GETFD) | FD_CLOEXEC); + } +#endif return mfd; } @@ -1463,8 +1741,9 @@ prof_dump_maps(bool propagate_err) { goto label_return; } } - nread = read(mfd, &prof_dump_buf[prof_dump_buf_end], - PROF_DUMP_BUFSIZE - prof_dump_buf_end); + nread = malloc_read_fd(mfd, + &prof_dump_buf[prof_dump_buf_end], PROF_DUMP_BUFSIZE + - prof_dump_buf_end); } while (nread > 0); } else { ret = true; @@ -1772,7 +2051,7 @@ prof_idump(tsdn_t *tsdn) { cassert(config_prof); - if (!prof_booted || tsdn_null(tsdn)) { + if (!prof_booted || tsdn_null(tsdn) || !prof_active_get_unlocked()) { return; } tsd = tsdn_tsd(tsdn); @@ -1829,7 +2108,7 @@ prof_gdump(tsdn_t *tsdn) { cassert(config_prof); - if (!prof_booted || tsdn_null(tsdn)) { + if (!prof_booted || tsdn_null(tsdn) || !prof_active_get_unlocked()) { return; } tsd = tsdn_tsd(tsdn); @@ -1878,6 +2157,33 @@ prof_bt_keycomp(const void *k1, const void *k2) { return (memcmp(bt1->vec, bt2->vec, bt1->len * sizeof(void *)) == 0); } +static void +prof_bt_node_hash(const void *key, size_t r_hash[2]) { + const prof_bt_node_t *bt_node = (prof_bt_node_t *)key; + prof_bt_hash((void *)(&bt_node->bt), r_hash); +} + +static bool +prof_bt_node_keycomp(const void *k1, const void *k2) { + const prof_bt_node_t *bt_node1 = (prof_bt_node_t *)k1; + const prof_bt_node_t *bt_node2 = (prof_bt_node_t *)k2; + return prof_bt_keycomp((void *)(&bt_node1->bt), + (void *)(&bt_node2->bt)); +} + +static void +prof_thr_node_hash(const void *key, size_t r_hash[2]) { + const prof_thr_node_t *thr_node = (prof_thr_node_t *)key; + hash(&thr_node->thr_uid, sizeof(uint64_t), 0x94122f35U, r_hash); +} + +static bool +prof_thr_node_keycomp(const void *k1, const void *k2) { + const prof_thr_node_t *thr_node1 = (prof_thr_node_t *)k1; + const prof_thr_node_t *thr_node2 = (prof_thr_node_t *)k2; + return thr_node1->thr_uid == thr_node2->thr_uid; +} + static uint64_t prof_thr_uid_alloc(tsdn_t *tsdn) { uint64_t thr_uid; @@ -2110,6 +2416,368 @@ prof_active_set(tsdn_t *tsdn, bool active) { return prof_active_old; } +#ifdef JEMALLOC_JET +size_t +prof_log_bt_count(void) { + size_t cnt = 0; + prof_bt_node_t *node = log_bt_first; + while (node != NULL) { + cnt++; + node = node->next; + } + return cnt; +} + +size_t +prof_log_alloc_count(void) { + size_t cnt = 0; + prof_alloc_node_t *node = log_alloc_first; + while (node != NULL) { + cnt++; + node = node->next; + } + return cnt; +} + +size_t +prof_log_thr_count(void) { + size_t cnt = 0; + prof_thr_node_t *node = log_thr_first; + while (node != NULL) { + cnt++; + node = node->next; + } + return cnt; +} + +bool +prof_log_is_logging(void) { + return prof_logging_state == prof_logging_state_started; +} + +bool +prof_log_rep_check(void) { + if (prof_logging_state == prof_logging_state_stopped + && log_tables_initialized) { + return true; + } + + if (log_bt_last != NULL && log_bt_last->next != NULL) { + return true; + } + if (log_thr_last != NULL && log_thr_last->next != NULL) { + return true; + } + if (log_alloc_last != NULL && log_alloc_last->next != NULL) { + return true; + } + + size_t bt_count = prof_log_bt_count(); + size_t thr_count = prof_log_thr_count(); + size_t alloc_count = prof_log_alloc_count(); + + + if (prof_logging_state == prof_logging_state_stopped) { + if (bt_count != 0 || thr_count != 0 || alloc_count || 0) { + return true; + } + } + + prof_alloc_node_t *node = log_alloc_first; + while (node != NULL) { + if (node->alloc_bt_ind >= bt_count) { + return true; + } + if (node->free_bt_ind >= bt_count) { + return true; + } + if (node->alloc_thr_ind >= thr_count) { + return true; + } + if (node->free_thr_ind >= thr_count) { + return true; + } + if (node->alloc_time_ns > node->free_time_ns) { + return true; + } + node = node->next; + } + + return false; +} + +void +prof_log_dummy_set(bool new_value) { + prof_log_dummy = new_value; +} +#endif + +bool +prof_log_start(tsdn_t *tsdn, const char *filename) { + if (!opt_prof || !prof_booted) { + return true; + } + + bool ret = false; + size_t buf_size = PATH_MAX + 1; + + malloc_mutex_lock(tsdn, &log_mtx); + + if (prof_logging_state != prof_logging_state_stopped) { + ret = true; + } else if (filename == NULL) { + /* Make default name. */ + malloc_snprintf(log_filename, buf_size, "%s.%d.%"FMTu64".json", + opt_prof_prefix, prof_getpid(), log_seq); + log_seq++; + prof_logging_state = prof_logging_state_started; + } else if (strlen(filename) >= buf_size) { + ret = true; + } else { + strcpy(log_filename, filename); + prof_logging_state = prof_logging_state_started; + } + + if (!ret) { + nstime_update(&log_start_timestamp); + } + + malloc_mutex_unlock(tsdn, &log_mtx); + + return ret; +} + +/* Used as an atexit function to stop logging on exit. */ +static void +prof_log_stop_final(void) { + tsd_t *tsd = tsd_fetch(); + prof_log_stop(tsd_tsdn(tsd)); +} + +struct prof_emitter_cb_arg_s { + int fd; + ssize_t ret; +}; + +static void +prof_emitter_write_cb(void *opaque, const char *to_write) { + struct prof_emitter_cb_arg_s *arg = + (struct prof_emitter_cb_arg_s *)opaque; + size_t bytes = strlen(to_write); +#ifdef JEMALLOC_JET + if (prof_log_dummy) { + return; + } +#endif + arg->ret = write(arg->fd, (void *)to_write, bytes); +} + +/* + * prof_log_emit_{...} goes through the appropriate linked list, emitting each + * node to the json and deallocating it. + */ +static void +prof_log_emit_threads(tsd_t *tsd, emitter_t *emitter) { + emitter_json_array_kv_begin(emitter, "threads"); + prof_thr_node_t *thr_node = log_thr_first; + prof_thr_node_t *thr_old_node; + while (thr_node != NULL) { + emitter_json_object_begin(emitter); + + emitter_json_kv(emitter, "thr_uid", emitter_type_uint64, + &thr_node->thr_uid); + + char *thr_name = thr_node->name; + + emitter_json_kv(emitter, "thr_name", emitter_type_string, + &thr_name); + + emitter_json_object_end(emitter); + thr_old_node = thr_node; + thr_node = thr_node->next; + idalloc(tsd, thr_old_node); + } + emitter_json_array_end(emitter); +} + +static void +prof_log_emit_traces(tsd_t *tsd, emitter_t *emitter) { + emitter_json_array_kv_begin(emitter, "stack_traces"); + prof_bt_node_t *bt_node = log_bt_first; + prof_bt_node_t *bt_old_node; + /* + * Calculate how many hex digits we need: twice number of bytes, two for + * "0x", and then one more for terminating '\0'. + */ + char buf[2 * sizeof(intptr_t) + 3]; + size_t buf_sz = sizeof(buf); + while (bt_node != NULL) { + emitter_json_array_begin(emitter); + size_t i; + for (i = 0; i < bt_node->bt.len; i++) { + malloc_snprintf(buf, buf_sz, "%p", bt_node->bt.vec[i]); + char *trace_str = buf; + emitter_json_value(emitter, emitter_type_string, + &trace_str); + } + emitter_json_array_end(emitter); + + bt_old_node = bt_node; + bt_node = bt_node->next; + idalloc(tsd, bt_old_node); + } + emitter_json_array_end(emitter); +} + +static void +prof_log_emit_allocs(tsd_t *tsd, emitter_t *emitter) { + emitter_json_array_kv_begin(emitter, "allocations"); + prof_alloc_node_t *alloc_node = log_alloc_first; + prof_alloc_node_t *alloc_old_node; + while (alloc_node != NULL) { + emitter_json_object_begin(emitter); + + emitter_json_kv(emitter, "alloc_thread", emitter_type_size, + &alloc_node->alloc_thr_ind); + + emitter_json_kv(emitter, "free_thread", emitter_type_size, + &alloc_node->free_thr_ind); + + emitter_json_kv(emitter, "alloc_trace", emitter_type_size, + &alloc_node->alloc_bt_ind); + + emitter_json_kv(emitter, "free_trace", emitter_type_size, + &alloc_node->free_bt_ind); + + emitter_json_kv(emitter, "alloc_timestamp", + emitter_type_uint64, &alloc_node->alloc_time_ns); + + emitter_json_kv(emitter, "free_timestamp", emitter_type_uint64, + &alloc_node->free_time_ns); + + emitter_json_kv(emitter, "usize", emitter_type_uint64, + &alloc_node->usize); + + emitter_json_object_end(emitter); + + alloc_old_node = alloc_node; + alloc_node = alloc_node->next; + idalloc(tsd, alloc_old_node); + } + emitter_json_array_end(emitter); +} + +static void +prof_log_emit_metadata(emitter_t *emitter) { + emitter_json_object_kv_begin(emitter, "info"); + + nstime_t now = NSTIME_ZERO_INITIALIZER; + + nstime_update(&now); + uint64_t ns = nstime_ns(&now) - nstime_ns(&log_start_timestamp); + emitter_json_kv(emitter, "duration", emitter_type_uint64, &ns); + + char *vers = JEMALLOC_VERSION; + emitter_json_kv(emitter, "version", + emitter_type_string, &vers); + + emitter_json_kv(emitter, "lg_sample_rate", + emitter_type_int, &lg_prof_sample); + + int pid = prof_getpid(); + emitter_json_kv(emitter, "pid", emitter_type_int, &pid); + + emitter_json_object_end(emitter); +} + + +bool +prof_log_stop(tsdn_t *tsdn) { + if (!opt_prof || !prof_booted) { + return true; + } + + tsd_t *tsd = tsdn_tsd(tsdn); + malloc_mutex_lock(tsdn, &log_mtx); + + if (prof_logging_state != prof_logging_state_started) { + malloc_mutex_unlock(tsdn, &log_mtx); + return true; + } + + /* + * Set the state to dumping. We'll set it to stopped when we're done. + * Since other threads won't be able to start/stop/log when the state is + * dumping, we don't have to hold the lock during the whole method. + */ + prof_logging_state = prof_logging_state_dumping; + malloc_mutex_unlock(tsdn, &log_mtx); + + + emitter_t emitter; + + /* Create a file. */ + + int fd; +#ifdef JEMALLOC_JET + if (prof_log_dummy) { + fd = 0; + } else { + fd = creat(log_filename, 0644); + } +#else + fd = creat(log_filename, 0644); +#endif + + if (fd == -1) { + malloc_printf(": creat() for log file \"%s\" " + " failed with %d\n", log_filename, errno); + if (opt_abort) { + abort(); + } + return true; + } + + /* Emit to json. */ + struct prof_emitter_cb_arg_s arg; + arg.fd = fd; + emitter_init(&emitter, emitter_output_json, &prof_emitter_write_cb, + (void *)(&arg)); + + emitter_json_object_begin(&emitter); + prof_log_emit_metadata(&emitter); + prof_log_emit_threads(tsd, &emitter); + prof_log_emit_traces(tsd, &emitter); + prof_log_emit_allocs(tsd, &emitter); + emitter_json_object_end(&emitter); + + /* Reset global state. */ + if (log_tables_initialized) { + ckh_delete(tsd, &log_bt_node_set); + ckh_delete(tsd, &log_thr_node_set); + } + log_tables_initialized = false; + log_bt_index = 0; + log_thr_index = 0; + log_bt_first = NULL; + log_bt_last = NULL; + log_thr_first = NULL; + log_thr_last = NULL; + log_alloc_first = NULL; + log_alloc_last = NULL; + + malloc_mutex_lock(tsdn, &log_mtx); + prof_logging_state = prof_logging_state_stopped; + malloc_mutex_unlock(tsdn, &log_mtx); + +#ifdef JEMALLOC_JET + if (prof_log_dummy) { + return false; + } +#endif + return close(fd); +} + const char * prof_thread_name_get(tsd_t *tsd) { prof_tdata_t *tdata; @@ -2346,6 +3014,35 @@ prof_boot2(tsd_t *tsd) { } } + if (opt_prof_log) { + prof_log_start(tsd_tsdn(tsd), NULL); + } + + if (atexit(prof_log_stop_final) != 0) { + malloc_write(": Error in atexit() " + "for logging\n"); + if (opt_abort) { + abort(); + } + } + + if (malloc_mutex_init(&log_mtx, "prof_log", + WITNESS_RANK_PROF_LOG, malloc_mutex_rank_exclusive)) { + return true; + } + + if (ckh_new(tsd, &log_bt_node_set, PROF_CKH_MINITEMS, + prof_bt_node_hash, prof_bt_node_keycomp)) { + return true; + } + + if (ckh_new(tsd, &log_thr_node_set, PROF_CKH_MINITEMS, + prof_thr_node_hash, prof_thr_node_keycomp)) { + return true; + } + + log_tables_initialized = true; + gctx_locks = (malloc_mutex_t *)base_alloc(tsd_tsdn(tsd), b0get(), PROF_NCTX_LOCKS * sizeof(malloc_mutex_t), CACHELINE); @@ -2373,16 +3070,14 @@ prof_boot2(tsd_t *tsd) { return true; } } - } - #ifdef JEMALLOC_PROF_LIBGCC - /* - * Cause the backtracing machinery to allocate its internal state - * before enabling profiling. - */ - _Unwind_Backtrace(prof_unwind_init_callback, NULL); + /* + * Cause the backtracing machinery to allocate its internal + * state before enabling profiling. + */ + _Unwind_Backtrace(prof_unwind_init_callback, NULL); #endif - + } prof_booted = true; return false; diff --git a/kbe/src/lib/dependencies/jemalloc/src/rtree.c b/kbe/src/lib/dependencies/jemalloc/src/rtree.c index 53702cf723..4ae41fe2fe 100755 --- a/kbe/src/lib/dependencies/jemalloc/src/rtree.c +++ b/kbe/src/lib/dependencies/jemalloc/src/rtree.c @@ -39,7 +39,7 @@ rtree_node_dalloc_impl(tsdn_t *tsdn, rtree_t *rtree, rtree_node_elm_t *node) { /* Nodes are never deleted during normal operation. */ not_reached(); } -UNUSED rtree_node_dalloc_t *JET_MUTABLE rtree_node_dalloc = +rtree_node_dalloc_t *JET_MUTABLE rtree_node_dalloc = rtree_node_dalloc_impl; static rtree_leaf_elm_t * @@ -54,7 +54,7 @@ rtree_leaf_dalloc_impl(tsdn_t *tsdn, rtree_t *rtree, rtree_leaf_elm_t *leaf) { /* Leaves are never deleted during normal operation. */ not_reached(); } -UNUSED rtree_leaf_dalloc_t *JET_MUTABLE rtree_leaf_dalloc = +rtree_leaf_dalloc_t *JET_MUTABLE rtree_leaf_dalloc = rtree_leaf_dalloc_impl; #ifdef JEMALLOC_JET diff --git a/kbe/src/lib/dependencies/jemalloc/src/sc.c b/kbe/src/lib/dependencies/jemalloc/src/sc.c new file mode 100644 index 0000000000..89ddb6ba6a --- /dev/null +++ b/kbe/src/lib/dependencies/jemalloc/src/sc.c @@ -0,0 +1,313 @@ +#include "jemalloc/internal/jemalloc_preamble.h" + +#include "jemalloc/internal/assert.h" +#include "jemalloc/internal/bit_util.h" +#include "jemalloc/internal/bitmap.h" +#include "jemalloc/internal/pages.h" +#include "jemalloc/internal/sc.h" + +/* + * This module computes the size classes used to satisfy allocations. The logic + * here was ported more or less line-by-line from a shell script, and because of + * that is not the most idiomatic C. Eventually we should fix this, but for now + * at least the damage is compartmentalized to this file. + */ + +sc_data_t sc_data_global; + +static size_t +reg_size_compute(int lg_base, int lg_delta, int ndelta) { + return (ZU(1) << lg_base) + (ZU(ndelta) << lg_delta); +} + +/* Returns the number of pages in the slab. */ +static int +slab_size(int lg_page, int lg_base, int lg_delta, int ndelta) { + size_t page = (ZU(1) << lg_page); + size_t reg_size = reg_size_compute(lg_base, lg_delta, ndelta); + + size_t try_slab_size = page; + size_t try_nregs = try_slab_size / reg_size; + size_t perfect_slab_size = 0; + bool perfect = false; + /* + * This loop continues until we find the least common multiple of the + * page size and size class size. Size classes are all of the form + * base + ndelta * delta == (ndelta + base/ndelta) * delta, which is + * (ndelta + ngroup) * delta. The way we choose slabbing strategies + * means that delta is at most the page size and ndelta < ngroup. So + * the loop executes for at most 2 * ngroup - 1 iterations, which is + * also the bound on the number of pages in a slab chosen by default. + * With the current default settings, this is at most 7. + */ + while (!perfect) { + perfect_slab_size = try_slab_size; + size_t perfect_nregs = try_nregs; + try_slab_size += page; + try_nregs = try_slab_size / reg_size; + if (perfect_slab_size == perfect_nregs * reg_size) { + perfect = true; + } + } + return (int)(perfect_slab_size / page); +} + +static void +size_class( + /* Output. */ + sc_t *sc, + /* Configuration decisions. */ + int lg_max_lookup, int lg_page, int lg_ngroup, + /* Inputs specific to the size class. */ + int index, int lg_base, int lg_delta, int ndelta) { + sc->index = index; + sc->lg_base = lg_base; + sc->lg_delta = lg_delta; + sc->ndelta = ndelta; + sc->psz = (reg_size_compute(lg_base, lg_delta, ndelta) + % (ZU(1) << lg_page) == 0); + size_t size = (ZU(1) << lg_base) + (ZU(ndelta) << lg_delta); + if (index == 0) { + assert(!sc->psz); + } + if (size < (ZU(1) << (lg_page + lg_ngroup))) { + sc->bin = true; + sc->pgs = slab_size(lg_page, lg_base, lg_delta, ndelta); + } else { + sc->bin = false; + sc->pgs = 0; + } + if (size <= (ZU(1) << lg_max_lookup)) { + sc->lg_delta_lookup = lg_delta; + } else { + sc->lg_delta_lookup = 0; + } +} + +static void +size_classes( + /* Output. */ + sc_data_t *sc_data, + /* Determined by the system. */ + size_t lg_ptr_size, int lg_quantum, + /* Configuration decisions. */ + int lg_tiny_min, int lg_max_lookup, int lg_page, int lg_ngroup) { + int ptr_bits = (1 << lg_ptr_size) * 8; + int ngroup = (1 << lg_ngroup); + int ntiny = 0; + int nlbins = 0; + int lg_tiny_maxclass = (unsigned)-1; + int nbins = 0; + int npsizes = 0; + + int index = 0; + + int ndelta = 0; + int lg_base = lg_tiny_min; + int lg_delta = lg_base; + + /* Outputs that we update as we go. */ + size_t lookup_maxclass = 0; + size_t small_maxclass = 0; + int lg_large_minclass = 0; + size_t large_maxclass = 0; + + /* Tiny size classes. */ + while (lg_base < lg_quantum) { + sc_t *sc = &sc_data->sc[index]; + size_class(sc, lg_max_lookup, lg_page, lg_ngroup, index, + lg_base, lg_delta, ndelta); + if (sc->lg_delta_lookup != 0) { + nlbins = index + 1; + } + if (sc->psz) { + npsizes++; + } + if (sc->bin) { + nbins++; + } + ntiny++; + /* Final written value is correct. */ + lg_tiny_maxclass = lg_base; + index++; + lg_delta = lg_base; + lg_base++; + } + + /* First non-tiny (pseudo) group. */ + if (ntiny != 0) { + sc_t *sc = &sc_data->sc[index]; + /* + * See the note in sc.h; the first non-tiny size class has an + * unusual encoding. + */ + lg_base--; + ndelta = 1; + size_class(sc, lg_max_lookup, lg_page, lg_ngroup, index, + lg_base, lg_delta, ndelta); + index++; + lg_base++; + lg_delta++; + if (sc->psz) { + npsizes++; + } + if (sc->bin) { + nbins++; + } + } + while (ndelta < ngroup) { + sc_t *sc = &sc_data->sc[index]; + size_class(sc, lg_max_lookup, lg_page, lg_ngroup, index, + lg_base, lg_delta, ndelta); + index++; + ndelta++; + if (sc->psz) { + npsizes++; + } + if (sc->bin) { + nbins++; + } + } + + /* All remaining groups. */ + lg_base = lg_base + lg_ngroup; + while (lg_base < ptr_bits - 1) { + ndelta = 1; + int ndelta_limit; + if (lg_base == ptr_bits - 2) { + ndelta_limit = ngroup - 1; + } else { + ndelta_limit = ngroup; + } + while (ndelta <= ndelta_limit) { + sc_t *sc = &sc_data->sc[index]; + size_class(sc, lg_max_lookup, lg_page, lg_ngroup, index, + lg_base, lg_delta, ndelta); + if (sc->lg_delta_lookup != 0) { + nlbins = index + 1; + /* Final written value is correct. */ + lookup_maxclass = (ZU(1) << lg_base) + + (ZU(ndelta) << lg_delta); + } + if (sc->psz) { + npsizes++; + } + if (sc->bin) { + nbins++; + /* Final written value is correct. */ + small_maxclass = (ZU(1) << lg_base) + + (ZU(ndelta) << lg_delta); + if (lg_ngroup > 0) { + lg_large_minclass = lg_base + 1; + } else { + lg_large_minclass = lg_base + 2; + } + } + large_maxclass = (ZU(1) << lg_base) + + (ZU(ndelta) << lg_delta); + index++; + ndelta++; + } + lg_base++; + lg_delta++; + } + /* Additional outputs. */ + int nsizes = index; + unsigned lg_ceil_nsizes = lg_ceil(nsizes); + + /* Fill in the output data. */ + sc_data->ntiny = ntiny; + sc_data->nlbins = nlbins; + sc_data->nbins = nbins; + sc_data->nsizes = nsizes; + sc_data->lg_ceil_nsizes = lg_ceil_nsizes; + sc_data->npsizes = npsizes; + sc_data->lg_tiny_maxclass = lg_tiny_maxclass; + sc_data->lookup_maxclass = lookup_maxclass; + sc_data->small_maxclass = small_maxclass; + sc_data->lg_large_minclass = lg_large_minclass; + sc_data->large_minclass = (ZU(1) << lg_large_minclass); + sc_data->large_maxclass = large_maxclass; + + /* + * We compute these values in two ways: + * - Incrementally, as above. + * - In macros, in sc.h. + * The computation is easier when done incrementally, but putting it in + * a constant makes it available to the fast paths without having to + * touch the extra global cacheline. We assert, however, that the two + * computations are equivalent. + */ + assert(sc_data->npsizes == SC_NPSIZES); + assert(sc_data->lg_tiny_maxclass == SC_LG_TINY_MAXCLASS); + assert(sc_data->small_maxclass == SC_SMALL_MAXCLASS); + assert(sc_data->large_minclass == SC_LARGE_MINCLASS); + assert(sc_data->lg_large_minclass == SC_LG_LARGE_MINCLASS); + assert(sc_data->large_maxclass == SC_LARGE_MAXCLASS); + + /* + * In the allocation fastpath, we want to assume that we can + * unconditionally subtract the requested allocation size from + * a ssize_t, and detect passing through 0 correctly. This + * results in optimal generated code. For this to work, the + * maximum allocation size must be less than SSIZE_MAX. + */ + assert(SC_LARGE_MAXCLASS < SSIZE_MAX); +} + +void +sc_data_init(sc_data_t *sc_data) { + assert(!sc_data->initialized); + + int lg_max_lookup = 12; + + size_classes(sc_data, LG_SIZEOF_PTR, LG_QUANTUM, SC_LG_TINY_MIN, + lg_max_lookup, LG_PAGE, 2); + + sc_data->initialized = true; +} + +static void +sc_data_update_sc_slab_size(sc_t *sc, size_t reg_size, size_t pgs_guess) { + size_t min_pgs = reg_size / PAGE; + if (reg_size % PAGE != 0) { + min_pgs++; + } + /* + * BITMAP_MAXBITS is actually determined by putting the smallest + * possible size-class on one page, so this can never be 0. + */ + size_t max_pgs = BITMAP_MAXBITS * reg_size / PAGE; + + assert(min_pgs <= max_pgs); + assert(min_pgs > 0); + assert(max_pgs >= 1); + if (pgs_guess < min_pgs) { + sc->pgs = (int)min_pgs; + } else if (pgs_guess > max_pgs) { + sc->pgs = (int)max_pgs; + } else { + sc->pgs = (int)pgs_guess; + } +} + +void +sc_data_update_slab_size(sc_data_t *data, size_t begin, size_t end, int pgs) { + assert(data->initialized); + for (int i = 0; i < data->nsizes; i++) { + sc_t *sc = &data->sc[i]; + if (!sc->bin) { + break; + } + size_t reg_size = reg_size_compute(sc->lg_base, sc->lg_delta, + sc->ndelta); + if (begin <= reg_size && reg_size <= end) { + sc_data_update_sc_slab_size(sc, reg_size, pgs); + } + } +} + +void +sc_boot(sc_data_t *data) { + sc_data_init(data); +} diff --git a/kbe/src/lib/dependencies/jemalloc/src/spin.c b/kbe/src/lib/dependencies/jemalloc/src/spin.c deleted file mode 100755 index 24372c26c9..0000000000 --- a/kbe/src/lib/dependencies/jemalloc/src/spin.c +++ /dev/null @@ -1,4 +0,0 @@ -#define JEMALLOC_SPIN_C_ -#include "jemalloc/internal/jemalloc_preamble.h" - -#include "jemalloc/internal/spin.h" diff --git a/kbe/src/lib/dependencies/jemalloc/src/stats.c b/kbe/src/lib/dependencies/jemalloc/src/stats.c index 087df7676e..4c427e0d35 100755 --- a/kbe/src/lib/dependencies/jemalloc/src/stats.c +++ b/kbe/src/lib/dependencies/jemalloc/src/stats.c @@ -4,6 +4,7 @@ #include "jemalloc/internal/assert.h" #include "jemalloc/internal/ctl.h" +#include "jemalloc/internal/emitter.h" #include "jemalloc/internal/mutex.h" #include "jemalloc/internal/mutex_prof.h" @@ -51,6 +52,20 @@ char opt_stats_print_opts[stats_print_tot_num_options+1] = ""; /******************************************************************************/ +static uint64_t +rate_per_second(uint64_t value, uint64_t uptime_ns) { + uint64_t billion = 1000000000; + if (uptime_ns == 0 || value == 0) { + return 0; + } + if (uptime_ns < billion) { + return value; + } else { + uint64_t uptime_s = uptime_ns / billion; + return value / uptime_s; + } +} + /* Calculate x.yyy and output a string (takes a fixed sized char array). */ static bool get_rate_str(uint64_t dividend, uint64_t divisor, char str[6]) { @@ -84,41 +99,175 @@ gen_mutex_ctl_str(char *str, size_t buf_len, const char *prefix, } static void -read_arena_bin_mutex_stats(unsigned arena_ind, unsigned bin_ind, - uint64_t results[mutex_prof_num_counters]) { +mutex_stats_init_cols(emitter_row_t *row, const char *table_name, + emitter_col_t *name, + emitter_col_t col_uint64_t[mutex_prof_num_uint64_t_counters], + emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters]) { + mutex_prof_uint64_t_counter_ind_t k_uint64_t = 0; + mutex_prof_uint32_t_counter_ind_t k_uint32_t = 0; + + emitter_col_t *col; + + if (name != NULL) { + emitter_col_init(name, row); + name->justify = emitter_justify_left; + name->width = 21; + name->type = emitter_type_title; + name->str_val = table_name; + } + +#define WIDTH_uint32_t 12 +#define WIDTH_uint64_t 16 +#define OP(counter, counter_type, human, derived, base_counter) \ + col = &col_##counter_type[k_##counter_type]; \ + ++k_##counter_type; \ + emitter_col_init(col, row); \ + col->justify = emitter_justify_right; \ + col->width = derived ? 8 : WIDTH_##counter_type; \ + col->type = emitter_type_title; \ + col->str_val = human; + MUTEX_PROF_COUNTERS +#undef OP +#undef WIDTH_uint32_t +#undef WIDTH_uint64_t + col_uint64_t[mutex_counter_total_wait_time_ps].width = 10; +} + +static void +mutex_stats_read_global(const char *name, emitter_col_t *col_name, + emitter_col_t col_uint64_t[mutex_prof_num_uint64_t_counters], + emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters], + uint64_t uptime) { + char cmd[MUTEX_CTL_STR_MAX_LENGTH]; + + col_name->str_val = name; + + emitter_col_t *dst; +#define EMITTER_TYPE_uint32_t emitter_type_uint32 +#define EMITTER_TYPE_uint64_t emitter_type_uint64 +#define OP(counter, counter_type, human, derived, base_counter) \ + dst = &col_##counter_type[mutex_counter_##counter]; \ + dst->type = EMITTER_TYPE_##counter_type; \ + if (!derived) { \ + gen_mutex_ctl_str(cmd, MUTEX_CTL_STR_MAX_LENGTH, \ + "mutexes", name, #counter); \ + CTL_GET(cmd, (counter_type *)&dst->bool_val, counter_type); \ + } else { \ + emitter_col_t *base = &col_##counter_type[mutex_counter_##base_counter]; \ + dst->counter_type##_val = rate_per_second(base->counter_type##_val, uptime); \ + } + MUTEX_PROF_COUNTERS +#undef OP +#undef EMITTER_TYPE_uint32_t +#undef EMITTER_TYPE_uint64_t +} + +static void +mutex_stats_read_arena(unsigned arena_ind, mutex_prof_arena_ind_t mutex_ind, + const char *name, emitter_col_t *col_name, + emitter_col_t col_uint64_t[mutex_prof_num_uint64_t_counters], + emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters], + uint64_t uptime) { char cmd[MUTEX_CTL_STR_MAX_LENGTH]; -#define OP(c, t) \ - gen_mutex_ctl_str(cmd, MUTEX_CTL_STR_MAX_LENGTH, \ - "arenas.0.bins.0","mutex", #c); \ - CTL_M2_M4_GET(cmd, arena_ind, bin_ind, \ - (t *)&results[mutex_counter_##c], t); -MUTEX_PROF_COUNTERS + + col_name->str_val = name; + + emitter_col_t *dst; +#define EMITTER_TYPE_uint32_t emitter_type_uint32 +#define EMITTER_TYPE_uint64_t emitter_type_uint64 +#define OP(counter, counter_type, human, derived, base_counter) \ + dst = &col_##counter_type[mutex_counter_##counter]; \ + dst->type = EMITTER_TYPE_##counter_type; \ + if (!derived) { \ + gen_mutex_ctl_str(cmd, MUTEX_CTL_STR_MAX_LENGTH, \ + "arenas.0.mutexes", arena_mutex_names[mutex_ind], #counter);\ + CTL_M2_GET(cmd, arena_ind, (counter_type *)&dst->bool_val, counter_type); \ + } else { \ + emitter_col_t *base = &col_##counter_type[mutex_counter_##base_counter]; \ + dst->counter_type##_val = rate_per_second(base->counter_type##_val, uptime); \ + } + MUTEX_PROF_COUNTERS #undef OP +#undef EMITTER_TYPE_uint32_t +#undef EMITTER_TYPE_uint64_t } static void -mutex_stats_output_json(void (*write_cb)(void *, const char *), void *cbopaque, - const char *name, uint64_t stats[mutex_prof_num_counters], - const char *json_indent, bool last) { - malloc_cprintf(write_cb, cbopaque, "%s\"%s\": {\n", json_indent, name); - - mutex_prof_counter_ind_t k = 0; - char *fmt_str[2] = {"%s\t\"%s\": %"FMTu32"%s\n", - "%s\t\"%s\": %"FMTu64"%s\n"}; -#define OP(c, t) \ - malloc_cprintf(write_cb, cbopaque, \ - fmt_str[sizeof(t) / sizeof(uint32_t) - 1], \ - json_indent, #c, (t)stats[mutex_counter_##c], \ - (++k == mutex_prof_num_counters) ? "" : ","); -MUTEX_PROF_COUNTERS +mutex_stats_read_arena_bin(unsigned arena_ind, unsigned bin_ind, + emitter_col_t col_uint64_t[mutex_prof_num_uint64_t_counters], + emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters], + uint64_t uptime) { + char cmd[MUTEX_CTL_STR_MAX_LENGTH]; + emitter_col_t *dst; + +#define EMITTER_TYPE_uint32_t emitter_type_uint32 +#define EMITTER_TYPE_uint64_t emitter_type_uint64 +#define OP(counter, counter_type, human, derived, base_counter) \ + dst = &col_##counter_type[mutex_counter_##counter]; \ + dst->type = EMITTER_TYPE_##counter_type; \ + if (!derived) { \ + gen_mutex_ctl_str(cmd, MUTEX_CTL_STR_MAX_LENGTH, \ + "arenas.0.bins.0","mutex", #counter); \ + CTL_M2_M4_GET(cmd, arena_ind, bin_ind, \ + (counter_type *)&dst->bool_val, counter_type); \ + } else { \ + emitter_col_t *base = &col_##counter_type[mutex_counter_##base_counter]; \ + dst->counter_type##_val = rate_per_second(base->counter_type##_val, uptime); \ + } + MUTEX_PROF_COUNTERS +#undef OP +#undef EMITTER_TYPE_uint32_t +#undef EMITTER_TYPE_uint64_t +} + +/* "row" can be NULL to avoid emitting in table mode. */ +static void +mutex_stats_emit(emitter_t *emitter, emitter_row_t *row, + emitter_col_t col_uint64_t[mutex_prof_num_uint64_t_counters], + emitter_col_t col_uint32_t[mutex_prof_num_uint32_t_counters]) { + if (row != NULL) { + emitter_table_row(emitter, row); + } + + mutex_prof_uint64_t_counter_ind_t k_uint64_t = 0; + mutex_prof_uint32_t_counter_ind_t k_uint32_t = 0; + + emitter_col_t *col; + +#define EMITTER_TYPE_uint32_t emitter_type_uint32 +#define EMITTER_TYPE_uint64_t emitter_type_uint64 +#define OP(counter, type, human, derived, base_counter) \ + if (!derived) { \ + col = &col_##type[k_##type]; \ + ++k_##type; \ + emitter_json_kv(emitter, #counter, EMITTER_TYPE_##type, \ + (const void *)&col->bool_val); \ + } + MUTEX_PROF_COUNTERS; #undef OP - malloc_cprintf(write_cb, cbopaque, "%s}%s\n", json_indent, - last ? "" : ","); +#undef EMITTER_TYPE_uint32_t +#undef EMITTER_TYPE_uint64_t } +#define COL(row_name, column_name, left_or_right, col_width, etype) \ + emitter_col_t col_##column_name; \ + emitter_col_init(&col_##column_name, &row_name); \ + col_##column_name.justify = emitter_justify_##left_or_right; \ + col_##column_name.width = col_width; \ + col_##column_name.type = emitter_type_##etype; + +#define COL_HDR(row_name, column_name, human, left_or_right, col_width, etype) \ + COL(row_name, column_name, left_or_right, col_width, etype) \ + emitter_col_t header_##column_name; \ + emitter_col_init(&header_##column_name, &header_##row_name); \ + header_##column_name.justify = emitter_justify_##left_or_right; \ + header_##column_name.width = col_width; \ + header_##column_name.type = emitter_type_title; \ + header_##column_name.str_val = human ? human : #column_name; + + static void -stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, - bool json, bool large, bool mutex, unsigned i) { +stats_arena_bins_print(emitter_t *emitter, bool mutex, unsigned i, uint64_t uptime) { size_t page; bool in_gap, in_gap_prev; unsigned nbins, j; @@ -126,23 +275,69 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, CTL_GET("arenas.page", &page, size_t); CTL_GET("arenas.nbins", &nbins, unsigned); - if (json) { - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\"bins\": [\n"); - } else { - char *mutex_counters = " n_lock_ops n_waiting" - " n_spin_acq total_wait_ns max_wait_ns\n"; - malloc_cprintf(write_cb, cbopaque, - "bins: size ind allocated nmalloc" - " ndalloc nrequests curregs curslabs regs" - " pgs util nfills nflushes newslabs" - " reslabs%s", mutex ? mutex_counters : "\n"); + + emitter_row_t header_row; + emitter_row_init(&header_row); + + emitter_row_t row; + emitter_row_init(&row); + + COL_HDR(row, size, NULL, right, 20, size) + COL_HDR(row, ind, NULL, right, 4, unsigned) + COL_HDR(row, allocated, NULL, right, 13, uint64) + COL_HDR(row, nmalloc, NULL, right, 13, uint64) + COL_HDR(row, nmalloc_ps, "(#/sec)", right, 8, uint64) + COL_HDR(row, ndalloc, NULL, right, 13, uint64) + COL_HDR(row, ndalloc_ps, "(#/sec)", right, 8, uint64) + COL_HDR(row, nrequests, NULL, right, 13, uint64) + COL_HDR(row, nrequests_ps, "(#/sec)", right, 10, uint64) + COL_HDR(row, nshards, NULL, right, 9, unsigned) + COL_HDR(row, curregs, NULL, right, 13, size) + COL_HDR(row, curslabs, NULL, right, 13, size) + COL_HDR(row, regs, NULL, right, 5, unsigned) + COL_HDR(row, pgs, NULL, right, 4, size) + /* To buffer a right- and left-justified column. */ + COL_HDR(row, justify_spacer, NULL, right, 1, title) + COL_HDR(row, util, NULL, right, 6, title) + COL_HDR(row, nfills, NULL, right, 13, uint64) + COL_HDR(row, nfills_ps, "(#/sec)", right, 8, uint64) + COL_HDR(row, nflushes, NULL, right, 13, uint64) + COL_HDR(row, nflushes_ps, "(#/sec)", right, 8, uint64) + COL_HDR(row, nslabs, NULL, right, 13, uint64) + COL_HDR(row, nreslabs, NULL, right, 13, uint64) + COL_HDR(row, nreslabs_ps, "(#/sec)", right, 8, uint64) + + /* Don't want to actually print the name. */ + header_justify_spacer.str_val = " "; + col_justify_spacer.str_val = " "; + + emitter_col_t col_mutex64[mutex_prof_num_uint64_t_counters]; + emitter_col_t col_mutex32[mutex_prof_num_uint32_t_counters]; + + emitter_col_t header_mutex64[mutex_prof_num_uint64_t_counters]; + emitter_col_t header_mutex32[mutex_prof_num_uint32_t_counters]; + + if (mutex) { + mutex_stats_init_cols(&row, NULL, NULL, col_mutex64, + col_mutex32); + mutex_stats_init_cols(&header_row, NULL, NULL, header_mutex64, + header_mutex32); } + + /* + * We print a "bins:" header as part of the table row; we need to adjust + * the header size column to compensate. + */ + header_size.width -=5; + emitter_table_printf(emitter, "bins:"); + emitter_table_row(emitter, &header_row); + emitter_json_array_kv_begin(emitter, "bins"); + for (j = 0, in_gap = false; j < nbins; j++) { uint64_t nslabs; size_t reg_size, slab_size, curregs; size_t curslabs; - uint32_t nregs; + uint32_t nregs, nshards; uint64_t nmalloc, ndalloc, nrequests, nfills, nflushes; uint64_t nreslabs; @@ -151,14 +346,15 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, in_gap_prev = in_gap; in_gap = (nslabs == 0); - if (!json && in_gap_prev && !in_gap) { - malloc_cprintf(write_cb, cbopaque, + if (in_gap_prev && !in_gap) { + emitter_table_printf(emitter, " ---\n"); } CTL_M2_GET("arenas.bin.0.size", j, ®_size, size_t); CTL_M2_GET("arenas.bin.0.nregs", j, &nregs, uint32_t); CTL_M2_GET("arenas.bin.0.slab_size", j, &slab_size, size_t); + CTL_M2_GET("arenas.bin.0.nshards", j, &nshards, uint32_t); CTL_M2_M4_GET("stats.arenas.0.bins.0.nmalloc", i, j, &nmalloc, uint64_t); @@ -177,105 +373,122 @@ stats_arena_bins_print(void (*write_cb)(void *, const char *), void *cbopaque, CTL_M2_M4_GET("stats.arenas.0.bins.0.curslabs", i, j, &curslabs, size_t); - if (json) { - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\t{\n" - "\t\t\t\t\t\t\"nmalloc\": %"FMTu64",\n" - "\t\t\t\t\t\t\"ndalloc\": %"FMTu64",\n" - "\t\t\t\t\t\t\"curregs\": %zu,\n" - "\t\t\t\t\t\t\"nrequests\": %"FMTu64",\n" - "\t\t\t\t\t\t\"nfills\": %"FMTu64",\n" - "\t\t\t\t\t\t\"nflushes\": %"FMTu64",\n" - "\t\t\t\t\t\t\"nreslabs\": %"FMTu64",\n" - "\t\t\t\t\t\t\"curslabs\": %zu%s\n", - nmalloc, ndalloc, curregs, nrequests, nfills, - nflushes, nreslabs, curslabs, mutex ? "," : ""); - if (mutex) { - uint64_t mutex_stats[mutex_prof_num_counters]; - read_arena_bin_mutex_stats(i, j, mutex_stats); - mutex_stats_output_json(write_cb, cbopaque, - "mutex", mutex_stats, "\t\t\t\t\t\t", true); - } - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\t}%s\n", - (j + 1 < nbins) ? "," : ""); - } else if (!in_gap) { - size_t availregs = nregs * curslabs; - char util[6]; - if (get_rate_str((uint64_t)curregs, (uint64_t)availregs, - util)) { - if (availregs == 0) { - malloc_snprintf(util, sizeof(util), - "1"); - } else if (curregs > availregs) { - /* - * Race detected: the counters were read - * in separate mallctl calls and - * concurrent operations happened in - * between. In this case no meaningful - * utilization can be computed. - */ - malloc_snprintf(util, sizeof(util), - " race"); - } else { - not_reached(); - } - } - uint64_t mutex_stats[mutex_prof_num_counters]; - if (mutex) { - read_arena_bin_mutex_stats(i, j, mutex_stats); - } + if (mutex) { + mutex_stats_read_arena_bin(i, j, col_mutex64, + col_mutex32, uptime); + } - malloc_cprintf(write_cb, cbopaque, "%20zu %3u %12zu %12" - FMTu64" %12"FMTu64" %12"FMTu64" %12zu %12zu %4u" - " %3zu %-5s %12"FMTu64" %12"FMTu64" %12"FMTu64 - " %12"FMTu64, reg_size, j, curregs * reg_size, - nmalloc, ndalloc, nrequests, curregs, curslabs, - nregs, slab_size / page, util, nfills, nflushes, - nslabs, nreslabs); - - /* Output less info for bin mutexes to save space. */ - if (mutex) { - malloc_cprintf(write_cb, cbopaque, - " %12"FMTu64" %12"FMTu64" %12"FMTu64 - " %14"FMTu64" %12"FMTu64"\n", - mutex_stats[mutex_counter_num_ops], - mutex_stats[mutex_counter_num_wait], - mutex_stats[mutex_counter_num_spin_acq], - mutex_stats[mutex_counter_total_wait_time], - mutex_stats[mutex_counter_max_wait_time]); + emitter_json_object_begin(emitter); + emitter_json_kv(emitter, "nmalloc", emitter_type_uint64, + &nmalloc); + emitter_json_kv(emitter, "ndalloc", emitter_type_uint64, + &ndalloc); + emitter_json_kv(emitter, "curregs", emitter_type_size, + &curregs); + emitter_json_kv(emitter, "nrequests", emitter_type_uint64, + &nrequests); + emitter_json_kv(emitter, "nfills", emitter_type_uint64, + &nfills); + emitter_json_kv(emitter, "nflushes", emitter_type_uint64, + &nflushes); + emitter_json_kv(emitter, "nreslabs", emitter_type_uint64, + &nreslabs); + emitter_json_kv(emitter, "curslabs", emitter_type_size, + &curslabs); + if (mutex) { + emitter_json_object_kv_begin(emitter, "mutex"); + mutex_stats_emit(emitter, NULL, col_mutex64, + col_mutex32); + emitter_json_object_end(emitter); + } + emitter_json_object_end(emitter); + + size_t availregs = nregs * curslabs; + char util[6]; + if (get_rate_str((uint64_t)curregs, (uint64_t)availregs, util)) + { + if (availregs == 0) { + malloc_snprintf(util, sizeof(util), "1"); + } else if (curregs > availregs) { + /* + * Race detected: the counters were read in + * separate mallctl calls and concurrent + * operations happened in between. In this case + * no meaningful utilization can be computed. + */ + malloc_snprintf(util, sizeof(util), " race"); } else { - malloc_cprintf(write_cb, cbopaque, "\n"); + not_reached(); } } + + col_size.size_val = reg_size; + col_ind.unsigned_val = j; + col_allocated.size_val = curregs * reg_size; + col_nmalloc.uint64_val = nmalloc; + col_nmalloc_ps.uint64_val = rate_per_second(nmalloc, uptime); + col_ndalloc.uint64_val = ndalloc; + col_ndalloc_ps.uint64_val = rate_per_second(ndalloc, uptime); + col_nrequests.uint64_val = nrequests; + col_nrequests_ps.uint64_val = rate_per_second(nrequests, uptime); + col_nshards.unsigned_val = nshards; + col_curregs.size_val = curregs; + col_curslabs.size_val = curslabs; + col_regs.unsigned_val = nregs; + col_pgs.size_val = slab_size / page; + col_util.str_val = util; + col_nfills.uint64_val = nfills; + col_nfills_ps.uint64_val = rate_per_second(nfills, uptime); + col_nflushes.uint64_val = nflushes; + col_nflushes_ps.uint64_val = rate_per_second(nflushes, uptime); + col_nslabs.uint64_val = nslabs; + col_nreslabs.uint64_val = nreslabs; + col_nreslabs_ps.uint64_val = rate_per_second(nreslabs, uptime); + + /* + * Note that mutex columns were initialized above, if mutex == + * true. + */ + + emitter_table_row(emitter, &row); } - if (json) { - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t]%s\n", large ? "," : ""); - } else { - if (in_gap) { - malloc_cprintf(write_cb, cbopaque, - " ---\n"); - } + emitter_json_array_end(emitter); /* Close "bins". */ + + if (in_gap) { + emitter_table_printf(emitter, " ---\n"); } } static void -stats_arena_lextents_print(void (*write_cb)(void *, const char *), - void *cbopaque, bool json, unsigned i) { +stats_arena_lextents_print(emitter_t *emitter, unsigned i, uint64_t uptime) { unsigned nbins, nlextents, j; bool in_gap, in_gap_prev; CTL_GET("arenas.nbins", &nbins, unsigned); CTL_GET("arenas.nlextents", &nlextents, unsigned); - if (json) { - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\"lextents\": [\n"); - } else { - malloc_cprintf(write_cb, cbopaque, - "large: size ind allocated nmalloc" - " ndalloc nrequests curlextents\n"); - } + + emitter_row_t header_row; + emitter_row_init(&header_row); + emitter_row_t row; + emitter_row_init(&row); + + COL_HDR(row, size, NULL, right, 20, size) + COL_HDR(row, ind, NULL, right, 4, unsigned) + COL_HDR(row, allocated, NULL, right, 13, size) + COL_HDR(row, nmalloc, NULL, right, 13, uint64) + COL_HDR(row, nmalloc_ps, "(#/sec)", right, 8, uint64) + COL_HDR(row, ndalloc, NULL, right, 13, uint64) + COL_HDR(row, ndalloc_ps, "(#/sec)", right, 8, uint64) + COL_HDR(row, nrequests, NULL, right, 13, uint64) + COL_HDR(row, nrequests_ps, "(#/sec)", right, 8, uint64) + COL_HDR(row, curlextents, NULL, right, 13, size) + + /* As with bins, we label the large extents table. */ + header_size.width -= 6; + emitter_table_printf(emitter, "large:"); + emitter_table_row(emitter, &header_row); + emitter_json_array_kv_begin(emitter, "lextents"); + for (j = 0, in_gap = false; j < nlextents; j++) { uint64_t nmalloc, ndalloc, nrequests; size_t lextent_size, curlextents; @@ -289,119 +502,162 @@ stats_arena_lextents_print(void (*write_cb)(void *, const char *), in_gap_prev = in_gap; in_gap = (nrequests == 0); - if (!json && in_gap_prev && !in_gap) { - malloc_cprintf(write_cb, cbopaque, + if (in_gap_prev && !in_gap) { + emitter_table_printf(emitter, " ---\n"); } CTL_M2_GET("arenas.lextent.0.size", j, &lextent_size, size_t); CTL_M2_M4_GET("stats.arenas.0.lextents.0.curlextents", i, j, &curlextents, size_t); - if (json) { - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\t{\n" - "\t\t\t\t\t\t\"curlextents\": %zu\n" - "\t\t\t\t\t}%s\n", - curlextents, - (j + 1 < nlextents) ? "," : ""); - } else if (!in_gap) { - malloc_cprintf(write_cb, cbopaque, - "%20zu %3u %12zu %12"FMTu64" %12"FMTu64 - " %12"FMTu64" %12zu\n", - lextent_size, nbins + j, - curlextents * lextent_size, nmalloc, ndalloc, - nrequests, curlextents); + + emitter_json_object_begin(emitter); + emitter_json_kv(emitter, "curlextents", emitter_type_size, + &curlextents); + emitter_json_object_end(emitter); + + col_size.size_val = lextent_size; + col_ind.unsigned_val = nbins + j; + col_allocated.size_val = curlextents * lextent_size; + col_nmalloc.uint64_val = nmalloc; + col_nmalloc_ps.uint64_val = rate_per_second(nmalloc, uptime); + col_ndalloc.uint64_val = ndalloc; + col_ndalloc_ps.uint64_val = rate_per_second(ndalloc, uptime); + col_nrequests.uint64_val = nrequests; + col_nrequests_ps.uint64_val = rate_per_second(nrequests, uptime); + col_curlextents.size_val = curlextents; + + if (!in_gap) { + emitter_table_row(emitter, &row); } } - if (json) { - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t]\n"); - } else { - if (in_gap) { - malloc_cprintf(write_cb, cbopaque, - " ---\n"); - } + emitter_json_array_end(emitter); /* Close "lextents". */ + if (in_gap) { + emitter_table_printf(emitter, " ---\n"); } } static void -read_arena_mutex_stats(unsigned arena_ind, - uint64_t results[mutex_prof_num_arena_mutexes][mutex_prof_num_counters]) { - char cmd[MUTEX_CTL_STR_MAX_LENGTH]; - - mutex_prof_arena_ind_t i; - for (i = 0; i < mutex_prof_num_arena_mutexes; i++) { -#define OP(c, t) \ - gen_mutex_ctl_str(cmd, MUTEX_CTL_STR_MAX_LENGTH, \ - "arenas.0.mutexes", arena_mutex_names[i], #c); \ - CTL_M2_GET(cmd, arena_ind, \ - (t *)&results[i][mutex_counter_##c], t); -MUTEX_PROF_COUNTERS -#undef OP - } -} +stats_arena_extents_print(emitter_t *emitter, unsigned i) { + unsigned j; + bool in_gap, in_gap_prev; + emitter_row_t header_row; + emitter_row_init(&header_row); + emitter_row_t row; + emitter_row_init(&row); + + COL_HDR(row, size, NULL, right, 20, size) + COL_HDR(row, ind, NULL, right, 4, unsigned) + COL_HDR(row, ndirty, NULL, right, 13, size) + COL_HDR(row, dirty, NULL, right, 13, size) + COL_HDR(row, nmuzzy, NULL, right, 13, size) + COL_HDR(row, muzzy, NULL, right, 13, size) + COL_HDR(row, nretained, NULL, right, 13, size) + COL_HDR(row, retained, NULL, right, 13, size) + COL_HDR(row, ntotal, NULL, right, 13, size) + COL_HDR(row, total, NULL, right, 13, size) + + /* Label this section. */ + header_size.width -= 8; + emitter_table_printf(emitter, "extents:"); + emitter_table_row(emitter, &header_row); + emitter_json_array_kv_begin(emitter, "extents"); + + in_gap = false; + for (j = 0; j < SC_NPSIZES; j++) { + size_t ndirty, nmuzzy, nretained, total, dirty_bytes, + muzzy_bytes, retained_bytes, total_bytes; + CTL_M2_M4_GET("stats.arenas.0.extents.0.ndirty", i, j, + &ndirty, size_t); + CTL_M2_M4_GET("stats.arenas.0.extents.0.nmuzzy", i, j, + &nmuzzy, size_t); + CTL_M2_M4_GET("stats.arenas.0.extents.0.nretained", i, j, + &nretained, size_t); + CTL_M2_M4_GET("stats.arenas.0.extents.0.dirty_bytes", i, j, + &dirty_bytes, size_t); + CTL_M2_M4_GET("stats.arenas.0.extents.0.muzzy_bytes", i, j, + &muzzy_bytes, size_t); + CTL_M2_M4_GET("stats.arenas.0.extents.0.retained_bytes", i, j, + &retained_bytes, size_t); + total = ndirty + nmuzzy + nretained; + total_bytes = dirty_bytes + muzzy_bytes + retained_bytes; -static void -mutex_stats_output(void (*write_cb)(void *, const char *), void *cbopaque, - const char *name, uint64_t stats[mutex_prof_num_counters], - bool first_mutex) { - if (first_mutex) { - /* Print title. */ - malloc_cprintf(write_cb, cbopaque, - " n_lock_ops n_waiting" - " n_spin_acq n_owner_switch total_wait_ns" - " max_wait_ns max_n_thds\n"); - } + in_gap_prev = in_gap; + in_gap = (total == 0); - malloc_cprintf(write_cb, cbopaque, "%s", name); - malloc_cprintf(write_cb, cbopaque, ":%*c", - (int)(20 - strlen(name)), ' '); + if (in_gap_prev && !in_gap) { + emitter_table_printf(emitter, + " ---\n"); + } - char *fmt_str[2] = {"%12"FMTu32, "%16"FMTu64}; -#define OP(c, t) \ - malloc_cprintf(write_cb, cbopaque, \ - fmt_str[sizeof(t) / sizeof(uint32_t) - 1], \ - (t)stats[mutex_counter_##c]); -MUTEX_PROF_COUNTERS -#undef OP - malloc_cprintf(write_cb, cbopaque, "\n"); + emitter_json_object_begin(emitter); + emitter_json_kv(emitter, "ndirty", emitter_type_size, &ndirty); + emitter_json_kv(emitter, "nmuzzy", emitter_type_size, &nmuzzy); + emitter_json_kv(emitter, "nretained", emitter_type_size, + &nretained); + + emitter_json_kv(emitter, "dirty_bytes", emitter_type_size, + &dirty_bytes); + emitter_json_kv(emitter, "muzzy_bytes", emitter_type_size, + &muzzy_bytes); + emitter_json_kv(emitter, "retained_bytes", emitter_type_size, + &retained_bytes); + emitter_json_object_end(emitter); + + col_size.size_val = sz_pind2sz(j); + col_ind.size_val = j; + col_ndirty.size_val = ndirty; + col_dirty.size_val = dirty_bytes; + col_nmuzzy.size_val = nmuzzy; + col_muzzy.size_val = muzzy_bytes; + col_nretained.size_val = nretained; + col_retained.size_val = retained_bytes; + col_ntotal.size_val = total; + col_total.size_val = total_bytes; + + if (!in_gap) { + emitter_table_row(emitter, &row); + } + } + emitter_json_array_end(emitter); /* Close "extents". */ + if (in_gap) { + emitter_table_printf(emitter, " ---\n"); + } } static void -stats_arena_mutexes_print(void (*write_cb)(void *, const char *), - void *cbopaque, bool json, bool json_end, unsigned arena_ind) { - uint64_t mutex_stats[mutex_prof_num_arena_mutexes][mutex_prof_num_counters]; - read_arena_mutex_stats(arena_ind, mutex_stats); - - /* Output mutex stats. */ - if (json) { - malloc_cprintf(write_cb, cbopaque, "\t\t\t\t\"mutexes\": {\n"); - mutex_prof_arena_ind_t i, last_mutex; - last_mutex = mutex_prof_num_arena_mutexes - 1; - for (i = 0; i < mutex_prof_num_arena_mutexes; i++) { - mutex_stats_output_json(write_cb, cbopaque, - arena_mutex_names[i], mutex_stats[i], - "\t\t\t\t\t", (i == last_mutex)); - } - malloc_cprintf(write_cb, cbopaque, "\t\t\t\t}%s\n", - json_end ? "" : ","); - } else { - mutex_prof_arena_ind_t i; - for (i = 0; i < mutex_prof_num_arena_mutexes; i++) { - mutex_stats_output(write_cb, cbopaque, - arena_mutex_names[i], mutex_stats[i], i == 0); - } +stats_arena_mutexes_print(emitter_t *emitter, unsigned arena_ind, uint64_t uptime) { + emitter_row_t row; + emitter_col_t col_name; + emitter_col_t col64[mutex_prof_num_uint64_t_counters]; + emitter_col_t col32[mutex_prof_num_uint32_t_counters]; + + emitter_row_init(&row); + mutex_stats_init_cols(&row, "", &col_name, col64, col32); + + emitter_json_object_kv_begin(emitter, "mutexes"); + emitter_table_row(emitter, &row); + + for (mutex_prof_arena_ind_t i = 0; i < mutex_prof_num_arena_mutexes; + i++) { + const char *name = arena_mutex_names[i]; + emitter_json_object_kv_begin(emitter, name); + mutex_stats_read_arena(arena_ind, i, name, &col_name, col64, + col32, uptime); + mutex_stats_emit(emitter, &row, col64, col32); + emitter_json_object_end(emitter); /* Close the mutex dict. */ } + emitter_json_object_end(emitter); /* End "mutexes". */ } static void -stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque, - bool json, unsigned i, bool bins, bool large, bool mutex) { +stats_arena_print(emitter_t *emitter, unsigned i, bool bins, bool large, + bool mutex, bool extents) { unsigned nthreads; const char *dss; ssize_t dirty_decay_ms, muzzy_decay_ms; size_t page, pactive, pdirty, pmuzzy, mapped, retained; - size_t base, internal, resident; + size_t base, internal, resident, metadata_thp, extent_avail; uint64_t dirty_npurge, dirty_nmadvise, dirty_purged; uint64_t muzzy_npurge, muzzy_nmadvise, muzzy_purged; size_t small_allocated; @@ -414,31 +670,16 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque, CTL_GET("arenas.page", &page, size_t); CTL_M2_GET("stats.arenas.0.nthreads", i, &nthreads, unsigned); - if (json) { - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\"nthreads\": %u,\n", nthreads); - } else { - malloc_cprintf(write_cb, cbopaque, - "assigned threads: %u\n", nthreads); - } + emitter_kv(emitter, "nthreads", "assigned threads", + emitter_type_unsigned, &nthreads); CTL_M2_GET("stats.arenas.0.uptime", i, &uptime, uint64_t); - if (json) { - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\"uptime_ns\": %"FMTu64",\n", uptime); - } else { - malloc_cprintf(write_cb, cbopaque, - "uptime: %"FMTu64"\n", uptime); - } + emitter_kv(emitter, "uptime_ns", "uptime", emitter_type_uint64, + &uptime); CTL_M2_GET("stats.arenas.0.dss", i, &dss, const char *); - if (json) { - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\"dss\": \"%s\",\n", dss); - } else { - malloc_cprintf(write_cb, cbopaque, - "dss allocation precedence: %s\n", dss); - } + emitter_kv(emitter, "dss", "dss allocation precedence", + emitter_type_string, &dss); CTL_M2_GET("stats.arenas.0.dirty_decay_ms", i, &dirty_decay_ms, ssize_t); @@ -455,205 +696,259 @@ stats_arena_print(void (*write_cb)(void *, const char *), void *cbopaque, CTL_M2_GET("stats.arenas.0.muzzy_nmadvise", i, &muzzy_nmadvise, uint64_t); CTL_M2_GET("stats.arenas.0.muzzy_purged", i, &muzzy_purged, uint64_t); - if (json) { - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\"dirty_decay_ms\": %zd,\n", dirty_decay_ms); - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\"muzzy_decay_ms\": %zd,\n", muzzy_decay_ms); - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\"pactive\": %zu,\n", pactive); - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\"pdirty\": %zu,\n", pdirty); - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\"pmuzzy\": %zu,\n", pmuzzy); - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\"dirty_npurge\": %"FMTu64",\n", dirty_npurge); - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\"dirty_nmadvise\": %"FMTu64",\n", dirty_nmadvise); - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\"dirty_purged\": %"FMTu64",\n", dirty_purged); - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\"muzzy_npurge\": %"FMTu64",\n", muzzy_npurge); - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\"muzzy_nmadvise\": %"FMTu64",\n", muzzy_nmadvise); - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\"muzzy_purged\": %"FMTu64",\n", muzzy_purged); - } else { - malloc_cprintf(write_cb, cbopaque, - "decaying: time npages sweeps madvises" - " purged\n"); - if (dirty_decay_ms >= 0) { - malloc_cprintf(write_cb, cbopaque, - " dirty: %5zd %12zu %12"FMTu64" %12"FMTu64" %12" - FMTu64"\n", dirty_decay_ms, pdirty, dirty_npurge, - dirty_nmadvise, dirty_purged); - } else { - malloc_cprintf(write_cb, cbopaque, - " dirty: N/A %12zu %12"FMTu64" %12"FMTu64" %12" - FMTu64"\n", pdirty, dirty_npurge, dirty_nmadvise, - dirty_purged); - } - if (muzzy_decay_ms >= 0) { - malloc_cprintf(write_cb, cbopaque, - " muzzy: %5zd %12zu %12"FMTu64" %12"FMTu64" %12" - FMTu64"\n", muzzy_decay_ms, pmuzzy, muzzy_npurge, - muzzy_nmadvise, muzzy_purged); - } else { - malloc_cprintf(write_cb, cbopaque, - " muzzy: N/A %12zu %12"FMTu64" %12"FMTu64" %12" - FMTu64"\n", pmuzzy, muzzy_npurge, muzzy_nmadvise, - muzzy_purged); - } - } - CTL_M2_GET("stats.arenas.0.small.allocated", i, &small_allocated, - size_t); - CTL_M2_GET("stats.arenas.0.small.nmalloc", i, &small_nmalloc, uint64_t); - CTL_M2_GET("stats.arenas.0.small.ndalloc", i, &small_ndalloc, uint64_t); - CTL_M2_GET("stats.arenas.0.small.nrequests", i, &small_nrequests, - uint64_t); - if (json) { - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\"small\": {\n"); - - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\t\"allocated\": %zu,\n", small_allocated); - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\t\"nmalloc\": %"FMTu64",\n", small_nmalloc); - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\t\"ndalloc\": %"FMTu64",\n", small_ndalloc); - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\t\"nrequests\": %"FMTu64"\n", small_nrequests); - - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t},\n"); - } else { - malloc_cprintf(write_cb, cbopaque, - " allocated nmalloc" - " ndalloc nrequests\n"); - malloc_cprintf(write_cb, cbopaque, - "small: %12zu %12"FMTu64" %12"FMTu64 - " %12"FMTu64"\n", - small_allocated, small_nmalloc, small_ndalloc, - small_nrequests); - } + emitter_row_t decay_row; + emitter_row_init(&decay_row); - CTL_M2_GET("stats.arenas.0.large.allocated", i, &large_allocated, - size_t); - CTL_M2_GET("stats.arenas.0.large.nmalloc", i, &large_nmalloc, uint64_t); - CTL_M2_GET("stats.arenas.0.large.ndalloc", i, &large_ndalloc, uint64_t); - CTL_M2_GET("stats.arenas.0.large.nrequests", i, &large_nrequests, - uint64_t); - if (json) { - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\"large\": {\n"); - - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\t\"allocated\": %zu,\n", large_allocated); - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\t\"nmalloc\": %"FMTu64",\n", large_nmalloc); - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\t\"ndalloc\": %"FMTu64",\n", large_ndalloc); - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\t\"nrequests\": %"FMTu64"\n", large_nrequests); - - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t},\n"); - } else { - malloc_cprintf(write_cb, cbopaque, - "large: %12zu %12"FMTu64" %12"FMTu64 - " %12"FMTu64"\n", - large_allocated, large_nmalloc, large_ndalloc, - large_nrequests); - malloc_cprintf(write_cb, cbopaque, - "total: %12zu %12"FMTu64" %12"FMTu64 - " %12"FMTu64"\n", - small_allocated + large_allocated, small_nmalloc + - large_nmalloc, small_ndalloc + large_ndalloc, - small_nrequests + large_nrequests); - } - if (!json) { - malloc_cprintf(write_cb, cbopaque, - "active: %12zu\n", pactive * page); - } + /* JSON-style emission. */ + emitter_json_kv(emitter, "dirty_decay_ms", emitter_type_ssize, + &dirty_decay_ms); + emitter_json_kv(emitter, "muzzy_decay_ms", emitter_type_ssize, + &muzzy_decay_ms); - CTL_M2_GET("stats.arenas.0.mapped", i, &mapped, size_t); - if (json) { - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\"mapped\": %zu,\n", mapped); - } else { - malloc_cprintf(write_cb, cbopaque, - "mapped: %12zu\n", mapped); - } + emitter_json_kv(emitter, "pactive", emitter_type_size, &pactive); + emitter_json_kv(emitter, "pdirty", emitter_type_size, &pdirty); + emitter_json_kv(emitter, "pmuzzy", emitter_type_size, &pmuzzy); - CTL_M2_GET("stats.arenas.0.retained", i, &retained, size_t); - if (json) { - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\"retained\": %zu,\n", retained); - } else { - malloc_cprintf(write_cb, cbopaque, - "retained: %12zu\n", retained); - } + emitter_json_kv(emitter, "dirty_npurge", emitter_type_uint64, + &dirty_npurge); + emitter_json_kv(emitter, "dirty_nmadvise", emitter_type_uint64, + &dirty_nmadvise); + emitter_json_kv(emitter, "dirty_purged", emitter_type_uint64, + &dirty_purged); - CTL_M2_GET("stats.arenas.0.base", i, &base, size_t); - if (json) { - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\"base\": %zu,\n", base); - } else { - malloc_cprintf(write_cb, cbopaque, - "base: %12zu\n", base); - } + emitter_json_kv(emitter, "muzzy_npurge", emitter_type_uint64, + &muzzy_npurge); + emitter_json_kv(emitter, "muzzy_nmadvise", emitter_type_uint64, + &muzzy_nmadvise); + emitter_json_kv(emitter, "muzzy_purged", emitter_type_uint64, + &muzzy_purged); - CTL_M2_GET("stats.arenas.0.internal", i, &internal, size_t); - if (json) { - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\"internal\": %zu,\n", internal); - } else { - malloc_cprintf(write_cb, cbopaque, - "internal: %12zu\n", internal); - } + /* Table-style emission. */ + COL(decay_row, decay_type, right, 9, title); + col_decay_type.str_val = "decaying:"; + + COL(decay_row, decay_time, right, 6, title); + col_decay_time.str_val = "time"; + + COL(decay_row, decay_npages, right, 13, title); + col_decay_npages.str_val = "npages"; + + COL(decay_row, decay_sweeps, right, 13, title); + col_decay_sweeps.str_val = "sweeps"; + + COL(decay_row, decay_madvises, right, 13, title); + col_decay_madvises.str_val = "madvises"; + + COL(decay_row, decay_purged, right, 13, title); + col_decay_purged.str_val = "purged"; + + /* Title row. */ + emitter_table_row(emitter, &decay_row); - CTL_M2_GET("stats.arenas.0.tcache_bytes", i, &tcache_bytes, size_t); - if (json) { - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\"tcache\": %zu,\n", tcache_bytes); + /* Dirty row. */ + col_decay_type.str_val = "dirty:"; + + if (dirty_decay_ms >= 0) { + col_decay_time.type = emitter_type_ssize; + col_decay_time.ssize_val = dirty_decay_ms; } else { - malloc_cprintf(write_cb, cbopaque, - "tcache: %12zu\n", tcache_bytes); + col_decay_time.type = emitter_type_title; + col_decay_time.str_val = "N/A"; } - CTL_M2_GET("stats.arenas.0.resident", i, &resident, size_t); - if (json) { - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\"resident\": %zu%s\n", resident, - (bins || large || mutex) ? "," : ""); + col_decay_npages.type = emitter_type_size; + col_decay_npages.size_val = pdirty; + + col_decay_sweeps.type = emitter_type_uint64; + col_decay_sweeps.uint64_val = dirty_npurge; + + col_decay_madvises.type = emitter_type_uint64; + col_decay_madvises.uint64_val = dirty_nmadvise; + + col_decay_purged.type = emitter_type_uint64; + col_decay_purged.uint64_val = dirty_purged; + + emitter_table_row(emitter, &decay_row); + + /* Muzzy row. */ + col_decay_type.str_val = "muzzy:"; + + if (muzzy_decay_ms >= 0) { + col_decay_time.type = emitter_type_ssize; + col_decay_time.ssize_val = muzzy_decay_ms; } else { - malloc_cprintf(write_cb, cbopaque, - "resident: %12zu\n", resident); + col_decay_time.type = emitter_type_title; + col_decay_time.str_val = "N/A"; } + col_decay_npages.type = emitter_type_size; + col_decay_npages.size_val = pmuzzy; + + col_decay_sweeps.type = emitter_type_uint64; + col_decay_sweeps.uint64_val = muzzy_npurge; + + col_decay_madvises.type = emitter_type_uint64; + col_decay_madvises.uint64_val = muzzy_nmadvise; + + col_decay_purged.type = emitter_type_uint64; + col_decay_purged.uint64_val = muzzy_purged; + + emitter_table_row(emitter, &decay_row); + + /* Small / large / total allocation counts. */ + emitter_row_t alloc_count_row; + emitter_row_init(&alloc_count_row); + + COL(alloc_count_row, count_title, left, 21, title); + col_count_title.str_val = ""; + + COL(alloc_count_row, count_allocated, right, 16, title); + col_count_allocated.str_val = "allocated"; + + COL(alloc_count_row, count_nmalloc, right, 16, title); + col_count_nmalloc.str_val = "nmalloc"; + COL(alloc_count_row, count_nmalloc_ps, right, 8, title); + col_count_nmalloc_ps.str_val = "(#/sec)"; + + COL(alloc_count_row, count_ndalloc, right, 16, title); + col_count_ndalloc.str_val = "ndalloc"; + COL(alloc_count_row, count_ndalloc_ps, right, 8, title); + col_count_ndalloc_ps.str_val = "(#/sec)"; + + COL(alloc_count_row, count_nrequests, right, 16, title); + col_count_nrequests.str_val = "nrequests"; + COL(alloc_count_row, count_nrequests_ps, right, 10, title); + col_count_nrequests_ps.str_val = "(#/sec)"; + + emitter_table_row(emitter, &alloc_count_row); + + col_count_nmalloc_ps.type = emitter_type_uint64; + col_count_ndalloc_ps.type = emitter_type_uint64; + col_count_nrequests_ps.type = emitter_type_uint64; + +#define GET_AND_EMIT_ALLOC_STAT(small_or_large, name, valtype) \ + CTL_M2_GET("stats.arenas.0." #small_or_large "." #name, i, \ + &small_or_large##_##name, valtype##_t); \ + emitter_json_kv(emitter, #name, emitter_type_##valtype, \ + &small_or_large##_##name); \ + col_count_##name.type = emitter_type_##valtype; \ + col_count_##name.valtype##_val = small_or_large##_##name; + + emitter_json_object_kv_begin(emitter, "small"); + col_count_title.str_val = "small:"; + + GET_AND_EMIT_ALLOC_STAT(small, allocated, size) + GET_AND_EMIT_ALLOC_STAT(small, nmalloc, uint64) + col_count_nmalloc_ps.uint64_val = + rate_per_second(col_count_nmalloc.uint64_val, uptime); + GET_AND_EMIT_ALLOC_STAT(small, ndalloc, uint64) + col_count_ndalloc_ps.uint64_val = + rate_per_second(col_count_ndalloc.uint64_val, uptime); + GET_AND_EMIT_ALLOC_STAT(small, nrequests, uint64) + col_count_nrequests_ps.uint64_val = + rate_per_second(col_count_nrequests.uint64_val, uptime); + + emitter_table_row(emitter, &alloc_count_row); + emitter_json_object_end(emitter); /* Close "small". */ + + emitter_json_object_kv_begin(emitter, "large"); + col_count_title.str_val = "large:"; + + GET_AND_EMIT_ALLOC_STAT(large, allocated, size) + GET_AND_EMIT_ALLOC_STAT(large, nmalloc, uint64) + col_count_nmalloc_ps.uint64_val = + rate_per_second(col_count_nmalloc.uint64_val, uptime); + GET_AND_EMIT_ALLOC_STAT(large, ndalloc, uint64) + col_count_ndalloc_ps.uint64_val = + rate_per_second(col_count_ndalloc.uint64_val, uptime); + GET_AND_EMIT_ALLOC_STAT(large, nrequests, uint64) + col_count_nrequests_ps.uint64_val = + rate_per_second(col_count_nrequests.uint64_val, uptime); + + emitter_table_row(emitter, &alloc_count_row); + emitter_json_object_end(emitter); /* Close "large". */ + +#undef GET_AND_EMIT_ALLOC_STAT + + /* Aggregated small + large stats are emitter only in table mode. */ + col_count_title.str_val = "total:"; + col_count_allocated.size_val = small_allocated + large_allocated; + col_count_nmalloc.uint64_val = small_nmalloc + large_nmalloc; + col_count_ndalloc.uint64_val = small_ndalloc + large_ndalloc; + col_count_nrequests.uint64_val = small_nrequests + large_nrequests; + col_count_nmalloc_ps.uint64_val = + rate_per_second(col_count_nmalloc.uint64_val, uptime); + col_count_ndalloc_ps.uint64_val = + rate_per_second(col_count_ndalloc.uint64_val, uptime); + col_count_nrequests_ps.uint64_val = + rate_per_second(col_count_nrequests.uint64_val, uptime); + emitter_table_row(emitter, &alloc_count_row); + + emitter_row_t mem_count_row; + emitter_row_init(&mem_count_row); + + emitter_col_t mem_count_title; + emitter_col_init(&mem_count_title, &mem_count_row); + mem_count_title.justify = emitter_justify_left; + mem_count_title.width = 21; + mem_count_title.type = emitter_type_title; + mem_count_title.str_val = ""; + + emitter_col_t mem_count_val; + emitter_col_init(&mem_count_val, &mem_count_row); + mem_count_val.justify = emitter_justify_right; + mem_count_val.width = 16; + mem_count_val.type = emitter_type_title; + mem_count_val.str_val = ""; + + emitter_table_row(emitter, &mem_count_row); + mem_count_val.type = emitter_type_size; + + /* Active count in bytes is emitted only in table mode. */ + mem_count_title.str_val = "active:"; + mem_count_val.size_val = pactive * page; + emitter_table_row(emitter, &mem_count_row); + +#define GET_AND_EMIT_MEM_STAT(stat) \ + CTL_M2_GET("stats.arenas.0."#stat, i, &stat, size_t); \ + emitter_json_kv(emitter, #stat, emitter_type_size, &stat); \ + mem_count_title.str_val = #stat":"; \ + mem_count_val.size_val = stat; \ + emitter_table_row(emitter, &mem_count_row); + + GET_AND_EMIT_MEM_STAT(mapped) + GET_AND_EMIT_MEM_STAT(retained) + GET_AND_EMIT_MEM_STAT(base) + GET_AND_EMIT_MEM_STAT(internal) + GET_AND_EMIT_MEM_STAT(metadata_thp) + GET_AND_EMIT_MEM_STAT(tcache_bytes) + GET_AND_EMIT_MEM_STAT(resident) + GET_AND_EMIT_MEM_STAT(extent_avail) +#undef GET_AND_EMIT_MEM_STAT + if (mutex) { - stats_arena_mutexes_print(write_cb, cbopaque, json, - !(bins || large), i); + stats_arena_mutexes_print(emitter, i, uptime); } if (bins) { - stats_arena_bins_print(write_cb, cbopaque, json, large, mutex, - i); + stats_arena_bins_print(emitter, mutex, i, uptime); } if (large) { - stats_arena_lextents_print(write_cb, cbopaque, json, i); + stats_arena_lextents_print(emitter, i, uptime); + } + if (extents) { + stats_arena_extents_print(emitter, i); } } static void -stats_general_print(void (*write_cb)(void *, const char *), void *cbopaque, - bool json, bool more) { +stats_general_print(emitter_t *emitter) { const char *cpv; - bool bv; + bool bv, bv2; unsigned uv; uint32_t u32v; uint64_t u64v; - ssize_t ssv; + ssize_t ssv, ssv2; size_t sv, bsz, usz, ssz, sssz, cpsz; bsz = sizeof(bool); @@ -663,365 +958,255 @@ stats_general_print(void (*write_cb)(void *, const char *), void *cbopaque, cpsz = sizeof(const char *); CTL_GET("version", &cpv, const char *); - if (json) { - malloc_cprintf(write_cb, cbopaque, - "\t\t\"version\": \"%s\",\n", cpv); - } else { - malloc_cprintf(write_cb, cbopaque, "Version: %s\n", cpv); - } + emitter_kv(emitter, "version", "Version", emitter_type_string, &cpv); /* config. */ -#define CONFIG_WRITE_BOOL_JSON(n, c) \ - if (json) { \ - CTL_GET("config."#n, &bv, bool); \ - malloc_cprintf(write_cb, cbopaque, \ - "\t\t\t\""#n"\": %s%s\n", bv ? "true" : "false", \ - (c)); \ + emitter_dict_begin(emitter, "config", "Build-time option settings"); +#define CONFIG_WRITE_BOOL(name) \ + do { \ + CTL_GET("config."#name, &bv, bool); \ + emitter_kv(emitter, #name, "config."#name, \ + emitter_type_bool, &bv); \ + } while (0) + + CONFIG_WRITE_BOOL(cache_oblivious); + CONFIG_WRITE_BOOL(debug); + CONFIG_WRITE_BOOL(fill); + CONFIG_WRITE_BOOL(lazy_lock); + emitter_kv(emitter, "malloc_conf", "config.malloc_conf", + emitter_type_string, &config_malloc_conf); + + CONFIG_WRITE_BOOL(prof); + CONFIG_WRITE_BOOL(prof_libgcc); + CONFIG_WRITE_BOOL(prof_libunwind); + CONFIG_WRITE_BOOL(stats); + CONFIG_WRITE_BOOL(utrace); + CONFIG_WRITE_BOOL(xmalloc); +#undef CONFIG_WRITE_BOOL + emitter_dict_end(emitter); /* Close "config" dict. */ + + /* opt. */ +#define OPT_WRITE(name, var, size, emitter_type) \ + if (je_mallctl("opt."name, (void *)&var, &size, NULL, 0) == \ + 0) { \ + emitter_kv(emitter, name, "opt."name, emitter_type, \ + &var); \ } - if (json) { - malloc_cprintf(write_cb, cbopaque, - "\t\t\"config\": {\n"); +#define OPT_WRITE_MUTABLE(name, var1, var2, size, emitter_type, \ + altname) \ + if (je_mallctl("opt."name, (void *)&var1, &size, NULL, 0) == \ + 0 && je_mallctl(altname, (void *)&var2, &size, NULL, 0) \ + == 0) { \ + emitter_kv_note(emitter, name, "opt."name, \ + emitter_type, &var1, altname, emitter_type, \ + &var2); \ } - CONFIG_WRITE_BOOL_JSON(cache_oblivious, ",") +#define OPT_WRITE_BOOL(name) OPT_WRITE(name, bv, bsz, emitter_type_bool) +#define OPT_WRITE_BOOL_MUTABLE(name, altname) \ + OPT_WRITE_MUTABLE(name, bv, bv2, bsz, emitter_type_bool, altname) + +#define OPT_WRITE_UNSIGNED(name) \ + OPT_WRITE(name, uv, usz, emitter_type_unsigned) + +#define OPT_WRITE_SIZE_T(name) \ + OPT_WRITE(name, sv, ssz, emitter_type_size) +#define OPT_WRITE_SSIZE_T(name) \ + OPT_WRITE(name, ssv, sssz, emitter_type_ssize) +#define OPT_WRITE_SSIZE_T_MUTABLE(name, altname) \ + OPT_WRITE_MUTABLE(name, ssv, ssv2, sssz, emitter_type_ssize, \ + altname) + +#define OPT_WRITE_CHAR_P(name) \ + OPT_WRITE(name, cpv, cpsz, emitter_type_string) + + emitter_dict_begin(emitter, "opt", "Run-time option settings"); + + OPT_WRITE_BOOL("abort") + OPT_WRITE_BOOL("abort_conf") + OPT_WRITE_BOOL("retain") + OPT_WRITE_CHAR_P("dss") + OPT_WRITE_UNSIGNED("narenas") + OPT_WRITE_CHAR_P("percpu_arena") + OPT_WRITE_SIZE_T("oversize_threshold") + OPT_WRITE_CHAR_P("metadata_thp") + OPT_WRITE_BOOL_MUTABLE("background_thread", "background_thread") + OPT_WRITE_SSIZE_T_MUTABLE("dirty_decay_ms", "arenas.dirty_decay_ms") + OPT_WRITE_SSIZE_T_MUTABLE("muzzy_decay_ms", "arenas.muzzy_decay_ms") + OPT_WRITE_SIZE_T("lg_extent_max_active_fit") + OPT_WRITE_CHAR_P("junk") + OPT_WRITE_BOOL("zero") + OPT_WRITE_BOOL("utrace") + OPT_WRITE_BOOL("xmalloc") + OPT_WRITE_BOOL("tcache") + OPT_WRITE_SSIZE_T("lg_tcache_max") + OPT_WRITE_CHAR_P("thp") + OPT_WRITE_BOOL("prof") + OPT_WRITE_CHAR_P("prof_prefix") + OPT_WRITE_BOOL_MUTABLE("prof_active", "prof.active") + OPT_WRITE_BOOL_MUTABLE("prof_thread_active_init", + "prof.thread_active_init") + OPT_WRITE_SSIZE_T_MUTABLE("lg_prof_sample", "prof.lg_sample") + OPT_WRITE_BOOL("prof_accum") + OPT_WRITE_SSIZE_T("lg_prof_interval") + OPT_WRITE_BOOL("prof_gdump") + OPT_WRITE_BOOL("prof_final") + OPT_WRITE_BOOL("prof_leak") + OPT_WRITE_BOOL("stats_print") + OPT_WRITE_CHAR_P("stats_print_opts") + + emitter_dict_end(emitter); + +#undef OPT_WRITE +#undef OPT_WRITE_MUTABLE +#undef OPT_WRITE_BOOL +#undef OPT_WRITE_BOOL_MUTABLE +#undef OPT_WRITE_UNSIGNED +#undef OPT_WRITE_SSIZE_T +#undef OPT_WRITE_SSIZE_T_MUTABLE +#undef OPT_WRITE_CHAR_P - CTL_GET("config.debug", &bv, bool); - if (json) { - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\"debug\": %s,\n", bv ? "true" : "false"); - } else { - malloc_cprintf(write_cb, cbopaque, "Assertions %s\n", - bv ? "enabled" : "disabled"); - } + /* prof. */ + if (config_prof) { + emitter_dict_begin(emitter, "prof", "Profiling settings"); - CONFIG_WRITE_BOOL_JSON(fill, ",") - CONFIG_WRITE_BOOL_JSON(lazy_lock, ",") + CTL_GET("prof.thread_active_init", &bv, bool); + emitter_kv(emitter, "thread_active_init", + "prof.thread_active_init", emitter_type_bool, &bv); - if (json) { - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\"malloc_conf\": \"%s\",\n", - config_malloc_conf); - } else { - malloc_cprintf(write_cb, cbopaque, - "config.malloc_conf: \"%s\"\n", config_malloc_conf); - } + CTL_GET("prof.active", &bv, bool); + emitter_kv(emitter, "active", "prof.active", emitter_type_bool, + &bv); - CONFIG_WRITE_BOOL_JSON(prof, ",") - CONFIG_WRITE_BOOL_JSON(prof_libgcc, ",") - CONFIG_WRITE_BOOL_JSON(prof_libunwind, ",") - CONFIG_WRITE_BOOL_JSON(stats, ",") - CONFIG_WRITE_BOOL_JSON(thp, ",") - CONFIG_WRITE_BOOL_JSON(utrace, ",") - CONFIG_WRITE_BOOL_JSON(xmalloc, "") - - if (json) { - malloc_cprintf(write_cb, cbopaque, - "\t\t},\n"); - } -#undef CONFIG_WRITE_BOOL_JSON + CTL_GET("prof.gdump", &bv, bool); + emitter_kv(emitter, "gdump", "prof.gdump", emitter_type_bool, + &bv); - /* opt. */ -#define OPT_WRITE_BOOL(n, c) \ - if (je_mallctl("opt."#n, (void *)&bv, &bsz, NULL, 0) == 0) { \ - if (json) { \ - malloc_cprintf(write_cb, cbopaque, \ - "\t\t\t\""#n"\": %s%s\n", bv ? "true" : \ - "false", (c)); \ - } else { \ - malloc_cprintf(write_cb, cbopaque, \ - " opt."#n": %s\n", bv ? "true" : "false"); \ - } \ - } -#define OPT_WRITE_BOOL_MUTABLE(n, m, c) { \ - bool bv2; \ - if (je_mallctl("opt."#n, (void *)&bv, &bsz, NULL, 0) == 0 && \ - je_mallctl(#m, (void *)&bv2, &bsz, NULL, 0) == 0) { \ - if (json) { \ - malloc_cprintf(write_cb, cbopaque, \ - "\t\t\t\""#n"\": %s%s\n", bv ? "true" : \ - "false", (c)); \ - } else { \ - malloc_cprintf(write_cb, cbopaque, \ - " opt."#n": %s ("#m": %s)\n", bv ? "true" \ - : "false", bv2 ? "true" : "false"); \ - } \ - } \ -} -#define OPT_WRITE_UNSIGNED(n, c) \ - if (je_mallctl("opt."#n, (void *)&uv, &usz, NULL, 0) == 0) { \ - if (json) { \ - malloc_cprintf(write_cb, cbopaque, \ - "\t\t\t\""#n"\": %u%s\n", uv, (c)); \ - } else { \ - malloc_cprintf(write_cb, cbopaque, \ - " opt."#n": %u\n", uv); \ - } \ - } -#define OPT_WRITE_SSIZE_T(n, c) \ - if (je_mallctl("opt."#n, (void *)&ssv, &sssz, NULL, 0) == 0) { \ - if (json) { \ - malloc_cprintf(write_cb, cbopaque, \ - "\t\t\t\""#n"\": %zd%s\n", ssv, (c)); \ - } else { \ - malloc_cprintf(write_cb, cbopaque, \ - " opt."#n": %zd\n", ssv); \ - } \ - } -#define OPT_WRITE_SSIZE_T_MUTABLE(n, m, c) { \ - ssize_t ssv2; \ - if (je_mallctl("opt."#n, (void *)&ssv, &sssz, NULL, 0) == 0 && \ - je_mallctl(#m, (void *)&ssv2, &sssz, NULL, 0) == 0) { \ - if (json) { \ - malloc_cprintf(write_cb, cbopaque, \ - "\t\t\t\""#n"\": %zd%s\n", ssv, (c)); \ - } else { \ - malloc_cprintf(write_cb, cbopaque, \ - " opt."#n": %zd ("#m": %zd)\n", \ - ssv, ssv2); \ - } \ - } \ -} -#define OPT_WRITE_CHAR_P(n, c) \ - if (je_mallctl("opt."#n, (void *)&cpv, &cpsz, NULL, 0) == 0) { \ - if (json) { \ - malloc_cprintf(write_cb, cbopaque, \ - "\t\t\t\""#n"\": \"%s\"%s\n", cpv, (c)); \ - } else { \ - malloc_cprintf(write_cb, cbopaque, \ - " opt."#n": \"%s\"\n", cpv); \ - } \ - } + CTL_GET("prof.interval", &u64v, uint64_t); + emitter_kv(emitter, "interval", "prof.interval", + emitter_type_uint64, &u64v); - if (json) { - malloc_cprintf(write_cb, cbopaque, - "\t\t\"opt\": {\n"); - } else { - malloc_cprintf(write_cb, cbopaque, - "Run-time option settings:\n"); - } - OPT_WRITE_BOOL(abort, ",") - OPT_WRITE_BOOL(abort_conf, ",") - OPT_WRITE_BOOL(retain, ",") - OPT_WRITE_CHAR_P(dss, ",") - OPT_WRITE_UNSIGNED(narenas, ",") - OPT_WRITE_CHAR_P(percpu_arena, ",") - OPT_WRITE_BOOL_MUTABLE(background_thread, background_thread, ",") - OPT_WRITE_SSIZE_T_MUTABLE(dirty_decay_ms, arenas.dirty_decay_ms, ",") - OPT_WRITE_SSIZE_T_MUTABLE(muzzy_decay_ms, arenas.muzzy_decay_ms, ",") - OPT_WRITE_CHAR_P(junk, ",") - OPT_WRITE_BOOL(zero, ",") - OPT_WRITE_BOOL(utrace, ",") - OPT_WRITE_BOOL(xmalloc, ",") - OPT_WRITE_BOOL(tcache, ",") - OPT_WRITE_SSIZE_T(lg_tcache_max, ",") - OPT_WRITE_BOOL(prof, ",") - OPT_WRITE_CHAR_P(prof_prefix, ",") - OPT_WRITE_BOOL_MUTABLE(prof_active, prof.active, ",") - OPT_WRITE_BOOL_MUTABLE(prof_thread_active_init, prof.thread_active_init, - ",") - OPT_WRITE_SSIZE_T_MUTABLE(lg_prof_sample, prof.lg_sample, ",") - OPT_WRITE_BOOL(prof_accum, ",") - OPT_WRITE_SSIZE_T(lg_prof_interval, ",") - OPT_WRITE_BOOL(prof_gdump, ",") - OPT_WRITE_BOOL(prof_final, ",") - OPT_WRITE_BOOL(prof_leak, ",") - OPT_WRITE_BOOL(stats_print, ",") - if (json || opt_stats_print) { - /* - * stats_print_opts is always emitted for JSON, so as long as it - * comes last it's safe to unconditionally omit the comma here - * (rather than having to conditionally omit it elsewhere - * depending on configuration). - */ - OPT_WRITE_CHAR_P(stats_print_opts, "") - } - if (json) { - malloc_cprintf(write_cb, cbopaque, - "\t\t},\n"); - } + CTL_GET("prof.lg_sample", &ssv, ssize_t); + emitter_kv(emitter, "lg_sample", "prof.lg_sample", + emitter_type_ssize, &ssv); -#undef OPT_WRITE_BOOL -#undef OPT_WRITE_BOOL_MUTABLE -#undef OPT_WRITE_SSIZE_T -#undef OPT_WRITE_CHAR_P + emitter_dict_end(emitter); /* Close "prof". */ + } /* arenas. */ - if (json) { - malloc_cprintf(write_cb, cbopaque, - "\t\t\"arenas\": {\n"); - } + /* + * The json output sticks arena info into an "arenas" dict; the table + * output puts them at the top-level. + */ + emitter_json_object_kv_begin(emitter, "arenas"); CTL_GET("arenas.narenas", &uv, unsigned); - if (json) { - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\"narenas\": %u,\n", uv); - } else { - malloc_cprintf(write_cb, cbopaque, "Arenas: %u\n", uv); - } + emitter_kv(emitter, "narenas", "Arenas", emitter_type_unsigned, &uv); - if (json) { - CTL_GET("arenas.dirty_decay_ms", &ssv, ssize_t); - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\"dirty_decay_ms\": %zd,\n", ssv); + /* + * Decay settings are emitted only in json mode; in table mode, they're + * emitted as notes with the opt output, above. + */ + CTL_GET("arenas.dirty_decay_ms", &ssv, ssize_t); + emitter_json_kv(emitter, "dirty_decay_ms", emitter_type_ssize, &ssv); - CTL_GET("arenas.muzzy_decay_ms", &ssv, ssize_t); - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\"muzzy_decay_ms\": %zd,\n", ssv); - } + CTL_GET("arenas.muzzy_decay_ms", &ssv, ssize_t); + emitter_json_kv(emitter, "muzzy_decay_ms", emitter_type_ssize, &ssv); CTL_GET("arenas.quantum", &sv, size_t); - if (json) { - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\"quantum\": %zu,\n", sv); - } else { - malloc_cprintf(write_cb, cbopaque, "Quantum size: %zu\n", sv); - } + emitter_kv(emitter, "quantum", "Quantum size", emitter_type_size, &sv); CTL_GET("arenas.page", &sv, size_t); - if (json) { - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\"page\": %zu,\n", sv); - } else { - malloc_cprintf(write_cb, cbopaque, "Page size: %zu\n", sv); - } + emitter_kv(emitter, "page", "Page size", emitter_type_size, &sv); if (je_mallctl("arenas.tcache_max", (void *)&sv, &ssz, NULL, 0) == 0) { - if (json) { - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\"tcache_max\": %zu,\n", sv); - } else { - malloc_cprintf(write_cb, cbopaque, - "Maximum thread-cached size class: %zu\n", sv); - } + emitter_kv(emitter, "tcache_max", + "Maximum thread-cached size class", emitter_type_size, &sv); } - if (json) { - unsigned nbins, nlextents, i; - - CTL_GET("arenas.nbins", &nbins, unsigned); - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\"nbins\": %u,\n", nbins); + unsigned nbins; + CTL_GET("arenas.nbins", &nbins, unsigned); + emitter_kv(emitter, "nbins", "Number of bin size classes", + emitter_type_unsigned, &nbins); - CTL_GET("arenas.nhbins", &uv, unsigned); - malloc_cprintf(write_cb, cbopaque, "\t\t\t\"nhbins\": %u,\n", - uv); + unsigned nhbins; + CTL_GET("arenas.nhbins", &nhbins, unsigned); + emitter_kv(emitter, "nhbins", "Number of thread-cache bin size classes", + emitter_type_unsigned, &nhbins); - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\"bin\": [\n"); - for (i = 0; i < nbins; i++) { - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t{\n"); + /* + * We do enough mallctls in a loop that we actually want to omit them + * (not just omit the printing). + */ + if (emitter->output == emitter_output_json) { + emitter_json_array_kv_begin(emitter, "bin"); + for (unsigned i = 0; i < nbins; i++) { + emitter_json_object_begin(emitter); CTL_M2_GET("arenas.bin.0.size", i, &sv, size_t); - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\t\"size\": %zu,\n", sv); + emitter_json_kv(emitter, "size", emitter_type_size, + &sv); CTL_M2_GET("arenas.bin.0.nregs", i, &u32v, uint32_t); - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\t\"nregs\": %"FMTu32",\n", u32v); + emitter_json_kv(emitter, "nregs", emitter_type_uint32, + &u32v); CTL_M2_GET("arenas.bin.0.slab_size", i, &sv, size_t); - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\t\"slab_size\": %zu\n", sv); + emitter_json_kv(emitter, "slab_size", emitter_type_size, + &sv); - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t}%s\n", (i + 1 < nbins) ? "," : ""); - } - malloc_cprintf(write_cb, cbopaque, - "\t\t\t],\n"); - - CTL_GET("arenas.nlextents", &nlextents, unsigned); - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\"nlextents\": %u,\n", nlextents); - - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\"lextent\": [\n"); - for (i = 0; i < nlextents; i++) { - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t{\n"); - - CTL_M2_GET("arenas.lextent.0.size", i, &sv, size_t); - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\t\"size\": %zu\n", sv); + CTL_M2_GET("arenas.bin.0.nshards", i, &u32v, uint32_t); + emitter_json_kv(emitter, "nshards", emitter_type_uint32, + &u32v); - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t}%s\n", (i + 1 < nlextents) ? "," : ""); + emitter_json_object_end(emitter); } - malloc_cprintf(write_cb, cbopaque, - "\t\t\t]\n"); - - malloc_cprintf(write_cb, cbopaque, - "\t\t}%s\n", (config_prof || more) ? "," : ""); + emitter_json_array_end(emitter); /* Close "bin". */ } - /* prof. */ - if (config_prof && json) { - malloc_cprintf(write_cb, cbopaque, - "\t\t\"prof\": {\n"); - - CTL_GET("prof.thread_active_init", &bv, bool); - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\"thread_active_init\": %s,\n", bv ? "true" : - "false"); - - CTL_GET("prof.active", &bv, bool); - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\"active\": %s,\n", bv ? "true" : "false"); - - CTL_GET("prof.gdump", &bv, bool); - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\"gdump\": %s,\n", bv ? "true" : "false"); + unsigned nlextents; + CTL_GET("arenas.nlextents", &nlextents, unsigned); + emitter_kv(emitter, "nlextents", "Number of large size classes", + emitter_type_unsigned, &nlextents); - CTL_GET("prof.interval", &u64v, uint64_t); - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\"interval\": %"FMTu64",\n", u64v); + if (emitter->output == emitter_output_json) { + emitter_json_array_kv_begin(emitter, "lextent"); + for (unsigned i = 0; i < nlextents; i++) { + emitter_json_object_begin(emitter); - CTL_GET("prof.lg_sample", &ssv, ssize_t); - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\"lg_sample\": %zd\n", ssv); + CTL_M2_GET("arenas.lextent.0.size", i, &sv, size_t); + emitter_json_kv(emitter, "size", emitter_type_size, + &sv); - malloc_cprintf(write_cb, cbopaque, - "\t\t}%s\n", more ? "," : ""); + emitter_json_object_end(emitter); + } + emitter_json_array_end(emitter); /* Close "lextent". */ } -} -static void -read_global_mutex_stats( - uint64_t results[mutex_prof_num_global_mutexes][mutex_prof_num_counters]) { - char cmd[MUTEX_CTL_STR_MAX_LENGTH]; - - mutex_prof_global_ind_t i; - for (i = 0; i < mutex_prof_num_global_mutexes; i++) { -#define OP(c, t) \ - gen_mutex_ctl_str(cmd, MUTEX_CTL_STR_MAX_LENGTH, \ - "mutexes", global_mutex_names[i], #c); \ - CTL_GET(cmd, (t *)&results[i][mutex_counter_##c], t); -MUTEX_PROF_COUNTERS -#undef OP - } + emitter_json_object_end(emitter); /* Close "arenas" */ } static void -stats_print_helper(void (*write_cb)(void *, const char *), void *cbopaque, - bool json, bool merged, bool destroyed, bool unmerged, bool bins, - bool large, bool mutex) { - size_t allocated, active, metadata, resident, mapped, retained; +stats_print_helper(emitter_t *emitter, bool merged, bool destroyed, + bool unmerged, bool bins, bool large, bool mutex, bool extents) { + /* + * These should be deleted. We keep them around for a while, to aid in + * the transition to the emitter code. + */ + size_t allocated, active, metadata, metadata_thp, resident, mapped, + retained; size_t num_background_threads; uint64_t background_thread_num_runs, background_thread_run_interval; CTL_GET("stats.allocated", &allocated, size_t); CTL_GET("stats.active", &active, size_t); CTL_GET("stats.metadata", &metadata, size_t); + CTL_GET("stats.metadata_thp", &metadata_thp, size_t); CTL_GET("stats.resident", &resident, size_t); CTL_GET("stats.mapped", &mapped, size_t); CTL_GET("stats.retained", &retained, size_t); - uint64_t mutex_stats[mutex_prof_num_global_mutexes][mutex_prof_num_counters]; - if (mutex) { - read_global_mutex_stats(mutex_stats); - } - if (have_background_thread) { CTL_GET("stats.background_thread.num_threads", &num_background_threads, size_t); @@ -1035,182 +1220,133 @@ stats_print_helper(void (*write_cb)(void *, const char *), void *cbopaque, background_thread_run_interval = 0; } - if (json) { - malloc_cprintf(write_cb, cbopaque, - "\t\t\"stats\": {\n"); - - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\"allocated\": %zu,\n", allocated); - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\"active\": %zu,\n", active); - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\"metadata\": %zu,\n", metadata); - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\"resident\": %zu,\n", resident); - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\"mapped\": %zu,\n", mapped); - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\"retained\": %zu,\n", retained); - - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\"background_thread\": {\n"); - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\"num_threads\": %zu,\n", num_background_threads); - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\"num_runs\": %"FMTu64",\n", - background_thread_num_runs); - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\t\"run_interval\": %"FMTu64"\n", - background_thread_run_interval); - malloc_cprintf(write_cb, cbopaque, "\t\t\t}%s\n", - mutex ? "," : ""); + /* Generic global stats. */ + emitter_json_object_kv_begin(emitter, "stats"); + emitter_json_kv(emitter, "allocated", emitter_type_size, &allocated); + emitter_json_kv(emitter, "active", emitter_type_size, &active); + emitter_json_kv(emitter, "metadata", emitter_type_size, &metadata); + emitter_json_kv(emitter, "metadata_thp", emitter_type_size, + &metadata_thp); + emitter_json_kv(emitter, "resident", emitter_type_size, &resident); + emitter_json_kv(emitter, "mapped", emitter_type_size, &mapped); + emitter_json_kv(emitter, "retained", emitter_type_size, &retained); + + emitter_table_printf(emitter, "Allocated: %zu, active: %zu, " + "metadata: %zu (n_thp %zu), resident: %zu, mapped: %zu, " + "retained: %zu\n", allocated, active, metadata, metadata_thp, + resident, mapped, retained); + + /* Background thread stats. */ + emitter_json_object_kv_begin(emitter, "background_thread"); + emitter_json_kv(emitter, "num_threads", emitter_type_size, + &num_background_threads); + emitter_json_kv(emitter, "num_runs", emitter_type_uint64, + &background_thread_num_runs); + emitter_json_kv(emitter, "run_interval", emitter_type_uint64, + &background_thread_run_interval); + emitter_json_object_end(emitter); /* Close "background_thread". */ + + emitter_table_printf(emitter, "Background threads: %zu, " + "num_runs: %"FMTu64", run_interval: %"FMTu64" ns\n", + num_background_threads, background_thread_num_runs, + background_thread_run_interval); - if (mutex) { - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\"mutexes\": {\n"); - mutex_prof_global_ind_t i; - for (i = 0; i < mutex_prof_num_global_mutexes; i++) { - mutex_stats_output_json(write_cb, cbopaque, - global_mutex_names[i], mutex_stats[i], - "\t\t\t\t", - i == mutex_prof_num_global_mutexes - 1); - } - malloc_cprintf(write_cb, cbopaque, "\t\t\t}\n"); - } - malloc_cprintf(write_cb, cbopaque, - "\t\t}%s\n", (merged || unmerged || destroyed) ? "," : ""); - } else { - malloc_cprintf(write_cb, cbopaque, - "Allocated: %zu, active: %zu, metadata: %zu," - " resident: %zu, mapped: %zu, retained: %zu\n", - allocated, active, metadata, resident, mapped, retained); - - if (have_background_thread && num_background_threads > 0) { - malloc_cprintf(write_cb, cbopaque, - "Background threads: %zu, num_runs: %"FMTu64", " - "run_interval: %"FMTu64" ns\n", - num_background_threads, - background_thread_num_runs, - background_thread_run_interval); - } - if (mutex) { - mutex_prof_global_ind_t i; - for (i = 0; i < mutex_prof_num_global_mutexes; i++) { - mutex_stats_output(write_cb, cbopaque, - global_mutex_names[i], mutex_stats[i], - i == 0); - } + if (mutex) { + emitter_row_t row; + emitter_col_t name; + emitter_col_t col64[mutex_prof_num_uint64_t_counters]; + emitter_col_t col32[mutex_prof_num_uint32_t_counters]; + uint64_t uptime; + + emitter_row_init(&row); + mutex_stats_init_cols(&row, "", &name, col64, col32); + + emitter_table_row(emitter, &row); + emitter_json_object_kv_begin(emitter, "mutexes"); + + CTL_M2_GET("stats.arenas.0.uptime", 0, &uptime, uint64_t); + + for (int i = 0; i < mutex_prof_num_global_mutexes; i++) { + mutex_stats_read_global(global_mutex_names[i], &name, + col64, col32, uptime); + emitter_json_object_kv_begin(emitter, global_mutex_names[i]); + mutex_stats_emit(emitter, &row, col64, col32); + emitter_json_object_end(emitter); } + + emitter_json_object_end(emitter); /* Close "mutexes". */ } + emitter_json_object_end(emitter); /* Close "stats". */ + if (merged || destroyed || unmerged) { unsigned narenas; - if (json) { - malloc_cprintf(write_cb, cbopaque, - "\t\t\"stats.arenas\": {\n"); - } + emitter_json_object_kv_begin(emitter, "stats.arenas"); CTL_GET("arenas.narenas", &narenas, unsigned); - { - size_t mib[3]; - size_t miblen = sizeof(mib) / sizeof(size_t); - size_t sz; - VARIABLE_ARRAY(bool, initialized, narenas); - bool destroyed_initialized; - unsigned i, j, ninitialized; - - xmallctlnametomib("arena.0.initialized", mib, &miblen); - for (i = ninitialized = 0; i < narenas; i++) { - mib[1] = i; - sz = sizeof(bool); - xmallctlbymib(mib, miblen, &initialized[i], &sz, - NULL, 0); - if (initialized[i]) { - ninitialized++; - } - } - mib[1] = MALLCTL_ARENAS_DESTROYED; + size_t mib[3]; + size_t miblen = sizeof(mib) / sizeof(size_t); + size_t sz; + VARIABLE_ARRAY(bool, initialized, narenas); + bool destroyed_initialized; + unsigned i, j, ninitialized; + + xmallctlnametomib("arena.0.initialized", mib, &miblen); + for (i = ninitialized = 0; i < narenas; i++) { + mib[1] = i; sz = sizeof(bool); - xmallctlbymib(mib, miblen, &destroyed_initialized, &sz, + xmallctlbymib(mib, miblen, &initialized[i], &sz, NULL, 0); - - /* Merged stats. */ - if (merged && (ninitialized > 1 || !unmerged)) { - /* Print merged arena stats. */ - if (json) { - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\"merged\": {\n"); - } else { - malloc_cprintf(write_cb, cbopaque, - "\nMerged arenas stats:\n"); - } - stats_arena_print(write_cb, cbopaque, json, - MALLCTL_ARENAS_ALL, bins, large, mutex); - if (json) { - malloc_cprintf(write_cb, cbopaque, - "\t\t\t}%s\n", - ((destroyed_initialized && - destroyed) || unmerged) ? "," : - ""); - } + if (initialized[i]) { + ninitialized++; } + } + mib[1] = MALLCTL_ARENAS_DESTROYED; + sz = sizeof(bool); + xmallctlbymib(mib, miblen, &destroyed_initialized, &sz, + NULL, 0); + + /* Merged stats. */ + if (merged && (ninitialized > 1 || !unmerged)) { + /* Print merged arena stats. */ + emitter_table_printf(emitter, "Merged arenas stats:\n"); + emitter_json_object_kv_begin(emitter, "merged"); + stats_arena_print(emitter, MALLCTL_ARENAS_ALL, bins, + large, mutex, extents); + emitter_json_object_end(emitter); /* Close "merged". */ + } - /* Destroyed stats. */ - if (destroyed_initialized && destroyed) { - /* Print destroyed arena stats. */ - if (json) { - malloc_cprintf(write_cb, cbopaque, - "\t\t\t\"destroyed\": {\n"); - } else { - malloc_cprintf(write_cb, cbopaque, - "\nDestroyed arenas stats:\n"); - } - stats_arena_print(write_cb, cbopaque, json, - MALLCTL_ARENAS_DESTROYED, bins, large, - mutex); - if (json) { - malloc_cprintf(write_cb, cbopaque, - "\t\t\t}%s\n", unmerged ? "," : - ""); - } - } + /* Destroyed stats. */ + if (destroyed_initialized && destroyed) { + /* Print destroyed arena stats. */ + emitter_table_printf(emitter, + "Destroyed arenas stats:\n"); + emitter_json_object_kv_begin(emitter, "destroyed"); + stats_arena_print(emitter, MALLCTL_ARENAS_DESTROYED, + bins, large, mutex, extents); + emitter_json_object_end(emitter); /* Close "destroyed". */ + } - /* Unmerged stats. */ - if (unmerged) { - for (i = j = 0; i < narenas; i++) { - if (initialized[i]) { - if (json) { - j++; - malloc_cprintf(write_cb, - cbopaque, - "\t\t\t\"%u\": {\n", - i); - } else { - malloc_cprintf(write_cb, - cbopaque, - "\narenas[%u]:\n", - i); - } - stats_arena_print(write_cb, - cbopaque, json, i, bins, - large, mutex); - if (json) { - malloc_cprintf(write_cb, - cbopaque, - "\t\t\t}%s\n", (j < - ninitialized) ? "," - : ""); - } - } + /* Unmerged stats. */ + if (unmerged) { + for (i = j = 0; i < narenas; i++) { + if (initialized[i]) { + char arena_ind_str[20]; + malloc_snprintf(arena_ind_str, + sizeof(arena_ind_str), "%u", i); + emitter_json_object_kv_begin(emitter, + arena_ind_str); + emitter_table_printf(emitter, + "arenas[%s]:\n", arena_ind_str); + stats_arena_print(emitter, i, bins, + large, mutex, extents); + /* Close "". */ + emitter_json_object_end(emitter); } } } - - if (json) { - malloc_cprintf(write_cb, cbopaque, - "\t\t}\n"); - } + emitter_json_object_end(emitter); /* Close "stats.arenas". */ } } @@ -1257,29 +1393,23 @@ stats_print(void (*write_cb)(void *, const char *), void *cbopaque, } } - if (json) { - malloc_cprintf(write_cb, cbopaque, - "{\n" - "\t\"jemalloc\": {\n"); - } else { - malloc_cprintf(write_cb, cbopaque, - "___ Begin jemalloc statistics ___\n"); - } + emitter_t emitter; + emitter_init(&emitter, + json ? emitter_output_json : emitter_output_table, write_cb, + cbopaque); + emitter_begin(&emitter); + emitter_table_printf(&emitter, "___ Begin jemalloc statistics ___\n"); + emitter_json_object_kv_begin(&emitter, "jemalloc"); if (general) { - stats_general_print(write_cb, cbopaque, json, config_stats); + stats_general_print(&emitter); } if (config_stats) { - stats_print_helper(write_cb, cbopaque, json, merged, destroyed, - unmerged, bins, large, mutex); + stats_print_helper(&emitter, merged, destroyed, unmerged, + bins, large, mutex, extents); } - if (json) { - malloc_cprintf(write_cb, cbopaque, - "\t}\n" - "}\n"); - } else { - malloc_cprintf(write_cb, cbopaque, - "--- End jemalloc statistics ---\n"); - } + emitter_json_object_end(&emitter); /* Closes the "jemalloc" dict. */ + emitter_table_printf(&emitter, "--- End jemalloc statistics ---\n"); + emitter_end(&emitter); } diff --git a/kbe/src/lib/dependencies/jemalloc/src/sz.c b/kbe/src/lib/dependencies/jemalloc/src/sz.c index 0986615f71..8633fb0500 100755 --- a/kbe/src/lib/dependencies/jemalloc/src/sz.c +++ b/kbe/src/lib/dependencies/jemalloc/src/sz.c @@ -2,105 +2,63 @@ #include "jemalloc/internal/sz.h" JEMALLOC_ALIGNED(CACHELINE) -const size_t sz_pind2sz_tab[NPSIZES+1] = { -#define PSZ_yes(lg_grp, ndelta, lg_delta) \ - (((ZU(1)<sc[i]; + if (sc->psz) { + sz_pind2sz_tab[pind] = (ZU(1) << sc->lg_base) + + (ZU(sc->ndelta) << sc->lg_delta); + pind++; + } + } + for (int i = pind; i <= (int)SC_NPSIZES; i++) { + sz_pind2sz_tab[pind] = sc_data->large_maxclass + PAGE; + } +} JEMALLOC_ALIGNED(CACHELINE) -const size_t sz_index2size_tab[NSIZES] = { -#define SC(index, lg_grp, lg_delta, ndelta, psz, bin, pgs, lg_delta_lookup) \ - ((ZU(1)<sc[i]; + sz_index2size_tab[i] = (ZU(1) << sc->lg_base) + + (ZU(sc->ndelta) << (sc->lg_delta)); + } +} +/* + * To keep this table small, we divide sizes by the tiny min size, which gives + * the smallest interval for which the result can change. + */ JEMALLOC_ALIGNED(CACHELINE) -const uint8_t sz_size2index_tab[] = { -#if LG_TINY_MIN == 0 -#warning "Dangerous LG_TINY_MIN" -#define S2B_0(i) i, -#elif LG_TINY_MIN == 1 -#warning "Dangerous LG_TINY_MIN" -#define S2B_1(i) i, -#elif LG_TINY_MIN == 2 -#warning "Dangerous LG_TINY_MIN" -#define S2B_2(i) i, -#elif LG_TINY_MIN == 3 -#define S2B_3(i) i, -#elif LG_TINY_MIN == 4 -#define S2B_4(i) i, -#elif LG_TINY_MIN == 5 -#define S2B_5(i) i, -#elif LG_TINY_MIN == 6 -#define S2B_6(i) i, -#elif LG_TINY_MIN == 7 -#define S2B_7(i) i, -#elif LG_TINY_MIN == 8 -#define S2B_8(i) i, -#elif LG_TINY_MIN == 9 -#define S2B_9(i) i, -#elif LG_TINY_MIN == 10 -#define S2B_10(i) i, -#elif LG_TINY_MIN == 11 -#define S2B_11(i) i, -#else -#error "Unsupported LG_TINY_MIN" -#endif -#if LG_TINY_MIN < 1 -#define S2B_1(i) S2B_0(i) S2B_0(i) -#endif -#if LG_TINY_MIN < 2 -#define S2B_2(i) S2B_1(i) S2B_1(i) -#endif -#if LG_TINY_MIN < 3 -#define S2B_3(i) S2B_2(i) S2B_2(i) -#endif -#if LG_TINY_MIN < 4 -#define S2B_4(i) S2B_3(i) S2B_3(i) -#endif -#if LG_TINY_MIN < 5 -#define S2B_5(i) S2B_4(i) S2B_4(i) -#endif -#if LG_TINY_MIN < 6 -#define S2B_6(i) S2B_5(i) S2B_5(i) -#endif -#if LG_TINY_MIN < 7 -#define S2B_7(i) S2B_6(i) S2B_6(i) -#endif -#if LG_TINY_MIN < 8 -#define S2B_8(i) S2B_7(i) S2B_7(i) -#endif -#if LG_TINY_MIN < 9 -#define S2B_9(i) S2B_8(i) S2B_8(i) -#endif -#if LG_TINY_MIN < 10 -#define S2B_10(i) S2B_9(i) S2B_9(i) -#endif -#if LG_TINY_MIN < 11 -#define S2B_11(i) S2B_10(i) S2B_10(i) -#endif -#define S2B_no(i) -#define SC(index, lg_grp, lg_delta, ndelta, psz, bin, pgs, lg_delta_lookup) \ - S2B_##lg_delta_lookup(index) - SIZE_CLASSES -#undef S2B_3 -#undef S2B_4 -#undef S2B_5 -#undef S2B_6 -#undef S2B_7 -#undef S2B_8 -#undef S2B_9 -#undef S2B_10 -#undef S2B_11 -#undef S2B_no -#undef SC -}; +uint8_t sz_size2index_tab[(SC_LOOKUP_MAXCLASS >> SC_LG_TINY_MIN) + 1]; + +static void +sz_boot_size2index_tab(const sc_data_t *sc_data) { + size_t dst_max = (SC_LOOKUP_MAXCLASS >> SC_LG_TINY_MIN) + 1; + size_t dst_ind = 0; + for (unsigned sc_ind = 0; sc_ind < SC_NSIZES && dst_ind < dst_max; + sc_ind++) { + const sc_t *sc = &sc_data->sc[sc_ind]; + size_t sz = (ZU(1) << sc->lg_base) + + (ZU(sc->ndelta) << sc->lg_delta); + size_t max_ind = ((sz + (ZU(1) << SC_LG_TINY_MIN) - 1) + >> SC_LG_TINY_MIN); + for (; dst_ind <= max_ind && dst_ind < dst_max; dst_ind++) { + sz_size2index_tab[dst_ind] = sc_ind; + } + } +} + +void +sz_boot(const sc_data_t *sc_data) { + sz_boot_pind2sz_tab(sc_data); + sz_boot_index2size_tab(sc_data); + sz_boot_size2index_tab(sc_data); +} diff --git a/kbe/src/lib/dependencies/jemalloc/src/tcache.c b/kbe/src/lib/dependencies/jemalloc/src/tcache.c index 936ef3140d..e7b970d904 100755 --- a/kbe/src/lib/dependencies/jemalloc/src/tcache.c +++ b/kbe/src/lib/dependencies/jemalloc/src/tcache.c @@ -4,7 +4,7 @@ #include "jemalloc/internal/assert.h" #include "jemalloc/internal/mutex.h" -#include "jemalloc/internal/size_classes.h" +#include "jemalloc/internal/sc.h" /******************************************************************************/ /* Data. */ @@ -12,7 +12,7 @@ bool opt_tcache = true; ssize_t opt_lg_tcache_max = LG_TCACHE_MAXCLASS_DEFAULT; -tcache_bin_info_t *tcache_bin_info; +cache_bin_info_t *tcache_bin_info; static unsigned stack_nelms; /* Total stack elms per tcache. */ unsigned nhbins; @@ -40,8 +40,8 @@ void tcache_event_hard(tsd_t *tsd, tcache_t *tcache) { szind_t binind = tcache->next_gc_bin; - tcache_bin_t *tbin; - if (binind < NBINS) { + cache_bin_t *tbin; + if (binind < SC_NBINS) { tbin = tcache_small_bin_get(tcache, binind); } else { tbin = tcache_large_bin_get(tcache, binind); @@ -50,7 +50,7 @@ tcache_event_hard(tsd_t *tsd, tcache_t *tcache) { /* * Flush (ceiling) 3/4 of the objects below the low water mark. */ - if (binind < NBINS) { + if (binind < SC_NBINS) { tcache_bin_flush_small(tsd, tcache, tbin, binind, tbin->ncached - tbin->low_water + (tbin->low_water >> 2)); @@ -58,7 +58,7 @@ tcache_event_hard(tsd_t *tsd, tcache_t *tcache) { * Reduce fill count by 2X. Limit lg_fill_div such that * the fill count is always at least 1. */ - tcache_bin_info_t *tbin_info = &tcache_bin_info[binind]; + cache_bin_info_t *tbin_info = &tcache_bin_info[binind]; if ((tbin_info->ncached_max >> (tcache->lg_fill_div[binind] + 1)) >= 1) { tcache->lg_fill_div[binind]++; @@ -72,7 +72,7 @@ tcache_event_hard(tsd_t *tsd, tcache_t *tcache) { * Increase fill count by 2X for small bins. Make sure * lg_fill_div stays greater than 0. */ - if (binind < NBINS && tcache->lg_fill_div[binind] > 1) { + if (binind < SC_NBINS && tcache->lg_fill_div[binind] > 1) { tcache->lg_fill_div[binind]--; } } @@ -86,7 +86,7 @@ tcache_event_hard(tsd_t *tsd, tcache_t *tcache) { void * tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache, - tcache_bin_t *tbin, szind_t binind, bool *tcache_success) { + cache_bin_t *tbin, szind_t binind, bool *tcache_success) { void *ret; assert(tcache->arena != NULL); @@ -95,33 +95,73 @@ tcache_alloc_small_hard(tsdn_t *tsdn, arena_t *arena, tcache_t *tcache, if (config_prof) { tcache->prof_accumbytes = 0; } - ret = tcache_alloc_easy(tbin, tcache_success); + ret = cache_bin_alloc_easy(tbin, tcache_success); return ret; } +/* Enabled with --enable-extra-size-check. */ +#ifdef JEMALLOC_EXTRA_SIZE_CHECK +static void +tbin_extents_lookup_size_check(tsdn_t *tsdn, cache_bin_t *tbin, szind_t binind, + size_t nflush, extent_t **extents){ + rtree_ctx_t rtree_ctx_fallback; + rtree_ctx_t *rtree_ctx = tsdn_rtree_ctx(tsdn, &rtree_ctx_fallback); + + /* + * Verify that the items in the tcache all have the correct size; this + * is useful for catching sized deallocation bugs, also to fail early + * instead of corrupting metadata. Since this can be turned on for opt + * builds, avoid the branch in the loop. + */ + szind_t szind; + size_t sz_sum = binind * nflush; + for (unsigned i = 0 ; i < nflush; i++) { + rtree_extent_szind_read(tsdn, &extents_rtree, + rtree_ctx, (uintptr_t)*(tbin->avail - 1 - i), true, + &extents[i], &szind); + sz_sum -= szind; + } + if (sz_sum != 0) { + malloc_printf(": size mismatch in thread cache " + "detected, likely caused by sized deallocation bugs by " + "application. Abort.\n"); + abort(); + } +} +#endif + void -tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin, +tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, cache_bin_t *tbin, szind_t binind, unsigned rem) { bool merged_stats = false; - assert(binind < NBINS); - assert(rem <= tbin->ncached); + assert(binind < SC_NBINS); + assert((cache_bin_sz_t)rem <= tbin->ncached); arena_t *arena = tcache->arena; assert(arena != NULL); unsigned nflush = tbin->ncached - rem; VARIABLE_ARRAY(extent_t *, item_extent, nflush); + +#ifndef JEMALLOC_EXTRA_SIZE_CHECK /* Look up extent once per item. */ for (unsigned i = 0 ; i < nflush; i++) { item_extent[i] = iealloc(tsd_tsdn(tsd), *(tbin->avail - 1 - i)); } - +#else + tbin_extents_lookup_size_check(tsd_tsdn(tsd), tbin, binind, nflush, + item_extent); +#endif while (nflush > 0) { /* Lock the arena bin associated with the first object. */ extent_t *extent = item_extent[0]; - arena_t *bin_arena = extent_arena_get(extent); - arena_bin_t *bin = &bin_arena->bins[binind]; + unsigned bin_arena_ind = extent_arena_ind_get(extent); + arena_t *bin_arena = arena_get(tsd_tsdn(tsd), bin_arena_ind, + false); + unsigned binshard = extent_binshard_get(extent); + assert(binshard < bin_infos[binind].n_shards); + bin_t *bin = &bin_arena->bins[binind].bin_shards[binshard]; if (config_prof && bin_arena == arena) { if (arena_prof_accum(tsd_tsdn(tsd), arena, @@ -132,8 +172,7 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin, } malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); - if (config_stats && bin_arena == arena) { - assert(!merged_stats); + if (config_stats && bin_arena == arena && !merged_stats) { merged_stats = true; bin->stats.nflushes++; bin->stats.nrequests += tbin->tstats.nrequests; @@ -145,9 +184,10 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin, extent = item_extent[i]; assert(ptr != NULL && extent != NULL); - if (extent_arena_get(extent) == bin_arena) { + if (extent_arena_ind_get(extent) == bin_arena_ind + && extent_binshard_get(extent) == binshard) { arena_dalloc_bin_junked_locked(tsd_tsdn(tsd), - bin_arena, extent, ptr); + bin_arena, bin, binind, extent, ptr); } else { /* * This object was allocated via a different @@ -169,8 +209,9 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin, * The flush loop didn't happen to flush to this thread's * arena, so the stats didn't get merged. Manually do so now. */ - arena_bin_t *bin = &arena->bins[binind]; - malloc_mutex_lock(tsd_tsdn(tsd), &bin->lock); + unsigned binshard; + bin_t *bin = arena_bin_choose_lock(tsd_tsdn(tsd), arena, binind, + &binshard); bin->stats.nflushes++; bin->stats.nrequests += tbin->tstats.nrequests; tbin->tstats.nrequests = 0; @@ -180,63 +221,76 @@ tcache_bin_flush_small(tsd_t *tsd, tcache_t *tcache, tcache_bin_t *tbin, memmove(tbin->avail - rem, tbin->avail - tbin->ncached, rem * sizeof(void *)); tbin->ncached = rem; - if ((low_water_t)tbin->ncached < tbin->low_water) { + if (tbin->ncached < tbin->low_water) { tbin->low_water = tbin->ncached; } } void -tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind, +tcache_bin_flush_large(tsd_t *tsd, cache_bin_t *tbin, szind_t binind, unsigned rem, tcache_t *tcache) { bool merged_stats = false; assert(binind < nhbins); - assert(rem <= tbin->ncached); + assert((cache_bin_sz_t)rem <= tbin->ncached); - arena_t *arena = tcache->arena; - assert(arena != NULL); + arena_t *tcache_arena = tcache->arena; + assert(tcache_arena != NULL); unsigned nflush = tbin->ncached - rem; VARIABLE_ARRAY(extent_t *, item_extent, nflush); + +#ifndef JEMALLOC_EXTRA_SIZE_CHECK /* Look up extent once per item. */ for (unsigned i = 0 ; i < nflush; i++) { item_extent[i] = iealloc(tsd_tsdn(tsd), *(tbin->avail - 1 - i)); } - +#else + tbin_extents_lookup_size_check(tsd_tsdn(tsd), tbin, binind, nflush, + item_extent); +#endif while (nflush > 0) { /* Lock the arena associated with the first object. */ extent_t *extent = item_extent[0]; - arena_t *locked_arena = extent_arena_get(extent); - UNUSED bool idump; + unsigned locked_arena_ind = extent_arena_ind_get(extent); + arena_t *locked_arena = arena_get(tsd_tsdn(tsd), + locked_arena_ind, false); + bool idump; if (config_prof) { idump = false; } - malloc_mutex_lock(tsd_tsdn(tsd), &locked_arena->large_mtx); + bool lock_large = !arena_is_auto(locked_arena); + if (lock_large) { + malloc_mutex_lock(tsd_tsdn(tsd), &locked_arena->large_mtx); + } for (unsigned i = 0; i < nflush; i++) { void *ptr = *(tbin->avail - 1 - i); assert(ptr != NULL); extent = item_extent[i]; - if (extent_arena_get(extent) == locked_arena) { + if (extent_arena_ind_get(extent) == locked_arena_ind) { large_dalloc_prep_junked_locked(tsd_tsdn(tsd), extent); } } - if ((config_prof || config_stats) && locked_arena == arena) { + if ((config_prof || config_stats) && + (locked_arena == tcache_arena)) { if (config_prof) { - idump = arena_prof_accum(tsd_tsdn(tsd), arena, - tcache->prof_accumbytes); + idump = arena_prof_accum(tsd_tsdn(tsd), + tcache_arena, tcache->prof_accumbytes); tcache->prof_accumbytes = 0; } if (config_stats) { merged_stats = true; arena_stats_large_nrequests_add(tsd_tsdn(tsd), - &arena->stats, binind, + &tcache_arena->stats, binind, tbin->tstats.nrequests); tbin->tstats.nrequests = 0; } } - malloc_mutex_unlock(tsd_tsdn(tsd), &locked_arena->large_mtx); + if (lock_large) { + malloc_mutex_unlock(tsd_tsdn(tsd), &locked_arena->large_mtx); + } unsigned ndeferred = 0; for (unsigned i = 0; i < nflush; i++) { @@ -244,7 +298,7 @@ tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind, extent = item_extent[i]; assert(ptr != NULL && extent != NULL); - if (extent_arena_get(extent) == locked_arena) { + if (extent_arena_ind_get(extent) == locked_arena_ind) { large_dalloc_finish(tsd_tsdn(tsd), extent); } else { /* @@ -270,15 +324,15 @@ tcache_bin_flush_large(tsd_t *tsd, tcache_bin_t *tbin, szind_t binind, * The flush loop didn't happen to flush to this thread's * arena, so the stats didn't get merged. Manually do so now. */ - arena_stats_large_nrequests_add(tsd_tsdn(tsd), &arena->stats, - binind, tbin->tstats.nrequests); + arena_stats_large_nrequests_add(tsd_tsdn(tsd), + &tcache_arena->stats, binind, tbin->tstats.nrequests); tbin->tstats.nrequests = 0; } memmove(tbin->avail - rem, tbin->avail - tbin->ncached, rem * sizeof(void *)); tbin->ncached = rem; - if ((low_water_t)tbin->ncached < tbin->low_water) { + if (tbin->ncached < tbin->low_water) { tbin->low_water = tbin->ncached; } } @@ -291,8 +345,15 @@ tcache_arena_associate(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) { if (config_stats) { /* Link into list of extant tcaches. */ malloc_mutex_lock(tsdn, &arena->tcache_ql_mtx); + ql_elm_new(tcache, link); ql_tail_insert(&arena->tcache_ql, tcache, link); + cache_bin_array_descriptor_init( + &tcache->cache_bin_array_descriptor, tcache->bins_small, + tcache->bins_large); + ql_tail_insert(&arena->cache_bin_array_descriptor_ql, + &tcache->cache_bin_array_descriptor, link); + malloc_mutex_unlock(tsdn, &arena->tcache_ql_mtx); } } @@ -316,6 +377,8 @@ tcache_arena_dissociate(tsdn_t *tsdn, tcache_t *tcache) { assert(in_ql); } ql_remove(&arena->tcache_ql, tcache, link); + ql_remove(&arena->cache_bin_array_descriptor_ql, + &tcache->cache_bin_array_descriptor, link); tcache_stats_merge(tsdn, tcache, arena); malloc_mutex_unlock(tsdn, &arena->tcache_ql_mtx); } @@ -354,10 +417,10 @@ tcache_init(tsd_t *tsd, tcache_t *tcache, void *avail_stack) { size_t stack_offset = 0; assert((TCACHE_NSLOTS_SMALL_MAX & 1U) == 0); - memset(tcache->tbins_small, 0, sizeof(tcache_bin_t) * NBINS); - memset(tcache->tbins_large, 0, sizeof(tcache_bin_t) * (nhbins - NBINS)); + memset(tcache->bins_small, 0, sizeof(cache_bin_t) * SC_NBINS); + memset(tcache->bins_large, 0, sizeof(cache_bin_t) * (nhbins - SC_NBINS)); unsigned i = 0; - for (; i < NBINS; i++) { + for (; i < SC_NBINS; i++) { tcache->lg_fill_div[i] = 1; stack_offset += tcache_bin_info[i].ncached_max * sizeof(void *); /* @@ -449,16 +512,16 @@ static void tcache_flush_cache(tsd_t *tsd, tcache_t *tcache) { assert(tcache->arena != NULL); - for (unsigned i = 0; i < NBINS; i++) { - tcache_bin_t *tbin = tcache_small_bin_get(tcache, i); + for (unsigned i = 0; i < SC_NBINS; i++) { + cache_bin_t *tbin = tcache_small_bin_get(tcache, i); tcache_bin_flush_small(tsd, tcache, tbin, i, 0); if (config_stats) { assert(tbin->tstats.nrequests == 0); } } - for (unsigned i = NBINS; i < nhbins; i++) { - tcache_bin_t *tbin = tcache_large_bin_get(tcache, i); + for (unsigned i = SC_NBINS; i < nhbins; i++) { + cache_bin_t *tbin = tcache_large_bin_get(tcache, i); tcache_bin_flush_large(tsd, tbin, i, 0, tcache); if (config_stats) { @@ -482,6 +545,7 @@ tcache_flush(tsd_t *tsd) { static void tcache_destroy(tsd_t *tsd, tcache_t *tcache, bool tsd_tcache) { tcache_flush_cache(tsd, tcache); + arena_t *arena = tcache->arena; tcache_arena_dissociate(tsd_tsdn(tsd), tcache); if (tsd_tcache) { @@ -494,6 +558,23 @@ tcache_destroy(tsd_t *tsd, tcache_t *tcache, bool tsd_tcache) { /* Release both the tcache struct and avail array. */ idalloctm(tsd_tsdn(tsd), tcache, NULL, NULL, true, true); } + + /* + * The deallocation and tcache flush above may not trigger decay since + * we are on the tcache shutdown path (potentially with non-nominal + * tsd). Manually trigger decay to avoid pathological cases. Also + * include arena 0 because the tcache array is allocated from it. + */ + arena_decay(tsd_tsdn(tsd), arena_get(tsd_tsdn(tsd), 0, false), + false, false); + + if (arena_nthreads_get(arena, false) == 0 && + !background_thread_enabled()) { + /* Force purging when no threads assigned to the arena anymore. */ + arena_decay(tsd_tsdn(tsd), arena, false, true); + } else { + arena_decay(tsd_tsdn(tsd), arena, false, false); + } } /* For auto tcache (embedded in TSD) only. */ @@ -523,17 +604,17 @@ tcache_stats_merge(tsdn_t *tsdn, tcache_t *tcache, arena_t *arena) { cassert(config_stats); /* Merge and reset tcache stats. */ - for (i = 0; i < NBINS; i++) { - arena_bin_t *bin = &arena->bins[i]; - tcache_bin_t *tbin = tcache_small_bin_get(tcache, i); - malloc_mutex_lock(tsdn, &bin->lock); + for (i = 0; i < SC_NBINS; i++) { + cache_bin_t *tbin = tcache_small_bin_get(tcache, i); + unsigned binshard; + bin_t *bin = arena_bin_choose_lock(tsdn, arena, i, &binshard); bin->stats.nrequests += tbin->tstats.nrequests; malloc_mutex_unlock(tsdn, &bin->lock); tbin->tstats.nrequests = 0; } for (; i < nhbins; i++) { - tcache_bin_t *tbin = tcache_large_bin_get(tcache, i); + cache_bin_t *tbin = tcache_large_bin_get(tcache, i); arena_stats_large_nrequests_add(tsdn, &arena->stats, i, tbin->tstats.nrequests); tbin->tstats.nrequests = 0; @@ -605,23 +686,32 @@ tcaches_create(tsd_t *tsd, unsigned *r_ind) { } static tcache_t * -tcaches_elm_remove(tsd_t *tsd, tcaches_t *elm) { +tcaches_elm_remove(tsd_t *tsd, tcaches_t *elm, bool allow_reinit) { malloc_mutex_assert_owner(tsd_tsdn(tsd), &tcaches_mtx); if (elm->tcache == NULL) { return NULL; } tcache_t *tcache = elm->tcache; - elm->tcache = NULL; + if (allow_reinit) { + elm->tcache = TCACHES_ELM_NEED_REINIT; + } else { + elm->tcache = NULL; + } + + if (tcache == TCACHES_ELM_NEED_REINIT) { + return NULL; + } return tcache; } void tcaches_flush(tsd_t *tsd, unsigned ind) { malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx); - tcache_t *tcache = tcaches_elm_remove(tsd, &tcaches[ind]); + tcache_t *tcache = tcaches_elm_remove(tsd, &tcaches[ind], true); malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx); if (tcache != NULL) { + /* Destroy the tcache; recreate in tcaches_get() if needed. */ tcache_destroy(tsd, tcache, false); } } @@ -630,7 +720,7 @@ void tcaches_destroy(tsd_t *tsd, unsigned ind) { malloc_mutex_lock(tsd_tsdn(tsd), &tcaches_mtx); tcaches_t *elm = &tcaches[ind]; - tcache_t *tcache = tcaches_elm_remove(tsd, elm); + tcache_t *tcache = tcaches_elm_remove(tsd, elm, false); elm->next = tcaches_avail; tcaches_avail = elm; malloc_mutex_unlock(tsd_tsdn(tsd), &tcaches_mtx); @@ -643,8 +733,8 @@ bool tcache_boot(tsdn_t *tsdn) { /* If necessary, clamp opt_lg_tcache_max. */ if (opt_lg_tcache_max < 0 || (ZU(1) << opt_lg_tcache_max) < - SMALL_MAXCLASS) { - tcache_maxclass = SMALL_MAXCLASS; + SC_SMALL_MAXCLASS) { + tcache_maxclass = SC_SMALL_MAXCLASS; } else { tcache_maxclass = (ZU(1) << opt_lg_tcache_max); } @@ -657,21 +747,21 @@ tcache_boot(tsdn_t *tsdn) { nhbins = sz_size2index(tcache_maxclass) + 1; /* Initialize tcache_bin_info. */ - tcache_bin_info = (tcache_bin_info_t *)base_alloc(tsdn, b0get(), nhbins - * sizeof(tcache_bin_info_t), CACHELINE); + tcache_bin_info = (cache_bin_info_t *)base_alloc(tsdn, b0get(), nhbins + * sizeof(cache_bin_info_t), CACHELINE); if (tcache_bin_info == NULL) { return true; } stack_nelms = 0; unsigned i; - for (i = 0; i < NBINS; i++) { - if ((arena_bin_info[i].nregs << 1) <= TCACHE_NSLOTS_SMALL_MIN) { + for (i = 0; i < SC_NBINS; i++) { + if ((bin_infos[i].nregs << 1) <= TCACHE_NSLOTS_SMALL_MIN) { tcache_bin_info[i].ncached_max = TCACHE_NSLOTS_SMALL_MIN; - } else if ((arena_bin_info[i].nregs << 1) <= + } else if ((bin_infos[i].nregs << 1) <= TCACHE_NSLOTS_SMALL_MAX) { tcache_bin_info[i].ncached_max = - (arena_bin_info[i].nregs << 1); + (bin_infos[i].nregs << 1); } else { tcache_bin_info[i].ncached_max = TCACHE_NSLOTS_SMALL_MAX; diff --git a/kbe/src/lib/dependencies/jemalloc/src/hooks.c b/kbe/src/lib/dependencies/jemalloc/src/test_hooks.c old mode 100755 new mode 100644 similarity index 79% rename from kbe/src/lib/dependencies/jemalloc/src/hooks.c rename to kbe/src/lib/dependencies/jemalloc/src/test_hooks.c index 6266ecd47f..ace00d9c46 --- a/kbe/src/lib/dependencies/jemalloc/src/hooks.c +++ b/kbe/src/lib/dependencies/jemalloc/src/test_hooks.c @@ -6,7 +6,7 @@ * from outside the generated library, so that we can use them in test code. */ JEMALLOC_EXPORT -void (*hooks_arena_new_hook)() = NULL; +void (*test_hooks_arena_new_hook)() = NULL; JEMALLOC_EXPORT -void (*hooks_libc_hook)() = NULL; +void (*test_hooks_libc_hook)() = NULL; diff --git a/kbe/src/lib/dependencies/jemalloc/src/tsd.c b/kbe/src/lib/dependencies/jemalloc/src/tsd.c index f968992f2b..d5fb4d6f82 100755 --- a/kbe/src/lib/dependencies/jemalloc/src/tsd.c +++ b/kbe/src/lib/dependencies/jemalloc/src/tsd.c @@ -12,6 +12,10 @@ static unsigned ncleanups; static malloc_tsd_cleanup_t cleanups[MALLOC_TSD_CLEANUPS_MAX]; +/* TSD_INITIALIZER triggers "-Wmissing-field-initializer" */ +JEMALLOC_DIAGNOSTIC_PUSH +JEMALLOC_DIAGNOSTIC_IGNORE_MISSING_STRUCT_FIELD_INITIALIZERS + #ifdef JEMALLOC_MALLOC_THREAD_CLEANUP __thread tsd_t JEMALLOC_TLS_MODEL tsd_tls = TSD_INITIALIZER; __thread bool JEMALLOC_TLS_MODEL tsd_initialized = false; @@ -41,6 +45,7 @@ tsd_init_head_t tsd_init_head = { ql_head_initializer(blocks), MALLOC_MUTEX_INITIALIZER }; + tsd_wrapper_t tsd_boot_wrapper = { false, TSD_INITIALIZER @@ -48,17 +53,164 @@ tsd_wrapper_t tsd_boot_wrapper = { bool tsd_booted = false; #endif +JEMALLOC_DIAGNOSTIC_POP /******************************************************************************/ +/* A list of all the tsds in the nominal state. */ +typedef ql_head(tsd_t) tsd_list_t; +static tsd_list_t tsd_nominal_tsds = ql_head_initializer(tsd_nominal_tsds); +static malloc_mutex_t tsd_nominal_tsds_lock; + +/* How many slow-path-enabling features are turned on. */ +static atomic_u32_t tsd_global_slow_count = ATOMIC_INIT(0); + +static bool +tsd_in_nominal_list(tsd_t *tsd) { + tsd_t *tsd_list; + bool found = false; + /* + * We don't know that tsd is nominal; it might not be safe to get data + * out of it here. + */ + malloc_mutex_lock(TSDN_NULL, &tsd_nominal_tsds_lock); + ql_foreach(tsd_list, &tsd_nominal_tsds, TSD_MANGLE(tcache).tsd_link) { + if (tsd == tsd_list) { + found = true; + break; + } + } + malloc_mutex_unlock(TSDN_NULL, &tsd_nominal_tsds_lock); + return found; +} + +static void +tsd_add_nominal(tsd_t *tsd) { + assert(!tsd_in_nominal_list(tsd)); + assert(tsd_state_get(tsd) <= tsd_state_nominal_max); + ql_elm_new(tsd, TSD_MANGLE(tcache).tsd_link); + malloc_mutex_lock(tsd_tsdn(tsd), &tsd_nominal_tsds_lock); + ql_tail_insert(&tsd_nominal_tsds, tsd, TSD_MANGLE(tcache).tsd_link); + malloc_mutex_unlock(tsd_tsdn(tsd), &tsd_nominal_tsds_lock); +} + +static void +tsd_remove_nominal(tsd_t *tsd) { + assert(tsd_in_nominal_list(tsd)); + assert(tsd_state_get(tsd) <= tsd_state_nominal_max); + malloc_mutex_lock(tsd_tsdn(tsd), &tsd_nominal_tsds_lock); + ql_remove(&tsd_nominal_tsds, tsd, TSD_MANGLE(tcache).tsd_link); + malloc_mutex_unlock(tsd_tsdn(tsd), &tsd_nominal_tsds_lock); +} + +static void +tsd_force_recompute(tsdn_t *tsdn) { + /* + * The stores to tsd->state here need to synchronize with the exchange + * in tsd_slow_update. + */ + atomic_fence(ATOMIC_RELEASE); + malloc_mutex_lock(tsdn, &tsd_nominal_tsds_lock); + tsd_t *remote_tsd; + ql_foreach(remote_tsd, &tsd_nominal_tsds, TSD_MANGLE(tcache).tsd_link) { + assert(tsd_atomic_load(&remote_tsd->state, ATOMIC_RELAXED) + <= tsd_state_nominal_max); + tsd_atomic_store(&remote_tsd->state, tsd_state_nominal_recompute, + ATOMIC_RELAXED); + } + malloc_mutex_unlock(tsdn, &tsd_nominal_tsds_lock); +} + +void +tsd_global_slow_inc(tsdn_t *tsdn) { + atomic_fetch_add_u32(&tsd_global_slow_count, 1, ATOMIC_RELAXED); + /* + * We unconditionally force a recompute, even if the global slow count + * was already positive. If we didn't, then it would be possible for us + * to return to the user, have the user synchronize externally with some + * other thread, and then have that other thread not have picked up the + * update yet (since the original incrementing thread might still be + * making its way through the tsd list). + */ + tsd_force_recompute(tsdn); +} + +void tsd_global_slow_dec(tsdn_t *tsdn) { + atomic_fetch_sub_u32(&tsd_global_slow_count, 1, ATOMIC_RELAXED); + /* See the note in ..._inc(). */ + tsd_force_recompute(tsdn); +} + +static bool +tsd_local_slow(tsd_t *tsd) { + return !tsd_tcache_enabled_get(tsd) + || tsd_reentrancy_level_get(tsd) > 0; +} + +bool +tsd_global_slow() { + return atomic_load_u32(&tsd_global_slow_count, ATOMIC_RELAXED) > 0; +} + +/******************************************************************************/ + +static uint8_t +tsd_state_compute(tsd_t *tsd) { + if (!tsd_nominal(tsd)) { + return tsd_state_get(tsd); + } + /* We're in *a* nominal state; but which one? */ + if (malloc_slow || tsd_local_slow(tsd) || tsd_global_slow()) { + return tsd_state_nominal_slow; + } else { + return tsd_state_nominal; + } +} + void tsd_slow_update(tsd_t *tsd) { - if (tsd_nominal(tsd)) { - if (malloc_slow || !tsd_tcache_enabled_get(tsd) || - tsd_reentrancy_level_get(tsd) > 0) { - tsd->state = tsd_state_nominal_slow; + uint8_t old_state; + do { + uint8_t new_state = tsd_state_compute(tsd); + old_state = tsd_atomic_exchange(&tsd->state, new_state, + ATOMIC_ACQUIRE); + } while (old_state == tsd_state_nominal_recompute); +} + +void +tsd_state_set(tsd_t *tsd, uint8_t new_state) { + /* Only the tsd module can change the state *to* recompute. */ + assert(new_state != tsd_state_nominal_recompute); + uint8_t old_state = tsd_atomic_load(&tsd->state, ATOMIC_RELAXED); + if (old_state > tsd_state_nominal_max) { + /* + * Not currently in the nominal list, but it might need to be + * inserted there. + */ + assert(!tsd_in_nominal_list(tsd)); + tsd_atomic_store(&tsd->state, new_state, ATOMIC_RELAXED); + if (new_state <= tsd_state_nominal_max) { + tsd_add_nominal(tsd); + } + } else { + /* + * We're currently nominal. If the new state is non-nominal, + * great; we take ourselves off the list and just enter the new + * state. + */ + assert(tsd_in_nominal_list(tsd)); + if (new_state > tsd_state_nominal_max) { + tsd_remove_nominal(tsd); + tsd_atomic_store(&tsd->state, new_state, + ATOMIC_RELAXED); } else { - tsd->state = tsd_state_nominal; + /* + * This is the tricky case. We're transitioning from + * one nominal state to another. The caller can't know + * about any races that are occuring at the same time, + * so we always have to recompute no matter what. + */ + tsd_slow_update(tsd); } } } @@ -71,12 +223,23 @@ tsd_data_init(tsd_t *tsd) { */ rtree_ctx_data_init(tsd_rtree_ctxp_get_unsafe(tsd)); + /* + * A nondeterministic seed based on the address of tsd reduces + * the likelihood of lockstep non-uniform cache index + * utilization among identical concurrent processes, but at the + * cost of test repeatability. For debug builds, instead use a + * deterministic seed. + */ + *tsd_offset_statep_get(tsd) = config_debug ? 0 : + (uint64_t)(uintptr_t)tsd; + return tsd_tcache_enabled_data_init(tsd); } static void assert_tsd_data_cleanup_done(tsd_t *tsd) { assert(!tsd_nominal(tsd)); + assert(!tsd_in_nominal_list(tsd)); assert(*tsd_arenap_get_unsafe(tsd) == NULL); assert(*tsd_iarenap_get_unsafe(tsd) == NULL); assert(*tsd_arenas_tdata_bypassp_get_unsafe(tsd) == true); @@ -87,8 +250,8 @@ assert_tsd_data_cleanup_done(tsd_t *tsd) { static bool tsd_data_init_nocleanup(tsd_t *tsd) { - assert(tsd->state == tsd_state_reincarnated || - tsd->state == tsd_state_minimal_initialized); + assert(tsd_state_get(tsd) == tsd_state_reincarnated || + tsd_state_get(tsd) == tsd_state_minimal_initialized); /* * During reincarnation, there is no guarantee that the cleanup function * will be called (deallocation may happen after all tsd destructors). @@ -107,27 +270,33 @@ tsd_t * tsd_fetch_slow(tsd_t *tsd, bool minimal) { assert(!tsd_fast(tsd)); - if (tsd->state == tsd_state_nominal_slow) { - /* On slow path but no work needed. */ - assert(malloc_slow || !tsd_tcache_enabled_get(tsd) || - tsd_reentrancy_level_get(tsd) > 0 || - *tsd_arenas_tdata_bypassp_get(tsd)); - } else if (tsd->state == tsd_state_uninitialized) { + if (tsd_state_get(tsd) == tsd_state_nominal_slow) { + /* + * On slow path but no work needed. Note that we can't + * necessarily *assert* that we're slow, because we might be + * slow because of an asynchronous modification to global state, + * which might be asynchronously modified *back*. + */ + } else if (tsd_state_get(tsd) == tsd_state_nominal_recompute) { + tsd_slow_update(tsd); + } else if (tsd_state_get(tsd) == tsd_state_uninitialized) { if (!minimal) { - tsd->state = tsd_state_nominal; - tsd_slow_update(tsd); - /* Trigger cleanup handler registration. */ - tsd_set(tsd); - tsd_data_init(tsd); + if (tsd_booted) { + tsd_state_set(tsd, tsd_state_nominal); + tsd_slow_update(tsd); + /* Trigger cleanup handler registration. */ + tsd_set(tsd); + tsd_data_init(tsd); + } } else { - tsd->state = tsd_state_minimal_initialized; + tsd_state_set(tsd, tsd_state_minimal_initialized); tsd_set(tsd); tsd_data_init_nocleanup(tsd); } - } else if (tsd->state == tsd_state_minimal_initialized) { + } else if (tsd_state_get(tsd) == tsd_state_minimal_initialized) { if (!minimal) { /* Switch to fully initialized. */ - tsd->state = tsd_state_nominal; + tsd_state_set(tsd, tsd_state_nominal); assert(*tsd_reentrancy_levelp_get(tsd) >= 1); (*tsd_reentrancy_levelp_get(tsd))--; tsd_slow_update(tsd); @@ -135,12 +304,12 @@ tsd_fetch_slow(tsd_t *tsd, bool minimal) { } else { assert_tsd_data_cleanup_done(tsd); } - } else if (tsd->state == tsd_state_purgatory) { - tsd->state = tsd_state_reincarnated; + } else if (tsd_state_get(tsd) == tsd_state_purgatory) { + tsd_state_set(tsd, tsd_state_reincarnated); tsd_set(tsd); tsd_data_init_nocleanup(tsd); } else { - assert(tsd->state == tsd_state_reincarnated); + assert(tsd_state_get(tsd) == tsd_state_reincarnated); } return tsd; @@ -204,7 +373,7 @@ void tsd_cleanup(void *arg) { tsd_t *tsd = (tsd_t *)arg; - switch (tsd->state) { + switch (tsd_state_get(tsd)) { case tsd_state_uninitialized: /* Do nothing. */ break; @@ -222,7 +391,7 @@ tsd_cleanup(void *arg) { case tsd_state_nominal: case tsd_state_nominal_slow: tsd_do_data_cleanup(tsd); - tsd->state = tsd_state_purgatory; + tsd_state_set(tsd, tsd_state_purgatory); tsd_set(tsd); break; case tsd_state_purgatory: @@ -250,6 +419,10 @@ malloc_tsd_boot0(void) { tsd_t *tsd; ncleanups = 0; + if (malloc_mutex_init(&tsd_nominal_tsds_lock, "tsd_nominal_tsds_lock", + WITNESS_RANK_OMIT, malloc_mutex_rank_exclusive)) { + return NULL; + } if (tsd_boot0()) { return NULL; } @@ -300,7 +473,7 @@ _tls_callback(HINSTANCE hinstDLL, DWORD fdwReason, LPVOID lpvReserved) { # pragma comment(linker, "/INCLUDE:_tls_callback") # else # pragma comment(linker, "/INCLUDE:_tls_used") -# pragma comment(linker, "/INCLUDE:tls_callback") +# pragma comment(linker, "/INCLUDE:" STRINGIFY(tls_callback) ) # endif # pragma section(".CRT$XLY",long,read) #endif @@ -339,3 +512,23 @@ tsd_init_finish(tsd_init_head_t *head, tsd_init_block_t *block) { malloc_mutex_unlock(TSDN_NULL, &head->lock); } #endif + +void +tsd_prefork(tsd_t *tsd) { + malloc_mutex_prefork(tsd_tsdn(tsd), &tsd_nominal_tsds_lock); +} + +void +tsd_postfork_parent(tsd_t *tsd) { + malloc_mutex_postfork_parent(tsd_tsdn(tsd), &tsd_nominal_tsds_lock); +} + +void +tsd_postfork_child(tsd_t *tsd) { + malloc_mutex_postfork_child(tsd_tsdn(tsd), &tsd_nominal_tsds_lock); + ql_new(&tsd_nominal_tsds); + + if (tsd_state_get(tsd) <= tsd_state_nominal_max) { + tsd_add_nominal(tsd); + } +} diff --git a/kbe/src/lib/dependencies/jemalloc/src/zone.c b/kbe/src/lib/dependencies/jemalloc/src/zone.c index 9d3b7b4952..23dfdd04a9 100755 --- a/kbe/src/lib/dependencies/jemalloc/src/zone.c +++ b/kbe/src/lib/dependencies/jemalloc/src/zone.c @@ -89,6 +89,7 @@ JEMALLOC_ATTR(weak_import); static malloc_zone_t *default_zone, *purgeable_zone; static malloc_zone_t jemalloc_zone; static struct malloc_introspection_t jemalloc_zone_introspect; +static pid_t zone_force_lock_pid = -1; /******************************************************************************/ /* Function prototypes for non-inline static functions. */ @@ -270,6 +271,12 @@ zone_log(malloc_zone_t *zone, void *address) { static void zone_force_lock(malloc_zone_t *zone) { if (isthreaded) { + /* + * See the note in zone_force_unlock, below, to see why we need + * this. + */ + assert(zone_force_lock_pid == -1); + zone_force_lock_pid = getpid(); jemalloc_prefork(); } } @@ -277,14 +284,25 @@ zone_force_lock(malloc_zone_t *zone) { static void zone_force_unlock(malloc_zone_t *zone) { /* - * Call jemalloc_postfork_child() rather than - * jemalloc_postfork_parent(), because this function is executed by both - * parent and child. The parent can tolerate having state - * reinitialized, but the child cannot unlock mutexes that were locked - * by the parent. + * zone_force_lock and zone_force_unlock are the entry points to the + * forking machinery on OS X. The tricky thing is, the child is not + * allowed to unlock mutexes locked in the parent, even if owned by the + * forking thread (and the mutex type we use in OS X will fail an assert + * if we try). In the child, we can get away with reinitializing all + * the mutexes, which has the effect of unlocking them. In the parent, + * doing this would mean we wouldn't wake any waiters blocked on the + * mutexes we unlock. So, we record the pid of the current thread in + * zone_force_lock, and use that to detect if we're in the parent or + * child here, to decide which unlock logic we need. */ if (isthreaded) { - jemalloc_postfork_child(); + assert(zone_force_lock_pid != -1); + if (getpid() == zone_force_lock_pid) { + jemalloc_postfork_parent(); + } else { + jemalloc_postfork_child(); + } + zone_force_lock_pid = -1; } } diff --git a/kbe/src/lib/dependencies/jemalloc/test/include/test/extent_hooks.h b/kbe/src/lib/dependencies/jemalloc/test/include/test/extent_hooks.h index ea0128573a..1f0620154d 100755 --- a/kbe/src/lib/dependencies/jemalloc/test/include/test/extent_hooks.h +++ b/kbe/src/lib/dependencies/jemalloc/test/include/test/extent_hooks.h @@ -266,6 +266,8 @@ extent_merge_hook(extent_hooks_t *extent_hooks, void *addr_a, size_t size_a, "extent_hooks should be same as pointer used to set hooks"); assert_ptr_eq(extent_hooks->merge, extent_merge_hook, "Wrong hook function"); + assert_ptr_eq((void *)((uintptr_t)addr_a + size_a), addr_b, + "Extents not mergeable"); called_merge = true; if (!try_merge) { return true; diff --git a/kbe/src/lib/dependencies/jemalloc/test/include/test/jemalloc_test.h.in b/kbe/src/lib/dependencies/jemalloc/test/include/test/jemalloc_test.h.in index 67caa86bf6..c46af5d9b2 100755 --- a/kbe/src/lib/dependencies/jemalloc/test/include/test/jemalloc_test.h.in +++ b/kbe/src/lib/dependencies/jemalloc/test/include/test/jemalloc_test.h.in @@ -25,7 +25,7 @@ extern "C" { #include "test/jemalloc_test_defs.h" -#ifdef JEMALLOC_OSSPIN +#if defined(JEMALLOC_OSATOMIC) # include #endif @@ -69,7 +69,7 @@ static const bool config_debug = # define JEMALLOC_N(n) @private_namespace@##n # include "jemalloc/internal/private_namespace.h" -# include "jemalloc/internal/hooks.h" +# include "jemalloc/internal/test_hooks.h" /* Hermetic headers. */ # include "jemalloc/internal/assert.h" diff --git a/kbe/src/lib/dependencies/jemalloc/test/include/test/mtx.h b/kbe/src/lib/dependencies/jemalloc/test/include/test/mtx.h index 58afbc3d13..066a213790 100755 --- a/kbe/src/lib/dependencies/jemalloc/test/include/test/mtx.h +++ b/kbe/src/lib/dependencies/jemalloc/test/include/test/mtx.h @@ -10,8 +10,6 @@ typedef struct { CRITICAL_SECTION lock; #elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) os_unfair_lock lock; -#elif (defined(JEMALLOC_OSSPIN)) - OSSpinLock lock; #else pthread_mutex_t lock; #endif diff --git a/kbe/src/lib/dependencies/jemalloc/test/integration/aligned_alloc.c b/kbe/src/lib/dependencies/jemalloc/test/integration/aligned_alloc.c index 536b67ea87..4375b172ad 100755 --- a/kbe/src/lib/dependencies/jemalloc/test/integration/aligned_alloc.c +++ b/kbe/src/lib/dependencies/jemalloc/test/integration/aligned_alloc.c @@ -34,6 +34,17 @@ TEST_BEGIN(test_alignment_errors) { } TEST_END + +/* + * GCC "-Walloc-size-larger-than" warning detects when one of the memory + * allocation functions is called with a size larger than the maximum size that + * they support. Here we want to explicitly test that the allocation functions + * do indeed fail properly when this is the case, which triggers the warning. + * Therefore we disable the warning for these tests. + */ +JEMALLOC_DIAGNOSTIC_PUSH +JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN + TEST_BEGIN(test_oom_errors) { size_t alignment, size; void *p; @@ -78,6 +89,9 @@ TEST_BEGIN(test_oom_errors) { } TEST_END +/* Re-enable the "-Walloc-size-larger-than=" warning */ +JEMALLOC_DIAGNOSTIC_POP + TEST_BEGIN(test_alignment_and_size) { #define NITER 4 size_t alignment, size, total; @@ -124,10 +138,20 @@ TEST_BEGIN(test_alignment_and_size) { } TEST_END +TEST_BEGIN(test_zero_alloc) { + void *res = aligned_alloc(8, 0); + assert(res); + size_t usable = malloc_usable_size(res); + assert(usable > 0); + free(res); +} +TEST_END + int main(void) { return test( test_alignment_errors, test_oom_errors, - test_alignment_and_size); + test_alignment_and_size, + test_zero_alloc); } diff --git a/kbe/src/lib/dependencies/jemalloc/test/integration/cpp/basic.cpp b/kbe/src/lib/dependencies/jemalloc/test/integration/cpp/basic.cpp new file mode 100644 index 0000000000..65890ecd55 --- /dev/null +++ b/kbe/src/lib/dependencies/jemalloc/test/integration/cpp/basic.cpp @@ -0,0 +1,25 @@ +#include +#include "test/jemalloc_test.h" + +TEST_BEGIN(test_basic) { + auto foo = new long(4); + assert_ptr_not_null(foo, "Unexpected new[] failure"); + delete foo; + // Test nullptr handling. + foo = nullptr; + delete foo; + + auto bar = new long; + assert_ptr_not_null(bar, "Unexpected new failure"); + delete bar; + // Test nullptr handling. + bar = nullptr; + delete bar; +} +TEST_END + +int +main() { + return test( + test_basic); +} diff --git a/kbe/src/lib/dependencies/jemalloc/test/integration/extent.c b/kbe/src/lib/dependencies/jemalloc/test/integration/extent.c index 1dcf217630..b5db087665 100755 --- a/kbe/src/lib/dependencies/jemalloc/test/integration/extent.c +++ b/kbe/src/lib/dependencies/jemalloc/test/integration/extent.c @@ -98,7 +98,45 @@ test_extent_body(unsigned arena_ind) { dallocx(p, flags); } -TEST_BEGIN(test_extent_manual_hook) { +static void +test_manual_hook_auto_arena(void) { + unsigned narenas; + size_t old_size, new_size, sz; + size_t hooks_mib[3]; + size_t hooks_miblen; + extent_hooks_t *new_hooks, *old_hooks; + + extent_hooks_prep(); + + sz = sizeof(unsigned); + /* Get number of auto arenas. */ + assert_d_eq(mallctl("opt.narenas", (void *)&narenas, &sz, NULL, 0), + 0, "Unexpected mallctl() failure"); + if (narenas == 1) { + return; + } + + /* Install custom extent hooks on arena 1 (might not be initialized). */ + hooks_miblen = sizeof(hooks_mib)/sizeof(size_t); + assert_d_eq(mallctlnametomib("arena.0.extent_hooks", hooks_mib, + &hooks_miblen), 0, "Unexpected mallctlnametomib() failure"); + hooks_mib[1] = 1; + old_size = sizeof(extent_hooks_t *); + new_hooks = &hooks; + new_size = sizeof(extent_hooks_t *); + assert_d_eq(mallctlbymib(hooks_mib, hooks_miblen, (void *)&old_hooks, + &old_size, (void *)&new_hooks, new_size), 0, + "Unexpected extent_hooks error"); + static bool auto_arena_created = false; + if (old_hooks != &hooks) { + assert_b_eq(auto_arena_created, false, + "Expected auto arena 1 created only once."); + auto_arena_created = true; + } +} + +static void +test_manual_hook_body(void) { unsigned arena_ind; size_t old_size, new_size, sz; size_t hooks_mib[3]; @@ -139,8 +177,9 @@ TEST_BEGIN(test_extent_manual_hook) { assert_ptr_ne(old_hooks->merge, extent_merge_hook, "Unexpected extent_hooks error"); - test_skip_if(check_background_thread_enabled()); - test_extent_body(arena_ind); + if (!check_background_thread_enabled()) { + test_extent_body(arena_ind); + } /* Restore extent hooks. */ assert_d_eq(mallctlbymib(hooks_mib, hooks_miblen, NULL, NULL, @@ -165,6 +204,22 @@ TEST_BEGIN(test_extent_manual_hook) { assert_ptr_eq(old_hooks->merge, default_hooks->merge, "Unexpected extent_hooks error"); } + +TEST_BEGIN(test_extent_manual_hook) { + test_manual_hook_auto_arena(); + test_manual_hook_body(); + + /* Test failure paths. */ + try_split = false; + test_manual_hook_body(); + try_merge = false; + test_manual_hook_body(); + try_purge_lazy = false; + try_purge_forced = false; + test_manual_hook_body(); + + try_split = try_merge = try_purge_lazy = try_purge_forced = true; +} TEST_END TEST_BEGIN(test_extent_auto_hook) { diff --git a/kbe/src/lib/dependencies/jemalloc/test/integration/malloc.c b/kbe/src/lib/dependencies/jemalloc/test/integration/malloc.c new file mode 100644 index 0000000000..8b33bc8f3b --- /dev/null +++ b/kbe/src/lib/dependencies/jemalloc/test/integration/malloc.c @@ -0,0 +1,16 @@ +#include "test/jemalloc_test.h" + +TEST_BEGIN(test_zero_alloc) { + void *res = malloc(0); + assert(res); + size_t usable = malloc_usable_size(res); + assert(usable > 0); + free(res); +} +TEST_END + +int +main(void) { + return test( + test_zero_alloc); +} diff --git a/kbe/src/lib/dependencies/jemalloc/test/integration/mallocx.c b/kbe/src/lib/dependencies/jemalloc/test/integration/mallocx.c index b0b5cdac23..645d4db48f 100755 --- a/kbe/src/lib/dependencies/jemalloc/test/integration/mallocx.c +++ b/kbe/src/lib/dependencies/jemalloc/test/integration/mallocx.c @@ -51,6 +51,16 @@ purge(void) { "Unexpected mallctl error"); } +/* + * GCC "-Walloc-size-larger-than" warning detects when one of the memory + * allocation functions is called with a size larger than the maximum size that + * they support. Here we want to explicitly test that the allocation functions + * do indeed fail properly when this is the case, which triggers the warning. + * Therefore we disable the warning for these tests. + */ +JEMALLOC_DIAGNOSTIC_PUSH +JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN + TEST_BEGIN(test_overflow) { size_t largemax; @@ -71,6 +81,38 @@ TEST_BEGIN(test_overflow) { } TEST_END +static void * +remote_alloc(void *arg) { + unsigned arena; + size_t sz = sizeof(unsigned); + assert_d_eq(mallctl("arenas.create", (void *)&arena, &sz, NULL, 0), 0, + "Unexpected mallctl() failure"); + size_t large_sz; + sz = sizeof(size_t); + assert_d_eq(mallctl("arenas.lextent.0.size", (void *)&large_sz, &sz, + NULL, 0), 0, "Unexpected mallctl failure"); + + void *ptr = mallocx(large_sz, MALLOCX_ARENA(arena) + | MALLOCX_TCACHE_NONE); + void **ret = (void **)arg; + *ret = ptr; + + return NULL; +} + +TEST_BEGIN(test_remote_free) { + thd_t thd; + void *ret; + thd_create(&thd, remote_alloc, (void *)&ret); + thd_join(thd, NULL); + assert_ptr_not_null(ret, "Unexpected mallocx failure"); + + /* Avoid TCACHE_NONE to explicitly test tcache_flush(). */ + dallocx(ret, 0); + mallctl("thread.tcache.flush", NULL, NULL, NULL, 0); +} +TEST_END + TEST_BEGIN(test_oom) { size_t largemax; bool oom; @@ -84,7 +126,7 @@ TEST_BEGIN(test_oom) { largemax = get_large_size(get_nlarge()-1); oom = false; for (i = 0; i < sizeof(ptrs) / sizeof(void *); i++) { - ptrs[i] = mallocx(largemax, 0); + ptrs[i] = mallocx(largemax, MALLOCX_ARENA(0)); if (ptrs[i] == NULL) { oom = true; } @@ -113,6 +155,9 @@ TEST_BEGIN(test_oom) { } TEST_END +/* Re-enable the "-Walloc-size-larger-than=" warning */ +JEMALLOC_DIAGNOSTIC_POP + TEST_BEGIN(test_basic) { #define MAXSZ (((size_t)1) << 23) size_t sz; @@ -151,9 +196,17 @@ TEST_BEGIN(test_basic) { TEST_END TEST_BEGIN(test_alignment_and_size) { + const char *percpu_arena; + size_t sz = sizeof(percpu_arena); + + if(mallctl("opt.percpu_arena", (void *)&percpu_arena, &sz, NULL, 0) || + strcmp(percpu_arena, "disabled") != 0) { + test_skip("test_alignment_and_size skipped: " + "not working with percpu arena."); + }; #define MAXALIGN (((size_t)1) << 23) #define NITER 4 - size_t nsz, rsz, sz, alignment, total; + size_t nsz, rsz, alignment, total; unsigned i; void *ps[NITER]; @@ -170,12 +223,12 @@ TEST_BEGIN(test_alignment_and_size) { sz += (alignment >> (LG_SIZEOF_PTR-1)) - 1) { for (i = 0; i < NITER; i++) { nsz = nallocx(sz, MALLOCX_ALIGN(alignment) | - MALLOCX_ZERO); + MALLOCX_ZERO | MALLOCX_ARENA(0)); assert_zu_ne(nsz, 0, "nallocx() error for alignment=%zu, " "size=%zu (%#zx)", alignment, sz, sz); ps[i] = mallocx(sz, MALLOCX_ALIGN(alignment) | - MALLOCX_ZERO); + MALLOCX_ZERO | MALLOCX_ARENA(0)); assert_ptr_not_null(ps[i], "mallocx() error for alignment=%zu, " "size=%zu (%#zx)", alignment, sz, sz); @@ -215,6 +268,7 @@ main(void) { return test( test_overflow, test_oom, + test_remote_free, test_basic, test_alignment_and_size); } diff --git a/kbe/src/lib/dependencies/jemalloc/test/integration/overflow.c b/kbe/src/lib/dependencies/jemalloc/test/integration/overflow.c index 6a9785b2e2..748ebb6772 100755 --- a/kbe/src/lib/dependencies/jemalloc/test/integration/overflow.c +++ b/kbe/src/lib/dependencies/jemalloc/test/integration/overflow.c @@ -1,5 +1,15 @@ #include "test/jemalloc_test.h" +/* + * GCC "-Walloc-size-larger-than" warning detects when one of the memory + * allocation functions is called with a size larger than the maximum size that + * they support. Here we want to explicitly test that the allocation functions + * do indeed fail properly when this is the case, which triggers the warning. + * Therefore we disable the warning for these tests. + */ +JEMALLOC_DIAGNOSTIC_PUSH +JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN + TEST_BEGIN(test_overflow) { unsigned nlextents; size_t mib[4]; @@ -39,6 +49,9 @@ TEST_BEGIN(test_overflow) { } TEST_END +/* Re-enable the "-Walloc-size-larger-than=" warning */ +JEMALLOC_DIAGNOSTIC_POP + int main(void) { return test( diff --git a/kbe/src/lib/dependencies/jemalloc/test/integration/rallocx.c b/kbe/src/lib/dependencies/jemalloc/test/integration/rallocx.c index 7821ca5f56..08ed08d3fb 100755 --- a/kbe/src/lib/dependencies/jemalloc/test/integration/rallocx.c +++ b/kbe/src/lib/dependencies/jemalloc/test/integration/rallocx.c @@ -208,6 +208,16 @@ TEST_BEGIN(test_lg_align_and_zero) { } TEST_END +/* + * GCC "-Walloc-size-larger-than" warning detects when one of the memory + * allocation functions is called with a size larger than the maximum size that + * they support. Here we want to explicitly test that the allocation functions + * do indeed fail properly when this is the case, which triggers the warning. + * Therefore we disable the warning for these tests. + */ +JEMALLOC_DIAGNOSTIC_PUSH +JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN + TEST_BEGIN(test_overflow) { size_t largemax; void *p; @@ -234,6 +244,9 @@ TEST_BEGIN(test_overflow) { } TEST_END +/* Re-enable the "-Walloc-size-larger-than=" warning */ +JEMALLOC_DIAGNOSTIC_POP + int main(void) { return test( diff --git a/kbe/src/lib/dependencies/jemalloc/test/integration/sdallocx.c b/kbe/src/lib/dependencies/jemalloc/test/integration/sdallocx.c index e7ea1d82c0..ca0144855a 100755 --- a/kbe/src/lib/dependencies/jemalloc/test/integration/sdallocx.c +++ b/kbe/src/lib/dependencies/jemalloc/test/integration/sdallocx.c @@ -49,7 +49,7 @@ TEST_END int main(void) { - return test( + return test_no_reentrancy( test_basic, test_alignment_and_size); } diff --git a/kbe/src/lib/dependencies/jemalloc/test/integration/slab_sizes.c b/kbe/src/lib/dependencies/jemalloc/test/integration/slab_sizes.c new file mode 100644 index 0000000000..af250c3f46 --- /dev/null +++ b/kbe/src/lib/dependencies/jemalloc/test/integration/slab_sizes.c @@ -0,0 +1,80 @@ +#include "test/jemalloc_test.h" + +/* Note that this test relies on the unusual slab sizes set in slab_sizes.sh. */ + +TEST_BEGIN(test_slab_sizes) { + unsigned nbins; + size_t page; + size_t sizemib[4]; + size_t slabmib[4]; + size_t len; + + len = sizeof(nbins); + assert_d_eq(mallctl("arenas.nbins", &nbins, &len, NULL, 0), 0, + "nbins mallctl failure"); + + len = sizeof(page); + assert_d_eq(mallctl("arenas.page", &page, &len, NULL, 0), 0, + "page mallctl failure"); + + len = 4; + assert_d_eq(mallctlnametomib("arenas.bin.0.size", sizemib, &len), 0, + "bin size mallctlnametomib failure"); + + len = 4; + assert_d_eq(mallctlnametomib("arenas.bin.0.slab_size", slabmib, &len), + 0, "slab size mallctlnametomib failure"); + + size_t biggest_slab_seen = 0; + + for (unsigned i = 0; i < nbins; i++) { + size_t bin_size; + size_t slab_size; + len = sizeof(size_t); + sizemib[2] = i; + slabmib[2] = i; + assert_d_eq(mallctlbymib(sizemib, 4, (void *)&bin_size, &len, + NULL, 0), 0, "bin size mallctlbymib failure"); + + len = sizeof(size_t); + assert_d_eq(mallctlbymib(slabmib, 4, (void *)&slab_size, &len, + NULL, 0), 0, "slab size mallctlbymib failure"); + + if (bin_size < 100) { + /* + * Then we should be as close to 17 as possible. Since + * not all page sizes are valid (because of bitmap + * limitations on the number of items in a slab), we + * should at least make sure that the number of pages + * goes up. + */ + assert_zu_ge(slab_size, biggest_slab_seen, + "Slab sizes should go up"); + biggest_slab_seen = slab_size; + } else if ( + (100 <= bin_size && bin_size < 128) + || (128 < bin_size && bin_size <= 200)) { + assert_zu_eq(slab_size, page, + "Forced-small slabs should be small"); + } else if (bin_size == 128) { + assert_zu_eq(slab_size, 2 * page, + "Forced-2-page slab should be 2 pages"); + } else if (200 < bin_size && bin_size <= 4096) { + assert_zu_ge(slab_size, biggest_slab_seen, + "Slab sizes should go up"); + biggest_slab_seen = slab_size; + } + } + /* + * For any reasonable configuration, 17 pages should be a valid slab + * size for 4096-byte items. + */ + assert_zu_eq(biggest_slab_seen, 17 * page, "Didn't hit page target"); +} +TEST_END + +int +main(void) { + return test( + test_slab_sizes); +} diff --git a/kbe/src/lib/dependencies/jemalloc/test/integration/slab_sizes.sh b/kbe/src/lib/dependencies/jemalloc/test/integration/slab_sizes.sh new file mode 100644 index 0000000000..07e3db81bc --- /dev/null +++ b/kbe/src/lib/dependencies/jemalloc/test/integration/slab_sizes.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +# Some screwy-looking slab sizes. +export MALLOC_CONF="slab_sizes:1-4096:17|100-200:1|128-128:2" diff --git a/kbe/src/lib/dependencies/jemalloc/test/integration/smallocx.c b/kbe/src/lib/dependencies/jemalloc/test/integration/smallocx.c new file mode 100644 index 0000000000..2486752beb --- /dev/null +++ b/kbe/src/lib/dependencies/jemalloc/test/integration/smallocx.c @@ -0,0 +1,312 @@ +#include "test/jemalloc_test.h" +#include "jemalloc/jemalloc_macros.h" + +#define STR_HELPER(x) #x +#define STR(x) STR_HELPER(x) + +#ifndef JEMALLOC_VERSION_GID_IDENT + #error "JEMALLOC_VERSION_GID_IDENT not defined" +#endif + +#define JOIN(x, y) x ## y +#define JOIN2(x, y) JOIN(x, y) +#define smallocx JOIN2(smallocx_, JEMALLOC_VERSION_GID_IDENT) + +typedef struct { + void *ptr; + size_t size; +} smallocx_return_t; + +extern smallocx_return_t +smallocx(size_t size, int flags); + +static unsigned +get_nsizes_impl(const char *cmd) { + unsigned ret; + size_t z; + + z = sizeof(unsigned); + assert_d_eq(mallctl(cmd, (void *)&ret, &z, NULL, 0), 0, + "Unexpected mallctl(\"%s\", ...) failure", cmd); + + return ret; +} + +static unsigned +get_nlarge(void) { + return get_nsizes_impl("arenas.nlextents"); +} + +static size_t +get_size_impl(const char *cmd, size_t ind) { + size_t ret; + size_t z; + size_t mib[4]; + size_t miblen = 4; + + z = sizeof(size_t); + assert_d_eq(mallctlnametomib(cmd, mib, &miblen), + 0, "Unexpected mallctlnametomib(\"%s\", ...) failure", cmd); + mib[2] = ind; + z = sizeof(size_t); + assert_d_eq(mallctlbymib(mib, miblen, (void *)&ret, &z, NULL, 0), + 0, "Unexpected mallctlbymib([\"%s\", %zu], ...) failure", cmd, ind); + + return ret; +} + +static size_t +get_large_size(size_t ind) { + return get_size_impl("arenas.lextent.0.size", ind); +} + +/* + * On systems which can't merge extents, tests that call this function generate + * a lot of dirty memory very quickly. Purging between cycles mitigates + * potential OOM on e.g. 32-bit Windows. + */ +static void +purge(void) { + assert_d_eq(mallctl("arena.0.purge", NULL, NULL, NULL, 0), 0, + "Unexpected mallctl error"); +} + +/* + * GCC "-Walloc-size-larger-than" warning detects when one of the memory + * allocation functions is called with a size larger than the maximum size that + * they support. Here we want to explicitly test that the allocation functions + * do indeed fail properly when this is the case, which triggers the warning. + * Therefore we disable the warning for these tests. + */ +JEMALLOC_DIAGNOSTIC_PUSH +JEMALLOC_DIAGNOSTIC_IGNORE_ALLOC_SIZE_LARGER_THAN + +TEST_BEGIN(test_overflow) { + size_t largemax; + + largemax = get_large_size(get_nlarge()-1); + + assert_ptr_null(smallocx(largemax+1, 0).ptr, + "Expected OOM for smallocx(size=%#zx, 0)", largemax+1); + + assert_ptr_null(smallocx(ZU(PTRDIFF_MAX)+1, 0).ptr, + "Expected OOM for smallocx(size=%#zx, 0)", ZU(PTRDIFF_MAX)+1); + + assert_ptr_null(smallocx(SIZE_T_MAX, 0).ptr, + "Expected OOM for smallocx(size=%#zx, 0)", SIZE_T_MAX); + + assert_ptr_null(smallocx(1, MALLOCX_ALIGN(ZU(PTRDIFF_MAX)+1)).ptr, + "Expected OOM for smallocx(size=1, MALLOCX_ALIGN(%#zx))", + ZU(PTRDIFF_MAX)+1); +} +TEST_END + +static void * +remote_alloc(void *arg) { + unsigned arena; + size_t sz = sizeof(unsigned); + assert_d_eq(mallctl("arenas.create", (void *)&arena, &sz, NULL, 0), 0, + "Unexpected mallctl() failure"); + size_t large_sz; + sz = sizeof(size_t); + assert_d_eq(mallctl("arenas.lextent.0.size", (void *)&large_sz, &sz, + NULL, 0), 0, "Unexpected mallctl failure"); + + smallocx_return_t r + = smallocx(large_sz, MALLOCX_ARENA(arena) | MALLOCX_TCACHE_NONE); + void *ptr = r.ptr; + assert_zu_eq(r.size, + nallocx(large_sz, MALLOCX_ARENA(arena) | MALLOCX_TCACHE_NONE), + "Expected smalloc(size,flags).size == nallocx(size,flags)"); + void **ret = (void **)arg; + *ret = ptr; + + return NULL; +} + +TEST_BEGIN(test_remote_free) { + thd_t thd; + void *ret; + thd_create(&thd, remote_alloc, (void *)&ret); + thd_join(thd, NULL); + assert_ptr_not_null(ret, "Unexpected smallocx failure"); + + /* Avoid TCACHE_NONE to explicitly test tcache_flush(). */ + dallocx(ret, 0); + mallctl("thread.tcache.flush", NULL, NULL, NULL, 0); +} +TEST_END + +TEST_BEGIN(test_oom) { + size_t largemax; + bool oom; + void *ptrs[3]; + unsigned i; + + /* + * It should be impossible to allocate three objects that each consume + * nearly half the virtual address space. + */ + largemax = get_large_size(get_nlarge()-1); + oom = false; + for (i = 0; i < sizeof(ptrs) / sizeof(void *); i++) { + ptrs[i] = smallocx(largemax, 0).ptr; + if (ptrs[i] == NULL) { + oom = true; + } + } + assert_true(oom, + "Expected OOM during series of calls to smallocx(size=%zu, 0)", + largemax); + for (i = 0; i < sizeof(ptrs) / sizeof(void *); i++) { + if (ptrs[i] != NULL) { + dallocx(ptrs[i], 0); + } + } + purge(); + +#if LG_SIZEOF_PTR == 3 + assert_ptr_null(smallocx(0x8000000000000000ULL, + MALLOCX_ALIGN(0x8000000000000000ULL)).ptr, + "Expected OOM for smallocx()"); + assert_ptr_null(smallocx(0x8000000000000000ULL, + MALLOCX_ALIGN(0x80000000)).ptr, + "Expected OOM for smallocx()"); +#else + assert_ptr_null(smallocx(0x80000000UL, MALLOCX_ALIGN(0x80000000UL)).ptr, + "Expected OOM for smallocx()"); +#endif +} +TEST_END + +/* Re-enable the "-Walloc-size-larger-than=" warning */ +JEMALLOC_DIAGNOSTIC_POP + +TEST_BEGIN(test_basic) { +#define MAXSZ (((size_t)1) << 23) + size_t sz; + + for (sz = 1; sz < MAXSZ; sz = nallocx(sz, 0) + 1) { + smallocx_return_t ret; + size_t nsz, rsz, smz; + void *p; + nsz = nallocx(sz, 0); + assert_zu_ne(nsz, 0, "Unexpected nallocx() error"); + ret = smallocx(sz, 0); + p = ret.ptr; + smz = ret.size; + assert_ptr_not_null(p, + "Unexpected smallocx(size=%zx, flags=0) error", sz); + rsz = sallocx(p, 0); + assert_zu_ge(rsz, sz, "Real size smaller than expected"); + assert_zu_eq(nsz, rsz, "nallocx()/sallocx() size mismatch"); + assert_zu_eq(nsz, smz, "nallocx()/smallocx() size mismatch"); + dallocx(p, 0); + + ret = smallocx(sz, 0); + p = ret.ptr; + smz = ret.size; + assert_ptr_not_null(p, + "Unexpected smallocx(size=%zx, flags=0) error", sz); + dallocx(p, 0); + + nsz = nallocx(sz, MALLOCX_ZERO); + assert_zu_ne(nsz, 0, "Unexpected nallocx() error"); + assert_zu_ne(smz, 0, "Unexpected smallocx() error"); + ret = smallocx(sz, MALLOCX_ZERO); + p = ret.ptr; + assert_ptr_not_null(p, + "Unexpected smallocx(size=%zx, flags=MALLOCX_ZERO) error", + nsz); + rsz = sallocx(p, 0); + assert_zu_eq(nsz, rsz, "nallocx()/sallocx() rsize mismatch"); + assert_zu_eq(nsz, smz, "nallocx()/smallocx() size mismatch"); + dallocx(p, 0); + purge(); + } +#undef MAXSZ +} +TEST_END + +TEST_BEGIN(test_alignment_and_size) { + const char *percpu_arena; + size_t sz = sizeof(percpu_arena); + + if(mallctl("opt.percpu_arena", (void *)&percpu_arena, &sz, NULL, 0) || + strcmp(percpu_arena, "disabled") != 0) { + test_skip("test_alignment_and_size skipped: " + "not working with percpu arena."); + }; +#define MAXALIGN (((size_t)1) << 23) +#define NITER 4 + size_t nsz, rsz, smz, alignment, total; + unsigned i; + void *ps[NITER]; + + for (i = 0; i < NITER; i++) { + ps[i] = NULL; + } + + for (alignment = 8; + alignment <= MAXALIGN; + alignment <<= 1) { + total = 0; + for (sz = 1; + sz < 3 * alignment && sz < (1U << 31); + sz += (alignment >> (LG_SIZEOF_PTR-1)) - 1) { + for (i = 0; i < NITER; i++) { + nsz = nallocx(sz, MALLOCX_ALIGN(alignment) | + MALLOCX_ZERO); + assert_zu_ne(nsz, 0, + "nallocx() error for alignment=%zu, " + "size=%zu (%#zx)", alignment, sz, sz); + smallocx_return_t ret + = smallocx(sz, MALLOCX_ALIGN(alignment) | MALLOCX_ZERO); + ps[i] = ret.ptr; + assert_ptr_not_null(ps[i], + "smallocx() error for alignment=%zu, " + "size=%zu (%#zx)", alignment, sz, sz); + rsz = sallocx(ps[i], 0); + smz = ret.size; + assert_zu_ge(rsz, sz, + "Real size smaller than expected for " + "alignment=%zu, size=%zu", alignment, sz); + assert_zu_eq(nsz, rsz, + "nallocx()/sallocx() size mismatch for " + "alignment=%zu, size=%zu", alignment, sz); + assert_zu_eq(nsz, smz, + "nallocx()/smallocx() size mismatch for " + "alignment=%zu, size=%zu", alignment, sz); + assert_ptr_null( + (void *)((uintptr_t)ps[i] & (alignment-1)), + "%p inadequately aligned for" + " alignment=%zu, size=%zu", ps[i], + alignment, sz); + total += rsz; + if (total >= (MAXALIGN << 1)) { + break; + } + } + for (i = 0; i < NITER; i++) { + if (ps[i] != NULL) { + dallocx(ps[i], 0); + ps[i] = NULL; + } + } + } + purge(); + } +#undef MAXALIGN +#undef NITER +} +TEST_END + +int +main(void) { + return test( + test_overflow, + test_oom, + test_remote_free, + test_basic, + test_alignment_and_size); +} diff --git a/kbe/src/lib/dependencies/jemalloc/test/integration/smallocx.sh b/kbe/src/lib/dependencies/jemalloc/test/integration/smallocx.sh new file mode 100644 index 0000000000..d07f10f3ce --- /dev/null +++ b/kbe/src/lib/dependencies/jemalloc/test/integration/smallocx.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +if [ "x${enable_fill}" = "x1" ] ; then + export MALLOC_CONF="junk:false" +fi diff --git a/kbe/src/lib/dependencies/jemalloc/test/integration/thread_tcache_enabled.c b/kbe/src/lib/dependencies/jemalloc/test/integration/thread_tcache_enabled.c index 0c343a6c56..95c9acc138 100755 --- a/kbe/src/lib/dependencies/jemalloc/test/integration/thread_tcache_enabled.c +++ b/kbe/src/lib/dependencies/jemalloc/test/integration/thread_tcache_enabled.c @@ -60,8 +60,6 @@ thd_start(void *arg) { free(malloc(1)); return NULL; - test_skip("\"thread.tcache.enabled\" mallctl not available"); - return NULL; } TEST_BEGIN(test_main_thread) { diff --git a/kbe/src/lib/dependencies/jemalloc/test/src/mtx.c b/kbe/src/lib/dependencies/jemalloc/test/src/mtx.c index a393c01fc6..d9ce375c35 100755 --- a/kbe/src/lib/dependencies/jemalloc/test/src/mtx.c +++ b/kbe/src/lib/dependencies/jemalloc/test/src/mtx.c @@ -13,8 +13,6 @@ mtx_init(mtx_t *mtx) { } #elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) mtx->lock = OS_UNFAIR_LOCK_INIT; -#elif (defined(JEMALLOC_OSSPIN)) - mtx->lock = 0; #else pthread_mutexattr_t attr; @@ -35,7 +33,6 @@ void mtx_fini(mtx_t *mtx) { #ifdef _WIN32 #elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) -#elif (defined(JEMALLOC_OSSPIN)) #else pthread_mutex_destroy(&mtx->lock); #endif @@ -47,8 +44,6 @@ mtx_lock(mtx_t *mtx) { EnterCriticalSection(&mtx->lock); #elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) os_unfair_lock_lock(&mtx->lock); -#elif (defined(JEMALLOC_OSSPIN)) - OSSpinLockLock(&mtx->lock); #else pthread_mutex_lock(&mtx->lock); #endif @@ -60,8 +55,6 @@ mtx_unlock(mtx_t *mtx) { LeaveCriticalSection(&mtx->lock); #elif (defined(JEMALLOC_OS_UNFAIR_LOCK)) os_unfair_lock_unlock(&mtx->lock); -#elif (defined(JEMALLOC_OSSPIN)) - OSSpinLockUnlock(&mtx->lock); #else pthread_mutex_unlock(&mtx->lock); #endif diff --git a/kbe/src/lib/dependencies/jemalloc/test/src/test.c b/kbe/src/lib/dependencies/jemalloc/test/src/test.c index 01a4d7380b..f97ce4d18d 100755 --- a/kbe/src/lib/dependencies/jemalloc/test/src/test.c +++ b/kbe/src/lib/dependencies/jemalloc/test/src/test.c @@ -48,12 +48,12 @@ do_hook(bool *hook_ran, void (**hook)()) { static void libc_reentrancy_hook() { - do_hook(&libc_hook_ran, &hooks_libc_hook); + do_hook(&libc_hook_ran, &test_hooks_libc_hook); } static void arena_new_reentrancy_hook() { - do_hook(&arena_new_hook_ran, &hooks_arena_new_hook); + do_hook(&arena_new_hook_ran, &test_hooks_arena_new_hook); } /* Actual test infrastructure. */ @@ -110,6 +110,20 @@ p_test_fini(void) { test_status_string(test_status)); } +static void +check_global_slow(test_status_t *status) { +#ifdef JEMALLOC_UNIT_TEST + /* + * This check needs to peek into tsd internals, which is why it's only + * exposed in unit tests. + */ + if (tsd_global_slow()) { + malloc_printf("Testing increased global slow count\n"); + *status = test_status_fail; + } +#endif +} + static test_status_t p_test_impl(bool do_malloc_init, bool do_reentrant, test_t *t, va_list ap) { test_status_t ret; @@ -131,28 +145,31 @@ p_test_impl(bool do_malloc_init, bool do_reentrant, test_t *t, va_list ap) { for (; t != NULL; t = va_arg(ap, test_t *)) { /* Non-reentrant run. */ reentrancy = non_reentrant; - hooks_arena_new_hook = hooks_libc_hook = NULL; + test_hooks_arena_new_hook = test_hooks_libc_hook = NULL; t(); if (test_status > ret) { ret = test_status; } + check_global_slow(&ret); /* Reentrant run. */ if (do_reentrant) { reentrancy = libc_reentrant; - hooks_arena_new_hook = NULL; - hooks_libc_hook = &libc_reentrancy_hook; + test_hooks_arena_new_hook = NULL; + test_hooks_libc_hook = &libc_reentrancy_hook; t(); if (test_status > ret) { ret = test_status; } + check_global_slow(&ret); reentrancy = arena_new_reentrant; - hooks_libc_hook = NULL; - hooks_arena_new_hook = &arena_new_reentrancy_hook; + test_hooks_libc_hook = NULL; + test_hooks_arena_new_hook = &arena_new_reentrancy_hook; t(); if (test_status > ret) { ret = test_status; } + check_global_slow(&ret); } } diff --git a/kbe/src/lib/dependencies/jemalloc/test/stress/hookbench.c b/kbe/src/lib/dependencies/jemalloc/test/stress/hookbench.c new file mode 100644 index 0000000000..97e90b0e35 --- /dev/null +++ b/kbe/src/lib/dependencies/jemalloc/test/stress/hookbench.c @@ -0,0 +1,73 @@ +#include "test/jemalloc_test.h" + +static void +noop_alloc_hook(void *extra, hook_alloc_t type, void *result, + uintptr_t result_raw, uintptr_t args_raw[3]) { +} + +static void +noop_dalloc_hook(void *extra, hook_dalloc_t type, void *address, + uintptr_t args_raw[3]) { +} + +static void +noop_expand_hook(void *extra, hook_expand_t type, void *address, + size_t old_usize, size_t new_usize, uintptr_t result_raw, + uintptr_t args_raw[4]) { +} + +static void +malloc_free_loop(int iters) { + for (int i = 0; i < iters; i++) { + void *p = mallocx(1, 0); + free(p); + } +} + +static void +test_hooked(int iters) { + hooks_t hooks = {&noop_alloc_hook, &noop_dalloc_hook, &noop_expand_hook, + NULL}; + + int err; + void *handles[HOOK_MAX]; + size_t sz = sizeof(handles[0]); + + for (int i = 0; i < HOOK_MAX; i++) { + err = mallctl("experimental.hooks.install", &handles[i], + &sz, &hooks, sizeof(hooks)); + assert(err == 0); + + timedelta_t timer; + timer_start(&timer); + malloc_free_loop(iters); + timer_stop(&timer); + malloc_printf("With %d hook%s: %"FMTu64"us\n", i + 1, + i + 1 == 1 ? "" : "s", timer_usec(&timer)); + } + for (int i = 0; i < HOOK_MAX; i++) { + err = mallctl("experimental.hooks.remove", NULL, NULL, + &handles[i], sizeof(handles[i])); + assert(err == 0); + } +} + +static void +test_unhooked(int iters) { + timedelta_t timer; + timer_start(&timer); + malloc_free_loop(iters); + timer_stop(&timer); + + malloc_printf("Without hooks: %"FMTu64"us\n", timer_usec(&timer)); +} + +int +main(void) { + /* Initialize */ + free(mallocx(1, 0)); + int iters = 10 * 1000 * 1000; + malloc_printf("Benchmarking hooks with %d iterations:\n", iters); + test_hooked(iters); + test_unhooked(iters); +} diff --git a/kbe/src/lib/dependencies/jemalloc/test/unit/arena_reset.c b/kbe/src/lib/dependencies/jemalloc/test/unit/arena_reset.c index f5fb24d1ed..96b042ddea 100755 --- a/kbe/src/lib/dependencies/jemalloc/test/unit/arena_reset.c +++ b/kbe/src/lib/dependencies/jemalloc/test/unit/arena_reset.c @@ -77,7 +77,7 @@ vsalloc(tsdn_t *tsdn, const void *ptr) { return 0; } - if (szind == NSIZES) { + if (szind == SC_NSIZES) { return 0; } @@ -142,7 +142,7 @@ do_arena_reset_post(void **ptrs, unsigned nptrs, unsigned arena_ind) { if (have_background_thread) { malloc_mutex_lock(tsdn, - &background_thread_info[arena_ind % ncpus].mtx); + &background_thread_info_get(arena_ind)->mtx); } /* Verify allocations no longer exist. */ for (i = 0; i < nptrs; i++) { @@ -151,7 +151,7 @@ do_arena_reset_post(void **ptrs, unsigned nptrs, unsigned arena_ind) { } if (have_background_thread) { malloc_mutex_unlock(tsdn, - &background_thread_info[arena_ind % ncpus].mtx); + &background_thread_info_get(arena_ind)->mtx); } free(ptrs); diff --git a/kbe/src/lib/dependencies/jemalloc/test/unit/background_thread_enable.c b/kbe/src/lib/dependencies/jemalloc/test/unit/background_thread_enable.c new file mode 100644 index 0000000000..d894e93716 --- /dev/null +++ b/kbe/src/lib/dependencies/jemalloc/test/unit/background_thread_enable.c @@ -0,0 +1,85 @@ +#include "test/jemalloc_test.h" + +const char *malloc_conf = "background_thread:false,narenas:1,max_background_threads:20"; + +TEST_BEGIN(test_deferred) { + test_skip_if(!have_background_thread); + + unsigned id; + size_t sz_u = sizeof(unsigned); + + /* + * 10 here is somewhat arbitrary, except insofar as we want to ensure + * that the number of background threads is smaller than the number of + * arenas. I'll ragequit long before we have to spin up 10 threads per + * cpu to handle background purging, so this is a conservative + * approximation. + */ + for (unsigned i = 0; i < 10 * ncpus; i++) { + assert_d_eq(mallctl("arenas.create", &id, &sz_u, NULL, 0), 0, + "Failed to create arena"); + } + + bool enable = true; + size_t sz_b = sizeof(bool); + assert_d_eq(mallctl("background_thread", NULL, NULL, &enable, sz_b), 0, + "Failed to enable background threads"); + enable = false; + assert_d_eq(mallctl("background_thread", NULL, NULL, &enable, sz_b), 0, + "Failed to disable background threads"); +} +TEST_END + +TEST_BEGIN(test_max_background_threads) { + test_skip_if(!have_background_thread); + + size_t max_n_thds; + size_t opt_max_n_thds; + size_t sz_m = sizeof(max_n_thds); + assert_d_eq(mallctl("opt.max_background_threads", + &opt_max_n_thds, &sz_m, NULL, 0), 0, + "Failed to get opt.max_background_threads"); + assert_d_eq(mallctl("max_background_threads", &max_n_thds, &sz_m, NULL, + 0), 0, "Failed to get max background threads"); + assert_zu_eq(opt_max_n_thds, max_n_thds, + "max_background_threads and " + "opt.max_background_threads should match"); + assert_d_eq(mallctl("max_background_threads", NULL, NULL, &max_n_thds, + sz_m), 0, "Failed to set max background threads"); + + unsigned id; + size_t sz_u = sizeof(unsigned); + + for (unsigned i = 0; i < 10 * ncpus; i++) { + assert_d_eq(mallctl("arenas.create", &id, &sz_u, NULL, 0), 0, + "Failed to create arena"); + } + + bool enable = true; + size_t sz_b = sizeof(bool); + assert_d_eq(mallctl("background_thread", NULL, NULL, &enable, sz_b), 0, + "Failed to enable background threads"); + assert_zu_eq(n_background_threads, max_n_thds, + "Number of background threads should not change.\n"); + size_t new_max_thds = max_n_thds - 1; + if (new_max_thds > 0) { + assert_d_eq(mallctl("max_background_threads", NULL, NULL, + &new_max_thds, sz_m), 0, + "Failed to set max background threads"); + assert_zu_eq(n_background_threads, new_max_thds, + "Number of background threads should decrease by 1.\n"); + } + new_max_thds = 1; + assert_d_eq(mallctl("max_background_threads", NULL, NULL, &new_max_thds, + sz_m), 0, "Failed to set max background threads"); + assert_zu_eq(n_background_threads, new_max_thds, + "Number of background threads should be 1.\n"); +} +TEST_END + +int +main(void) { + return test_no_reentrancy( + test_deferred, + test_max_background_threads); +} diff --git a/kbe/src/lib/dependencies/jemalloc/test/unit/base.c b/kbe/src/lib/dependencies/jemalloc/test/unit/base.c index 7fa24ac035..6b792cf21a 100755 --- a/kbe/src/lib/dependencies/jemalloc/test/unit/base.c +++ b/kbe/src/lib/dependencies/jemalloc/test/unit/base.c @@ -28,22 +28,28 @@ static extent_hooks_t hooks_not_null = { TEST_BEGIN(test_base_hooks_default) { base_t *base; - size_t allocated0, allocated1, resident, mapped; + size_t allocated0, allocated1, resident, mapped, n_thp; tsdn_t *tsdn = tsd_tsdn(tsd_fetch()); base = base_new(tsdn, 0, (extent_hooks_t *)&extent_hooks_default); if (config_stats) { - base_stats_get(tsdn, base, &allocated0, &resident, &mapped); + base_stats_get(tsdn, base, &allocated0, &resident, &mapped, + &n_thp); assert_zu_ge(allocated0, sizeof(base_t), "Base header should count as allocated"); + if (opt_metadata_thp == metadata_thp_always) { + assert_zu_gt(n_thp, 0, + "Base should have 1 THP at least."); + } } assert_ptr_not_null(base_alloc(tsdn, base, 42, 1), "Unexpected base_alloc() failure"); if (config_stats) { - base_stats_get(tsdn, base, &allocated1, &resident, &mapped); + base_stats_get(tsdn, base, &allocated1, &resident, &mapped, + &n_thp); assert_zu_ge(allocated1 - allocated0, 42, "At least 42 bytes were allocated by base_alloc()"); } @@ -55,7 +61,7 @@ TEST_END TEST_BEGIN(test_base_hooks_null) { extent_hooks_t hooks_orig; base_t *base; - size_t allocated0, allocated1, resident, mapped; + size_t allocated0, allocated1, resident, mapped, n_thp; extent_hooks_prep(); try_dalloc = false; @@ -71,16 +77,22 @@ TEST_BEGIN(test_base_hooks_null) { assert_ptr_not_null(base, "Unexpected base_new() failure"); if (config_stats) { - base_stats_get(tsdn, base, &allocated0, &resident, &mapped); + base_stats_get(tsdn, base, &allocated0, &resident, &mapped, + &n_thp); assert_zu_ge(allocated0, sizeof(base_t), "Base header should count as allocated"); + if (opt_metadata_thp == metadata_thp_always) { + assert_zu_gt(n_thp, 0, + "Base should have 1 THP at least."); + } } assert_ptr_not_null(base_alloc(tsdn, base, 42, 1), "Unexpected base_alloc() failure"); if (config_stats) { - base_stats_get(tsdn, base, &allocated1, &resident, &mapped); + base_stats_get(tsdn, base, &allocated1, &resident, &mapped, + &n_thp); assert_zu_ge(allocated1 - allocated0, 42, "At least 42 bytes were allocated by base_alloc()"); } diff --git a/kbe/src/lib/dependencies/jemalloc/test/unit/binshard.c b/kbe/src/lib/dependencies/jemalloc/test/unit/binshard.c new file mode 100644 index 0000000000..d7a8df8fc0 --- /dev/null +++ b/kbe/src/lib/dependencies/jemalloc/test/unit/binshard.c @@ -0,0 +1,154 @@ +#include "test/jemalloc_test.h" + +/* Config -- "narenas:1,bin_shards:1-160:16|129-512:4|256-256:8" */ + +#define NTHREADS 16 +#define REMOTE_NALLOC 256 + +static void * +thd_producer(void *varg) { + void **mem = varg; + unsigned arena, i; + size_t sz; + + sz = sizeof(arena); + /* Remote arena. */ + assert_d_eq(mallctl("arenas.create", (void *)&arena, &sz, NULL, 0), 0, + "Unexpected mallctl() failure"); + for (i = 0; i < REMOTE_NALLOC / 2; i++) { + mem[i] = mallocx(1, MALLOCX_TCACHE_NONE | MALLOCX_ARENA(arena)); + } + + /* Remote bin. */ + for (; i < REMOTE_NALLOC; i++) { + mem[i] = mallocx(1, MALLOCX_TCACHE_NONE | MALLOCX_ARENA(0)); + } + + return NULL; +} + +TEST_BEGIN(test_producer_consumer) { + thd_t thds[NTHREADS]; + void *mem[NTHREADS][REMOTE_NALLOC]; + unsigned i; + + /* Create producer threads to allocate. */ + for (i = 0; i < NTHREADS; i++) { + thd_create(&thds[i], thd_producer, mem[i]); + } + for (i = 0; i < NTHREADS; i++) { + thd_join(thds[i], NULL); + } + /* Remote deallocation by the current thread. */ + for (i = 0; i < NTHREADS; i++) { + for (unsigned j = 0; j < REMOTE_NALLOC; j++) { + assert_ptr_not_null(mem[i][j], + "Unexpected remote allocation failure"); + dallocx(mem[i][j], 0); + } + } +} +TEST_END + +static void * +thd_start(void *varg) { + void *ptr, *ptr2; + extent_t *extent; + unsigned shard1, shard2; + + tsdn_t *tsdn = tsdn_fetch(); + /* Try triggering allocations from sharded bins. */ + for (unsigned i = 0; i < 1024; i++) { + ptr = mallocx(1, MALLOCX_TCACHE_NONE); + ptr2 = mallocx(129, MALLOCX_TCACHE_NONE); + + extent = iealloc(tsdn, ptr); + shard1 = extent_binshard_get(extent); + dallocx(ptr, 0); + assert_u_lt(shard1, 16, "Unexpected bin shard used"); + + extent = iealloc(tsdn, ptr2); + shard2 = extent_binshard_get(extent); + dallocx(ptr2, 0); + assert_u_lt(shard2, 4, "Unexpected bin shard used"); + + if (shard1 > 0 || shard2 > 0) { + /* Triggered sharded bin usage. */ + return (void *)(uintptr_t)shard1; + } + } + + return NULL; +} + +TEST_BEGIN(test_bin_shard_mt) { + test_skip_if(have_percpu_arena && + PERCPU_ARENA_ENABLED(opt_percpu_arena)); + + thd_t thds[NTHREADS]; + unsigned i; + for (i = 0; i < NTHREADS; i++) { + thd_create(&thds[i], thd_start, NULL); + } + bool sharded = false; + for (i = 0; i < NTHREADS; i++) { + void *ret; + thd_join(thds[i], &ret); + if (ret != NULL) { + sharded = true; + } + } + assert_b_eq(sharded, true, "Did not find sharded bins"); +} +TEST_END + +TEST_BEGIN(test_bin_shard) { + unsigned nbins, i; + size_t mib[4], mib2[4]; + size_t miblen, miblen2, len; + + len = sizeof(nbins); + assert_d_eq(mallctl("arenas.nbins", (void *)&nbins, &len, NULL, 0), 0, + "Unexpected mallctl() failure"); + + miblen = 4; + assert_d_eq(mallctlnametomib("arenas.bin.0.nshards", mib, &miblen), 0, + "Unexpected mallctlnametomib() failure"); + miblen2 = 4; + assert_d_eq(mallctlnametomib("arenas.bin.0.size", mib2, &miblen2), 0, + "Unexpected mallctlnametomib() failure"); + + for (i = 0; i < nbins; i++) { + uint32_t nshards; + size_t size, sz1, sz2; + + mib[2] = i; + sz1 = sizeof(nshards); + assert_d_eq(mallctlbymib(mib, miblen, (void *)&nshards, &sz1, + NULL, 0), 0, "Unexpected mallctlbymib() failure"); + + mib2[2] = i; + sz2 = sizeof(size); + assert_d_eq(mallctlbymib(mib2, miblen2, (void *)&size, &sz2, + NULL, 0), 0, "Unexpected mallctlbymib() failure"); + + if (size >= 1 && size <= 128) { + assert_u_eq(nshards, 16, "Unexpected nshards"); + } else if (size == 256) { + assert_u_eq(nshards, 8, "Unexpected nshards"); + } else if (size > 128 && size <= 512) { + assert_u_eq(nshards, 4, "Unexpected nshards"); + } else { + assert_u_eq(nshards, 1, "Unexpected nshards"); + } + } +} +TEST_END + +int +main(void) { + return test_no_reentrancy( + test_bin_shard, + test_bin_shard_mt, + test_producer_consumer); +} diff --git a/kbe/src/lib/dependencies/jemalloc/test/unit/binshard.sh b/kbe/src/lib/dependencies/jemalloc/test/unit/binshard.sh new file mode 100644 index 0000000000..c1d58c8837 --- /dev/null +++ b/kbe/src/lib/dependencies/jemalloc/test/unit/binshard.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +export MALLOC_CONF="narenas:1,bin_shards:1-160:16|129-512:4|256-256:8" diff --git a/kbe/src/lib/dependencies/jemalloc/test/unit/bit_util.c b/kbe/src/lib/dependencies/jemalloc/test/unit/bit_util.c index 42a97013d0..b747deb433 100755 --- a/kbe/src/lib/dependencies/jemalloc/test/unit/bit_util.c +++ b/kbe/src/lib/dependencies/jemalloc/test/unit/bit_util.c @@ -48,10 +48,64 @@ TEST_BEGIN(test_pow2_ceil_zu) { } TEST_END +void +assert_lg_ceil_range(size_t input, unsigned answer) { + if (input == 1) { + assert_u_eq(0, answer, "Got %u as lg_ceil of 1", answer); + return; + } + assert_zu_le(input, (ZU(1) << answer), + "Got %u as lg_ceil of %zu", answer, input); + assert_zu_gt(input, (ZU(1) << (answer - 1)), + "Got %u as lg_ceil of %zu", answer, input); +} + +void +assert_lg_floor_range(size_t input, unsigned answer) { + if (input == 1) { + assert_u_eq(0, answer, "Got %u as lg_floor of 1", answer); + return; + } + assert_zu_ge(input, (ZU(1) << answer), + "Got %u as lg_floor of %zu", answer, input); + assert_zu_lt(input, (ZU(1) << (answer + 1)), + "Got %u as lg_floor of %zu", answer, input); +} + +TEST_BEGIN(test_lg_ceil_floor) { + for (size_t i = 1; i < 10 * 1000 * 1000; i++) { + assert_lg_ceil_range(i, lg_ceil(i)); + assert_lg_ceil_range(i, LG_CEIL(i)); + assert_lg_floor_range(i, lg_floor(i)); + assert_lg_floor_range(i, LG_FLOOR(i)); + } + for (int i = 10; i < 8 * (1 << LG_SIZEOF_PTR) - 5; i++) { + for (size_t j = 0; j < (1 << 4); j++) { + size_t num1 = ((size_t)1 << i) + - j * ((size_t)1 << (i - 4)); + size_t num2 = ((size_t)1 << i) + + j * ((size_t)1 << (i - 4)); + assert_zu_ne(num1, 0, "Invalid lg argument"); + assert_zu_ne(num2, 0, "Invalid lg argument"); + assert_lg_ceil_range(num1, lg_ceil(num1)); + assert_lg_ceil_range(num1, LG_CEIL(num1)); + assert_lg_ceil_range(num2, lg_ceil(num2)); + assert_lg_ceil_range(num2, LG_CEIL(num2)); + + assert_lg_floor_range(num1, lg_floor(num1)); + assert_lg_floor_range(num1, LG_FLOOR(num1)); + assert_lg_floor_range(num2, lg_floor(num2)); + assert_lg_floor_range(num2, LG_FLOOR(num2)); + } + } +} +TEST_END + int main(void) { return test( test_pow2_ceil_u64, test_pow2_ceil_u32, - test_pow2_ceil_zu); + test_pow2_ceil_zu, + test_lg_ceil_floor); } diff --git a/kbe/src/lib/dependencies/jemalloc/test/unit/decay.c b/kbe/src/lib/dependencies/jemalloc/test/unit/decay.c index f727bf9316..cf3c07960e 100755 --- a/kbe/src/lib/dependencies/jemalloc/test/unit/decay.c +++ b/kbe/src/lib/dependencies/jemalloc/test/unit/decay.c @@ -121,6 +121,12 @@ get_arena_dirty_npurge(unsigned arena_ind) { return get_arena_npurge_impl("stats.arenas.0.dirty_npurge", arena_ind); } +static uint64_t +get_arena_dirty_purged(unsigned arena_ind) { + do_epoch(); + return get_arena_npurge_impl("stats.arenas.0.dirty_purged", arena_ind); +} + static uint64_t get_arena_muzzy_npurge(unsigned arena_ind) { do_epoch(); @@ -559,7 +565,7 @@ TEST_BEGIN(test_decay_now) { TEST_END TEST_BEGIN(test_decay_never) { - test_skip_if(check_background_thread_enabled()); + test_skip_if(check_background_thread_enabled() || !config_stats); unsigned arena_ind = do_arena_create(-1, -1); int flags = MALLOCX_ARENA(arena_ind) | MALLOCX_TCACHE_NONE; @@ -579,8 +585,8 @@ TEST_BEGIN(test_decay_never) { dallocx(ptrs[i], flags); size_t pdirty = get_arena_pdirty(arena_ind); size_t pmuzzy = get_arena_pmuzzy(arena_ind); - assert_zu_gt(pdirty, pdirty_prev, - "Expected dirty pages to increase."); + assert_zu_gt(pdirty + (size_t)get_arena_dirty_purged(arena_ind), + pdirty_prev, "Expected dirty pages to increase."); assert_zu_eq(pmuzzy, 0, "Unexpected muzzy pages"); pdirty_prev = pdirty; } diff --git a/kbe/src/lib/dependencies/jemalloc/test/unit/div.c b/kbe/src/lib/dependencies/jemalloc/test/unit/div.c new file mode 100644 index 0000000000..b47f10b2ba --- /dev/null +++ b/kbe/src/lib/dependencies/jemalloc/test/unit/div.c @@ -0,0 +1,29 @@ +#include "test/jemalloc_test.h" + +#include "jemalloc/internal/div.h" + +TEST_BEGIN(test_div_exhaustive) { + for (size_t divisor = 2; divisor < 1000 * 1000; ++divisor) { + div_info_t div_info; + div_init(&div_info, divisor); + size_t max = 1000 * divisor; + if (max < 1000 * 1000) { + max = 1000 * 1000; + } + for (size_t dividend = 0; dividend < 1000 * divisor; + dividend += divisor) { + size_t quotient = div_compute( + &div_info, dividend); + assert_zu_eq(dividend, quotient * divisor, + "With divisor = %zu, dividend = %zu, " + "got quotient %zu", divisor, dividend, quotient); + } + } +} +TEST_END + +int +main(void) { + return test_no_reentrancy( + test_div_exhaustive); +} diff --git a/kbe/src/lib/dependencies/jemalloc/test/unit/emitter.c b/kbe/src/lib/dependencies/jemalloc/test/unit/emitter.c new file mode 100644 index 0000000000..b4a693f4b3 --- /dev/null +++ b/kbe/src/lib/dependencies/jemalloc/test/unit/emitter.c @@ -0,0 +1,469 @@ +#include "test/jemalloc_test.h" +#include "jemalloc/internal/emitter.h" + +/* + * This is so useful for debugging and feature work, we'll leave printing + * functionality committed but disabled by default. + */ +/* Print the text as it will appear. */ +static bool print_raw = false; +/* Print the text escaped, so it can be copied back into the test case. */ +static bool print_escaped = false; + +typedef struct buf_descriptor_s buf_descriptor_t; +struct buf_descriptor_s { + char *buf; + size_t len; + bool mid_quote; +}; + +/* + * Forwards all writes to the passed-in buf_v (which should be cast from a + * buf_descriptor_t *). + */ +static void +forwarding_cb(void *buf_descriptor_v, const char *str) { + buf_descriptor_t *buf_descriptor = (buf_descriptor_t *)buf_descriptor_v; + + if (print_raw) { + malloc_printf("%s", str); + } + if (print_escaped) { + const char *it = str; + while (*it != '\0') { + if (!buf_descriptor->mid_quote) { + malloc_printf("\""); + buf_descriptor->mid_quote = true; + } + switch (*it) { + case '\\': + malloc_printf("\\"); + break; + case '\"': + malloc_printf("\\\""); + break; + case '\t': + malloc_printf("\\t"); + break; + case '\n': + malloc_printf("\\n\"\n"); + buf_descriptor->mid_quote = false; + break; + default: + malloc_printf("%c", *it); + } + it++; + } + } + + size_t written = malloc_snprintf(buf_descriptor->buf, + buf_descriptor->len, "%s", str); + assert_zu_eq(written, strlen(str), "Buffer overflow!"); + buf_descriptor->buf += written; + buf_descriptor->len -= written; + assert_zu_gt(buf_descriptor->len, 0, "Buffer out of space!"); +} + +static void +assert_emit_output(void (*emit_fn)(emitter_t *), + const char *expected_json_output, const char *expected_table_output) { + emitter_t emitter; + char buf[MALLOC_PRINTF_BUFSIZE]; + buf_descriptor_t buf_descriptor; + + buf_descriptor.buf = buf; + buf_descriptor.len = MALLOC_PRINTF_BUFSIZE; + buf_descriptor.mid_quote = false; + + emitter_init(&emitter, emitter_output_json, &forwarding_cb, + &buf_descriptor); + (*emit_fn)(&emitter); + assert_str_eq(expected_json_output, buf, "json output failure"); + + buf_descriptor.buf = buf; + buf_descriptor.len = MALLOC_PRINTF_BUFSIZE; + buf_descriptor.mid_quote = false; + + emitter_init(&emitter, emitter_output_table, &forwarding_cb, + &buf_descriptor); + (*emit_fn)(&emitter); + assert_str_eq(expected_table_output, buf, "table output failure"); +} + +static void +emit_dict(emitter_t *emitter) { + bool b_false = false; + bool b_true = true; + int i_123 = 123; + const char *str = "a string"; + + emitter_begin(emitter); + emitter_dict_begin(emitter, "foo", "This is the foo table:"); + emitter_kv(emitter, "abc", "ABC", emitter_type_bool, &b_false); + emitter_kv(emitter, "def", "DEF", emitter_type_bool, &b_true); + emitter_kv_note(emitter, "ghi", "GHI", emitter_type_int, &i_123, + "note_key1", emitter_type_string, &str); + emitter_kv_note(emitter, "jkl", "JKL", emitter_type_string, &str, + "note_key2", emitter_type_bool, &b_false); + emitter_dict_end(emitter); + emitter_end(emitter); +} +static const char *dict_json = +"{\n" +"\t\"foo\": {\n" +"\t\t\"abc\": false,\n" +"\t\t\"def\": true,\n" +"\t\t\"ghi\": 123,\n" +"\t\t\"jkl\": \"a string\"\n" +"\t}\n" +"}\n"; +static const char *dict_table = +"This is the foo table:\n" +" ABC: false\n" +" DEF: true\n" +" GHI: 123 (note_key1: \"a string\")\n" +" JKL: \"a string\" (note_key2: false)\n"; + +TEST_BEGIN(test_dict) { + assert_emit_output(&emit_dict, dict_json, dict_table); +} +TEST_END + +static void +emit_table_printf(emitter_t *emitter) { + emitter_begin(emitter); + emitter_table_printf(emitter, "Table note 1\n"); + emitter_table_printf(emitter, "Table note 2 %s\n", + "with format string"); + emitter_end(emitter); +} + +static const char *table_printf_json = +"{\n" +"}\n"; + +static const char *table_printf_table = +"Table note 1\n" +"Table note 2 with format string\n"; + +TEST_BEGIN(test_table_printf) { + assert_emit_output(&emit_table_printf, table_printf_json, + table_printf_table); +} +TEST_END + +static void emit_nested_dict(emitter_t *emitter) { + int val = 123; + emitter_begin(emitter); + emitter_dict_begin(emitter, "json1", "Dict 1"); + emitter_dict_begin(emitter, "json2", "Dict 2"); + emitter_kv(emitter, "primitive", "A primitive", emitter_type_int, &val); + emitter_dict_end(emitter); /* Close 2 */ + emitter_dict_begin(emitter, "json3", "Dict 3"); + emitter_dict_end(emitter); /* Close 3 */ + emitter_dict_end(emitter); /* Close 1 */ + emitter_dict_begin(emitter, "json4", "Dict 4"); + emitter_kv(emitter, "primitive", "Another primitive", + emitter_type_int, &val); + emitter_dict_end(emitter); /* Close 4 */ + emitter_end(emitter); +} + +static const char *nested_object_json = +"{\n" +"\t\"json1\": {\n" +"\t\t\"json2\": {\n" +"\t\t\t\"primitive\": 123\n" +"\t\t},\n" +"\t\t\"json3\": {\n" +"\t\t}\n" +"\t},\n" +"\t\"json4\": {\n" +"\t\t\"primitive\": 123\n" +"\t}\n" +"}\n"; + +static const char *nested_object_table = +"Dict 1\n" +" Dict 2\n" +" A primitive: 123\n" +" Dict 3\n" +"Dict 4\n" +" Another primitive: 123\n"; + +TEST_BEGIN(test_nested_dict) { + assert_emit_output(&emit_nested_dict, nested_object_json, + nested_object_table); +} +TEST_END + +static void +emit_types(emitter_t *emitter) { + bool b = false; + int i = -123; + unsigned u = 123; + ssize_t zd = -456; + size_t zu = 456; + const char *str = "string"; + uint32_t u32 = 789; + uint64_t u64 = 10000000000ULL; + + emitter_begin(emitter); + emitter_kv(emitter, "k1", "K1", emitter_type_bool, &b); + emitter_kv(emitter, "k2", "K2", emitter_type_int, &i); + emitter_kv(emitter, "k3", "K3", emitter_type_unsigned, &u); + emitter_kv(emitter, "k4", "K4", emitter_type_ssize, &zd); + emitter_kv(emitter, "k5", "K5", emitter_type_size, &zu); + emitter_kv(emitter, "k6", "K6", emitter_type_string, &str); + emitter_kv(emitter, "k7", "K7", emitter_type_uint32, &u32); + emitter_kv(emitter, "k8", "K8", emitter_type_uint64, &u64); + /* + * We don't test the title type, since it's only used for tables. It's + * tested in the emitter_table_row tests. + */ + emitter_end(emitter); +} + +static const char *types_json = +"{\n" +"\t\"k1\": false,\n" +"\t\"k2\": -123,\n" +"\t\"k3\": 123,\n" +"\t\"k4\": -456,\n" +"\t\"k5\": 456,\n" +"\t\"k6\": \"string\",\n" +"\t\"k7\": 789,\n" +"\t\"k8\": 10000000000\n" +"}\n"; + +static const char *types_table = +"K1: false\n" +"K2: -123\n" +"K3: 123\n" +"K4: -456\n" +"K5: 456\n" +"K6: \"string\"\n" +"K7: 789\n" +"K8: 10000000000\n"; + +TEST_BEGIN(test_types) { + assert_emit_output(&emit_types, types_json, types_table); +} +TEST_END + +static void +emit_modal(emitter_t *emitter) { + int val = 123; + emitter_begin(emitter); + emitter_dict_begin(emitter, "j0", "T0"); + emitter_json_key(emitter, "j1"); + emitter_json_object_begin(emitter); + emitter_kv(emitter, "i1", "I1", emitter_type_int, &val); + emitter_json_kv(emitter, "i2", emitter_type_int, &val); + emitter_table_kv(emitter, "I3", emitter_type_int, &val); + emitter_table_dict_begin(emitter, "T1"); + emitter_kv(emitter, "i4", "I4", emitter_type_int, &val); + emitter_json_object_end(emitter); /* Close j1 */ + emitter_kv(emitter, "i5", "I5", emitter_type_int, &val); + emitter_table_dict_end(emitter); /* Close T1 */ + emitter_kv(emitter, "i6", "I6", emitter_type_int, &val); + emitter_dict_end(emitter); /* Close j0 / T0 */ + emitter_end(emitter); +} + +const char *modal_json = +"{\n" +"\t\"j0\": {\n" +"\t\t\"j1\": {\n" +"\t\t\t\"i1\": 123,\n" +"\t\t\t\"i2\": 123,\n" +"\t\t\t\"i4\": 123\n" +"\t\t},\n" +"\t\t\"i5\": 123,\n" +"\t\t\"i6\": 123\n" +"\t}\n" +"}\n"; + +const char *modal_table = +"T0\n" +" I1: 123\n" +" I3: 123\n" +" T1\n" +" I4: 123\n" +" I5: 123\n" +" I6: 123\n"; + +TEST_BEGIN(test_modal) { + assert_emit_output(&emit_modal, modal_json, modal_table); +} +TEST_END + +static void +emit_json_arr(emitter_t *emitter) { + int ival = 123; + + emitter_begin(emitter); + emitter_json_key(emitter, "dict"); + emitter_json_object_begin(emitter); + emitter_json_key(emitter, "arr"); + emitter_json_array_begin(emitter); + emitter_json_object_begin(emitter); + emitter_json_kv(emitter, "foo", emitter_type_int, &ival); + emitter_json_object_end(emitter); /* Close arr[0] */ + /* arr[1] and arr[2] are primitives. */ + emitter_json_value(emitter, emitter_type_int, &ival); + emitter_json_value(emitter, emitter_type_int, &ival); + emitter_json_object_begin(emitter); + emitter_json_kv(emitter, "bar", emitter_type_int, &ival); + emitter_json_kv(emitter, "baz", emitter_type_int, &ival); + emitter_json_object_end(emitter); /* Close arr[3]. */ + emitter_json_array_end(emitter); /* Close arr. */ + emitter_json_object_end(emitter); /* Close dict. */ + emitter_end(emitter); +} + +static const char *json_array_json = +"{\n" +"\t\"dict\": {\n" +"\t\t\"arr\": [\n" +"\t\t\t{\n" +"\t\t\t\t\"foo\": 123\n" +"\t\t\t},\n" +"\t\t\t123,\n" +"\t\t\t123,\n" +"\t\t\t{\n" +"\t\t\t\t\"bar\": 123,\n" +"\t\t\t\t\"baz\": 123\n" +"\t\t\t}\n" +"\t\t]\n" +"\t}\n" +"}\n"; + +static const char *json_array_table = ""; + +TEST_BEGIN(test_json_arr) { + assert_emit_output(&emit_json_arr, json_array_json, json_array_table); +} +TEST_END + +static void +emit_json_nested_array(emitter_t *emitter) { + int ival = 123; + char *sval = "foo"; + emitter_begin(emitter); + emitter_json_array_begin(emitter); + emitter_json_array_begin(emitter); + emitter_json_value(emitter, emitter_type_int, &ival); + emitter_json_value(emitter, emitter_type_string, &sval); + emitter_json_value(emitter, emitter_type_int, &ival); + emitter_json_value(emitter, emitter_type_string, &sval); + emitter_json_array_end(emitter); + emitter_json_array_begin(emitter); + emitter_json_value(emitter, emitter_type_int, &ival); + emitter_json_array_end(emitter); + emitter_json_array_begin(emitter); + emitter_json_value(emitter, emitter_type_string, &sval); + emitter_json_value(emitter, emitter_type_int, &ival); + emitter_json_array_end(emitter); + emitter_json_array_begin(emitter); + emitter_json_array_end(emitter); + emitter_json_array_end(emitter); + emitter_end(emitter); +} + +static const char *json_nested_array_json = +"{\n" +"\t[\n" +"\t\t[\n" +"\t\t\t123,\n" +"\t\t\t\"foo\",\n" +"\t\t\t123,\n" +"\t\t\t\"foo\"\n" +"\t\t],\n" +"\t\t[\n" +"\t\t\t123\n" +"\t\t],\n" +"\t\t[\n" +"\t\t\t\"foo\",\n" +"\t\t\t123\n" +"\t\t],\n" +"\t\t[\n" +"\t\t]\n" +"\t]\n" +"}\n"; + +TEST_BEGIN(test_json_nested_arr) { + assert_emit_output(&emit_json_nested_array, json_nested_array_json, + json_array_table); +} +TEST_END + +static void +emit_table_row(emitter_t *emitter) { + emitter_begin(emitter); + emitter_row_t row; + emitter_col_t abc = {emitter_justify_left, 10, emitter_type_title, {0}, {0, 0}}; + abc.str_val = "ABC title"; + emitter_col_t def = {emitter_justify_right, 15, emitter_type_title, {0}, {0, 0}}; + def.str_val = "DEF title"; + emitter_col_t ghi = {emitter_justify_right, 5, emitter_type_title, {0}, {0, 0}}; + ghi.str_val = "GHI"; + + emitter_row_init(&row); + emitter_col_init(&abc, &row); + emitter_col_init(&def, &row); + emitter_col_init(&ghi, &row); + + emitter_table_row(emitter, &row); + + abc.type = emitter_type_int; + def.type = emitter_type_bool; + ghi.type = emitter_type_int; + + abc.int_val = 123; + def.bool_val = true; + ghi.int_val = 456; + emitter_table_row(emitter, &row); + + abc.int_val = 789; + def.bool_val = false; + ghi.int_val = 1011; + emitter_table_row(emitter, &row); + + abc.type = emitter_type_string; + abc.str_val = "a string"; + def.bool_val = false; + ghi.type = emitter_type_title; + ghi.str_val = "ghi"; + emitter_table_row(emitter, &row); + + emitter_end(emitter); +} + +static const char *table_row_json = +"{\n" +"}\n"; + +static const char *table_row_table = +"ABC title DEF title GHI\n" +"123 true 456\n" +"789 false 1011\n" +"\"a string\" false ghi\n"; + +TEST_BEGIN(test_table_row) { + assert_emit_output(&emit_table_row, table_row_json, table_row_table); +} +TEST_END + +int +main(void) { + return test_no_reentrancy( + test_dict, + test_table_printf, + test_nested_dict, + test_types, + test_modal, + test_json_arr, + test_json_nested_arr, + test_table_row); +} diff --git a/kbe/src/lib/dependencies/jemalloc/test/unit/fork.c b/kbe/src/lib/dependencies/jemalloc/test/unit/fork.c index afe2214139..b1690750ad 100755 --- a/kbe/src/lib/dependencies/jemalloc/test/unit/fork.c +++ b/kbe/src/lib/dependencies/jemalloc/test/unit/fork.c @@ -4,6 +4,30 @@ #include #endif +#ifndef _WIN32 +static void +wait_for_child_exit(int pid) { + int status; + while (true) { + if (waitpid(pid, &status, 0) == -1) { + test_fail("Unexpected waitpid() failure."); + } + if (WIFSIGNALED(status)) { + test_fail("Unexpected child termination due to " + "signal %d", WTERMSIG(status)); + break; + } + if (WIFEXITED(status)) { + if (WEXITSTATUS(status) != 0) { + test_fail("Unexpected child exit value %d", + WEXITSTATUS(status)); + } + break; + } + } +} +#endif + TEST_BEGIN(test_fork) { #ifndef _WIN32 void *p; @@ -40,26 +64,67 @@ TEST_BEGIN(test_fork) { /* Child. */ _exit(0); } else { - int status; + wait_for_child_exit(pid); + } +#else + test_skip("fork(2) is irrelevant to Windows"); +#endif +} +TEST_END - /* Parent. */ - while (true) { - if (waitpid(pid, &status, 0) == -1) { - test_fail("Unexpected waitpid() failure"); - } - if (WIFSIGNALED(status)) { - test_fail("Unexpected child termination due to " - "signal %d", WTERMSIG(status)); - break; - } - if (WIFEXITED(status)) { - if (WEXITSTATUS(status) != 0) { - test_fail( - "Unexpected child exit value %d", - WEXITSTATUS(status)); - } - break; - } +#ifndef _WIN32 +static void * +do_fork_thd(void *arg) { + malloc(1); + int pid = fork(); + if (pid == -1) { + /* Error. */ + test_fail("Unexpected fork() failure"); + } else if (pid == 0) { + /* Child. */ + char *args[] = {"true", NULL}; + execvp(args[0], args); + test_fail("Exec failed"); + } else { + /* Parent */ + wait_for_child_exit(pid); + } + return NULL; +} +#endif + +#ifndef _WIN32 +static void +do_test_fork_multithreaded() { + thd_t child; + thd_create(&child, do_fork_thd, NULL); + do_fork_thd(NULL); + thd_join(child, NULL); +} +#endif + +TEST_BEGIN(test_fork_multithreaded) { +#ifndef _WIN32 + /* + * We've seen bugs involving hanging on arenas_lock (though the same + * class of bugs can happen on any mutex). The bugs are intermittent + * though, so we want to run the test multiple times. Since we hold the + * arenas lock only early in the process lifetime, we can't just run + * this test in a loop (since, after all the arenas are initialized, we + * won't acquire arenas_lock any further). We therefore repeat the test + * with multiple processes. + */ + for (int i = 0; i < 100; i++) { + int pid = fork(); + if (pid == -1) { + /* Error. */ + test_fail("Unexpected fork() failure,"); + } else if (pid == 0) { + /* Child. */ + do_test_fork_multithreaded(); + _exit(0); + } else { + wait_for_child_exit(pid); } } #else @@ -70,6 +135,7 @@ TEST_END int main(void) { - return test( - test_fork); + return test_no_reentrancy( + test_fork, + test_fork_multithreaded); } diff --git a/kbe/src/lib/dependencies/jemalloc/test/unit/hook.c b/kbe/src/lib/dependencies/jemalloc/test/unit/hook.c new file mode 100644 index 0000000000..72fcc433cc --- /dev/null +++ b/kbe/src/lib/dependencies/jemalloc/test/unit/hook.c @@ -0,0 +1,580 @@ +#include "test/jemalloc_test.h" + +#include "jemalloc/internal/hook.h" + +static void *arg_extra; +static int arg_type; +static void *arg_result; +static void *arg_address; +static size_t arg_old_usize; +static size_t arg_new_usize; +static uintptr_t arg_result_raw; +static uintptr_t arg_args_raw[4]; + +static int call_count = 0; + +static void +reset_args() { + arg_extra = NULL; + arg_type = 12345; + arg_result = NULL; + arg_address = NULL; + arg_old_usize = 0; + arg_new_usize = 0; + arg_result_raw = 0; + memset(arg_args_raw, 77, sizeof(arg_args_raw)); +} + +static void +alloc_free_size(size_t sz) { + void *ptr = mallocx(1, 0); + free(ptr); + ptr = mallocx(1, 0); + free(ptr); + ptr = mallocx(1, MALLOCX_TCACHE_NONE); + dallocx(ptr, MALLOCX_TCACHE_NONE); +} + +/* + * We want to support a degree of user reentrancy. This tests a variety of + * allocation scenarios. + */ +static void +be_reentrant() { + /* Let's make sure the tcache is non-empty if enabled. */ + alloc_free_size(1); + alloc_free_size(1024); + alloc_free_size(64 * 1024); + alloc_free_size(256 * 1024); + alloc_free_size(1024 * 1024); + + /* Some reallocation. */ + void *ptr = mallocx(129, 0); + ptr = rallocx(ptr, 130, 0); + free(ptr); + + ptr = mallocx(2 * 1024 * 1024, 0); + free(ptr); + ptr = mallocx(1 * 1024 * 1024, 0); + ptr = rallocx(ptr, 2 * 1024 * 1024, 0); + free(ptr); + + ptr = mallocx(1, 0); + ptr = rallocx(ptr, 1000, 0); + free(ptr); +} + +static void +set_args_raw(uintptr_t *args_raw, int nargs) { + memcpy(arg_args_raw, args_raw, sizeof(uintptr_t) * nargs); +} + +static void +assert_args_raw(uintptr_t *args_raw_expected, int nargs) { + int cmp = memcmp(args_raw_expected, arg_args_raw, + sizeof(uintptr_t) * nargs); + assert_d_eq(cmp, 0, "Raw args mismatch"); +} + +static void +reset() { + call_count = 0; + reset_args(); +} + +static void +test_alloc_hook(void *extra, hook_alloc_t type, void *result, + uintptr_t result_raw, uintptr_t args_raw[3]) { + call_count++; + arg_extra = extra; + arg_type = (int)type; + arg_result = result; + arg_result_raw = result_raw; + set_args_raw(args_raw, 3); + be_reentrant(); +} + +static void +test_dalloc_hook(void *extra, hook_dalloc_t type, void *address, + uintptr_t args_raw[3]) { + call_count++; + arg_extra = extra; + arg_type = (int)type; + arg_address = address; + set_args_raw(args_raw, 3); + be_reentrant(); +} + +static void +test_expand_hook(void *extra, hook_expand_t type, void *address, + size_t old_usize, size_t new_usize, uintptr_t result_raw, + uintptr_t args_raw[4]) { + call_count++; + arg_extra = extra; + arg_type = (int)type; + arg_address = address; + arg_old_usize = old_usize; + arg_new_usize = new_usize; + arg_result_raw = result_raw; + set_args_raw(args_raw, 4); + be_reentrant(); +} + +TEST_BEGIN(test_hooks_basic) { + /* Just verify that the record their arguments correctly. */ + hooks_t hooks = { + &test_alloc_hook, &test_dalloc_hook, &test_expand_hook, + (void *)111}; + void *handle = hook_install(TSDN_NULL, &hooks); + uintptr_t args_raw[4] = {10, 20, 30, 40}; + + /* Alloc */ + reset_args(); + hook_invoke_alloc(hook_alloc_posix_memalign, (void *)222, 333, + args_raw); + assert_ptr_eq(arg_extra, (void *)111, "Passed wrong user pointer"); + assert_d_eq((int)hook_alloc_posix_memalign, arg_type, + "Passed wrong alloc type"); + assert_ptr_eq((void *)222, arg_result, "Passed wrong result address"); + assert_u64_eq(333, arg_result_raw, "Passed wrong result"); + assert_args_raw(args_raw, 3); + + /* Dalloc */ + reset_args(); + hook_invoke_dalloc(hook_dalloc_sdallocx, (void *)222, args_raw); + assert_d_eq((int)hook_dalloc_sdallocx, arg_type, + "Passed wrong dalloc type"); + assert_ptr_eq((void *)111, arg_extra, "Passed wrong user pointer"); + assert_ptr_eq((void *)222, arg_address, "Passed wrong address"); + assert_args_raw(args_raw, 3); + + /* Expand */ + reset_args(); + hook_invoke_expand(hook_expand_xallocx, (void *)222, 333, 444, 555, + args_raw); + assert_d_eq((int)hook_expand_xallocx, arg_type, + "Passed wrong expand type"); + assert_ptr_eq((void *)111, arg_extra, "Passed wrong user pointer"); + assert_ptr_eq((void *)222, arg_address, "Passed wrong address"); + assert_zu_eq(333, arg_old_usize, "Passed wrong old usize"); + assert_zu_eq(444, arg_new_usize, "Passed wrong new usize"); + assert_zu_eq(555, arg_result_raw, "Passed wrong result"); + assert_args_raw(args_raw, 4); + + hook_remove(TSDN_NULL, handle); +} +TEST_END + +TEST_BEGIN(test_hooks_null) { + /* Null hooks should be ignored, not crash. */ + hooks_t hooks1 = {NULL, NULL, NULL, NULL}; + hooks_t hooks2 = {&test_alloc_hook, NULL, NULL, NULL}; + hooks_t hooks3 = {NULL, &test_dalloc_hook, NULL, NULL}; + hooks_t hooks4 = {NULL, NULL, &test_expand_hook, NULL}; + + void *handle1 = hook_install(TSDN_NULL, &hooks1); + void *handle2 = hook_install(TSDN_NULL, &hooks2); + void *handle3 = hook_install(TSDN_NULL, &hooks3); + void *handle4 = hook_install(TSDN_NULL, &hooks4); + + assert_ptr_ne(handle1, NULL, "Hook installation failed"); + assert_ptr_ne(handle2, NULL, "Hook installation failed"); + assert_ptr_ne(handle3, NULL, "Hook installation failed"); + assert_ptr_ne(handle4, NULL, "Hook installation failed"); + + uintptr_t args_raw[4] = {10, 20, 30, 40}; + + call_count = 0; + hook_invoke_alloc(hook_alloc_malloc, NULL, 0, args_raw); + assert_d_eq(call_count, 1, "Called wrong number of times"); + + call_count = 0; + hook_invoke_dalloc(hook_dalloc_free, NULL, args_raw); + assert_d_eq(call_count, 1, "Called wrong number of times"); + + call_count = 0; + hook_invoke_expand(hook_expand_realloc, NULL, 0, 0, 0, args_raw); + assert_d_eq(call_count, 1, "Called wrong number of times"); + + hook_remove(TSDN_NULL, handle1); + hook_remove(TSDN_NULL, handle2); + hook_remove(TSDN_NULL, handle3); + hook_remove(TSDN_NULL, handle4); +} +TEST_END + +TEST_BEGIN(test_hooks_remove) { + hooks_t hooks = {&test_alloc_hook, NULL, NULL, NULL}; + void *handle = hook_install(TSDN_NULL, &hooks); + assert_ptr_ne(handle, NULL, "Hook installation failed"); + call_count = 0; + uintptr_t args_raw[4] = {10, 20, 30, 40}; + hook_invoke_alloc(hook_alloc_malloc, NULL, 0, args_raw); + assert_d_eq(call_count, 1, "Hook not invoked"); + + call_count = 0; + hook_remove(TSDN_NULL, handle); + hook_invoke_alloc(hook_alloc_malloc, NULL, 0, NULL); + assert_d_eq(call_count, 0, "Hook invoked after removal"); + +} +TEST_END + +TEST_BEGIN(test_hooks_alloc_simple) { + /* "Simple" in the sense that we're not in a realloc variant. */ + hooks_t hooks = {&test_alloc_hook, NULL, NULL, (void *)123}; + void *handle = hook_install(TSDN_NULL, &hooks); + assert_ptr_ne(handle, NULL, "Hook installation failed"); + + /* Stop malloc from being optimized away. */ + volatile int err; + void *volatile ptr; + + /* malloc */ + reset(); + ptr = malloc(1); + assert_d_eq(call_count, 1, "Hook not called"); + assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); + assert_d_eq(arg_type, (int)hook_alloc_malloc, "Wrong hook type"); + assert_ptr_eq(ptr, arg_result, "Wrong result"); + assert_u64_eq((uintptr_t)ptr, (uintptr_t)arg_result_raw, + "Wrong raw result"); + assert_u64_eq((uintptr_t)1, arg_args_raw[0], "Wrong argument"); + free(ptr); + + /* posix_memalign */ + reset(); + err = posix_memalign((void **)&ptr, 1024, 1); + assert_d_eq(call_count, 1, "Hook not called"); + assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); + assert_d_eq(arg_type, (int)hook_alloc_posix_memalign, + "Wrong hook type"); + assert_ptr_eq(ptr, arg_result, "Wrong result"); + assert_u64_eq((uintptr_t)err, (uintptr_t)arg_result_raw, + "Wrong raw result"); + assert_u64_eq((uintptr_t)&ptr, arg_args_raw[0], "Wrong argument"); + assert_u64_eq((uintptr_t)1024, arg_args_raw[1], "Wrong argument"); + assert_u64_eq((uintptr_t)1, arg_args_raw[2], "Wrong argument"); + free(ptr); + + /* aligned_alloc */ + reset(); + ptr = aligned_alloc(1024, 1); + assert_d_eq(call_count, 1, "Hook not called"); + assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); + assert_d_eq(arg_type, (int)hook_alloc_aligned_alloc, + "Wrong hook type"); + assert_ptr_eq(ptr, arg_result, "Wrong result"); + assert_u64_eq((uintptr_t)ptr, (uintptr_t)arg_result_raw, + "Wrong raw result"); + assert_u64_eq((uintptr_t)1024, arg_args_raw[0], "Wrong argument"); + assert_u64_eq((uintptr_t)1, arg_args_raw[1], "Wrong argument"); + free(ptr); + + /* calloc */ + reset(); + ptr = calloc(11, 13); + assert_d_eq(call_count, 1, "Hook not called"); + assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); + assert_d_eq(arg_type, (int)hook_alloc_calloc, "Wrong hook type"); + assert_ptr_eq(ptr, arg_result, "Wrong result"); + assert_u64_eq((uintptr_t)ptr, (uintptr_t)arg_result_raw, + "Wrong raw result"); + assert_u64_eq((uintptr_t)11, arg_args_raw[0], "Wrong argument"); + assert_u64_eq((uintptr_t)13, arg_args_raw[1], "Wrong argument"); + free(ptr); + + /* memalign */ +#ifdef JEMALLOC_OVERRIDE_MEMALIGN + reset(); + ptr = memalign(1024, 1); + assert_d_eq(call_count, 1, "Hook not called"); + assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); + assert_d_eq(arg_type, (int)hook_alloc_memalign, "Wrong hook type"); + assert_ptr_eq(ptr, arg_result, "Wrong result"); + assert_u64_eq((uintptr_t)ptr, (uintptr_t)arg_result_raw, + "Wrong raw result"); + assert_u64_eq((uintptr_t)1024, arg_args_raw[0], "Wrong argument"); + assert_u64_eq((uintptr_t)1, arg_args_raw[1], "Wrong argument"); + free(ptr); +#endif /* JEMALLOC_OVERRIDE_MEMALIGN */ + + /* valloc */ +#ifdef JEMALLOC_OVERRIDE_VALLOC + reset(); + ptr = valloc(1); + assert_d_eq(call_count, 1, "Hook not called"); + assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); + assert_d_eq(arg_type, (int)hook_alloc_valloc, "Wrong hook type"); + assert_ptr_eq(ptr, arg_result, "Wrong result"); + assert_u64_eq((uintptr_t)ptr, (uintptr_t)arg_result_raw, + "Wrong raw result"); + assert_u64_eq((uintptr_t)1, arg_args_raw[0], "Wrong argument"); + free(ptr); +#endif /* JEMALLOC_OVERRIDE_VALLOC */ + + /* mallocx */ + reset(); + ptr = mallocx(1, MALLOCX_LG_ALIGN(10)); + assert_d_eq(call_count, 1, "Hook not called"); + assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); + assert_d_eq(arg_type, (int)hook_alloc_mallocx, "Wrong hook type"); + assert_ptr_eq(ptr, arg_result, "Wrong result"); + assert_u64_eq((uintptr_t)ptr, (uintptr_t)arg_result_raw, + "Wrong raw result"); + assert_u64_eq((uintptr_t)1, arg_args_raw[0], "Wrong argument"); + assert_u64_eq((uintptr_t)MALLOCX_LG_ALIGN(10), arg_args_raw[1], + "Wrong flags"); + free(ptr); + + hook_remove(TSDN_NULL, handle); +} +TEST_END + +TEST_BEGIN(test_hooks_dalloc_simple) { + /* "Simple" in the sense that we're not in a realloc variant. */ + hooks_t hooks = {NULL, &test_dalloc_hook, NULL, (void *)123}; + void *handle = hook_install(TSDN_NULL, &hooks); + assert_ptr_ne(handle, NULL, "Hook installation failed"); + + void *volatile ptr; + + /* free() */ + reset(); + ptr = malloc(1); + free(ptr); + assert_d_eq(call_count, 1, "Hook not called"); + assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); + assert_d_eq(arg_type, (int)hook_dalloc_free, "Wrong hook type"); + assert_ptr_eq(ptr, arg_address, "Wrong pointer freed"); + assert_u64_eq((uintptr_t)ptr, arg_args_raw[0], "Wrong raw arg"); + + /* dallocx() */ + reset(); + ptr = malloc(1); + dallocx(ptr, MALLOCX_TCACHE_NONE); + assert_d_eq(call_count, 1, "Hook not called"); + assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); + assert_d_eq(arg_type, (int)hook_dalloc_dallocx, "Wrong hook type"); + assert_ptr_eq(ptr, arg_address, "Wrong pointer freed"); + assert_u64_eq((uintptr_t)ptr, arg_args_raw[0], "Wrong raw arg"); + assert_u64_eq((uintptr_t)MALLOCX_TCACHE_NONE, arg_args_raw[1], + "Wrong raw arg"); + + /* sdallocx() */ + reset(); + ptr = malloc(1); + sdallocx(ptr, 1, MALLOCX_TCACHE_NONE); + assert_d_eq(call_count, 1, "Hook not called"); + assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); + assert_d_eq(arg_type, (int)hook_dalloc_sdallocx, "Wrong hook type"); + assert_ptr_eq(ptr, arg_address, "Wrong pointer freed"); + assert_u64_eq((uintptr_t)ptr, arg_args_raw[0], "Wrong raw arg"); + assert_u64_eq((uintptr_t)1, arg_args_raw[1], "Wrong raw arg"); + assert_u64_eq((uintptr_t)MALLOCX_TCACHE_NONE, arg_args_raw[2], + "Wrong raw arg"); + + hook_remove(TSDN_NULL, handle); +} +TEST_END + +TEST_BEGIN(test_hooks_expand_simple) { + /* "Simple" in the sense that we're not in a realloc variant. */ + hooks_t hooks = {NULL, NULL, &test_expand_hook, (void *)123}; + void *handle = hook_install(TSDN_NULL, &hooks); + assert_ptr_ne(handle, NULL, "Hook installation failed"); + + void *volatile ptr; + + /* xallocx() */ + reset(); + ptr = malloc(1); + size_t new_usize = xallocx(ptr, 100, 200, MALLOCX_TCACHE_NONE); + assert_d_eq(call_count, 1, "Hook not called"); + assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); + assert_d_eq(arg_type, (int)hook_expand_xallocx, "Wrong hook type"); + assert_ptr_eq(ptr, arg_address, "Wrong pointer expanded"); + assert_u64_eq(arg_old_usize, nallocx(1, 0), "Wrong old usize"); + assert_u64_eq(arg_new_usize, sallocx(ptr, 0), "Wrong new usize"); + assert_u64_eq(new_usize, arg_result_raw, "Wrong result"); + assert_u64_eq((uintptr_t)ptr, arg_args_raw[0], "Wrong arg"); + assert_u64_eq(100, arg_args_raw[1], "Wrong arg"); + assert_u64_eq(200, arg_args_raw[2], "Wrong arg"); + assert_u64_eq(MALLOCX_TCACHE_NONE, arg_args_raw[3], "Wrong arg"); + + hook_remove(TSDN_NULL, handle); +} +TEST_END + +TEST_BEGIN(test_hooks_realloc_as_malloc_or_free) { + hooks_t hooks = {&test_alloc_hook, &test_dalloc_hook, + &test_expand_hook, (void *)123}; + void *handle = hook_install(TSDN_NULL, &hooks); + assert_ptr_ne(handle, NULL, "Hook installation failed"); + + void *volatile ptr; + + /* realloc(NULL, size) as malloc */ + reset(); + ptr = realloc(NULL, 1); + assert_d_eq(call_count, 1, "Hook not called"); + assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); + assert_d_eq(arg_type, (int)hook_alloc_realloc, "Wrong hook type"); + assert_ptr_eq(ptr, arg_result, "Wrong result"); + assert_u64_eq((uintptr_t)ptr, (uintptr_t)arg_result_raw, + "Wrong raw result"); + assert_u64_eq((uintptr_t)NULL, arg_args_raw[0], "Wrong argument"); + assert_u64_eq((uintptr_t)1, arg_args_raw[1], "Wrong argument"); + free(ptr); + + /* realloc(ptr, 0) as free */ + ptr = malloc(1); + reset(); + realloc(ptr, 0); + assert_d_eq(call_count, 1, "Hook not called"); + assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); + assert_d_eq(arg_type, (int)hook_dalloc_realloc, "Wrong hook type"); + assert_ptr_eq(ptr, arg_address, "Wrong pointer freed"); + assert_u64_eq((uintptr_t)ptr, arg_args_raw[0], "Wrong raw arg"); + assert_u64_eq((uintptr_t)0, arg_args_raw[1], "Wrong raw arg"); + + /* realloc(NULL, 0) as malloc(0) */ + reset(); + ptr = realloc(NULL, 0); + assert_d_eq(call_count, 1, "Hook not called"); + assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); + assert_d_eq(arg_type, (int)hook_alloc_realloc, "Wrong hook type"); + assert_ptr_eq(ptr, arg_result, "Wrong result"); + assert_u64_eq((uintptr_t)ptr, (uintptr_t)arg_result_raw, + "Wrong raw result"); + assert_u64_eq((uintptr_t)NULL, arg_args_raw[0], "Wrong argument"); + assert_u64_eq((uintptr_t)0, arg_args_raw[1], "Wrong argument"); + free(ptr); + + hook_remove(TSDN_NULL, handle); +} +TEST_END + +static void +do_realloc_test(void *(*ralloc)(void *, size_t, int), int flags, + int expand_type, int dalloc_type) { + hooks_t hooks = {&test_alloc_hook, &test_dalloc_hook, + &test_expand_hook, (void *)123}; + void *handle = hook_install(TSDN_NULL, &hooks); + assert_ptr_ne(handle, NULL, "Hook installation failed"); + + void *volatile ptr; + void *volatile ptr2; + + /* Realloc in-place, small. */ + ptr = malloc(129); + reset(); + ptr2 = ralloc(ptr, 130, flags); + assert_ptr_eq(ptr, ptr2, "Small realloc moved"); + + assert_d_eq(call_count, 1, "Hook not called"); + assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); + assert_d_eq(arg_type, expand_type, "Wrong hook type"); + assert_ptr_eq(ptr, arg_address, "Wrong address"); + assert_u64_eq((uintptr_t)ptr, (uintptr_t)arg_result_raw, + "Wrong raw result"); + assert_u64_eq((uintptr_t)ptr, arg_args_raw[0], "Wrong argument"); + assert_u64_eq((uintptr_t)130, arg_args_raw[1], "Wrong argument"); + free(ptr); + + /* + * Realloc in-place, large. Since we can't guarantee the large case + * across all platforms, we stay resilient to moving results. + */ + ptr = malloc(2 * 1024 * 1024); + free(ptr); + ptr2 = malloc(1 * 1024 * 1024); + reset(); + ptr = ralloc(ptr2, 2 * 1024 * 1024, flags); + /* ptr is the new address, ptr2 is the old address. */ + if (ptr == ptr2) { + assert_d_eq(call_count, 1, "Hook not called"); + assert_d_eq(arg_type, expand_type, "Wrong hook type"); + } else { + assert_d_eq(call_count, 2, "Wrong hooks called"); + assert_ptr_eq(ptr, arg_result, "Wrong address"); + assert_d_eq(arg_type, dalloc_type, "Wrong hook type"); + } + assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); + assert_ptr_eq(ptr2, arg_address, "Wrong address"); + assert_u64_eq((uintptr_t)ptr, (uintptr_t)arg_result_raw, + "Wrong raw result"); + assert_u64_eq((uintptr_t)ptr2, arg_args_raw[0], "Wrong argument"); + assert_u64_eq((uintptr_t)2 * 1024 * 1024, arg_args_raw[1], + "Wrong argument"); + free(ptr); + + /* Realloc with move, small. */ + ptr = malloc(8); + reset(); + ptr2 = ralloc(ptr, 128, flags); + assert_ptr_ne(ptr, ptr2, "Small realloc didn't move"); + + assert_d_eq(call_count, 2, "Hook not called"); + assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); + assert_d_eq(arg_type, dalloc_type, "Wrong hook type"); + assert_ptr_eq(ptr, arg_address, "Wrong address"); + assert_ptr_eq(ptr2, arg_result, "Wrong address"); + assert_u64_eq((uintptr_t)ptr2, (uintptr_t)arg_result_raw, + "Wrong raw result"); + assert_u64_eq((uintptr_t)ptr, arg_args_raw[0], "Wrong argument"); + assert_u64_eq((uintptr_t)128, arg_args_raw[1], "Wrong argument"); + free(ptr2); + + /* Realloc with move, large. */ + ptr = malloc(1); + reset(); + ptr2 = ralloc(ptr, 2 * 1024 * 1024, flags); + assert_ptr_ne(ptr, ptr2, "Large realloc didn't move"); + + assert_d_eq(call_count, 2, "Hook not called"); + assert_ptr_eq(arg_extra, (void *)123, "Wrong extra"); + assert_d_eq(arg_type, dalloc_type, "Wrong hook type"); + assert_ptr_eq(ptr, arg_address, "Wrong address"); + assert_ptr_eq(ptr2, arg_result, "Wrong address"); + assert_u64_eq((uintptr_t)ptr2, (uintptr_t)arg_result_raw, + "Wrong raw result"); + assert_u64_eq((uintptr_t)ptr, arg_args_raw[0], "Wrong argument"); + assert_u64_eq((uintptr_t)2 * 1024 * 1024, arg_args_raw[1], + "Wrong argument"); + free(ptr2); + + hook_remove(TSDN_NULL, handle); +} + +static void * +realloc_wrapper(void *ptr, size_t size, UNUSED int flags) { + return realloc(ptr, size); +} + +TEST_BEGIN(test_hooks_realloc) { + do_realloc_test(&realloc_wrapper, 0, hook_expand_realloc, + hook_dalloc_realloc); +} +TEST_END + +TEST_BEGIN(test_hooks_rallocx) { + do_realloc_test(&rallocx, MALLOCX_TCACHE_NONE, hook_expand_rallocx, + hook_dalloc_rallocx); +} +TEST_END + +int +main(void) { + /* We assert on call counts. */ + return test_no_reentrancy( + test_hooks_basic, + test_hooks_null, + test_hooks_remove, + test_hooks_alloc_simple, + test_hooks_dalloc_simple, + test_hooks_expand_simple, + test_hooks_realloc_as_malloc_or_free, + test_hooks_realloc, + test_hooks_rallocx); +} diff --git a/kbe/src/lib/dependencies/jemalloc/test/unit/huge.c b/kbe/src/lib/dependencies/jemalloc/test/unit/huge.c new file mode 100644 index 0000000000..ab72cf0071 --- /dev/null +++ b/kbe/src/lib/dependencies/jemalloc/test/unit/huge.c @@ -0,0 +1,108 @@ +#include "test/jemalloc_test.h" + +/* Threshold: 2 << 20 = 2097152. */ +const char *malloc_conf = "oversize_threshold:2097152"; + +#define HUGE_SZ (2 << 20) +#define SMALL_SZ (8) + +TEST_BEGIN(huge_bind_thread) { + unsigned arena1, arena2; + size_t sz = sizeof(unsigned); + + /* Bind to a manual arena. */ + assert_d_eq(mallctl("arenas.create", &arena1, &sz, NULL, 0), 0, + "Failed to create arena"); + assert_d_eq(mallctl("thread.arena", NULL, NULL, &arena1, + sizeof(arena1)), 0, "Fail to bind thread"); + + void *ptr = mallocx(HUGE_SZ, 0); + assert_ptr_not_null(ptr, "Fail to allocate huge size"); + assert_d_eq(mallctl("arenas.lookup", &arena2, &sz, &ptr, + sizeof(ptr)), 0, "Unexpected mallctl() failure"); + assert_u_eq(arena1, arena2, "Wrong arena used after binding"); + dallocx(ptr, 0); + + /* Switch back to arena 0. */ + test_skip_if(have_percpu_arena && + PERCPU_ARENA_ENABLED(opt_percpu_arena)); + arena2 = 0; + assert_d_eq(mallctl("thread.arena", NULL, NULL, &arena2, + sizeof(arena2)), 0, "Fail to bind thread"); + ptr = mallocx(SMALL_SZ, MALLOCX_TCACHE_NONE); + assert_d_eq(mallctl("arenas.lookup", &arena2, &sz, &ptr, + sizeof(ptr)), 0, "Unexpected mallctl() failure"); + assert_u_eq(arena2, 0, "Wrong arena used after binding"); + dallocx(ptr, MALLOCX_TCACHE_NONE); + + /* Then huge allocation should use the huge arena. */ + ptr = mallocx(HUGE_SZ, 0); + assert_ptr_not_null(ptr, "Fail to allocate huge size"); + assert_d_eq(mallctl("arenas.lookup", &arena2, &sz, &ptr, + sizeof(ptr)), 0, "Unexpected mallctl() failure"); + assert_u_ne(arena2, 0, "Wrong arena used after binding"); + assert_u_ne(arena1, arena2, "Wrong arena used after binding"); + dallocx(ptr, 0); +} +TEST_END + +TEST_BEGIN(huge_mallocx) { + unsigned arena1, arena2; + size_t sz = sizeof(unsigned); + + assert_d_eq(mallctl("arenas.create", &arena1, &sz, NULL, 0), 0, + "Failed to create arena"); + void *huge = mallocx(HUGE_SZ, MALLOCX_ARENA(arena1)); + assert_ptr_not_null(huge, "Fail to allocate huge size"); + assert_d_eq(mallctl("arenas.lookup", &arena2, &sz, &huge, + sizeof(huge)), 0, "Unexpected mallctl() failure"); + assert_u_eq(arena1, arena2, "Wrong arena used for mallocx"); + dallocx(huge, MALLOCX_ARENA(arena1)); + + void *huge2 = mallocx(HUGE_SZ, 0); + assert_ptr_not_null(huge, "Fail to allocate huge size"); + assert_d_eq(mallctl("arenas.lookup", &arena2, &sz, &huge2, + sizeof(huge2)), 0, "Unexpected mallctl() failure"); + assert_u_ne(arena1, arena2, + "Huge allocation should not come from the manual arena."); + assert_u_ne(arena2, 0, + "Huge allocation should not come from the arena 0."); + dallocx(huge2, 0); +} +TEST_END + +TEST_BEGIN(huge_allocation) { + unsigned arena1, arena2; + + void *ptr = mallocx(HUGE_SZ, 0); + assert_ptr_not_null(ptr, "Fail to allocate huge size"); + size_t sz = sizeof(unsigned); + assert_d_eq(mallctl("arenas.lookup", &arena1, &sz, &ptr, sizeof(ptr)), + 0, "Unexpected mallctl() failure"); + assert_u_gt(arena1, 0, "Huge allocation should not come from arena 0"); + dallocx(ptr, 0); + + ptr = mallocx(HUGE_SZ >> 1, 0); + assert_ptr_not_null(ptr, "Fail to allocate half huge size"); + assert_d_eq(mallctl("arenas.lookup", &arena2, &sz, &ptr, + sizeof(ptr)), 0, "Unexpected mallctl() failure"); + assert_u_ne(arena1, arena2, "Wrong arena used for half huge"); + dallocx(ptr, 0); + + ptr = mallocx(SMALL_SZ, MALLOCX_TCACHE_NONE); + assert_ptr_not_null(ptr, "Fail to allocate small size"); + assert_d_eq(mallctl("arenas.lookup", &arena2, &sz, &ptr, + sizeof(ptr)), 0, "Unexpected mallctl() failure"); + assert_u_ne(arena1, arena2, + "Huge and small should be from different arenas"); + dallocx(ptr, 0); +} +TEST_END + +int +main(void) { + return test( + huge_allocation, + huge_mallocx, + huge_bind_thread); +} diff --git a/kbe/src/lib/dependencies/jemalloc/test/unit/junk.c b/kbe/src/lib/dependencies/jemalloc/test/unit/junk.c index fd0e65b1cd..57e3ad431a 100755 --- a/kbe/src/lib/dependencies/jemalloc/test/unit/junk.c +++ b/kbe/src/lib/dependencies/jemalloc/test/unit/junk.c @@ -15,7 +15,7 @@ watch_junking(void *p) { } static void -arena_dalloc_junk_small_intercept(void *ptr, const arena_bin_info_t *bin_info) { +arena_dalloc_junk_small_intercept(void *ptr, const bin_info_t *bin_info) { size_t i; arena_dalloc_junk_small_orig(ptr, bin_info); @@ -123,13 +123,13 @@ test_junk(size_t sz_min, size_t sz_max) { TEST_BEGIN(test_junk_small) { test_skip_if(!config_fill); - test_junk(1, SMALL_MAXCLASS-1); + test_junk(1, SC_SMALL_MAXCLASS - 1); } TEST_END TEST_BEGIN(test_junk_large) { test_skip_if(!config_fill); - test_junk(SMALL_MAXCLASS+1, (1U << (LG_LARGE_MINCLASS+1))); + test_junk(SC_SMALL_MAXCLASS + 1, (1U << (SC_LG_LARGE_MINCLASS + 1))); } TEST_END diff --git a/kbe/src/lib/dependencies/jemalloc/test/unit/log.c b/kbe/src/lib/dependencies/jemalloc/test/unit/log.c new file mode 100644 index 0000000000..a52bd737d4 --- /dev/null +++ b/kbe/src/lib/dependencies/jemalloc/test/unit/log.c @@ -0,0 +1,193 @@ +#include "test/jemalloc_test.h" + +#include "jemalloc/internal/log.h" + +static void +expect_no_logging(const char *names) { + log_var_t log_l1 = LOG_VAR_INIT("l1"); + log_var_t log_l2 = LOG_VAR_INIT("l2"); + log_var_t log_l2_a = LOG_VAR_INIT("l2.a"); + + strcpy(log_var_names, names); + + int count = 0; + + for (int i = 0; i < 10; i++) { + log_do_begin(log_l1) + count++; + log_do_end(log_l1) + + log_do_begin(log_l2) + count++; + log_do_end(log_l2) + + log_do_begin(log_l2_a) + count++; + log_do_end(log_l2_a) + } + assert_d_eq(count, 0, "Disabled logging not ignored!"); +} + +TEST_BEGIN(test_log_disabled) { + test_skip_if(!config_log); + atomic_store_b(&log_init_done, true, ATOMIC_RELAXED); + expect_no_logging(""); + expect_no_logging("abc"); + expect_no_logging("a.b.c"); + expect_no_logging("l12"); + expect_no_logging("l123|a456|b789"); + expect_no_logging("|||"); +} +TEST_END + +TEST_BEGIN(test_log_enabled_direct) { + test_skip_if(!config_log); + atomic_store_b(&log_init_done, true, ATOMIC_RELAXED); + log_var_t log_l1 = LOG_VAR_INIT("l1"); + log_var_t log_l1_a = LOG_VAR_INIT("l1.a"); + log_var_t log_l2 = LOG_VAR_INIT("l2"); + + int count; + + count = 0; + strcpy(log_var_names, "l1"); + for (int i = 0; i < 10; i++) { + log_do_begin(log_l1) + count++; + log_do_end(log_l1) + } + assert_d_eq(count, 10, "Mis-logged!"); + + count = 0; + strcpy(log_var_names, "l1.a"); + for (int i = 0; i < 10; i++) { + log_do_begin(log_l1_a) + count++; + log_do_end(log_l1_a) + } + assert_d_eq(count, 10, "Mis-logged!"); + + count = 0; + strcpy(log_var_names, "l1.a|abc|l2|def"); + for (int i = 0; i < 10; i++) { + log_do_begin(log_l1_a) + count++; + log_do_end(log_l1_a) + + log_do_begin(log_l2) + count++; + log_do_end(log_l2) + } + assert_d_eq(count, 20, "Mis-logged!"); +} +TEST_END + +TEST_BEGIN(test_log_enabled_indirect) { + test_skip_if(!config_log); + atomic_store_b(&log_init_done, true, ATOMIC_RELAXED); + strcpy(log_var_names, "l0|l1|abc|l2.b|def"); + + /* On. */ + log_var_t log_l1 = LOG_VAR_INIT("l1"); + /* Off. */ + log_var_t log_l1a = LOG_VAR_INIT("l1a"); + /* On. */ + log_var_t log_l1_a = LOG_VAR_INIT("l1.a"); + /* Off. */ + log_var_t log_l2_a = LOG_VAR_INIT("l2.a"); + /* On. */ + log_var_t log_l2_b_a = LOG_VAR_INIT("l2.b.a"); + /* On. */ + log_var_t log_l2_b_b = LOG_VAR_INIT("l2.b.b"); + + /* 4 are on total, so should sum to 40. */ + int count = 0; + for (int i = 0; i < 10; i++) { + log_do_begin(log_l1) + count++; + log_do_end(log_l1) + + log_do_begin(log_l1a) + count++; + log_do_end(log_l1a) + + log_do_begin(log_l1_a) + count++; + log_do_end(log_l1_a) + + log_do_begin(log_l2_a) + count++; + log_do_end(log_l2_a) + + log_do_begin(log_l2_b_a) + count++; + log_do_end(log_l2_b_a) + + log_do_begin(log_l2_b_b) + count++; + log_do_end(log_l2_b_b) + } + + assert_d_eq(count, 40, "Mis-logged!"); +} +TEST_END + +TEST_BEGIN(test_log_enabled_global) { + test_skip_if(!config_log); + atomic_store_b(&log_init_done, true, ATOMIC_RELAXED); + strcpy(log_var_names, "abc|.|def"); + + log_var_t log_l1 = LOG_VAR_INIT("l1"); + log_var_t log_l2_a_a = LOG_VAR_INIT("l2.a.a"); + + int count = 0; + for (int i = 0; i < 10; i++) { + log_do_begin(log_l1) + count++; + log_do_end(log_l1) + + log_do_begin(log_l2_a_a) + count++; + log_do_end(log_l2_a_a) + } + assert_d_eq(count, 20, "Mis-logged!"); +} +TEST_END + +TEST_BEGIN(test_logs_if_no_init) { + test_skip_if(!config_log); + atomic_store_b(&log_init_done, false, ATOMIC_RELAXED); + + log_var_t l = LOG_VAR_INIT("definitely.not.enabled"); + + int count = 0; + for (int i = 0; i < 10; i++) { + log_do_begin(l) + count++; + log_do_end(l) + } + assert_d_eq(count, 0, "Logging shouldn't happen if not initialized."); +} +TEST_END + +/* + * This really just checks to make sure that this usage compiles; we don't have + * any test code to run. + */ +TEST_BEGIN(test_log_only_format_string) { + if (false) { + LOG("log_str", "No arguments follow this format string."); + } +} +TEST_END + +int +main(void) { + return test( + test_log_disabled, + test_log_enabled_direct, + test_log_enabled_indirect, + test_log_enabled_global, + test_logs_if_no_init, + test_log_only_format_string); +} diff --git a/kbe/src/lib/dependencies/jemalloc/test/unit/mallctl.c b/kbe/src/lib/dependencies/jemalloc/test/unit/mallctl.c index f611654946..498f9e06ac 100755 --- a/kbe/src/lib/dependencies/jemalloc/test/unit/mallctl.c +++ b/kbe/src/lib/dependencies/jemalloc/test/unit/mallctl.c @@ -1,5 +1,6 @@ #include "test/jemalloc_test.h" +#include "jemalloc/internal/hook.h" #include "jemalloc/internal/util.h" TEST_BEGIN(test_mallctl_errors) { @@ -157,10 +158,13 @@ TEST_BEGIN(test_mallctl_opt) { } while (0) TEST_MALLCTL_OPT(bool, abort, always); + TEST_MALLCTL_OPT(bool, abort_conf, always); + TEST_MALLCTL_OPT(const char *, metadata_thp, always); TEST_MALLCTL_OPT(bool, retain, always); TEST_MALLCTL_OPT(const char *, dss, always); TEST_MALLCTL_OPT(unsigned, narenas, always); TEST_MALLCTL_OPT(const char *, percpu_arena, always); + TEST_MALLCTL_OPT(size_t, oversize_threshold, always); TEST_MALLCTL_OPT(bool, background_thread, always); TEST_MALLCTL_OPT(ssize_t, dirty_decay_ms, always); TEST_MALLCTL_OPT(ssize_t, muzzy_decay_ms, always); @@ -170,7 +174,9 @@ TEST_BEGIN(test_mallctl_opt) { TEST_MALLCTL_OPT(bool, utrace, utrace); TEST_MALLCTL_OPT(bool, xmalloc, xmalloc); TEST_MALLCTL_OPT(bool, tcache, always); + TEST_MALLCTL_OPT(size_t, lg_extent_max_active_fit, always); TEST_MALLCTL_OPT(size_t, lg_tcache_max, always); + TEST_MALLCTL_OPT(const char *, thp, always); TEST_MALLCTL_OPT(bool, prof, prof); TEST_MALLCTL_OPT(const char *, prof_prefix, prof); TEST_MALLCTL_OPT(bool, prof_active, prof); @@ -330,12 +336,15 @@ TEST_BEGIN(test_thread_arena) { const char *opa; size_t sz = sizeof(opa); - assert_d_eq(mallctl("opt.percpu_arena", &opa, &sz, NULL, 0), 0, + assert_d_eq(mallctl("opt.percpu_arena", (void *)&opa, &sz, NULL, 0), 0, "Unexpected mallctl() failure"); sz = sizeof(unsigned); assert_d_eq(mallctl("arenas.narenas", (void *)&narenas, &sz, NULL, 0), 0, "Unexpected mallctl() failure"); + if (opt_oversize_threshold != 0) { + narenas--; + } assert_u_eq(narenas, opt_narenas, "Number of arenas incorrect"); if (strcmp(opa, "disabled") == 0) { @@ -554,6 +563,54 @@ TEST_BEGIN(test_arena_i_dss) { } TEST_END +TEST_BEGIN(test_arena_i_retain_grow_limit) { + size_t old_limit, new_limit, default_limit; + size_t mib[3]; + size_t miblen; + + bool retain_enabled; + size_t sz = sizeof(retain_enabled); + assert_d_eq(mallctl("opt.retain", &retain_enabled, &sz, NULL, 0), + 0, "Unexpected mallctl() failure"); + test_skip_if(!retain_enabled); + + sz = sizeof(default_limit); + miblen = sizeof(mib)/sizeof(size_t); + assert_d_eq(mallctlnametomib("arena.0.retain_grow_limit", mib, &miblen), + 0, "Unexpected mallctlnametomib() error"); + + assert_d_eq(mallctlbymib(mib, miblen, &default_limit, &sz, NULL, 0), 0, + "Unexpected mallctl() failure"); + assert_zu_eq(default_limit, SC_LARGE_MAXCLASS, + "Unexpected default for retain_grow_limit"); + + new_limit = PAGE - 1; + assert_d_eq(mallctlbymib(mib, miblen, NULL, NULL, &new_limit, + sizeof(new_limit)), EFAULT, "Unexpected mallctl() success"); + + new_limit = PAGE + 1; + assert_d_eq(mallctlbymib(mib, miblen, NULL, NULL, &new_limit, + sizeof(new_limit)), 0, "Unexpected mallctl() failure"); + assert_d_eq(mallctlbymib(mib, miblen, &old_limit, &sz, NULL, 0), 0, + "Unexpected mallctl() failure"); + assert_zu_eq(old_limit, PAGE, + "Unexpected value for retain_grow_limit"); + + /* Expect grow less than psize class 10. */ + new_limit = sz_pind2sz(10) - 1; + assert_d_eq(mallctlbymib(mib, miblen, NULL, NULL, &new_limit, + sizeof(new_limit)), 0, "Unexpected mallctl() failure"); + assert_d_eq(mallctlbymib(mib, miblen, &old_limit, &sz, NULL, 0), 0, + "Unexpected mallctl() failure"); + assert_zu_eq(old_limit, sz_pind2sz(9), + "Unexpected value for retain_grow_limit"); + + /* Restore to default. */ + assert_d_eq(mallctlbymib(mib, miblen, NULL, NULL, &default_limit, + sizeof(default_limit)), 0, "Unexpected mallctl() failure"); +} +TEST_END + TEST_BEGIN(test_arenas_dirty_decay_ms) { ssize_t dirty_decay_ms, orig_dirty_decay_ms, prev_dirty_decay_ms; size_t sz = sizeof(ssize_t); @@ -629,8 +686,8 @@ TEST_BEGIN(test_arenas_constants) { TEST_ARENAS_CONSTANT(size_t, quantum, QUANTUM); TEST_ARENAS_CONSTANT(size_t, page, PAGE); - TEST_ARENAS_CONSTANT(unsigned, nbins, NBINS); - TEST_ARENAS_CONSTANT(unsigned, nlextents, NSIZES - NBINS); + TEST_ARENAS_CONSTANT(unsigned, nbins, SC_NBINS); + TEST_ARENAS_CONSTANT(unsigned, nlextents, SC_NSIZES - SC_NBINS); #undef TEST_ARENAS_CONSTANT } @@ -645,10 +702,11 @@ TEST_BEGIN(test_arenas_bin_constants) { assert_zu_eq(name, expected, "Incorrect "#name" size"); \ } while (0) - TEST_ARENAS_BIN_CONSTANT(size_t, size, arena_bin_info[0].reg_size); - TEST_ARENAS_BIN_CONSTANT(uint32_t, nregs, arena_bin_info[0].nregs); + TEST_ARENAS_BIN_CONSTANT(size_t, size, bin_infos[0].reg_size); + TEST_ARENAS_BIN_CONSTANT(uint32_t, nregs, bin_infos[0].nregs); TEST_ARENAS_BIN_CONSTANT(size_t, slab_size, - arena_bin_info[0].slab_size); + bin_infos[0].slab_size); + TEST_ARENAS_BIN_CONSTANT(uint32_t, nshards, bin_infos[0].n_shards); #undef TEST_ARENAS_BIN_CONSTANT } @@ -663,7 +721,8 @@ TEST_BEGIN(test_arenas_lextent_constants) { assert_zu_eq(name, expected, "Incorrect "#name" size"); \ } while (0) - TEST_ARENAS_LEXTENT_CONSTANT(size_t, size, LARGE_MINCLASS); + TEST_ARENAS_LEXTENT_CONSTANT(size_t, size, + SC_LARGE_MINCLASS); #undef TEST_ARENAS_LEXTENT_CONSTANT } @@ -686,6 +745,22 @@ TEST_BEGIN(test_arenas_create) { } TEST_END +TEST_BEGIN(test_arenas_lookup) { + unsigned arena, arena1; + void *ptr; + size_t sz = sizeof(unsigned); + + assert_d_eq(mallctl("arenas.create", (void *)&arena, &sz, NULL, 0), 0, + "Unexpected mallctl() failure"); + ptr = mallocx(42, MALLOCX_ARENA(arena) | MALLOCX_TCACHE_NONE); + assert_ptr_not_null(ptr, "Unexpected mallocx() failure"); + assert_d_eq(mallctl("arenas.lookup", &arena1, &sz, &ptr, sizeof(ptr)), + 0, "Unexpected mallctl() failure"); + assert_u_eq(arena, arena1, "Unexpected arena index"); + dallocx(ptr, 0); +} +TEST_END + TEST_BEGIN(test_stats_arenas) { #define TEST_STATS_ARENAS(t, name) do { \ t name; \ @@ -705,6 +780,79 @@ TEST_BEGIN(test_stats_arenas) { } TEST_END +static void +alloc_hook(void *extra, UNUSED hook_alloc_t type, UNUSED void *result, + UNUSED uintptr_t result_raw, UNUSED uintptr_t args_raw[3]) { + *(bool *)extra = true; +} + +static void +dalloc_hook(void *extra, UNUSED hook_dalloc_t type, + UNUSED void *address, UNUSED uintptr_t args_raw[3]) { + *(bool *)extra = true; +} + +TEST_BEGIN(test_hooks) { + bool hook_called = false; + hooks_t hooks = {&alloc_hook, &dalloc_hook, NULL, &hook_called}; + void *handle = NULL; + size_t sz = sizeof(handle); + int err = mallctl("experimental.hooks.install", &handle, &sz, &hooks, + sizeof(hooks)); + assert_d_eq(err, 0, "Hook installation failed"); + assert_ptr_ne(handle, NULL, "Hook installation gave null handle"); + void *ptr = mallocx(1, 0); + assert_true(hook_called, "Alloc hook not called"); + hook_called = false; + free(ptr); + assert_true(hook_called, "Free hook not called"); + + err = mallctl("experimental.hooks.remove", NULL, NULL, &handle, + sizeof(handle)); + assert_d_eq(err, 0, "Hook removal failed"); + hook_called = false; + ptr = mallocx(1, 0); + free(ptr); + assert_false(hook_called, "Hook called after removal"); +} +TEST_END + +TEST_BEGIN(test_hooks_exhaustion) { + bool hook_called = false; + hooks_t hooks = {&alloc_hook, &dalloc_hook, NULL, &hook_called}; + + void *handle; + void *handles[HOOK_MAX]; + size_t sz = sizeof(handle); + int err; + for (int i = 0; i < HOOK_MAX; i++) { + handle = NULL; + err = mallctl("experimental.hooks.install", &handle, &sz, + &hooks, sizeof(hooks)); + assert_d_eq(err, 0, "Error installation hooks"); + assert_ptr_ne(handle, NULL, "Got NULL handle"); + handles[i] = handle; + } + err = mallctl("experimental.hooks.install", &handle, &sz, &hooks, + sizeof(hooks)); + assert_d_eq(err, EAGAIN, "Should have failed hook installation"); + for (int i = 0; i < HOOK_MAX; i++) { + err = mallctl("experimental.hooks.remove", NULL, NULL, + &handles[i], sizeof(handles[i])); + assert_d_eq(err, 0, "Hook removal failed"); + } + /* Insertion failed, but then we removed some; it should work now. */ + handle = NULL; + err = mallctl("experimental.hooks.install", &handle, &sz, &hooks, + sizeof(hooks)); + assert_d_eq(err, 0, "Hook insertion failed"); + assert_ptr_ne(handle, NULL, "Got NULL handle"); + err = mallctl("experimental.hooks.remove", NULL, NULL, &handle, + sizeof(handle)); + assert_d_eq(err, 0, "Hook removal failed"); +} +TEST_END + int main(void) { return test( @@ -725,11 +873,15 @@ main(void) { test_arena_i_purge, test_arena_i_decay, test_arena_i_dss, + test_arena_i_retain_grow_limit, test_arenas_dirty_decay_ms, test_arenas_muzzy_decay_ms, test_arenas_constants, test_arenas_bin_constants, test_arenas_lextent_constants, test_arenas_create, - test_stats_arenas); + test_arenas_lookup, + test_stats_arenas, + test_hooks, + test_hooks_exhaustion); } diff --git a/kbe/src/lib/dependencies/jemalloc/test/unit/pack.c b/kbe/src/lib/dependencies/jemalloc/test/unit/pack.c index edfc548ff2..fc188b0033 100755 --- a/kbe/src/lib/dependencies/jemalloc/test/unit/pack.c +++ b/kbe/src/lib/dependencies/jemalloc/test/unit/pack.c @@ -88,6 +88,12 @@ arena_reset_mallctl(unsigned arena_ind) { } TEST_BEGIN(test_pack) { + bool prof_enabled; + size_t sz = sizeof(prof_enabled); + if (mallctl("opt.prof", (void *)&prof_enabled, &sz, NULL, 0) == 0) { + test_skip_if(prof_enabled); + } + unsigned arena_ind = arenas_create_mallctl(); size_t nregs_per_run = nregs_per_run_compute(); size_t nregs = nregs_per_run * NSLABS; diff --git a/kbe/src/lib/dependencies/jemalloc/test/unit/pages.c b/kbe/src/lib/dependencies/jemalloc/test/unit/pages.c index 67dbb4cd2f..ee729eece8 100755 --- a/kbe/src/lib/dependencies/jemalloc/test/unit/pages.c +++ b/kbe/src/lib/dependencies/jemalloc/test/unit/pages.c @@ -10,11 +10,13 @@ TEST_BEGIN(test_pages_huge) { pages = pages_map(NULL, alloc_size, PAGE, &commit); assert_ptr_not_null(pages, "Unexpected pages_map() error"); - hugepage = (void *)(ALIGNMENT_CEILING((uintptr_t)pages, HUGEPAGE)); - assert_b_ne(pages_huge(hugepage, HUGEPAGE), config_thp, - "Unexpected pages_huge() result"); - assert_false(pages_nohuge(hugepage, HUGEPAGE), - "Unexpected pages_nohuge() result"); + if (init_system_thp_mode == thp_mode_default) { + hugepage = (void *)(ALIGNMENT_CEILING((uintptr_t)pages, HUGEPAGE)); + assert_b_ne(pages_huge(hugepage, HUGEPAGE), have_madvise_huge, + "Unexpected pages_huge() result"); + assert_false(pages_nohuge(hugepage, HUGEPAGE), + "Unexpected pages_nohuge() result"); + } pages_unmap(pages, alloc_size); } diff --git a/kbe/src/lib/dependencies/jemalloc/test/unit/prof_gdump.c b/kbe/src/lib/dependencies/jemalloc/test/unit/prof_gdump.c index fcb434cb94..f7e0aac766 100755 --- a/kbe/src/lib/dependencies/jemalloc/test/unit/prof_gdump.c +++ b/kbe/src/lib/dependencies/jemalloc/test/unit/prof_gdump.c @@ -29,12 +29,12 @@ TEST_BEGIN(test_gdump) { prof_dump_open = prof_dump_open_intercept; did_prof_dump_open = false; - p = mallocx((1U << LG_LARGE_MINCLASS), 0); + p = mallocx((1U << SC_LG_LARGE_MINCLASS), 0); assert_ptr_not_null(p, "Unexpected mallocx() failure"); assert_true(did_prof_dump_open, "Expected a profile dump"); did_prof_dump_open = false; - q = mallocx((1U << LG_LARGE_MINCLASS), 0); + q = mallocx((1U << SC_LG_LARGE_MINCLASS), 0); assert_ptr_not_null(q, "Unexpected mallocx() failure"); assert_true(did_prof_dump_open, "Expected a profile dump"); @@ -45,7 +45,7 @@ TEST_BEGIN(test_gdump) { "Unexpected mallctl failure while disabling prof.gdump"); assert(gdump_old); did_prof_dump_open = false; - r = mallocx((1U << LG_LARGE_MINCLASS), 0); + r = mallocx((1U << SC_LG_LARGE_MINCLASS), 0); assert_ptr_not_null(q, "Unexpected mallocx() failure"); assert_false(did_prof_dump_open, "Unexpected profile dump"); @@ -56,7 +56,7 @@ TEST_BEGIN(test_gdump) { "Unexpected mallctl failure while enabling prof.gdump"); assert(!gdump_old); did_prof_dump_open = false; - s = mallocx((1U << LG_LARGE_MINCLASS), 0); + s = mallocx((1U << SC_LG_LARGE_MINCLASS), 0); assert_ptr_not_null(q, "Unexpected mallocx() failure"); assert_true(did_prof_dump_open, "Expected a profile dump"); diff --git a/kbe/src/lib/dependencies/jemalloc/test/unit/prof_log.c b/kbe/src/lib/dependencies/jemalloc/test/unit/prof_log.c new file mode 100644 index 0000000000..6a3464b420 --- /dev/null +++ b/kbe/src/lib/dependencies/jemalloc/test/unit/prof_log.c @@ -0,0 +1,146 @@ +#include "test/jemalloc_test.h" + +#define N_PARAM 100 +#define N_THREADS 10 + +static void assert_rep() { + assert_b_eq(prof_log_rep_check(), false, "Rep check failed"); +} + +static void assert_log_empty() { + assert_zu_eq(prof_log_bt_count(), 0, + "The log has backtraces; it isn't empty"); + assert_zu_eq(prof_log_thr_count(), 0, + "The log has threads; it isn't empty"); + assert_zu_eq(prof_log_alloc_count(), 0, + "The log has allocations; it isn't empty"); +} + +void *buf[N_PARAM]; + +static void f() { + int i; + for (i = 0; i < N_PARAM; i++) { + buf[i] = malloc(100); + } + for (i = 0; i < N_PARAM; i++) { + free(buf[i]); + } +} + +TEST_BEGIN(test_prof_log_many_logs) { + int i; + + test_skip_if(!config_prof); + + for (i = 0; i < N_PARAM; i++) { + assert_b_eq(prof_log_is_logging(), false, + "Logging shouldn't have started yet"); + assert_d_eq(mallctl("prof.log_start", NULL, NULL, NULL, 0), 0, + "Unexpected mallctl failure when starting logging"); + assert_b_eq(prof_log_is_logging(), true, + "Logging should be started by now"); + assert_log_empty(); + assert_rep(); + f(); + assert_zu_eq(prof_log_thr_count(), 1, "Wrong thread count"); + assert_rep(); + assert_b_eq(prof_log_is_logging(), true, + "Logging should still be on"); + assert_d_eq(mallctl("prof.log_stop", NULL, NULL, NULL, 0), 0, + "Unexpected mallctl failure when stopping logging"); + assert_b_eq(prof_log_is_logging(), false, + "Logging should have turned off"); + } +} +TEST_END + +thd_t thr_buf[N_THREADS]; + +static void *f_thread(void *unused) { + int i; + for (i = 0; i < N_PARAM; i++) { + void *p = malloc(100); + memset(p, 100, sizeof(char)); + free(p); + } + + return NULL; +} + +TEST_BEGIN(test_prof_log_many_threads) { + + test_skip_if(!config_prof); + + int i; + assert_d_eq(mallctl("prof.log_start", NULL, NULL, NULL, 0), 0, + "Unexpected mallctl failure when starting logging"); + for (i = 0; i < N_THREADS; i++) { + thd_create(&thr_buf[i], &f_thread, NULL); + } + + for (i = 0; i < N_THREADS; i++) { + thd_join(thr_buf[i], NULL); + } + assert_zu_eq(prof_log_thr_count(), N_THREADS, + "Wrong number of thread entries"); + assert_rep(); + assert_d_eq(mallctl("prof.log_stop", NULL, NULL, NULL, 0), 0, + "Unexpected mallctl failure when stopping logging"); +} +TEST_END + +static void f3() { + void *p = malloc(100); + free(p); +} + +static void f1() { + void *p = malloc(100); + f3(); + free(p); +} + +static void f2() { + void *p = malloc(100); + free(p); +} + +TEST_BEGIN(test_prof_log_many_traces) { + + test_skip_if(!config_prof); + + assert_d_eq(mallctl("prof.log_start", NULL, NULL, NULL, 0), 0, + "Unexpected mallctl failure when starting logging"); + int i; + assert_rep(); + assert_log_empty(); + for (i = 0; i < N_PARAM; i++) { + assert_rep(); + f1(); + assert_rep(); + f2(); + assert_rep(); + f3(); + assert_rep(); + } + /* + * There should be 8 total backtraces: two for malloc/free in f1(), + * two for malloc/free in f2(), two for malloc/free in f3(), and then + * two for malloc/free in f1()'s call to f3(). + */ + assert_zu_eq(prof_log_bt_count(), 8, + "Wrong number of backtraces given sample workload"); + assert_d_eq(mallctl("prof.log_stop", NULL, NULL, NULL, 0), 0, + "Unexpected mallctl failure when stopping logging"); +} +TEST_END + +int +main(void) { + prof_log_dummy_set(true); + return test_no_reentrancy( + test_prof_log_many_logs, + test_prof_log_many_traces, + test_prof_log_many_threads); +} diff --git a/kbe/src/lib/dependencies/jemalloc/test/unit/prof_log.sh b/kbe/src/lib/dependencies/jemalloc/test/unit/prof_log.sh new file mode 100644 index 0000000000..8fcc7d8a79 --- /dev/null +++ b/kbe/src/lib/dependencies/jemalloc/test/unit/prof_log.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +if [ "x${enable_prof}" = "x1" ] ; then + export MALLOC_CONF="prof:true,lg_prof_sample:0" +fi diff --git a/kbe/src/lib/dependencies/jemalloc/test/unit/rtree.c b/kbe/src/lib/dependencies/jemalloc/test/unit/rtree.c index 814837bf1b..b017bc0327 100755 --- a/kbe/src/lib/dependencies/jemalloc/test/unit/rtree.c +++ b/kbe/src/lib/dependencies/jemalloc/test/unit/rtree.c @@ -85,11 +85,11 @@ TEST_END TEST_BEGIN(test_rtree_extrema) { extent_t extent_a, extent_b; - extent_init(&extent_a, NULL, NULL, LARGE_MINCLASS, false, - sz_size2index(LARGE_MINCLASS), 0, extent_state_active, false, - false); - extent_init(&extent_b, NULL, NULL, 0, false, NSIZES, 0, - extent_state_active, false, false); + extent_init(&extent_a, NULL, NULL, SC_LARGE_MINCLASS, false, + sz_size2index(SC_LARGE_MINCLASS), 0, + extent_state_active, false, false, true); + extent_init(&extent_b, NULL, NULL, 0, false, SC_NSIZES, 0, + extent_state_active, false, false, true); tsdn_t *tsdn = tsdn_fetch(); @@ -125,8 +125,8 @@ TEST_BEGIN(test_rtree_bits) { PAGE + (((uintptr_t)1) << LG_PAGE) - 1}; extent_t extent; - extent_init(&extent, NULL, NULL, 0, false, NSIZES, 0, - extent_state_active, false, false); + extent_init(&extent, NULL, NULL, 0, false, SC_NSIZES, 0, + extent_state_active, false, false, true); rtree_t *rtree = &test_rtree; rtree_ctx_t rtree_ctx; @@ -135,7 +135,7 @@ TEST_BEGIN(test_rtree_bits) { for (unsigned i = 0; i < sizeof(keys)/sizeof(uintptr_t); i++) { assert_false(rtree_write(tsdn, rtree, &rtree_ctx, keys[i], - &extent, NSIZES, false), + &extent, SC_NSIZES, false), "Unexpected rtree_write() failure"); for (unsigned j = 0; j < sizeof(keys)/sizeof(uintptr_t); j++) { assert_ptr_eq(rtree_extent_read(tsdn, rtree, &rtree_ctx, @@ -166,8 +166,8 @@ TEST_BEGIN(test_rtree_random) { rtree_ctx_data_init(&rtree_ctx); extent_t extent; - extent_init(&extent, NULL, NULL, 0, false, NSIZES, 0, - extent_state_active, false, false); + extent_init(&extent, NULL, NULL, 0, false, SC_NSIZES, 0, + extent_state_active, false, false, true); assert_false(rtree_new(rtree, false), "Unexpected rtree_new() failure"); @@ -177,7 +177,8 @@ TEST_BEGIN(test_rtree_random) { &rtree_ctx, keys[i], false, true); assert_ptr_not_null(elm, "Unexpected rtree_leaf_elm_lookup() failure"); - rtree_leaf_elm_write(tsdn, rtree, elm, &extent, NSIZES, false); + rtree_leaf_elm_write(tsdn, rtree, elm, &extent, SC_NSIZES, + false); assert_ptr_eq(rtree_extent_read(tsdn, rtree, &rtree_ctx, keys[i], true), &extent, "rtree_extent_read() should return previously set value"); diff --git a/kbe/src/lib/dependencies/jemalloc/test/unit/sc.c b/kbe/src/lib/dependencies/jemalloc/test/unit/sc.c new file mode 100644 index 0000000000..bf51d8e59f --- /dev/null +++ b/kbe/src/lib/dependencies/jemalloc/test/unit/sc.c @@ -0,0 +1,33 @@ +#include "test/jemalloc_test.h" + +TEST_BEGIN(test_update_slab_size) { + sc_data_t data; + memset(&data, 0, sizeof(data)); + sc_data_init(&data); + sc_t *tiny = &data.sc[0]; + size_t tiny_size = (ZU(1) << tiny->lg_base) + + (ZU(tiny->ndelta) << tiny->lg_delta); + size_t pgs_too_big = (tiny_size * BITMAP_MAXBITS + PAGE - 1) / PAGE + 1; + sc_data_update_slab_size(&data, tiny_size, tiny_size, (int)pgs_too_big); + assert_zu_lt((size_t)tiny->pgs, pgs_too_big, "Allowed excessive pages"); + + sc_data_update_slab_size(&data, 1, 10 * PAGE, 1); + for (int i = 0; i < data.nbins; i++) { + sc_t *sc = &data.sc[i]; + size_t reg_size = (ZU(1) << sc->lg_base) + + (ZU(sc->ndelta) << sc->lg_delta); + if (reg_size <= PAGE) { + assert_d_eq(sc->pgs, 1, "Ignored valid page size hint"); + } else { + assert_d_gt(sc->pgs, 1, + "Allowed invalid page size hint"); + } + } +} +TEST_END + +int +main(void) { + return test( + test_update_slab_size); +} diff --git a/kbe/src/lib/dependencies/jemalloc/test/unit/seq.c b/kbe/src/lib/dependencies/jemalloc/test/unit/seq.c new file mode 100644 index 0000000000..19613b0b24 --- /dev/null +++ b/kbe/src/lib/dependencies/jemalloc/test/unit/seq.c @@ -0,0 +1,95 @@ +#include "test/jemalloc_test.h" + +#include "jemalloc/internal/seq.h" + +typedef struct data_s data_t; +struct data_s { + int arr[10]; +}; + +static void +set_data(data_t *data, int num) { + for (int i = 0; i < 10; i++) { + data->arr[i] = num; + } +} + +static void +assert_data(data_t *data) { + int num = data->arr[0]; + for (int i = 0; i < 10; i++) { + assert_d_eq(num, data->arr[i], "Data consistency error"); + } +} + +seq_define(data_t, data) + +typedef struct thd_data_s thd_data_t; +struct thd_data_s { + seq_data_t data; +}; + +static void * +seq_reader_thd(void *arg) { + thd_data_t *thd_data = (thd_data_t *)arg; + int iter = 0; + data_t local_data; + while (iter < 1000 * 1000 - 1) { + bool success = seq_try_load_data(&local_data, &thd_data->data); + if (success) { + assert_data(&local_data); + assert_d_le(iter, local_data.arr[0], + "Seq read went back in time."); + iter = local_data.arr[0]; + } + } + return NULL; +} + +static void * +seq_writer_thd(void *arg) { + thd_data_t *thd_data = (thd_data_t *)arg; + data_t local_data; + memset(&local_data, 0, sizeof(local_data)); + for (int i = 0; i < 1000 * 1000; i++) { + set_data(&local_data, i); + seq_store_data(&thd_data->data, &local_data); + } + return NULL; +} + +TEST_BEGIN(test_seq_threaded) { + thd_data_t thd_data; + memset(&thd_data, 0, sizeof(thd_data)); + + thd_t reader; + thd_t writer; + + thd_create(&reader, seq_reader_thd, &thd_data); + thd_create(&writer, seq_writer_thd, &thd_data); + + thd_join(reader, NULL); + thd_join(writer, NULL); +} +TEST_END + +TEST_BEGIN(test_seq_simple) { + data_t data; + seq_data_t seq; + memset(&seq, 0, sizeof(seq)); + for (int i = 0; i < 1000 * 1000; i++) { + set_data(&data, i); + seq_store_data(&seq, &data); + set_data(&data, 0); + bool success = seq_try_load_data(&data, &seq); + assert_b_eq(success, true, "Failed non-racing read"); + assert_data(&data); + } +} +TEST_END + +int main(void) { + return test_no_reentrancy( + test_seq_simple, + test_seq_threaded); +} diff --git a/kbe/src/lib/dependencies/jemalloc/test/unit/size_classes.c b/kbe/src/lib/dependencies/jemalloc/test/unit/size_classes.c index bcff560983..694733635b 100755 --- a/kbe/src/lib/dependencies/jemalloc/test/unit/size_classes.c +++ b/kbe/src/lib/dependencies/jemalloc/test/unit/size_classes.c @@ -108,8 +108,13 @@ TEST_BEGIN(test_psize_classes) { size_class, sz_psz2ind(size_class), sz_pind2sz(sz_psz2ind(size_class))); - assert_u_eq(pind+1, sz_psz2ind(size_class+1), - "Next size_class does not round up properly"); + if (size_class == SC_LARGE_MAXCLASS) { + assert_u_eq(SC_NPSIZES, sz_psz2ind(size_class + 1), + "Next size_class does not round up properly"); + } else { + assert_u_eq(pind + 1, sz_psz2ind(size_class + 1), + "Next size_class does not round up properly"); + } assert_zu_eq(size_class, (pind > 0) ? sz_psz2u(sz_pind2sz(pind-1)+1) : sz_psz2u(1), @@ -142,11 +147,11 @@ TEST_BEGIN(test_overflow) { max_size_class = get_max_size_class(); max_psz = max_size_class + PAGE; - assert_u_eq(sz_size2index(max_size_class+1), NSIZES, + assert_u_eq(sz_size2index(max_size_class+1), SC_NSIZES, "sz_size2index() should return NSIZES on overflow"); - assert_u_eq(sz_size2index(ZU(PTRDIFF_MAX)+1), NSIZES, + assert_u_eq(sz_size2index(ZU(PTRDIFF_MAX)+1), SC_NSIZES, "sz_size2index() should return NSIZES on overflow"); - assert_u_eq(sz_size2index(SIZE_T_MAX), NSIZES, + assert_u_eq(sz_size2index(SIZE_T_MAX), SC_NSIZES, "sz_size2index() should return NSIZES on overflow"); assert_zu_eq(sz_s2u(max_size_class+1), 0, @@ -156,11 +161,11 @@ TEST_BEGIN(test_overflow) { assert_zu_eq(sz_s2u(SIZE_T_MAX), 0, "sz_s2u() should return 0 on overflow"); - assert_u_eq(sz_psz2ind(max_size_class+1), NPSIZES, + assert_u_eq(sz_psz2ind(max_size_class+1), SC_NPSIZES, "sz_psz2ind() should return NPSIZES on overflow"); - assert_u_eq(sz_psz2ind(ZU(PTRDIFF_MAX)+1), NPSIZES, + assert_u_eq(sz_psz2ind(ZU(PTRDIFF_MAX)+1), SC_NPSIZES, "sz_psz2ind() should return NPSIZES on overflow"); - assert_u_eq(sz_psz2ind(SIZE_T_MAX), NPSIZES, + assert_u_eq(sz_psz2ind(SIZE_T_MAX), SC_NPSIZES, "sz_psz2ind() should return NPSIZES on overflow"); assert_zu_eq(sz_psz2u(max_size_class+1), max_psz, diff --git a/kbe/src/lib/dependencies/jemalloc/test/unit/slab.c b/kbe/src/lib/dependencies/jemalloc/test/unit/slab.c index 6f40aeef6c..ef7188215c 100755 --- a/kbe/src/lib/dependencies/jemalloc/test/unit/slab.c +++ b/kbe/src/lib/dependencies/jemalloc/test/unit/slab.c @@ -3,13 +3,13 @@ TEST_BEGIN(test_arena_slab_regind) { szind_t binind; - for (binind = 0; binind < NBINS; binind++) { + for (binind = 0; binind < SC_NBINS; binind++) { size_t regind; extent_t slab; - const arena_bin_info_t *bin_info = &arena_bin_info[binind]; + const bin_info_t *bin_info = &bin_infos[binind]; extent_init(&slab, NULL, mallocx(bin_info->slab_size, MALLOCX_LG_ALIGN(LG_PAGE)), bin_info->slab_size, true, - binind, 0, extent_state_active, false, true); + binind, 0, extent_state_active, false, true, true); assert_ptr_not_null(extent_addr_get(&slab), "Unexpected malloc() failure"); for (regind = 0; regind < bin_info->nregs; regind++) { diff --git a/kbe/src/lib/dependencies/jemalloc/test/unit/stats.c b/kbe/src/lib/dependencies/jemalloc/test/unit/stats.c index d9849d800b..4323bfa3b2 100755 --- a/kbe/src/lib/dependencies/jemalloc/test/unit/stats.c +++ b/kbe/src/lib/dependencies/jemalloc/test/unit/stats.c @@ -33,7 +33,7 @@ TEST_BEGIN(test_stats_large) { size_t sz; int expected = config_stats ? 0 : ENOENT; - p = mallocx(SMALL_MAXCLASS+1, MALLOCX_ARENA(0)); + p = mallocx(SC_SMALL_MAXCLASS + 1, MALLOCX_ARENA(0)); assert_ptr_not_null(p, "Unexpected mallocx() failure"); assert_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sizeof(epoch)), @@ -74,9 +74,10 @@ TEST_BEGIN(test_stats_arenas_summary) { uint64_t dirty_npurge, dirty_nmadvise, dirty_purged; uint64_t muzzy_npurge, muzzy_nmadvise, muzzy_purged; - little = mallocx(SMALL_MAXCLASS, MALLOCX_ARENA(0)); + little = mallocx(SC_SMALL_MAXCLASS, MALLOCX_ARENA(0)); assert_ptr_not_null(little, "Unexpected mallocx() failure"); - large = mallocx((1U << LG_LARGE_MINCLASS), MALLOCX_ARENA(0)); + large = mallocx((1U << SC_LG_LARGE_MINCLASS), + MALLOCX_ARENA(0)); assert_ptr_not_null(large, "Unexpected mallocx() failure"); dallocx(little, 0); @@ -148,7 +149,7 @@ TEST_BEGIN(test_stats_arenas_small) { no_lazy_lock(); /* Lazy locking would dodge tcache testing. */ - p = mallocx(SMALL_MAXCLASS, MALLOCX_ARENA(0)); + p = mallocx(SC_SMALL_MAXCLASS, MALLOCX_ARENA(0)); assert_ptr_not_null(p, "Unexpected mallocx() failure"); assert_d_eq(mallctl("thread.tcache.flush", NULL, NULL, NULL, 0), @@ -191,7 +192,7 @@ TEST_BEGIN(test_stats_arenas_large) { uint64_t epoch, nmalloc, ndalloc; int expected = config_stats ? 0 : ENOENT; - p = mallocx((1U << LG_LARGE_MINCLASS), MALLOCX_ARENA(0)); + p = mallocx((1U << SC_LG_LARGE_MINCLASS), MALLOCX_ARENA(0)); assert_ptr_not_null(p, "Unexpected mallocx() failure"); assert_d_eq(mallctl("epoch", NULL, NULL, (void *)&epoch, sizeof(epoch)), @@ -245,7 +246,7 @@ TEST_BEGIN(test_stats_arenas_bins) { (void *)&arena_ind, sizeof(arena_ind)), 0, "Unexpected mallctl() failure"); - p = malloc(arena_bin_info[0].reg_size); + p = malloc(bin_infos[0].reg_size); assert_ptr_not_null(p, "Unexpected malloc() failure"); assert_d_eq(mallctl("thread.tcache.flush", NULL, NULL, NULL, 0), diff --git a/kbe/src/lib/dependencies/jemalloc/test/unit/stats_print.c b/kbe/src/lib/dependencies/jemalloc/test/unit/stats_print.c index acb26b068e..014d002fd4 100755 --- a/kbe/src/lib/dependencies/jemalloc/test/unit/stats_print.c +++ b/kbe/src/lib/dependencies/jemalloc/test/unit/stats_print.c @@ -67,7 +67,7 @@ token_error(token_t *token) { token->col); break; } - UNUSED ssize_t err = write(STDERR_FILENO, + UNUSED ssize_t err = malloc_write_fd(STDERR_FILENO, &token->parser->buf[token->pos], token->len); malloc_printf("\n"); } diff --git a/kbe/src/lib/dependencies/jemalloc/test/unit/hooks.c b/kbe/src/lib/dependencies/jemalloc/test/unit/test_hooks.c old mode 100755 new mode 100644 similarity index 82% rename from kbe/src/lib/dependencies/jemalloc/test/unit/hooks.c rename to kbe/src/lib/dependencies/jemalloc/test/unit/test_hooks.c index b70172e13c..ded8698bc7 --- a/kbe/src/lib/dependencies/jemalloc/test/unit/hooks.c +++ b/kbe/src/lib/dependencies/jemalloc/test/unit/test_hooks.c @@ -12,10 +12,10 @@ func_to_hook(int arg1, int arg2) { return arg1 + arg2; } -#define func_to_hook JEMALLOC_HOOK(func_to_hook, hooks_libc_hook) +#define func_to_hook JEMALLOC_HOOK(func_to_hook, test_hooks_libc_hook) TEST_BEGIN(unhooked_call) { - hooks_libc_hook = NULL; + test_hooks_libc_hook = NULL; hook_called = false; assert_d_eq(3, func_to_hook(1, 2), "Hooking changed return value."); assert_false(hook_called, "Nulling out hook didn't take."); @@ -23,7 +23,7 @@ TEST_BEGIN(unhooked_call) { TEST_END TEST_BEGIN(hooked_call) { - hooks_libc_hook = &hook; + test_hooks_libc_hook = &hook; hook_called = false; assert_d_eq(3, func_to_hook(1, 2), "Hooking changed return value."); assert_true(hook_called, "Hook should have executed."); diff --git a/kbe/src/lib/dependencies/jemalloc/test/unit/tsd.c b/kbe/src/lib/dependencies/jemalloc/test/unit/tsd.c index 6c479139bd..917884dcf0 100755 --- a/kbe/src/lib/dependencies/jemalloc/test/unit/tsd.c +++ b/kbe/src/lib/dependencies/jemalloc/test/unit/tsd.c @@ -1,5 +1,10 @@ #include "test/jemalloc_test.h" +/* + * If we're e.g. in debug mode, we *never* enter the fast path, and so shouldn't + * be asserting that we're on one. + */ +static bool originally_fast; static int data_cleanup_count; void @@ -98,11 +103,11 @@ thd_start_reincarnated(void *arg) { tsd_cleanup((void *)tsd); assert_ptr_null(*tsd_arenap_get_unsafe(tsd), "TSD arena should have been cleared."); - assert_u_eq(tsd->state, tsd_state_purgatory, + assert_u_eq(tsd_state_get(tsd), tsd_state_purgatory, "TSD state should be purgatory\n"); free(p); - assert_u_eq(tsd->state, tsd_state_reincarnated, + assert_u_eq(tsd_state_get(tsd), tsd_state_reincarnated, "TSD state should be reincarnated\n"); p = mallocx(1, MALLOCX_TCACHE_NONE); assert_ptr_not_null(p, "Unexpected malloc() failure"); @@ -124,6 +129,128 @@ TEST_BEGIN(test_tsd_reincarnation) { } TEST_END +typedef struct { + atomic_u32_t phase; + atomic_b_t error; +} global_slow_data_t; + +static void * +thd_start_global_slow(void *arg) { + /* PHASE 0 */ + global_slow_data_t *data = (global_slow_data_t *)arg; + free(mallocx(1, 0)); + + tsd_t *tsd = tsd_fetch(); + /* + * No global slowness has happened yet; there was an error if we were + * originally fast but aren't now. + */ + atomic_store_b(&data->error, originally_fast && !tsd_fast(tsd), + ATOMIC_SEQ_CST); + atomic_store_u32(&data->phase, 1, ATOMIC_SEQ_CST); + + /* PHASE 2 */ + while (atomic_load_u32(&data->phase, ATOMIC_SEQ_CST) != 2) { + } + free(mallocx(1, 0)); + atomic_store_b(&data->error, tsd_fast(tsd), ATOMIC_SEQ_CST); + atomic_store_u32(&data->phase, 3, ATOMIC_SEQ_CST); + + /* PHASE 4 */ + while (atomic_load_u32(&data->phase, ATOMIC_SEQ_CST) != 4) { + } + free(mallocx(1, 0)); + atomic_store_b(&data->error, tsd_fast(tsd), ATOMIC_SEQ_CST); + atomic_store_u32(&data->phase, 5, ATOMIC_SEQ_CST); + + /* PHASE 6 */ + while (atomic_load_u32(&data->phase, ATOMIC_SEQ_CST) != 6) { + } + free(mallocx(1, 0)); + /* Only one decrement so far. */ + atomic_store_b(&data->error, tsd_fast(tsd), ATOMIC_SEQ_CST); + atomic_store_u32(&data->phase, 7, ATOMIC_SEQ_CST); + + /* PHASE 8 */ + while (atomic_load_u32(&data->phase, ATOMIC_SEQ_CST) != 8) { + } + free(mallocx(1, 0)); + /* + * Both decrements happened; we should be fast again (if we ever + * were) + */ + atomic_store_b(&data->error, originally_fast && !tsd_fast(tsd), + ATOMIC_SEQ_CST); + atomic_store_u32(&data->phase, 9, ATOMIC_SEQ_CST); + + return NULL; +} + +TEST_BEGIN(test_tsd_global_slow) { + global_slow_data_t data = {ATOMIC_INIT(0), ATOMIC_INIT(false)}; + /* + * Note that the "mallocx" here (vs. malloc) is important, since the + * compiler is allowed to optimize away free(malloc(1)) but not + * free(mallocx(1)). + */ + free(mallocx(1, 0)); + tsd_t *tsd = tsd_fetch(); + originally_fast = tsd_fast(tsd); + + thd_t thd; + thd_create(&thd, thd_start_global_slow, (void *)&data.phase); + /* PHASE 1 */ + while (atomic_load_u32(&data.phase, ATOMIC_SEQ_CST) != 1) { + /* + * We don't have a portable condvar/semaphore mechanism. + * Spin-wait. + */ + } + assert_false(atomic_load_b(&data.error, ATOMIC_SEQ_CST), ""); + tsd_global_slow_inc(tsd_tsdn(tsd)); + free(mallocx(1, 0)); + assert_false(tsd_fast(tsd), ""); + atomic_store_u32(&data.phase, 2, ATOMIC_SEQ_CST); + + /* PHASE 3 */ + while (atomic_load_u32(&data.phase, ATOMIC_SEQ_CST) != 3) { + } + assert_false(atomic_load_b(&data.error, ATOMIC_SEQ_CST), ""); + /* Increase again, so that we can test multiple fast/slow changes. */ + tsd_global_slow_inc(tsd_tsdn(tsd)); + atomic_store_u32(&data.phase, 4, ATOMIC_SEQ_CST); + free(mallocx(1, 0)); + assert_false(tsd_fast(tsd), ""); + + /* PHASE 5 */ + while (atomic_load_u32(&data.phase, ATOMIC_SEQ_CST) != 5) { + } + assert_false(atomic_load_b(&data.error, ATOMIC_SEQ_CST), ""); + tsd_global_slow_dec(tsd_tsdn(tsd)); + atomic_store_u32(&data.phase, 6, ATOMIC_SEQ_CST); + /* We only decreased once; things should still be slow. */ + free(mallocx(1, 0)); + assert_false(tsd_fast(tsd), ""); + + /* PHASE 7 */ + while (atomic_load_u32(&data.phase, ATOMIC_SEQ_CST) != 7) { + } + assert_false(atomic_load_b(&data.error, ATOMIC_SEQ_CST), ""); + tsd_global_slow_dec(tsd_tsdn(tsd)); + atomic_store_u32(&data.phase, 8, ATOMIC_SEQ_CST); + /* We incremented and then decremented twice; we should be fast now. */ + free(mallocx(1, 0)); + assert_true(!originally_fast || tsd_fast(tsd), ""); + + /* PHASE 9 */ + while (atomic_load_u32(&data.phase, ATOMIC_SEQ_CST) != 9) { + } + assert_false(atomic_load_b(&data.error, ATOMIC_SEQ_CST), ""); + + thd_join(thd, NULL); +} +TEST_END + int main(void) { /* Ensure tsd bootstrapped. */ @@ -135,5 +262,6 @@ main(void) { return test_no_reentrancy( test_tsd_main_thread, test_tsd_sub_thread, - test_tsd_reincarnation); + test_tsd_reincarnation, + test_tsd_global_slow); } diff --git a/kbe/src/lib/dependencies/jemalloc/test/unit/zero.c b/kbe/src/lib/dependencies/jemalloc/test/unit/zero.c index 553692ba7b..271fd5cba4 100755 --- a/kbe/src/lib/dependencies/jemalloc/test/unit/zero.c +++ b/kbe/src/lib/dependencies/jemalloc/test/unit/zero.c @@ -41,13 +41,13 @@ test_zero(size_t sz_min, size_t sz_max) { TEST_BEGIN(test_zero_small) { test_skip_if(!config_fill); - test_zero(1, SMALL_MAXCLASS-1); + test_zero(1, SC_SMALL_MAXCLASS - 1); } TEST_END TEST_BEGIN(test_zero_large) { test_skip_if(!config_fill); - test_zero(SMALL_MAXCLASS+1, (1U << (LG_LARGE_MINCLASS+1))); + test_zero(SC_SMALL_MAXCLASS + 1, 1U << (SC_LG_LARGE_MINCLASS + 1)); } TEST_END diff --git a/kbe/src/lib/entitydef/datatype.cpp b/kbe/src/lib/entitydef/datatype.cpp index 3d0c3fda33..7a09a7da3c 100644 --- a/kbe/src/lib/entitydef/datatype.cpp +++ b/kbe/src/lib/entitydef/datatype.cpp @@ -1,3133 +1,3122 @@ -// Copyright 2008-2018 Yolo Technologies, Inc. All Rights Reserved. https://www.comblockengine.com - -#include "datatype.h" -#include "datatypes.h" -#include "entitydef.h" -#include "fixeddict.h" -#include "fixedarray.h" -#include "entity_call.h" -#include "py_entitydef.h" -#include "property.h" -#include "entity_component.h" -#include "scriptdef_module.h" -#include "pyscript/vector2.h" -#include "pyscript/vector3.h" -#include "pyscript/vector4.h" -#include "pyscript/copy.h" -#include "pyscript/py_memorystream.h" - -#ifndef CODE_INLINE -#include "datatype.inl" -#endif - -namespace KBEngine{ -static DATATYPE_UID _g_dataTypeUID = 1; - -static bool isVecotr(std::string str, std::size_t n, std::vector& nums) -{ - if (str.empty()) - { - return false; - } - - if (n < 2) - { - return false; - } - - std::string strtemp = str; - - strutil::kbe_replace(strtemp, " ", ""); - strutil::kbe_replace(strtemp, "(", ""); - strutil::kbe_replace(strtemp, ")", ""); - - std::vector result; - strutil::kbe_splits(strtemp, ",", result); - - if (result.size() != n) - { - return false; - } - - nums.clear(); - for (auto ite = result.begin(); ite != result.end(); ite++) - { - try - { - float num = 0.f; - StringConv::str2value(num, (*ite).c_str()); - nums.push_back(num); - } - catch (...) - { - return false; - } - } - - return true; -} - -static bool isVector2(std::string str, std::vector& nums) -{ - return isVecotr(str, 2, nums); -} - -static bool isVector3(std::string str, std::vector& nums) -{ - return isVecotr(str, 3, nums); -} - -static bool isVector4(std::string str, std::vector& nums) -{ - return isVecotr(str, 4, nums); -} - -//------------------------------------------------------------------------------------- -DataType::DataType(DATATYPE_UID did): -id_(did), -aliasName_() -{ - if(id_ == 0) - id_ = _g_dataTypeUID++; - - DataTypes::addDataType(id_, this); - - EntityDef::md5().append((void*)this->aliasName(), (int)strlen(this->aliasName())); - EntityDef::md5().append((void*)&id_, sizeof(DATATYPE_UID)); -} - -//------------------------------------------------------------------------------------- -DataType::~DataType() -{ - finalise(); -} - -//------------------------------------------------------------------------------------- -bool DataType::finalise() -{ - _g_dataTypeUID = 1; - return true; -} - -//------------------------------------------------------------------------------------- -bool DataType::initialize(XML* xml, TiXmlNode* node) -{ - return true; -} - -//------------------------------------------------------------------------------------- -UInt64Type::UInt64Type(DATATYPE_UID did): -DataType(did) -{ -} - -//------------------------------------------------------------------------------------- -UInt64Type::~UInt64Type() -{ -} - -//------------------------------------------------------------------------------------- -bool UInt64Type::isSameType(PyObject* pyValue) -{ - if(pyValue == NULL) - { - OUT_TYPE_ERROR("UINT64"); - return false; - } - - if (!PyLong_Check(pyValue)) - { - OUT_TYPE_ERROR("UINT64"); - return false; - } - - PyLong_AsUnsignedLongLong(pyValue); - if (!PyErr_Occurred()) - return true; - - PyErr_Clear(); - PyLong_AsUnsignedLong(pyValue); - if (!PyErr_Occurred()) - return true; - - PyErr_Clear(); - long v = PyLong_AsLong(pyValue); - if (!PyErr_Occurred()) - { - if(v < 0) - { - OUT_TYPE_ERROR("UINT64"); - return false; - } - return true; - } - - PyErr_Clear(); - OUT_TYPE_ERROR("UINT64"); - return false; -} - -//------------------------------------------------------------------------------------- -PyObject* UInt64Type::parseDefaultStr(std::string defaultVal) -{ - uint64 val = 0; - if(!defaultVal.empty()) - { - std::stringstream stream; - stream << defaultVal; - stream >> val; - } - - PyObject* pyval = PyLong_FromUnsignedLongLong(val); - - if (PyErr_Occurred()) - { - PyErr_Clear(); - PyErr_Format(PyExc_TypeError, "UInt64Type::parseDefaultStr: defaultVal(%s) error! val=[%s]", - pyval != NULL ? pyval->ob_type->tp_name : "NULL", defaultVal.c_str()); - - PyErr_PrintEx(0); - - S_RELEASE(pyval); - return PyLong_FromUnsignedLongLong(0); - } - - return pyval; -} - -//------------------------------------------------------------------------------------- -void UInt64Type::addToStream(MemoryStream* mstream, PyObject* pyValue) -{ - uint64 v = static_cast(PyLong_AsUnsignedLongLong(pyValue)); - - if (PyErr_Occurred()) - { - PyErr_Clear(); - v = (uint64)PyLong_AsUnsignedLong(pyValue); - if (PyErr_Occurred()) - { - PyErr_Clear(); - v = (uint64)PyLong_AsLong(pyValue); - - if(PyErr_Occurred()) - { - PyErr_Clear(); - PyErr_Format(PyExc_TypeError, "UInt64Type::addToStream: pyValue(%s) is wrong!", - (pyValue == NULL) ? "NULL": pyValue->ob_type->tp_name); - - PyErr_PrintEx(0); - - v = 0; - } - } - } - - (*mstream) << v; -} - -//------------------------------------------------------------------------------------- -PyObject* UInt64Type::createFromStream(MemoryStream* mstream) -{ - uint64 val = 0; - if(mstream) - (*mstream) >> val; - - PyObject* pyval = PyLong_FromUnsignedLongLong(val); - - if (PyErr_Occurred()) - { - PyErr_Format(PyExc_TypeError, "UInt64Type::createFromStream: errval=%" PRIu64 ", default return is 0", val); - PyErr_PrintEx(0); - S_RELEASE(pyval); - return PyLong_FromUnsignedLongLong(0); - } - - return pyval; -} - -//------------------------------------------------------------------------------------- -UInt32Type::UInt32Type(DATATYPE_UID did): -DataType(did) -{ -} - -//------------------------------------------------------------------------------------- -UInt32Type::~UInt32Type() -{ -} - -//------------------------------------------------------------------------------------- -bool UInt32Type::isSameType(PyObject* pyValue) -{ - if(pyValue == NULL) - { - OUT_TYPE_ERROR("UINT32"); - return false; - } - - if (!PyLong_Check(pyValue)) - { - OUT_TYPE_ERROR("UINT32"); - return false; - } - - PyLong_AsUnsignedLong(pyValue); - if (!PyErr_Occurred()) - return true; - - PyErr_Clear(); - long v = PyLong_AsLong(pyValue); - if (!PyErr_Occurred()) - { - if(v < 0) - { - OUT_TYPE_ERROR("UINT32"); - return false; - } - return true; - } - - PyErr_Clear(); - OUT_TYPE_ERROR("UINT32"); - return false; -} - -//------------------------------------------------------------------------------------- -PyObject* UInt32Type::parseDefaultStr(std::string defaultVal) -{ - uint32 val = 0; - if(!defaultVal.empty()) - { - std::stringstream stream; - stream << defaultVal; - stream >> val; - } - - PyObject* pyval = PyLong_FromUnsignedLong(val); - - if (PyErr_Occurred()) - { - PyErr_Clear(); - PyErr_Format(PyExc_TypeError, "UInt32Type::parseDefaultStr: defaultVal(%s) error! val=[%s]", - pyval != NULL ? pyval->ob_type->tp_name : "NULL", defaultVal.c_str()); - - PyErr_PrintEx(0); - - S_RELEASE(pyval); - return PyLong_FromUnsignedLong(0); - } - - return pyval; -} - -//------------------------------------------------------------------------------------- -void UInt32Type::addToStream(MemoryStream* mstream, PyObject* pyValue) -{ - uint32 v = PyLong_AsUnsignedLong(pyValue); - - if (PyErr_Occurred()) - { - PyErr_Clear(); - v = (uint32)PyLong_AsLong(pyValue); - - if(PyErr_Occurred()) - { - PyErr_Clear(); - - PyErr_Format(PyExc_TypeError, "UInt32Type::addToStream: pyValue(%s) is wrong!", - (pyValue == NULL) ? "NULL": pyValue->ob_type->tp_name); - - PyErr_PrintEx(0); - - v = 0; - } - } - - (*mstream) << v; -} - -//------------------------------------------------------------------------------------- -PyObject* UInt32Type::createFromStream(MemoryStream* mstream) -{ - uint32 val = 0; - if(mstream) - (*mstream) >> val; - - PyObject* pyval = PyLong_FromUnsignedLong(val); - - if (PyErr_Occurred()) - { - PyErr_Format(PyExc_TypeError, "UInt32Type::createFromStream: errval=%u, default return is 0", val); - PyErr_PrintEx(0); - S_RELEASE(pyval); - return PyLong_FromUnsignedLong(0); - } - - return pyval; -} - -//------------------------------------------------------------------------------------- -Int64Type::Int64Type(DATATYPE_UID did): -DataType(did) -{ -} - -//------------------------------------------------------------------------------------- -Int64Type::~Int64Type() -{ -} - -//------------------------------------------------------------------------------------- -bool Int64Type::isSameType(PyObject* pyValue) -{ - if(pyValue == NULL) - { - OUT_TYPE_ERROR("INT64"); - return false; - } - - if (!PyLong_Check(pyValue)) - { - OUT_TYPE_ERROR("INT64"); - return false; - } - - PyLong_AsLongLong(pyValue); - if (!PyErr_Occurred()) - return true; - - PyErr_Clear(); - PyLong_AsLong(pyValue); - if (!PyErr_Occurred()) - { - return true; - } - - PyErr_Clear(); - OUT_TYPE_ERROR("INT64"); - return false; -} - -//------------------------------------------------------------------------------------- -PyObject* Int64Type::parseDefaultStr(std::string defaultVal) -{ - int64 val = 0; - if(!defaultVal.empty()) - { - std::stringstream stream; - stream << defaultVal; - stream >> val; - } - - PyObject* pyval = PyLong_FromLongLong(val); - - if (PyErr_Occurred()) - { - PyErr_Clear(); - PyErr_Format(PyExc_TypeError, "Int64Type::parseDefaultStr: defaultVal(%s) error! val=[%s]", - pyval != NULL ? pyval->ob_type->tp_name : "NULL", defaultVal.c_str()); - - PyErr_PrintEx(0); - - S_RELEASE(pyval); - return PyLong_FromLongLong(0); - } - - return pyval; -} - -//------------------------------------------------------------------------------------- -void Int64Type::addToStream(MemoryStream* mstream, PyObject* pyValue) -{ - int64 v = PyLong_AsLongLong(pyValue); - - if (PyErr_Occurred()) - { - PyErr_Clear(); - v = (uint32)PyLong_AsLong(pyValue); - - if(PyErr_Occurred()) - { - PyErr_Clear(); - - PyErr_Format(PyExc_TypeError, "Int64Type::addToStream: pyValue(%s) is wrong!", - (pyValue == NULL) ? "NULL": pyValue->ob_type->tp_name); - - PyErr_PrintEx(0); - - v = 0; - } - } - - (*mstream) << v; -} - -//------------------------------------------------------------------------------------- -PyObject* Int64Type::createFromStream(MemoryStream* mstream) -{ - int64 val = 0; - if(mstream) - (*mstream) >> val; - - PyObject* pyval = PyLong_FromLongLong(val); - - if (PyErr_Occurred()) - { - PyErr_Format(PyExc_TypeError, "Int64Type::createFromStream: errval=%" PRI64 ", default return is 0", val); - PyErr_PrintEx(0); - S_RELEASE(pyval); - return PyLong_FromLongLong(0); - } - - return pyval; -} - -//------------------------------------------------------------------------------------- -FloatType::FloatType(DATATYPE_UID did): -DataType(did) -{ -} - -//------------------------------------------------------------------------------------- -FloatType::~FloatType() -{ -} - -//------------------------------------------------------------------------------------- -bool FloatType::isSameType(PyObject* pyValue) -{ - if(pyValue == NULL) - { - OUT_TYPE_ERROR("FLOAT"); - return false; - } - - bool ret = PyFloat_Check(pyValue); - if(!ret) - OUT_TYPE_ERROR("FLOAT"); - return ret; -} - -//------------------------------------------------------------------------------------- -PyObject* FloatType::parseDefaultStr(std::string defaultVal) -{ - float val = 0.0f; - if(!defaultVal.empty()) - { - std::stringstream stream; - stream << defaultVal; - stream >> val; - } - - PyObject* pyval = PyFloat_FromDouble(val); - - if (PyErr_Occurred()) - { - PyErr_Clear(); - PyErr_Format(PyExc_TypeError, "FloatType::parseDefaultStr: defaultVal(%s) error! val=[%s]", - pyval != NULL ? pyval->ob_type->tp_name : "NULL", defaultVal.c_str()); - - PyErr_PrintEx(0); - - S_RELEASE(pyval); - return PyFloat_FromDouble(0.0f); - } - - return pyval; -} - -//------------------------------------------------------------------------------------- -void FloatType::addToStream(MemoryStream* mstream, PyObject* pyValue) -{ - if(!PyFloat_Check(pyValue)) - { - PyErr_Format(PyExc_TypeError, "FloatType::addToStream: pyValue(%s) is wrong!", - (pyValue == NULL) ? "NULL": pyValue->ob_type->tp_name); - - PyErr_PrintEx(0); - return; - } - - float val = (float)PyFloat_AsDouble(pyValue); - (*mstream) << val; -} - -//------------------------------------------------------------------------------------- -PyObject* FloatType::createFromStream(MemoryStream* mstream) -{ - float val = 0.0; - if(mstream) - (*mstream) >> val; - - PyObject* pyval = PyFloat_FromDouble(val); - - if (PyErr_Occurred()) - { - PyErr_Format(PyExc_TypeError, "FloatType::createFromStream: errval=%f, default return is 0", val); - PyErr_PrintEx(0); - S_RELEASE(pyval); - return PyFloat_FromDouble(0); - } - - return pyval; -} - -//------------------------------------------------------------------------------------- -DoubleType::DoubleType(DATATYPE_UID did): -DataType(did) -{ -} - -//------------------------------------------------------------------------------------- -DoubleType::~DoubleType() -{ -} - -//------------------------------------------------------------------------------------- -bool DoubleType::isSameType(PyObject* pyValue) -{ - if(pyValue == NULL) - { - OUT_TYPE_ERROR("DOUBLE"); - return false; - } - - bool ret = PyFloat_Check(pyValue); - if(!ret) - OUT_TYPE_ERROR("DOUBLE"); - return ret; -} - -//------------------------------------------------------------------------------------- -PyObject* DoubleType::parseDefaultStr(std::string defaultVal) -{ - double val = 0.0f; - if(!defaultVal.empty()) - { - std::stringstream stream; - stream << defaultVal; - stream >> val; - } - - PyObject* pyval = PyFloat_FromDouble(val); - - if (PyErr_Occurred()) - { - PyErr_Clear(); - PyErr_Format(PyExc_TypeError, "DoubleType::parseDefaultStr: defaultVal(%s) error! val=[%s]", - pyval != NULL ? pyval->ob_type->tp_name : "NULL", defaultVal.c_str()); - - PyErr_PrintEx(0); - - S_RELEASE(pyval); - return PyFloat_FromDouble(0.0f); - } - - return pyval; -} - -//------------------------------------------------------------------------------------- -void DoubleType::addToStream(MemoryStream* mstream, PyObject* pyValue) -{ - if(!PyFloat_Check(pyValue)) - { - PyErr_Format(PyExc_TypeError, "DoubleType::addToStream: pyValue(%s) is wrong!", - (pyValue == NULL) ? "NULL": pyValue->ob_type->tp_name); - - PyErr_PrintEx(0); - return; - } - - (*mstream) << PyFloat_AsDouble(pyValue); -} - -//------------------------------------------------------------------------------------- -PyObject* DoubleType::createFromStream(MemoryStream* mstream) -{ - double val = 0.0; - if(mstream) - (*mstream) >> val; - - PyObject* pyval = PyFloat_FromDouble(val); - - if (PyErr_Occurred()) - { - PyErr_Format(PyExc_TypeError, "UInt32Type::createFromStream: errval=%lf, default return is 0", val); - PyErr_PrintEx(0); - S_RELEASE(pyval); - return PyFloat_FromDouble(0); - } - - return pyval; -} - -//------------------------------------------------------------------------------------- -Vector2Type::Vector2Type(DATATYPE_UID did) : -DataType(did) -{ -} - -//------------------------------------------------------------------------------------- -Vector2Type::~Vector2Type() -{ -} - -//------------------------------------------------------------------------------------- -bool Vector2Type::isSameType(PyObject* pyValue) -{ - if(pyValue == NULL) - { - PyErr_Format(PyExc_TypeError, - "must be set to a VECTOR2 type."); - - PyErr_PrintEx(0); - return false; - } - - if(!PySequence_Check(pyValue) || (uint32)PySequence_Size(pyValue) != 2) - { - PyErr_Format(PyExc_TypeError, - "must be set to a VECTOR2 type."); - - PyErr_PrintEx(0); - return false; - } - - for(uint32 index=0; index<2; ++index) - { - PyObject* pyVal = PySequence_GetItem(pyValue, index); - if(!PyFloat_Check(pyVal) && !PyLong_Check(pyVal) && !PyLong_AsLongLong(pyVal)) - { - PyErr_Format(PyExc_TypeError, - "VECTOR2 item is not digit."); - - PyErr_PrintEx(0); - return false; - } - - Py_DECREF(pyVal); - } - - return true; -} - -//------------------------------------------------------------------------------------- -PyObject* Vector2Type::parseDefaultStr(std::string defaultVal) -{ - float x = 0.0f, y = 0.0f; - - if (!defaultVal.empty()) - { - std::vector result; - if (isVector2(defaultVal, result)) - { - x = result[0]; - y = result[1]; - return new script::ScriptVector2(float(x), float(y)); - } - - PyErr_Format(PyExc_TypeError, "Vector2::parseDefaultStr: defaultVal=%s format error!", defaultVal.c_str()); - PyErr_PrintEx(0); - } - - return new script::ScriptVector2(float(x), float(y)); -} - -//------------------------------------------------------------------------------------- -void Vector2Type::addToStream(MemoryStream* mstream, PyObject* pyValue) -{ - for(ArraySize index=0; index<2; ++index) - { - PyObject* pyVal = PySequence_GetItem(pyValue, index); -#ifdef CLIENT_NO_FLOAT - int32 v = (int32)PyFloat_AsDouble(pyVal); -#else - float v = (float)PyFloat_AsDouble(pyVal); -#endif - (*mstream) << v; - Py_DECREF(pyVal); - } -} - -//------------------------------------------------------------------------------------- -PyObject* Vector2Type::createFromStream(MemoryStream* mstream) -{ -#ifdef CLIENT_NO_FLOAT - int32 x = 0, y = 0; -#else - float x = 0.0f, y = 0.0f; -#endif - - if(mstream) - (*mstream) >> x >> y; - - return new script::ScriptVector2(float(x), float(y)); -} - -//------------------------------------------------------------------------------------- -Vector3Type::Vector3Type(DATATYPE_UID did) : -DataType(did) -{ -} - -//------------------------------------------------------------------------------------- -Vector3Type::~Vector3Type() -{ -} - -//------------------------------------------------------------------------------------- -bool Vector3Type::isSameType(PyObject* pyValue) -{ - if (pyValue == NULL) - { - PyErr_Format(PyExc_TypeError, - "must be set to a VECTOR3 type."); - - PyErr_PrintEx(0); - return false; - } - - if (!PySequence_Check(pyValue) || (uint32)PySequence_Size(pyValue) != 3) - { - PyErr_Format(PyExc_TypeError, - "must be set to a VECTOR3 type."); - - PyErr_PrintEx(0); - return false; - } - - for (uint32 index = 0; index<3; ++index) - { - PyObject* pyVal = PySequence_GetItem(pyValue, index); - if (!PyFloat_Check(pyVal) && !PyLong_Check(pyVal) && !PyLong_AsLongLong(pyVal)) - { - PyErr_Format(PyExc_TypeError, - "VECTOR3 item is not digit."); - - PyErr_PrintEx(0); - return false; - } - - Py_DECREF(pyVal); - } - - return true; -} - -//------------------------------------------------------------------------------------- -PyObject* Vector3Type::parseDefaultStr(std::string defaultVal) -{ - float x = 0.0f, y = 0.0f, z = 0.0f; - - if (!defaultVal.empty()) - { - std::vector result; - if (isVector3(defaultVal, result)) - { - x = result[0]; - y = result[1]; - z = result[2]; - return new script::ScriptVector3(float(x), float(y), float(z)); - } - - PyErr_Format(PyExc_TypeError, "Vector3::parseDefaultStr: defaultVal=%s format error!", defaultVal.c_str()); - PyErr_PrintEx(0); - } - - return new script::ScriptVector3(float(x), float(y), float(z)); -} - -//------------------------------------------------------------------------------------- -void Vector3Type::addToStream(MemoryStream* mstream, PyObject* pyValue) -{ - for (ArraySize index = 0; index<3; ++index) - { - PyObject* pyVal = PySequence_GetItem(pyValue, index); -#ifdef CLIENT_NO_FLOAT - int32 v = (int32)PyFloat_AsDouble(pyVal); -#else - float v = (float)PyFloat_AsDouble(pyVal); -#endif - (*mstream) << v; - Py_DECREF(pyVal); - } -} - -//------------------------------------------------------------------------------------- -PyObject* Vector3Type::createFromStream(MemoryStream* mstream) -{ -#ifdef CLIENT_NO_FLOAT - int32 x = 0, y = 0, z = 0; -#else - float x = 0.0f, y = 0.0f, z = 0.0f; -#endif - - if (mstream) - (*mstream) >> x >> y >> z; - - return new script::ScriptVector3(float(x), float(y), float(z)); -} - -//------------------------------------------------------------------------------------- -Vector4Type::Vector4Type(DATATYPE_UID did) : -DataType(did) -{ -} - -//------------------------------------------------------------------------------------- -Vector4Type::~Vector4Type() -{ -} - -//------------------------------------------------------------------------------------- -bool Vector4Type::isSameType(PyObject* pyValue) -{ - if (pyValue == NULL) - { - PyErr_Format(PyExc_TypeError, - "must be set to a VECTOR4 type."); - - PyErr_PrintEx(0); - return false; - } - - if (!PySequence_Check(pyValue) || (uint32)PySequence_Size(pyValue) != 4) - { - PyErr_Format(PyExc_TypeError, - "must be set to a VECTOR4 type."); - - PyErr_PrintEx(0); - return false; - } - - for (uint32 index = 0; index<4; ++index) - { - PyObject* pyVal = PySequence_GetItem(pyValue, index); - if (!PyFloat_Check(pyVal) && !PyLong_Check(pyVal) && !PyLong_AsLongLong(pyVal)) - { - PyErr_Format(PyExc_TypeError, - "VECTOR4 item is not digit."); - - PyErr_PrintEx(0); - return false; - } - - Py_DECREF(pyVal); - } - - return true; -} - -//------------------------------------------------------------------------------------- -PyObject* Vector4Type::parseDefaultStr(std::string defaultVal) -{ - float x = 0.0f, y = 0.0f, z = 0.0f, w = 0.0f; - - if (!defaultVal.empty()) - { - std::vector result; - if (isVector4(defaultVal, result)) - { - x = result[0]; - y = result[1]; - z = result[2]; - w = result[3]; - return new script::ScriptVector4(float(x), float(y), float(z), float(w)); - } - - PyErr_Format(PyExc_TypeError, "Vector4::parseDefaultStr: defaultVal=%s format error!", defaultVal.c_str()); - PyErr_PrintEx(0); - } - - return new script::ScriptVector4(float(x), float(y), float(z), float(w)); -} - -//------------------------------------------------------------------------------------- -void Vector4Type::addToStream(MemoryStream* mstream, PyObject* pyValue) -{ - for (ArraySize index = 0; index<4; ++index) - { - PyObject* pyVal = PySequence_GetItem(pyValue, index); -#ifdef CLIENT_NO_FLOAT - int32 v = (int32)PyFloat_AsDouble(pyVal); -#else - float v = (float)PyFloat_AsDouble(pyVal); -#endif - (*mstream) << v; - Py_DECREF(pyVal); - } -} - -//------------------------------------------------------------------------------------- -PyObject* Vector4Type::createFromStream(MemoryStream* mstream) -{ -#ifdef CLIENT_NO_FLOAT - int32 x = 0, y = 0, z = 0, w = 0; -#else - float x = 0.0f, y = 0.0f, z = 0.0f, w = 0.0f; -#endif - - if (mstream) - (*mstream) >> x >> y >> z >> w; - - return new script::ScriptVector4(float(x), float(y), float(z), float(w)); -} - -//------------------------------------------------------------------------------------- -StringType::StringType(DATATYPE_UID did): -DataType(did) -{ -} - -//------------------------------------------------------------------------------------- -StringType::~StringType() -{ -} - -//------------------------------------------------------------------------------------- -bool StringType::isSameType(PyObject* pyValue) -{ - if(pyValue == NULL) - { - OUT_TYPE_ERROR("STRING"); - return false; - } - - bool ret = PyUnicode_Check(pyValue); - if(!ret) - { - OUT_TYPE_ERROR("STRING"); - } - else - { - PyObject* pyfunc = PyObject_GetAttrString(pyValue, "encode"); - if(pyfunc == NULL) - { - SCRIPT_ERROR_CHECK(); - ret = false; - } - else - { - PyObject* pyRet = PyObject_CallFunction(pyfunc, - const_cast("(s)"), "ascii"); - - S_RELEASE(pyfunc); - - if(!pyRet) - { - SCRIPT_ERROR_CHECK(); - ret = false; - } - else - { - S_RELEASE(pyRet); - } - } - } - - return ret; -} - -//------------------------------------------------------------------------------------- -PyObject* StringType::parseDefaultStr(std::string defaultVal) -{ - PyObject* pyobj = PyUnicode_FromString(defaultVal.c_str()); - - if (pyobj && !PyErr_Occurred()) - return pyobj; - - PyErr_Clear(); - PyErr_Format(PyExc_TypeError, "StringType::parseDefaultStr: defaultVal(%s) error! val=[%s]", - pyobj != NULL ? pyobj->ob_type->tp_name : "NULL", defaultVal.c_str()); - - PyErr_PrintEx(0); - S_RELEASE(pyobj); - - return PyUnicode_FromString(""); -} - -//------------------------------------------------------------------------------------- -void StringType::addToStream(MemoryStream* mstream, PyObject* pyValue) -{ - const char* s = PyUnicode_AsUTF8AndSize(pyValue, NULL); - - if (s == NULL) - { - OUT_TYPE_ERROR("STRING"); - return; - } - - (*mstream) << s; -} - -//------------------------------------------------------------------------------------- -PyObject* StringType::createFromStream(MemoryStream* mstream) -{ - std::string val = ""; - if(mstream) - (*mstream) >> val; - - PyObject* pyobj = PyUnicode_FromString(val.c_str()); - - if (pyobj && !PyErr_Occurred()) - return pyobj; - - PyErr_PrintEx(0); - S_RELEASE(pyobj); - - return NULL; -} - -//------------------------------------------------------------------------------------- -UnicodeType::UnicodeType(DATATYPE_UID did): -DataType(did) -{ -} - -//------------------------------------------------------------------------------------- -UnicodeType::~UnicodeType() -{ -} - -//------------------------------------------------------------------------------------- -bool UnicodeType::isSameType(PyObject* pyValue) -{ - if(pyValue == NULL) - { - OUT_TYPE_ERROR("UNICODE"); - return false; - } - - bool ret = PyUnicode_Check(pyValue); - if(!ret) - OUT_TYPE_ERROR("UNICODE"); - - return ret; -} - -//------------------------------------------------------------------------------------- -PyObject* UnicodeType::parseDefaultStr(std::string defaultVal) -{ - PyObject* pyobj = PyUnicode_DecodeUTF8(defaultVal.data(), defaultVal.size(), ""); - - if(pyobj && !PyErr_Occurred()) - { - return pyobj; - } - - PyErr_Clear(); - PyErr_Format(PyExc_TypeError, "UnicodeType::parseDefaultStr: defaultVal(%s) error! val=[%s]", - pyobj != NULL ? pyobj->ob_type->tp_name : "NULL", defaultVal.c_str()); - - PyErr_PrintEx(0); - S_RELEASE(pyobj); - - return PyUnicode_DecodeUTF8("", 0, ""); -} - -//------------------------------------------------------------------------------------- -void UnicodeType::addToStream(MemoryStream* mstream, PyObject* pyValue) -{ - Py_ssize_t size; - const char* s = PyUnicode_AsUTF8AndSize(pyValue, &size); - - if (s == NULL) - { - OUT_TYPE_ERROR("UNICODE"); - return; - } - - mstream->appendBlob(s, size); -} - -//------------------------------------------------------------------------------------- -PyObject* UnicodeType::createFromStream(MemoryStream* mstream) -{ - std::string val = ""; - if(mstream) - { - mstream->readBlob(val); - } - - PyObject* pyobj = PyUnicode_DecodeUTF8(val.data(), val.size(), ""); - - if(pyobj && !PyErr_Occurred()) - { - return pyobj; - } - - S_RELEASE(pyobj); - ::PyErr_PrintEx(0); - - return NULL; -} - -//------------------------------------------------------------------------------------- -PythonType::PythonType(DATATYPE_UID did): -DataType(did) -{ -} - -//------------------------------------------------------------------------------------- -PythonType::~PythonType() -{ -} - -//------------------------------------------------------------------------------------- -bool PythonType::isSameType(PyObject* pyValue) -{ - if(pyValue == NULL) - { - OUT_TYPE_ERROR("PYTHON"); - return false; - } - - bool ret = script::Pickler::pickle(pyValue).empty(); - if(ret) - OUT_TYPE_ERROR("PYTHON"); - - return !ret; -} - -//------------------------------------------------------------------------------------- -PyObject* PythonType::parseDefaultStr(std::string defaultVal) -{ - if(defaultVal.size() > 0) - { - PyObject* module = PyImport_AddModule("__main__"); - if(module == NULL) - { - PyErr_SetString(PyExc_SystemError, - "PythonType::createObject:PyImport_AddModule __main__ error!"); - - PyErr_PrintEx(0); - S_Return; - } - - PyObject* mdict = PyModule_GetDict(module); // Borrowed reference. - - PyObject* result = PyRun_String(const_cast(defaultVal.c_str()), - Py_eval_input, mdict, mdict); - - if (result == NULL) - { - SCRIPT_ERROR_CHECK(); - S_Return; - } - - return result; - } - - S_Return; -} - -//------------------------------------------------------------------------------------- -void PythonType::addToStream(MemoryStream* mstream, PyObject* pyValue) -{ - std::string datas = script::Pickler::pickle(pyValue); - mstream->appendBlob(datas); -} - -//------------------------------------------------------------------------------------- -PyObject* PythonType::createFromStream(MemoryStream* mstream) -{ - std::string datas = ""; - mstream->readBlob(datas); - - if (datas.size() == 0) - Py_RETURN_NONE; - - return script::Pickler::unpickle(datas); -} - -//------------------------------------------------------------------------------------- -PyDictType::PyDictType(DATATYPE_UID did): -PythonType(did) -{ -} - -//------------------------------------------------------------------------------------- -PyDictType::~PyDictType() -{ -} - -//------------------------------------------------------------------------------------- -bool PyDictType::isSameType(PyObject* pyValue) -{ - if(pyValue == NULL) - { - OUT_TYPE_ERROR("PY_DICT"); - return false; - } - - if(!PyDict_Check(pyValue)) - { - OUT_TYPE_ERROR("PY_DICT"); - return false; - } - - return true; -} - -//------------------------------------------------------------------------------------- -PyObject* PyDictType::parseDefaultStr(std::string defaultVal) -{ - PyObject* pyVal = PythonType::parseDefaultStr(defaultVal); - if(PyDict_Check(pyVal)) - { - return pyVal; - } - - if(pyVal) - Py_DECREF(pyVal); - - return PyDict_New(); -} - -//------------------------------------------------------------------------------------- -PyObject* PyDictType::createFromStream(MemoryStream* mstream) -{ - std::string datas = ""; - mstream->readBlob(datas); - - if (datas.size() == 0) - return PyDict_New(); - - return script::Pickler::unpickle(datas); -} - -//------------------------------------------------------------------------------------- -PyTupleType::PyTupleType(DATATYPE_UID did): -PythonType(did) -{ -} - -//------------------------------------------------------------------------------------- -PyTupleType::~PyTupleType() -{ -} - -//------------------------------------------------------------------------------------- -bool PyTupleType::isSameType(PyObject* pyValue) -{ - if(pyValue == NULL) - { - OUT_TYPE_ERROR("PY_TUPLE"); - return false; - } - - if(!PyTuple_Check(pyValue)) - { - OUT_TYPE_ERROR("PY_TUPLE"); - return false; - } - - return true; -} - -//------------------------------------------------------------------------------------- -PyObject* PyTupleType::parseDefaultStr(std::string defaultVal) -{ - PyObject* pyVal = PythonType::parseDefaultStr(defaultVal); - if(PyTuple_Check(pyVal)) - { - return pyVal; - } - - if(pyVal) - Py_DECREF(pyVal); - - return PyTuple_New(0); -} - -//------------------------------------------------------------------------------------- -PyObject* PyTupleType::createFromStream(MemoryStream* mstream) -{ - std::string datas = ""; - mstream->readBlob(datas); - - if (datas.size() == 0) - return PyTuple_New(0); - - return script::Pickler::unpickle(datas); -} - -//------------------------------------------------------------------------------------- -PyListType::PyListType(DATATYPE_UID did): -PythonType(did) -{ -} - -//------------------------------------------------------------------------------------- -PyListType::~PyListType() -{ -} - -//------------------------------------------------------------------------------------- -bool PyListType::isSameType(PyObject* pyValue) -{ - if(pyValue == NULL) - { - OUT_TYPE_ERROR("PY_LIST"); - return false; - } - - if(!PyList_Check(pyValue)) - { - OUT_TYPE_ERROR("PY_LIST"); - return false; - } - - return true; -} - -//------------------------------------------------------------------------------------- -PyObject* PyListType::parseDefaultStr(std::string defaultVal) -{ - PyObject* pyVal = PythonType::parseDefaultStr(defaultVal); - if(PyList_Check(pyVal)) - { - return pyVal; - } - - if(pyVal) - Py_DECREF(pyVal); - - return PyList_New(0); -} - -//------------------------------------------------------------------------------------- -PyObject* PyListType::createFromStream(MemoryStream* mstream) -{ - std::string datas = ""; - mstream->readBlob(datas); - - if (datas.size() == 0) - return PyList_New(0); - - return script::Pickler::unpickle(datas); -} - -//------------------------------------------------------------------------------------- -BlobType::BlobType(DATATYPE_UID did): -DataType(did) -{ -} - -//------------------------------------------------------------------------------------- -BlobType::~BlobType() -{ -} - -//------------------------------------------------------------------------------------- -bool BlobType::isSameType(PyObject* pyValue) -{ - if(pyValue == NULL) - { - OUT_TYPE_ERROR("BLOB"); - return false; - } - - if (!PyBytes_Check(pyValue) && - !PyObject_TypeCheck(pyValue, script::PyMemoryStream::getScriptType())) - { - OUT_TYPE_ERROR("BLOB"); - return false; - } - - return true; -} - -//------------------------------------------------------------------------------------- -PyObject* BlobType::parseDefaultStr(std::string defaultVal) -{ - return PyBytes_FromStringAndSize(defaultVal.data(), defaultVal.size()); -} - -//------------------------------------------------------------------------------------- -void BlobType::addToStream(MemoryStream* mstream, PyObject* pyValue) -{ - if (!PyBytes_Check(pyValue)) - { - script::PyMemoryStream* pPyMemoryStream = static_cast(pyValue); - MemoryStream& m = pPyMemoryStream->stream(); - mstream->appendBlob((const char*)m.data() + m.rpos(), (ArraySize)m.length()); - } - else - { - Py_ssize_t datasize = PyBytes_GET_SIZE(pyValue); - char* datas = PyBytes_AsString(pyValue); - mstream->appendBlob(datas, (ArraySize)datasize); - } -} - -//------------------------------------------------------------------------------------- -PyObject* BlobType::createFromStream(MemoryStream* mstream) -{ - std::string datas; - mstream->readBlob(datas); - return PyBytes_FromStringAndSize(datas.data(), datas.size()); -} - -//------------------------------------------------------------------------------------- -EntityCallType::EntityCallType(DATATYPE_UID did): -DataType(did) -{ -} - -//------------------------------------------------------------------------------------- -EntityCallType::~EntityCallType() -{ -} - -//------------------------------------------------------------------------------------- -bool EntityCallType::isSameType(PyObject* pyValue) -{ - if(pyValue == NULL) - { - OUT_TYPE_ERROR("ENTITYCALL"); - return false; - } - - if(!(PyObject_TypeCheck(pyValue, EntityCall::getScriptType()) || pyValue == Py_None)) - { - PyTypeObject* type = script::ScriptObject::getScriptObjectType("Entity"); - if(!type || !(PyObject_IsInstance(pyValue, (PyObject *)type))) - { - OUT_TYPE_ERROR("ENTITYCALL"); - return false; - } - } - - return true; -} - -//------------------------------------------------------------------------------------- -PyObject* EntityCallType::parseDefaultStr(std::string defaultVal) -{ - Py_RETURN_NONE; -} - -//------------------------------------------------------------------------------------- -void EntityCallType::addToStream(MemoryStream* mstream, PyObject* pyValue) -{ - COMPONENT_ID cid = 0; - ENTITY_ID id = 0; - uint16 type = 0; - ENTITY_SCRIPT_UID utype = 0; - - if(pyValue != Py_None) - { - PyTypeObject* stype = script::ScriptObject::getScriptObjectType("Entity"); - { - // Ƿһentity? - if(PyObject_IsInstance(pyValue, (PyObject *)stype)) - { - PyObject* pyid = PyObject_GetAttrString(pyValue, "id"); - - if (pyid) - { - id = PyLong_AsLong(pyid); - Py_DECREF(pyid); - - cid = g_componentID; - - if (g_componentType == BASEAPP_TYPE) - type = (uint16)ENTITYCALL_TYPE_BASE; - else if (g_componentType == CELLAPP_TYPE) - type = (uint16)ENTITYCALL_TYPE_CELL; - else - type = (uint16)ENTITYCALL_TYPE_CLIENT; - - PyObject* pyClass = PyObject_GetAttrString(pyValue, "__class__"); - PyObject* pyClassName = PyObject_GetAttrString(pyClass, "__name__"); - - const char* ccattr = PyUnicode_AsUTF8AndSize(pyClassName, NULL); - - Py_DECREF(pyClass); - Py_DECREF(pyClassName); - - ScriptDefModule* pScriptDefModule = EntityDef::findScriptModule(ccattr); - - utype = pScriptDefModule->getUType(); - } - else - { - // ijЩ»ΪNULL 磺ʹweakproxyentityCallѾΪNULL - SCRIPT_ERROR_CHECK(); - id = 0; - cid = 0; - } - } - } - - // ֻentityCall - if(id == 0) - { - EntityCallAbstract* pEntityCallAbstract = static_cast(pyValue); - cid = pEntityCallAbstract->componentID(); - id = pEntityCallAbstract->id(); - type = static_cast(pEntityCallAbstract->type());; - utype = pEntityCallAbstract->utype(); - } - } - - (*mstream) << id; - (*mstream) << cid; - (*mstream) << type; - (*mstream) << utype; -} - -//------------------------------------------------------------------------------------- -PyObject* EntityCallType::createFromStream(MemoryStream* mstream) -{ - if(mstream) - { - COMPONENT_ID cid = 0; - ENTITY_ID id = 0; - uint16 type; - ENTITY_SCRIPT_UID utype; - - (*mstream) >> id >> cid >> type >> utype; - - // Py_None - if(id > 0) - { - if (entityCallType2ComponentType((ENTITYCALL_TYPE)type) == g_componentType) - { - PyObject* entity = EntityDef::tryGetEntity(cid, id); - if (entity != NULL) - { - Py_INCREF(entity); - return entity; - } - } - - return new EntityCall(EntityDef::findScriptModule(utype), NULL, cid, - id, (ENTITYCALL_TYPE)type); - } - } - - Py_RETURN_NONE; -} - -//------------------------------------------------------------------------------------- -FixedArrayType::FixedArrayType(DATATYPE_UID did): -DataType(did) -{ -} - -//------------------------------------------------------------------------------------- -FixedArrayType::~FixedArrayType() -{ - if(dataType_) - dataType_->decRef(); -} - -//------------------------------------------------------------------------------------- -PyObject* FixedArrayType::createNewItemFromObj(PyObject* pyobj) -{ - if(!isSameItemType(pyobj)) - { - Py_RETURN_NONE; - } - - return dataType_->createNewFromObj(pyobj); -} - -//------------------------------------------------------------------------------------- -PyObject* FixedArrayType::createNewFromObj(PyObject* pyobj) -{ - if(!isSameType(pyobj)) - { - Py_RETURN_NONE; - } - - if(PyObject_TypeCheck(pyobj, FixedArray::getScriptType())) - { - Py_INCREF(pyobj); - return pyobj; - } - - FixedArray* pFixedArray = new FixedArray(this); - pFixedArray->initialize(pyobj); - return pFixedArray; -} - -//------------------------------------------------------------------------------------- -bool FixedArrayType::initialize(XML* xml, TiXmlNode* node, const std::string& parentName) -{ - dataType_ = NULL; - - TiXmlNode* arrayNode = xml->enterNode(node, "of"); - if (arrayNode == NULL) - { - ERROR_MSG("FixedArrayType::initialize: not found \"of\".\n"); - return false; - } - - std::string strType = xml->getValStr(arrayNode); - - if (strType == "ARRAY") - { - FixedArrayType* dataType = new FixedArrayType(); - - std::string childName = std::string("_") + parentName + - dataType->aliasName() + "_ArrayType"; - - if (dataType->initialize(xml, arrayNode, childName)) - { - dataType_ = dataType; - dataType_->incRef(); - - DataTypes::addDataType(childName, dataType); - } - else - { - ERROR_MSG("FixedArrayType::initialize: Array is wrong.\n"); - delete dataType; - return false; - } - } - else - { - DataType* dataType = DataTypes::getDataType(strType); - if (dataType != NULL) - { - dataType_ = dataType; - dataType_->incRef(); - } - else - { - ERROR_MSG(fmt::format("FixedArrayType::initialize: key[{}] did not find type[{}]!\n", - "ARRAY", strType.c_str())); - - return false; - } - } - - if (dataType_ == NULL) - { - ERROR_MSG("FixedArrayType::initialize: dataType is NULL.\n"); - return false; - } - - DATATYPE_UID uid = dataType_->id(); - EntityDef::md5().append((void*)&uid, sizeof(DATATYPE_UID)); - EntityDef::md5().append((void*)strType.c_str(), (int)strType.size()); - return true; -} - -//------------------------------------------------------------------------------------- -bool FixedArrayType::initialize(script::entitydef::DefContext* pDefContext, const std::string& parentName) -{ - KBE_ASSERT(pDefContext->type == script::entitydef::DefContext::DC_TYPE_FIXED_ARRAY); - - dataType_ = NULL; - std::string strType; - - if (pDefContext->propertys.size() != 1) - { - ERROR_MSG(fmt::format("PyEntityDef::FixedArrayType::initialize: parse {} error, not found item type! file: \"{}\"\n", - pDefContext->moduleName.c_str(), pDefContext->pyObjectSourceFile)); - - return false; - } - - script::entitydef::DefContext& defContextItemType = pDefContext->propertys[0]; - KBE_ASSERT(defContextItemType.type == script::entitydef::DefContext::DC_TYPE_FIXED_ITEM); - - script::entitydef::DefContext* pDefContextItem = script::entitydef::DefContext::findDefContext(defContextItemType.returnType); - if (pDefContextItem) - { - if (pDefContextItem->type == script::entitydef::DefContext::DC_TYPE_FIXED_ARRAY) - { - FixedArrayType* dataType = new FixedArrayType(); - - std::string childName = std::string("_") + parentName + dataType->aliasName() + "_ArrayType"; - strType += childName; - - if (dataType->initialize(pDefContextItem, childName)) - { - dataType_ = dataType; - dataType_->incRef(); - DataTypes::addDataType(childName, dataType); - } - else - { - ERROR_MSG("PyEntityDef::FixedArrayType::initialize: Array is wrong.\n"); - delete dataType; - return false; - } - } - else - { - goto FIND_IN_DATATYPES; - } - } - else - { - FIND_IN_DATATYPES: - dataType_ = DataTypes::getDataType(defContextItemType.returnType, false); - if (dataType_ != NULL) - { - dataType_->incRef(); - } - else - { - ERROR_MSG(fmt::format("PyEntityDef::registerDefTypes: parse {} error, item({}={}), not a legal data type, file: \"{}\"\n", - pDefContext->moduleName.c_str(), defContextItemType.attrName, defContextItemType.returnType, defContextItemType.pyObjectSourceFile)); - - return false; - } - } - - if (dataType_ == NULL) - { - ERROR_MSG("PyEntityDef::FixedArrayType::initialize: dataType is NULL.\n"); - return false; - } - - strType += defContextItemType.returnType; - DATATYPE_UID uid = dataType_->id(); - EntityDef::md5().append((void*)&uid, sizeof(DATATYPE_UID)); - EntityDef::md5().append((void*)strType.c_str(), (int)strType.size()); - return true; -} - -//------------------------------------------------------------------------------------- -bool FixedArrayType::isSameItemType(PyObject* pyValue) -{ - return dataType_->isSameType(pyValue); -} - -//------------------------------------------------------------------------------------- -bool FixedArrayType::isSameType(PyObject* pyValue) -{ - if(pyValue == NULL) - { - OUT_TYPE_ERROR("ARRAY"); - return false; - } - - if(!PySequence_Check(pyValue)) - { - OUT_TYPE_ERROR("ARRAY"); - return false; - } - - Py_ssize_t size = PySequence_Size(pyValue); - for(Py_ssize_t i=0; iisSameType(pyVal); - Py_DECREF(pyVal); - - if(!ok) - return false; - } - - return true; -} - -//------------------------------------------------------------------------------------- -PyObject* FixedArrayType::parseDefaultStr(std::string defaultVal) -{ - FixedArray* pFixedArray = new FixedArray(this); - pFixedArray->initialize(defaultVal); - return pFixedArray; -} - -//------------------------------------------------------------------------------------- -void FixedArrayType::addToStream(MemoryStream* mstream, PyObject* pyValue) -{ - addToStreamEx(mstream, pyValue, false); -} - -//------------------------------------------------------------------------------------- -void FixedArrayType::addToStreamEx(MemoryStream* mstream, PyObject* pyValue, bool onlyPersistents) -{ - ArraySize size = (ArraySize)PySequence_Size(pyValue); - (*mstream) << size; - - for(ArraySize i=0; itype() == DATA_TYPE_FIXEDDICT) - ((FixedDictType*)dataType_)->addToStreamEx(mstream, pyVal, onlyPersistents); - else if(dataType_->type() == DATA_TYPE_FIXEDARRAY) - ((FixedArrayType*)dataType_)->addToStreamEx(mstream, pyVal, onlyPersistents); - else - dataType_->addToStream(mstream, pyVal); - - Py_DECREF(pyVal); - } -} - -//------------------------------------------------------------------------------------- -PyObject* FixedArrayType::createFromStream(MemoryStream* mstream) -{ - return createFromStreamEx(mstream, false); -} - -//------------------------------------------------------------------------------------- -PyObject* FixedArrayType::createFromStreamEx(MemoryStream* mstream, bool onlyPersistents) -{ - ArraySize size; - - FixedArray* pFixedArray = new FixedArray(this); - pFixedArray->initialize(""); - - if(mstream) - { - (*mstream) >> size; - - std::vector& vals = pFixedArray->getValues(); - for(ArraySize i=0; ilength() == 0) - { - ERROR_MSG(fmt::format("FixedArrayType::createFromStream: {} invalid(size={}), stream no space!\n", - aliasName(), size)); - - break; - } - - PyObject* pyVal = NULL; - - if(dataType_->type() == DATA_TYPE_FIXEDDICT) - pyVal = ((FixedDictType*)dataType_)->createFromStreamEx(mstream, onlyPersistents); - else if(dataType_->type() == DATA_TYPE_FIXEDARRAY) - pyVal = ((FixedArrayType*)dataType_)->createFromStreamEx(mstream, onlyPersistents); - else - pyVal = dataType_->createFromStream(mstream); - - if(pyVal) - { - vals.push_back(pyVal); - } - else - { - ERROR_MSG(fmt::format("FixedArrayType::createFromStream: {}, pyVal is NULL! size={}\n", - aliasName(), size)); - - break; - } - } - } - - return pFixedArray; -} - -//------------------------------------------------------------------------------------- -FixedDictType::FixedDictType(DATATYPE_UID did): -DataType(did), -keyTypes_(), -implObj_(NULL), -pycreateObjFromDict_(NULL), -pygetDictFromObj_(NULL), -pyisSameType_(NULL), -moduleName_() -{ -} - -//------------------------------------------------------------------------------------- -FixedDictType::~FixedDictType() -{ - FIXEDDICT_KEYTYPE_MAP::iterator iter = keyTypes_.begin(); - for(; iter != keyTypes_.end(); ++iter) - { - iter->second->dataType->decRef(); - } - - keyTypes_.clear(); - - S_RELEASE(pycreateObjFromDict_); - S_RELEASE(pygetDictFromObj_); - S_RELEASE(pyisSameType_); - S_RELEASE(implObj_); -} - -//------------------------------------------------------------------------------------- -std::string FixedDictType::getKeyNames(void) -{ - std::string keyNames = ""; - - FIXEDDICT_KEYTYPE_MAP::iterator iter = keyTypes_.begin(); - for(; iter != keyTypes_.end(); ++iter) - { - keyNames += iter->first + ","; - } - - return keyNames; -} - -//------------------------------------------------------------------------------------- -std::string FixedDictType::debugInfos(void) -{ - std::string retstr = ""; - - FIXEDDICT_KEYTYPE_MAP::iterator iter = keyTypes_.begin(); - for(; iter != keyTypes_.end(); ++iter) - { - retstr += iter->first; - retstr += "("; - retstr += iter->second->dataType->aliasName(); - retstr += "), "; - } - - if(retstr.size() > 0) - retstr.erase(retstr.size() - 2, 2); - - return retstr; -} - -//------------------------------------------------------------------------------------- -std::string FixedDictType::getNotFoundKeys(PyObject* dict) -{ - std::string notFoundKeys = ""; - - FIXEDDICT_KEYTYPE_MAP::iterator iter = keyTypes_.begin(); - for (; iter != keyTypes_.end(); ++iter) - { - PyObject* pyObject = PyDict_GetItemString(dict, const_cast(iter->first.c_str())); - if (pyObject == NULL) - { - notFoundKeys += iter->first.c_str(); - notFoundKeys += ", "; - - if (PyErr_Occurred()) - PyErr_Clear(); - } - } - - if (notFoundKeys.size() > 0) - notFoundKeys.erase(notFoundKeys.size() - 2, 2); - - return notFoundKeys; -} - -//------------------------------------------------------------------------------------- -PyObject* FixedDictType::createNewItemFromObj(const char* keyName, PyObject* pyobj) -{ - DataType* dataType = isSameItemType(keyName, pyobj); - if(!dataType) - { - Py_RETURN_NONE; - } - - FIXEDDICT_KEYTYPE_MAP::iterator iter = keyTypes_.begin(); - for(; iter != keyTypes_.end(); ++iter) - { - if(iter->first == keyName) - { - return iter->second->dataType->createNewFromObj(pyobj); - } - } - - Py_RETURN_NONE; -} - -//------------------------------------------------------------------------------------- -PyObject* FixedDictType::createNewFromObj(PyObject* pyobj) -{ - if(!isSameType(pyobj)) - { - Py_RETURN_NONE; - } - - if(this->hasImpl()) - { - return impl_createObjFromDict(pyobj); - } - - // ڴʱѾFixedDict, ΪparseDefaultStr - // ʼΪն - if(PyObject_TypeCheck(pyobj, FixedDict::getScriptType())) - { - Py_INCREF(pyobj); - return pyobj; - } - - FixedDict* pFixedDict = new FixedDict(this); - pFixedDict->initialize(pyobj); - return pFixedDict; -} - -//------------------------------------------------------------------------------------- -bool FixedDictType::initialize(XML* xml, TiXmlNode* node, std::string& parentName) -{ - TiXmlNode* propertiesNode = xml->enterNode(node, "Properties"); - if(propertiesNode == NULL) - { - ERROR_MSG("FixedDictType::initialize: not found \"Properties\".\n"); - return false; - } - - std::string strType = "", typeName = ""; - - XML_FOR_BEGIN(propertiesNode) - { - typeName = xml->getKey(propertiesNode); - - TiXmlNode* typeNode = xml->enterNode(propertiesNode->FirstChild(), "Type"); - TiXmlNode* PersistentNode = xml->enterNode(propertiesNode->FirstChild(), "Persistent"); - TiXmlNode* DatabaseLengthNode = xml->enterNode(propertiesNode->FirstChild(), "DatabaseLength"); - - bool persistent = true; - if(PersistentNode) - { - std::string strval = xml->getValStr(PersistentNode); - if(strval == "false") - { - persistent = false; - } - } - - uint32 databaseLength = 0; - if (DatabaseLengthNode) - { - databaseLength = xml->getValInt(DatabaseLengthNode); - } - - if(typeNode) - { - strType = xml->getValStr(typeNode); - - if(strType == "ARRAY") - { - FixedArrayType* dataType = new FixedArrayType(); - DictItemDataTypePtr pDictItemDataType(new DictItemDataType()); - pDictItemDataType->dataType = dataType; - - std::string childName = std::string("_") + parentName + std::string("_") + typeName + "_ArrayType"; - if (dataType->initialize(xml, typeNode, childName)) - { - DATATYPE_UID uid = dataType->id(); - EntityDef::md5().append((void*)&uid, sizeof(DATATYPE_UID)); - EntityDef::md5().append((void*)strType.c_str(), (int)strType.size()); - EntityDef::md5().append((void*)typeName.c_str(), (int)typeName.size()); - - keyTypes_.push_back(std::pair< std::string, DictItemDataTypePtr >(typeName, pDictItemDataType)); - dataType->incRef(); - - if (dataType->getDataType()->type() == DATA_TYPE_ENTITYCALL) - { - persistent = false; - } - - pDictItemDataType->persistent = persistent; - pDictItemDataType->databaseLength = databaseLength; - EntityDef::md5().append((void*)&persistent, sizeof(bool)); - EntityDef::md5().append((void*)&databaseLength, sizeof(uint32)); - DataTypes::addDataType(childName, dataType); - } - else - { - ERROR_MSG(fmt::format("FixedDictType::initialize: key[{}] did not find array-type[{}]!\n", - typeName.c_str(), strType.c_str())); - - return false; - } - } - else - { - DataType* dataType = DataTypes::getDataType(strType); - DictItemDataTypePtr pDictItemDataType(new DictItemDataType()); - pDictItemDataType->dataType = dataType; - - if(dataType != NULL) - { - DATATYPE_UID uid = dataType->id(); - EntityDef::md5().append((void*)&uid, sizeof(DATATYPE_UID)); - EntityDef::md5().append((void*)strType.c_str(), (int)strType.size()); - EntityDef::md5().append((void*)typeName.c_str(), (int)typeName.size()); - - keyTypes_.push_back(std::pair< std::string, DictItemDataTypePtr >(typeName, pDictItemDataType)); - dataType->incRef(); - - if(dataType->type() == DATA_TYPE_ENTITYCALL) - { - persistent = false; - } - - pDictItemDataType->persistent = persistent; - pDictItemDataType->databaseLength = databaseLength; - EntityDef::md5().append((void*)&persistent, sizeof(bool)); - EntityDef::md5().append((void*)&databaseLength, sizeof(uint32)); - } - else - { - ERROR_MSG(fmt::format("FixedDictType::initialize: key[{}] did not find type[{}]!\n", - typeName.c_str(), strType.c_str())); - - return false; - } - } - } - else - { - ERROR_MSG(fmt::format("FixedDictType::initialize: key[{}] no label[\"Type\"], key[{}] will be ignored!\n", - typeName, typeName)); - } - } - XML_FOR_END(propertiesNode); - - TiXmlNode* implementedByNode = xml->enterNode(node, "implementedBy"); - if(implementedByNode) - { - strType = xml->getValStr(implementedByNode); - - if(g_componentType == CELLAPP_TYPE || g_componentType == BASEAPP_TYPE || - g_componentType == CLIENT_TYPE) - { - if(strType.size() > 0 && !loadImplModule(strType)) - return false; - - moduleName_ = strType; - } - - if(strType.size() > 0) - EntityDef::md5().append((void*)strType.c_str(), (int)strType.size()); - } - - if (keyTypes_.size() == 0) - { - ERROR_MSG(fmt::format("FixedDictType::initialize(): FIXED_DICT({}) no keys! \n", - this->aliasName())); - - return false; - } - - return true; -} - -//------------------------------------------------------------------------------------- -bool FixedDictType::initialize(script::entitydef::DefContext* pDefContext, const std::string& parentName) -{ - KBE_ASSERT(pDefContext->type == script::entitydef::DefContext::DC_TYPE_FIXED_DICT); - - if (pDefContext->implementedByModuleName.size() > 0) - { - if (g_componentType == CELLAPP_TYPE || g_componentType == BASEAPP_TYPE || - g_componentType == CLIENT_TYPE) - { - PyObject* implementedBy = pDefContext->implementedBy.get(); - Py_INCREF(implementedBy); - - if (!setImplModule(implementedBy)) - { - ERROR_MSG(fmt::format("PyEntityDef::FixedDictType::initialize(): FIXED_DICT({}) setImplModule error!, file: \"{}\"!\n", - this->aliasName(), pDefContext->pyObjectSourceFile)); - - return false; - } - - moduleName_ = pDefContext->implementedByModuleName; - } - } - - script::entitydef::DefContext::DEF_CONTEXTS::iterator propIter = pDefContext->propertys.begin(); - for (; propIter != pDefContext->propertys.end(); ++propIter) - { - script::entitydef::DefContext& defContextItem = (*propIter); - KBE_ASSERT(defContextItem.type == script::entitydef::DefContext::DC_TYPE_FIXED_ITEM); - - bool persistent = true; - if (defContextItem.persistent != -1) - { - if(defContextItem.persistent <= 0) - persistent = false; - } - - uint32 databaseLength = defContextItem.databaseLength; - - script::entitydef::DefContext* pDefContextItemType = script::entitydef::DefContext::findDefContext(defContextItem.returnType); - if (pDefContextItemType) - { - if (pDefContextItemType->type == script::entitydef::DefContext::DC_TYPE_FIXED_ARRAY) - { - FixedArrayType* dataType = new FixedArrayType(); - DictItemDataTypePtr pDictItemDataType(new DictItemDataType()); - pDictItemDataType->dataType = dataType; - - std::string childName = std::string("_") + parentName + std::string("_") + defContextItem.attrName + "_ArrayType"; - if (dataType->initialize(pDefContextItemType, childName)) - { - DATATYPE_UID uid = dataType->id(); - EntityDef::md5().append((void*)&uid, sizeof(DATATYPE_UID)); - EntityDef::md5().append((void*)defContextItem.attrName.c_str(), (int)defContextItem.attrName.size()); - EntityDef::md5().append((void*)childName.c_str(), (int)childName.size()); - - keyTypes_.push_back(std::pair< std::string, DictItemDataTypePtr >(defContextItem.attrName, pDictItemDataType)); - dataType->incRef(); - - if (dataType->getDataType()->type() == DATA_TYPE_ENTITYCALL) - { - persistent = false; - } - - pDictItemDataType->persistent = persistent; - pDictItemDataType->databaseLength = databaseLength; - EntityDef::md5().append((void*)&persistent, sizeof(bool)); - EntityDef::md5().append((void*)&databaseLength, sizeof(uint32)); - DataTypes::addDataType(childName, dataType); - } - else - { - return false; - } - } - else - { - goto FIND_IN_DATATYPES; - } - } - else - { - FIND_IN_DATATYPES: - DataType* dataType = DataTypes::getDataType(defContextItem.returnType, false); - DictItemDataTypePtr pDictItemDataType(new DictItemDataType()); - pDictItemDataType->dataType = dataType; - - if (dataType != NULL) - { - DATATYPE_UID uid = dataType->id(); - EntityDef::md5().append((void*)&uid, sizeof(DATATYPE_UID)); - EntityDef::md5().append((void*)defContextItem.attrName.c_str(), (int)defContextItem.attrName.size()); - EntityDef::md5().append((void*)defContextItem.returnType.c_str(), (int)defContextItem.returnType.size()); - - keyTypes_.push_back(std::pair< std::string, DictItemDataTypePtr >(defContextItem.attrName, pDictItemDataType)); - dataType->incRef(); - - if (dataType->type() == DATA_TYPE_ENTITYCALL) - { - persistent = false; - } - - pDictItemDataType->persistent = persistent; - pDictItemDataType->databaseLength = databaseLength; - EntityDef::md5().append((void*)&persistent, sizeof(bool)); - EntityDef::md5().append((void*)&databaseLength, sizeof(uint32)); - } - else - { - ERROR_MSG(fmt::format("PyEntityDef::FixedDictType::initialize: {}.{} is not a legal data type[{}], file: \"{}\"!\n", - defContextItem.moduleName.c_str(), defContextItem.attrName.c_str(), defContextItem.returnType, defContextItem.pyObjectSourceFile)); - - return false; - } - } - } - - if (keyTypes_.size() == 0) - { - ERROR_MSG(fmt::format("PyEntityDef::FixedDictType::initialize(): FIXED_DICT({}) no keys!, file: \"{}\"!\n", - this->aliasName(), pDefContext->pyObjectSourceFile)); - - return false; - } - - return true; -} - -//------------------------------------------------------------------------------------- -bool FixedDictType::loadImplModule(std::string moduleName) -{ - KBE_ASSERT(implObj_ == NULL); - - std::vector res_; - strutil::kbe_split(moduleName, '.', res_); - - if (res_.size() != 2) - { - ERROR_MSG(fmt::format("FixedDictType::loadImplModule: {} impl error! like:[moduleName.ClassName|moduleName.xxInstance]\n", - moduleName.c_str())); - - return false; - } - - PyObject* implModule = PyImport_ImportModule(res_[0].c_str()); - if (!implModule) - { - SCRIPT_ERROR_CHECK(); - return false; - } - - PyObject* pyImplObj = PyObject_GetAttrString(implModule, res_[1].c_str()); - bool ret = setImplModule(pyImplObj); - Py_DECREF(implModule); - return ret; -} - -//------------------------------------------------------------------------------------- -bool FixedDictType::setImplModule(PyObject* pyobj) -{ - implObj_ = pyobj; - - if (!pyobj) - { - SCRIPT_ERROR_CHECK() - return false; - } - - if (PyType_Check(implObj_)) - { - PyObject* implClass = implObj_; - implObj_ = PyObject_CallObject(implClass, NULL); - Py_DECREF(implClass); - - if (!implObj_) - { - SCRIPT_ERROR_CHECK() - return false; - } - } - - pycreateObjFromDict_ = PyObject_GetAttrString(implObj_, "createObjFromDict"); - if (!pycreateObjFromDict_) - { - SCRIPT_ERROR_CHECK() - return false; - } - - pygetDictFromObj_ = PyObject_GetAttrString(implObj_, "getDictFromObj"); - if (!pygetDictFromObj_) - { - SCRIPT_ERROR_CHECK() - return false; - } - - pyisSameType_ = PyObject_GetAttrString(implObj_, "isSameType"); - if (!pyisSameType_) - { - SCRIPT_ERROR_CHECK() - return false; - } - - return true; -} - -//------------------------------------------------------------------------------------- -PyObject* FixedDictType::impl_createObjFromDict(PyObject* dictData) -{ - // ڴʱѾû, ΪparseDefaultStr - // ʼΪն - if(!PyObject_TypeCheck(dictData, FixedDict::getScriptType()) && impl_isSameType(dictData)) - { - Py_INCREF(dictData); - return dictData; - } - - PyObject* pyRet = PyObject_CallFunction(pycreateObjFromDict_, - const_cast("(O)"), dictData); - - if(pyRet == NULL || !impl_isSameType(pyRet)) - { - SCRIPT_ERROR_CHECK(); - - ERROR_MSG(fmt::format("FixedDictType::impl_createObjFromDict: {}.isSameType() is failed!\n", - moduleName_.c_str())); - - Py_RETURN_NONE; - } - - return pyRet; -} - -//------------------------------------------------------------------------------------- -PyObject* FixedDictType::impl_getDictFromObj(PyObject* pyobj) -{ - PyObject* pyRet = PyObject_CallFunction(pygetDictFromObj_, - const_cast("(O)"), pyobj); - - if(pyRet == NULL) - { - SCRIPT_ERROR_CHECK(); - return parseDefaultStr(""); - } - - return pyRet; -} - -//------------------------------------------------------------------------------------- -bool FixedDictType::impl_isSameType(PyObject* pyobj) -{ - PyObject* pyRet = PyObject_CallFunction(pyisSameType_, - const_cast("(O)"), pyobj); - - if(pyRet == NULL) - { - SCRIPT_ERROR_CHECK(); - return false; - } - - bool ret = Py_True == pyRet; - Py_DECREF(pyRet); - return ret; -} - -//------------------------------------------------------------------------------------- -DataType* FixedDictType::isSameItemType(const char* keyName, PyObject* pyValue) -{ - FIXEDDICT_KEYTYPE_MAP::iterator iter = keyTypes_.begin(); - for(; iter != keyTypes_.end(); ++iter) - { - if(iter->first == keyName) - { - if(pyValue == NULL || !iter->second->dataType->isSameType(pyValue)) - { - PyErr_Format(PyExc_TypeError, - "set FIXED_DICT(%s) error! at key: %s(%s), keyNames=[%s].", - this->aliasName(), - iter->first.c_str(), - (pyValue == NULL ? "NULL" : pyValue->ob_type->tp_name), - debugInfos().c_str()); - - PyErr_PrintEx(0); - return NULL; - } - else - { - return iter->second->dataType; - } - } - } - - return NULL; -} - -//------------------------------------------------------------------------------------- -bool FixedDictType::isSameType(PyObject* pyValue) -{ - if(pyValue == NULL) - { - OUT_TYPE_ERROR("DICT"); - return false; - } - - if(hasImpl()) - { - // ﷵfalse󻹼жϵԭisSameTypeΪ - // fixeddictûӦǺϷ - if(impl_isSameType(pyValue)) - return true; - } - - if(PyObject_TypeCheck(pyValue, FixedDict::getScriptType())) - { - if(static_cast(pyValue)->getDataType()->id() == this->id()) - return true; - else - return false; - } - - if(!PyDict_Check(pyValue)) - { - OUT_TYPE_ERROR("DICT"); - return false; - } - - Py_ssize_t dictSize = PyDict_Size(pyValue); - if(dictSize != (Py_ssize_t)keyTypes_.size()) - { - PyErr_Format(PyExc_TypeError, - "FIXED_DICT(%s) key does not match! giveKeySize=%d, dictKeySize=%d, dictKeyNames=[%s], notFoundKeys=[%s].", - this->aliasName(), dictSize, keyTypes_.size(), - debugInfos().c_str(), getNotFoundKeys(pyValue).c_str()); - - PyErr_PrintEx(0); - return false; - } - - FIXEDDICT_KEYTYPE_MAP::iterator iter = keyTypes_.begin(); - for(; iter != keyTypes_.end(); ++iter) - { - PyObject* pyObject = PyDict_GetItemString(pyValue, const_cast(iter->first.c_str())); - if(pyObject == NULL) - { - PyErr_Format(PyExc_TypeError, - "set FIXED_DICT(%s) error! keys[%s] not found, allKeyNames=[%s].", - this->aliasName(), getNotFoundKeys(pyValue).c_str(), debugInfos().c_str()); - - PyErr_PrintEx(0); - return false; - } - else if (!iter->second->dataType->isSameType(pyObject)) - { - PyErr_Format(PyExc_TypeError, - "set FIXED_DICT(%s) error! at key: %s(%s), allKeyNames=[%s].", - this->aliasName(), - iter->first.c_str(), - pyObject->ob_type->tp_name, - debugInfos().c_str()); - - PyErr_PrintEx(0); - return false; - } - } - - return true; -} - -//------------------------------------------------------------------------------------- -PyObject* FixedDictType::parseDefaultStr(std::string defaultVal) -{ - FixedDict* pydict = new FixedDict(this); - pydict->initialize(defaultVal); - - if (hasImpl()) - { - PyObject* pyValue = impl_createObjFromDict(pydict); - Py_DECREF(pydict); - return pyValue; - } - - return pydict; -} - -//------------------------------------------------------------------------------------- -void FixedDictType::addToStream(MemoryStream* mstream, PyObject* pyValue) -{ - addToStreamEx(mstream, pyValue, false); -} - -//------------------------------------------------------------------------------------- -void FixedDictType::addToStreamEx(MemoryStream* mstream, PyObject* pyValue, bool onlyPersistents) -{ - if(hasImpl()) - { - pyValue = impl_getDictFromObj(pyValue); - } - - PyObject* pydict = pyValue; - if(PyObject_TypeCheck(pyValue, FixedDict::getScriptType())) - { - pydict = static_cast(pyValue)->getDictObject(); - } - - FIXEDDICT_KEYTYPE_MAP::iterator iter = keyTypes_.begin(); - for(; iter != keyTypes_.end(); ++iter) - { - if(onlyPersistents) - { - if(!iter->second->persistent) - continue; - } - - PyObject* pyObject = - PyDict_GetItemString(pydict, const_cast(iter->first.c_str())); - - if(pyObject == NULL) - { - ERROR_MSG(fmt::format("FixedDictType::addToStreamEx: {} not found key[{}]. keyNames[{}]\n", - this->aliasName_, iter->first, this->debugInfos())); - - // KBE_ASSERT(pyObject != NULL); - PyObject* pobj = iter->second->dataType->parseDefaultStr(""); - - if(iter->second->dataType->type() == DATA_TYPE_FIXEDDICT) - ((FixedDictType*)iter->second->dataType)->addToStreamEx(mstream, pobj, onlyPersistents); - else if(iter->second->dataType->type() == DATA_TYPE_FIXEDARRAY) - ((FixedArrayType*)iter->second->dataType)->addToStreamEx(mstream, pobj, onlyPersistents); - else - iter->second->dataType->addToStream(mstream, pobj); - - Py_DECREF(pobj); - continue; - } - - if(!iter->second->dataType->isSameType(pyObject)) - { - // KBE_ASSERT(pyObject != NULL); - PyObject* pobj = iter->second->dataType->parseDefaultStr(""); - - if(iter->second->dataType->type() == DATA_TYPE_FIXEDDICT) - ((FixedDictType*)iter->second->dataType)->addToStreamEx(mstream, pobj, onlyPersistents); - else if(iter->second->dataType->type() == DATA_TYPE_FIXEDARRAY) - ((FixedArrayType*)iter->second->dataType)->addToStreamEx(mstream, pobj, onlyPersistents); - else - iter->second->dataType->addToStream(mstream, pobj); - - Py_DECREF(pobj); - continue; - } - - if(iter->second->dataType->type() == DATA_TYPE_FIXEDDICT) - ((FixedDictType*)iter->second->dataType)->addToStreamEx(mstream, pyObject, onlyPersistents); - else if(iter->second->dataType->type() == DATA_TYPE_FIXEDARRAY) - ((FixedArrayType*)iter->second->dataType)->addToStreamEx(mstream, pyObject, onlyPersistents); - else - iter->second->dataType->addToStream(mstream, pyObject); - } - - if(hasImpl()) - { - Py_DECREF(pyValue); - } -} - -//------------------------------------------------------------------------------------- -PyObject* FixedDictType::createFromStream(MemoryStream* mstream) -{ - return createFromStreamEx(mstream, false); -} - -//------------------------------------------------------------------------------------- -PyObject* FixedDictType::createFromStreamEx(MemoryStream* mstream, bool onlyPersistents) -{ - FixedDict* pydict = new FixedDict(this, onlyPersistents); - pydict->initialize(mstream, onlyPersistents); - - if(hasImpl()) - { - PyObject* pyValue = impl_createObjFromDict(pydict); - Py_DECREF(pydict); - return pyValue; - } - - return pydict; -} - -//------------------------------------------------------------------------------------- -EntityComponentType::EntityComponentType(ScriptDefModule* pScriptDefModule, DATATYPE_UID did) : - DataType(did), - pScriptDefModule_(pScriptDefModule) -{ -} - -//------------------------------------------------------------------------------------- -EntityComponentType::~EntityComponentType() -{ -} - -//------------------------------------------------------------------------------------- -bool EntityComponentType::isSameType(PyObject* pyValue) -{ - if (pyValue == NULL || !(PyObject_TypeCheck(pyValue, EntityComponent::getScriptType()))) - { - OUT_TYPE_ERROR("ENTITY_COMPONENT"); - return false; - } - - EntityComponent* pEntityComponent = static_cast(pyValue); - return pEntityComponent->isSameType(pyValue); -} - -//------------------------------------------------------------------------------------- -bool EntityComponentType::isSamePersistentType(PyObject* pyValue) -{ - if (pyValue == NULL) - { - OUT_TYPE_ERROR("ENTITY_COMPONENT"); - return false; - } - - if (!(PyObject_TypeCheck(pyValue, EntityComponent::getScriptType()))) - { - if (!PyDict_Check(pyValue)) - { - OUT_TYPE_ERROR("ENTITY_COMPONENT"); - } - - return isSameCellDataType(pyValue); - } - - EntityComponent* pEntityComponent = static_cast(pyValue); - return pEntityComponent->isSamePersistentType(pyValue); -} - -//------------------------------------------------------------------------------------- -bool EntityComponentType::isSameCellDataType(PyObject* pyValue) -{ - if (!PyDict_Check(pyValue)) - return false; - - ScriptDefModule::PROPERTYDESCRIPTION_MAP& propertyDescrs = pScriptDefModule_->getPersistentPropertyDescriptions(); - ScriptDefModule::PROPERTYDESCRIPTION_MAP::const_iterator iter = propertyDescrs.begin(); - - for (; iter != propertyDescrs.end(); ++iter) - { - PropertyDescription* propertyDescription = iter->second; - - if (!propertyDescription->hasCell()) - continue; - - PyObject* pyVal = PyDict_GetItemString(pyValue, propertyDescription->getName()); - - if (pyVal) - { - if (!propertyDescription->getDataType()->isSameType(pyVal)) - { - ERROR_MSG(fmt::format("EntityComponent::isSameCellDataType: {} type(curr_py: {} != {}) error! name={}, utype={}, domain={}.\n", - propertyDescription->getName(), (pyVal ? pyVal->ob_type->tp_name : "unknown"), propertyDescription->getDataType()->getName(), - pScriptDefModule_ ? pScriptDefModule_->getName() : "", pScriptDefModule_ ? pScriptDefModule_->getUType() : 0, - COMPONENT_NAME_EX(CELLAPP_TYPE))); - - return false; - } - } - else - { - SCRIPT_ERROR_CHECK(); - - ERROR_MSG(fmt::format("EntityComponent::isSameCellDataType: not found property({}), use default values! name={}, utype={}, domain={}.\n", - propertyDescription->getName(), pScriptDefModule_ ? pScriptDefModule_->getName() : "", pScriptDefModule_ ? pScriptDefModule_->getUType() : 0, - COMPONENT_NAME_EX(CELLAPP_TYPE))); - - return false; - } - } - - return true; -} - -//------------------------------------------------------------------------------------- -PyObject* EntityComponentType::parseDefaultStr(std::string defaultVal) -{ - KBE_ASSERT(EntityDef::context().currEntityID > 0); - - PyObject* pyobj = pScriptDefModule_->createObject(); - - // ִEntityĹ캯 - return new(pyobj) EntityComponent(EntityDef::context().currEntityID, pScriptDefModule_, EntityDef::context().currComponentType); -} - -//------------------------------------------------------------------------------------- -void EntityComponentType::addToStream(MemoryStream* mstream, PyObject* pyValue) -{ - EntityComponent* pEntityComponent = static_cast(pyValue); - pEntityComponent->addToStream(mstream, pyValue); -} - -//------------------------------------------------------------------------------------- -void EntityComponentType::addPersistentToStream(MemoryStream* mstream, PyObject* pyValue) -{ - if (PyDict_Check(pyValue)) - { - ScriptDefModule::PROPERTYDESCRIPTION_MAP& propertyDescrs = pScriptDefModule_->getPersistentPropertyDescriptions(); - ScriptDefModule::PROPERTYDESCRIPTION_MAP::const_iterator iter = propertyDescrs.begin(); - - for (; iter != propertyDescrs.end(); ++iter) - { - PropertyDescription* propertyDescription = iter->second; - - // ﴫһֵ䣬ô϶һcelldataֵ䣬ֻcell - if (!propertyDescription->hasCell()) - continue; - - PyObject* pyVal = PyDict_GetItemString(pyValue, propertyDescription->getName()); - - if (pyVal) - { - propertyDescription->getDataType()->addToStream(mstream, pyVal); - } - else - { - SCRIPT_ERROR_CHECK(); - - ERROR_MSG(fmt::format("EntityComponent::addPersistentToStream: not found property({}), use default values! name={}, utype={}, domain={}.\n", - propertyDescription->getName(), pScriptDefModule_ ? pScriptDefModule_->getName() : "", pScriptDefModule_ ? pScriptDefModule_->getUType() : 0, - COMPONENT_NAME_EX(CELLAPP_TYPE))); - - propertyDescription->addPersistentToStream(mstream, NULL); - } - } - - return; - } - - EntityComponent* pEntityComponent = static_cast(pyValue); - pEntityComponent->addPersistentToStream(mstream, pyValue); -} - -//------------------------------------------------------------------------------------- -void EntityComponentType::addPersistentToStream(MemoryStream* mstream) -{ - ScriptDefModule::PROPERTYDESCRIPTION_MAP& propertyDescrs = pScriptDefModule_->getPersistentPropertyDescriptions(); - ScriptDefModule::PROPERTYDESCRIPTION_MAP::const_iterator iter = propertyDescrs.begin(); - - for (; iter != propertyDescrs.end(); ++iter) - { - PropertyDescription* propertyDescription = iter->second; - - PyObject* pyDefVal = propertyDescription->newDefaultVal(); - propertyDescription->getDataType()->addToStream(mstream, pyDefVal); - Py_DECREF(pyDefVal); - } -} -//------------------------------------------------------------------------------------- -void EntityComponentType::addPersistentToStreamTemplates(ScriptDefModule* pScriptModule, MemoryStream* mstream) -{ - ScriptDefModule::PROPERTYDESCRIPTION_MAP& propertyDescrs = pScriptDefModule_->getPersistentPropertyDescriptions(); - ScriptDefModule::PROPERTYDESCRIPTION_MAP::const_iterator iter = propertyDescrs.begin(); - - for (; iter != propertyDescrs.end(); ++iter) - { - PropertyDescription* propertyDescription = iter->second; - - if (propertyDescription->hasCell()) - { - // һЩʵûcell֣ cellԺ - if (!pScriptModule->hasCell()) - continue; - } - - propertyDescription->addPersistentToStream(mstream, NULL); - } -} -//------------------------------------------------------------------------------------- -void EntityComponentType::addCellDataToStream(MemoryStream* mstream, uint32 flags, PyObject* pyValue, - ENTITY_ID ownerID, PropertyDescription* parentPropertyDescription, COMPONENT_TYPE sendtoComponentType, bool checkValue) -{ - KBE_ASSERT(PyDict_Check(pyValue)); - - (*mstream) << sendtoComponentType << ownerID << pScriptDefModule_->getUType(); - - ScriptDefModule::PROPERTYDESCRIPTION_MAP& propertyDescrs = pScriptDefModule_->getCellPropertyDescriptions(); - - uint16 count = 0; - - ScriptDefModule::PROPERTYDESCRIPTION_MAP::const_iterator iter = propertyDescrs.begin(); - - for (; iter != propertyDescrs.end(); ++iter) - { - PropertyDescription* propertyDescription = iter->second; - - if (!propertyDescription->hasCell()) - continue; - - if (flags == 0 || (flags & propertyDescription->getFlags()) > 0) - { - ++count; - } - } - - (*mstream) << count; - - iter = propertyDescrs.begin(); - - for (; iter != propertyDescrs.end(); ++iter) - { - PropertyDescription* propertyDescription = iter->second; - - if (!propertyDescription->hasCell()) - continue; - - if (flags == 0 || (flags & propertyDescription->getFlags()) > 0) - { - PyObject* pyVal = PyDict_GetItemString(pyValue, propertyDescription->getName()); - - if (pyVal) - { - if (checkValue && !propertyDescription->getDataType()->isSameType(pyVal)) - { - ERROR_MSG(fmt::format("EntityComponent::addCellDataToStream: {} type(curr_py: {} != {}) error, use default values! name={}, utype={}, domain={}.\n", - propertyDescription->getName(), (pyVal ? pyVal->ob_type->tp_name : "unknown"), propertyDescription->getDataType()->getName(), - pScriptDefModule_ ? pScriptDefModule_->getName() : "", pScriptDefModule_ ? pScriptDefModule_->getUType() : 0, - COMPONENT_NAME_EX(sendtoComponentType))); - - if (parentPropertyDescription) - { - if (sendtoComponentType == CLIENT_TYPE && pScriptDefModule_->usePropertyDescrAlias()) - { - (*mstream) << parentPropertyDescription->aliasIDAsUint8(); - (*mstream) << propertyDescription->aliasIDAsUint8(); - } - else - { - (*mstream) << parentPropertyDescription->getUType(); - (*mstream) << propertyDescription->getUType(); - } - } - - propertyDescription->addToStream(mstream, NULL); - } - else - { - if (parentPropertyDescription) - { - if (sendtoComponentType == CLIENT_TYPE && pScriptDefModule_->usePropertyDescrAlias()) - { - (*mstream) << parentPropertyDescription->aliasIDAsUint8(); - (*mstream) << propertyDescription->aliasIDAsUint8(); - } - else - { - (*mstream) << parentPropertyDescription->getUType(); - (*mstream) << propertyDescription->getUType(); - } - } - - propertyDescription->getDataType()->addToStream(mstream, pyVal); - } - } - else - { - SCRIPT_ERROR_CHECK(); - - ERROR_MSG(fmt::format("EntityComponent::addCellDataToStream: not found property({}), use default values! name={}, utype={}, domain={}.\n", - propertyDescription->getName(), pScriptDefModule_ ? pScriptDefModule_->getName() : "", pScriptDefModule_ ? pScriptDefModule_->getUType() : 0, - COMPONENT_NAME_EX(sendtoComponentType))); - - if (parentPropertyDescription) - { - if (sendtoComponentType == CLIENT_TYPE && pScriptDefModule_->usePropertyDescrAlias()) - { - (*mstream) << parentPropertyDescription->aliasIDAsUint8(); - (*mstream) << propertyDescription->aliasIDAsUint8(); - } - else - { - (*mstream) << parentPropertyDescription->getUType(); - (*mstream) << propertyDescription->getUType(); - } - } - - propertyDescription->addToStream(mstream, NULL); - } - } - } -} - -//------------------------------------------------------------------------------------- -PyObject* EntityComponentType::createFromStream(MemoryStream* mstream) -{ - KBE_ASSERT(EntityDef::context().currEntityID > 0); - - PyObject* pyobj = pScriptDefModule_->createObject(); - - // ִEntityĹ캯 - PyObject* pyEntityComponent = new(pyobj) EntityComponent(EntityDef::context().currEntityID, pScriptDefModule_, EntityDef::context().currComponentType); - - EntityComponent* pEntityComponent = static_cast(pyEntityComponent); - return pEntityComponent->createFromStream(mstream); -} - -//------------------------------------------------------------------------------------- -PyObject* EntityComponentType::createFromPersistentStream(ScriptDefModule* pScriptDefModule, MemoryStream* mstream) -{ - KBE_ASSERT(EntityDef::context().currEntityID > 0); - - PyObject* pyobj = pScriptDefModule_->createObject(); - - // ִEntityĹ캯 - PyObject* pyEntityComponent = new(pyobj) EntityComponent(EntityDef::context().currEntityID, pScriptDefModule_, EntityDef::context().currComponentType); - - EntityComponent* pEntityComponent = static_cast(pyEntityComponent); - return pEntityComponent->createFromPersistentStream(pScriptDefModule, mstream); -} - -//------------------------------------------------------------------------------------- -PyObject* EntityComponentType::createCellData() -{ - PyObject* cellDataDict = PyDict_New(); - - ScriptDefModule::PROPERTYDESCRIPTION_MAP& propertyDescrs = pScriptDefModule_->getCellPropertyDescriptions(); - ScriptDefModule::PROPERTYDESCRIPTION_MAP::const_iterator iter = propertyDescrs.begin(); - for (; iter != propertyDescrs.end(); ++iter) - { - PropertyDescription* propertyDescription = iter->second; - DataType* dataType = propertyDescription->getDataType(); - - if (dataType) - { - PyObject* pyObj = propertyDescription->newDefaultVal(); - PyDict_SetItemString(cellDataDict, propertyDescription->getName(), pyObj); - Py_DECREF(pyObj); - } - else - { - ERROR_MSG(fmt::format("EntityComponentType::createCellData: {} PropertyDescription the dataType is NULL! component={}\n", - propertyDescription->getName(), pScriptDefModule_->getName())); - } - - SCRIPT_ERROR_CHECK(); - } - - return cellDataDict; -} - -//------------------------------------------------------------------------------------- -PyObject* EntityComponentType::createCellDataFromStream(MemoryStream* mstream) -{ - COMPONENT_TYPE componentType; - ENTITY_SCRIPT_UID ComponentDescrsType; - uint16 count; - ENTITY_ID ownerID; - - (*mstream) >> componentType >> ownerID >> ComponentDescrsType >> count; - - PyObject* cellDataDict = PyDict_New(); - - ScriptDefModule::PROPERTYDESCRIPTION_MAP& propertyDescrs = pScriptDefModule_->getCellPropertyDescriptions(); - ScriptDefModule::PROPERTYDESCRIPTION_MAP::const_iterator iter = propertyDescrs.begin(); - - for (; iter != propertyDescrs.end(); ++iter) - { - PropertyDescription* propertyDescription = iter->second; - - ENTITY_PROPERTY_UID uid; - (*mstream) >> uid >> uid; - - KBE_ASSERT(propertyDescription->getUType() == uid); - - PyObject* pyobj = propertyDescription->createFromStream(mstream); - - if (pyobj == NULL) - { - SCRIPT_ERROR_CHECK(); - - ERROR_MSG(fmt::format("EntityComponentType::createCellDataFromStream: property({}) error, use default values! name={}, utype={}.\n", - propertyDescription->getName(), pScriptDefModule_ ? pScriptDefModule_->getName() : "", pScriptDefModule_ ? pScriptDefModule_->getUType() : 0)); - - pyobj = propertyDescription->newDefaultVal(); - } - - PyDict_SetItemString(cellDataDict, propertyDescription->getName(), pyobj); - Py_DECREF(pyobj); - } - - return cellDataDict; -} - -//------------------------------------------------------------------------------------- -PyObject* EntityComponentType::createCellDataFromPersistentStream(MemoryStream* mstream) -{ - PyObject* cellDataDict = PyDict_New(); - - ScriptDefModule::PROPERTYDESCRIPTION_MAP& propertyDescrs = pScriptDefModule_->getPersistentPropertyDescriptions(); - ScriptDefModule::PROPERTYDESCRIPTION_MAP::const_iterator iter = propertyDescrs.begin(); - - for (; iter != propertyDescrs.end(); ++iter) - { - PropertyDescription* propertyDescription = iter->second; - - if (!propertyDescription->hasCell()) - continue; - - PyObject* pyobj = NULL; - - if(mstream) - pyobj = propertyDescription->createFromStream(mstream); - - if (pyobj == NULL) - { - SCRIPT_ERROR_CHECK(); - - ERROR_MSG(fmt::format("EntityComponentType::createCellDataFromPersistentStream: property({}) error, use default values! name={}, utype={}.\n", - propertyDescription->getName(), pScriptDefModule_ ? pScriptDefModule_->getName() : "", pScriptDefModule_ ? pScriptDefModule_->getUType() : 0)); - - pyobj = propertyDescription->newDefaultVal(); - } - - PyDict_SetItemString(cellDataDict, propertyDescription->getName(), pyobj); - Py_DECREF(pyobj); - } - - return cellDataDict; -} - -//------------------------------------------------------------------------------------- -} +// Copyright 2008-2018 Yolo Technologies, Inc. All Rights Reserved. https://www.comblockengine.com + +#include "datatype.h" +#include "datatypes.h" +#include "entitydef.h" +#include "fixeddict.h" +#include "fixedarray.h" +#include "entity_call.h" +#include "py_entitydef.h" +#include "property.h" +#include "entity_component.h" +#include "scriptdef_module.h" +#include "pyscript/vector2.h" +#include "pyscript/vector3.h" +#include "pyscript/vector4.h" +#include "pyscript/copy.h" +#include "pyscript/py_memorystream.h" + +#ifndef CODE_INLINE +#include "datatype.inl" +#endif + +namespace KBEngine{ +static DATATYPE_UID _g_dataTypeUID = 1; + +static bool isVecotr(std::string str, std::size_t n, std::vector& nums) +{ + if (str.empty()) + { + return false; + } + + if (n < 2) + { + return false; + } + + std::string strtemp = str; + + strutil::kbe_replace(strtemp, " ", ""); + strutil::kbe_replace(strtemp, "(", ""); + strutil::kbe_replace(strtemp, ")", ""); + + std::vector result; + strutil::kbe_splits(strtemp, ",", result); + + if (result.size() != n) + { + return false; + } + + nums.clear(); + for (auto ite = result.begin(); ite != result.end(); ite++) + { + try + { + float num = 0.f; + StringConv::str2value(num, (*ite).c_str()); + nums.push_back(num); + } + catch (...) + { + return false; + } + } + + return true; +} + +static bool isVector2(std::string str, std::vector& nums) +{ + return isVecotr(str, 2, nums); +} + +static bool isVector3(std::string str, std::vector& nums) +{ + return isVecotr(str, 3, nums); +} + +static bool isVector4(std::string str, std::vector& nums) +{ + return isVecotr(str, 4, nums); +} + +//------------------------------------------------------------------------------------- +DataType::DataType(DATATYPE_UID did): +id_(did), +aliasName_() +{ + if(id_ == 0) + id_ = _g_dataTypeUID++; + + DataTypes::addDataType(id_, this); + + EntityDef::md5().append((void*)this->aliasName(), (int)strlen(this->aliasName())); + EntityDef::md5().append((void*)&id_, sizeof(DATATYPE_UID)); +} + +//------------------------------------------------------------------------------------- +DataType::~DataType() +{ + finalise(); +} + +//------------------------------------------------------------------------------------- +bool DataType::finalise() +{ + _g_dataTypeUID = 1; + return true; +} + +//------------------------------------------------------------------------------------- +bool DataType::initialize(XML* xml, TiXmlNode* node) +{ + return true; +} + +//------------------------------------------------------------------------------------- +UInt64Type::UInt64Type(DATATYPE_UID did): +DataType(did) +{ +} + +//------------------------------------------------------------------------------------- +UInt64Type::~UInt64Type() +{ +} + +//------------------------------------------------------------------------------------- +bool UInt64Type::isSameType(PyObject* pyValue) +{ + if(pyValue == NULL) + { + OUT_TYPE_ERROR("UINT64"); + return false; + } + + if (!PyLong_Check(pyValue)) + { + OUT_TYPE_ERROR("UINT64"); + return false; + } + + PyLong_AsUnsignedLongLong(pyValue); + if (!PyErr_Occurred()) + return true; + + PyErr_Clear(); + PyLong_AsUnsignedLong(pyValue); + if (!PyErr_Occurred()) + return true; + + PyErr_Clear(); + long v = PyLong_AsLong(pyValue); + if (!PyErr_Occurred()) + { + if(v < 0) + { + OUT_TYPE_ERROR("UINT64"); + return false; + } + return true; + } + + PyErr_Clear(); + OUT_TYPE_ERROR("UINT64"); + return false; +} + +//------------------------------------------------------------------------------------- +PyObject* UInt64Type::parseDefaultStr(std::string defaultVal) +{ + uint64 val = 0; + if(!defaultVal.empty()) + { + std::stringstream stream; + stream << defaultVal; + stream >> val; + } + + PyObject* pyval = PyLong_FromUnsignedLongLong(val); + + if (PyErr_Occurred()) + { + PyErr_Clear(); + PyErr_Format(PyExc_TypeError, "UInt64Type::parseDefaultStr: defaultVal(%s) error! val=[%s]", + pyval != NULL ? pyval->ob_type->tp_name : "NULL", defaultVal.c_str()); + + PyErr_PrintEx(0); + + S_RELEASE(pyval); + return PyLong_FromUnsignedLongLong(0); + } + + return pyval; +} + +//------------------------------------------------------------------------------------- +void UInt64Type::addToStream(MemoryStream* mstream, PyObject* pyValue) +{ + uint64 v = static_cast(PyLong_AsUnsignedLongLong(pyValue)); + + if (PyErr_Occurred()) + { + PyErr_Clear(); + v = (uint64)PyLong_AsUnsignedLong(pyValue); + if (PyErr_Occurred()) + { + PyErr_Clear(); + v = (uint64)PyLong_AsLong(pyValue); + + if(PyErr_Occurred()) + { + PyErr_Clear(); + PyErr_Format(PyExc_TypeError, "UInt64Type::addToStream: pyValue(%s) is wrong!", + (pyValue == NULL) ? "NULL": pyValue->ob_type->tp_name); + + PyErr_PrintEx(0); + + v = 0; + } + } + } + + (*mstream) << v; +} + +//------------------------------------------------------------------------------------- +PyObject* UInt64Type::createFromStream(MemoryStream* mstream) +{ + uint64 val = 0; + if(mstream) + (*mstream) >> val; + + PyObject* pyval = PyLong_FromUnsignedLongLong(val); + + if (PyErr_Occurred()) + { + PyErr_Format(PyExc_TypeError, "UInt64Type::createFromStream: errval=%" PRIu64 ", default return is 0", val); + PyErr_PrintEx(0); + S_RELEASE(pyval); + return PyLong_FromUnsignedLongLong(0); + } + + return pyval; +} + +//------------------------------------------------------------------------------------- +UInt32Type::UInt32Type(DATATYPE_UID did): +DataType(did) +{ +} + +//------------------------------------------------------------------------------------- +UInt32Type::~UInt32Type() +{ +} + +//------------------------------------------------------------------------------------- +bool UInt32Type::isSameType(PyObject* pyValue) +{ + if(pyValue == NULL) + { + OUT_TYPE_ERROR("UINT32"); + return false; + } + + if (!PyLong_Check(pyValue)) + { + OUT_TYPE_ERROR("UINT32"); + return false; + } + + PyLong_AsUnsignedLong(pyValue); + if (!PyErr_Occurred()) + return true; + + PyErr_Clear(); + long v = PyLong_AsLong(pyValue); + if (!PyErr_Occurred()) + { + if(v < 0) + { + OUT_TYPE_ERROR("UINT32"); + return false; + } + return true; + } + + PyErr_Clear(); + OUT_TYPE_ERROR("UINT32"); + return false; +} + +//------------------------------------------------------------------------------------- +PyObject* UInt32Type::parseDefaultStr(std::string defaultVal) +{ + uint32 val = 0; + if(!defaultVal.empty()) + { + std::stringstream stream; + stream << defaultVal; + stream >> val; + } + + PyObject* pyval = PyLong_FromUnsignedLong(val); + + if (PyErr_Occurred()) + { + PyErr_Clear(); + PyErr_Format(PyExc_TypeError, "UInt32Type::parseDefaultStr: defaultVal(%s) error! val=[%s]", + pyval != NULL ? pyval->ob_type->tp_name : "NULL", defaultVal.c_str()); + + PyErr_PrintEx(0); + + S_RELEASE(pyval); + return PyLong_FromUnsignedLong(0); + } + + return pyval; +} + +//------------------------------------------------------------------------------------- +void UInt32Type::addToStream(MemoryStream* mstream, PyObject* pyValue) +{ + uint32 v = PyLong_AsUnsignedLong(pyValue); + + if (PyErr_Occurred()) + { + PyErr_Clear(); + v = (uint32)PyLong_AsLong(pyValue); + + if(PyErr_Occurred()) + { + PyErr_Clear(); + + PyErr_Format(PyExc_TypeError, "UInt32Type::addToStream: pyValue(%s) is wrong!", + (pyValue == NULL) ? "NULL": pyValue->ob_type->tp_name); + + PyErr_PrintEx(0); + + v = 0; + } + } + + (*mstream) << v; +} + +//------------------------------------------------------------------------------------- +PyObject* UInt32Type::createFromStream(MemoryStream* mstream) +{ + uint32 val = 0; + if(mstream) + (*mstream) >> val; + + PyObject* pyval = PyLong_FromUnsignedLong(val); + + if (PyErr_Occurred()) + { + PyErr_Format(PyExc_TypeError, "UInt32Type::createFromStream: errval=%u, default return is 0", val); + PyErr_PrintEx(0); + S_RELEASE(pyval); + return PyLong_FromUnsignedLong(0); + } + + return pyval; +} + +//------------------------------------------------------------------------------------- +Int64Type::Int64Type(DATATYPE_UID did): +DataType(did) +{ +} + +//------------------------------------------------------------------------------------- +Int64Type::~Int64Type() +{ +} + +//------------------------------------------------------------------------------------- +bool Int64Type::isSameType(PyObject* pyValue) +{ + if(pyValue == NULL) + { + OUT_TYPE_ERROR("INT64"); + return false; + } + + if (!PyLong_Check(pyValue)) + { + OUT_TYPE_ERROR("INT64"); + return false; + } + + PyLong_AsLongLong(pyValue); + if (!PyErr_Occurred()) + return true; + + PyErr_Clear(); + PyLong_AsLong(pyValue); + if (!PyErr_Occurred()) + { + return true; + } + + PyErr_Clear(); + OUT_TYPE_ERROR("INT64"); + return false; +} + +//------------------------------------------------------------------------------------- +PyObject* Int64Type::parseDefaultStr(std::string defaultVal) +{ + int64 val = 0; + if(!defaultVal.empty()) + { + std::stringstream stream; + stream << defaultVal; + stream >> val; + } + + PyObject* pyval = PyLong_FromLongLong(val); + + if (PyErr_Occurred()) + { + PyErr_Clear(); + PyErr_Format(PyExc_TypeError, "Int64Type::parseDefaultStr: defaultVal(%s) error! val=[%s]", + pyval != NULL ? pyval->ob_type->tp_name : "NULL", defaultVal.c_str()); + + PyErr_PrintEx(0); + + S_RELEASE(pyval); + return PyLong_FromLongLong(0); + } + + return pyval; +} + +//------------------------------------------------------------------------------------- +void Int64Type::addToStream(MemoryStream* mstream, PyObject* pyValue) +{ + int64 v = PyLong_AsLongLong(pyValue); + + if (PyErr_Occurred()) + { + PyErr_Clear(); + v = (uint32)PyLong_AsLong(pyValue); + + if(PyErr_Occurred()) + { + PyErr_Clear(); + + PyErr_Format(PyExc_TypeError, "Int64Type::addToStream: pyValue(%s) is wrong!", + (pyValue == NULL) ? "NULL": pyValue->ob_type->tp_name); + + PyErr_PrintEx(0); + + v = 0; + } + } + + (*mstream) << v; +} + +//------------------------------------------------------------------------------------- +PyObject* Int64Type::createFromStream(MemoryStream* mstream) +{ + int64 val = 0; + if(mstream) + (*mstream) >> val; + + PyObject* pyval = PyLong_FromLongLong(val); + + if (PyErr_Occurred()) + { + PyErr_Format(PyExc_TypeError, "Int64Type::createFromStream: errval=%" PRI64 ", default return is 0", val); + PyErr_PrintEx(0); + S_RELEASE(pyval); + return PyLong_FromLongLong(0); + } + + return pyval; +} + +//------------------------------------------------------------------------------------- +FloatType::FloatType(DATATYPE_UID did): +DataType(did) +{ +} + +//------------------------------------------------------------------------------------- +FloatType::~FloatType() +{ +} + +//------------------------------------------------------------------------------------- +bool FloatType::isSameType(PyObject* pyValue) +{ + if(pyValue == NULL) + { + OUT_TYPE_ERROR("FLOAT"); + return false; + } + + bool ret = PyFloat_Check(pyValue); + if(!ret) + OUT_TYPE_ERROR("FLOAT"); + return ret; +} + +//------------------------------------------------------------------------------------- +PyObject* FloatType::parseDefaultStr(std::string defaultVal) +{ + float val = 0.0f; + if(!defaultVal.empty()) + { + std::stringstream stream; + stream << defaultVal; + stream >> val; + } + + PyObject* pyval = PyFloat_FromDouble(val); + + if (PyErr_Occurred()) + { + PyErr_Clear(); + PyErr_Format(PyExc_TypeError, "FloatType::parseDefaultStr: defaultVal(%s) error! val=[%s]", + pyval != NULL ? pyval->ob_type->tp_name : "NULL", defaultVal.c_str()); + + PyErr_PrintEx(0); + + S_RELEASE(pyval); + return PyFloat_FromDouble(0.0f); + } + + return pyval; +} + +//------------------------------------------------------------------------------------- +void FloatType::addToStream(MemoryStream* mstream, PyObject* pyValue) +{ + if(!PyFloat_Check(pyValue)) + { + PyErr_Format(PyExc_TypeError, "FloatType::addToStream: pyValue(%s) is wrong!", + (pyValue == NULL) ? "NULL": pyValue->ob_type->tp_name); + + PyErr_PrintEx(0); + return; + } + + float val = (float)PyFloat_AsDouble(pyValue); + (*mstream) << val; +} + +//------------------------------------------------------------------------------------- +PyObject* FloatType::createFromStream(MemoryStream* mstream) +{ + float val = 0.0; + if(mstream) + (*mstream) >> val; + + PyObject* pyval = PyFloat_FromDouble(val); + + if (PyErr_Occurred()) + { + PyErr_Format(PyExc_TypeError, "FloatType::createFromStream: errval=%f, default return is 0", val); + PyErr_PrintEx(0); + S_RELEASE(pyval); + return PyFloat_FromDouble(0); + } + + return pyval; +} + +//------------------------------------------------------------------------------------- +DoubleType::DoubleType(DATATYPE_UID did): +DataType(did) +{ +} + +//------------------------------------------------------------------------------------- +DoubleType::~DoubleType() +{ +} + +//------------------------------------------------------------------------------------- +bool DoubleType::isSameType(PyObject* pyValue) +{ + if(pyValue == NULL) + { + OUT_TYPE_ERROR("DOUBLE"); + return false; + } + + bool ret = PyFloat_Check(pyValue); + if(!ret) + OUT_TYPE_ERROR("DOUBLE"); + return ret; +} + +//------------------------------------------------------------------------------------- +PyObject* DoubleType::parseDefaultStr(std::string defaultVal) +{ + double val = 0.0f; + if(!defaultVal.empty()) + { + std::stringstream stream; + stream << defaultVal; + stream >> val; + } + + PyObject* pyval = PyFloat_FromDouble(val); + + if (PyErr_Occurred()) + { + PyErr_Clear(); + PyErr_Format(PyExc_TypeError, "DoubleType::parseDefaultStr: defaultVal(%s) error! val=[%s]", + pyval != NULL ? pyval->ob_type->tp_name : "NULL", defaultVal.c_str()); + + PyErr_PrintEx(0); + + S_RELEASE(pyval); + return PyFloat_FromDouble(0.0f); + } + + return pyval; +} + +//------------------------------------------------------------------------------------- +void DoubleType::addToStream(MemoryStream* mstream, PyObject* pyValue) +{ + if(!PyFloat_Check(pyValue)) + { + PyErr_Format(PyExc_TypeError, "DoubleType::addToStream: pyValue(%s) is wrong!", + (pyValue == NULL) ? "NULL": pyValue->ob_type->tp_name); + + PyErr_PrintEx(0); + return; + } + + (*mstream) << PyFloat_AsDouble(pyValue); +} + +//------------------------------------------------------------------------------------- +PyObject* DoubleType::createFromStream(MemoryStream* mstream) +{ + double val = 0.0; + if(mstream) + (*mstream) >> val; + + PyObject* pyval = PyFloat_FromDouble(val); + + if (PyErr_Occurred()) + { + PyErr_Format(PyExc_TypeError, "UInt32Type::createFromStream: errval=%lf, default return is 0", val); + PyErr_PrintEx(0); + S_RELEASE(pyval); + return PyFloat_FromDouble(0); + } + + return pyval; +} + +//------------------------------------------------------------------------------------- +Vector2Type::Vector2Type(DATATYPE_UID did) : +DataType(did) +{ +} + +//------------------------------------------------------------------------------------- +Vector2Type::~Vector2Type() +{ +} + +//------------------------------------------------------------------------------------- +bool Vector2Type::isSameType(PyObject* pyValue) +{ + if(pyValue == NULL) + { + PyErr_Format(PyExc_TypeError, + "must be set to a VECTOR2 type."); + + PyErr_PrintEx(0); + return false; + } + + if(!PySequence_Check(pyValue) || (uint32)PySequence_Size(pyValue) != 2) + { + PyErr_Format(PyExc_TypeError, + "must be set to a VECTOR2 type."); + + PyErr_PrintEx(0); + return false; + } + + for(uint32 index=0; index<2; ++index) + { + PyObject* pyVal = PySequence_GetItem(pyValue, index); + if(!PyFloat_Check(pyVal) && !PyLong_Check(pyVal) && !PyLong_AsLongLong(pyVal)) + { + PyErr_Format(PyExc_TypeError, + "VECTOR2 item is not digit."); + + PyErr_PrintEx(0); + return false; + } + + Py_DECREF(pyVal); + } + + return true; +} + +//------------------------------------------------------------------------------------- +PyObject* Vector2Type::parseDefaultStr(std::string defaultVal) +{ + float x = 0.0f, y = 0.0f; + + if (!defaultVal.empty()) + { + std::vector result; + if (isVector2(defaultVal, result)) + { + x = result[0]; + y = result[1]; + return new script::ScriptVector2(float(x), float(y)); + } + + PyErr_Format(PyExc_TypeError, "Vector2::parseDefaultStr: defaultVal=%s format error!", defaultVal.c_str()); + PyErr_PrintEx(0); + } + + return new script::ScriptVector2(float(x), float(y)); +} + +//------------------------------------------------------------------------------------- +void Vector2Type::addToStream(MemoryStream* mstream, PyObject* pyValue) +{ + for(ArraySize index=0; index<2; ++index) + { + PyObject* pyVal = PySequence_GetItem(pyValue, index); +#ifdef CLIENT_NO_FLOAT + int32 v = (int32)PyFloat_AsDouble(pyVal); +#else + float v = (float)PyFloat_AsDouble(pyVal); +#endif + (*mstream) << v; + Py_DECREF(pyVal); + } +} + +//------------------------------------------------------------------------------------- +PyObject* Vector2Type::createFromStream(MemoryStream* mstream) +{ +#ifdef CLIENT_NO_FLOAT + int32 x = 0, y = 0; +#else + float x = 0.0f, y = 0.0f; +#endif + + if(mstream) + (*mstream) >> x >> y; + + return new script::ScriptVector2(float(x), float(y)); +} + +//------------------------------------------------------------------------------------- +Vector3Type::Vector3Type(DATATYPE_UID did) : +DataType(did) +{ +} + +//------------------------------------------------------------------------------------- +Vector3Type::~Vector3Type() +{ +} + +//------------------------------------------------------------------------------------- +bool Vector3Type::isSameType(PyObject* pyValue) +{ + if (pyValue == NULL) + { + PyErr_Format(PyExc_TypeError, + "must be set to a VECTOR3 type."); + + PyErr_PrintEx(0); + return false; + } + + if (!PySequence_Check(pyValue) || (uint32)PySequence_Size(pyValue) != 3) + { + PyErr_Format(PyExc_TypeError, + "must be set to a VECTOR3 type."); + + PyErr_PrintEx(0); + return false; + } + + for (uint32 index = 0; index<3; ++index) + { + PyObject* pyVal = PySequence_GetItem(pyValue, index); + if (!PyFloat_Check(pyVal) && !PyLong_Check(pyVal) && !PyLong_AsLongLong(pyVal)) + { + PyErr_Format(PyExc_TypeError, + "VECTOR3 item is not digit."); + + PyErr_PrintEx(0); + return false; + } + + Py_DECREF(pyVal); + } + + return true; +} + +//------------------------------------------------------------------------------------- +PyObject* Vector3Type::parseDefaultStr(std::string defaultVal) +{ + float x = 0.0f, y = 0.0f, z = 0.0f; + + if (!defaultVal.empty()) + { + std::vector result; + if (isVector3(defaultVal, result)) + { + x = result[0]; + y = result[1]; + z = result[2]; + return new script::ScriptVector3(float(x), float(y), float(z)); + } + + PyErr_Format(PyExc_TypeError, "Vector3::parseDefaultStr: defaultVal=%s format error!", defaultVal.c_str()); + PyErr_PrintEx(0); + } + + return new script::ScriptVector3(float(x), float(y), float(z)); +} + +//------------------------------------------------------------------------------------- +void Vector3Type::addToStream(MemoryStream* mstream, PyObject* pyValue) +{ + for (ArraySize index = 0; index<3; ++index) + { + PyObject* pyVal = PySequence_GetItem(pyValue, index); +#ifdef CLIENT_NO_FLOAT + int32 v = (int32)PyFloat_AsDouble(pyVal); +#else + float v = (float)PyFloat_AsDouble(pyVal); +#endif + (*mstream) << v; + Py_DECREF(pyVal); + } +} + +//------------------------------------------------------------------------------------- +PyObject* Vector3Type::createFromStream(MemoryStream* mstream) +{ +#ifdef CLIENT_NO_FLOAT + int32 x = 0, y = 0, z = 0; +#else + float x = 0.0f, y = 0.0f, z = 0.0f; +#endif + + if (mstream) + (*mstream) >> x >> y >> z; + + return new script::ScriptVector3(float(x), float(y), float(z)); +} + +//------------------------------------------------------------------------------------- +Vector4Type::Vector4Type(DATATYPE_UID did) : +DataType(did) +{ +} + +//------------------------------------------------------------------------------------- +Vector4Type::~Vector4Type() +{ +} + +//------------------------------------------------------------------------------------- +bool Vector4Type::isSameType(PyObject* pyValue) +{ + if (pyValue == NULL) + { + PyErr_Format(PyExc_TypeError, + "must be set to a VECTOR4 type."); + + PyErr_PrintEx(0); + return false; + } + + if (!PySequence_Check(pyValue) || (uint32)PySequence_Size(pyValue) != 4) + { + PyErr_Format(PyExc_TypeError, + "must be set to a VECTOR4 type."); + + PyErr_PrintEx(0); + return false; + } + + for (uint32 index = 0; index<4; ++index) + { + PyObject* pyVal = PySequence_GetItem(pyValue, index); + if (!PyFloat_Check(pyVal) && !PyLong_Check(pyVal) && !PyLong_AsLongLong(pyVal)) + { + PyErr_Format(PyExc_TypeError, + "VECTOR4 item is not digit."); + + PyErr_PrintEx(0); + return false; + } + + Py_DECREF(pyVal); + } + + return true; +} + +//------------------------------------------------------------------------------------- +PyObject* Vector4Type::parseDefaultStr(std::string defaultVal) +{ + float x = 0.0f, y = 0.0f, z = 0.0f, w = 0.0f; + + if (!defaultVal.empty()) + { + std::vector result; + if (isVector4(defaultVal, result)) + { + x = result[0]; + y = result[1]; + z = result[2]; + w = result[3]; + return new script::ScriptVector4(float(x), float(y), float(z), float(w)); + } + + PyErr_Format(PyExc_TypeError, "Vector4::parseDefaultStr: defaultVal=%s format error!", defaultVal.c_str()); + PyErr_PrintEx(0); + } + + return new script::ScriptVector4(float(x), float(y), float(z), float(w)); +} + +//------------------------------------------------------------------------------------- +void Vector4Type::addToStream(MemoryStream* mstream, PyObject* pyValue) +{ + for (ArraySize index = 0; index<4; ++index) + { + PyObject* pyVal = PySequence_GetItem(pyValue, index); +#ifdef CLIENT_NO_FLOAT + int32 v = (int32)PyFloat_AsDouble(pyVal); +#else + float v = (float)PyFloat_AsDouble(pyVal); +#endif + (*mstream) << v; + Py_DECREF(pyVal); + } +} + +//------------------------------------------------------------------------------------- +PyObject* Vector4Type::createFromStream(MemoryStream* mstream) +{ +#ifdef CLIENT_NO_FLOAT + int32 x = 0, y = 0, z = 0, w = 0; +#else + float x = 0.0f, y = 0.0f, z = 0.0f, w = 0.0f; +#endif + + if (mstream) + (*mstream) >> x >> y >> z >> w; + + return new script::ScriptVector4(float(x), float(y), float(z), float(w)); +} + +//------------------------------------------------------------------------------------- +StringType::StringType(DATATYPE_UID did): +DataType(did) +{ +} + +//------------------------------------------------------------------------------------- +StringType::~StringType() +{ +} + +//------------------------------------------------------------------------------------- +bool StringType::isSameType(PyObject* pyValue) +{ + if(pyValue == NULL) + { + OUT_TYPE_ERROR("STRING"); + return false; + } + + bool ret = PyUnicode_Check(pyValue); + if(!ret) + { + OUT_TYPE_ERROR("STRING"); + } + else + { + PyObject* pyfunc = PyObject_GetAttrString(pyValue, "encode"); + if(pyfunc == NULL) + { + SCRIPT_ERROR_CHECK(); + ret = false; + } + else + { + PyObject* pyRet = PyObject_CallFunction(pyfunc, + const_cast("(s)"), "ascii"); + + S_RELEASE(pyfunc); + + if(!pyRet) + { + SCRIPT_ERROR_CHECK(); + ret = false; + } + else + { + S_RELEASE(pyRet); + } + } + } + + return ret; +} + +//------------------------------------------------------------------------------------- +PyObject* StringType::parseDefaultStr(std::string defaultVal) +{ + PyObject* pyobj = PyUnicode_FromString(defaultVal.c_str()); + + if (pyobj && !PyErr_Occurred()) + return pyobj; + + PyErr_Clear(); + PyErr_Format(PyExc_TypeError, "StringType::parseDefaultStr: defaultVal(%s) error! val=[%s]", + pyobj != NULL ? pyobj->ob_type->tp_name : "NULL", defaultVal.c_str()); + + PyErr_PrintEx(0); + S_RELEASE(pyobj); + + return PyUnicode_FromString(""); +} + +//------------------------------------------------------------------------------------- +void StringType::addToStream(MemoryStream* mstream, PyObject* pyValue) +{ + const char* s = PyUnicode_AsUTF8AndSize(pyValue, NULL); + + if (s == NULL) + { + OUT_TYPE_ERROR("STRING"); + return; + } + + (*mstream) << s; +} + +//------------------------------------------------------------------------------------- +PyObject* StringType::createFromStream(MemoryStream* mstream) +{ + std::string val = ""; + if(mstream) + (*mstream) >> val; + + PyObject* pyobj = PyUnicode_FromString(val.c_str()); + + if (pyobj && !PyErr_Occurred()) + return pyobj; + + PyErr_PrintEx(0); + S_RELEASE(pyobj); + + return NULL; +} + +//------------------------------------------------------------------------------------- +UnicodeType::UnicodeType(DATATYPE_UID did): +DataType(did) +{ +} + +//------------------------------------------------------------------------------------- +UnicodeType::~UnicodeType() +{ +} + +//------------------------------------------------------------------------------------- +bool UnicodeType::isSameType(PyObject* pyValue) +{ + if(pyValue == NULL) + { + OUT_TYPE_ERROR("UNICODE"); + return false; + } + + bool ret = PyUnicode_Check(pyValue); + if(!ret) + OUT_TYPE_ERROR("UNICODE"); + + return ret; +} + +//------------------------------------------------------------------------------------- +PyObject* UnicodeType::parseDefaultStr(std::string defaultVal) +{ + PyObject* pyobj = PyUnicode_DecodeUTF8(defaultVal.data(), defaultVal.size(), ""); + + if(pyobj && !PyErr_Occurred()) + { + return pyobj; + } + + PyErr_Clear(); + PyErr_Format(PyExc_TypeError, "UnicodeType::parseDefaultStr: defaultVal(%s) error! val=[%s]", + pyobj != NULL ? pyobj->ob_type->tp_name : "NULL", defaultVal.c_str()); + + PyErr_PrintEx(0); + S_RELEASE(pyobj); + + return PyUnicode_DecodeUTF8("", 0, ""); +} + +//------------------------------------------------------------------------------------- +void UnicodeType::addToStream(MemoryStream* mstream, PyObject* pyValue) +{ + Py_ssize_t size; + const char* s = PyUnicode_AsUTF8AndSize(pyValue, &size); + + if (s == NULL) + { + OUT_TYPE_ERROR("UNICODE"); + return; + } + + mstream->appendBlob(s, size); +} + +//------------------------------------------------------------------------------------- +PyObject* UnicodeType::createFromStream(MemoryStream* mstream) +{ + std::string val = ""; + if(mstream) + { + mstream->readBlob(val); + } + + PyObject* pyobj = PyUnicode_DecodeUTF8(val.data(), val.size(), ""); + + if(pyobj && !PyErr_Occurred()) + { + return pyobj; + } + + S_RELEASE(pyobj); + ::PyErr_PrintEx(0); + + return NULL; +} + +//------------------------------------------------------------------------------------- +PythonType::PythonType(DATATYPE_UID did): +DataType(did) +{ +} + +//------------------------------------------------------------------------------------- +PythonType::~PythonType() +{ +} + +//------------------------------------------------------------------------------------- +bool PythonType::isSameType(PyObject* pyValue) +{ + if(pyValue == NULL) + { + OUT_TYPE_ERROR("PYTHON"); + return false; + } + + bool ret = script::Pickler::pickle(pyValue).empty(); + if(ret) + OUT_TYPE_ERROR("PYTHON"); + + return !ret; +} + +//------------------------------------------------------------------------------------- +PyObject* PythonType::parseDefaultStr(std::string defaultVal) +{ + if(defaultVal.size() > 0) + { + PyObject* module = PyImport_AddModule("__main__"); + if(module == NULL) + { + PyErr_SetString(PyExc_SystemError, + "PythonType::createObject:PyImport_AddModule __main__ error!"); + + PyErr_PrintEx(0); + S_Return; + } + + PyObject* mdict = PyModule_GetDict(module); // Borrowed reference. + + PyObject* result = PyRun_String(const_cast(defaultVal.c_str()), + Py_eval_input, mdict, mdict); + + if (result == NULL) + { + SCRIPT_ERROR_CHECK(); + S_Return; + } + + return result; + } + + S_Return; +} + +//------------------------------------------------------------------------------------- +void PythonType::addToStream(MemoryStream* mstream, PyObject* pyValue) +{ + std::string datas = script::Pickler::pickle(pyValue); + mstream->appendBlob(datas); +} + +//------------------------------------------------------------------------------------- +PyObject* PythonType::createFromStream(MemoryStream* mstream) +{ + std::string datas = ""; + mstream->readBlob(datas); + + if (datas.size() == 0) + Py_RETURN_NONE; + + return script::Pickler::unpickle(datas); +} + +//------------------------------------------------------------------------------------- +PyDictType::PyDictType(DATATYPE_UID did): +PythonType(did) +{ +} + +//------------------------------------------------------------------------------------- +PyDictType::~PyDictType() +{ +} + +//------------------------------------------------------------------------------------- +bool PyDictType::isSameType(PyObject* pyValue) +{ + if(pyValue == NULL) + { + OUT_TYPE_ERROR("PY_DICT"); + return false; + } + + if(!PyDict_Check(pyValue)) + { + OUT_TYPE_ERROR("PY_DICT"); + return false; + } + + return true; +} + +//------------------------------------------------------------------------------------- +PyObject* PyDictType::parseDefaultStr(std::string defaultVal) +{ + PyObject* pyVal = PythonType::parseDefaultStr(defaultVal); + if(PyDict_Check(pyVal)) + { + return pyVal; + } + + if(pyVal) + Py_DECREF(pyVal); + + return PyDict_New(); +} + +//------------------------------------------------------------------------------------- +PyObject* PyDictType::createFromStream(MemoryStream* mstream) +{ + std::string datas = ""; + mstream->readBlob(datas); + + if (datas.size() == 0) + return PyDict_New(); + + return script::Pickler::unpickle(datas); +} + +//------------------------------------------------------------------------------------- +PyTupleType::PyTupleType(DATATYPE_UID did): +PythonType(did) +{ +} + +//------------------------------------------------------------------------------------- +PyTupleType::~PyTupleType() +{ +} + +//------------------------------------------------------------------------------------- +bool PyTupleType::isSameType(PyObject* pyValue) +{ + if(pyValue == NULL) + { + OUT_TYPE_ERROR("PY_TUPLE"); + return false; + } + + if(!PyTuple_Check(pyValue)) + { + OUT_TYPE_ERROR("PY_TUPLE"); + return false; + } + + return true; +} + +//------------------------------------------------------------------------------------- +PyObject* PyTupleType::parseDefaultStr(std::string defaultVal) +{ + PyObject* pyVal = PythonType::parseDefaultStr(defaultVal); + if(PyTuple_Check(pyVal)) + { + return pyVal; + } + + if(pyVal) + Py_DECREF(pyVal); + + return PyTuple_New(0); +} + +//------------------------------------------------------------------------------------- +PyObject* PyTupleType::createFromStream(MemoryStream* mstream) +{ + std::string datas = ""; + mstream->readBlob(datas); + + if (datas.size() == 0) + return PyTuple_New(0); + + return script::Pickler::unpickle(datas); +} + +//------------------------------------------------------------------------------------- +PyListType::PyListType(DATATYPE_UID did): +PythonType(did) +{ +} + +//------------------------------------------------------------------------------------- +PyListType::~PyListType() +{ +} + +//------------------------------------------------------------------------------------- +bool PyListType::isSameType(PyObject* pyValue) +{ + if(pyValue == NULL) + { + OUT_TYPE_ERROR("PY_LIST"); + return false; + } + + if(!PyList_Check(pyValue)) + { + OUT_TYPE_ERROR("PY_LIST"); + return false; + } + + return true; +} + +//------------------------------------------------------------------------------------- +PyObject* PyListType::parseDefaultStr(std::string defaultVal) +{ + PyObject* pyVal = PythonType::parseDefaultStr(defaultVal); + if(PyList_Check(pyVal)) + { + return pyVal; + } + + if(pyVal) + Py_DECREF(pyVal); + + return PyList_New(0); +} + +//------------------------------------------------------------------------------------- +PyObject* PyListType::createFromStream(MemoryStream* mstream) +{ + std::string datas = ""; + mstream->readBlob(datas); + + if (datas.size() == 0) + return PyList_New(0); + + return script::Pickler::unpickle(datas); +} + +//------------------------------------------------------------------------------------- +BlobType::BlobType(DATATYPE_UID did): +DataType(did) +{ +} + +//------------------------------------------------------------------------------------- +BlobType::~BlobType() +{ +} + +//------------------------------------------------------------------------------------- +bool BlobType::isSameType(PyObject* pyValue) +{ + if(pyValue == NULL) + { + OUT_TYPE_ERROR("BLOB"); + return false; + } + + if (!PyBytes_Check(pyValue) && + !PyObject_TypeCheck(pyValue, script::PyMemoryStream::getScriptType())) + { + OUT_TYPE_ERROR("BLOB"); + return false; + } + + return true; +} + +//------------------------------------------------------------------------------------- +PyObject* BlobType::parseDefaultStr(std::string defaultVal) +{ + return PyBytes_FromStringAndSize(defaultVal.data(), defaultVal.size()); +} + +//------------------------------------------------------------------------------------- +void BlobType::addToStream(MemoryStream* mstream, PyObject* pyValue) +{ + if (!PyBytes_Check(pyValue)) + { + script::PyMemoryStream* pPyMemoryStream = static_cast(pyValue); + MemoryStream& m = pPyMemoryStream->stream(); + mstream->appendBlob((const char*)m.data() + m.rpos(), (ArraySize)m.length()); + } + else + { + Py_ssize_t datasize = PyBytes_GET_SIZE(pyValue); + char* datas = PyBytes_AsString(pyValue); + mstream->appendBlob(datas, (ArraySize)datasize); + } +} + +//------------------------------------------------------------------------------------- +PyObject* BlobType::createFromStream(MemoryStream* mstream) +{ + std::string datas; + mstream->readBlob(datas); + return PyBytes_FromStringAndSize(datas.data(), datas.size()); +} + +//------------------------------------------------------------------------------------- +EntityCallType::EntityCallType(DATATYPE_UID did): +DataType(did) +{ +} + +//------------------------------------------------------------------------------------- +EntityCallType::~EntityCallType() +{ +} + +//------------------------------------------------------------------------------------- +bool EntityCallType::isSameType(PyObject* pyValue) +{ + if(pyValue == NULL) + { + OUT_TYPE_ERROR("ENTITYCALL"); + return false; + } + + if(!(PyObject_TypeCheck(pyValue, EntityCall::getScriptType()) || pyValue == Py_None)) + { + PyTypeObject* type = script::ScriptObject::getScriptObjectType("Entity"); + if(!type || !(PyObject_IsInstance(pyValue, (PyObject *)type))) + { + OUT_TYPE_ERROR("ENTITYCALL"); + return false; + } + } + + return true; +} + +//------------------------------------------------------------------------------------- +PyObject* EntityCallType::parseDefaultStr(std::string defaultVal) +{ + Py_RETURN_NONE; +} + +//------------------------------------------------------------------------------------- +void EntityCallType::addToStream(MemoryStream* mstream, PyObject* pyValue) +{ + COMPONENT_ID cid = 0; + ENTITY_ID id = 0; + uint16 type = 0; + ENTITY_SCRIPT_UID utype = 0; + + if(pyValue != Py_None) + { + PyTypeObject* stype = script::ScriptObject::getScriptObjectType("Entity"); + { + // Ƿһentity? + if(PyObject_IsInstance(pyValue, (PyObject *)stype)) + { + PyObject* pyid = PyObject_GetAttrString(pyValue, "id"); + + if (pyid) + { + id = PyLong_AsLong(pyid); + Py_DECREF(pyid); + + cid = g_componentID; + + if (g_componentType == BASEAPP_TYPE) + type = (uint16)ENTITYCALL_TYPE_BASE; + else if (g_componentType == CELLAPP_TYPE) + type = (uint16)ENTITYCALL_TYPE_CELL; + else + type = (uint16)ENTITYCALL_TYPE_CLIENT; + + PyObject* pyClass = PyObject_GetAttrString(pyValue, "__class__"); + PyObject* pyClassName = PyObject_GetAttrString(pyClass, "__name__"); + + const char* ccattr = PyUnicode_AsUTF8AndSize(pyClassName, NULL); + + Py_DECREF(pyClass); + Py_DECREF(pyClassName); + + ScriptDefModule* pScriptDefModule = EntityDef::findScriptModule(ccattr); + + utype = pScriptDefModule->getUType(); + } + else + { + // ijЩ»ΪNULL 磺ʹweakproxyentityCallѾΪNULL + SCRIPT_ERROR_CHECK(); + id = 0; + cid = 0; + } + } + } + + // ֻentityCall + if(id == 0) + { + EntityCallAbstract* pEntityCallAbstract = static_cast(pyValue); + cid = pEntityCallAbstract->componentID(); + id = pEntityCallAbstract->id(); + type = static_cast(pEntityCallAbstract->type());; + utype = pEntityCallAbstract->utype(); + } + } + + (*mstream) << id; + (*mstream) << cid; + (*mstream) << type; + (*mstream) << utype; +} + +//------------------------------------------------------------------------------------- +PyObject* EntityCallType::createFromStream(MemoryStream* mstream) +{ + if(mstream) + { + COMPONENT_ID cid = 0; + ENTITY_ID id = 0; + uint16 type; + ENTITY_SCRIPT_UID utype; + + (*mstream) >> id >> cid >> type >> utype; + + // Py_None + if(id > 0) + { + if (entityCallType2ComponentType((ENTITYCALL_TYPE)type) == g_componentType) + { + PyObject* entity = EntityDef::tryGetEntity(cid, id); + if (entity != NULL) + { + Py_INCREF(entity); + return entity; + } + } + + return new EntityCall(EntityDef::findScriptModule(utype), NULL, cid, + id, (ENTITYCALL_TYPE)type); + } + } + + Py_RETURN_NONE; +} + +//------------------------------------------------------------------------------------- +FixedArrayType::FixedArrayType(DATATYPE_UID did): +DataType(did) +{ +} + +//------------------------------------------------------------------------------------- +FixedArrayType::~FixedArrayType() +{ + if(dataType_) + dataType_->decRef(); +} + +//------------------------------------------------------------------------------------- +PyObject* FixedArrayType::createNewItemFromObj(PyObject* pyobj) +{ + if(!isSameItemType(pyobj)) + { + Py_RETURN_NONE; + } + + return dataType_->createNewFromObj(pyobj); +} + +//------------------------------------------------------------------------------------- +PyObject* FixedArrayType::createNewFromObj(PyObject* pyobj) +{ + if(!isSameType(pyobj)) + { + Py_RETURN_NONE; + } + + if(PyObject_TypeCheck(pyobj, FixedArray::getScriptType())) + { + Py_INCREF(pyobj); + return pyobj; + } + + FixedArray* pFixedArray = new FixedArray(this); + pFixedArray->initialize(pyobj); + return pFixedArray; +} + +//------------------------------------------------------------------------------------- +bool FixedArrayType::initialize(XML* xml, TiXmlNode* node, const std::string& parentName) +{ + dataType_ = NULL; + + TiXmlNode* arrayNode = xml->enterNode(node, "of"); + if (arrayNode == NULL) + { + ERROR_MSG("FixedArrayType::initialize: not found \"of\".\n"); + return false; + } + + std::string strType = xml->getValStr(arrayNode); + + if (strType == "ARRAY") + { + FixedArrayType* dataType = new FixedArrayType(); + + std::string childName = std::string("_") + parentName + + dataType->aliasName() + "_ArrayType"; + + if (dataType->initialize(xml, arrayNode, childName)) + { + dataType_ = dataType; + dataType_->incRef(); + + DataTypes::addDataType(childName, dataType); + } + else + { + ERROR_MSG("FixedArrayType::initialize: Array is wrong.\n"); + delete dataType; + return false; + } + } + else + { + DataType* dataType = DataTypes::getDataType(strType); + if (dataType != NULL) + { + dataType_ = dataType; + dataType_->incRef(); + } + else + { + ERROR_MSG(fmt::format("FixedArrayType::initialize: key[{}] did not find type[{}]!\n", + "ARRAY", strType.c_str())); + + return false; + } + } + + if (dataType_ == NULL) + { + ERROR_MSG("FixedArrayType::initialize: dataType is NULL.\n"); + return false; + } + + DATATYPE_UID uid = dataType_->id(); + EntityDef::md5().append((void*)&uid, sizeof(DATATYPE_UID)); + EntityDef::md5().append((void*)strType.c_str(), (int)strType.size()); + return true; +} + +//------------------------------------------------------------------------------------- +bool FixedArrayType::initialize(script::entitydef::DefContext* pDefContext, const std::string& parentName) +{ + KBE_ASSERT(pDefContext->type == script::entitydef::DefContext::DC_TYPE_FIXED_ARRAY); + + dataType_ = NULL; + std::string strType; + + script::entitydef::DefContext* pDefContextItem = script::entitydef::DefContext::findDefContext(pDefContext->returnType); + if (pDefContextItem) + { + if (pDefContextItem->type == script::entitydef::DefContext::DC_TYPE_FIXED_ARRAY) + { + FixedArrayType* dataType = new FixedArrayType(); + + std::string childName = std::string("_") + parentName + dataType->aliasName() + "_ArrayType"; + strType += childName; + + if (dataType->initialize(pDefContextItem, childName)) + { + dataType_ = dataType; + dataType_->incRef(); + DataTypes::addDataType(childName, dataType); + } + else + { + ERROR_MSG("PyEntityDef::FixedArrayType::initialize: Array is wrong.\n"); + delete dataType; + return false; + } + } + else + { + goto FIND_IN_DATATYPES; + } + } + else + { + FIND_IN_DATATYPES: + dataType_ = DataTypes::getDataType(pDefContext->returnType, false); + if (dataType_ != NULL) + { + dataType_->incRef(); + } + else + { + ERROR_MSG(fmt::format("PyEntityDef::registerDefTypes: parse {} error, item({}), not a legal data type, file: \"{}\"\n", + pDefContext->moduleName.c_str(), pDefContext->returnType, pDefContext->pyObjectSourceFile)); + + return false; + } + } + + if (dataType_ == NULL) + { + ERROR_MSG("PyEntityDef::FixedArrayType::initialize: dataType is NULL.\n"); + return false; + } + + strType += pDefContext->returnType; + DATATYPE_UID uid = dataType_->id(); + EntityDef::md5().append((void*)&uid, sizeof(DATATYPE_UID)); + EntityDef::md5().append((void*)strType.c_str(), (int)strType.size()); + return true; +} + +//------------------------------------------------------------------------------------- +bool FixedArrayType::isSameItemType(PyObject* pyValue) +{ + return dataType_->isSameType(pyValue); +} + +//------------------------------------------------------------------------------------- +bool FixedArrayType::isSameType(PyObject* pyValue) +{ + if(pyValue == NULL) + { + OUT_TYPE_ERROR("ARRAY"); + return false; + } + + if(!PySequence_Check(pyValue)) + { + OUT_TYPE_ERROR("ARRAY"); + return false; + } + + Py_ssize_t size = PySequence_Size(pyValue); + for(Py_ssize_t i=0; iisSameType(pyVal); + Py_DECREF(pyVal); + + if(!ok) + return false; + } + + return true; +} + +//------------------------------------------------------------------------------------- +PyObject* FixedArrayType::parseDefaultStr(std::string defaultVal) +{ + FixedArray* pFixedArray = new FixedArray(this); + pFixedArray->initialize(defaultVal); + return pFixedArray; +} + +//------------------------------------------------------------------------------------- +void FixedArrayType::addToStream(MemoryStream* mstream, PyObject* pyValue) +{ + addToStreamEx(mstream, pyValue, false); +} + +//------------------------------------------------------------------------------------- +void FixedArrayType::addToStreamEx(MemoryStream* mstream, PyObject* pyValue, bool onlyPersistents) +{ + ArraySize size = (ArraySize)PySequence_Size(pyValue); + (*mstream) << size; + + for(ArraySize i=0; itype() == DATA_TYPE_FIXEDDICT) + ((FixedDictType*)dataType_)->addToStreamEx(mstream, pyVal, onlyPersistents); + else if(dataType_->type() == DATA_TYPE_FIXEDARRAY) + ((FixedArrayType*)dataType_)->addToStreamEx(mstream, pyVal, onlyPersistents); + else + dataType_->addToStream(mstream, pyVal); + + Py_DECREF(pyVal); + } +} + +//------------------------------------------------------------------------------------- +PyObject* FixedArrayType::createFromStream(MemoryStream* mstream) +{ + return createFromStreamEx(mstream, false); +} + +//------------------------------------------------------------------------------------- +PyObject* FixedArrayType::createFromStreamEx(MemoryStream* mstream, bool onlyPersistents) +{ + ArraySize size; + + FixedArray* pFixedArray = new FixedArray(this); + pFixedArray->initialize(""); + + if(mstream) + { + (*mstream) >> size; + + std::vector& vals = pFixedArray->getValues(); + for(ArraySize i=0; ilength() == 0) + { + ERROR_MSG(fmt::format("FixedArrayType::createFromStream: {} invalid(size={}), stream no space!\n", + aliasName(), size)); + + break; + } + + PyObject* pyVal = NULL; + + if(dataType_->type() == DATA_TYPE_FIXEDDICT) + pyVal = ((FixedDictType*)dataType_)->createFromStreamEx(mstream, onlyPersistents); + else if(dataType_->type() == DATA_TYPE_FIXEDARRAY) + pyVal = ((FixedArrayType*)dataType_)->createFromStreamEx(mstream, onlyPersistents); + else + pyVal = dataType_->createFromStream(mstream); + + if(pyVal) + { + vals.push_back(pyVal); + } + else + { + ERROR_MSG(fmt::format("FixedArrayType::createFromStream: {}, pyVal is NULL! size={}\n", + aliasName(), size)); + + break; + } + } + } + + return pFixedArray; +} + +//------------------------------------------------------------------------------------- +FixedDictType::FixedDictType(DATATYPE_UID did): +DataType(did), +keyTypes_(), +implObj_(NULL), +pycreateObjFromDict_(NULL), +pygetDictFromObj_(NULL), +pyisSameType_(NULL), +moduleName_() +{ +} + +//------------------------------------------------------------------------------------- +FixedDictType::~FixedDictType() +{ + FIXEDDICT_KEYTYPE_MAP::iterator iter = keyTypes_.begin(); + for(; iter != keyTypes_.end(); ++iter) + { + iter->second->dataType->decRef(); + } + + keyTypes_.clear(); + + S_RELEASE(pycreateObjFromDict_); + S_RELEASE(pygetDictFromObj_); + S_RELEASE(pyisSameType_); + S_RELEASE(implObj_); +} + +//------------------------------------------------------------------------------------- +std::string FixedDictType::getKeyNames(void) +{ + std::string keyNames = ""; + + FIXEDDICT_KEYTYPE_MAP::iterator iter = keyTypes_.begin(); + for(; iter != keyTypes_.end(); ++iter) + { + keyNames += iter->first + ","; + } + + return keyNames; +} + +//------------------------------------------------------------------------------------- +std::string FixedDictType::debugInfos(void) +{ + std::string retstr = ""; + + FIXEDDICT_KEYTYPE_MAP::iterator iter = keyTypes_.begin(); + for(; iter != keyTypes_.end(); ++iter) + { + retstr += iter->first; + retstr += "("; + retstr += iter->second->dataType->aliasName(); + retstr += "), "; + } + + if(retstr.size() > 0) + retstr.erase(retstr.size() - 2, 2); + + return retstr; +} + +//------------------------------------------------------------------------------------- +std::string FixedDictType::getNotFoundKeys(PyObject* dict) +{ + std::string notFoundKeys = ""; + + FIXEDDICT_KEYTYPE_MAP::iterator iter = keyTypes_.begin(); + for (; iter != keyTypes_.end(); ++iter) + { + PyObject* pyObject = PyDict_GetItemString(dict, const_cast(iter->first.c_str())); + if (pyObject == NULL) + { + notFoundKeys += iter->first.c_str(); + notFoundKeys += ", "; + + if (PyErr_Occurred()) + PyErr_Clear(); + } + } + + if (notFoundKeys.size() > 0) + notFoundKeys.erase(notFoundKeys.size() - 2, 2); + + return notFoundKeys; +} + +//------------------------------------------------------------------------------------- +PyObject* FixedDictType::createNewItemFromObj(const char* keyName, PyObject* pyobj) +{ + DataType* dataType = isSameItemType(keyName, pyobj); + if(!dataType) + { + Py_RETURN_NONE; + } + + FIXEDDICT_KEYTYPE_MAP::iterator iter = keyTypes_.begin(); + for(; iter != keyTypes_.end(); ++iter) + { + if(iter->first == keyName) + { + return iter->second->dataType->createNewFromObj(pyobj); + } + } + + Py_RETURN_NONE; +} + +//------------------------------------------------------------------------------------- +PyObject* FixedDictType::createNewFromObj(PyObject* pyobj) +{ + if(!isSameType(pyobj)) + { + Py_RETURN_NONE; + } + + if(this->hasImpl()) + { + return impl_createObjFromDict(pyobj); + } + + // ڴʱѾFixedDict, ΪparseDefaultStr + // ʼΪն + if(PyObject_TypeCheck(pyobj, FixedDict::getScriptType())) + { + Py_INCREF(pyobj); + return pyobj; + } + + FixedDict* pFixedDict = new FixedDict(this); + pFixedDict->initialize(pyobj); + return pFixedDict; +} + +//------------------------------------------------------------------------------------- +bool FixedDictType::initialize(XML* xml, TiXmlNode* node, std::string& parentName) +{ + TiXmlNode* propertiesNode = xml->enterNode(node, "Properties"); + if(propertiesNode == NULL) + { + ERROR_MSG("FixedDictType::initialize: not found \"Properties\".\n"); + return false; + } + + std::string strType = "", typeName = ""; + + XML_FOR_BEGIN(propertiesNode) + { + typeName = xml->getKey(propertiesNode); + + TiXmlNode* typeNode = xml->enterNode(propertiesNode->FirstChild(), "Type"); + TiXmlNode* PersistentNode = xml->enterNode(propertiesNode->FirstChild(), "Persistent"); + TiXmlNode* DatabaseLengthNode = xml->enterNode(propertiesNode->FirstChild(), "DatabaseLength"); + + bool persistent = true; + if(PersistentNode) + { + std::string strval = xml->getValStr(PersistentNode); + if(strval == "false") + { + persistent = false; + } + } + + uint32 databaseLength = 0; + if (DatabaseLengthNode) + { + databaseLength = xml->getValInt(DatabaseLengthNode); + } + + if(typeNode) + { + strType = xml->getValStr(typeNode); + + if(strType == "ARRAY") + { + FixedArrayType* dataType = new FixedArrayType(); + DictItemDataTypePtr pDictItemDataType(new DictItemDataType()); + pDictItemDataType->dataType = dataType; + + std::string childName = std::string("_") + parentName + std::string("_") + typeName + "_ArrayType"; + if (dataType->initialize(xml, typeNode, childName)) + { + DATATYPE_UID uid = dataType->id(); + EntityDef::md5().append((void*)&uid, sizeof(DATATYPE_UID)); + EntityDef::md5().append((void*)strType.c_str(), (int)strType.size()); + EntityDef::md5().append((void*)typeName.c_str(), (int)typeName.size()); + + keyTypes_.push_back(std::pair< std::string, DictItemDataTypePtr >(typeName, pDictItemDataType)); + dataType->incRef(); + + if (dataType->getDataType()->type() == DATA_TYPE_ENTITYCALL) + { + persistent = false; + } + + pDictItemDataType->persistent = persistent; + pDictItemDataType->databaseLength = databaseLength; + EntityDef::md5().append((void*)&persistent, sizeof(bool)); + EntityDef::md5().append((void*)&databaseLength, sizeof(uint32)); + DataTypes::addDataType(childName, dataType); + } + else + { + ERROR_MSG(fmt::format("FixedDictType::initialize: key[{}] did not find array-type[{}]!\n", + typeName.c_str(), strType.c_str())); + + return false; + } + } + else + { + DataType* dataType = DataTypes::getDataType(strType); + DictItemDataTypePtr pDictItemDataType(new DictItemDataType()); + pDictItemDataType->dataType = dataType; + + if(dataType != NULL) + { + DATATYPE_UID uid = dataType->id(); + EntityDef::md5().append((void*)&uid, sizeof(DATATYPE_UID)); + EntityDef::md5().append((void*)strType.c_str(), (int)strType.size()); + EntityDef::md5().append((void*)typeName.c_str(), (int)typeName.size()); + + keyTypes_.push_back(std::pair< std::string, DictItemDataTypePtr >(typeName, pDictItemDataType)); + dataType->incRef(); + + if(dataType->type() == DATA_TYPE_ENTITYCALL) + { + persistent = false; + } + + pDictItemDataType->persistent = persistent; + pDictItemDataType->databaseLength = databaseLength; + EntityDef::md5().append((void*)&persistent, sizeof(bool)); + EntityDef::md5().append((void*)&databaseLength, sizeof(uint32)); + } + else + { + ERROR_MSG(fmt::format("FixedDictType::initialize: key[{}] did not find type[{}]!\n", + typeName.c_str(), strType.c_str())); + + return false; + } + } + } + else + { + ERROR_MSG(fmt::format("FixedDictType::initialize: key[{}] no label[\"Type\"], key[{}] will be ignored!\n", + typeName, typeName)); + } + } + XML_FOR_END(propertiesNode); + + TiXmlNode* implementedByNode = xml->enterNode(node, "implementedBy"); + if(implementedByNode) + { + strType = xml->getValStr(implementedByNode); + + if(g_componentType == CELLAPP_TYPE || g_componentType == BASEAPP_TYPE || + g_componentType == CLIENT_TYPE) + { + if(strType.size() > 0 && !loadImplModule(strType)) + return false; + + moduleName_ = strType; + } + + if(strType.size() > 0) + EntityDef::md5().append((void*)strType.c_str(), (int)strType.size()); + } + + if (keyTypes_.size() == 0) + { + ERROR_MSG(fmt::format("FixedDictType::initialize(): FIXED_DICT({}) no keys! \n", + this->aliasName())); + + return false; + } + + return true; +} + +//------------------------------------------------------------------------------------- +bool FixedDictType::initialize(script::entitydef::DefContext* pDefContext, const std::string& parentName) +{ + KBE_ASSERT(pDefContext->type == script::entitydef::DefContext::DC_TYPE_FIXED_DICT); + + if (pDefContext->implementedByModuleName.size() > 0) + { + if (g_componentType == CELLAPP_TYPE || g_componentType == BASEAPP_TYPE || + g_componentType == CLIENT_TYPE) + { + PyObject* implementedBy = pDefContext->implementedBy.get(); + Py_INCREF(implementedBy); + + if (!setImplModule(implementedBy)) + { + ERROR_MSG(fmt::format("PyEntityDef::FixedDictType::initialize(): FIXED_DICT({}) setImplModule error!, file: \"{}\"!\n", + this->aliasName(), pDefContext->pyObjectSourceFile)); + + return false; + } + + moduleName_ = pDefContext->implementedByModuleName; + } + } + + script::entitydef::DefContext::DEF_CONTEXTS::iterator propIter = pDefContext->propertys.begin(); + for (; propIter != pDefContext->propertys.end(); ++propIter) + { + script::entitydef::DefContext& defContextItem = (*propIter); + KBE_ASSERT(defContextItem.type == script::entitydef::DefContext::DC_TYPE_FIXED_ITEM); + + bool persistent = true; + if (defContextItem.persistent != -1) + { + if(defContextItem.persistent <= 0) + persistent = false; + } + + uint32 databaseLength = defContextItem.databaseLength; + + script::entitydef::DefContext* pDefContextItemType = script::entitydef::DefContext::findDefContext(defContextItem.returnType); + if (pDefContextItemType) + { + if (pDefContextItemType->type == script::entitydef::DefContext::DC_TYPE_FIXED_ARRAY) + { + FixedArrayType* dataType = new FixedArrayType(); + DictItemDataTypePtr pDictItemDataType(new DictItemDataType()); + pDictItemDataType->dataType = dataType; + + std::string childName = std::string("_") + parentName + std::string("_") + defContextItem.attrName + "_ArrayType"; + if (dataType->initialize(pDefContextItemType, childName)) + { + DATATYPE_UID uid = dataType->id(); + EntityDef::md5().append((void*)&uid, sizeof(DATATYPE_UID)); + EntityDef::md5().append((void*)defContextItem.attrName.c_str(), (int)defContextItem.attrName.size()); + EntityDef::md5().append((void*)childName.c_str(), (int)childName.size()); + + keyTypes_.push_back(std::pair< std::string, DictItemDataTypePtr >(defContextItem.attrName, pDictItemDataType)); + dataType->incRef(); + + if (dataType->getDataType()->type() == DATA_TYPE_ENTITYCALL) + { + persistent = false; + } + + pDictItemDataType->persistent = persistent; + pDictItemDataType->databaseLength = databaseLength; + EntityDef::md5().append((void*)&persistent, sizeof(bool)); + EntityDef::md5().append((void*)&databaseLength, sizeof(uint32)); + DataTypes::addDataType(childName, dataType); + } + else + { + return false; + } + } + else + { + goto FIND_IN_DATATYPES; + } + } + else + { + FIND_IN_DATATYPES: + DataType* dataType = DataTypes::getDataType(defContextItem.returnType, false); + DictItemDataTypePtr pDictItemDataType(new DictItemDataType()); + pDictItemDataType->dataType = dataType; + + if (dataType != NULL) + { + DATATYPE_UID uid = dataType->id(); + EntityDef::md5().append((void*)&uid, sizeof(DATATYPE_UID)); + EntityDef::md5().append((void*)defContextItem.attrName.c_str(), (int)defContextItem.attrName.size()); + EntityDef::md5().append((void*)defContextItem.returnType.c_str(), (int)defContextItem.returnType.size()); + + keyTypes_.push_back(std::pair< std::string, DictItemDataTypePtr >(defContextItem.attrName, pDictItemDataType)); + dataType->incRef(); + + if (dataType->type() == DATA_TYPE_ENTITYCALL) + { + persistent = false; + } + + pDictItemDataType->persistent = persistent; + pDictItemDataType->databaseLength = databaseLength; + EntityDef::md5().append((void*)&persistent, sizeof(bool)); + EntityDef::md5().append((void*)&databaseLength, sizeof(uint32)); + } + else + { + ERROR_MSG(fmt::format("PyEntityDef::FixedDictType::initialize: {}.{} is not a legal data type[{}], file: \"{}\"!\n", + defContextItem.moduleName.c_str(), defContextItem.attrName.c_str(), defContextItem.returnType, defContextItem.pyObjectSourceFile)); + + return false; + } + } + } + + if (keyTypes_.size() == 0) + { + ERROR_MSG(fmt::format("PyEntityDef::FixedDictType::initialize(): FIXED_DICT({}) no keys!, file: \"{}\"!\n", + this->aliasName(), pDefContext->pyObjectSourceFile)); + + return false; + } + + return true; +} + +//------------------------------------------------------------------------------------- +bool FixedDictType::loadImplModule(std::string moduleName) +{ + KBE_ASSERT(implObj_ == NULL); + + std::vector res_; + strutil::kbe_split(moduleName, '.', res_); + + if (res_.size() != 2) + { + ERROR_MSG(fmt::format("FixedDictType::loadImplModule: {} impl error! like:[moduleName.ClassName|moduleName.xxInstance]\n", + moduleName.c_str())); + + return false; + } + + PyObject* implModule = PyImport_ImportModule(res_[0].c_str()); + if (!implModule) + { + SCRIPT_ERROR_CHECK(); + return false; + } + + PyObject* pyImplObj = PyObject_GetAttrString(implModule, res_[1].c_str()); + bool ret = setImplModule(pyImplObj); + Py_DECREF(implModule); + return ret; +} + +//------------------------------------------------------------------------------------- +bool FixedDictType::setImplModule(PyObject* pyobj) +{ + implObj_ = pyobj; + + if (!pyobj) + { + SCRIPT_ERROR_CHECK() + return false; + } + + if (PyType_Check(implObj_)) + { + PyObject* implClass = implObj_; + implObj_ = PyObject_CallObject(implClass, NULL); + Py_DECREF(implClass); + + if (!implObj_) + { + SCRIPT_ERROR_CHECK() + return false; + } + } + + pycreateObjFromDict_ = PyObject_GetAttrString(implObj_, "createObjFromDict"); + if (!pycreateObjFromDict_) + { + SCRIPT_ERROR_CHECK() + return false; + } + + pygetDictFromObj_ = PyObject_GetAttrString(implObj_, "getDictFromObj"); + if (!pygetDictFromObj_) + { + SCRIPT_ERROR_CHECK() + return false; + } + + pyisSameType_ = PyObject_GetAttrString(implObj_, "isSameType"); + if (!pyisSameType_) + { + SCRIPT_ERROR_CHECK() + return false; + } + + return true; +} + +//------------------------------------------------------------------------------------- +PyObject* FixedDictType::impl_createObjFromDict(PyObject* dictData) +{ + // ڴʱѾû, ΪparseDefaultStr + // ʼΪն + if(!PyObject_TypeCheck(dictData, FixedDict::getScriptType()) && impl_isSameType(dictData)) + { + Py_INCREF(dictData); + return dictData; + } + + PyObject* pyRet = PyObject_CallFunction(pycreateObjFromDict_, + const_cast("(O)"), dictData); + + if(pyRet == NULL || !impl_isSameType(pyRet)) + { + SCRIPT_ERROR_CHECK(); + + ERROR_MSG(fmt::format("FixedDictType::impl_createObjFromDict: {}.isSameType() is failed!\n", + moduleName_.c_str())); + + Py_RETURN_NONE; + } + + return pyRet; +} + +//------------------------------------------------------------------------------------- +PyObject* FixedDictType::impl_getDictFromObj(PyObject* pyobj) +{ + PyObject* pyRet = PyObject_CallFunction(pygetDictFromObj_, + const_cast("(O)"), pyobj); + + if(pyRet == NULL) + { + SCRIPT_ERROR_CHECK(); + return parseDefaultStr(""); + } + + return pyRet; +} + +//------------------------------------------------------------------------------------- +bool FixedDictType::impl_isSameType(PyObject* pyobj) +{ + PyObject* pyRet = PyObject_CallFunction(pyisSameType_, + const_cast("(O)"), pyobj); + + if(pyRet == NULL) + { + SCRIPT_ERROR_CHECK(); + return false; + } + + bool ret = Py_True == pyRet; + Py_DECREF(pyRet); + return ret; +} + +//------------------------------------------------------------------------------------- +DataType* FixedDictType::isSameItemType(const char* keyName, PyObject* pyValue) +{ + FIXEDDICT_KEYTYPE_MAP::iterator iter = keyTypes_.begin(); + for(; iter != keyTypes_.end(); ++iter) + { + if(iter->first == keyName) + { + if(pyValue == NULL || !iter->second->dataType->isSameType(pyValue)) + { + PyErr_Format(PyExc_TypeError, + "set FIXED_DICT(%s) error! at key: %s(%s), keyNames=[%s].", + this->aliasName(), + iter->first.c_str(), + (pyValue == NULL ? "NULL" : pyValue->ob_type->tp_name), + debugInfos().c_str()); + + PyErr_PrintEx(0); + return NULL; + } + else + { + return iter->second->dataType; + } + } + } + + return NULL; +} + +//------------------------------------------------------------------------------------- +bool FixedDictType::isSameType(PyObject* pyValue) +{ + if(pyValue == NULL) + { + OUT_TYPE_ERROR("DICT"); + return false; + } + + if(hasImpl()) + { + // ﷵfalse󻹼жϵԭisSameTypeΪ + // fixeddictûӦǺϷ + if(impl_isSameType(pyValue)) + return true; + } + + if(PyObject_TypeCheck(pyValue, FixedDict::getScriptType())) + { + if(static_cast(pyValue)->getDataType()->id() == this->id()) + return true; + else + return false; + } + + if(!PyDict_Check(pyValue)) + { + OUT_TYPE_ERROR("DICT"); + return false; + } + + Py_ssize_t dictSize = PyDict_Size(pyValue); + if(dictSize != (Py_ssize_t)keyTypes_.size()) + { + PyErr_Format(PyExc_TypeError, + "FIXED_DICT(%s) key does not match! giveKeySize=%d, dictKeySize=%d, dictKeyNames=[%s], notFoundKeys=[%s].", + this->aliasName(), dictSize, keyTypes_.size(), + debugInfos().c_str(), getNotFoundKeys(pyValue).c_str()); + + PyErr_PrintEx(0); + return false; + } + + FIXEDDICT_KEYTYPE_MAP::iterator iter = keyTypes_.begin(); + for(; iter != keyTypes_.end(); ++iter) + { + PyObject* pyObject = PyDict_GetItemString(pyValue, const_cast(iter->first.c_str())); + if(pyObject == NULL) + { + PyErr_Format(PyExc_TypeError, + "set FIXED_DICT(%s) error! keys[%s] not found, allKeyNames=[%s].", + this->aliasName(), getNotFoundKeys(pyValue).c_str(), debugInfos().c_str()); + + PyErr_PrintEx(0); + return false; + } + else if (!iter->second->dataType->isSameType(pyObject)) + { + PyErr_Format(PyExc_TypeError, + "set FIXED_DICT(%s) error! at key: %s(%s), allKeyNames=[%s].", + this->aliasName(), + iter->first.c_str(), + pyObject->ob_type->tp_name, + debugInfos().c_str()); + + PyErr_PrintEx(0); + return false; + } + } + + return true; +} + +//------------------------------------------------------------------------------------- +PyObject* FixedDictType::parseDefaultStr(std::string defaultVal) +{ + FixedDict* pydict = new FixedDict(this); + pydict->initialize(defaultVal); + + if (hasImpl()) + { + PyObject* pyValue = impl_createObjFromDict(pydict); + Py_DECREF(pydict); + return pyValue; + } + + return pydict; +} + +//------------------------------------------------------------------------------------- +void FixedDictType::addToStream(MemoryStream* mstream, PyObject* pyValue) +{ + addToStreamEx(mstream, pyValue, false); +} + +//------------------------------------------------------------------------------------- +void FixedDictType::addToStreamEx(MemoryStream* mstream, PyObject* pyValue, bool onlyPersistents) +{ + if(hasImpl()) + { + pyValue = impl_getDictFromObj(pyValue); + } + + PyObject* pydict = pyValue; + if(PyObject_TypeCheck(pyValue, FixedDict::getScriptType())) + { + pydict = static_cast(pyValue)->getDictObject(); + } + + FIXEDDICT_KEYTYPE_MAP::iterator iter = keyTypes_.begin(); + for(; iter != keyTypes_.end(); ++iter) + { + if(onlyPersistents) + { + if(!iter->second->persistent) + continue; + } + + PyObject* pyObject = + PyDict_GetItemString(pydict, const_cast(iter->first.c_str())); + + if(pyObject == NULL) + { + ERROR_MSG(fmt::format("FixedDictType::addToStreamEx: {} not found key[{}]. keyNames[{}]\n", + this->aliasName_, iter->first, this->debugInfos())); + + // KBE_ASSERT(pyObject != NULL); + PyObject* pobj = iter->second->dataType->parseDefaultStr(""); + + if(iter->second->dataType->type() == DATA_TYPE_FIXEDDICT) + ((FixedDictType*)iter->second->dataType)->addToStreamEx(mstream, pobj, onlyPersistents); + else if(iter->second->dataType->type() == DATA_TYPE_FIXEDARRAY) + ((FixedArrayType*)iter->second->dataType)->addToStreamEx(mstream, pobj, onlyPersistents); + else + iter->second->dataType->addToStream(mstream, pobj); + + Py_DECREF(pobj); + continue; + } + + if(!iter->second->dataType->isSameType(pyObject)) + { + // KBE_ASSERT(pyObject != NULL); + PyObject* pobj = iter->second->dataType->parseDefaultStr(""); + + if(iter->second->dataType->type() == DATA_TYPE_FIXEDDICT) + ((FixedDictType*)iter->second->dataType)->addToStreamEx(mstream, pobj, onlyPersistents); + else if(iter->second->dataType->type() == DATA_TYPE_FIXEDARRAY) + ((FixedArrayType*)iter->second->dataType)->addToStreamEx(mstream, pobj, onlyPersistents); + else + iter->second->dataType->addToStream(mstream, pobj); + + Py_DECREF(pobj); + continue; + } + + if(iter->second->dataType->type() == DATA_TYPE_FIXEDDICT) + ((FixedDictType*)iter->second->dataType)->addToStreamEx(mstream, pyObject, onlyPersistents); + else if(iter->second->dataType->type() == DATA_TYPE_FIXEDARRAY) + ((FixedArrayType*)iter->second->dataType)->addToStreamEx(mstream, pyObject, onlyPersistents); + else + iter->second->dataType->addToStream(mstream, pyObject); + } + + if(hasImpl()) + { + Py_DECREF(pyValue); + } +} + +//------------------------------------------------------------------------------------- +PyObject* FixedDictType::createFromStream(MemoryStream* mstream) +{ + return createFromStreamEx(mstream, false); +} + +//------------------------------------------------------------------------------------- +PyObject* FixedDictType::createFromStreamEx(MemoryStream* mstream, bool onlyPersistents) +{ + FixedDict* pydict = new FixedDict(this, onlyPersistents); + pydict->initialize(mstream, onlyPersistents); + + if(hasImpl()) + { + PyObject* pyValue = impl_createObjFromDict(pydict); + Py_DECREF(pydict); + return pyValue; + } + + return pydict; +} + +//------------------------------------------------------------------------------------- +EntityComponentType::EntityComponentType(ScriptDefModule* pScriptDefModule, DATATYPE_UID did) : + DataType(did), + pScriptDefModule_(pScriptDefModule) +{ +} + +//------------------------------------------------------------------------------------- +EntityComponentType::~EntityComponentType() +{ +} + +//------------------------------------------------------------------------------------- +bool EntityComponentType::isSameType(PyObject* pyValue) +{ + if (pyValue == NULL || !(PyObject_TypeCheck(pyValue, EntityComponent::getScriptType()))) + { + OUT_TYPE_ERROR("ENTITY_COMPONENT"); + return false; + } + + EntityComponent* pEntityComponent = static_cast(pyValue); + return pEntityComponent->isSameType(pyValue); +} + +//------------------------------------------------------------------------------------- +bool EntityComponentType::isSamePersistentType(PyObject* pyValue) +{ + if (pyValue == NULL) + { + OUT_TYPE_ERROR("ENTITY_COMPONENT"); + return false; + } + + if (!(PyObject_TypeCheck(pyValue, EntityComponent::getScriptType()))) + { + if (!PyDict_Check(pyValue)) + { + OUT_TYPE_ERROR("ENTITY_COMPONENT"); + } + + return isSameCellDataType(pyValue); + } + + EntityComponent* pEntityComponent = static_cast(pyValue); + return pEntityComponent->isSamePersistentType(pyValue); +} + +//------------------------------------------------------------------------------------- +bool EntityComponentType::isSameCellDataType(PyObject* pyValue) +{ + if (!PyDict_Check(pyValue)) + return false; + + ScriptDefModule::PROPERTYDESCRIPTION_MAP& propertyDescrs = pScriptDefModule_->getPersistentPropertyDescriptions(); + ScriptDefModule::PROPERTYDESCRIPTION_MAP::const_iterator iter = propertyDescrs.begin(); + + for (; iter != propertyDescrs.end(); ++iter) + { + PropertyDescription* propertyDescription = iter->second; + + if (!propertyDescription->hasCell()) + continue; + + PyObject* pyVal = PyDict_GetItemString(pyValue, propertyDescription->getName()); + + if (pyVal) + { + if (!propertyDescription->getDataType()->isSameType(pyVal)) + { + ERROR_MSG(fmt::format("EntityComponent::isSameCellDataType: {} type(curr_py: {} != {}) error! name={}, utype={}, domain={}.\n", + propertyDescription->getName(), (pyVal ? pyVal->ob_type->tp_name : "unknown"), propertyDescription->getDataType()->getName(), + pScriptDefModule_ ? pScriptDefModule_->getName() : "", pScriptDefModule_ ? pScriptDefModule_->getUType() : 0, + COMPONENT_NAME_EX(CELLAPP_TYPE))); + + return false; + } + } + else + { + SCRIPT_ERROR_CHECK(); + + ERROR_MSG(fmt::format("EntityComponent::isSameCellDataType: not found property({}), use default values! name={}, utype={}, domain={}.\n", + propertyDescription->getName(), pScriptDefModule_ ? pScriptDefModule_->getName() : "", pScriptDefModule_ ? pScriptDefModule_->getUType() : 0, + COMPONENT_NAME_EX(CELLAPP_TYPE))); + + return false; + } + } + + return true; +} + +//------------------------------------------------------------------------------------- +PyObject* EntityComponentType::parseDefaultStr(std::string defaultVal) +{ + KBE_ASSERT(EntityDef::context().currEntityID > 0); + + PyObject* pyobj = pScriptDefModule_->createObject(); + + // ִEntityĹ캯 + return new(pyobj) EntityComponent(EntityDef::context().currEntityID, pScriptDefModule_, EntityDef::context().currComponentType); +} + +//------------------------------------------------------------------------------------- +void EntityComponentType::addToStream(MemoryStream* mstream, PyObject* pyValue) +{ + EntityComponent* pEntityComponent = static_cast(pyValue); + pEntityComponent->addToStream(mstream, pyValue); +} + +//------------------------------------------------------------------------------------- +void EntityComponentType::addPersistentToStream(MemoryStream* mstream, PyObject* pyValue) +{ + if (PyDict_Check(pyValue)) + { + ScriptDefModule::PROPERTYDESCRIPTION_MAP& propertyDescrs = pScriptDefModule_->getPersistentPropertyDescriptions(); + ScriptDefModule::PROPERTYDESCRIPTION_MAP::const_iterator iter = propertyDescrs.begin(); + + for (; iter != propertyDescrs.end(); ++iter) + { + PropertyDescription* propertyDescription = iter->second; + + // ﴫһֵ䣬ô϶һcelldataֵ䣬ֻcell + if (!propertyDescription->hasCell()) + continue; + + PyObject* pyVal = PyDict_GetItemString(pyValue, propertyDescription->getName()); + + if (pyVal) + { + propertyDescription->getDataType()->addToStream(mstream, pyVal); + } + else + { + SCRIPT_ERROR_CHECK(); + + ERROR_MSG(fmt::format("EntityComponent::addPersistentToStream: not found property({}), use default values! name={}, utype={}, domain={}.\n", + propertyDescription->getName(), pScriptDefModule_ ? pScriptDefModule_->getName() : "", pScriptDefModule_ ? pScriptDefModule_->getUType() : 0, + COMPONENT_NAME_EX(CELLAPP_TYPE))); + + propertyDescription->addPersistentToStream(mstream, NULL); + } + } + + return; + } + + EntityComponent* pEntityComponent = static_cast(pyValue); + pEntityComponent->addPersistentToStream(mstream, pyValue); +} + +//------------------------------------------------------------------------------------- +void EntityComponentType::addPersistentToStream(MemoryStream* mstream) +{ + ScriptDefModule::PROPERTYDESCRIPTION_MAP& propertyDescrs = pScriptDefModule_->getPersistentPropertyDescriptions(); + ScriptDefModule::PROPERTYDESCRIPTION_MAP::const_iterator iter = propertyDescrs.begin(); + + for (; iter != propertyDescrs.end(); ++iter) + { + PropertyDescription* propertyDescription = iter->second; + + PyObject* pyDefVal = propertyDescription->newDefaultVal(); + propertyDescription->getDataType()->addToStream(mstream, pyDefVal); + Py_DECREF(pyDefVal); + } +} +//------------------------------------------------------------------------------------- +void EntityComponentType::addPersistentToStreamTemplates(ScriptDefModule* pScriptModule, MemoryStream* mstream) +{ + ScriptDefModule::PROPERTYDESCRIPTION_MAP& propertyDescrs = pScriptDefModule_->getPersistentPropertyDescriptions(); + ScriptDefModule::PROPERTYDESCRIPTION_MAP::const_iterator iter = propertyDescrs.begin(); + + for (; iter != propertyDescrs.end(); ++iter) + { + PropertyDescription* propertyDescription = iter->second; + + if (propertyDescription->hasCell()) + { + // һЩʵûcell֣ cellԺ + if (!pScriptModule->hasCell()) + continue; + } + + propertyDescription->addPersistentToStream(mstream, NULL); + } +} +//------------------------------------------------------------------------------------- +void EntityComponentType::addCellDataToStream(MemoryStream* mstream, uint32 flags, PyObject* pyValue, + ENTITY_ID ownerID, PropertyDescription* parentPropertyDescription, COMPONENT_TYPE sendtoComponentType, bool checkValue) +{ + KBE_ASSERT(PyDict_Check(pyValue)); + + (*mstream) << sendtoComponentType << ownerID << pScriptDefModule_->getUType(); + + ScriptDefModule::PROPERTYDESCRIPTION_MAP& propertyDescrs = pScriptDefModule_->getCellPropertyDescriptions(); + + uint16 count = 0; + + ScriptDefModule::PROPERTYDESCRIPTION_MAP::const_iterator iter = propertyDescrs.begin(); + + for (; iter != propertyDescrs.end(); ++iter) + { + PropertyDescription* propertyDescription = iter->second; + + if (!propertyDescription->hasCell()) + continue; + + if (flags == 0 || (flags & propertyDescription->getFlags()) > 0) + { + ++count; + } + } + + (*mstream) << count; + + iter = propertyDescrs.begin(); + + for (; iter != propertyDescrs.end(); ++iter) + { + PropertyDescription* propertyDescription = iter->second; + + if (!propertyDescription->hasCell()) + continue; + + if (flags == 0 || (flags & propertyDescription->getFlags()) > 0) + { + PyObject* pyVal = PyDict_GetItemString(pyValue, propertyDescription->getName()); + + if (pyVal) + { + if (checkValue && !propertyDescription->getDataType()->isSameType(pyVal)) + { + ERROR_MSG(fmt::format("EntityComponent::addCellDataToStream: {} type(curr_py: {} != {}) error, use default values! name={}, utype={}, domain={}.\n", + propertyDescription->getName(), (pyVal ? pyVal->ob_type->tp_name : "unknown"), propertyDescription->getDataType()->getName(), + pScriptDefModule_ ? pScriptDefModule_->getName() : "", pScriptDefModule_ ? pScriptDefModule_->getUType() : 0, + COMPONENT_NAME_EX(sendtoComponentType))); + + if (parentPropertyDescription) + { + if (sendtoComponentType == CLIENT_TYPE && pScriptDefModule_->usePropertyDescrAlias()) + { + (*mstream) << parentPropertyDescription->aliasIDAsUint8(); + (*mstream) << propertyDescription->aliasIDAsUint8(); + } + else + { + (*mstream) << parentPropertyDescription->getUType(); + (*mstream) << propertyDescription->getUType(); + } + } + + propertyDescription->addToStream(mstream, NULL); + } + else + { + if (parentPropertyDescription) + { + if (sendtoComponentType == CLIENT_TYPE && pScriptDefModule_->usePropertyDescrAlias()) + { + (*mstream) << parentPropertyDescription->aliasIDAsUint8(); + (*mstream) << propertyDescription->aliasIDAsUint8(); + } + else + { + (*mstream) << parentPropertyDescription->getUType(); + (*mstream) << propertyDescription->getUType(); + } + } + + propertyDescription->getDataType()->addToStream(mstream, pyVal); + } + } + else + { + SCRIPT_ERROR_CHECK(); + + ERROR_MSG(fmt::format("EntityComponent::addCellDataToStream: not found property({}), use default values! name={}, utype={}, domain={}.\n", + propertyDescription->getName(), pScriptDefModule_ ? pScriptDefModule_->getName() : "", pScriptDefModule_ ? pScriptDefModule_->getUType() : 0, + COMPONENT_NAME_EX(sendtoComponentType))); + + if (parentPropertyDescription) + { + if (sendtoComponentType == CLIENT_TYPE && pScriptDefModule_->usePropertyDescrAlias()) + { + (*mstream) << parentPropertyDescription->aliasIDAsUint8(); + (*mstream) << propertyDescription->aliasIDAsUint8(); + } + else + { + (*mstream) << parentPropertyDescription->getUType(); + (*mstream) << propertyDescription->getUType(); + } + } + + propertyDescription->addToStream(mstream, NULL); + } + } + } +} + +//------------------------------------------------------------------------------------- +PyObject* EntityComponentType::createFromStream(MemoryStream* mstream) +{ + KBE_ASSERT(EntityDef::context().currEntityID > 0); + + PyObject* pyobj = pScriptDefModule_->createObject(); + + // ִEntityĹ캯 + PyObject* pyEntityComponent = new(pyobj) EntityComponent(EntityDef::context().currEntityID, pScriptDefModule_, EntityDef::context().currComponentType); + + EntityComponent* pEntityComponent = static_cast(pyEntityComponent); + return pEntityComponent->createFromStream(mstream); +} + +//------------------------------------------------------------------------------------- +PyObject* EntityComponentType::createFromPersistentStream(ScriptDefModule* pScriptDefModule, MemoryStream* mstream) +{ + KBE_ASSERT(EntityDef::context().currEntityID > 0); + + PyObject* pyobj = pScriptDefModule_->createObject(); + + // ִEntityĹ캯 + PyObject* pyEntityComponent = new(pyobj) EntityComponent(EntityDef::context().currEntityID, pScriptDefModule_, EntityDef::context().currComponentType); + + EntityComponent* pEntityComponent = static_cast(pyEntityComponent); + return pEntityComponent->createFromPersistentStream(pScriptDefModule, mstream); +} + +//------------------------------------------------------------------------------------- +PyObject* EntityComponentType::createCellData() +{ + PyObject* cellDataDict = PyDict_New(); + + ScriptDefModule::PROPERTYDESCRIPTION_MAP& propertyDescrs = pScriptDefModule_->getCellPropertyDescriptions(); + ScriptDefModule::PROPERTYDESCRIPTION_MAP::const_iterator iter = propertyDescrs.begin(); + for (; iter != propertyDescrs.end(); ++iter) + { + PropertyDescription* propertyDescription = iter->second; + DataType* dataType = propertyDescription->getDataType(); + + if (dataType) + { + PyObject* pyObj = propertyDescription->newDefaultVal(); + PyDict_SetItemString(cellDataDict, propertyDescription->getName(), pyObj); + Py_DECREF(pyObj); + } + else + { + ERROR_MSG(fmt::format("EntityComponentType::createCellData: {} PropertyDescription the dataType is NULL! component={}\n", + propertyDescription->getName(), pScriptDefModule_->getName())); + } + + SCRIPT_ERROR_CHECK(); + } + + return cellDataDict; +} + +//------------------------------------------------------------------------------------- +PyObject* EntityComponentType::createCellDataFromStream(MemoryStream* mstream) +{ + COMPONENT_TYPE componentType; + ENTITY_SCRIPT_UID ComponentDescrsType; + uint16 count; + ENTITY_ID ownerID; + + (*mstream) >> componentType >> ownerID >> ComponentDescrsType >> count; + + PyObject* cellDataDict = PyDict_New(); + + ScriptDefModule::PROPERTYDESCRIPTION_MAP& propertyDescrs = pScriptDefModule_->getCellPropertyDescriptions(); + ScriptDefModule::PROPERTYDESCRIPTION_MAP::const_iterator iter = propertyDescrs.begin(); + + for (; iter != propertyDescrs.end(); ++iter) + { + PropertyDescription* propertyDescription = iter->second; + + ENTITY_PROPERTY_UID uid; + (*mstream) >> uid >> uid; + + KBE_ASSERT(propertyDescription->getUType() == uid); + + PyObject* pyobj = propertyDescription->createFromStream(mstream); + + if (pyobj == NULL) + { + SCRIPT_ERROR_CHECK(); + + ERROR_MSG(fmt::format("EntityComponentType::createCellDataFromStream: property({}) error, use default values! name={}, utype={}.\n", + propertyDescription->getName(), pScriptDefModule_ ? pScriptDefModule_->getName() : "", pScriptDefModule_ ? pScriptDefModule_->getUType() : 0)); + + pyobj = propertyDescription->newDefaultVal(); + } + + PyDict_SetItemString(cellDataDict, propertyDescription->getName(), pyobj); + Py_DECREF(pyobj); + } + + return cellDataDict; +} + +//------------------------------------------------------------------------------------- +PyObject* EntityComponentType::createCellDataFromPersistentStream(MemoryStream* mstream) +{ + PyObject* cellDataDict = PyDict_New(); + + ScriptDefModule::PROPERTYDESCRIPTION_MAP& propertyDescrs = pScriptDefModule_->getPersistentPropertyDescriptions(); + ScriptDefModule::PROPERTYDESCRIPTION_MAP::const_iterator iter = propertyDescrs.begin(); + + for (; iter != propertyDescrs.end(); ++iter) + { + PropertyDescription* propertyDescription = iter->second; + + if (!propertyDescription->hasCell()) + continue; + + PyObject* pyobj = NULL; + + if(mstream) + pyobj = propertyDescription->createFromStream(mstream); + + if (pyobj == NULL) + { + SCRIPT_ERROR_CHECK(); + + ERROR_MSG(fmt::format("EntityComponentType::createCellDataFromPersistentStream: property({}) error, use default values! name={}, utype={}.\n", + propertyDescription->getName(), pScriptDefModule_ ? pScriptDefModule_->getName() : "", pScriptDefModule_ ? pScriptDefModule_->getUType() : 0)); + + pyobj = propertyDescription->newDefaultVal(); + } + + PyDict_SetItemString(cellDataDict, propertyDescription->getName(), pyobj); + Py_DECREF(pyobj); + } + + return cellDataDict; +} + +//------------------------------------------------------------------------------------- +} diff --git a/kbe/src/lib/entitydef/entitydef.cpp b/kbe/src/lib/entitydef/entitydef.cpp index 60306eae49..ac8324e3da 100644 --- a/kbe/src/lib/entitydef/entitydef.cpp +++ b/kbe/src/lib/entitydef/entitydef.cpp @@ -1,2396 +1,2479 @@ -// Copyright 2008-2018 Yolo Technologies, Inc. All Rights Reserved. https://www.comblockengine.com - - -#include "entitydef.h" -#include "scriptdef_module.h" -#include "datatypes.h" -#include "common.h" -#include "py_entitydef.h" -#include "entity_component.h" -#include "pyscript/py_memorystream.h" -#include "resmgr/resmgr.h" -#include "common/smartpointer.h" -#include "entitydef/volatileinfo.h" -#include "entitydef/entity_call.h" -#include "entitydef/entity_component_call.h" - -#ifndef CODE_INLINE -#include "entitydef.inl" -#endif - -namespace KBEngine{ -std::vector EntityDef::__scriptModules; -std::vector EntityDef::__oldScriptModules; - -std::map EntityDef::__scriptTypeMappingUType; -std::map EntityDef::__oldScriptTypeMappingUType; - -COMPONENT_TYPE EntityDef::__loadComponentType; -std::vector EntityDef::__scriptBaseTypes; -std::string EntityDef::__entitiesPath; - -KBE_MD5 EntityDef::__md5; -bool EntityDef::_isInit = false; -bool g_isReload = false; - -bool EntityDef::__entityAliasID = false; -bool EntityDef::__entitydefAliasID = false; - -EntityDef::Context EntityDef::__context; - -// ʱԶutypeõ -ENTITY_METHOD_UID g_methodUtypeAuto = 1; -std::vector g_methodCusUtypes; - -ENTITY_PROPERTY_UID g_propertyUtypeAuto = 1; -std::vector g_propertyUtypes; - -// µĽűģʱԶutype -ENTITY_SCRIPT_UID g_scriptUtype = 1; - -// ijentityĺַ -EntityDef::GetEntityFunc EntityDef::__getEntityFunc; - -static std::map > g_logComponentPropertys; - -//------------------------------------------------------------------------------------- -EntityDef::EntityDef() -{ -} - -//------------------------------------------------------------------------------------- -EntityDef::~EntityDef() -{ - EntityDef::finalise(); -} - -//------------------------------------------------------------------------------------- -bool EntityDef::finalise(bool isReload) -{ - PropertyDescription::resetDescriptionCount(); - MethodDescription::resetDescriptionCount(); - - EntityDef::__md5.clear(); - g_methodUtypeAuto = 1; - EntityDef::_isInit = false; - - g_propertyUtypeAuto = 1; - g_propertyUtypes.clear(); - - if(!isReload) - { - std::vector::iterator iter = EntityDef::__scriptModules.begin(); - for(; iter != EntityDef::__scriptModules.end(); ++iter) - { - (*iter)->finalise(); - } - - iter = EntityDef::__oldScriptModules.begin(); - for(; iter != EntityDef::__oldScriptModules.end(); ++iter) - { - (*iter)->finalise(); - } - - EntityDef::__oldScriptModules.clear(); - EntityDef::__oldScriptTypeMappingUType.clear(); - } - - g_scriptUtype = 1; - EntityDef::__scriptModules.clear(); - EntityDef::__scriptTypeMappingUType.clear(); - g_methodCusUtypes.clear(); - DataType::finalise(); - DataTypes::finalise(); - return true; -} - -//------------------------------------------------------------------------------------- -PyObject* EntityDef::tryGetEntity(COMPONENT_ID componentID, ENTITY_ID entityID) -{ - return __getEntityFunc(componentID, entityID); -} - -//------------------------------------------------------------------------------------- -bool EntityDef::isReload() -{ - return g_isReload; -} - -//------------------------------------------------------------------------------------- -void EntityDef::reload(bool fullReload) -{ - g_isReload = true; - - script::entitydef::reload(fullReload); - - if(fullReload) - { - EntityDef::__oldScriptModules.clear(); - EntityDef::__oldScriptTypeMappingUType.clear(); - - std::vector::iterator iter = EntityDef::__scriptModules.begin(); - for(; iter != EntityDef::__scriptModules.end(); ++iter) - { - __oldScriptModules.push_back((*iter)); - __oldScriptTypeMappingUType[(*iter)->getName()] = (*iter)->getUType(); - } - - bool ret = finalise(true); - KBE_ASSERT(ret && "EntityDef::reload: finalise error!"); - - ret = initialize(EntityDef::__scriptBaseTypes, EntityDef::__loadComponentType); - KBE_ASSERT(ret && "EntityDef::reload: initialize error!"); - } - else - { - loadAllEntityScriptModules(EntityDef::__entitiesPath, EntityDef::__scriptBaseTypes); - } - - EntityDef::_isInit = true; -} - -//------------------------------------------------------------------------------------- -bool EntityDef::initialize(std::vector& scriptBaseTypes, - COMPONENT_TYPE loadComponentType) -{ - __loadComponentType = loadComponentType; - __scriptBaseTypes = scriptBaseTypes; - - __entitiesPath = Resmgr::getSingleton().getPyUserScriptsPath(); - - g_entityFlagMapping["CELL"] = ED_FLAG_CELL_PUBLIC; - g_entityFlagMapping["CELL_AND_CLIENT"] = ED_FLAG_CELL_PUBLIC_AND_OWN; - g_entityFlagMapping["CELL_AND_CLIENTS"] = ED_FLAG_ALL_CLIENTS; - g_entityFlagMapping["CELL_AND_OTHER_CLIENTS"] = ED_FLAG_OTHER_CLIENTS; - g_entityFlagMapping["BASE_AND_CLIENT"] = ED_FLAG_BASE_AND_CLIENT; - g_entityFlagMapping["BASE"] = ED_FLAG_BASE; - - g_entityFlagMapping["CELL_PUBLIC"] = ED_FLAG_CELL_PUBLIC; - g_entityFlagMapping["CELL_PRIVATE"] = ED_FLAG_CELL_PRIVATE; - g_entityFlagMapping["ALL_CLIENTS"] = ED_FLAG_ALL_CLIENTS; - g_entityFlagMapping["CELL_PUBLIC_AND_OWN"] = ED_FLAG_CELL_PUBLIC_AND_OWN; - - g_entityFlagMapping["OTHER_CLIENTS"] = ED_FLAG_OTHER_CLIENTS; - g_entityFlagMapping["OWN_CLIENT"] = ED_FLAG_OWN_CLIENT; - - std::string entitiesFile = __entitiesPath + "entities.xml"; - std::string defFilePath = __entitiesPath + "entity_defs/"; - - // ʼ - // assets/scripts/entity_defs/types.xml - if(!DataTypes::initialize(defFilePath + "types.xml")) - return false; - - // entities.xmlļ - // ű壬ûļ - if (access(entitiesFile.c_str(), 0) == 0) - { - SmartPointer xml(new XML()); - if (!xml->openSection(entitiesFile.c_str())) - return false; - - // entities.xmlڵ, ûжһentityôֱӷtrue - TiXmlNode* node = xml->getRootNode(); - if (node == NULL) - return true; - - // ʼеentityڵ - XML_FOR_BEGIN(node) - { - std::string moduleName = xml.get()->getKey(node); - - ScriptDefModule* pScriptModule = registerNewScriptDefModule(moduleName); - - std::string deffile = defFilePath + moduleName + ".def"; - SmartPointer defxml(new XML()); - - if (!defxml->openSection(deffile.c_str())) - return false; - - TiXmlNode* defNode = defxml->getRootNode(); - if (defNode == NULL) - { - // rootڵûӽڵ - continue; - } - - // defļеĶ - if (!loadDefInfo(defFilePath, moduleName, defxml.get(), defNode, pScriptModule)) - { - ERROR_MSG(fmt::format("EntityDef::initialize: failed to load entity({}) module!\n", - moduleName.c_str())); - - return false; - } - - // entityļмdetailLevel - if (!loadDetailLevelInfo(defFilePath, moduleName, defxml.get(), defNode, pScriptModule)) - { - ERROR_MSG(fmt::format("EntityDef::initialize: failed to load entity({}) DetailLevelInfo!\n", - moduleName.c_str())); - - return false; - } - - pScriptModule->onLoaded(); - } - XML_FOR_END(node); - } - - if (!script::entitydef::initialize()) - return false; - - EntityDef::md5().final(); - - if(loadComponentType == DBMGR_TYPE) - return true; - - return loadAllEntityScriptModules(__entitiesPath, scriptBaseTypes) && - initializeWatcher(); -} - -//------------------------------------------------------------------------------------- -ScriptDefModule* EntityDef::registerNewScriptDefModule(const std::string& moduleName) -{ - ScriptDefModule* pScriptModule = findScriptModule(moduleName.c_str(), false); - - if (!pScriptModule) - { - __scriptTypeMappingUType[moduleName] = g_scriptUtype; - pScriptModule = new ScriptDefModule(moduleName, g_scriptUtype++); - EntityDef::__scriptModules.push_back(pScriptModule); - } - - return pScriptModule; -} - -//------------------------------------------------------------------------------------- -bool EntityDef::loadDefInfo(const std::string& defFilePath, - const std::string& moduleName, - XML* defxml, - TiXmlNode* defNode, - ScriptDefModule* pScriptModule) -{ - if(!loadAllDefDescriptions(moduleName, defxml, defNode, pScriptModule)) - { - ERROR_MSG(fmt::format("EntityDef::loadDefInfo: failed to loadAllDefDescription(), entity:{}\n", - moduleName.c_str())); - - return false; - } - - // еinterface ǵķԼ뵽ģ - if(!loadInterfaces(defFilePath, moduleName, defxml, defNode, pScriptModule)) - { - ERROR_MSG(fmt::format("EntityDef::loadDefInfo: failed to load entity:{} interface.\n", - moduleName.c_str())); - - return false; - } - - // еinterface ǵķԼ뵽ģ - if (!loadComponents(defFilePath, moduleName, defxml, defNode, pScriptModule)) - { - ERROR_MSG(fmt::format("EntityDef::loadDefInfo: failed to load entity:{} component.\n", - moduleName.c_str())); - - return false; - } - - // ظе - if(!loadParentClass(defFilePath, moduleName, defxml, defNode, pScriptModule)) - { - ERROR_MSG(fmt::format("EntityDef::loadDefInfo: failed to load entity:{} parentClass.\n", - moduleName.c_str())); - - return false; - } - - // ԼdetailLevel - if(!loadDetailLevelInfo(defFilePath, moduleName, defxml, defNode, pScriptModule)) - { - ERROR_MSG(fmt::format("EntityDef::loadDefInfo: failed to load entity:{} DetailLevelInfo.\n", - moduleName.c_str())); - - return false; - } - - // ԼVolatileInfo - if(!loadVolatileInfo(defFilePath, moduleName, defxml, defNode, pScriptModule)) - { - ERROR_MSG(fmt::format("EntityDef::loadDefInfo: failed to load entity:{} VolatileInfo.\n", - moduleName.c_str())); - - return false; - } - - pScriptModule->autoMatchCompOwn(); - return true; -} - -//------------------------------------------------------------------------------------- -bool EntityDef::loadDetailLevelInfo(const std::string& defFilePath, - const std::string& moduleName, - XML* defxml, - TiXmlNode* defNode, - ScriptDefModule* pScriptModule) -{ - TiXmlNode* detailLevelNode = defxml->enterNode(defNode, "DetailLevels"); - if(detailLevelNode == NULL) - return true; - - DetailLevel& dlInfo = pScriptModule->getDetailLevel(); - - TiXmlNode* node = defxml->enterNode(detailLevelNode, "NEAR"); - TiXmlNode* radiusNode = defxml->enterNode(node, "radius"); - TiXmlNode* hystNode = defxml->enterNode(node, "hyst"); - if(node == NULL || radiusNode == NULL || hystNode == NULL) - { - ERROR_MSG(fmt::format("EntityDef::loadDetailLevelInfo: failed to load entity:{} NEAR-DetailLevelInfo.\n", - moduleName.c_str())); - - return false; - } - - dlInfo.level[DETAIL_LEVEL_NEAR].radius = (float)defxml->getValFloat(radiusNode); - dlInfo.level[DETAIL_LEVEL_NEAR].hyst = (float)defxml->getValFloat(hystNode); - - node = defxml->enterNode(detailLevelNode, "MEDIUM"); - radiusNode = defxml->enterNode(node, "radius"); - hystNode = defxml->enterNode(node, "hyst"); - if(node == NULL || radiusNode == NULL || hystNode == NULL) - { - ERROR_MSG(fmt::format("EntityDef::loadDetailLevelInfo: failed to load entity:{} MEDIUM-DetailLevelInfo.\n", - moduleName.c_str())); - - return false; - } - - dlInfo.level[DETAIL_LEVEL_MEDIUM].radius = (float)defxml->getValFloat(radiusNode); - - dlInfo.level[DETAIL_LEVEL_MEDIUM].radius += dlInfo.level[DETAIL_LEVEL_NEAR].radius + - dlInfo.level[DETAIL_LEVEL_NEAR].hyst; - - dlInfo.level[DETAIL_LEVEL_MEDIUM].hyst = (float)defxml->getValFloat(hystNode); - - node = defxml->enterNode(detailLevelNode, "FAR"); - radiusNode = defxml->enterNode(node, "radius"); - hystNode = defxml->enterNode(node, "hyst"); - if(node == NULL || radiusNode == NULL || hystNode == NULL) - { - ERROR_MSG(fmt::format("EntityDef::loadDetailLevelInfo: failed to load entity:{} FAR-DetailLevelInfo.\n", - moduleName.c_str())); - - return false; - } - - dlInfo.level[DETAIL_LEVEL_FAR].radius = (float)defxml->getValFloat(radiusNode); - - dlInfo.level[DETAIL_LEVEL_FAR].radius += dlInfo.level[DETAIL_LEVEL_MEDIUM].radius + - dlInfo.level[DETAIL_LEVEL_MEDIUM].hyst; - - dlInfo.level[DETAIL_LEVEL_FAR].hyst = (float)defxml->getValFloat(hystNode); - - return true; - -} - -//------------------------------------------------------------------------------------- -bool EntityDef::loadVolatileInfo(const std::string& defFilePath, - const std::string& moduleName, - XML* defxml, - TiXmlNode* defNode, - ScriptDefModule* pScriptModule) -{ - TiXmlNode* pNode = defxml->enterNode(defNode, "Volatile"); - if(pNode == NULL) - return true; - - VolatileInfo* pVolatileInfo = pScriptModule->getPVolatileInfo(); - - TiXmlNode* node = defxml->enterNode(pNode, "position"); - if(node) - { - pVolatileInfo->position((float)defxml->getValFloat(node)); - } - else - { - if(defxml->hasNode(pNode, "position")) - pVolatileInfo->position(VolatileInfo::ALWAYS); - else - pVolatileInfo->position(-1.f); - } - - node = defxml->enterNode(pNode, "yaw"); - if(node) - { - pVolatileInfo->yaw((float)defxml->getValFloat(node)); - } - else - { - if(defxml->hasNode(pNode, "yaw")) - pVolatileInfo->yaw(VolatileInfo::ALWAYS); - else - pVolatileInfo->yaw(-1.f); - } - - node = defxml->enterNode(pNode, "pitch"); - if(node) - { - pVolatileInfo->pitch((float)defxml->getValFloat(node)); - } - else - { - if(defxml->hasNode(pNode, "pitch")) - pVolatileInfo->pitch(VolatileInfo::ALWAYS); - else - pVolatileInfo->pitch(-1.f); - } - - node = defxml->enterNode(pNode, "roll"); - if(node) - { - pVolatileInfo->roll((float)defxml->getValFloat(node)); - } - else - { - if(defxml->hasNode(pNode, "roll")) - pVolatileInfo->roll(VolatileInfo::ALWAYS); - else - pVolatileInfo->roll(-1.f); - } - - node = defxml->enterNode(pNode, "optimized"); - if (node) - { - pVolatileInfo->optimized(defxml->getBool(node)); - } - else - { - if (defxml->hasNode(pNode, "optimized")) - pVolatileInfo->optimized(true); - else - pVolatileInfo->optimized(true); - } - - return true; -} - -//------------------------------------------------------------------------------------- -bool EntityDef::loadInterfaces(const std::string& defFilePath, - const std::string& moduleName, - XML* defxml, - TiXmlNode* defNode, - ScriptDefModule* pScriptModule, bool ignoreComponents) -{ - TiXmlNode* implementsNode = defxml->enterNode(defNode, "Interfaces"); - if(implementsNode == NULL) - return true; - - XML_FOR_BEGIN(implementsNode) - { - if (defxml->getKey(implementsNode) != "interface" && defxml->getKey(implementsNode) != "Interface" && - defxml->getKey(implementsNode) != "type" && defxml->getKey(implementsNode) != "Type") - continue; - - TiXmlNode* interfaceNode = defxml->enterNode(implementsNode, "Interface"); - if (!interfaceNode) - { - interfaceNode = defxml->enterNode(implementsNode, "interface"); - if (!interfaceNode) - { - interfaceNode = defxml->enterNode(implementsNode, "Type"); - if (!interfaceNode) - { - interfaceNode = defxml->enterNode(implementsNode, "type"); - if (!interfaceNode) - { - continue; - } - } - } - } - - std::string interfaceName = defxml->getKey(interfaceNode); - std::string interfacefile = defFilePath + "interfaces/" + interfaceName + ".def"; - SmartPointer interfaceXml(new XML()); - if(!interfaceXml.get()->openSection(interfacefile.c_str())) - return false; - - TiXmlNode* interfaceRootNode = interfaceXml->getRootNode(); - if(interfaceRootNode == NULL) - { - // rootڵûӽڵ - return true; - } - - if(!loadAllDefDescriptions(moduleName, interfaceXml.get(), interfaceRootNode, pScriptModule)) - { - ERROR_MSG(fmt::format("EntityDef::initialize: interface[{}] error!\n", - interfaceName.c_str())); - - return false; - } - - // ԼdetailLevel - if(!loadDetailLevelInfo(defFilePath, moduleName, interfaceXml.get(), interfaceRootNode, pScriptModule)) - { - ERROR_MSG(fmt::format("EntityDef::loadInterfaces: failed to load entity:{} DetailLevelInfo.\n", - moduleName.c_str())); - - return false; - } - - // еinterface ǵķԼ뵽ģ - if (!ignoreComponents) - { - if (!loadComponents(defFilePath, moduleName, interfaceXml.get(), interfaceRootNode, pScriptModule)) - { - ERROR_MSG(fmt::format("EntityDef::loadInterfaces: failed to load entity:{} component.\n", - moduleName.c_str())); - - return false; - } - } - - // еinterface ǵķԼ뵽ģ - if(!loadInterfaces(defFilePath, moduleName, interfaceXml.get(), interfaceRootNode, pScriptModule)) - { - ERROR_MSG(fmt::format("EntityDef::loadInterfaces: failed to load entity:{} interface.\n", - moduleName.c_str())); - - return false; - } - } - XML_FOR_END(implementsNode); - - return true; -} - -//------------------------------------------------------------------------------------- -bool EntityDef::loadComponents(const std::string& defFilePath, - const std::string& moduleName, - XML* defxml, - TiXmlNode* defNode, - ScriptDefModule* pScriptModule) -{ - TiXmlNode* implementsNode = defxml->enterNode(defNode, "Components"); - if (implementsNode == NULL) - return true; - - XML_FOR_BEGIN(implementsNode) - { - std::string componentName = defxml->getKey(implementsNode); - - TiXmlNode* componentNode = defxml->enterNode(implementsNode, componentName.c_str()); - if (!componentNode) - continue; - - if (!validDefPropertyName(componentName)) - { - ERROR_MSG(fmt::format("EntityDef::loadComponents: '{}' is limited, in module({})!\n", - componentName, moduleName)); - - return false; - } - - std::string componentTypeName = ""; - TiXmlNode* componentTypeNameNode = defxml->enterNode(componentNode, "Type"); - if (componentTypeNameNode) - componentTypeName = defxml->getKey(componentTypeNameNode); - - if (componentTypeName == "") - { - ERROR_MSG(fmt::format("EntityDef::loadComponents: component name is NULL.\n", - componentName.c_str())); - - return false; - } - - std::string componentfile = defFilePath + "components/" + componentTypeName + ".def"; - SmartPointer componentXml(new XML()); - if (!componentXml.get()->openSection(componentfile.c_str())) - return false; - - // һʵ - ENTITY_PROPERTY_UID futype = 0; - uint32 flags = ED_FLAG_BASE | ED_FLAG_CELL_PUBLIC | ENTITY_CLIENT_DATA_FLAGS; - bool isPersistent = true; - bool isIdentifier = false; // Ƿһ - uint32 databaseLength = 0; // ݿеij - std::string indexType = ""; - DETAIL_TYPE detailLevel = DETAIL_LEVEL_FAR; - std::string detailLevelStr = ""; - std::string strisPersistent; - std::string defaultStr = ""; - - TiXmlNode* utypeValNode = defxml->enterNode(componentNode, "Utype"); - - if (!calcDefPropertyUType(moduleName, componentName, (utypeValNode ? defxml->getValInt(utypeValNode) : -1), pScriptModule, futype)) - return false; - - TiXmlNode* persistentNode = defxml->enterNode(componentNode, "Persistent"); - if (persistentNode) - { - strisPersistent = defxml->getValStr(persistentNode); - - std::transform(strisPersistent.begin(), strisPersistent.end(), - strisPersistent.begin(), tolower); - - if (strisPersistent == "false") - isPersistent = false; - } - - // Ƿģ飬˵Ѿعٴμ - ScriptDefModule* pCompScriptDefModule = findScriptModule(componentTypeName.c_str(), false); - - if (!pCompScriptDefModule) - { - pCompScriptDefModule = registerNewScriptDefModule(componentTypeName); - pCompScriptDefModule->isPersistent(false); - pCompScriptDefModule->isComponentModule(true); - } - else - { - flags = ED_FLAG_UNKOWN; - - if (pCompScriptDefModule->hasBase()) - flags |= ED_FLAG_BASE; - - if (pCompScriptDefModule->hasCell()) - flags |= ED_FLAG_CELL_PUBLIC; - - if (pCompScriptDefModule->hasClient()) - { - if (pCompScriptDefModule->hasBase()) - flags |= ED_FLAG_BASE_AND_CLIENT; - else - flags |= (ED_FLAG_ALL_CLIENTS | ED_FLAG_CELL_PUBLIC_AND_OWN | ED_FLAG_OTHER_CLIENTS | ED_FLAG_OWN_CLIENT); - } - - addComponentProperty(futype, componentTypeName, componentName, flags, isPersistent, isIdentifier, - indexType, databaseLength, defaultStr, detailLevel, pScriptModule, pCompScriptDefModule); - - pScriptModule->addComponentDescription(componentName.c_str(), pCompScriptDefModule); - continue; - } - - TiXmlNode* componentRootNode = componentXml->getRootNode(); - if (componentRootNode == NULL) - { - // rootڵûӽڵ - return true; - } - - if (!loadAllDefDescriptions(componentTypeName, componentXml.get(), componentRootNode, pCompScriptDefModule)) - { - ERROR_MSG(fmt::format("EntityDef::initialize: component[{}] error!\n", - componentTypeName.c_str())); - - return false; - } - - // еinterface ǵķԼ뵽ģ - if (!loadInterfaces(defFilePath, componentTypeName, componentXml.get(), componentRootNode, pCompScriptDefModule, true)) - { - ERROR_MSG(fmt::format("EntityDef::loadComponents: failed to load component:{} interface.\n", - componentTypeName.c_str())); - - return false; - } - - // ظе - if (!loadParentClass(defFilePath + "components/", componentTypeName, componentXml.get(), componentRootNode, pCompScriptDefModule)) - { - ERROR_MSG(fmt::format("EntityDef::loadComponents: failed to load component:{} parentClass.\n", - componentTypeName.c_str())); - - return false; - } - - // ԼdetailLevel - if (!loadDetailLevelInfo(defFilePath, componentTypeName, componentXml.get(), componentRootNode, pCompScriptDefModule)) - { - ERROR_MSG(fmt::format("EntityDef::loadComponents: failed to load component:{} DetailLevelInfo.\n", - componentTypeName.c_str())); - - return false; - } - - pCompScriptDefModule->autoMatchCompOwn(); - - flags = ED_FLAG_UNKOWN; - - if (pCompScriptDefModule->hasBase()) - flags |= ED_FLAG_BASE; - - if (pCompScriptDefModule->hasCell()) - flags |= ED_FLAG_CELL_PUBLIC; - - if (pCompScriptDefModule->hasClient()) - { - if (pCompScriptDefModule->hasBase()) - flags |= ED_FLAG_BASE_AND_CLIENT; - - if (pCompScriptDefModule->hasCell()) - flags |= (ED_FLAG_ALL_CLIENTS | ED_FLAG_CELL_PUBLIC_AND_OWN | ED_FLAG_OTHER_CLIENTS | ED_FLAG_OWN_CLIENT); - } - - addComponentProperty(futype, componentTypeName, componentName, flags, isPersistent, isIdentifier, - indexType, databaseLength, defaultStr, detailLevel, pScriptModule, pCompScriptDefModule); - - pScriptModule->addComponentDescription(componentName.c_str(), pCompScriptDefModule); - } - XML_FOR_END(implementsNode); - - return true; -} - -//------------------------------------------------------------------------------------- -PropertyDescription* EntityDef::addComponentProperty(ENTITY_PROPERTY_UID utype, - const std::string& componentTypeName, - const std::string& componentName, - uint32 flags, - bool isPersistent, - bool isIdentifier, - std::string indexType, - uint32 databaseLength, - const std::string& defaultStr, - DETAIL_TYPE detailLevel, - ScriptDefModule* pScriptModule, - ScriptDefModule* pCompScriptDefModule) -{ - DataType* pEntityComponentType = DataTypes::getDataType(componentTypeName, false); - - if (!pEntityComponentType) - pEntityComponentType = new EntityComponentType(pCompScriptDefModule); - - PropertyDescription* propertyDescription = PropertyDescription::createDescription(utype, "EntityComponent", - componentName, flags, isPersistent, - pEntityComponentType, isIdentifier, indexType, - databaseLength, defaultStr, - detailLevel); - - bool ret = true; - - int32 hasBaseFlags = 0; - int32 hasCellFlags = 0; - int32 hasClientFlags = 0; - - hasBaseFlags = flags & ENTITY_BASE_DATA_FLAGS; - if (hasBaseFlags > 0) - pScriptModule->setBase(true); - - hasCellFlags = flags & ENTITY_CELL_DATA_FLAGS; - if (hasCellFlags > 0) - pScriptModule->setCell(true); - - hasClientFlags = flags & ENTITY_CLIENT_DATA_FLAGS; - if (hasClientFlags > 0) - pScriptModule->setClient(true); - - // ӵģ - if (hasCellFlags > 0) - ret = pScriptModule->addPropertyDescription(componentName.c_str(), - propertyDescription, CELLAPP_TYPE); - - if (hasBaseFlags > 0) - ret = pScriptModule->addPropertyDescription(componentName.c_str(), - propertyDescription, BASEAPP_TYPE); - - if (hasClientFlags > 0) - ret = pScriptModule->addPropertyDescription(componentName.c_str(), - propertyDescription, CLIENT_TYPE); - - if (!ret) - { - ERROR_MSG(fmt::format("EntityDef::addComponentProperty({}): {}.\n", - pScriptModule->getName(), componentName)); - - SAFE_RELEASE(propertyDescription); - return NULL; - } - - g_logComponentPropertys[pScriptModule->getName()].push_back(propertyDescription); - return propertyDescription; -} - -//------------------------------------------------------------------------------------- -bool EntityDef::loadParentClass(const std::string& defFilePath, - const std::string& moduleName, - XML* defxml, - TiXmlNode* defNode, - ScriptDefModule* pScriptModule) -{ - TiXmlNode* parentClassNode = defxml->enterNode(defNode, "Parent"); - if(parentClassNode == NULL) - return true; - - std::string parentClassName = defxml->getKey(parentClassNode); - std::string parentClassfile = defFilePath + parentClassName + ".def"; - - SmartPointer parentClassXml(new XML()); - if(!parentClassXml->openSection(parentClassfile.c_str())) - return false; - - TiXmlNode* parentClassdefNode = parentClassXml->getRootNode(); - if(parentClassdefNode == NULL) - { - // rootڵûӽڵ - return true; - } - - // defļеĶ - if(!loadDefInfo(defFilePath, parentClassName, parentClassXml.get(), parentClassdefNode, pScriptModule)) - { - ERROR_MSG(fmt::format("EntityDef::loadParentClass: failed to load entity:{} parentClass.\n", - moduleName.c_str())); - - return false; - } - - return true; -} - -//------------------------------------------------------------------------------------- -bool EntityDef::loadAllDefDescriptions(const std::string& moduleName, - XML* defxml, - TiXmlNode* defNode, - ScriptDefModule* pScriptModule) -{ - // - if(!loadDefPropertys(moduleName, defxml, defxml->enterNode(defNode, "Properties"), pScriptModule)) - return false; - - // cell - if(!loadDefCellMethods(moduleName, defxml, defxml->enterNode(defNode, "CellMethods"), pScriptModule)) - { - ERROR_MSG(fmt::format("EntityDef::loadAllDefDescription:loadDefCellMethods[{}] is failed!\n", - moduleName.c_str())); - - return false; - } - - // base - if(!loadDefBaseMethods(moduleName, defxml, defxml->enterNode(defNode, "BaseMethods"), pScriptModule)) - { - ERROR_MSG(fmt::format("EntityDef::loadAllDefDescription:loadDefBaseMethods[{}] is failed!\n", - moduleName.c_str())); - - return false; - } - - // client - if(!loadDefClientMethods(moduleName, defxml, defxml->enterNode(defNode, "ClientMethods"), pScriptModule)) - { - ERROR_MSG(fmt::format("EntityDef::loadAllDefDescription:loadDefClientMethods[{}] is failed!\n", - moduleName.c_str())); - - return false; - } - - return true; -} - -//------------------------------------------------------------------------------------- -bool EntityDef::validDefPropertyName(const std::string& name) -{ - int i = 0; - - while(true) - { - std::string limited = ENTITY_LIMITED_PROPERTYS[i]; - - if(limited == "") - break; - - if(name == limited) - return false; - - ++i; - }; - - PyObject* pyKBEModule = - PyImport_ImportModule(const_cast("KBEngine")); - - PyObject* pyEntityModule = - PyObject_GetAttrString(pyKBEModule, const_cast("Entity")); - - Py_DECREF(pyKBEModule); - - if (pyEntityModule != NULL) - { - PyObject* pyEntityAttr = - PyObject_GetAttrString(pyEntityModule, const_cast(name.c_str())); - - if (pyEntityAttr != NULL) - { - Py_DECREF(pyEntityAttr); - Py_DECREF(pyEntityModule); - return false; - } - else - { - PyErr_Clear(); - } - } - else - { - PyErr_Clear(); - } - - Py_XDECREF(pyEntityModule); - return true; -} - -//------------------------------------------------------------------------------------- -bool EntityDef::calcDefPropertyUType(const std::string& moduleName, - const std::string& name, int iUtype, ScriptDefModule* pScriptModule, ENTITY_PROPERTY_UID& outUtype) -{ - ENTITY_PROPERTY_UID futype = 0; - outUtype = futype; - - if (iUtype > 0) - { - futype = iUtype; - - if (iUtype != int(futype)) - { - ERROR_MSG(fmt::format("EntityDef::calcDefPropertyUType: 'Utype' has overflowed({} > 65535), is {}.{}!\n", - iUtype, moduleName, name.c_str())); - - return false; - } - - // ǷظUtype - std::vector::iterator iter = - std::find(g_propertyUtypes.begin(), g_propertyUtypes.end(), futype); - - if (iter != g_propertyUtypes.end()) - { - bool foundConflict = false; - - PropertyDescription* pConflictPropertyDescription = pScriptModule->findPropertyDescription(futype, BASEAPP_TYPE); - if (pConflictPropertyDescription) - { - ERROR_MSG(fmt::format("EntityDef::calcDefPropertyUType: {}.{}, 'Utype' {} Conflict({}.{} 'Utype' {})!\n", - moduleName, name.c_str(), iUtype, moduleName, pConflictPropertyDescription->getName(), iUtype)); - - foundConflict = true; - } - - pConflictPropertyDescription = pScriptModule->findPropertyDescription(futype, CELLAPP_TYPE); - if (pConflictPropertyDescription) - { - ERROR_MSG(fmt::format("EntityDef::calcDefPropertyUType: {}.{}, 'Utype' {} Conflict({}.{} 'Utype' {})!\n", - moduleName, name.c_str(), iUtype, moduleName, pConflictPropertyDescription->getName(), iUtype)); - - foundConflict = true; - } - - pConflictPropertyDescription = pScriptModule->findPropertyDescription(futype, CLIENT_TYPE); - if (pConflictPropertyDescription) - { - ERROR_MSG(fmt::format("EntityDef::calcDefPropertyUType: {}.{}, 'Utype' {} Conflict({}.{} 'Utype' {})!\n", - moduleName, name.c_str(), iUtype, moduleName, pConflictPropertyDescription->getName(), iUtype)); - - foundConflict = true; - } - - if (foundConflict) - return false; - } - - g_propertyUtypes.push_back(futype); - } - else - { - while (true) - { - futype = g_propertyUtypeAuto++; - std::vector::iterator iter = - std::find(g_propertyUtypes.begin(), g_propertyUtypes.end(), futype); - - if (iter == g_propertyUtypes.end()) - break; - } - - g_propertyUtypes.push_back(futype); - } - - outUtype = futype; - return true; -} - -//------------------------------------------------------------------------------------- -bool EntityDef::loadDefPropertys(const std::string& moduleName, - XML* xml, - TiXmlNode* defPropertyNode, - ScriptDefModule* pScriptModule) -{ - if(defPropertyNode) - { - XML_FOR_BEGIN(defPropertyNode) - { - ENTITY_PROPERTY_UID futype = 0; - uint32 flags = 0; - int32 hasBaseFlags = 0; - int32 hasCellFlags = 0; - int32 hasClientFlags = 0; - DataType* dataType = NULL; - bool isPersistent = false; - bool isIdentifier = false; // Ƿһ - uint32 databaseLength = 0; // ݿеij - std::string indexType; - DETAIL_TYPE detailLevel = DETAIL_LEVEL_FAR; - std::string detailLevelStr = ""; - std::string strType; - std::string strisPersistent; - std::string strFlags; - std::string strIdentifierNode; - std::string defaultStr; - std::string name = ""; - - name = xml->getKey(defPropertyNode); - if(!validDefPropertyName(name)) - { - ERROR_MSG(fmt::format("EntityDef::loadDefPropertys: '{}' is limited, in module({})!\n", - name, moduleName)); - - return false; - } - - TiXmlNode* flagsNode = xml->enterNode(defPropertyNode->FirstChild(), "Flags"); - if(flagsNode) - { - strFlags = xml->getValStr(flagsNode); - std::transform(strFlags.begin(), strFlags.end(), strFlags.begin(), toupper); - - ENTITYFLAGMAP::iterator iter = g_entityFlagMapping.find(strFlags.c_str()); - if(iter == g_entityFlagMapping.end()) - { - ERROR_MSG(fmt::format("EntityDef::loadDefPropertys: not fount flags[{}], is {}.{}!\n", - strFlags, moduleName, name)); - - return false; - } - - flags = iter->second; - hasBaseFlags = flags & ENTITY_BASE_DATA_FLAGS; - if(hasBaseFlags > 0) - pScriptModule->setBase(true); - - hasCellFlags = flags & ENTITY_CELL_DATA_FLAGS; - if(hasCellFlags > 0) - pScriptModule->setCell(true); - - hasClientFlags = flags & ENTITY_CLIENT_DATA_FLAGS; - if(hasClientFlags > 0) - pScriptModule->setClient(true); - - if(hasBaseFlags <= 0 && hasCellFlags <= 0) - { - ERROR_MSG(fmt::format("EntityDef::loadDefPropertys: not fount flags[{}], is {}.{}!\n", - strFlags.c_str(), moduleName, name.c_str())); - - return false; - } - } - else - { - ERROR_MSG(fmt::format("EntityDef::loadDefPropertys: not fount flagsNode, is {}.{}!\n", - moduleName, name.c_str())); - - return false; - } - - TiXmlNode* persistentNode = xml->enterNode(defPropertyNode->FirstChild(), "Persistent"); - if(persistentNode) - { - strisPersistent = xml->getValStr(persistentNode); - - std::transform(strisPersistent.begin(), strisPersistent.end(), - strisPersistent.begin(), tolower); - - if(strisPersistent == "true") - isPersistent = true; - } - - TiXmlNode* typeNode = xml->enterNode(defPropertyNode->FirstChild(), "Type"); - if(typeNode) - { - strType = xml->getValStr(typeNode); - - if(strType == "ARRAY") - { - FixedArrayType* dataType1 = new FixedArrayType(); - if(dataType1->initialize(xml, typeNode, moduleName + "_" + name)) - dataType = dataType1; - else - return false; - } - else - { - dataType = DataTypes::getDataType(strType); - } - - if(dataType == NULL) - { - return false; - } - } - else - { - ERROR_MSG(fmt::format("EntityDef::loadDefPropertys: not fount TypeNode, is {}.{}!\n", - moduleName, name.c_str())); - - return false; - } - - TiXmlNode* indexTypeNode = xml->enterNode(defPropertyNode->FirstChild(), "Index"); - if(indexTypeNode) - { - indexType = xml->getValStr(indexTypeNode); - - std::transform(indexType.begin(), indexType.end(), - indexType.begin(), toupper); - } - - TiXmlNode* identifierNode = xml->enterNode(defPropertyNode->FirstChild(), "Identifier"); - if(identifierNode) - { - strIdentifierNode = xml->getValStr(identifierNode); - std::transform(strIdentifierNode.begin(), strIdentifierNode.end(), - strIdentifierNode.begin(), tolower); - - if(strIdentifierNode == "true") - isIdentifier = true; - } - - TiXmlNode* databaseLengthNode = xml->enterNode(defPropertyNode->FirstChild(), "DatabaseLength"); - if(databaseLengthNode) - { - databaseLength = xml->getValInt(databaseLengthNode); - } - - TiXmlNode* defaultValNode = - xml->enterNode(defPropertyNode->FirstChild(), "Default"); - - if(defaultValNode) - { - defaultStr = xml->getValStr(defaultValNode); - } - - TiXmlNode* detailLevelNode = - xml->enterNode(defPropertyNode->FirstChild(), "DetailLevel"); - - if(detailLevelNode) - { - detailLevelStr = xml->getValStr(detailLevelNode); - if(detailLevelStr == "FAR") - detailLevel = DETAIL_LEVEL_FAR; - else if(detailLevelStr == "MEDIUM") - detailLevel = DETAIL_LEVEL_MEDIUM; - else if(detailLevelStr == "NEAR") - detailLevel = DETAIL_LEVEL_NEAR; - else - detailLevel = DETAIL_LEVEL_FAR; - } - - TiXmlNode* utypeValNode = - xml->enterNode(defPropertyNode->FirstChild(), "Utype"); - - if (!calcDefPropertyUType(moduleName, name, (utypeValNode ? xml->getValInt(utypeValNode) : -1), pScriptModule, futype)) - return false; - - // һʵ - PropertyDescription* propertyDescription = PropertyDescription::createDescription(futype, strType, - name, flags, isPersistent, - dataType, isIdentifier, indexType, - databaseLength, defaultStr, - detailLevel); - - bool ret = true; - - // ӵģ - if(hasCellFlags > 0) - ret = pScriptModule->addPropertyDescription(name.c_str(), - propertyDescription, CELLAPP_TYPE); - - if(hasBaseFlags > 0) - ret = pScriptModule->addPropertyDescription(name.c_str(), - propertyDescription, BASEAPP_TYPE); - - if(hasClientFlags > 0) - ret = pScriptModule->addPropertyDescription(name.c_str(), - propertyDescription, CLIENT_TYPE); - - if(!ret) - { - ERROR_MSG(fmt::format("EntityDef::addPropertyDescription({}): {}.\n", - moduleName.c_str(), xml->getTxdoc()->Value())); - - return false; - } - } - XML_FOR_END(defPropertyNode); - } - - return true; -} - -//------------------------------------------------------------------------------------- -bool EntityDef::loadDefCellMethods(const std::string& moduleName, - XML* xml, - TiXmlNode* defMethodNode, - ScriptDefModule* pScriptModule) -{ - if(defMethodNode) - { - XML_FOR_BEGIN(defMethodNode) - { - std::string name = xml->getKey(defMethodNode); - MethodDescription* methodDescription = new MethodDescription(0, CELLAPP_TYPE, name); - TiXmlNode* argNode = defMethodNode->FirstChild(); - - // ûв - if(argNode) - { - XML_FOR_BEGIN(argNode) - { - std::string argType = xml->getKey(argNode); - - if(argType == "Exposed") - { - methodDescription->setExposed(); - } - else if(argType == "Arg") - { - DataType* dataType = NULL; - TiXmlNode* typeNode = argNode->FirstChild(); - std::string strType = xml->getValStr(typeNode); - - if(strType == "ARRAY") - { - FixedArrayType* dataType1 = new FixedArrayType(); - if(dataType1->initialize(xml, typeNode, moduleName + "_" + name)) - dataType = dataType1; - } - else - { - dataType = DataTypes::getDataType(strType); - } - - if(dataType == NULL) - { - ERROR_MSG(fmt::format("EntityDef::loadDefCellMethods: dataType[{}] not found, in {}!\n", - strType.c_str(), name.c_str())); - - return false; - } - - methodDescription->pushArgType(dataType); - } - else if(argType == "Utype") - { - TiXmlNode* typeNode = argNode->FirstChild(); - - int iUtype = xml->getValInt(typeNode); - ENTITY_METHOD_UID muid = iUtype; - - if (iUtype != int(muid)) - { - ERROR_MSG(fmt::format("EntityDef::loadDefCellMethods: 'Utype' has overflowed({} > 65535), is {}.{}!\n", - iUtype, moduleName, name.c_str())); - - return false; - } - - methodDescription->setUType(muid); - g_methodCusUtypes.push_back(muid); - } - } - XML_FOR_END(argNode); - } - - // ûùutype, - if(methodDescription->getUType() <= 0) - { - ENTITY_METHOD_UID muid = 0; - while(true) - { - muid = g_methodUtypeAuto++; - std::vector::iterator iterutype = - std::find(g_methodCusUtypes.begin(), g_methodCusUtypes.end(), muid); - - if(iterutype == g_methodCusUtypes.end()) - { - break; - } - } - - methodDescription->setUType(muid); - g_methodCusUtypes.push_back(muid); - } - else - { - // ǷظUtype - ENTITY_METHOD_UID muid = methodDescription->getUType(); - std::vector::iterator iter = - std::find(g_methodCusUtypes.begin(), g_methodCusUtypes.end(), muid); - - if (iter != g_methodCusUtypes.end()) - { - bool foundConflict = false; - - MethodDescription* pConflictMethodDescription = pScriptModule->findBaseMethodDescription(muid); - if (pConflictMethodDescription) - { - ERROR_MSG(fmt::format("EntityDef::loadDefCellMethods: {}.{}, 'Utype' {} Conflict({}.{} 'Utype' {})!\n", - moduleName, name.c_str(), muid, moduleName, pConflictMethodDescription->getName(), muid)); - - foundConflict = true; - } - - pConflictMethodDescription = pScriptModule->findCellMethodDescription(muid); - if (pConflictMethodDescription) - { - ERROR_MSG(fmt::format("EntityDef::loadDefCellMethods: {}.{}, 'Utype' {} Conflict({}.{} 'Utype' {})!\n", - moduleName, name.c_str(), muid, moduleName, pConflictMethodDescription->getName(), muid)); - - foundConflict = true; - } - - pConflictMethodDescription = pScriptModule->findClientMethodDescription(muid); - if (pConflictMethodDescription) - { - ERROR_MSG(fmt::format("EntityDef::loadDefCellMethods: {}.{}, 'Utype' {} Conflict({}.{} 'Utype' {})!\n", - moduleName, name.c_str(), muid, moduleName, pConflictMethodDescription->getName(), muid)); - - foundConflict = true; - } - - if (foundConflict) - return false; - } - } - - if(!pScriptModule->addCellMethodDescription(name.c_str(), methodDescription)) - return false; - } - XML_FOR_END(defMethodNode); - } - - return true; -} - -//------------------------------------------------------------------------------------- -bool EntityDef::loadDefBaseMethods(const std::string& moduleName, XML* xml, - TiXmlNode* defMethodNode, ScriptDefModule* pScriptModule) -{ - if(defMethodNode) - { - XML_FOR_BEGIN(defMethodNode) - { - std::string name = xml->getKey(defMethodNode); - MethodDescription* methodDescription = new MethodDescription(0, BASEAPP_TYPE, name); - TiXmlNode* argNode = defMethodNode->FirstChild(); - - // ûв - if(argNode) - { - XML_FOR_BEGIN(argNode) - { - std::string argType = xml->getKey(argNode); - - if(argType == "Exposed") - { - methodDescription->setExposed(); - } - else if(argType == "Arg") - { - DataType* dataType = NULL; - TiXmlNode* typeNode = argNode->FirstChild(); - std::string strType = xml->getValStr(typeNode); - - if(strType == "ARRAY") - { - FixedArrayType* dataType1 = new FixedArrayType(); - if(dataType1->initialize(xml, typeNode, moduleName + "_" + name)) - dataType = dataType1; - } - else - { - dataType = DataTypes::getDataType(strType); - } - - if(dataType == NULL) - { - ERROR_MSG(fmt::format("EntityDef::loadDefBaseMethods: dataType[{}] not found, in {}!\n", - strType.c_str(), name.c_str())); - - return false; - } - - methodDescription->pushArgType(dataType); - } - else if(argType == "Utype") - { - TiXmlNode* typeNode = argNode->FirstChild(); - - int iUtype = xml->getValInt(typeNode); - ENTITY_METHOD_UID muid = iUtype; - - if (iUtype != int(muid)) - { - ERROR_MSG(fmt::format("EntityDef::loadDefBaseMethods: 'Utype' has overflowed({} > 65535), is {}.{}!\n", - iUtype, moduleName, name.c_str())); - - return false; - } - - methodDescription->setUType(muid); - g_methodCusUtypes.push_back(muid); - } - } - XML_FOR_END(argNode); - } - - // ûùutype, - if(methodDescription->getUType() <= 0) - { - ENTITY_METHOD_UID muid = 0; - while(true) - { - muid = g_methodUtypeAuto++; - std::vector::iterator iterutype = - std::find(g_methodCusUtypes.begin(), g_methodCusUtypes.end(), muid); - - if(iterutype == g_methodCusUtypes.end()) - { - break; - } - } - - methodDescription->setUType(muid); - g_methodCusUtypes.push_back(muid); - } - else - { - // ǷظUtype - ENTITY_METHOD_UID muid = methodDescription->getUType(); - std::vector::iterator iter = - std::find(g_methodCusUtypes.begin(), g_methodCusUtypes.end(), muid); - - if (iter != g_methodCusUtypes.end()) - { - bool foundConflict = false; - - MethodDescription* pConflictMethodDescription = pScriptModule->findBaseMethodDescription(muid); - if (pConflictMethodDescription) - { - ERROR_MSG(fmt::format("EntityDef::loadDefBaseMethods: {}.{}, 'Utype' {} Conflict({}.{} 'Utype' {})!\n", - moduleName, name.c_str(), muid, moduleName, pConflictMethodDescription->getName(), muid)); - - foundConflict = true; - } - - pConflictMethodDescription = pScriptModule->findCellMethodDescription(muid); - if (pConflictMethodDescription) - { - ERROR_MSG(fmt::format("EntityDef::loadDefBaseMethods: {}.{}, 'Utype' {} Conflict({}.{} 'Utype' {})!\n", - moduleName, name.c_str(), muid, moduleName, pConflictMethodDescription->getName(), muid)); - - foundConflict = true; - } - - pConflictMethodDescription = pScriptModule->findClientMethodDescription(muid); - if (pConflictMethodDescription) - { - ERROR_MSG(fmt::format("EntityDef::loadDefBaseMethods: {}.{}, 'Utype' {} Conflict({}.{} 'Utype' {})!\n", - moduleName, name.c_str(), muid, moduleName, pConflictMethodDescription->getName(), muid)); - - foundConflict = true; - } - - if (foundConflict) - return false; - } - } - - if(!pScriptModule->addBaseMethodDescription(name.c_str(), methodDescription)) - return false; - } - XML_FOR_END(defMethodNode); - } - - return true; -} - -//------------------------------------------------------------------------------------- -bool EntityDef::loadDefClientMethods(const std::string& moduleName, XML* xml, - TiXmlNode* defMethodNode, ScriptDefModule* pScriptModule) -{ - if(defMethodNode) - { - XML_FOR_BEGIN(defMethodNode) - { - std::string name = xml->getKey(defMethodNode); - MethodDescription* methodDescription = new MethodDescription(0, CLIENT_TYPE, name); - TiXmlNode* argNode = defMethodNode->FirstChild(); - - // ûв - if(argNode) - { - XML_FOR_BEGIN(argNode) - { - std::string argType = xml->getKey(argNode); - - if(argType == "Arg") - { - DataType* dataType = NULL; - TiXmlNode* typeNode = argNode->FirstChild(); - std::string strType = xml->getValStr(typeNode); - - if(strType == "ARRAY") - { - FixedArrayType* dataType1 = new FixedArrayType(); - if(dataType1->initialize(xml, typeNode, moduleName + "_" + name)) - dataType = dataType1; - } - else - { - dataType = DataTypes::getDataType(strType); - } - - if(dataType == NULL) - { - ERROR_MSG(fmt::format("EntityDef::loadDefClientMethods: dataType[{}] not found, in {}!\n", - strType.c_str(), name.c_str())); - - return false; - } - - methodDescription->pushArgType(dataType); - } - else if(argType == "Utype") - { - TiXmlNode* typeNode = argNode->FirstChild(); - - int iUtype = xml->getValInt(typeNode); - ENTITY_METHOD_UID muid = iUtype; - - if (iUtype != int(muid)) - { - ERROR_MSG(fmt::format("EntityDef::loadDefClientMethods: 'Utype' has overflowed({} > 65535), is {}.{}!\n", - iUtype, moduleName, name.c_str())); - - return false; - } - - methodDescription->setUType(muid); - g_methodCusUtypes.push_back(muid); - } - } - XML_FOR_END(argNode); - } - - // ûùutype, - if(methodDescription->getUType() <= 0) - { - ENTITY_METHOD_UID muid = 0; - while(true) - { - muid = g_methodUtypeAuto++; - std::vector::iterator iterutype = - std::find(g_methodCusUtypes.begin(), g_methodCusUtypes.end(), muid); - - if(iterutype == g_methodCusUtypes.end()) - { - break; - } - } - - methodDescription->setUType(muid); - g_methodCusUtypes.push_back(muid); - } - else - { - // ǷظUtype - ENTITY_METHOD_UID muid = methodDescription->getUType(); - std::vector::iterator iter = - std::find(g_methodCusUtypes.begin(), g_methodCusUtypes.end(), muid); - - if (iter != g_methodCusUtypes.end()) - { - bool foundConflict = false; - - MethodDescription* pConflictMethodDescription = pScriptModule->findBaseMethodDescription(muid); - if (pConflictMethodDescription) - { - ERROR_MSG(fmt::format("EntityDef::loadDefClientMethods: {}.{}, 'Utype' {} Conflict({}.{} 'Utype' {})!\n", - moduleName, name.c_str(), muid, moduleName, pConflictMethodDescription->getName(), muid)); - - foundConflict = true; - } - - pConflictMethodDescription = pScriptModule->findCellMethodDescription(muid); - if (pConflictMethodDescription) - { - ERROR_MSG(fmt::format("EntityDef::loadDefClientMethods: {}.{}, 'Utype' {} Conflict({}.{} 'Utype' {})!\n", - moduleName, name.c_str(), muid, moduleName, pConflictMethodDescription->getName(), muid)); - - foundConflict = true; - } - - pConflictMethodDescription = pScriptModule->findClientMethodDescription(muid); - if (pConflictMethodDescription) - { - ERROR_MSG(fmt::format("EntityDef::loadDefClientMethods: {}.{}, 'Utype' {} Conflict({}.{} 'Utype' {})!\n", - moduleName, name.c_str(), muid, moduleName, pConflictMethodDescription->getName(), muid)); - - foundConflict = true; - } - - if (foundConflict) - return false; - } - } - - if(!pScriptModule->addClientMethodDescription(name.c_str(), methodDescription)) - return false; - } - XML_FOR_END(defMethodNode); - } - - return true; -} - -//------------------------------------------------------------------------------------- -bool EntityDef::isLoadScriptModule(ScriptDefModule* pScriptModule) -{ - switch(__loadComponentType) - { - case BASEAPP_TYPE: - { - if(!pScriptModule->hasBase()) - return false; - - break; - } - case CELLAPP_TYPE: - { - if(!pScriptModule->hasCell()) - return false; - - break; - } - case CLIENT_TYPE: - case BOTS_TYPE: - { - if(!pScriptModule->hasClient()) - return false; - - break; - } - case TOOL_TYPE: - { - return false; - break; - } - default: - { - if(!pScriptModule->hasCell()) - return false; - - break; - } - }; - - return true; -} - -//------------------------------------------------------------------------------------- -bool EntityDef::checkDefMethod(ScriptDefModule* pScriptModule, - PyObject* moduleObj, const std::string& moduleName) -{ - ScriptDefModule::METHODDESCRIPTION_MAP* methodDescrsPtr = NULL; - - PyObject* pyInspectModule = - PyImport_ImportModule(const_cast("inspect")); - - PyObject* pyGetfullargspec = NULL; - if (pyInspectModule) - { - Py_DECREF(pyInspectModule); - - pyGetfullargspec = - PyObject_GetAttrString(pyInspectModule, const_cast("getfullargspec")); - } - else - { - SCRIPT_ERROR_CHECK(); - } - - switch (__loadComponentType) - { - case BASEAPP_TYPE: - methodDescrsPtr = - (ScriptDefModule::METHODDESCRIPTION_MAP*)&pScriptModule->getBaseMethodDescriptions(); - break; - case CELLAPP_TYPE: - methodDescrsPtr = - (ScriptDefModule::METHODDESCRIPTION_MAP*)&pScriptModule->getCellMethodDescriptions(); - break; - case CLIENT_TYPE: - case BOTS_TYPE: - methodDescrsPtr = - (ScriptDefModule::METHODDESCRIPTION_MAP*)&pScriptModule->getClientMethodDescriptions(); - break; - default: - methodDescrsPtr = - (ScriptDefModule::METHODDESCRIPTION_MAP*)&pScriptModule->getCellMethodDescriptions(); - break; - }; - - ScriptDefModule::METHODDESCRIPTION_MAP::iterator iter = methodDescrsPtr->begin(); - for (; iter != methodDescrsPtr->end(); ++iter) - { - PyObject* pyMethod = - PyObject_GetAttrString(moduleObj, const_cast(iter->first.c_str())); - - if (pyMethod != NULL) - { - if (pyGetfullargspec) - { - // defеIJ - size_t methodArgsSize = iter->second->getArgSize(); - - PyObject* pyGetMethodArgs = PyObject_CallFunction(pyGetfullargspec, - const_cast("(O)"), pyMethod); - - if (!pyGetMethodArgs) - { - SCRIPT_ERROR_CHECK(); - } - else - { - PyObject* pyGetMethodArgsResult = PyObject_GetAttrString(pyGetMethodArgs, const_cast("args")); - Py_DECREF(pyGetMethodArgs); - - if (!pyGetMethodArgsResult) - { - SCRIPT_ERROR_CHECK(); - } - else - { - size_t argsSize = (size_t)PyObject_Size(pyGetMethodArgsResult); - - // ȥself - KBE_ASSERT(argsSize > 0); - argsSize -= 1; - - Py_DECREF(pyGetMethodArgsResult); - - // ĸǷƥ - if (methodArgsSize != argsSize) - { - // ƥ䣬 һexposedһΪʾļ˵һcallerIDڽű - // һΪȷ - if (iter->second->isExposed() && methodArgsSize + 1 == argsSize) - { - iter->second->setExposed(MethodDescription::EXPOSED_AND_CALLER_CHECK); - } - else - { - ERROR_MSG(fmt::format("EntityDef::checkDefMethod: {}.{} parameter is incorrect, script argssize({}) != {}! defined in {}.def!\n", - moduleName.c_str(), iter->first.c_str(), methodArgsSize, argsSize, moduleName)); - - Py_DECREF(pyMethod); - Py_XDECREF(pyGetfullargspec); - return false; - } - } - - if (iter->second->isExposed()) - { - if (iter->second->isExposed() != MethodDescription::EXPOSED_AND_CALLER_CHECK && iter->second->isCell()) - { - WARNING_MSG(fmt::format("EntityDef::checkDefMethod: exposed of method: {}.{}{}!\n", - moduleName.c_str(), iter->first.c_str(), (iter->second->isExposed() == MethodDescription::EXPOSED_AND_CALLER_CHECK ? - "" : fmt::format(", check the caller can use \"def {}(self, callerID, ...)\", such as: if callerID == self.id", iter->first)))); - } - } - } - } - } - - Py_DECREF(pyMethod); - } - else - { - ERROR_MSG(fmt::format("EntityDef::checkDefMethod: class {} does not have method[{}], defined in {}.def!\n", - moduleName.c_str(), iter->first.c_str(), moduleName)); - - PyErr_Clear(); - Py_XDECREF(pyGetfullargspec); - return false; - } - } - - Py_XDECREF(pyGetfullargspec); - return true; -} - -//------------------------------------------------------------------------------------- -void EntityDef::setScriptModuleHasComponentEntity(ScriptDefModule* pScriptModule, - bool has) -{ - switch(__loadComponentType) - { - case BASEAPP_TYPE: - pScriptModule->setBase(has); - return; - case CELLAPP_TYPE: - pScriptModule->setCell(has); - return; - case CLIENT_TYPE: - case BOTS_TYPE: - pScriptModule->setClient(has); - return; - default: - pScriptModule->setCell(has); - return; - }; -} - -//------------------------------------------------------------------------------------- -PyObject* EntityDef::loadScriptModule(std::string moduleName) -{ - PyObject* pyModule = - PyImport_ImportModule(const_cast(moduleName.c_str())); - - if (g_isReload && pyModule) - pyModule = PyImport_ReloadModule(pyModule); - - // ģ·ǷKBEűĿ¼µģֹûȡpythonģƳͻϵͳģ - if (pyModule) - { - std::string userScriptsPath = Resmgr::getSingleton().getPyUserScriptsPath(); - std::string pyModulePath = ""; - - PyObject *fileobj = NULL; - - fileobj = PyModule_GetFilenameObject(pyModule); - if (fileobj) - pyModulePath = PyUnicode_AsUTF8(fileobj); - - Py_DECREF(fileobj); - - strutil::kbe_replace(userScriptsPath, "/", ""); - strutil::kbe_replace(userScriptsPath, "\\", ""); - strutil::kbe_replace(pyModulePath, "/", ""); - strutil::kbe_replace(pyModulePath, "\\", ""); - - if (pyModulePath.find(userScriptsPath) == std::string::npos) - { - WARNING_MSG(fmt::format("EntityDef::initialize: The script module name[{}] and system module name conflict!\n", - moduleName.c_str())); - - pyModule = NULL; - } - } - - return pyModule; -} - -//------------------------------------------------------------------------------------- -bool EntityDef::loadAllComponentScriptModules(std::string entitiesPath, std::vector& scriptBaseTypes) -{ - std::string entitiesFile = entitiesPath + "entities.xml"; - - SmartPointer xml(new XML()); - if (!xml->openSection(entitiesFile.c_str())) - return false; - - TiXmlNode* node = xml->getRootNode(); - if (node == NULL) - return true; - - // Ҫؽű - std::set componentTypes; - - XML_FOR_BEGIN(node) - { - std::string moduleName = xml.get()->getKey(node); - ScriptDefModule* pScriptModule = findScriptModule(moduleName.c_str()); - - const ScriptDefModule::COMPONENTDESCRIPTION_MAP& componentDescrs = pScriptModule->getComponentDescrs(); - ScriptDefModule::COMPONENTDESCRIPTION_MAP::const_iterator comp_iter = componentDescrs.begin(); - for (; comp_iter != componentDescrs.end(); ++comp_iter) - { - componentTypes.insert(comp_iter->second->getName()); - } - } - XML_FOR_END(node); - - // ʵĽű - std::set::iterator comp_iter = componentTypes.begin(); - for (; comp_iter != componentTypes.end(); ++comp_iter) - { - std::string componentScriptName = (*comp_iter); - ScriptDefModule* pScriptModule = findScriptModule(componentScriptName.c_str()); - PyObject* pyModule = loadScriptModule(componentScriptName); - - if (pyModule == NULL) - { - // ǰǹߣkbcmd ô޷ؽűijģûпͻɾ - if (g_componentType == TOOL_TYPE) - { - if (!pScriptModule->hasClient()) - { - goto ERASE_PROPERTYS; - } - else - { - PyErr_Clear(); - continue; - } - } - - // Ƿģ ȡǷdefļж뵱ǰصķԣ - if (isLoadScriptModule(pScriptModule)) - { - ERROR_MSG(fmt::format("EntityDef::initialize: Could not load EntityComponentModule[{}]\n", - componentScriptName.c_str())); - - PyErr_Print(); - return false; - } - -ERASE_PROPERTYS: - std::vector::iterator entityScriptModuleIter = EntityDef::__scriptModules.begin(); - for (; entityScriptModuleIter != EntityDef::__scriptModules.end(); ++entityScriptModuleIter) - { - ScriptDefModule::PROPERTYDESCRIPTION_MAP& propertyDescrs = (*entityScriptModuleIter)->getPropertyDescrs(); - ScriptDefModule::PROPERTYDESCRIPTION_MAP::iterator compPropertyInter = propertyDescrs.begin(); - for (; compPropertyInter != propertyDescrs.end();) - { - if (compPropertyInter->second->getDataType()->type() == DATA_TYPE_ENTITY_COMPONENT) - { - ScriptDefModule* pCompScriptModule = static_cast(compPropertyInter->second->getDataType())->pScriptDefModule(); - if (pCompScriptModule->getName() == componentScriptName) - { - uint32 flags = compPropertyInter->second->getFlags(); - - if (g_componentType == BASEAPP_TYPE) - { - flags &= ~ENTITY_BASE_DATA_FLAGS; - flags &= ~ED_FLAG_BASE_AND_CLIENT; - } - else if (g_componentType == CELLAPP_TYPE) - { - flags &= ~ENTITY_CELL_DATA_FLAGS; - flags &= ~(ED_FLAG_ALL_CLIENTS | ED_FLAG_CELL_PUBLIC_AND_OWN | ED_FLAG_OTHER_CLIENTS | ED_FLAG_OWN_CLIENT); - } - else - { - flags &= ~ENTITY_CLIENT_DATA_FLAGS; - } - - compPropertyInter->second->setFlags(flags); - compPropertyInter->second->decRef(); - - propertyDescrs.erase(compPropertyInter++); - continue; - } - } - - compPropertyInter++; - } - } - - PyErr_Clear(); - - // ã ֮ǰûᵼisLoadScriptModuleʧЧӶûд - setScriptModuleHasComponentEntity(pScriptModule, false); - continue; - } - - setScriptModuleHasComponentEntity(pScriptModule, true); - - { - std::vector::iterator entityScriptModuleIter = EntityDef::__scriptModules.begin(); - for (; entityScriptModuleIter != EntityDef::__scriptModules.end(); ++entityScriptModuleIter) - { - std::vector& componentPropertys = g_logComponentPropertys[(*entityScriptModuleIter)->getName()]; - std::vector::iterator componentPropertysIter = componentPropertys.begin(); - for (; componentPropertysIter != componentPropertys.end(); ++componentPropertysIter) - { - PropertyDescription* pComponentPropertyDescription = (*componentPropertysIter); - ScriptDefModule* pCompScriptModule = static_cast(pComponentPropertyDescription->getDataType())->pScriptDefModule(); - - if (pCompScriptModule->getName() != componentScriptName) - continue; - - uint32 pflags = pComponentPropertyDescription->getFlags(); - - if (g_componentType == BASEAPP_TYPE) - { - pflags |= ENTITY_BASE_DATA_FLAGS; - - if(pCompScriptModule->hasClient()) - pflags |= ED_FLAG_BASE_AND_CLIENT; - } - else if (g_componentType == CELLAPP_TYPE) - { - pflags |= ENTITY_CELL_DATA_FLAGS; - - if (pCompScriptModule->hasClient()) - pflags |= (ED_FLAG_ALL_CLIENTS | ED_FLAG_CELL_PUBLIC_AND_OWN | ED_FLAG_OTHER_CLIENTS | ED_FLAG_OWN_CLIENT); - } - else - { - pflags |= ENTITY_CLIENT_DATA_FLAGS; - } - - pComponentPropertyDescription->setFlags(pflags); - if (pComponentPropertyDescription->isPersistent() && pCompScriptModule->numPropertys() == 0) - { - pComponentPropertyDescription->isPersistent(false); - - if ((*entityScriptModuleIter)->findPersistentPropertyDescription(pComponentPropertyDescription->getUType())) - { - (*entityScriptModuleIter)->getPersistentPropertyDescriptions().erase(pComponentPropertyDescription->getName()); - (*entityScriptModuleIter)->getPersistentPropertyDescriptions_uidmap().erase(pComponentPropertyDescription->getUType()); - } - } - - if ((*entityScriptModuleIter)->findPropertyDescription(pComponentPropertyDescription->getName(), g_componentType) != pComponentPropertyDescription) - { - (*entityScriptModuleIter)->addPropertyDescription(pComponentPropertyDescription->getName(), pComponentPropertyDescription, g_componentType, true); - } - } - } - } - - PyObject* pyClass = - PyObject_GetAttrString(pyModule, const_cast(componentScriptName.c_str())); - - if (pyClass == NULL) - { - ERROR_MSG(fmt::format("EntityDef::initialize: Could not find ComponentClass[{}]\n", - componentScriptName.c_str())); - - return false; - } - else - { - std::string typeNames = ""; - bool valid = false; - std::vector::iterator iter = scriptBaseTypes.begin(); - for (; iter != scriptBaseTypes.end(); ++iter) - { - if (!PyObject_IsSubclass(pyClass, (PyObject *)(*iter))) - { - typeNames += "'"; - typeNames += (*iter)->tp_name; - typeNames += "'"; - } - else - { - valid = true; - break; - } - } - - if (!valid) - { - ERROR_MSG(fmt::format("EntityDef::initialize: ComponentClass {} is not derived from KBEngine.[{}]\n", - componentScriptName.c_str(), typeNames.c_str())); - - return false; - } - } - - if (!PyType_Check(pyClass)) - { - ERROR_MSG(fmt::format("EntityDef::initialize: ComponentClass[{}] is valid!\n", - componentScriptName.c_str())); - - return false; - } - - if (!checkDefMethod(pScriptModule, pyClass, componentScriptName)) - { - ERROR_MSG(fmt::format("EntityDef::initialize: ComponentClass[{}] checkDefMethod is failed!\n", - componentScriptName.c_str())); - - return false; - } - - DEBUG_MSG(fmt::format("loaded component-script:{}({}).\n", componentScriptName.c_str(), - pScriptModule->getUType())); - - pScriptModule->setScriptType((PyTypeObject *)pyClass); - S_RELEASE(pyModule); - } - - g_logComponentPropertys.clear(); - return true; -} - -//------------------------------------------------------------------------------------- -bool EntityDef::loadAllEntityScriptModules(std::string entitiesPath, - std::vector& scriptBaseTypes) -{ - // ű壬ûļ - if (access(entitiesPath.c_str(), 0) != 0) - return true; - - if (!loadAllComponentScriptModules(entitiesPath, scriptBaseTypes)) - return false; - - std::string entitiesFile = entitiesPath + "entities.xml"; - - SmartPointer xml(new XML()); - if(!xml->openSection(entitiesFile.c_str())) - return false; - - TiXmlNode* node = xml->getRootNode(); - if(node == NULL) - return true; - - // Ҫؽű - std::set checkedComponentTypes; - - XML_FOR_BEGIN(node) - { - std::string moduleName = xml.get()->getKey(node); - ScriptDefModule* pScriptModule = findScriptModule(moduleName.c_str()); - - PyObject* pyModule = loadScriptModule(moduleName); - if (pyModule == NULL) - { - // Ƿģ ȡǷdefļж뵱ǰصķԣ - if(isLoadScriptModule(pScriptModule)) - { - ERROR_MSG(fmt::format("EntityDef::initialize: Could not load EntityModule[{}]\n", - moduleName.c_str())); - - PyErr_Print(); - return false; - } - - PyErr_Clear(); - - // ã ֮ǰûᵼisLoadScriptModuleʧЧӶûд - setScriptModuleHasComponentEntity(pScriptModule, false); - continue; - } - - setScriptModuleHasComponentEntity(pScriptModule, true); - - PyObject* pyClass = - PyObject_GetAttrString(pyModule, const_cast(moduleName.c_str())); - - if (pyClass == NULL) - { - ERROR_MSG(fmt::format("EntityDef::initialize: Could not find EntityClass[{}]\n", - moduleName.c_str())); - - return false; - } - else - { - std::string typeNames = ""; - bool valid = false; - std::vector::iterator iter = scriptBaseTypes.begin(); - for(; iter != scriptBaseTypes.end(); ++iter) - { - if(!PyObject_IsSubclass(pyClass, (PyObject *)(*iter))) - { - typeNames += "'"; - typeNames += (*iter)->tp_name; - typeNames += "'"; - } - else - { - valid = true; - break; - } - } - - if(!valid) - { - ERROR_MSG(fmt::format("EntityDef::initialize: EntityClass {} is not derived from KBEngine.[{}]\n", - moduleName.c_str(), typeNames.c_str())); - - return false; - } - } - - if(!PyType_Check(pyClass)) - { - ERROR_MSG(fmt::format("EntityDef::initialize: EntityClass[{}] is valid!\n", - moduleName.c_str())); - - return false; - } - - if(!checkDefMethod(pScriptModule, pyClass, moduleName)) - { - ERROR_MSG(fmt::format("EntityDef::initialize: EntityClass[{}] checkDefMethod is failed!\n", - moduleName.c_str())); - - return false; - } - - DEBUG_MSG(fmt::format("loaded entity-script:{}({}).\n", moduleName.c_str(), - pScriptModule->getUType())); - - pScriptModule->setScriptType((PyTypeObject *)pyClass); - S_RELEASE(pyModule); - - // ʵڸýǷӦҪʵֵʵûʾ - const ScriptDefModule::COMPONENTDESCRIPTION_MAP& componentDescrs = pScriptModule->getComponentDescrs(); - ScriptDefModule::COMPONENTDESCRIPTION_MAP::const_iterator comp_iter = componentDescrs.begin(); - for (; comp_iter != componentDescrs.end(); ++comp_iter) - { - std::string componentScriptName = comp_iter->second->getName(); - - std::set::iterator fiter = checkedComponentTypes.find(componentScriptName); - if (fiter != checkedComponentTypes.end()) - continue; - - ScriptDefModule* pComponentScriptModule = findScriptModule(componentScriptName.c_str()); - - // Ƿģ飬Ҫҵǰûģʾ - if (!pComponentScriptModule->getScriptType() && isLoadScriptModule(pComponentScriptModule)) - { - ERROR_MSG(fmt::format("EntityDef::initialize: Could not load ComponentModule[{}]\n", - componentScriptName.c_str())); - - PyErr_Print(); - return false; - } - - checkedComponentTypes.insert(componentScriptName); - } - } - XML_FOR_END(node); - - return true; -} - -//------------------------------------------------------------------------------------- -ScriptDefModule* EntityDef::findScriptModule(ENTITY_SCRIPT_UID utype, bool notFoundOutErr) -{ - // utype СΪ1 - if (utype == 0 || utype >= __scriptModules.size() + 1) - { - if (notFoundOutErr) - { - ERROR_MSG(fmt::format("EntityDef::findScriptModule: is not exist(utype:{})!\n", utype)); - } - - return NULL; - } - - return __scriptModules[utype - 1].get(); -} - -//------------------------------------------------------------------------------------- -ScriptDefModule* EntityDef::findScriptModule(const char* scriptName, bool notFoundOutErr) -{ - std::map::iterator iter = - __scriptTypeMappingUType.find(scriptName); - - if(iter == __scriptTypeMappingUType.end()) - { - if (notFoundOutErr) - { - ERROR_MSG(fmt::format("EntityDef::findScriptModule: [{}] not found!\n", scriptName)); - } - - return NULL; - } - - return findScriptModule(iter->second); -} - -//------------------------------------------------------------------------------------- -ScriptDefModule* EntityDef::findOldScriptModule(const char* scriptName, bool notFoundOutErr) -{ - std::map::iterator iter = - __oldScriptTypeMappingUType.find(scriptName); - - if(iter == __oldScriptTypeMappingUType.end()) - { - if (notFoundOutErr) - { - ERROR_MSG(fmt::format("EntityDef::findOldScriptModule: [{}] not found!\n", scriptName)); - } - - return NULL; - } - - if (iter->second >= __oldScriptModules.size() + 1) - { - if (notFoundOutErr) - { - ERROR_MSG(fmt::format("EntityDef::findOldScriptModule: is not exist(utype:{})!\n", iter->second)); - } - - return NULL; - } - - return __oldScriptModules[iter->second - 1].get(); - -} - -//------------------------------------------------------------------------------------- -bool EntityDef::installScript(PyObject* mod) -{ - if(_isInit) - return true; - - script::PyMemoryStream::installScript(NULL); - APPEND_SCRIPT_MODULE_METHOD(mod, MemoryStream, script::PyMemoryStream::py_new, METH_VARARGS, 0); - - EntityCall::installScript(NULL); - EntityComponentCall::installScript(NULL); - FixedArray::installScript(NULL); - FixedDict::installScript(NULL); - VolatileInfo::installScript(NULL); - script::entitydef::installModule("EntityDef"); - - _isInit = true; - return true; -} - -//------------------------------------------------------------------------------------- -bool EntityDef::uninstallScript() -{ - if(_isInit) - { - script::PyMemoryStream::uninstallScript(); - EntityCall::uninstallScript(); - EntityComponentCall::uninstallScript(); - FixedArray::uninstallScript(); - FixedDict::uninstallScript(); - VolatileInfo::uninstallScript(); - script::entitydef::uninstallModule(); - } - - return script::entitydef::finalise() && EntityDef::finalise(); -} - -//------------------------------------------------------------------------------------- -bool EntityDef::initializeWatcher() -{ - return script::entitydef::initializeWatcher(); -} - -//------------------------------------------------------------------------------------- -} +// Copyright 2008-2018 Yolo Technologies, Inc. All Rights Reserved. https://www.comblockengine.com + + +#include "entitydef.h" +#include "scriptdef_module.h" +#include "datatypes.h" +#include "common.h" +#include "py_entitydef.h" +#include "entity_component.h" +#include "pyscript/py_memorystream.h" +#include "resmgr/resmgr.h" +#include "common/smartpointer.h" +#include "entitydef/volatileinfo.h" +#include "entitydef/entity_call.h" +#include "entitydef/entity_component_call.h" + +#ifndef CODE_INLINE +#include "entitydef.inl" +#endif + +namespace KBEngine{ +std::vector EntityDef::__scriptModules; +std::vector EntityDef::__oldScriptModules; + +std::map EntityDef::__scriptTypeMappingUType; +std::map EntityDef::__oldScriptTypeMappingUType; + +COMPONENT_TYPE EntityDef::__loadComponentType; +std::vector EntityDef::__scriptBaseTypes; +std::string EntityDef::__entitiesPath; + +KBE_MD5 EntityDef::__md5; +bool EntityDef::_isInit = false; +bool g_isReload = false; + +bool EntityDef::__entityAliasID = false; +bool EntityDef::__entitydefAliasID = false; + +EntityDef::Context EntityDef::__context; + +// ʱԶutypeõ +ENTITY_METHOD_UID g_methodUtypeAuto = 1; +std::vector g_methodCusUtypes; + +ENTITY_PROPERTY_UID g_propertyUtypeAuto = 1; +std::vector g_propertyUtypes; + +// µĽűģʱԶutype +ENTITY_SCRIPT_UID g_scriptUtype = 1; + +// ijentityĺַ +EntityDef::GetEntityFunc EntityDef::__getEntityFunc; + +static std::map > g_logComponentPropertys; + +//------------------------------------------------------------------------------------- +EntityDef::EntityDef() +{ +} + +//------------------------------------------------------------------------------------- +EntityDef::~EntityDef() +{ + EntityDef::finalise(); +} + +//------------------------------------------------------------------------------------- +bool EntityDef::finalise(bool isReload) +{ + PropertyDescription::resetDescriptionCount(); + MethodDescription::resetDescriptionCount(); + + EntityDef::__md5.clear(); + g_methodUtypeAuto = 1; + EntityDef::_isInit = false; + + g_propertyUtypeAuto = 1; + g_propertyUtypes.clear(); + + if(!isReload) + { + std::vector::iterator iter = EntityDef::__scriptModules.begin(); + for(; iter != EntityDef::__scriptModules.end(); ++iter) + { + (*iter)->finalise(); + } + + iter = EntityDef::__oldScriptModules.begin(); + for(; iter != EntityDef::__oldScriptModules.end(); ++iter) + { + (*iter)->finalise(); + } + + EntityDef::__oldScriptModules.clear(); + EntityDef::__oldScriptTypeMappingUType.clear(); + } + + g_scriptUtype = 1; + EntityDef::__scriptModules.clear(); + EntityDef::__scriptTypeMappingUType.clear(); + g_methodCusUtypes.clear(); + DataType::finalise(); + DataTypes::finalise(); + return true; +} + +//------------------------------------------------------------------------------------- +PyObject* EntityDef::tryGetEntity(COMPONENT_ID componentID, ENTITY_ID entityID) +{ + return __getEntityFunc(componentID, entityID); +} + +//------------------------------------------------------------------------------------- +bool EntityDef::isReload() +{ + return g_isReload; +} + +//------------------------------------------------------------------------------------- +void EntityDef::reload(bool fullReload) +{ + g_isReload = true; + + script::entitydef::reload(fullReload); + + if(fullReload) + { + EntityDef::__oldScriptModules.clear(); + EntityDef::__oldScriptTypeMappingUType.clear(); + + std::vector::iterator iter = EntityDef::__scriptModules.begin(); + for(; iter != EntityDef::__scriptModules.end(); ++iter) + { + __oldScriptModules.push_back((*iter)); + __oldScriptTypeMappingUType[(*iter)->getName()] = (*iter)->getUType(); + } + + bool ret = finalise(true); + KBE_ASSERT(ret && "EntityDef::reload: finalise error!"); + + ret = initialize(EntityDef::__scriptBaseTypes, EntityDef::__loadComponentType); + KBE_ASSERT(ret && "EntityDef::reload: initialize error!"); + } + else + { + loadAllEntityScriptModules(EntityDef::__entitiesPath, EntityDef::__scriptBaseTypes); + } + + EntityDef::_isInit = true; +} + +//------------------------------------------------------------------------------------- +bool EntityDef::initialize(std::vector& scriptBaseTypes, + COMPONENT_TYPE loadComponentType) +{ + __loadComponentType = loadComponentType; + __scriptBaseTypes = scriptBaseTypes; + + __entitiesPath = Resmgr::getSingleton().getPyUserScriptsPath(); + + g_entityFlagMapping["CELL"] = ED_FLAG_CELL_PUBLIC; + g_entityFlagMapping["CELL_AND_CLIENT"] = ED_FLAG_CELL_PUBLIC_AND_OWN; + g_entityFlagMapping["CELL_AND_CLIENTS"] = ED_FLAG_ALL_CLIENTS; + g_entityFlagMapping["CELL_AND_OTHER_CLIENTS"] = ED_FLAG_OTHER_CLIENTS; + g_entityFlagMapping["BASE_AND_CLIENT"] = ED_FLAG_BASE_AND_CLIENT; + g_entityFlagMapping["BASE"] = ED_FLAG_BASE; + + g_entityFlagMapping["CELL_PUBLIC"] = ED_FLAG_CELL_PUBLIC; + g_entityFlagMapping["CELL_PRIVATE"] = ED_FLAG_CELL_PRIVATE; + g_entityFlagMapping["ALL_CLIENTS"] = ED_FLAG_ALL_CLIENTS; + g_entityFlagMapping["CELL_PUBLIC_AND_OWN"] = ED_FLAG_CELL_PUBLIC_AND_OWN; + + g_entityFlagMapping["OTHER_CLIENTS"] = ED_FLAG_OTHER_CLIENTS; + g_entityFlagMapping["OWN_CLIENT"] = ED_FLAG_OWN_CLIENT; + + std::string entitiesFile = __entitiesPath + "entities.xml"; + std::string defFilePath = __entitiesPath + "entity_defs/"; + + // ʼ + // assets/scripts/entity_defs/types.xml + if(!DataTypes::initialize(defFilePath + "types.xml")) + return false; + + // entities.xmlļ + // ű壬ûļ + if (access(entitiesFile.c_str(), 0) == 0) + { + SmartPointer xml(new XML()); + if (!xml->openSection(entitiesFile.c_str())) + return false; + + // entities.xmlڵ, ûжһentityôֱӷtrue + TiXmlNode* node = xml->getRootNode(); + if (node == NULL) + return true; + + // ʼеentityڵ + XML_FOR_BEGIN(node) + { + std::string moduleName = xml.get()->getKey(node); + + ScriptDefModule* pScriptModule = registerNewScriptDefModule(moduleName); + + std::string deffile = defFilePath + moduleName + ".def"; + SmartPointer defxml(new XML()); + + if (!defxml->openSection(deffile.c_str())) + return false; + + TiXmlNode* defNode = defxml->getRootNode(); + if (defNode == NULL) + { + // rootڵûӽڵ + continue; + } + + // defļеĶ + if (!loadDefInfo(defFilePath, moduleName, defxml.get(), defNode, pScriptModule)) + { + ERROR_MSG(fmt::format("EntityDef::initialize: failed to load entity({}) module!\n", + moduleName.c_str())); + + return false; + } + + // entityļмdetailLevel + if (!loadDetailLevelInfo(defFilePath, moduleName, defxml.get(), defNode, pScriptModule)) + { + ERROR_MSG(fmt::format("EntityDef::initialize: failed to load entity({}) DetailLevelInfo!\n", + moduleName.c_str())); + + return false; + } + + pScriptModule->onLoaded(); + } + XML_FOR_END(node); + } + + if (!script::entitydef::initialize()) + return false; + + EntityDef::md5().final(); + + if(loadComponentType == DBMGR_TYPE) + return true; + + return loadAllEntityScriptModules(__entitiesPath, scriptBaseTypes) && + initializeWatcher(); +} + +//------------------------------------------------------------------------------------- +ScriptDefModule* EntityDef::registerNewScriptDefModule(const std::string& moduleName) +{ + ScriptDefModule* pScriptModule = findScriptModule(moduleName.c_str(), false); + + if (!pScriptModule) + { + __scriptTypeMappingUType[moduleName] = g_scriptUtype; + pScriptModule = new ScriptDefModule(moduleName, g_scriptUtype++); + EntityDef::__scriptModules.push_back(pScriptModule); + } + + return pScriptModule; +} + +//------------------------------------------------------------------------------------- +MethodDescription* EntityDef::createMethodDescription(ScriptDefModule* pScriptModule, ENTITY_METHOD_UID utype, COMPONENT_ID domain, const std::string& name, MethodDescription::EXPOSED_TYPE exposedType) +{ + if(utype > 0) + g_methodCusUtypes.push_back(utype); + + // ûùutype, + if (utype == 0) + { + ENTITY_METHOD_UID muid = 0; + while (true) + { + muid = g_methodUtypeAuto++; + std::vector::iterator iterutype = + std::find(g_methodCusUtypes.begin(), g_methodCusUtypes.end(), muid); + + if (iterutype == g_methodCusUtypes.end()) + { + break; + } + } + + utype = muid; + g_methodCusUtypes.push_back(muid); + } + else + { + // ǷظUtype + ENTITY_METHOD_UID muid = utype; + std::vector::iterator iter = + std::find(g_methodCusUtypes.begin(), g_methodCusUtypes.end(), muid); + + if (iter != g_methodCusUtypes.end()) + { + bool foundConflict = false; + + MethodDescription* pConflictMethodDescription = pScriptModule->findBaseMethodDescription(muid); + if (pConflictMethodDescription) + { + ERROR_MSG(fmt::format("EntityDef::loadDefMethods: {}.{}, 'Utype' {} Conflict({}.{} 'Utype' {})! componentType={}\n", + pScriptModule->getName(), name.c_str(), muid, pScriptModule->getName(), pConflictMethodDescription->getName(), muid, COMPONENT_NAME_EX((COMPONENT_TYPE)domain))); + + foundConflict = true; + } + + pConflictMethodDescription = pScriptModule->findCellMethodDescription(muid); + if (pConflictMethodDescription) + { + ERROR_MSG(fmt::format("EntityDef::loadDefMethods: {}.{}, 'Utype' {} Conflict({}.{} 'Utype' {})! componentType={}\n", + pScriptModule->getName(), name.c_str(), muid, pScriptModule->getName(), pConflictMethodDescription->getName(), muid, COMPONENT_NAME_EX((COMPONENT_TYPE)domain))); + + foundConflict = true; + } + + pConflictMethodDescription = pScriptModule->findClientMethodDescription(muid); + if (pConflictMethodDescription) + { + ERROR_MSG(fmt::format("EntityDef::loadDefMethods: {}.{}, 'Utype' {} Conflict({}.{} 'Utype' {})! componentType={}\n", + pScriptModule->getName(), name.c_str(), muid, pScriptModule->getName(), pConflictMethodDescription->getName(), muid, COMPONENT_NAME_EX((COMPONENT_TYPE)domain))); + + foundConflict = true; + } + + if (foundConflict) + return NULL; + } + } + + return new MethodDescription(utype, domain, name, exposedType); +} + +//------------------------------------------------------------------------------------- +bool EntityDef::loadDefInfo(const std::string& defFilePath, + const std::string& moduleName, + XML* defxml, + TiXmlNode* defNode, + ScriptDefModule* pScriptModule) +{ + if(!loadAllDefDescriptions(moduleName, defxml, defNode, pScriptModule)) + { + ERROR_MSG(fmt::format("EntityDef::loadDefInfo: failed to loadAllDefDescription(), entity:{}\n", + moduleName.c_str())); + + return false; + } + + // еinterface ǵķԼ뵽ģ + if(!loadInterfaces(defFilePath, moduleName, defxml, defNode, pScriptModule)) + { + ERROR_MSG(fmt::format("EntityDef::loadDefInfo: failed to load entity:{} interface.\n", + moduleName.c_str())); + + return false; + } + + // еcomponent Լ뵽ģ + if (!loadComponents(defFilePath, moduleName, defxml, defNode, pScriptModule)) + { + ERROR_MSG(fmt::format("EntityDef::loadDefInfo: failed to load entity:{} component.\n", + moduleName.c_str())); + + return false; + } + + // ظе + if(!loadParentClass(defFilePath, moduleName, defxml, defNode, pScriptModule)) + { + ERROR_MSG(fmt::format("EntityDef::loadDefInfo: failed to load entity:{} parentClass.\n", + moduleName.c_str())); + + return false; + } + + // ԼdetailLevel + if(!loadDetailLevelInfo(defFilePath, moduleName, defxml, defNode, pScriptModule)) + { + ERROR_MSG(fmt::format("EntityDef::loadDefInfo: failed to load entity:{} DetailLevelInfo.\n", + moduleName.c_str())); + + return false; + } + + // ԼVolatileInfo + if(!loadVolatileInfo(defFilePath, moduleName, defxml, defNode, pScriptModule)) + { + ERROR_MSG(fmt::format("EntityDef::loadDefInfo: failed to load entity:{} VolatileInfo.\n", + moduleName.c_str())); + + return false; + } + + pScriptModule->autoMatchCompOwn(); + return true; +} + +//------------------------------------------------------------------------------------- +bool EntityDef::loadDetailLevelInfo(const std::string& defFilePath, + const std::string& moduleName, + XML* defxml, + TiXmlNode* defNode, + ScriptDefModule* pScriptModule) +{ + TiXmlNode* detailLevelNode = defxml->enterNode(defNode, "DetailLevels"); + if(detailLevelNode == NULL) + return true; + + DetailLevel& dlInfo = pScriptModule->getDetailLevel(); + + TiXmlNode* node = defxml->enterNode(detailLevelNode, "NEAR"); + TiXmlNode* radiusNode = defxml->enterNode(node, "radius"); + TiXmlNode* hystNode = defxml->enterNode(node, "hyst"); + if(node == NULL || radiusNode == NULL || hystNode == NULL) + { + ERROR_MSG(fmt::format("EntityDef::loadDetailLevelInfo: failed to load entity:{} NEAR-DetailLevelInfo.\n", + moduleName.c_str())); + + return false; + } + + dlInfo.level[DETAIL_LEVEL_NEAR].radius = (float)defxml->getValFloat(radiusNode); + dlInfo.level[DETAIL_LEVEL_NEAR].hyst = (float)defxml->getValFloat(hystNode); + + node = defxml->enterNode(detailLevelNode, "MEDIUM"); + radiusNode = defxml->enterNode(node, "radius"); + hystNode = defxml->enterNode(node, "hyst"); + if(node == NULL || radiusNode == NULL || hystNode == NULL) + { + ERROR_MSG(fmt::format("EntityDef::loadDetailLevelInfo: failed to load entity:{} MEDIUM-DetailLevelInfo.\n", + moduleName.c_str())); + + return false; + } + + dlInfo.level[DETAIL_LEVEL_MEDIUM].radius = (float)defxml->getValFloat(radiusNode); + + dlInfo.level[DETAIL_LEVEL_MEDIUM].radius += dlInfo.level[DETAIL_LEVEL_NEAR].radius + + dlInfo.level[DETAIL_LEVEL_NEAR].hyst; + + dlInfo.level[DETAIL_LEVEL_MEDIUM].hyst = (float)defxml->getValFloat(hystNode); + + node = defxml->enterNode(detailLevelNode, "FAR"); + radiusNode = defxml->enterNode(node, "radius"); + hystNode = defxml->enterNode(node, "hyst"); + if(node == NULL || radiusNode == NULL || hystNode == NULL) + { + ERROR_MSG(fmt::format("EntityDef::loadDetailLevelInfo: failed to load entity:{} FAR-DetailLevelInfo.\n", + moduleName.c_str())); + + return false; + } + + dlInfo.level[DETAIL_LEVEL_FAR].radius = (float)defxml->getValFloat(radiusNode); + + dlInfo.level[DETAIL_LEVEL_FAR].radius += dlInfo.level[DETAIL_LEVEL_MEDIUM].radius + + dlInfo.level[DETAIL_LEVEL_MEDIUM].hyst; + + dlInfo.level[DETAIL_LEVEL_FAR].hyst = (float)defxml->getValFloat(hystNode); + + return true; + +} + +//------------------------------------------------------------------------------------- +bool EntityDef::loadVolatileInfo(const std::string& defFilePath, + const std::string& moduleName, + XML* defxml, + TiXmlNode* defNode, + ScriptDefModule* pScriptModule) +{ + TiXmlNode* pNode = defxml->enterNode(defNode, "Volatile"); + if(pNode == NULL) + return true; + + VolatileInfo* pVolatileInfo = pScriptModule->getPVolatileInfo(); + + TiXmlNode* node = defxml->enterNode(pNode, "position"); + if(node) + { + pVolatileInfo->position((float)defxml->getValFloat(node)); + } + else + { + if(defxml->hasNode(pNode, "position")) + pVolatileInfo->position(VolatileInfo::ALWAYS); + else + pVolatileInfo->position(-1.f); + } + + node = defxml->enterNode(pNode, "yaw"); + if(node) + { + pVolatileInfo->yaw((float)defxml->getValFloat(node)); + } + else + { + if(defxml->hasNode(pNode, "yaw")) + pVolatileInfo->yaw(VolatileInfo::ALWAYS); + else + pVolatileInfo->yaw(-1.f); + } + + node = defxml->enterNode(pNode, "pitch"); + if(node) + { + pVolatileInfo->pitch((float)defxml->getValFloat(node)); + } + else + { + if(defxml->hasNode(pNode, "pitch")) + pVolatileInfo->pitch(VolatileInfo::ALWAYS); + else + pVolatileInfo->pitch(-1.f); + } + + node = defxml->enterNode(pNode, "roll"); + if(node) + { + pVolatileInfo->roll((float)defxml->getValFloat(node)); + } + else + { + if(defxml->hasNode(pNode, "roll")) + pVolatileInfo->roll(VolatileInfo::ALWAYS); + else + pVolatileInfo->roll(-1.f); + } + + node = defxml->enterNode(pNode, "optimized"); + if (node) + { + pVolatileInfo->optimized(defxml->getBool(node)); + } + else + { + if (defxml->hasNode(pNode, "optimized")) + pVolatileInfo->optimized(true); + else + pVolatileInfo->optimized(true); + } + + return true; +} + +//------------------------------------------------------------------------------------- +bool EntityDef::loadInterfaces(const std::string& defFilePath, + const std::string& moduleName, + XML* defxml, + TiXmlNode* defNode, + ScriptDefModule* pScriptModule, bool ignoreComponents) +{ + TiXmlNode* implementsNode = defxml->enterNode(defNode, "Interfaces"); + if(implementsNode == NULL) + return true; + + XML_FOR_BEGIN(implementsNode) + { + if (defxml->getKey(implementsNode) != "interface" && defxml->getKey(implementsNode) != "Interface" && + defxml->getKey(implementsNode) != "type" && defxml->getKey(implementsNode) != "Type") + continue; + + TiXmlNode* interfaceNode = defxml->enterNode(implementsNode, "Interface"); + if (!interfaceNode) + { + interfaceNode = defxml->enterNode(implementsNode, "interface"); + if (!interfaceNode) + { + interfaceNode = defxml->enterNode(implementsNode, "Type"); + if (!interfaceNode) + { + interfaceNode = defxml->enterNode(implementsNode, "type"); + if (!interfaceNode) + { + continue; + } + } + } + } + + std::string interfaceName = defxml->getKey(interfaceNode); + std::string interfacefile = defFilePath + "interfaces/" + interfaceName + ".def"; + SmartPointer interfaceXml(new XML()); + if(!interfaceXml.get()->openSection(interfacefile.c_str())) + return false; + + TiXmlNode* interfaceRootNode = interfaceXml->getRootNode(); + if(interfaceRootNode == NULL) + { + // rootڵûӽڵ + return true; + } + + if(!loadAllDefDescriptions(moduleName, interfaceXml.get(), interfaceRootNode, pScriptModule)) + { + ERROR_MSG(fmt::format("EntityDef::initialize: interface[{}] error!\n", + interfaceName.c_str())); + + return false; + } + + // ԼdetailLevel + if(!loadDetailLevelInfo(defFilePath, moduleName, interfaceXml.get(), interfaceRootNode, pScriptModule)) + { + ERROR_MSG(fmt::format("EntityDef::loadInterfaces: failed to load entity:{} DetailLevelInfo.\n", + moduleName.c_str())); + + return false; + } + + // еinterface ǵķԼ뵽ģ + if (!ignoreComponents) + { + if (!loadComponents(defFilePath, moduleName, interfaceXml.get(), interfaceRootNode, pScriptModule)) + { + ERROR_MSG(fmt::format("EntityDef::loadInterfaces: failed to load entity:{} component.\n", + moduleName.c_str())); + + return false; + } + } + + // еinterface ǵķԼ뵽ģ + if(!loadInterfaces(defFilePath, moduleName, interfaceXml.get(), interfaceRootNode, pScriptModule)) + { + ERROR_MSG(fmt::format("EntityDef::loadInterfaces: failed to load entity:{} interface.\n", + moduleName.c_str())); + + return false; + } + } + XML_FOR_END(implementsNode); + + return true; +} + +//------------------------------------------------------------------------------------- +bool EntityDef::loadComponents(const std::string& defFilePath, + const std::string& moduleName, + XML* defxml, + TiXmlNode* defNode, + ScriptDefModule* pScriptModule) +{ + TiXmlNode* implementsNode = defxml->enterNode(defNode, "Components"); + if (implementsNode == NULL) + return true; + + XML_FOR_BEGIN(implementsNode) + { + std::string componentName = defxml->getKey(implementsNode); + + TiXmlNode* componentNode = defxml->enterNode(implementsNode, componentName.c_str()); + if (!componentNode) + continue; + + if (!validDefPropertyName(componentName)) + { + ERROR_MSG(fmt::format("EntityDef::loadComponents: '{}' is limited, in module({})!\n", + componentName, moduleName)); + + return false; + } + + std::string componentTypeName = ""; + TiXmlNode* componentTypeNameNode = defxml->enterNode(componentNode, "Type"); + if (componentTypeNameNode) + componentTypeName = defxml->getKey(componentTypeNameNode); + + if (componentTypeName == "") + { + ERROR_MSG(fmt::format("EntityDef::loadComponents: component name is NULL.\n", + componentName.c_str())); + + return false; + } + + std::string componentfile = defFilePath + "components/" + componentTypeName + ".def"; + SmartPointer componentXml(new XML()); + if (!componentXml.get()->openSection(componentfile.c_str())) + return false; + + // һʵ + ENTITY_PROPERTY_UID futype = 0; + uint32 flags = ED_FLAG_BASE | ED_FLAG_CELL_PUBLIC | ENTITY_CLIENT_DATA_FLAGS; + bool isPersistent = true; + bool isIdentifier = false; // Ƿһ + uint32 databaseLength = 0; // ݿеij + std::string indexType = ""; + DETAIL_TYPE detailLevel = DETAIL_LEVEL_FAR; + std::string detailLevelStr = ""; + std::string strisPersistent; + std::string defaultStr = ""; + + TiXmlNode* utypeValNode = defxml->enterNode(componentNode, "Utype"); + + if (!calcDefPropertyUType(moduleName, componentName, (utypeValNode ? defxml->getValInt(utypeValNode) : -1), pScriptModule, futype)) + return false; + + TiXmlNode* persistentNode = defxml->enterNode(componentNode, "Persistent"); + if (persistentNode) + { + strisPersistent = defxml->getValStr(persistentNode); + + std::transform(strisPersistent.begin(), strisPersistent.end(), + strisPersistent.begin(), tolower); + + if (strisPersistent == "false") + isPersistent = false; + } + + // Ƿģ飬˵Ѿعٴμ + ScriptDefModule* pCompScriptDefModule = findScriptModule(componentTypeName.c_str(), false); + + if (!pCompScriptDefModule) + { + pCompScriptDefModule = registerNewScriptDefModule(componentTypeName); + pCompScriptDefModule->isPersistent(false); + pCompScriptDefModule->isComponentModule(true); + } + else + { + flags = ED_FLAG_UNKOWN; + + if (pCompScriptDefModule->hasBase()) + flags |= ED_FLAG_BASE; + + if (pCompScriptDefModule->hasCell()) + flags |= ED_FLAG_CELL_PUBLIC; + + if (pCompScriptDefModule->hasClient()) + { + if (pCompScriptDefModule->hasBase()) + flags |= ED_FLAG_BASE_AND_CLIENT; + else + flags |= (ED_FLAG_ALL_CLIENTS | ED_FLAG_CELL_PUBLIC_AND_OWN | ED_FLAG_OTHER_CLIENTS | ED_FLAG_OWN_CLIENT); + } + + addComponentProperty(futype, componentTypeName, componentName, flags, isPersistent, isIdentifier, + indexType, databaseLength, defaultStr, detailLevel, pScriptModule, pCompScriptDefModule); + + pScriptModule->addComponentDescription(componentName.c_str(), pCompScriptDefModule); + continue; + } + + TiXmlNode* componentRootNode = componentXml->getRootNode(); + if (componentRootNode == NULL) + { + // rootڵûӽڵ + return true; + } + + if (!loadAllDefDescriptions(componentTypeName, componentXml.get(), componentRootNode, pCompScriptDefModule)) + { + ERROR_MSG(fmt::format("EntityDef::initialize: component[{}] error!\n", + componentTypeName.c_str())); + + return false; + } + + // еinterface ǵķԼ뵽ģ + if (!loadInterfaces(defFilePath, componentTypeName, componentXml.get(), componentRootNode, pCompScriptDefModule, true)) + { + ERROR_MSG(fmt::format("EntityDef::loadComponents: failed to load component:{} interface.\n", + componentTypeName.c_str())); + + return false; + } + + // ظе + if (!loadParentClass(defFilePath + "components/", componentTypeName, componentXml.get(), componentRootNode, pCompScriptDefModule)) + { + ERROR_MSG(fmt::format("EntityDef::loadComponents: failed to load component:{} parentClass.\n", + componentTypeName.c_str())); + + return false; + } + + // ԼdetailLevel + if (!loadDetailLevelInfo(defFilePath, componentTypeName, componentXml.get(), componentRootNode, pCompScriptDefModule)) + { + ERROR_MSG(fmt::format("EntityDef::loadComponents: failed to load component:{} DetailLevelInfo.\n", + componentTypeName.c_str())); + + return false; + } + + pCompScriptDefModule->autoMatchCompOwn(); + + flags = ED_FLAG_UNKOWN; + + if (pCompScriptDefModule->hasBase()) + flags |= ED_FLAG_BASE; + + if (pCompScriptDefModule->hasCell()) + flags |= ED_FLAG_CELL_PUBLIC; + + if (pCompScriptDefModule->hasClient()) + { + if (pCompScriptDefModule->hasBase()) + flags |= ED_FLAG_BASE_AND_CLIENT; + + if (pCompScriptDefModule->hasCell()) + flags |= (ED_FLAG_ALL_CLIENTS | ED_FLAG_CELL_PUBLIC_AND_OWN | ED_FLAG_OTHER_CLIENTS | ED_FLAG_OWN_CLIENT); + } + + addComponentProperty(futype, componentTypeName, componentName, flags, isPersistent, isIdentifier, + indexType, databaseLength, defaultStr, detailLevel, pScriptModule, pCompScriptDefModule); + + pScriptModule->addComponentDescription(componentName.c_str(), pCompScriptDefModule); + } + XML_FOR_END(implementsNode); + + return true; +} + +//------------------------------------------------------------------------------------- +PropertyDescription* EntityDef::addComponentProperty(ENTITY_PROPERTY_UID utype, + const std::string& componentTypeName, + const std::string& componentName, + uint32 flags, + bool isPersistent, + bool isIdentifier, + std::string indexType, + uint32 databaseLength, + const std::string& defaultStr, + DETAIL_TYPE detailLevel, + ScriptDefModule* pScriptModule, + ScriptDefModule* pCompScriptDefModule) +{ + DataType* pEntityComponentType = DataTypes::getDataType(componentTypeName, false); + + if (!pEntityComponentType) + pEntityComponentType = new EntityComponentType(pCompScriptDefModule); + + PropertyDescription* propertyDescription = PropertyDescription::createDescription(utype, "EntityComponent", + componentName, flags, isPersistent, + pEntityComponentType, isIdentifier, indexType, + databaseLength, defaultStr, + detailLevel); + + bool ret = true; + + int32 hasBaseFlags = 0; + int32 hasCellFlags = 0; + int32 hasClientFlags = 0; + + hasBaseFlags = flags & ENTITY_BASE_DATA_FLAGS; + if (hasBaseFlags > 0) + pScriptModule->setBase(true); + + hasCellFlags = flags & ENTITY_CELL_DATA_FLAGS; + if (hasCellFlags > 0) + pScriptModule->setCell(true); + + hasClientFlags = flags & ENTITY_CLIENT_DATA_FLAGS; + if (hasClientFlags > 0) + pScriptModule->setClient(true); + + // ӵģ + if (hasCellFlags > 0) + ret = pScriptModule->addPropertyDescription(componentName.c_str(), + propertyDescription, CELLAPP_TYPE); + + if (hasBaseFlags > 0) + ret = pScriptModule->addPropertyDescription(componentName.c_str(), + propertyDescription, BASEAPP_TYPE); + + if (hasClientFlags > 0) + ret = pScriptModule->addPropertyDescription(componentName.c_str(), + propertyDescription, CLIENT_TYPE); + + if (!ret) + { + ERROR_MSG(fmt::format("EntityDef::addComponentProperty({}): {}.\n", + pScriptModule->getName(), componentName)); + + SAFE_RELEASE(propertyDescription); + return NULL; + } + + g_logComponentPropertys[pScriptModule->getName()].push_back(propertyDescription); + return propertyDescription; +} + +//------------------------------------------------------------------------------------- +bool EntityDef::loadParentClass(const std::string& defFilePath, + const std::string& moduleName, + XML* defxml, + TiXmlNode* defNode, + ScriptDefModule* pScriptModule) +{ + TiXmlNode* parentClassNode = defxml->enterNode(defNode, "Parent"); + if(parentClassNode == NULL) + return true; + + std::string parentClassName = defxml->getKey(parentClassNode); + std::string parentClassfile = defFilePath + parentClassName + ".def"; + + SmartPointer parentClassXml(new XML()); + if(!parentClassXml->openSection(parentClassfile.c_str())) + return false; + + TiXmlNode* parentClassdefNode = parentClassXml->getRootNode(); + if(parentClassdefNode == NULL) + { + // rootڵûӽڵ + return true; + } + + // defļеĶ + if(!loadDefInfo(defFilePath, parentClassName, parentClassXml.get(), parentClassdefNode, pScriptModule)) + { + ERROR_MSG(fmt::format("EntityDef::loadParentClass: failed to load entity:{} parentClass.\n", + moduleName.c_str())); + + return false; + } + + return true; +} + +//------------------------------------------------------------------------------------- +bool EntityDef::loadAllDefDescriptions(const std::string& moduleName, + XML* defxml, + TiXmlNode* defNode, + ScriptDefModule* pScriptModule) +{ + // + if(!loadDefPropertys(moduleName, defxml, defxml->enterNode(defNode, "Properties"), pScriptModule)) + return false; + + // cell + if(!loadDefCellMethods(moduleName, defxml, defxml->enterNode(defNode, "CellMethods"), pScriptModule)) + { + ERROR_MSG(fmt::format("EntityDef::loadAllDefDescription:loadDefCellMethods[{}] is failed!\n", + moduleName.c_str())); + + return false; + } + + // base + if(!loadDefBaseMethods(moduleName, defxml, defxml->enterNode(defNode, "BaseMethods"), pScriptModule)) + { + ERROR_MSG(fmt::format("EntityDef::loadAllDefDescription:loadDefBaseMethods[{}] is failed!\n", + moduleName.c_str())); + + return false; + } + + // client + if(!loadDefClientMethods(moduleName, defxml, defxml->enterNode(defNode, "ClientMethods"), pScriptModule)) + { + ERROR_MSG(fmt::format("EntityDef::loadAllDefDescription:loadDefClientMethods[{}] is failed!\n", + moduleName.c_str())); + + return false; + } + + return true; +} + +//------------------------------------------------------------------------------------- +bool EntityDef::validDefPropertyName(const std::string& name) +{ + int i = 0; + + while(true) + { + std::string limited = ENTITY_LIMITED_PROPERTYS[i]; + + if(limited == "") + break; + + if(name == limited) + return false; + + ++i; + }; + + PyObject* pyKBEModule = + PyImport_ImportModule(const_cast("KBEngine")); + + PyObject* pyEntityModule = + PyObject_GetAttrString(pyKBEModule, const_cast("Entity")); + + Py_DECREF(pyKBEModule); + + if (pyEntityModule != NULL) + { + PyObject* pyEntityAttr = + PyObject_GetAttrString(pyEntityModule, const_cast(name.c_str())); + + if (pyEntityAttr != NULL) + { + Py_DECREF(pyEntityAttr); + Py_DECREF(pyEntityModule); + return false; + } + else + { + PyErr_Clear(); + } + } + else + { + PyErr_Clear(); + } + + Py_XDECREF(pyEntityModule); + return true; +} + +//------------------------------------------------------------------------------------- +bool EntityDef::calcDefPropertyUType(const std::string& moduleName, + const std::string& name, int iUtype, ScriptDefModule* pScriptModule, ENTITY_PROPERTY_UID& outUtype) +{ + ENTITY_PROPERTY_UID futype = 0; + outUtype = futype; + + if (iUtype > 0) + { + futype = iUtype; + + if (iUtype != int(futype)) + { + ERROR_MSG(fmt::format("EntityDef::calcDefPropertyUType: 'Utype' has overflowed({} > 65535), is {}.{}!\n", + iUtype, moduleName, name.c_str())); + + return false; + } + + // ǷظUtype + std::vector::iterator iter = + std::find(g_propertyUtypes.begin(), g_propertyUtypes.end(), futype); + + if (iter != g_propertyUtypes.end()) + { + bool foundConflict = false; + + PropertyDescription* pConflictPropertyDescription = pScriptModule->findPropertyDescription(futype, BASEAPP_TYPE); + if (pConflictPropertyDescription) + { + ERROR_MSG(fmt::format("EntityDef::calcDefPropertyUType: {}.{}, 'Utype' {} Conflict({}.{} 'Utype' {})!\n", + moduleName, name.c_str(), iUtype, moduleName, pConflictPropertyDescription->getName(), iUtype)); + + foundConflict = true; + } + + pConflictPropertyDescription = pScriptModule->findPropertyDescription(futype, CELLAPP_TYPE); + if (pConflictPropertyDescription) + { + ERROR_MSG(fmt::format("EntityDef::calcDefPropertyUType: {}.{}, 'Utype' {} Conflict({}.{} 'Utype' {})!\n", + moduleName, name.c_str(), iUtype, moduleName, pConflictPropertyDescription->getName(), iUtype)); + + foundConflict = true; + } + + pConflictPropertyDescription = pScriptModule->findPropertyDescription(futype, CLIENT_TYPE); + if (pConflictPropertyDescription) + { + ERROR_MSG(fmt::format("EntityDef::calcDefPropertyUType: {}.{}, 'Utype' {} Conflict({}.{} 'Utype' {})!\n", + moduleName, name.c_str(), iUtype, moduleName, pConflictPropertyDescription->getName(), iUtype)); + + foundConflict = true; + } + + if (foundConflict) + return false; + } + + g_propertyUtypes.push_back(futype); + } + else + { + while (true) + { + futype = g_propertyUtypeAuto++; + std::vector::iterator iter = + std::find(g_propertyUtypes.begin(), g_propertyUtypes.end(), futype); + + if (iter == g_propertyUtypes.end()) + break; + } + + g_propertyUtypes.push_back(futype); + } + + outUtype = futype; + return true; +} + +//------------------------------------------------------------------------------------- +bool EntityDef::loadDefPropertys(const std::string& moduleName, + XML* xml, + TiXmlNode* defPropertyNode, + ScriptDefModule* pScriptModule) +{ + if(defPropertyNode) + { + XML_FOR_BEGIN(defPropertyNode) + { + ENTITY_PROPERTY_UID futype = 0; + uint32 flags = 0; + int32 hasBaseFlags = 0; + int32 hasCellFlags = 0; + int32 hasClientFlags = 0; + DataType* dataType = NULL; + bool isPersistent = false; + bool isIdentifier = false; // Ƿһ + uint32 databaseLength = 0; // ݿеij + std::string indexType; + DETAIL_TYPE detailLevel = DETAIL_LEVEL_FAR; + std::string detailLevelStr = ""; + std::string strType; + std::string strisPersistent; + std::string strFlags; + std::string strIdentifierNode; + std::string defaultStr; + std::string name = ""; + + name = xml->getKey(defPropertyNode); + if(!validDefPropertyName(name)) + { + ERROR_MSG(fmt::format("EntityDef::loadDefPropertys: '{}' is limited, in module({})!\n", + name, moduleName)); + + return false; + } + + TiXmlNode* flagsNode = xml->enterNode(defPropertyNode->FirstChild(), "Flags"); + if(flagsNode) + { + strFlags = xml->getValStr(flagsNode); + std::transform(strFlags.begin(), strFlags.end(), strFlags.begin(), toupper); + + ENTITYFLAGMAP::iterator iter = g_entityFlagMapping.find(strFlags.c_str()); + if(iter == g_entityFlagMapping.end()) + { + ERROR_MSG(fmt::format("EntityDef::loadDefPropertys: not fount flags[{}], is {}.{}!\n", + strFlags, moduleName, name)); + + return false; + } + + flags = iter->second; + hasBaseFlags = flags & ENTITY_BASE_DATA_FLAGS; + if(hasBaseFlags > 0) + pScriptModule->setBase(true); + + hasCellFlags = flags & ENTITY_CELL_DATA_FLAGS; + if(hasCellFlags > 0) + pScriptModule->setCell(true); + + hasClientFlags = flags & ENTITY_CLIENT_DATA_FLAGS; + if(hasClientFlags > 0) + pScriptModule->setClient(true); + + if(hasBaseFlags <= 0 && hasCellFlags <= 0) + { + ERROR_MSG(fmt::format("EntityDef::loadDefPropertys: not fount flags[{}], is {}.{}!\n", + strFlags.c_str(), moduleName, name.c_str())); + + return false; + } + } + else + { + ERROR_MSG(fmt::format("EntityDef::loadDefPropertys: not fount flagsNode, is {}.{}!\n", + moduleName, name.c_str())); + + return false; + } + + TiXmlNode* persistentNode = xml->enterNode(defPropertyNode->FirstChild(), "Persistent"); + if(persistentNode) + { + strisPersistent = xml->getValStr(persistentNode); + + std::transform(strisPersistent.begin(), strisPersistent.end(), + strisPersistent.begin(), tolower); + + if(strisPersistent == "true") + isPersistent = true; + } + + TiXmlNode* typeNode = xml->enterNode(defPropertyNode->FirstChild(), "Type"); + if(typeNode) + { + strType = xml->getValStr(typeNode); + + if(strType == "ARRAY") + { + FixedArrayType* dataType1 = new FixedArrayType(); + if(dataType1->initialize(xml, typeNode, moduleName + "_" + name)) + dataType = dataType1; + else + return false; + } + else + { + dataType = DataTypes::getDataType(strType); + } + + if(dataType == NULL) + { + return false; + } + } + else + { + ERROR_MSG(fmt::format("EntityDef::loadDefPropertys: not fount TypeNode, is {}.{}!\n", + moduleName, name.c_str())); + + return false; + } + + TiXmlNode* indexTypeNode = xml->enterNode(defPropertyNode->FirstChild(), "Index"); + if(indexTypeNode) + { + indexType = xml->getValStr(indexTypeNode); + + std::transform(indexType.begin(), indexType.end(), + indexType.begin(), toupper); + } + + TiXmlNode* identifierNode = xml->enterNode(defPropertyNode->FirstChild(), "Identifier"); + if(identifierNode) + { + strIdentifierNode = xml->getValStr(identifierNode); + std::transform(strIdentifierNode.begin(), strIdentifierNode.end(), + strIdentifierNode.begin(), tolower); + + if(strIdentifierNode == "true") + isIdentifier = true; + } + + TiXmlNode* databaseLengthNode = xml->enterNode(defPropertyNode->FirstChild(), "DatabaseLength"); + if(databaseLengthNode) + { + databaseLength = xml->getValInt(databaseLengthNode); + } + + TiXmlNode* defaultValNode = + xml->enterNode(defPropertyNode->FirstChild(), "Default"); + + if(defaultValNode) + { + defaultStr = xml->getValStr(defaultValNode); + } + + TiXmlNode* detailLevelNode = + xml->enterNode(defPropertyNode->FirstChild(), "DetailLevel"); + + if(detailLevelNode) + { + detailLevelStr = xml->getValStr(detailLevelNode); + if(detailLevelStr == "FAR") + detailLevel = DETAIL_LEVEL_FAR; + else if(detailLevelStr == "MEDIUM") + detailLevel = DETAIL_LEVEL_MEDIUM; + else if(detailLevelStr == "NEAR") + detailLevel = DETAIL_LEVEL_NEAR; + else + detailLevel = DETAIL_LEVEL_FAR; + } + + TiXmlNode* utypeValNode = + xml->enterNode(defPropertyNode->FirstChild(), "Utype"); + + if (!calcDefPropertyUType(moduleName, name, (utypeValNode ? xml->getValInt(utypeValNode) : -1), pScriptModule, futype)) + return false; + + // һʵ + PropertyDescription* propertyDescription = PropertyDescription::createDescription(futype, strType, + name, flags, isPersistent, + dataType, isIdentifier, indexType, + databaseLength, defaultStr, + detailLevel); + + bool ret = true; + + // ӵģ + if(hasCellFlags > 0) + ret = pScriptModule->addPropertyDescription(name.c_str(), + propertyDescription, CELLAPP_TYPE); + + if(hasBaseFlags > 0) + ret = pScriptModule->addPropertyDescription(name.c_str(), + propertyDescription, BASEAPP_TYPE); + + if(hasClientFlags > 0) + ret = pScriptModule->addPropertyDescription(name.c_str(), + propertyDescription, CLIENT_TYPE); + + if(!ret) + { + ERROR_MSG(fmt::format("EntityDef::addPropertyDescription({}): {}.\n", + moduleName.c_str(), xml->getTxdoc()->Value())); + + return false; + } + } + XML_FOR_END(defPropertyNode); + } + + return true; +} + +//------------------------------------------------------------------------------------- +bool EntityDef::loadDefCellMethods(const std::string& moduleName, + XML* xml, + TiXmlNode* defMethodNode, + ScriptDefModule* pScriptModule) +{ + if(defMethodNode) + { + XML_FOR_BEGIN(defMethodNode) + { + std::string name = xml->getKey(defMethodNode); + MethodDescription* methodDescription = new MethodDescription(0, CELLAPP_TYPE, name); + TiXmlNode* argNode = defMethodNode->FirstChild(); + + // ûв + if(argNode) + { + XML_FOR_BEGIN(argNode) + { + std::string argType = xml->getKey(argNode); + + if(argType == "Exposed") + { + methodDescription->setExposed(); + } + else if(argType == "Arg") + { + DataType* dataType = NULL; + TiXmlNode* typeNode = argNode->FirstChild(); + std::string strType = xml->getValStr(typeNode); + + if(strType == "ARRAY") + { + FixedArrayType* dataType1 = new FixedArrayType(); + if(dataType1->initialize(xml, typeNode, moduleName + "_" + name)) + dataType = dataType1; + } + else + { + dataType = DataTypes::getDataType(strType); + } + + if(dataType == NULL) + { + ERROR_MSG(fmt::format("EntityDef::loadDefCellMethods: dataType[{}] not found, in {}!\n", + strType.c_str(), name.c_str())); + + return false; + } + + methodDescription->pushArgType(dataType); + } + else if(argType == "Utype") + { + TiXmlNode* typeNode = argNode->FirstChild(); + + int iUtype = xml->getValInt(typeNode); + ENTITY_METHOD_UID muid = iUtype; + + if (iUtype != int(muid)) + { + ERROR_MSG(fmt::format("EntityDef::loadDefCellMethods: 'Utype' has overflowed({} > 65535), is {}.{}!\n", + iUtype, moduleName, name.c_str())); + + return false; + } + + methodDescription->setUType(muid); + g_methodCusUtypes.push_back(muid); + } + } + XML_FOR_END(argNode); + } + + // ûùutype, + if(methodDescription->getUType() <= 0) + { + ENTITY_METHOD_UID muid = 0; + while(true) + { + muid = g_methodUtypeAuto++; + std::vector::iterator iterutype = + std::find(g_methodCusUtypes.begin(), g_methodCusUtypes.end(), muid); + + if(iterutype == g_methodCusUtypes.end()) + { + break; + } + } + + methodDescription->setUType(muid); + g_methodCusUtypes.push_back(muid); + } + else + { + // ǷظUtype + ENTITY_METHOD_UID muid = methodDescription->getUType(); + std::vector::iterator iter = + std::find(g_methodCusUtypes.begin(), g_methodCusUtypes.end(), muid); + + if (iter != g_methodCusUtypes.end()) + { + bool foundConflict = false; + + MethodDescription* pConflictMethodDescription = pScriptModule->findBaseMethodDescription(muid); + if (pConflictMethodDescription) + { + ERROR_MSG(fmt::format("EntityDef::loadDefCellMethods: {}.{}, 'Utype' {} Conflict({}.{} 'Utype' {})!\n", + moduleName, name.c_str(), muid, moduleName, pConflictMethodDescription->getName(), muid)); + + foundConflict = true; + } + + pConflictMethodDescription = pScriptModule->findCellMethodDescription(muid); + if (pConflictMethodDescription) + { + ERROR_MSG(fmt::format("EntityDef::loadDefCellMethods: {}.{}, 'Utype' {} Conflict({}.{} 'Utype' {})!\n", + moduleName, name.c_str(), muid, moduleName, pConflictMethodDescription->getName(), muid)); + + foundConflict = true; + } + + pConflictMethodDescription = pScriptModule->findClientMethodDescription(muid); + if (pConflictMethodDescription) + { + ERROR_MSG(fmt::format("EntityDef::loadDefCellMethods: {}.{}, 'Utype' {} Conflict({}.{} 'Utype' {})!\n", + moduleName, name.c_str(), muid, moduleName, pConflictMethodDescription->getName(), muid)); + + foundConflict = true; + } + + if (foundConflict) + return false; + } + } + + if(!pScriptModule->addCellMethodDescription(name.c_str(), methodDescription)) + return false; + } + XML_FOR_END(defMethodNode); + } + + return true; +} + +//------------------------------------------------------------------------------------- +bool EntityDef::loadDefBaseMethods(const std::string& moduleName, XML* xml, + TiXmlNode* defMethodNode, ScriptDefModule* pScriptModule) +{ + if(defMethodNode) + { + XML_FOR_BEGIN(defMethodNode) + { + std::string name = xml->getKey(defMethodNode); + MethodDescription* methodDescription = new MethodDescription(0, BASEAPP_TYPE, name); + TiXmlNode* argNode = defMethodNode->FirstChild(); + + // ûв + if(argNode) + { + XML_FOR_BEGIN(argNode) + { + std::string argType = xml->getKey(argNode); + + if(argType == "Exposed") + { + methodDescription->setExposed(); + } + else if(argType == "Arg") + { + DataType* dataType = NULL; + TiXmlNode* typeNode = argNode->FirstChild(); + std::string strType = xml->getValStr(typeNode); + + if(strType == "ARRAY") + { + FixedArrayType* dataType1 = new FixedArrayType(); + if(dataType1->initialize(xml, typeNode, moduleName + "_" + name)) + dataType = dataType1; + } + else + { + dataType = DataTypes::getDataType(strType); + } + + if(dataType == NULL) + { + ERROR_MSG(fmt::format("EntityDef::loadDefBaseMethods: dataType[{}] not found, in {}!\n", + strType.c_str(), name.c_str())); + + return false; + } + + methodDescription->pushArgType(dataType); + } + else if(argType == "Utype") + { + TiXmlNode* typeNode = argNode->FirstChild(); + + int iUtype = xml->getValInt(typeNode); + ENTITY_METHOD_UID muid = iUtype; + + if (iUtype != int(muid)) + { + ERROR_MSG(fmt::format("EntityDef::loadDefBaseMethods: 'Utype' has overflowed({} > 65535), is {}.{}!\n", + iUtype, moduleName, name.c_str())); + + return false; + } + + methodDescription->setUType(muid); + g_methodCusUtypes.push_back(muid); + } + } + XML_FOR_END(argNode); + } + + // ûùutype, + if(methodDescription->getUType() <= 0) + { + ENTITY_METHOD_UID muid = 0; + while(true) + { + muid = g_methodUtypeAuto++; + std::vector::iterator iterutype = + std::find(g_methodCusUtypes.begin(), g_methodCusUtypes.end(), muid); + + if(iterutype == g_methodCusUtypes.end()) + { + break; + } + } + + methodDescription->setUType(muid); + g_methodCusUtypes.push_back(muid); + } + else + { + // ǷظUtype + ENTITY_METHOD_UID muid = methodDescription->getUType(); + std::vector::iterator iter = + std::find(g_methodCusUtypes.begin(), g_methodCusUtypes.end(), muid); + + if (iter != g_methodCusUtypes.end()) + { + bool foundConflict = false; + + MethodDescription* pConflictMethodDescription = pScriptModule->findBaseMethodDescription(muid); + if (pConflictMethodDescription) + { + ERROR_MSG(fmt::format("EntityDef::loadDefBaseMethods: {}.{}, 'Utype' {} Conflict({}.{} 'Utype' {})!\n", + moduleName, name.c_str(), muid, moduleName, pConflictMethodDescription->getName(), muid)); + + foundConflict = true; + } + + pConflictMethodDescription = pScriptModule->findCellMethodDescription(muid); + if (pConflictMethodDescription) + { + ERROR_MSG(fmt::format("EntityDef::loadDefBaseMethods: {}.{}, 'Utype' {} Conflict({}.{} 'Utype' {})!\n", + moduleName, name.c_str(), muid, moduleName, pConflictMethodDescription->getName(), muid)); + + foundConflict = true; + } + + pConflictMethodDescription = pScriptModule->findClientMethodDescription(muid); + if (pConflictMethodDescription) + { + ERROR_MSG(fmt::format("EntityDef::loadDefBaseMethods: {}.{}, 'Utype' {} Conflict({}.{} 'Utype' {})!\n", + moduleName, name.c_str(), muid, moduleName, pConflictMethodDescription->getName(), muid)); + + foundConflict = true; + } + + if (foundConflict) + return false; + } + } + + if(!pScriptModule->addBaseMethodDescription(name.c_str(), methodDescription)) + return false; + } + XML_FOR_END(defMethodNode); + } + + return true; +} + +//------------------------------------------------------------------------------------- +bool EntityDef::loadDefClientMethods(const std::string& moduleName, XML* xml, + TiXmlNode* defMethodNode, ScriptDefModule* pScriptModule) +{ + if(defMethodNode) + { + XML_FOR_BEGIN(defMethodNode) + { + std::string name = xml->getKey(defMethodNode); + MethodDescription* methodDescription = new MethodDescription(0, CLIENT_TYPE, name); + TiXmlNode* argNode = defMethodNode->FirstChild(); + + // ûв + if(argNode) + { + XML_FOR_BEGIN(argNode) + { + std::string argType = xml->getKey(argNode); + + if(argType == "Arg") + { + DataType* dataType = NULL; + TiXmlNode* typeNode = argNode->FirstChild(); + std::string strType = xml->getValStr(typeNode); + + if(strType == "ARRAY") + { + FixedArrayType* dataType1 = new FixedArrayType(); + if(dataType1->initialize(xml, typeNode, moduleName + "_" + name)) + dataType = dataType1; + } + else + { + dataType = DataTypes::getDataType(strType); + } + + if(dataType == NULL) + { + ERROR_MSG(fmt::format("EntityDef::loadDefClientMethods: dataType[{}] not found, in {}!\n", + strType.c_str(), name.c_str())); + + return false; + } + + methodDescription->pushArgType(dataType); + } + else if(argType == "Utype") + { + TiXmlNode* typeNode = argNode->FirstChild(); + + int iUtype = xml->getValInt(typeNode); + ENTITY_METHOD_UID muid = iUtype; + + if (iUtype != int(muid)) + { + ERROR_MSG(fmt::format("EntityDef::loadDefClientMethods: 'Utype' has overflowed({} > 65535), is {}.{}!\n", + iUtype, moduleName, name.c_str())); + + return false; + } + + methodDescription->setUType(muid); + g_methodCusUtypes.push_back(muid); + } + } + XML_FOR_END(argNode); + } + + // ûùutype, + if(methodDescription->getUType() <= 0) + { + ENTITY_METHOD_UID muid = 0; + while(true) + { + muid = g_methodUtypeAuto++; + std::vector::iterator iterutype = + std::find(g_methodCusUtypes.begin(), g_methodCusUtypes.end(), muid); + + if(iterutype == g_methodCusUtypes.end()) + { + break; + } + } + + methodDescription->setUType(muid); + g_methodCusUtypes.push_back(muid); + } + else + { + // ǷظUtype + ENTITY_METHOD_UID muid = methodDescription->getUType(); + std::vector::iterator iter = + std::find(g_methodCusUtypes.begin(), g_methodCusUtypes.end(), muid); + + if (iter != g_methodCusUtypes.end()) + { + bool foundConflict = false; + + MethodDescription* pConflictMethodDescription = pScriptModule->findBaseMethodDescription(muid); + if (pConflictMethodDescription) + { + ERROR_MSG(fmt::format("EntityDef::loadDefClientMethods: {}.{}, 'Utype' {} Conflict({}.{} 'Utype' {})!\n", + moduleName, name.c_str(), muid, moduleName, pConflictMethodDescription->getName(), muid)); + + foundConflict = true; + } + + pConflictMethodDescription = pScriptModule->findCellMethodDescription(muid); + if (pConflictMethodDescription) + { + ERROR_MSG(fmt::format("EntityDef::loadDefClientMethods: {}.{}, 'Utype' {} Conflict({}.{} 'Utype' {})!\n", + moduleName, name.c_str(), muid, moduleName, pConflictMethodDescription->getName(), muid)); + + foundConflict = true; + } + + pConflictMethodDescription = pScriptModule->findClientMethodDescription(muid); + if (pConflictMethodDescription) + { + ERROR_MSG(fmt::format("EntityDef::loadDefClientMethods: {}.{}, 'Utype' {} Conflict({}.{} 'Utype' {})!\n", + moduleName, name.c_str(), muid, moduleName, pConflictMethodDescription->getName(), muid)); + + foundConflict = true; + } + + if (foundConflict) + return false; + } + } + + if(!pScriptModule->addClientMethodDescription(name.c_str(), methodDescription)) + return false; + } + XML_FOR_END(defMethodNode); + } + + return true; +} + +//------------------------------------------------------------------------------------- +bool EntityDef::isLoadScriptModule(ScriptDefModule* pScriptModule) +{ + switch(__loadComponentType) + { + case BASEAPP_TYPE: + { + if(!pScriptModule->hasBase()) + return false; + + break; + } + case CELLAPP_TYPE: + { + if(!pScriptModule->hasCell()) + return false; + + break; + } + case CLIENT_TYPE: + case BOTS_TYPE: + { + if(!pScriptModule->hasClient()) + return false; + + break; + } + case TOOL_TYPE: + { + return false; + break; + } + default: + { + if(!pScriptModule->hasCell()) + return false; + + break; + } + }; + + return true; +} + +//------------------------------------------------------------------------------------- +bool EntityDef::checkDefMethod(ScriptDefModule* pScriptModule, + PyObject* moduleObj, const std::string& moduleName) +{ + ScriptDefModule::METHODDESCRIPTION_MAP* methodDescrsPtr = NULL; + + PyObject* pyInspectModule = + PyImport_ImportModule(const_cast("inspect")); + + PyObject* pyGetfullargspec = NULL; + if (pyInspectModule) + { + Py_DECREF(pyInspectModule); + + pyGetfullargspec = + PyObject_GetAttrString(pyInspectModule, const_cast("getfullargspec")); + } + else + { + SCRIPT_ERROR_CHECK(); + } + + switch (__loadComponentType) + { + case BASEAPP_TYPE: + methodDescrsPtr = + (ScriptDefModule::METHODDESCRIPTION_MAP*)&pScriptModule->getBaseMethodDescriptions(); + break; + case CELLAPP_TYPE: + methodDescrsPtr = + (ScriptDefModule::METHODDESCRIPTION_MAP*)&pScriptModule->getCellMethodDescriptions(); + break; + case CLIENT_TYPE: + case BOTS_TYPE: + methodDescrsPtr = + (ScriptDefModule::METHODDESCRIPTION_MAP*)&pScriptModule->getClientMethodDescriptions(); + break; + default: + methodDescrsPtr = + (ScriptDefModule::METHODDESCRIPTION_MAP*)&pScriptModule->getCellMethodDescriptions(); + break; + }; + + ScriptDefModule::METHODDESCRIPTION_MAP::iterator iter = methodDescrsPtr->begin(); + for (; iter != methodDescrsPtr->end(); ++iter) + { + PyObject* pyMethod = + PyObject_GetAttrString(moduleObj, const_cast(iter->first.c_str())); + + if (pyMethod != NULL) + { + if (pyGetfullargspec) + { + // defеIJ + size_t methodArgsSize = iter->second->getArgSize(); + + PyObject* pyGetMethodArgs = PyObject_CallFunction(pyGetfullargspec, + const_cast("(O)"), pyMethod); + + if (!pyGetMethodArgs) + { + SCRIPT_ERROR_CHECK(); + } + else + { + PyObject* pyGetMethodArgsResult = PyObject_GetAttrString(pyGetMethodArgs, const_cast("args")); + Py_DECREF(pyGetMethodArgs); + + if (!pyGetMethodArgsResult) + { + SCRIPT_ERROR_CHECK(); + } + else + { + size_t argsSize = (size_t)PyObject_Size(pyGetMethodArgsResult); + + // ȥself + KBE_ASSERT(argsSize > 0); + argsSize -= 1; + + Py_DECREF(pyGetMethodArgsResult); + + // ĸǷƥ + if (methodArgsSize != argsSize) + { + // ƥ䣬 һexposedһΪʾļ˵һcallerIDڽű + // һΪȷ + if (iter->second->isExposed() && methodArgsSize + 1 == argsSize) + { + iter->second->setExposed(MethodDescription::EXPOSED_AND_CALLER_CHECK); + } + else + { + ERROR_MSG(fmt::format("EntityDef::checkDefMethod: {}.{} parameter is incorrect, script argssize({}) != {}! defined in {}.def!\n", + moduleName.c_str(), iter->first.c_str(), methodArgsSize, argsSize, moduleName)); + + Py_DECREF(pyMethod); + Py_XDECREF(pyGetfullargspec); + return false; + } + } + + if (iter->second->isExposed()) + { + if (iter->second->isExposed() != MethodDescription::EXPOSED_AND_CALLER_CHECK && iter->second->isCell()) + { + WARNING_MSG(fmt::format("EntityDef::checkDefMethod: exposed of method: {}.{}{}!\n", + moduleName.c_str(), iter->first.c_str(), (iter->second->isExposed() == MethodDescription::EXPOSED_AND_CALLER_CHECK ? + "" : fmt::format(", check the caller can use \"def {}(self, callerID, ...)\", such as: if callerID == self.id", iter->first)))); + } + } + } + } + } + + Py_DECREF(pyMethod); + } + else + { + PyErr_Clear(); + + PyObject* pyClassStr = PyObject_Str(moduleObj); + const char* classStr = PyUnicode_AsUTF8AndSize(pyClassStr, NULL); + + ERROR_MSG(fmt::format("EntityDef::checkDefMethod: {} does not have method[{}], defined in {}.def!\n", + classStr, iter->first.c_str(), moduleName)); + + Py_DECREF(pyClassStr); + Py_XDECREF(pyGetfullargspec); + return false; + } + } + + Py_XDECREF(pyGetfullargspec); + return true; +} + +//------------------------------------------------------------------------------------- +void EntityDef::setScriptModuleHasComponentEntity(ScriptDefModule* pScriptModule, + bool has) +{ + switch(__loadComponentType) + { + case BASEAPP_TYPE: + pScriptModule->setBase(has); + return; + case CELLAPP_TYPE: + pScriptModule->setCell(has); + return; + case CLIENT_TYPE: + case BOTS_TYPE: + pScriptModule->setClient(has); + return; + default: + pScriptModule->setCell(has); + return; + }; +} + +//------------------------------------------------------------------------------------- +PyObject* EntityDef::loadScriptModule(std::string moduleName) +{ + PyObject* pyModule = + PyImport_ImportModule(const_cast(moduleName.c_str())); + + if (g_isReload && pyModule) + pyModule = PyImport_ReloadModule(pyModule); + + // ģ·ǷKBEűĿ¼µģֹûȡpythonģƳͻϵͳģ + if (pyModule) + { + std::string userScriptsPath = Resmgr::getSingleton().getPyUserScriptsPath(); + std::string pyModulePath = ""; + + PyObject *fileobj = NULL; + + fileobj = PyModule_GetFilenameObject(pyModule); + if (fileobj) + pyModulePath = PyUnicode_AsUTF8(fileobj); + + Py_DECREF(fileobj); + + strutil::kbe_replace(userScriptsPath, "/", ""); + strutil::kbe_replace(userScriptsPath, "\\", ""); + strutil::kbe_replace(pyModulePath, "/", ""); + strutil::kbe_replace(pyModulePath, "\\", ""); + + if (pyModulePath.find(userScriptsPath) == std::string::npos) + { + WARNING_MSG(fmt::format("EntityDef::initialize: The script module name[{}] and system module name conflict!\n", + moduleName.c_str())); + + pyModule = NULL; + } + } + + return pyModule; +} + +//------------------------------------------------------------------------------------- +bool EntityDef::loadAllComponentScriptModules(std::string entitiesPath, std::vector& scriptBaseTypes) +{ + std::string entitiesFile = entitiesPath + "entities.xml"; + + // entities.xmlļ + // ű壬ûļ + if (access(entitiesFile.c_str(), 0) != 0) + { + return true; + } + + SmartPointer xml(new XML()); + if (!xml->openSection(entitiesFile.c_str())) + return false; + + TiXmlNode* node = xml->getRootNode(); + if (node == NULL) + return true; + + // Ҫؽű + std::set componentTypes; + + XML_FOR_BEGIN(node) + { + std::string moduleName = xml.get()->getKey(node); + ScriptDefModule* pScriptModule = findScriptModule(moduleName.c_str()); + + const ScriptDefModule::COMPONENTDESCRIPTION_MAP& componentDescrs = pScriptModule->getComponentDescrs(); + ScriptDefModule::COMPONENTDESCRIPTION_MAP::const_iterator comp_iter = componentDescrs.begin(); + for (; comp_iter != componentDescrs.end(); ++comp_iter) + { + componentTypes.insert(comp_iter->second->getName()); + } + } + XML_FOR_END(node); + + // ʵĽű + std::set::iterator comp_iter = componentTypes.begin(); + for (; comp_iter != componentTypes.end(); ++comp_iter) + { + std::string componentScriptName = (*comp_iter); + ScriptDefModule* pScriptModule = findScriptModule(componentScriptName.c_str()); + PyObject* pyModule = loadScriptModule(componentScriptName); + + if (pyModule == NULL) + { + // ǰǹߣkbcmd ô޷ؽűijģûпͻɾ + if (g_componentType == TOOL_TYPE) + { + if (!pScriptModule->hasClient()) + { + goto ERASE_PROPERTYS; + } + else + { + PyErr_Clear(); + continue; + } + } + + // Ƿģ ȡǷdefļж뵱ǰصķԣ + if (isLoadScriptModule(pScriptModule)) + { + ERROR_MSG(fmt::format("EntityDef::initialize: Could not load EntityComponentModule[{}]\n", + componentScriptName.c_str())); + + PyErr_Print(); + return false; + } + +ERASE_PROPERTYS: + std::vector::iterator entityScriptModuleIter = EntityDef::__scriptModules.begin(); + for (; entityScriptModuleIter != EntityDef::__scriptModules.end(); ++entityScriptModuleIter) + { + ScriptDefModule::PROPERTYDESCRIPTION_MAP& propertyDescrs = (*entityScriptModuleIter)->getPropertyDescrs(); + ScriptDefModule::PROPERTYDESCRIPTION_MAP::iterator compPropertyInter = propertyDescrs.begin(); + for (; compPropertyInter != propertyDescrs.end();) + { + if (compPropertyInter->second->getDataType()->type() == DATA_TYPE_ENTITY_COMPONENT) + { + ScriptDefModule* pCompScriptModule = static_cast(compPropertyInter->second->getDataType())->pScriptDefModule(); + if (pCompScriptModule->getName() == componentScriptName) + { + uint32 flags = compPropertyInter->second->getFlags(); + + if (g_componentType == BASEAPP_TYPE) + { + flags &= ~ENTITY_BASE_DATA_FLAGS; + flags &= ~ED_FLAG_BASE_AND_CLIENT; + } + else if (g_componentType == CELLAPP_TYPE) + { + flags &= ~ENTITY_CELL_DATA_FLAGS; + flags &= ~(ED_FLAG_ALL_CLIENTS | ED_FLAG_CELL_PUBLIC_AND_OWN | ED_FLAG_OTHER_CLIENTS | ED_FLAG_OWN_CLIENT); + } + else + { + flags &= ~ENTITY_CLIENT_DATA_FLAGS; + } + + compPropertyInter->second->setFlags(flags); + compPropertyInter->second->decRef(); + + propertyDescrs.erase(compPropertyInter++); + continue; + } + } + + compPropertyInter++; + } + } + + PyErr_Clear(); + + // ã ֮ǰûᵼisLoadScriptModuleʧЧӶûд + setScriptModuleHasComponentEntity(pScriptModule, false); + continue; + } + + setScriptModuleHasComponentEntity(pScriptModule, true); + + { + std::vector::iterator entityScriptModuleIter = EntityDef::__scriptModules.begin(); + for (; entityScriptModuleIter != EntityDef::__scriptModules.end(); ++entityScriptModuleIter) + { + std::vector& componentPropertys = g_logComponentPropertys[(*entityScriptModuleIter)->getName()]; + std::vector::iterator componentPropertysIter = componentPropertys.begin(); + for (; componentPropertysIter != componentPropertys.end(); ++componentPropertysIter) + { + PropertyDescription* pComponentPropertyDescription = (*componentPropertysIter); + ScriptDefModule* pCompScriptModule = static_cast(pComponentPropertyDescription->getDataType())->pScriptDefModule(); + + if (pCompScriptModule->getName() != componentScriptName) + continue; + + uint32 pflags = pComponentPropertyDescription->getFlags(); + + if (g_componentType == BASEAPP_TYPE) + { + pflags |= ENTITY_BASE_DATA_FLAGS; + + if(pCompScriptModule->hasClient()) + pflags |= ED_FLAG_BASE_AND_CLIENT; + } + else if (g_componentType == CELLAPP_TYPE) + { + pflags |= ENTITY_CELL_DATA_FLAGS; + + if (pCompScriptModule->hasClient()) + pflags |= (ED_FLAG_ALL_CLIENTS | ED_FLAG_CELL_PUBLIC_AND_OWN | ED_FLAG_OTHER_CLIENTS | ED_FLAG_OWN_CLIENT); + } + else + { + pflags |= ENTITY_CLIENT_DATA_FLAGS; + } + + pComponentPropertyDescription->setFlags(pflags); + if (pComponentPropertyDescription->isPersistent() && pCompScriptModule->numPropertys() == 0) + { + pComponentPropertyDescription->isPersistent(false); + + if ((*entityScriptModuleIter)->findPersistentPropertyDescription(pComponentPropertyDescription->getUType())) + { + (*entityScriptModuleIter)->getPersistentPropertyDescriptions().erase(pComponentPropertyDescription->getName()); + (*entityScriptModuleIter)->getPersistentPropertyDescriptions_uidmap().erase(pComponentPropertyDescription->getUType()); + } + } + + if ((*entityScriptModuleIter)->findPropertyDescription(pComponentPropertyDescription->getName(), g_componentType) != pComponentPropertyDescription) + { + (*entityScriptModuleIter)->addPropertyDescription(pComponentPropertyDescription->getName(), pComponentPropertyDescription, g_componentType, true); + } + } + } + } + + PyObject* pyClass = + PyObject_GetAttrString(pyModule, const_cast(componentScriptName.c_str())); + + if (pyClass == NULL) + { + ERROR_MSG(fmt::format("EntityDef::initialize: Could not find ComponentClass[{}]\n", + componentScriptName.c_str())); + + return false; + } + else + { + std::string typeNames = ""; + bool valid = false; + std::vector::iterator iter = scriptBaseTypes.begin(); + for (; iter != scriptBaseTypes.end(); ++iter) + { + if (!PyObject_IsSubclass(pyClass, (PyObject *)(*iter))) + { + typeNames += "'"; + typeNames += (*iter)->tp_name; + typeNames += "'"; + } + else + { + valid = true; + break; + } + } + + if (!valid) + { + ERROR_MSG(fmt::format("EntityDef::initialize: ComponentClass {} is not derived from KBEngine.[{}]\n", + componentScriptName.c_str(), typeNames.c_str())); + + return false; + } + } + + if (!PyType_Check(pyClass)) + { + ERROR_MSG(fmt::format("EntityDef::initialize: ComponentClass[{}] is valid!\n", + componentScriptName.c_str())); + + return false; + } + + if (!checkDefMethod(pScriptModule, pyClass, componentScriptName)) + { + ERROR_MSG(fmt::format("EntityDef::initialize: ComponentClass[{}] checkDefMethod is failed!\n", + componentScriptName.c_str())); + + return false; + } + + DEBUG_MSG(fmt::format("loaded component-script:{}({}).\n", componentScriptName.c_str(), + pScriptModule->getUType())); + + pScriptModule->setScriptType((PyTypeObject *)pyClass); + S_RELEASE(pyModule); + } + + g_logComponentPropertys.clear(); + return true; +} + +//------------------------------------------------------------------------------------- +bool EntityDef::loadAllEntityScriptModules(std::string entitiesPath, + std::vector& scriptBaseTypes) +{ + std::string entitiesFile = entitiesPath + "entities.xml"; + + // ű壬ûļ + if (access(entitiesFile.c_str(), 0) != 0) + return true; + + if (!loadAllComponentScriptModules(entitiesPath, scriptBaseTypes)) + return false; + + SmartPointer xml(new XML()); + if(!xml->openSection(entitiesFile.c_str())) + return false; + + TiXmlNode* node = xml->getRootNode(); + if(node == NULL) + return true; + + // Ҫؽű + std::set checkedComponentTypes; + + XML_FOR_BEGIN(node) + { + std::string moduleName = xml.get()->getKey(node); + ScriptDefModule* pScriptModule = findScriptModule(moduleName.c_str()); + + PyObject* pyModule = loadScriptModule(moduleName); + if (pyModule == NULL) + { + // Ƿģ ȡǷdefļж뵱ǰصķԣ + if(isLoadScriptModule(pScriptModule)) + { + ERROR_MSG(fmt::format("EntityDef::initialize: Could not load EntityModule[{}]\n", + moduleName.c_str())); + + PyErr_Print(); + return false; + } + + PyErr_Clear(); + + // ã ֮ǰûᵼisLoadScriptModuleʧЧӶûд + setScriptModuleHasComponentEntity(pScriptModule, false); + continue; + } + + setScriptModuleHasComponentEntity(pScriptModule, true); + + PyObject* pyClass = + PyObject_GetAttrString(pyModule, const_cast(moduleName.c_str())); + + if (pyClass == NULL) + { + ERROR_MSG(fmt::format("EntityDef::initialize: Could not find EntityClass[{}]\n", + moduleName.c_str())); + + return false; + } + else + { + std::string typeNames = ""; + bool valid = false; + std::vector::iterator iter = scriptBaseTypes.begin(); + for(; iter != scriptBaseTypes.end(); ++iter) + { + if(!PyObject_IsSubclass(pyClass, (PyObject *)(*iter))) + { + typeNames += "'"; + typeNames += (*iter)->tp_name; + typeNames += "'"; + } + else + { + valid = true; + break; + } + } + + if(!valid) + { + ERROR_MSG(fmt::format("EntityDef::initialize: EntityClass {} is not derived from KBEngine.[{}]\n", + moduleName.c_str(), typeNames.c_str())); + + return false; + } + } + + if(!PyType_Check(pyClass)) + { + ERROR_MSG(fmt::format("EntityDef::initialize: EntityClass[{}] is valid!\n", + moduleName.c_str())); + + return false; + } + + if(!checkDefMethod(pScriptModule, pyClass, moduleName)) + { + ERROR_MSG(fmt::format("EntityDef::initialize: EntityClass[{}] checkDefMethod is failed!\n", + moduleName.c_str())); + + return false; + } + + DEBUG_MSG(fmt::format("loaded entity-script:{}({}).\n", moduleName.c_str(), + pScriptModule->getUType())); + + pScriptModule->setScriptType((PyTypeObject *)pyClass); + S_RELEASE(pyModule); + + // ʵڸýǷӦҪʵֵʵûʾ + const ScriptDefModule::COMPONENTDESCRIPTION_MAP& componentDescrs = pScriptModule->getComponentDescrs(); + ScriptDefModule::COMPONENTDESCRIPTION_MAP::const_iterator comp_iter = componentDescrs.begin(); + for (; comp_iter != componentDescrs.end(); ++comp_iter) + { + std::string componentScriptName = comp_iter->second->getName(); + + std::set::iterator fiter = checkedComponentTypes.find(componentScriptName); + if (fiter != checkedComponentTypes.end()) + continue; + + ScriptDefModule* pComponentScriptModule = findScriptModule(componentScriptName.c_str()); + + // Ƿģ飬Ҫҵǰûģʾ + if (!pComponentScriptModule->getScriptType() && isLoadScriptModule(pComponentScriptModule)) + { + ERROR_MSG(fmt::format("EntityDef::initialize: Could not load ComponentModule[{}]\n", + componentScriptName.c_str())); + + PyErr_Print(); + return false; + } + + checkedComponentTypes.insert(componentScriptName); + } + } + XML_FOR_END(node); + + return true; +} + +//------------------------------------------------------------------------------------- +ScriptDefModule* EntityDef::findScriptModule(ENTITY_SCRIPT_UID utype, bool notFoundOutErr) +{ + // utype СΪ1 + if (utype == 0 || utype >= __scriptModules.size() + 1) + { + if (notFoundOutErr) + { + ERROR_MSG(fmt::format("EntityDef::findScriptModule: is not exist(utype:{})!\n", utype)); + } + + return NULL; + } + + return __scriptModules[utype - 1].get(); +} + +//------------------------------------------------------------------------------------- +ScriptDefModule* EntityDef::findScriptModule(const char* scriptName, bool notFoundOutErr) +{ + std::map::iterator iter = + __scriptTypeMappingUType.find(scriptName); + + if(iter == __scriptTypeMappingUType.end()) + { + if (notFoundOutErr) + { + ERROR_MSG(fmt::format("EntityDef::findScriptModule: [{}] not found!\n", scriptName)); + } + + return NULL; + } + + return findScriptModule(iter->second); +} + +//------------------------------------------------------------------------------------- +ScriptDefModule* EntityDef::findOldScriptModule(const char* scriptName, bool notFoundOutErr) +{ + std::map::iterator iter = + __oldScriptTypeMappingUType.find(scriptName); + + if(iter == __oldScriptTypeMappingUType.end()) + { + if (notFoundOutErr) + { + ERROR_MSG(fmt::format("EntityDef::findOldScriptModule: [{}] not found!\n", scriptName)); + } + + return NULL; + } + + if (iter->second >= __oldScriptModules.size() + 1) + { + if (notFoundOutErr) + { + ERROR_MSG(fmt::format("EntityDef::findOldScriptModule: is not exist(utype:{})!\n", iter->second)); + } + + return NULL; + } + + return __oldScriptModules[iter->second - 1].get(); + +} + +//------------------------------------------------------------------------------------- +bool EntityDef::installScript(PyObject* mod) +{ + if(_isInit) + return true; + + script::PyMemoryStream::installScript(NULL); + APPEND_SCRIPT_MODULE_METHOD(mod, MemoryStream, script::PyMemoryStream::py_new, METH_VARARGS, 0); + + EntityCall::installScript(NULL); + EntityComponentCall::installScript(NULL); + FixedArray::installScript(NULL); + FixedDict::installScript(NULL); + VolatileInfo::installScript(NULL); + script::entitydef::installModule("EntityDef"); + + _isInit = true; + return true; +} + +//------------------------------------------------------------------------------------- +bool EntityDef::uninstallScript() +{ + if(_isInit) + { + script::PyMemoryStream::uninstallScript(); + EntityCall::uninstallScript(); + EntityComponentCall::uninstallScript(); + FixedArray::uninstallScript(); + FixedDict::uninstallScript(); + VolatileInfo::uninstallScript(); + script::entitydef::uninstallModule(); + } + + return script::entitydef::finalise() && EntityDef::finalise(); +} + +//------------------------------------------------------------------------------------- +bool EntityDef::initializeWatcher() +{ + return script::entitydef::initializeWatcher(); +} + +//------------------------------------------------------------------------------------- +} diff --git a/kbe/src/lib/entitydef/entitydef.h b/kbe/src/lib/entitydef/entitydef.h index e70f001c08..36738296c4 100644 --- a/kbe/src/lib/entitydef/entitydef.h +++ b/kbe/src/lib/entitydef/entitydef.h @@ -1,266 +1,267 @@ -// Copyright 2008-2018 Yolo Technologies, Inc. All Rights Reserved. https://www.comblockengine.com - - -#ifndef KBE_ENTITYDEF_H -#define KBE_ENTITYDEF_H - -#include "common/common.h" -#include "common/md5.h" -#if KBE_PLATFORM == PLATFORM_WIN32 -#pragma warning (disable : 4910) -#pragma warning (disable : 4251) -#endif - -#include "method.h" -#include "property.h" -#include "entity_call.h" -#include "math/math.h" -#include "pyscript/scriptobject.h" -#include "xml/xml.h" -#include "common/smartpointer.h" - - -namespace KBEngine{ - -class ScriptDefModule; -typedef SmartPointer ScriptDefModulePtr; - -class EntityDef -{ -public: - typedef std::vector SCRIPT_MODULES; - typedef std::map SCRIPT_MODULE_UID_MAP; - - typedef std::tr1::function GetEntityFunc; - typedef std::tr1::function FindChannelFunc; - - EntityDef(); - ~EntityDef(); - - /** - ʼ - */ - static bool initialize(std::vector& scriptBaseTypes, - COMPONENT_TYPE loadComponentType); - - static bool finalise(bool isReload = false); - - static void reload(bool fullReload); - - /** - ͨentityIDѰʵ - */ - static PyObject* tryGetEntity(COMPONENT_ID componentID, ENTITY_ID entityID); - - /** - entityCall__getEntityFuncַ - */ - static void setGetEntityFunc(GetEntityFunc func) { - __getEntityFunc = func; - }; - - /** - - */ - static bool loadAllEntityScriptModules(std::string entitiesPath, - std::vector& scriptBaseTypes); - - static bool loadAllComponentScriptModules(std::string entitiesPath, - std::vector& scriptBaseTypes); - - static bool loadAllDefDescriptions(const std::string& moduleName, - XML* defxml, - TiXmlNode* defNode, - ScriptDefModule* pScriptModule); - - static bool loadDefPropertys(const std::string& moduleName, - XML* xml, - TiXmlNode* defPropertyNode, - ScriptDefModule* pScriptModule); - - static bool calcDefPropertyUType(const std::string& moduleName, - const std::string& name, int iUtype, ScriptDefModule* pScriptModule, ENTITY_PROPERTY_UID& outUtype); - - static bool loadDefCellMethods(const std::string& moduleName, - XML* xml, - TiXmlNode* defMethodNode, - ScriptDefModule* pScriptModule); - - static bool loadDefBaseMethods(const std::string& moduleName, - XML* xml, - TiXmlNode* defMethodNode, - ScriptDefModule* pScriptModule); - - static bool loadDefClientMethods(const std::string& moduleName, - XML* xml, - TiXmlNode* defMethodNode, - ScriptDefModule* pScriptModule); - - static bool loadInterfaces(const std::string& defFilePath, - const std::string& moduleName, - XML* defxml, - TiXmlNode* defNode, - ScriptDefModule* pScriptModule, bool ignoreComponents = false); - - static bool loadComponents(const std::string& defFilePath, - const std::string& moduleName, - XML* defxml, - TiXmlNode* defNode, - ScriptDefModule* pScriptModule); - - static PropertyDescription* addComponentProperty(ENTITY_PROPERTY_UID utype, - const std::string& componentTypeName, - const std::string& componentName, - uint32 flags, - bool isPersistent, - bool isIdentifier, - std::string indexType, - uint32 databaseLength, - const std::string& defaultStr, - DETAIL_TYPE detailLevel, - ScriptDefModule* pScriptModule, - ScriptDefModule* pCompScriptDefModule); - - static bool loadParentClass(const std::string& defFilePath, - const std::string& moduleName, - XML* defxml, - TiXmlNode* defNode, - ScriptDefModule* pScriptModule); - - static bool loadDefInfo(const std::string& defFilePath, - const std::string& moduleName, - XML* defxml, - TiXmlNode* defNode, - ScriptDefModule* pScriptModule); - - static bool loadDetailLevelInfo(const std::string& defFilePath, - const std::string& moduleName, - XML* defxml, - TiXmlNode* defNode, - ScriptDefModule* pScriptModule); - - static bool loadVolatileInfo(const std::string& defFilePath, - const std::string& moduleName, - XML* defxml, - TiXmlNode* defNode, - ScriptDefModule* pScriptModule); - - static PyObject* loadScriptModule(std::string moduleName); - - /** - Ƿűģ - */ - static bool isLoadScriptModule(ScriptDefModule* pScriptModule); - - /** - ݵǰǷcell base - */ - static void setScriptModuleHasComponentEntity(ScriptDefModule* pScriptModule, bool has); - - /** - űģбķǷ - */ - static bool checkDefMethod(ScriptDefModule* pScriptModule, PyObject* moduleObj, - const std::string& moduleName); - - /** - űģбǷϷ - */ - static bool validDefPropertyName(const std::string& name); - - /** - ͨѰҵӦĽűģ - */ - static ScriptDefModule* findScriptModule(ENTITY_SCRIPT_UID utype, bool notFoundOutErr = true); - static ScriptDefModule* findScriptModule(const char* scriptName, bool notFoundOutErr = true); - static ScriptDefModule* findOldScriptModule(const char* scriptName, bool notFoundOutErr = true); - - static bool installScript(PyObject* mod); - static bool uninstallScript(); - - static const SCRIPT_MODULES& getScriptModules() { - return EntityDef::__scriptModules; - } - - static KBE_MD5& md5(){ return __md5; } - - static bool initializeWatcher(); - - static void entitydefAliasID(bool v) - { - __entitydefAliasID = v; - } - - static bool entitydefAliasID() - { - return __entitydefAliasID; - } - - static void entityAliasID(bool v) - { - __entityAliasID = v; - } - - static bool entityAliasID() - { - return __entityAliasID; - } - - static bool scriptModuleAliasID() - { - return __entitydefAliasID && __scriptModules.size() <= 255; - } - - struct Context - { - Context() - { - currEntityID = 0; - currClientappID = 0; - currComponentType = UNKNOWN_COMPONENT_TYPE; - } - - ENTITY_ID currEntityID; - COMPONENT_TYPE currComponentType; - int32 currClientappID; - }; - - static Context& context() { - return __context; - } - - static ScriptDefModule* registerNewScriptDefModule(const std::string& moduleName); - - static bool isReload(); - -private: - static SCRIPT_MODULES __scriptModules; // еչűģ鶼洢 - static SCRIPT_MODULES __oldScriptModules; // reloadʱɵģŵж - - static SCRIPT_MODULE_UID_MAP __scriptTypeMappingUType; // űӳutype - static SCRIPT_MODULE_UID_MAP __oldScriptTypeMappingUType; // reloadʱɵĽűӳutype - - static COMPONENT_TYPE __loadComponentType; // ϵ - static std::vector __scriptBaseTypes; - static std::string __entitiesPath; - - static KBE_MD5 __md5; // defs-md5 - - static bool _isInit; - - static bool __entityAliasID; // ŻEntityIDviewΧС255EntityID, 䵽clientʱʹ1ֽαID - static bool __entitydefAliasID; // ŻentityԺͷ㲥ʱռõĴentityͻԻ߿ͻ˲255ʱ uiduid䵽clientʱʹ1ֽڱID - - static GetEntityFunc __getEntityFunc; // һentityʵĺַ - - // õǰһЩ - static Context __context; -}; - -} - -#ifdef CODE_INLINE -#include "entitydef.inl" -#endif -#endif // KBE_ENTITYDEF_H - +// Copyright 2008-2018 Yolo Technologies, Inc. All Rights Reserved. https://www.comblockengine.com + + +#ifndef KBE_ENTITYDEF_H +#define KBE_ENTITYDEF_H + +#include "common/common.h" +#include "common/md5.h" +#if KBE_PLATFORM == PLATFORM_WIN32 +#pragma warning (disable : 4910) +#pragma warning (disable : 4251) +#endif + +#include "method.h" +#include "property.h" +#include "entity_call.h" +#include "math/math.h" +#include "pyscript/scriptobject.h" +#include "xml/xml.h" +#include "common/smartpointer.h" + + +namespace KBEngine{ + +class ScriptDefModule; +typedef SmartPointer ScriptDefModulePtr; + +class EntityDef +{ +public: + typedef std::vector SCRIPT_MODULES; + typedef std::map SCRIPT_MODULE_UID_MAP; + + typedef std::tr1::function GetEntityFunc; + typedef std::tr1::function FindChannelFunc; + + EntityDef(); + ~EntityDef(); + + /** + ʼ + */ + static bool initialize(std::vector& scriptBaseTypes, + COMPONENT_TYPE loadComponentType); + + static bool finalise(bool isReload = false); + + static void reload(bool fullReload); + + /** + ͨentityIDѰʵ + */ + static PyObject* tryGetEntity(COMPONENT_ID componentID, ENTITY_ID entityID); + + /** + entityCall__getEntityFuncַ + */ + static void setGetEntityFunc(GetEntityFunc func) { + __getEntityFunc = func; + }; + + /** + + */ + static bool loadAllEntityScriptModules(std::string entitiesPath, + std::vector& scriptBaseTypes); + + static bool loadAllComponentScriptModules(std::string entitiesPath, + std::vector& scriptBaseTypes); + + static bool loadAllDefDescriptions(const std::string& moduleName, + XML* defxml, + TiXmlNode* defNode, + ScriptDefModule* pScriptModule); + + static bool loadDefPropertys(const std::string& moduleName, + XML* xml, + TiXmlNode* defPropertyNode, + ScriptDefModule* pScriptModule); + + static bool calcDefPropertyUType(const std::string& moduleName, + const std::string& name, int iUtype, ScriptDefModule* pScriptModule, ENTITY_PROPERTY_UID& outUtype); + + static bool loadDefCellMethods(const std::string& moduleName, + XML* xml, + TiXmlNode* defMethodNode, + ScriptDefModule* pScriptModule); + + static bool loadDefBaseMethods(const std::string& moduleName, + XML* xml, + TiXmlNode* defMethodNode, + ScriptDefModule* pScriptModule); + + static bool loadDefClientMethods(const std::string& moduleName, + XML* xml, + TiXmlNode* defMethodNode, + ScriptDefModule* pScriptModule); + + static bool loadInterfaces(const std::string& defFilePath, + const std::string& moduleName, + XML* defxml, + TiXmlNode* defNode, + ScriptDefModule* pScriptModule, bool ignoreComponents = false); + + static bool loadComponents(const std::string& defFilePath, + const std::string& moduleName, + XML* defxml, + TiXmlNode* defNode, + ScriptDefModule* pScriptModule); + + static PropertyDescription* addComponentProperty(ENTITY_PROPERTY_UID utype, + const std::string& componentTypeName, + const std::string& componentName, + uint32 flags, + bool isPersistent, + bool isIdentifier, + std::string indexType, + uint32 databaseLength, + const std::string& defaultStr, + DETAIL_TYPE detailLevel, + ScriptDefModule* pScriptModule, + ScriptDefModule* pCompScriptDefModule); + + static bool loadParentClass(const std::string& defFilePath, + const std::string& moduleName, + XML* defxml, + TiXmlNode* defNode, + ScriptDefModule* pScriptModule); + + static bool loadDefInfo(const std::string& defFilePath, + const std::string& moduleName, + XML* defxml, + TiXmlNode* defNode, + ScriptDefModule* pScriptModule); + + static bool loadDetailLevelInfo(const std::string& defFilePath, + const std::string& moduleName, + XML* defxml, + TiXmlNode* defNode, + ScriptDefModule* pScriptModule); + + static bool loadVolatileInfo(const std::string& defFilePath, + const std::string& moduleName, + XML* defxml, + TiXmlNode* defNode, + ScriptDefModule* pScriptModule); + + static PyObject* loadScriptModule(std::string moduleName); + + /** + Ƿűģ + */ + static bool isLoadScriptModule(ScriptDefModule* pScriptModule); + + /** + ݵǰǷcell base + */ + static void setScriptModuleHasComponentEntity(ScriptDefModule* pScriptModule, bool has); + + /** + űģбķǷ + */ + static bool checkDefMethod(ScriptDefModule* pScriptModule, PyObject* moduleObj, + const std::string& moduleName); + + /** + űģбǷϷ + */ + static bool validDefPropertyName(const std::string& name); + + /** + ͨѰҵӦĽűģ + */ + static ScriptDefModule* findScriptModule(ENTITY_SCRIPT_UID utype, bool notFoundOutErr = true); + static ScriptDefModule* findScriptModule(const char* scriptName, bool notFoundOutErr = true); + static ScriptDefModule* findOldScriptModule(const char* scriptName, bool notFoundOutErr = true); + + static bool installScript(PyObject* mod); + static bool uninstallScript(); + + static const SCRIPT_MODULES& getScriptModules() { + return EntityDef::__scriptModules; + } + + static KBE_MD5& md5(){ return __md5; } + + static bool initializeWatcher(); + + static void entitydefAliasID(bool v) + { + __entitydefAliasID = v; + } + + static bool entitydefAliasID() + { + return __entitydefAliasID; + } + + static void entityAliasID(bool v) + { + __entityAliasID = v; + } + + static bool entityAliasID() + { + return __entityAliasID; + } + + static bool scriptModuleAliasID() + { + return __entitydefAliasID && __scriptModules.size() <= 255; + } + + struct Context + { + Context() + { + currEntityID = 0; + currClientappID = 0; + currComponentType = UNKNOWN_COMPONENT_TYPE; + } + + ENTITY_ID currEntityID; + COMPONENT_TYPE currComponentType; + int32 currClientappID; + }; + + static Context& context() { + return __context; + } + + static ScriptDefModule* registerNewScriptDefModule(const std::string& moduleName); + static MethodDescription* createMethodDescription(ScriptDefModule* pScriptModule, ENTITY_METHOD_UID utype, COMPONENT_ID domain, const std::string& name, MethodDescription::EXPOSED_TYPE exposedType); + + static bool isReload(); + +private: + static SCRIPT_MODULES __scriptModules; // еչűģ鶼洢 + static SCRIPT_MODULES __oldScriptModules; // reloadʱɵģŵж + + static SCRIPT_MODULE_UID_MAP __scriptTypeMappingUType; // űӳutype + static SCRIPT_MODULE_UID_MAP __oldScriptTypeMappingUType; // reloadʱɵĽűӳutype + + static COMPONENT_TYPE __loadComponentType; // ϵ + static std::vector __scriptBaseTypes; + static std::string __entitiesPath; + + static KBE_MD5 __md5; // defs-md5 + + static bool _isInit; + + static bool __entityAliasID; // ŻEntityIDviewΧС255EntityID, 䵽clientʱʹ1ֽαID + static bool __entitydefAliasID; // ŻentityԺͷ㲥ʱռõĴentityͻԻ߿ͻ˲255ʱ uiduid䵽clientʱʹ1ֽڱID + + static GetEntityFunc __getEntityFunc; // һentityʵĺַ + + // õǰһЩ + static Context __context; +}; + +} + +#ifdef CODE_INLINE +#include "entitydef.inl" +#endif +#endif // KBE_ENTITYDEF_H + diff --git a/kbe/src/lib/entitydef/method.cpp b/kbe/src/lib/entitydef/method.cpp index 3196f762cd..783278a378 100644 --- a/kbe/src/lib/entitydef/method.cpp +++ b/kbe/src/lib/entitydef/method.cpp @@ -1,258 +1,258 @@ -// Copyright 2008-2018 Yolo Technologies, Inc. All Rights Reserved. https://www.comblockengine.com - -#include "method.h" -#include "entitydef.h" -#include "network/bundle.h" - -#ifndef CODE_INLINE -#include "method.inl" -#endif - -namespace KBEngine{ - -uint32 MethodDescription::methodDescriptionCount_ = 0; - -//------------------------------------------------------------------------------------- -MethodDescription::MethodDescription(ENTITY_METHOD_UID utype, COMPONENT_ID domain, - std::string name, - bool isExposed): -methodDomain_(domain), -name_(name), -utype_(utype), -argTypes_(), -exposedType_(NO_EXPOSED), -aliasID_(-1) -{ - MethodDescription::methodDescriptionCount_++; - - EntityDef::md5().append((void*)name_.c_str(), (int)name_.size()); - EntityDef::md5().append((void*)&utype_, sizeof(ENTITY_METHOD_UID)); - EntityDef::md5().append((void*)&exposedType_, sizeof(EXPOSED_TYPE)); -} - -//------------------------------------------------------------------------------------- -MethodDescription::~MethodDescription() -{ - std::vector::iterator iter = argTypes_.begin(); - for(; iter != argTypes_.end(); ++iter) - (*iter)->decRef(); - - argTypes_.clear(); -} - -//------------------------------------------------------------------------------------- -void MethodDescription::setExposed(EXPOSED_TYPE type) -{ - // ֻܱΪһEXPOSED_AND_CALLER_CHECK - KBE_ASSERT(exposedType_ != EXPOSED_AND_CALLER_CHECK); - - exposedType_ = type; - EntityDef::md5().append((void*)&exposedType_, sizeof(EXPOSED_TYPE)); -} - -//------------------------------------------------------------------------------------- -bool MethodDescription::pushArgType(DataType* dataType) -{ - if(dataType == NULL) - { - ERROR_MSG("MethodDescription::pushArgType: dataType is NULL!\n"); - return false; - } - - dataType->incRef(); - argTypes_.push_back(dataType); - - DATATYPE_UID uid = dataType->id(); - EntityDef::md5().append((void*)&uid, sizeof(DATATYPE_UID)); - EntityDef::md5().append((void*)&exposedType_, sizeof(EXPOSED_TYPE)); - return true; -} - -//------------------------------------------------------------------------------------- -bool MethodDescription::checkArgs(PyObject* args) -{ - if (args == NULL || !PyTuple_Check(args)) - { - PyErr_Format(PyExc_AssertionError, "Method::checkArgs: method[%s] args is not a tuple.\n", - getName()); - - PyErr_PrintEx(0); - return false; - } - - int offset = (isExposed() == EXPOSED_AND_CALLER_CHECK && g_componentType == CELLAPP_TYPE && isCell()) ? 1 : 0; - uint8 argsSize = (uint8)argTypes_.size(); - uint8 giveArgsSize = (uint8)PyTuple_Size(args); - - if (giveArgsSize != argsSize + offset) - { - PyErr_Format(PyExc_AssertionError, "Method::checkArgs: method[%s] requires exactly %d argument%s%s; %d given", - getName(), - argsSize, - (offset > 0) ? " + exposed(1)" : "", - (argsSize == 1) ? "" : "s", - PyTuple_Size(args)); - - PyErr_PrintEx(0); - return false; - } - - - // Ƿһexposed - if(offset > 0) - { - PyObject* pyExposed = PyTuple_GetItem(args, 0); - if (!PyLong_Check(pyExposed)) - { - PyObject* pyeid = PyObject_GetAttrString(pyExposed, "id"); - if (pyeid == NULL || !PyLong_Check(pyeid)) - { - Py_XDECREF(pyeid); - PyErr_Format( PyExc_TypeError, - "Method::checkArgs: method[%s] requires None, an id, or an object with an " - "id as its first agument", getName()); - - PyErr_PrintEx(0); - return false; - } - - Py_DECREF(pyeid); - } - } - - for(uint8 i=0; i isSameType(pyArg)) - { - PyObject* pExample = argTypes_[i]->parseDefaultStr(""); - PyErr_Format(PyExc_AssertionError, - "Method::checkArgs: method[%s] argument %d: Expected %s, %s found", - getName(), - i+1, - pExample->ob_type->tp_name, - pyArg != NULL ? pyArg->ob_type->tp_name : "NULL"); - - PyErr_PrintEx(0); - Py_DECREF(pExample); - return false; - } - } - - return true; -} - -//------------------------------------------------------------------------------------- -void MethodDescription::addToStream(MemoryStream* mstream, PyObject* args) -{ - uint8 argsSize = argTypes_.size(); - int offset = 0; - - // utypeŽȥԶʶ - // aliasID_0һŻİ취 ʹ1ֽڴ - // ע⣺ڼdefʱָ˿ͻ˷aliasID˷ڲʹaliasID - if(aliasID_ <= 0) - { - (*mstream) << utype_; - } - else - { - uint8 utype = (uint8)aliasID_; - (*mstream) << utype; - } - - // exposedȽentityIDȥ - if(isExposed() == EXPOSED_AND_CALLER_CHECK && g_componentType == CELLAPP_TYPE && isCell()) - { - offset = 1; - } - - // ÿһӵ - for(uint8 i=0; i addToStream(mstream, pyArg); - } -} - -//------------------------------------------------------------------------------------- -PyObject* MethodDescription::createFromStream(MemoryStream* mstream) -{ - size_t argSize = getArgSize(); - PyObject* pyArgsTuple = NULL; - int offset = 0; - - if(isExposed() == EXPOSED_AND_CALLER_CHECK && g_componentType == CELLAPP_TYPE && isCell()) - { - offset = 1; - pyArgsTuple = PyTuple_New(argSize + offset); - - // һIDṩűжԴǷȷ - KBE_ASSERT(EntityDef::context().currEntityID > 0); - PyTuple_SET_ITEM(pyArgsTuple, 0, PyLong_FromLong(EntityDef::context().currEntityID)); - } - else - pyArgsTuple = PyTuple_New(argSize); - - for(size_t index=0; indexcreateFromStream(mstream); - - if(pyitem == NULL) - { - WARNING_MSG(fmt::format("MethodDescription::createFromStream: {} arg[{}][{}] is NULL.\n", - this->getName(), index, argTypes_[index]->getName())); - } - - PyTuple_SET_ITEM(pyArgsTuple, index + offset, pyitem); - } - - return pyArgsTuple; -} - -//------------------------------------------------------------------------------------- -size_t MethodDescription::getArgSize(void) -{ - return argTypes_.size(); -} - -//------------------------------------------------------------------------------------- -PyObject* MethodDescription::call(PyObject* func, PyObject* args) -{ - PyObject* pyResult = NULL; - if (!PyCallable_Check(func)) - { - PyErr_Format(PyExc_TypeError, "MethodDescription::call: method[%s] call attempted on a error object!", - getName()); - } - else - { - if(args == NULL) - { - pyResult = PyObject_CallObject(func, NULL); - } - else - { - if(checkArgs(args)) - pyResult = PyObject_CallObject(func, args); - } - } - - if (PyErr_Occurred()) - { - if (isExposed() == EXPOSED_AND_CALLER_CHECK && PyErr_ExceptionMatches(PyExc_TypeError)) - { - WARNING_MSG(fmt::format("MethodDescription::call: {} is exposed of method, if there is a missing arguments error, " - "try adding callerEntityID, For example: \ndef func(msg): => def func(callerEntityID, msg):\n", - this->getName())); - } - - PyErr_PrintEx(0); - } - - return pyResult; -} - -//------------------------------------------------------------------------------------- - -} +// Copyright 2008-2018 Yolo Technologies, Inc. All Rights Reserved. https://www.comblockengine.com + +#include "method.h" +#include "entitydef.h" +#include "network/bundle.h" + +#ifndef CODE_INLINE +#include "method.inl" +#endif + +namespace KBEngine{ + +uint32 MethodDescription::methodDescriptionCount_ = 0; + +//------------------------------------------------------------------------------------- +MethodDescription::MethodDescription(ENTITY_METHOD_UID utype, COMPONENT_ID domain, + std::string name, + EXPOSED_TYPE exposedType) : +methodDomain_(domain), +name_(name), +utype_(utype), +argTypes_(), +exposedType_(exposedType), +aliasID_(-1) +{ + MethodDescription::methodDescriptionCount_++; + + EntityDef::md5().append((void*)name_.c_str(), (int)name_.size()); + EntityDef::md5().append((void*)&utype_, sizeof(ENTITY_METHOD_UID)); + EntityDef::md5().append((void*)&exposedType_, sizeof(EXPOSED_TYPE)); +} + +//------------------------------------------------------------------------------------- +MethodDescription::~MethodDescription() +{ + std::vector::iterator iter = argTypes_.begin(); + for(; iter != argTypes_.end(); ++iter) + (*iter)->decRef(); + + argTypes_.clear(); +} + +//------------------------------------------------------------------------------------- +void MethodDescription::setExposed(EXPOSED_TYPE type) +{ + if (exposedType_ == EXPOSED_AND_CALLER_CHECK) + return; + + exposedType_ = type; + EntityDef::md5().append((void*)&exposedType_, sizeof(EXPOSED_TYPE)); +} + +//------------------------------------------------------------------------------------- +bool MethodDescription::pushArgType(DataType* dataType) +{ + if(dataType == NULL) + { + ERROR_MSG("MethodDescription::pushArgType: dataType is NULL!\n"); + return false; + } + + dataType->incRef(); + argTypes_.push_back(dataType); + + DATATYPE_UID uid = dataType->id(); + EntityDef::md5().append((void*)&uid, sizeof(DATATYPE_UID)); + EntityDef::md5().append((void*)&exposedType_, sizeof(EXPOSED_TYPE)); + return true; +} + +//------------------------------------------------------------------------------------- +bool MethodDescription::checkArgs(PyObject* args) +{ + if (args == NULL || !PyTuple_Check(args)) + { + PyErr_Format(PyExc_AssertionError, "Method::checkArgs: method[%s] args is not a tuple.\n", + getName()); + + PyErr_PrintEx(0); + return false; + } + + int offset = (isExposed() == EXPOSED_AND_CALLER_CHECK && g_componentType == CELLAPP_TYPE && isCell()) ? 1 : 0; + uint8 argsSize = (uint8)argTypes_.size(); + uint8 giveArgsSize = (uint8)PyTuple_Size(args); + + if (giveArgsSize != argsSize + offset) + { + PyErr_Format(PyExc_AssertionError, "Method::checkArgs: method[%s] requires exactly %d argument%s%s; %d given", + getName(), + argsSize, + (offset > 0) ? " + exposed(1)" : "", + (argsSize == 1) ? "" : "s", + PyTuple_Size(args)); + + PyErr_PrintEx(0); + return false; + } + + + // Ƿһexposed + if(offset > 0) + { + PyObject* pyExposed = PyTuple_GetItem(args, 0); + if (!PyLong_Check(pyExposed)) + { + PyObject* pyeid = PyObject_GetAttrString(pyExposed, "id"); + if (pyeid == NULL || !PyLong_Check(pyeid)) + { + Py_XDECREF(pyeid); + PyErr_Format( PyExc_TypeError, + "Method::checkArgs: method[%s] requires None, an id, or an object with an " + "id as its first agument", getName()); + + PyErr_PrintEx(0); + return false; + } + + Py_DECREF(pyeid); + } + } + + for(uint8 i=0; i isSameType(pyArg)) + { + PyObject* pExample = argTypes_[i]->parseDefaultStr(""); + PyErr_Format(PyExc_AssertionError, + "Method::checkArgs: method[%s] argument %d: Expected %s, %s found", + getName(), + i+1, + pExample->ob_type->tp_name, + pyArg != NULL ? pyArg->ob_type->tp_name : "NULL"); + + PyErr_PrintEx(0); + Py_DECREF(pExample); + return false; + } + } + + return true; +} + +//------------------------------------------------------------------------------------- +void MethodDescription::addToStream(MemoryStream* mstream, PyObject* args) +{ + uint8 argsSize = argTypes_.size(); + int offset = 0; + + // utypeŽȥԶʶ + // aliasID_0һŻİ취 ʹ1ֽڴ + // ע⣺ڼdefʱָ˿ͻ˷aliasID˷ڲʹaliasID + if(aliasID_ <= 0) + { + (*mstream) << utype_; + } + else + { + uint8 utype = (uint8)aliasID_; + (*mstream) << utype; + } + + // exposedȽentityIDȥ + if(isExposed() == EXPOSED_AND_CALLER_CHECK && g_componentType == CELLAPP_TYPE && isCell()) + { + offset = 1; + } + + // ÿһӵ + for(uint8 i=0; i addToStream(mstream, pyArg); + } +} + +//------------------------------------------------------------------------------------- +PyObject* MethodDescription::createFromStream(MemoryStream* mstream) +{ + size_t argSize = getArgSize(); + PyObject* pyArgsTuple = NULL; + int offset = 0; + + if(isExposed() == EXPOSED_AND_CALLER_CHECK && g_componentType == CELLAPP_TYPE && isCell()) + { + offset = 1; + pyArgsTuple = PyTuple_New(argSize + offset); + + // һIDṩűжԴǷȷ + KBE_ASSERT(EntityDef::context().currEntityID > 0); + PyTuple_SET_ITEM(pyArgsTuple, 0, PyLong_FromLong(EntityDef::context().currEntityID)); + } + else + pyArgsTuple = PyTuple_New(argSize); + + for(size_t index=0; indexcreateFromStream(mstream); + + if(pyitem == NULL) + { + WARNING_MSG(fmt::format("MethodDescription::createFromStream: {} arg[{}][{}] is NULL.\n", + this->getName(), index, argTypes_[index]->getName())); + } + + PyTuple_SET_ITEM(pyArgsTuple, index + offset, pyitem); + } + + return pyArgsTuple; +} + +//------------------------------------------------------------------------------------- +size_t MethodDescription::getArgSize(void) +{ + return argTypes_.size(); +} + +//------------------------------------------------------------------------------------- +PyObject* MethodDescription::call(PyObject* func, PyObject* args) +{ + PyObject* pyResult = NULL; + if (!PyCallable_Check(func)) + { + PyErr_Format(PyExc_TypeError, "MethodDescription::call: method[%s] call attempted on a error object!", + getName()); + } + else + { + if(args == NULL) + { + pyResult = PyObject_CallObject(func, NULL); + } + else + { + if(checkArgs(args)) + pyResult = PyObject_CallObject(func, args); + } + } + + if (PyErr_Occurred()) + { + if (isExposed() == EXPOSED_AND_CALLER_CHECK && PyErr_ExceptionMatches(PyExc_TypeError)) + { + WARNING_MSG(fmt::format("MethodDescription::call: {} is exposed of method, if there is a missing arguments error, " + "try adding callerEntityID, For example: \ndef func(msg): => def func(callerEntityID, msg):\n", + this->getName())); + } + + PyErr_PrintEx(0); + } + + return pyResult; +} + +//------------------------------------------------------------------------------------- + +} diff --git a/kbe/src/lib/entitydef/method.h b/kbe/src/lib/entitydef/method.h index c4ab058688..33ea25c148 100644 --- a/kbe/src/lib/entitydef/method.h +++ b/kbe/src/lib/entitydef/method.h @@ -1,119 +1,119 @@ -// Copyright 2008-2018 Yolo Technologies, Inc. All Rights Reserved. https://www.comblockengine.com - - -#ifndef KBENGINE_DEF_METHOD_H -#define KBENGINE_DEF_METHOD_H - -#include "common/common.h" -#if KBE_PLATFORM == PLATFORM_WIN32 -#pragma warning (disable : 4910) -#pragma warning (disable : 4251) -#endif - -#include "datatype.h" -#include "datatypes.h" -#include "helper/debug_helper.h" -#include "network/packet.h" -#include "entitycallabstract.h" -#include "pyscript/scriptobject.h" - - -namespace KBEngine{ - -class MethodDescription -{ -public: - // ¶ - enum EXPOSED_TYPE - { - // ĬϣDZ¶ - NO_EXPOSED = 0, - - // ĬϣűԲӵ߲ - EXPOSED = 1, - - // űһΪIDṩűߺϷ - EXPOSED_AND_CALLER_CHECK = 2 - }; - -public: - MethodDescription(ENTITY_METHOD_UID utype, COMPONENT_ID domain, - std::string name, - bool isExposed = false); - - virtual ~MethodDescription(); - - INLINE const char* getName(void) const; - - INLINE ENTITY_METHOD_UID getUType(void) const; - INLINE void setUType(ENTITY_METHOD_UID muid); - - static uint32 getDescriptionCount(void){ return methodDescriptionCount_; } - static void resetDescriptionCount(void){ methodDescriptionCount_ = 0; } - - INLINE EXPOSED_TYPE isExposed(void) const; - - void setExposed(EXPOSED_TYPE type = EXPOSED); - - bool pushArgType(DataType* dataType); - - INLINE std::vector& getArgTypes(void); - - size_t getArgSize(void); - - /** - һcallǷϷ - */ - bool checkArgs(PyObject* args); - - /** - ÿӵ - ϢڽűʱﴫIJ - */ - void addToStream(MemoryStream* mstream, PyObject* args); - - /** - һcall һPyObject͵args - */ - PyObject* createFromStream(MemoryStream* mstream); - - /** - һ - */ - PyObject* call(PyObject* func, PyObject* args); - - INLINE COMPONENT_ID domain() const; - - INLINE bool isClient() const; - INLINE bool isCell() const; - INLINE bool isBase() const; - - /** - id ¶ķ߹㲥ܸС255ʱ - Dzʹutypeʹ1ֽڵaliasID - */ - INLINE int16 aliasID() const; - INLINE uint8 aliasIDAsUint8() const; - INLINE void aliasID(int16 v); - -protected: - static uint32 methodDescriptionCount_; // е - - COMPONENT_ID methodDomain_; - - std::string name_; // - ENTITY_METHOD_UID utype_; // ϴʶ - - std::vector argTypes_; // ԵIJб - - EXPOSED_TYPE exposedType_; // Ƿһ¶ - - int16 aliasID_; // id ¶ķ߹㲥ܸС255ʱ Dzʹutypeʹ1ֽڵaliasID -}; - -} - -#ifdef CODE_INLINE -#include "method.inl" -#endif -#endif // KBENGINE_DEF_METHOD_H +// Copyright 2008-2018 Yolo Technologies, Inc. All Rights Reserved. https://www.comblockengine.com + + +#ifndef KBENGINE_DEF_METHOD_H +#define KBENGINE_DEF_METHOD_H + +#include "common/common.h" +#if KBE_PLATFORM == PLATFORM_WIN32 +#pragma warning (disable : 4910) +#pragma warning (disable : 4251) +#endif + +#include "datatype.h" +#include "datatypes.h" +#include "helper/debug_helper.h" +#include "network/packet.h" +#include "entitycallabstract.h" +#include "pyscript/scriptobject.h" + + +namespace KBEngine{ + +class MethodDescription +{ +public: + // ¶ + enum EXPOSED_TYPE + { + // ĬϣDZ¶ + NO_EXPOSED = 0, + + // ĬϣűԲӵ߲ + EXPOSED = 1, + + // űһΪIDṩűߺϷ + EXPOSED_AND_CALLER_CHECK = 2 + }; + +public: + MethodDescription(ENTITY_METHOD_UID utype, COMPONENT_ID domain, + std::string name, + EXPOSED_TYPE exposedType = NO_EXPOSED); + + virtual ~MethodDescription(); + + INLINE const char* getName(void) const; + + INLINE ENTITY_METHOD_UID getUType(void) const; + INLINE void setUType(ENTITY_METHOD_UID muid); + + static uint32 getDescriptionCount(void){ return methodDescriptionCount_; } + static void resetDescriptionCount(void){ methodDescriptionCount_ = 0; } + + INLINE EXPOSED_TYPE isExposed(void) const; + + void setExposed(EXPOSED_TYPE type = EXPOSED); + + bool pushArgType(DataType* dataType); + + INLINE std::vector& getArgTypes(void); + + size_t getArgSize(void); + + /** + һcallǷϷ + */ + bool checkArgs(PyObject* args); + + /** + ÿӵ + ϢڽűʱﴫIJ + */ + void addToStream(MemoryStream* mstream, PyObject* args); + + /** + һcall һPyObject͵args + */ + PyObject* createFromStream(MemoryStream* mstream); + + /** + һ + */ + PyObject* call(PyObject* func, PyObject* args); + + INLINE COMPONENT_ID domain() const; + + INLINE bool isClient() const; + INLINE bool isCell() const; + INLINE bool isBase() const; + + /** + id ¶ķ߹㲥ܸС255ʱ + Dzʹutypeʹ1ֽڵaliasID + */ + INLINE int16 aliasID() const; + INLINE uint8 aliasIDAsUint8() const; + INLINE void aliasID(int16 v); + +protected: + static uint32 methodDescriptionCount_; // е + + COMPONENT_ID methodDomain_; + + std::string name_; // + ENTITY_METHOD_UID utype_; // ϴʶ + + std::vector argTypes_; // ԵIJб + + EXPOSED_TYPE exposedType_; // Ƿһ¶ + + int16 aliasID_; // id ¶ķ߹㲥ܸС255ʱ Dzʹutypeʹ1ֽڵaliasID +}; + +} + +#ifdef CODE_INLINE +#include "method.inl" +#endif +#endif // KBENGINE_DEF_METHOD_H diff --git a/kbe/src/lib/entitydef/py_entitydef.cpp b/kbe/src/lib/entitydef/py_entitydef.cpp index f54396e8c7..acf64ac628 100644 --- a/kbe/src/lib/entitydef/py_entitydef.cpp +++ b/kbe/src/lib/entitydef/py_entitydef.cpp @@ -1,2360 +1,2782 @@ -// Copyright 2008-2018 Yolo Technologies, Inc. All Rights Reserved. https://www.comblockengine.com - -#include -#include -#include - -#include "common.h" -#include "entitydef.h" -#include "datatypes.h" -#include "py_entitydef.h" -#include "scriptdef_module.h" -#include "pyscript/py_platform.h" -#include "pyscript/script.h" -#include "pyscript/copy.h" -#include "resmgr/resmgr.h" - -namespace KBEngine{ namespace script{ namespace entitydef { - -struct CallContext -{ - PyObjectPtr pyArgs; - PyObjectPtr pyKwargs; - std::string optionName; -}; - -static std::stack g_callContexts; -static std::string pyDefModuleName = ""; - -DefContext::DEF_CONTEXT_MAP DefContext::allScriptDefContextMaps; -DefContext::DEF_CONTEXT_MAP DefContext::allScriptDefContextLineMaps; - -static bool g_inited = false; - -//------------------------------------------------------------------------------------- -static PyObject* __py_array(PyObject* self, PyObject* args) -{ - if (PyTuple_GET_SIZE(args) == 0) - { - PyErr_Format(PyExc_AssertionError, "EntityDef.ARRAY: does not set itemType! should be like this \"EntityDef.ARRAY(itemType)\"\n"); - return NULL; - } - - PyObject *entitydefModule = PyImport_AddModule(pyDefModuleName.c_str()); - PyObject* pyARRAY = PyObject_GetAttrString(entitydefModule, "ARRAY"); - - PyObject* pyArrayItemType = PyTuple_GET_ITEM(args, 0); - Py_INCREF(pyArrayItemType); - - PyObject* ret = PyTuple_New(2); - PyTuple_SET_ITEM(ret, 0, pyARRAY); - PyTuple_SET_ITEM(ret, 1, pyArrayItemType); - - return ret; -} - -//------------------------------------------------------------------------------------- -class Entity : public script::ScriptObject -{ - BASE_SCRIPT_HREADER(Entity, ScriptObject) -public: - Entity(PyTypeObject* pyType = getScriptType(), bool isInitialised = true) : - ScriptObject(pyType, isInitialised) {} - ~Entity() {} -}; - -SCRIPT_METHOD_DECLARE_BEGIN(Entity) -SCRIPT_METHOD_DECLARE_END() - -SCRIPT_MEMBER_DECLARE_BEGIN(Entity) -SCRIPT_MEMBER_DECLARE_END() - -SCRIPT_GETSET_DECLARE_BEGIN(Entity) -SCRIPT_GETSET_DECLARE_END() -BASE_SCRIPT_INIT(Entity, 0, 0, 0, 0, 0) - -//------------------------------------------------------------------------------------- -class Space : public script::ScriptObject -{ - BASE_SCRIPT_HREADER(Space, ScriptObject) -public: - Space(PyTypeObject* pyType = getScriptType(), bool isInitialised = true) : - ScriptObject(pyType, isInitialised) {} - ~Space() {} -}; - -SCRIPT_METHOD_DECLARE_BEGIN(Space) -SCRIPT_METHOD_DECLARE_END() - -SCRIPT_MEMBER_DECLARE_BEGIN(Space) -SCRIPT_MEMBER_DECLARE_END() - -SCRIPT_GETSET_DECLARE_BEGIN(Space) -SCRIPT_GETSET_DECLARE_END() -BASE_SCRIPT_INIT(Space, 0, 0, 0, 0, 0) - -//------------------------------------------------------------------------------------- -class Proxy : public script::ScriptObject -{ - BASE_SCRIPT_HREADER(Proxy, ScriptObject) -public: - Proxy(PyTypeObject* pyType = getScriptType(), bool isInitialised = true) : - ScriptObject(pyType, isInitialised) {} - ~Proxy() {} -}; - -SCRIPT_METHOD_DECLARE_BEGIN(Proxy) -SCRIPT_METHOD_DECLARE_END() - -SCRIPT_MEMBER_DECLARE_BEGIN(Proxy) -SCRIPT_MEMBER_DECLARE_END() - -SCRIPT_GETSET_DECLARE_BEGIN(Proxy) -SCRIPT_GETSET_DECLARE_END() -BASE_SCRIPT_INIT(Proxy, 0, 0, 0, 0, 0) - -//------------------------------------------------------------------------------------- -class EntityComponent : public script::ScriptObject -{ - BASE_SCRIPT_HREADER(EntityComponent, ScriptObject) -public: - EntityComponent(PyTypeObject* pyType = getScriptType(), bool isInitialised = true) : - ScriptObject(pyType, isInitialised) {} - ~EntityComponent() {} -}; - -SCRIPT_METHOD_DECLARE_BEGIN(EntityComponent) -SCRIPT_METHOD_DECLARE_END() - -SCRIPT_MEMBER_DECLARE_BEGIN(EntityComponent) -SCRIPT_MEMBER_DECLARE_END() - -SCRIPT_GETSET_DECLARE_BEGIN(EntityComponent) -SCRIPT_GETSET_DECLARE_END() -BASE_SCRIPT_INIT(EntityComponent, 0, 0, 0, 0, 0) - -//------------------------------------------------------------------------------------- -DefContext::DefContext() -{ - optionName = ""; - - moduleName = ""; - attrName = ""; - methodArgs = ""; - returnType = ""; - - isModuleScope = false; - - exposed = false; - hasClient = false; - persistent = -1; - databaseLength = 0; - utype = -1; - detailLevel = ""; - - propertyFlags = ""; - propertyIndex = ""; - propertyDefaultVal = ""; - - implementedByModuleName = ""; - implementedByModuleFile = ""; - pyObjectSourceFile = ""; - - inheritEngineModuleType = DC_TYPE_UNKNOWN; - type = DC_TYPE_UNKNOWN; - - componentType = UNKNOWN_COMPONENT_TYPE; -} - -//------------------------------------------------------------------------------------- -bool DefContext::addToStream(MemoryStream* pMemoryStream) -{ - (*pMemoryStream) << optionName; - (*pMemoryStream) << moduleName; - (*pMemoryStream) << attrName; - (*pMemoryStream) << methodArgs; - (*pMemoryStream) << returnType; - - (*pMemoryStream) << (int)argsvecs.size(); - std::vector< std::string >::iterator argsvecsIter = argsvecs.begin(); - for(; argsvecsIter != argsvecs.end(); ++argsvecsIter) - (*pMemoryStream) << (*argsvecsIter); - - (*pMemoryStream) << (int)annotationsMaps.size(); - std::map< std::string, std::string >::iterator annotationsMapsIter = annotationsMaps.begin(); - for (; annotationsMapsIter != annotationsMaps.end(); ++annotationsMapsIter) - (*pMemoryStream) << annotationsMapsIter->first << annotationsMapsIter->second; - - (*pMemoryStream) << isModuleScope; - (*pMemoryStream) << exposed; - (*pMemoryStream) << hasClient; - - (*pMemoryStream) << persistent; - (*pMemoryStream) << databaseLength; - (*pMemoryStream) << utype; - (*pMemoryStream) << detailLevel; - - (*pMemoryStream) << propertyFlags; - (*pMemoryStream) << propertyIndex; - (*pMemoryStream) << propertyDefaultVal; - - (*pMemoryStream) << implementedByModuleName; - (*pMemoryStream) << implementedByModuleFile; - (*pMemoryStream) << pyObjectSourceFile; - - (*pMemoryStream) << (int)baseClasses.size(); - std::vector< std::string >::iterator baseClassesIter = baseClasses.begin(); - for (; baseClassesIter != baseClasses.end(); ++baseClassesIter) - (*pMemoryStream) << (*baseClassesIter); - - (*pMemoryStream) << (int)inheritEngineModuleType; - (*pMemoryStream) << (int)type; - - (*pMemoryStream) << (int)methods.size(); - std::vector< DefContext >::iterator methodsIter = methods.begin(); - for (; methodsIter != methods.end(); ++methodsIter) - (*methodsIter).addToStream(pMemoryStream); - - (*pMemoryStream) << (int)client_methods.size(); - std::vector< DefContext >::iterator client_methodsIter = client_methods.begin(); - for (; client_methodsIter != client_methods.end(); ++client_methodsIter) - (*client_methodsIter).addToStream(pMemoryStream); - - (*pMemoryStream) << (int)propertys.size(); - std::vector< DefContext >::iterator propertysIter = propertys.begin(); - for (; propertysIter != propertys.end(); ++propertysIter) - (*propertysIter).addToStream(pMemoryStream); - - (*pMemoryStream) << (int)components.size(); - std::vector< std::string >::iterator componentsIter = components.begin(); - for (; componentsIter != components.end(); ++componentsIter) - (*pMemoryStream) << (*componentsIter); - - return true; -} - -//------------------------------------------------------------------------------------- -bool DefContext::createFromStream(MemoryStream* pMemoryStream) -{ - (*pMemoryStream) >> optionName; - (*pMemoryStream) >> moduleName; - (*pMemoryStream) >> attrName; - (*pMemoryStream) >> methodArgs; - (*pMemoryStream) >> returnType; - - int size = 0; - - (*pMemoryStream) >> size; - for (int i = 0; i < size; ++i) - { - std::string str; - (*pMemoryStream) >> str; - - argsvecs.push_back(str); - } - - (*pMemoryStream) >> size; - for (int i = 0; i < size; ++i) - { - std::string key, val; - (*pMemoryStream) >> key >> val; - - annotationsMaps[key] = val; - } - - (*pMemoryStream) >> isModuleScope; - (*pMemoryStream) >> exposed; - (*pMemoryStream) >> hasClient; - - (*pMemoryStream) >> persistent; - (*pMemoryStream) >> databaseLength; - (*pMemoryStream) >> utype; - (*pMemoryStream) >> detailLevel; - - (*pMemoryStream) >> propertyFlags; - (*pMemoryStream) >> propertyIndex; - (*pMemoryStream) >> propertyDefaultVal; - - (*pMemoryStream) >> implementedByModuleName; - (*pMemoryStream) >> implementedByModuleFile; - (*pMemoryStream) >> pyObjectSourceFile; - - (*pMemoryStream) >> size; - for (int i = 0; i < size; ++i) - { - std::string str; - (*pMemoryStream) >> str; - - baseClasses.push_back(str); - } - - int t_inheritEngineModuleType; - (*pMemoryStream) >> t_inheritEngineModuleType; - inheritEngineModuleType = (DCType)t_inheritEngineModuleType; - - int t_type; - (*pMemoryStream) >> t_type; - type = (DCType)t_type; - - (*pMemoryStream) >> size; - for (int i = 0; i < size; ++i) - { - DefContext dc; - dc.createFromStream(pMemoryStream); - - methods.push_back(dc); - } - - (*pMemoryStream) >> size; - for (int i = 0; i < size; ++i) - { - DefContext dc; - dc.createFromStream(pMemoryStream); - - client_methods.push_back(dc); - } - - (*pMemoryStream) >> size; - for (int i = 0; i < size; ++i) - { - DefContext dc; - dc.createFromStream(pMemoryStream); - - propertys.push_back(dc); - } - - (*pMemoryStream) >> size; - for (int i = 0; i < size; ++i) - { - std::string str; - (*pMemoryStream) >> str; - - components.push_back(str); - } - - return true; -} - -//------------------------------------------------------------------------------------- -bool DefContext::addChildContext(DefContext& defContext) -{ - std::vector< DefContext >* pContexts = NULL; - - if (defContext.type == DefContext::DC_TYPE_PROPERTY) - pContexts = &propertys; - else if (defContext.type == DefContext::DC_TYPE_METHOD) - pContexts = &methods; - else if (defContext.type == DefContext::DC_TYPE_CLIENT_METHOD) - pContexts = &client_methods; - else if (defContext.type == DefContext::DC_TYPE_FIXED_ITEM) - pContexts = &propertys; - else - KBE_ASSERT(false); - - std::vector< DefContext >::iterator iter = pContexts->begin(); - for (; iter != pContexts->end(); ++iter) - { - if ((*iter).attrName == defContext.attrName) - { - // assemblyContextsʱܻᷢ - // - if (moduleName != defContext.moduleName || (*iter).pyObjectSourceFile == defContext.pyObjectSourceFile) - return true; - - return false; - } - } - - pContexts->push_back(defContext); - return true; -} - -//------------------------------------------------------------------------------------- -static bool assemblyContexts(bool notfoundModuleError = false) -{ - std::vector< std::string > dels; - - DefContext::DEF_CONTEXT_MAP::iterator iter = DefContext::allScriptDefContextMaps.begin(); - for (; iter != DefContext::allScriptDefContextMaps.end(); ++iter) - { - DefContext& defContext = iter->second; - - if (defContext.type == DefContext::DC_TYPE_PROPERTY || - defContext.type == DefContext::DC_TYPE_METHOD || - defContext.type == DefContext::DC_TYPE_CLIENT_METHOD || - defContext.type == DefContext::DC_TYPE_FIXED_ITEM) - { - DefContext::DEF_CONTEXT_MAP::iterator fiter = DefContext::allScriptDefContextMaps.find(defContext.moduleName); - if (fiter == DefContext::allScriptDefContextMaps.end()) - { - if (notfoundModuleError) - { - PyErr_Format(PyExc_AssertionError, "PyEntityDef::process(): No \'%s\' module defined!\n", defContext.moduleName.c_str()); - return false; - } - - return true; - } - - if (!fiter->second.addChildContext(defContext)) - { - PyErr_Format(PyExc_AssertionError, "\'%s.%s\' already exists!\n", - fiter->second.moduleName.c_str(), defContext.attrName.c_str()); - - return false; - } - - dels.push_back(iter->first); - } - else - { - } - } - - std::vector< std::string >::iterator diter = dels.begin(); - for (; diter != dels.end(); ++diter) - { - DefContext::allScriptDefContextMaps.erase((*diter)); - } - - // ԽϢ䵽 - iter = DefContext::allScriptDefContextMaps.begin(); - for (; iter != DefContext::allScriptDefContextMaps.end(); ++iter) - { - DefContext& defContext = iter->second; - if (defContext.baseClasses.size() > 0) - { - for (size_t i = 0; i < defContext.baseClasses.size(); ++i) - { - std::string parentClass = defContext.baseClasses[i]; - - DefContext::DEF_CONTEXT_MAP::iterator fiter = DefContext::allScriptDefContextMaps.find(parentClass); - if (fiter == DefContext::allScriptDefContextMaps.end()) - { - //PyErr_Format(PyExc_AssertionError, "not found parentClass(\'%s\')!\n", parentClass.c_str()); - //return false; - continue; - } - - DefContext& parentDefContext = fiter->second; - std::vector< DefContext > childContexts; - - childContexts.insert(childContexts.end(), parentDefContext.methods.begin(), parentDefContext.methods.end()); - childContexts.insert(childContexts.end(), parentDefContext.client_methods.begin(), parentDefContext.client_methods.end()); - childContexts.insert(childContexts.end(), parentDefContext.propertys.begin(), parentDefContext.propertys.end()); - - std::vector< DefContext >::iterator itemIter = childContexts.begin(); - for (; itemIter != childContexts.end(); ++itemIter) - { - DefContext& parentDefContext = (*itemIter); - if (!defContext.addChildContext(parentDefContext)) - { - PyErr_Format(PyExc_AssertionError, "\'%s.%s\' already exists(%s.%s)!\n", - defContext.moduleName.c_str(), parentDefContext.attrName.c_str(), parentDefContext.moduleName.c_str(), parentDefContext.attrName.c_str()); - - return false; - } - } - } - } - } - - return true; -} - -//------------------------------------------------------------------------------------- -static bool registerDefContext(DefContext& defContext) -{ - DefContext::allScriptDefContextLineMaps[defContext.pyObjectSourceFile] = defContext; - - std::string name = defContext.moduleName; - - if (defContext.type == DefContext::DC_TYPE_PROPERTY) - { - if(!EntityDef::validDefPropertyName(name)) - { - PyErr_Format(PyExc_AssertionError, "EntityDef.%s: '%s' is limited!\n\n", - defContext.optionName.c_str(), name.c_str()); - - return false; - } - - // Ƿڸý - bool flagsGood = true; - if (defContext.componentType == BASEAPP_TYPE) - flagsGood = (stringToEntityDataFlags(defContext.propertyFlags) & ENTITY_BASE_DATA_FLAGS) != 0; - else if (defContext.componentType == CELLAPP_TYPE) - flagsGood = (stringToEntityDataFlags(defContext.propertyFlags) & ENTITY_CELL_DATA_FLAGS) != 0; - else if (defContext.componentType == CLIENT_TYPE) - flagsGood = (stringToEntityDataFlags(defContext.propertyFlags) & ENTITY_CLIENT_DATA_FLAGS) != 0; - - name += "." + defContext.attrName; - - if (!flagsGood) - { - PyErr_Format(PyExc_AssertionError, "EntityDef.%s: '%s'(%s) not a valid %s property flags!\n\n", - defContext.optionName.c_str(), name.c_str(), defContext.propertyFlags.c_str(), COMPONENT_NAME_EX(defContext.componentType)); - - return false; - } - } - else if(defContext.type == DefContext::DC_TYPE_METHOD || - defContext.type == DefContext::DC_TYPE_CLIENT_METHOD) - { - name += "." + defContext.attrName; - } - else if (defContext.type == DefContext::DC_TYPE_FIXED_ITEM) - { - name += "." + defContext.attrName; - } - else if (defContext.type == DefContext::DC_TYPE_FIXED_ARRAY || - defContext.type == DefContext::DC_TYPE_FIXED_DICT || - defContext.type == DefContext::DC_TYPE_RENAME) - { - if (!DataTypes::validTypeName(name)) - { - PyErr_Format(PyExc_AssertionError, "EntityDef.%s: Not allowed to use the prefix \"_\"! typeName=%s\n", - defContext.optionName.c_str(), name.c_str()); - - return false; - } - } - - DefContext::DEF_CONTEXT_MAP::iterator iter = DefContext::allScriptDefContextMaps.find(name); - if (iter != DefContext::allScriptDefContextMaps.end()) - { - if (iter->second.pyObjectSourceFile != defContext.pyObjectSourceFile) - { - // Dz̵ͬĽű֣ôҪкϲע - if (iter->second.componentType != defContext.componentType && DefContext::allScriptDefContextLineMaps.find(defContext.pyObjectSourceFile) != DefContext::allScriptDefContextLineMaps.end() && - (defContext.type == DefContext::DC_TYPE_ENTITY || defContext.type == DefContext::DC_TYPE_COMPONENT || defContext.type == DefContext::DC_TYPE_INTERFACE) && - iter->second.type == defContext.type) - { - std::vector< std::string >::iterator bciter = defContext.baseClasses.begin(); - for (; bciter != defContext.baseClasses.end(); ++bciter) - { - if (std::find(iter->second.baseClasses.begin(), iter->second.baseClasses.end(), (*bciter)) == iter->second.baseClasses.end()) - { - iter->second.baseClasses.push_back((*bciter)); - } - - KBE_ASSERT(defContext.methods.size() == 0 && defContext.client_methods.size() == 0 && defContext.components.size() == 0 && defContext.propertys.size() == 0); - if (defContext.hasClient) - iter->second.hasClient = true; - - } - - return true; - } - - PyErr_Format(PyExc_AssertionError, "EntityDef.%s: \'%s\' already exists!\n", - defContext.optionName.c_str(), name.c_str()); - - return false; - } - - return true; - } - - DefContext::allScriptDefContextMaps[name] = defContext; - return assemblyContexts(); -} - -//------------------------------------------------------------------------------------- - -static bool onDefRename(DefContext& defContext) -{ - defContext.type = DefContext::DC_TYPE_RENAME; - return registerDefContext(defContext); -} - -//------------------------------------------------------------------------------------- -static bool onDefFixedDict(DefContext& defContext) -{ - defContext.type = DefContext::DC_TYPE_FIXED_DICT; - return registerDefContext(defContext); -} - -//------------------------------------------------------------------------------------- -static bool onDefFixedArray(DefContext& defContext) -{ - defContext.type = DefContext::DC_TYPE_FIXED_ARRAY; - return registerDefContext(defContext); -} - -//------------------------------------------------------------------------------------- -static bool onDefFixedItem(DefContext& defContext) -{ - defContext.type = DefContext::DC_TYPE_FIXED_ITEM; - return registerDefContext(defContext); -} - -//------------------------------------------------------------------------------------- -static bool onDefProperty(DefContext& defContext) -{ - defContext.type = DefContext::DC_TYPE_PROPERTY; - return registerDefContext(defContext); -} - -//------------------------------------------------------------------------------------- -static bool onDefMethod(DefContext& defContext) -{ - defContext.type = DefContext::DC_TYPE_METHOD; - return registerDefContext(defContext); -} - -//------------------------------------------------------------------------------------- -static bool onDefClientMethod(DefContext& defContext) -{ - defContext.type = DefContext::DC_TYPE_CLIENT_METHOD; - return registerDefContext(defContext); -} - -//------------------------------------------------------------------------------------- -static bool onDefEntity(DefContext& defContext) -{ - defContext.type = DefContext::DC_TYPE_ENTITY; - return registerDefContext(defContext); -} - -//------------------------------------------------------------------------------------- -static bool onDefInterface(DefContext& defContext) -{ - defContext.type = DefContext::DC_TYPE_INTERFACE; - return registerDefContext(defContext); -} - -//------------------------------------------------------------------------------------- -static bool onDefComponent(DefContext& defContext) -{ - defContext.type = DefContext::DC_TYPE_COMPONENT; - return registerDefContext(defContext); -} - -//------------------------------------------------------------------------------------- -static bool isRefEntityDefModule(PyObject *pyObj) -{ - if(!pyObj) - return true; - - PyObject *entitydefModule = PyImport_AddModule(pyDefModuleName.c_str()); - PyObject* pydict = PyObject_GetAttrString(entitydefModule, "__dict__"); - - PyObject *key, *value; - Py_ssize_t pos = 0; - - while (PyDict_Next(pydict, &pos, &key, &value)) { - if (value == pyObj) - { - Py_DECREF(pydict); - return true; - } - } - - Py_DECREF(pydict); - return false; -} - -//------------------------------------------------------------------------------------- -#define PY_RETURN_ERROR { DefContext::allScriptDefContextLineMaps.clear(); DefContext::allScriptDefContextMaps.clear(); while(!g_callContexts.empty()) g_callContexts.pop(); return NULL; } - -#define PYOBJECT_SOURCEFILE(PYOBJ, OUT) \ -{ \ - PyObject* pyInspectModule = \ - PyImport_ImportModule(const_cast("inspect")); \ - \ - PyObject* pyGetsourcefile = NULL; \ - PyObject* pyGetLineno = NULL; \ - PyObject* pyGetCurrentFrame = NULL; \ - \ - if (pyInspectModule) \ - { \ - pyGetsourcefile = \ - PyObject_GetAttrString(pyInspectModule, const_cast("getsourcefile")); \ - pyGetLineno = \ - PyObject_GetAttrString(pyInspectModule, const_cast("getlineno")); \ - pyGetCurrentFrame = \ - PyObject_GetAttrString(pyInspectModule, const_cast("currentframe")); \ - \ - Py_DECREF(pyInspectModule); \ - } \ - else \ - { \ - PY_RETURN_ERROR; \ - } \ - \ - if (pyGetsourcefile) \ - { \ - PyObject* pyFile = PyObject_CallFunction(pyGetsourcefile, \ - const_cast("(O)"), PYOBJ); \ - \ - Py_DECREF(pyGetsourcefile); \ - \ - if (!pyFile) \ - { \ - Py_XDECREF(pyGetLineno); \ - Py_XDECREF(pyGetCurrentFrame); \ - PY_RETURN_ERROR; \ - } \ - else \ - { \ - /* ֹͬϵͳɵ·һ£޳ϵͳ· */ \ - OUT = PyUnicode_AsUTF8AndSize(pyFile, NULL); \ - strutil::kbe_replace(OUT, "\\\\", "/"); \ - strutil::kbe_replace(OUT, "\\", "/"); \ - strutil::kbe_replace(OUT, "//", "/"); \ - std::string kbe_root = Resmgr::getSingleton().getPyUserScriptsPath(); \ - strutil::kbe_replace(kbe_root, "\\\\", "/"); \ - strutil::kbe_replace(kbe_root, "\\", "/"); \ - strutil::kbe_replace(kbe_root, "/", "/"); \ - strutil::kbe_replace(OUT, kbe_root, ""); \ - Py_DECREF(pyFile); \ - } \ - } \ - else \ - { \ - Py_XDECREF(pyGetLineno); \ - Py_XDECREF(pyGetCurrentFrame); \ - PY_RETURN_ERROR; \ - } \ - \ - if (pyGetLineno) \ - { \ - if(!pyGetCurrentFrame) \ - { \ - Py_DECREF(pyGetLineno); \ - } \ - \ - PyObject* pyCurrentFrame = PyObject_CallFunction(pyGetCurrentFrame, \ - const_cast("()")); \ - \ - PyObject* pyLine = PyObject_CallFunction(pyGetLineno, \ - const_cast("(O)"), pyCurrentFrame); \ - \ - Py_DECREF(pyGetLineno); \ - Py_DECREF(pyGetCurrentFrame); \ - Py_DECREF(pyCurrentFrame); \ - \ - if (!pyLine) \ - { \ - PY_RETURN_ERROR; \ - } \ - else \ - { \ - /* кţͬļжζ */ \ - OUT += fmt::format("#{}", PyLong_AsLong(pyLine)); \ - Py_DECREF(pyLine); \ - } \ - } \ - else \ - { \ - Py_XDECREF(pyGetCurrentFrame); \ - PY_RETURN_ERROR; \ - } \ -} - -static PyObject* __py_def_parse(PyObject *self, PyObject* args) -{ - CallContext cc = g_callContexts.top(); - g_callContexts.pop(); - - DefContext defContext; - defContext.optionName = cc.optionName; - - PyObject* kbeModule = PyImport_AddModule("KBEngine"); - KBE_ASSERT(kbeModule); - - PyObject* pyComponentName = PyObject_GetAttrString(kbeModule, "component"); - if (!pyComponentName) - { - PyErr_Format(PyExc_AssertionError, "EntityDef.__py_def_call(): get KBEngine.component error!\n"); - PY_RETURN_ERROR; - } - - defContext.componentType = ComponentName2ComponentType(PyUnicode_AsUTF8AndSize(pyComponentName, NULL)); - Py_DECREF(pyComponentName); - - if (!args || PyTuple_Size(args) < 1) - { - PyErr_Format(PyExc_AssertionError, "EntityDef.__py_def_call(EntityDef.%s): error!\n", defContext.optionName.c_str()); - PY_RETURN_ERROR; - } - - PyObject* pyFunc = PyTuple_GET_ITEM(args, 0); - - PyObject* pyModuleQualname = PyObject_GetAttrString(pyFunc, "__qualname__"); - if (!pyModuleQualname) - { - PY_RETURN_ERROR; - } - - const char* moduleQualname = PyUnicode_AsUTF8AndSize(pyModuleQualname, NULL); - Py_DECREF(pyModuleQualname); - - defContext.pyObjectPtr = PyObjectPtr(pyFunc); - PYOBJECT_SOURCEFILE(defContext.pyObjectPtr.get(), defContext.pyObjectSourceFile); - - if (defContext.optionName == "method") - { - static char * keywords[] = - { - const_cast ("exposed"), - const_cast ("utype"), - NULL - }; - - PyObject* pyExposed = NULL; - PyObject* pyUtype = NULL; - - if (!PyArg_ParseTupleAndKeywords(cc.pyArgs.get(), cc.pyKwargs.get(), "|OO", - keywords, &pyExposed, &pyUtype)) - { - PY_RETURN_ERROR; - } - - if (pyExposed && !PyBool_Check(pyExposed)) - { - PyErr_Format(PyExc_AssertionError, "EntityDef.%s: \'exposed\' error! not a bool type.\n", defContext.optionName.c_str()); - PY_RETURN_ERROR; - } - - if (pyUtype && !PyLong_Check(pyUtype)) - { - PyErr_Format(PyExc_AssertionError, "EntityDef.%s: \'utype\' error! not a number type.\n", defContext.optionName.c_str()); - PY_RETURN_ERROR; - } - - defContext.exposed = pyExposed == Py_True; - - if (pyUtype) - defContext.utype = (int)PyLong_AsLong(pyUtype); - } - else if (defContext.optionName == "property" || defContext.optionName == "fixed_item") - { - if (defContext.optionName != "fixed_item") - { - static char * keywords[] = - { - const_cast ("flags"), - const_cast ("persistent"), - const_cast ("index"), - const_cast ("databaseLength"), - const_cast ("utype"), - NULL - }; - - PyObject* pyFlags = NULL; - PyObject* pyPersistent = NULL; - PyObject* pyIndex = NULL; - PyObject* pyDatabaseLength = NULL; - PyObject* pyUtype = NULL; - - if (!PyArg_ParseTupleAndKeywords(cc.pyArgs.get(), cc.pyKwargs.get(), "|OOOOO", - keywords, &pyFlags, &pyPersistent, &pyIndex, &pyDatabaseLength, &pyUtype)) - { - PY_RETURN_ERROR; - } - - if (!pyFlags || !isRefEntityDefModule(pyFlags)) - { - PyErr_Format(PyExc_AssertionError, "EntityDef.%s: \'flags\' must be referenced from the [EntityDef.ALL_CLIENTS, EntityDef.*] module!\n", defContext.optionName.c_str()); - PY_RETURN_ERROR; - } - - if (!isRefEntityDefModule(pyIndex)) - { - PyErr_Format(PyExc_AssertionError, "EntityDef.%s: \'index\' must be referenced from the [EntityDef.UNIQUE, EntityDef.INDEX] module!\n", defContext.optionName.c_str()); - PY_RETURN_ERROR; - } - - if (pyDatabaseLength && !PyLong_Check(pyDatabaseLength)) - { - PyErr_Format(PyExc_AssertionError, "EntityDef.%s: \'databaseLength\' error! not a number type.\n", defContext.optionName.c_str()); - PY_RETURN_ERROR; - } - - if (pyPersistent && !PyBool_Check(pyPersistent)) - { - PyErr_Format(PyExc_AssertionError, "EntityDef.%s: \'persistent\' error! not a bool type.\n", defContext.optionName.c_str()); - PY_RETURN_ERROR; - } - - if(pyUtype && !PyLong_Check(pyUtype)) - { - PyErr_Format(PyExc_AssertionError, "EntityDef.%s: \'utype\' error! not a number type.\n", defContext.optionName.c_str()); - PY_RETURN_ERROR; - } - - defContext.propertyFlags = PyUnicode_AsUTF8AndSize(pyFlags, NULL); - - if (pyPersistent) - defContext.persistent = pyPersistent == Py_True; - - if (pyIndex) - defContext.propertyIndex = PyUnicode_AsUTF8AndSize(pyIndex, NULL); - - if (pyDatabaseLength) - defContext.databaseLength = (int)PyLong_AsLong(pyDatabaseLength); - - if (pyUtype) - defContext.utype = (int)PyLong_AsLong(pyUtype); - } - - // ԣ Ҫ÷ֵΪĬֵ - PyObject* pyRet = PyObject_CallFunction(pyFunc, - const_cast("(O)"), Py_None); - - if (!pyRet) - return NULL; - - if (pyRet != Py_None) - { - PyObject* pyStrResult = PyObject_Str(pyRet); - Py_DECREF(pyRet); - - defContext.propertyDefaultVal = PyUnicode_AsUTF8AndSize(pyStrResult, NULL); - Py_DECREF(pyStrResult); - - // ַ֤ǷԻԭɶ - if (defContext.propertyDefaultVal.size() > 0) - { - PyObject* module = PyImport_AddModule("__main__"); - if (module == NULL) - return NULL; - - PyObject* mdict = PyModule_GetDict(module); // Borrowed reference. - PyObject* result = PyRun_String(const_cast(defContext.propertyDefaultVal.c_str()), - Py_eval_input, mdict, mdict); - - if (result == NULL) - return NULL; - - Py_DECREF(result); - } - } - else - { - Py_DECREF(pyRet); - } - } - else if (defContext.optionName == "entity") - { - defContext.isModuleScope = true; - - static char * keywords[] = - { - const_cast ("hasClient"), - NULL - }; - - PyObject* pyHasClient = NULL; - - if (!PyArg_ParseTupleAndKeywords(cc.pyArgs.get(), cc.pyKwargs.get(), "|O", - keywords, &pyHasClient)) - { - PY_RETURN_ERROR; - } - - defContext.hasClient = pyHasClient == Py_True; - } - else if (defContext.optionName == "interface") - { - defContext.isModuleScope = true; - defContext.inheritEngineModuleType = DefContext::DC_TYPE_INTERFACE; - } - else if (defContext.optionName == "component") - { - defContext.isModuleScope = true; - } - else if (defContext.optionName == "fixed_dict") - { - defContext.isModuleScope = true; - - static char * keywords[] = - { - const_cast ("implementedBy"), - NULL - }; - - PyObject* pImplementedBy = NULL; - - if (!PyArg_ParseTupleAndKeywords(cc.pyArgs.get(), cc.pyKwargs.get(), "|O", - keywords, &pImplementedBy)) - { - PY_RETURN_ERROR; - } - - if (pImplementedBy) - { - if (isRefEntityDefModule(pImplementedBy)) - { - if (std::string(PyUnicode_AsUTF8AndSize(pImplementedBy, NULL)) == "thisClass") - { - defContext.implementedBy = pyFunc; - } - } - else - { - defContext.implementedBy = pImplementedBy; - } - - PyObject* pyQualname = PyObject_GetAttrString(defContext.implementedBy.get(), "__qualname__"); - if (!pyQualname) - { - PY_RETURN_ERROR; - } - - defContext.implementedByModuleName = PyUnicode_AsUTF8AndSize(pyQualname, NULL); - Py_DECREF(pyQualname); - - PYOBJECT_SOURCEFILE(defContext.implementedBy.get(), defContext.implementedByModuleFile); - } - } - else if (defContext.optionName == "fixed_array") - { - defContext.isModuleScope = true; - } - else if (defContext.optionName == "fixed_item") - { - } - else if (defContext.optionName == "rename") - { - } - else - { - PyErr_Format(PyExc_AssertionError, "EntityDef.%s: not support!\n", defContext.optionName.c_str()); - PY_RETURN_ERROR; - } - - if (!defContext.isModuleScope) - { - std::vector outs; - - if (moduleQualname) - strutil::kbe_splits(moduleQualname, ".", outs); - - if (defContext.optionName != "rename") - { - if (outs.size() != 2) - { - if(PyFunction_Check(pyFunc)) - PyErr_Format(PyExc_AssertionError, "EntityDef.%s: \'%s\' must be defined in the entity module!\n", - defContext.optionName.c_str(), moduleQualname); - else - PyErr_Format(PyExc_AssertionError, "EntityDef.%s: please check the command format is: EntityDef.%s(..)\n", - defContext.optionName.c_str(), defContext.optionName.c_str()); - - PY_RETURN_ERROR; - } - - defContext.moduleName = outs[0]; - defContext.attrName = outs[1]; - } - else - { - if (outs.size() != 1) - { - PyErr_Format(PyExc_AssertionError, "EntityDef.%s: error! such as: @EntityDef.rename()\n\tdef ENTITY_ID() -> EntityDef.INT32: pass\n", defContext.optionName.c_str()); - PY_RETURN_ERROR; - } - - defContext.moduleName = outs[0]; - } - - PyObject* pyInspectModule = - PyImport_ImportModule(const_cast("inspect")); - - PyObject* pyGetfullargspec = NULL; - if (pyInspectModule) - { - pyGetfullargspec = - PyObject_GetAttrString(pyInspectModule, const_cast("getfullargspec")); - - Py_DECREF(pyInspectModule); - } - else - { - PY_RETURN_ERROR; - } - - if (pyGetfullargspec) - { - PyObject* pyGetMethodArgs = PyObject_CallFunction(pyGetfullargspec, - const_cast("(O)"), pyFunc); - - if (!pyGetMethodArgs) - { - PY_RETURN_ERROR; - } - else - { - PyObject* pyGetMethodArgsResult = PyObject_GetAttrString(pyGetMethodArgs, const_cast("args")); - PyObject* pyGetMethodAnnotationsResult = PyObject_GetAttrString(pyGetMethodArgs, const_cast("annotations")); - - Py_DECREF(pyGetMethodArgs); - - if (!pyGetMethodArgsResult || !pyGetMethodAnnotationsResult) - { - Py_XDECREF(pyGetMethodArgsResult); - Py_XDECREF(pyGetMethodAnnotationsResult); - PY_RETURN_ERROR; - } - - PyObjectPtr pyGetMethodArgsResultPtr = pyGetMethodArgsResult; - PyObjectPtr pyGetMethodAnnotationsResultPtr = pyGetMethodAnnotationsResult; - Py_DECREF(pyGetMethodArgsResult); - Py_DECREF(pyGetMethodAnnotationsResult); - - if (defContext.optionName != "rename") - { - Py_ssize_t argsSize = PyList_Size(pyGetMethodArgsResult); - if (argsSize == 0) - { - PyErr_Format(PyExc_AssertionError, "EntityDef.%s: \'%s\' did not find \'self\' parameter!\n", defContext.optionName.c_str(), moduleQualname); - PY_RETURN_ERROR; - } - - for (Py_ssize_t i = 1; i < argsSize; ++i) - { - PyObject* pyItem = PyList_GetItem(pyGetMethodArgsResult, i); - - const char* ccattr = PyUnicode_AsUTF8AndSize(pyItem, NULL); - if (!ccattr) - { - PY_RETURN_ERROR; - } - - defContext.argsvecs.push_back(ccattr); - } - } - - PyObject *key, *value; - Py_ssize_t pos = 0; - - while (PyDict_Next(pyGetMethodAnnotationsResult, &pos, &key, &value)) { - const char* skey = PyUnicode_AsUTF8AndSize(key, NULL); - if (!skey) - { - PY_RETURN_ERROR; - } - - std::string svalue = ""; - - // EntityDef.Array˴һtupleο__py_array - if (PyTuple_Check(value) && PyTuple_Size(value) == 2) - { - PyObject *entitydefModule = PyImport_AddModule(pyDefModuleName.c_str()); - PyObject* pyARRAY = PyObject_GetAttrString(entitydefModule, "ARRAY"); - PyObject* item0 = PyTuple_GET_ITEM(value, 0); - - if (pyARRAY == item0) - { - value = PyTuple_GET_ITEM(value, 1); - defContext.optionName = "anonymous_fixed_array"; - - if (std::string(skey) != "return") - { - PyErr_Format(PyExc_AssertionError, "EntityDef.%s: \'EntityDef.ARRAY\' Can only be used to define property types!\n", defContext.optionName.c_str()); - PY_RETURN_ERROR; - } - } - else - { - Py_DECREF(pyARRAY); - PY_RETURN_ERROR; - } - - Py_DECREF(pyARRAY); - } - - if (PyUnicode_Check(value)) - { - svalue = PyUnicode_AsUTF8AndSize(value, NULL); - } - else - { - PyObject* pyQualname = PyObject_GetAttrString(value, "__qualname__"); - if (!pyQualname) - { - PY_RETURN_ERROR; - } - - svalue = PyUnicode_AsUTF8AndSize(pyQualname, NULL); - Py_DECREF(pyQualname); - } - - if (svalue.size() == 0) - { - PY_RETURN_ERROR; - } - - if (std::string(skey) == "return") - defContext.returnType = svalue; - else - defContext.annotationsMaps[skey] = svalue; - } - } - } - else - { - PY_RETURN_ERROR; - } - } - else - { - defContext.moduleName = moduleQualname; - - PyObject* pyBases = PyObject_GetAttrString(pyFunc, "__bases__"); - if (!pyBases) - PY_RETURN_ERROR; - - Py_ssize_t basesSize = PyTuple_Size(pyBases); - if (basesSize == 0) - { - PyErr_Format(PyExc_AssertionError, "EntityDef.%s: \'%s\' does not inherit the KBEngine.Entity class!\n", defContext.optionName.c_str(), moduleQualname); - Py_XDECREF(pyBases); - PY_RETURN_ERROR; - } - - for (Py_ssize_t i = 0; i < basesSize; ++i) - { - PyObject* pyClass = PyTuple_GetItem(pyBases, i); - - PyObject* pyQualname = PyObject_GetAttrString(pyClass, "__qualname__"); - if (!pyQualname) - { - Py_XDECREF(pyBases); - PY_RETURN_ERROR; - } - - std::string parentClass = PyUnicode_AsUTF8AndSize(pyQualname, NULL); - Py_DECREF(pyQualname); - - if (parentClass == "object") - { - continue; - } - else if (parentClass == "Entity" || parentClass == "Proxy") - { - defContext.inheritEngineModuleType = DefContext::DC_TYPE_ENTITY; - continue; - } - else if (parentClass == "EntityComponent") - { - defContext.inheritEngineModuleType = DefContext::DC_TYPE_COMPONENT; - continue; - } - - defContext.baseClasses.push_back(parentClass); - } - - Py_XDECREF(pyBases); - } - - bool noerror = true; - - if (defContext.optionName == "method" || defContext.optionName == "clientmethod") - { - if (defContext.annotationsMaps.size() != defContext.argsvecs.size()) - { - PyErr_Format(PyExc_AssertionError, "EntityDef.%s: \'%s\' all parameters must have annotations!\n", defContext.optionName.c_str(), moduleQualname); - PY_RETURN_ERROR; - } - - if (defContext.optionName == "method") - noerror = onDefMethod(defContext); - else - noerror = onDefClientMethod(defContext); - } - else if (defContext.optionName == "rename") - { - noerror = onDefRename(defContext); - } - else if (defContext.optionName == "property") - { - noerror = onDefProperty(defContext); - } - else if (defContext.optionName == "entity") - { - noerror = onDefEntity(defContext); - } - else if (defContext.optionName == "interface") - { - noerror = onDefInterface(defContext); - } - else if (defContext.optionName == "component") - { - noerror = onDefComponent(defContext); - } - else if (defContext.optionName == "fixed_dict") - { - noerror = onDefFixedDict(defContext); - } - else if (defContext.optionName == "fixed_array") - { - noerror = onDefFixedArray(defContext); - } - else if (defContext.optionName == "anonymous_fixed_array") - { - // һ - DefContext arrayType; - arrayType = defContext; - arrayType.moduleName += ".anonymous_fixed_array"; - - // ֹ2Դļһ£ײΪظ - arrayType.pyObjectSourceFile += "(array)"; - - // һitem - DefContext itemType; - itemType = arrayType; - itemType.optionName = "fixed_item"; - - // ֹ2Դļһ£ײΪظ - itemType.pyObjectSourceFile += "(array_item)"; - - noerror = onDefFixedItem(itemType); - - arrayType.returnType = ""; - - if(noerror) - noerror = onDefFixedArray(arrayType); - - defContext.returnType = arrayType.moduleName; - - if (noerror) - noerror = onDefFixedItem(defContext); - } - else if (defContext.optionName == "fixed_item") - { - noerror = onDefFixedItem(defContext); - } - - if (!noerror) - { - PY_RETURN_ERROR; - } - - Py_INCREF(pyFunc); - return pyFunc; -} - -//------------------------------------------------------------------------------------- -static PyMethodDef __call_def_parse = { "_PyEntityDefParse", (PyCFunction)&__py_def_parse, METH_VARARGS, 0 }; - -#define PY_DEF_HOOK(NAME) \ - static PyObject* __py_def_##NAME(PyObject* self, PyObject* args, PyObject* kwargs) \ - { \ - CallContext cc; \ - cc.pyArgs = PyObjectPtr(Copy::deepcopy(args)); \ - cc.pyKwargs = kwargs ? PyObjectPtr(Copy::deepcopy(kwargs)) : PyObjectPtr(NULL); \ - cc.optionName = #NAME; \ - g_callContexts.push(cc); \ - Py_XDECREF(cc.pyArgs.get()); \ - Py_XDECREF(cc.pyKwargs.get()); \ - \ - return PyCFunction_New(&__call_def_parse, self); \ - } - -static PyObject* __py_def_rename(PyObject* self, PyObject* args, PyObject* kwargs) -{ - CallContext cc; - cc.pyArgs = PyObjectPtr(Copy::deepcopy(args)); - cc.pyKwargs = kwargs ? PyObjectPtr(Copy::deepcopy(kwargs)) : PyObjectPtr(NULL); - cc.optionName = "rename"; - - Py_XDECREF(cc.pyArgs.get()); - Py_XDECREF(cc.pyKwargs.get()); - - // ֶ巽ʽ EntityDef.rename(ENTITY_ID=EntityDef.INT32) - if (kwargs) - { - PyObject *key, *value; - Py_ssize_t pos = 0; - - while (PyDict_Next(kwargs, &pos, &key, &value)) { - if (!PyType_Check(value)) - { - PyErr_Format(PyExc_AssertionError, "EntityDef.%s: arg2 not legal type! such as: EntityDef.rename(ENTITY_ID=EntityDef.INT32)\n", cc.optionName.c_str()); - return NULL; - } - - PyObject* pyQualname = PyObject_GetAttrString(value, "__qualname__"); - if (!pyQualname) - { - PyErr_Format(PyExc_AssertionError, "EntityDef.%s: arg2 get __qualname__ error! such as: EntityDef.rename(ENTITY_ID=EntityDef.INT32)\n", cc.optionName.c_str()); - return NULL; - } - - std::string typeName = PyUnicode_AsUTF8AndSize(pyQualname, NULL); - Py_DECREF(pyQualname); - - if (!PyUnicode_Check(key)) - { - PyErr_Format(PyExc_AssertionError, "EntityDef.%s: arg1 must be a string! such as: EntityDef.rename(ENTITY_ID=EntityDef.INT32)\n", cc.optionName.c_str()); - return NULL; - } - - DefContext defContext; - defContext.optionName = cc.optionName; - defContext.moduleName = PyUnicode_AsUTF8AndSize(key, NULL); - defContext.returnType = typeName; - - PyObject* kbeModule = PyImport_AddModule("KBEngine"); - KBE_ASSERT(kbeModule); - - PyObject* pyComponentName = PyObject_GetAttrString(kbeModule, "component"); - if (!pyComponentName) - { - PyErr_Format(PyExc_AssertionError, "EntityDef.rename(): get KBEngine.component error!\n"); - PY_RETURN_ERROR; - } - - defContext.componentType = ComponentName2ComponentType(PyUnicode_AsUTF8AndSize(pyComponentName, NULL)); - Py_DECREF(pyComponentName); - - if (!onDefRename(defContext)) - return NULL; - } - - S_Return; - } - - g_callContexts.push(cc); - - // @EntityDef.rename() - // def ENTITY_ID() -> int: pass - return PyCFunction_New(&__call_def_parse, self); -} - -static PyObject* __py_def_fixed_array(PyObject* self, PyObject* args, PyObject* kwargs) -{ - CallContext cc; - cc.pyArgs = PyObjectPtr(Copy::deepcopy(args)); - cc.pyKwargs = kwargs ? PyObjectPtr(Copy::deepcopy(kwargs)) : PyObjectPtr(NULL); - cc.optionName = "fixed_array"; - - Py_XDECREF(cc.pyArgs.get()); - Py_XDECREF(cc.pyKwargs.get()); - - // ֶ巽ʽ EntityDef.fixed_array(XXArray=EntityDef.INT32) - if (kwargs) - { - PyObject *key, *value; - Py_ssize_t pos = 0; - - while (PyDict_Next(kwargs, &pos, &key, &value)) { - if (!PyType_Check(value)) - { - PyErr_Format(PyExc_AssertionError, "EntityDef.%s: arg2 not legal type! such as: EntityDef.fixed_array(XXArray=EntityDef.INT32)\n", cc.optionName.c_str()); - return NULL; - } - - PyObject* pyQualname = PyObject_GetAttrString(value, "__qualname__"); - if (!pyQualname) - { - PyErr_Format(PyExc_AssertionError, "EntityDef.%s: arg2 get __qualname__ error! such as: EntityDef.fixed_array(XXArray=EntityDef.INT32)\n", cc.optionName.c_str()); - return NULL; - } - - std::string typeName = PyUnicode_AsUTF8AndSize(pyQualname, NULL); - Py_DECREF(pyQualname); - - if (!PyUnicode_Check(key)) - { - PyErr_Format(PyExc_AssertionError, "EntityDef.%s: arg1 must be a string! such as: EntityDef.fixed_array(XXArray=EntityDef.INT32)\n", cc.optionName.c_str()); - return NULL; - } - - DefContext defContext; - defContext.optionName = cc.optionName; - defContext.moduleName = PyUnicode_AsUTF8AndSize(key, NULL); - defContext.returnType = typeName; - - PyObject* kbeModule = PyImport_AddModule("KBEngine"); - KBE_ASSERT(kbeModule); - - PyObject* pyComponentName = PyObject_GetAttrString(kbeModule, "component"); - if (!pyComponentName) - { - PyErr_Format(PyExc_AssertionError, "EntityDef.fixed_array(): get KBEngine.component error!\n"); - PY_RETURN_ERROR; - } - - defContext.componentType = ComponentName2ComponentType(PyUnicode_AsUTF8AndSize(pyComponentName, NULL)); - Py_DECREF(pyComponentName); - - if (!onDefRename(defContext)) - return NULL; - } - - S_Return; - } - - g_callContexts.push(cc); - - // @EntityDef.fixed_array() - // class XXXArray - return PyCFunction_New(&__call_def_parse, self); -} - -#define PY_ADD_METHOD(NAME, DOCS) APPEND_SCRIPT_MODULE_METHOD(entitydefModule, NAME, __py_def_##NAME, METH_VARARGS | METH_KEYWORDS, 0); - -#ifdef interface -#undef interface -#endif - -#ifdef property -#undef property -#endif - -PY_DEF_HOOK(method) -PY_DEF_HOOK(clientmethod) -PY_DEF_HOOK(property) -PY_DEF_HOOK(entity) -PY_DEF_HOOK(interface) -PY_DEF_HOOK(component) -PY_DEF_HOOK(fixed_dict) -PY_DEF_HOOK(fixed_item) - -//------------------------------------------------------------------------------------- -bool installModule(const char* moduleName) -{ - pyDefModuleName = moduleName; - - PyObject *entitydefModule = PyImport_AddModule(pyDefModuleName.c_str()); - PyObject_SetAttrString(entitydefModule, "__doc__", PyUnicode_FromString("This module is created by KBEngine!")); - - PY_ADD_METHOD(rename, ""); - PY_ADD_METHOD(method, ""); - PY_ADD_METHOD(clientmethod, ""); - PY_ADD_METHOD(property, ""); - PY_ADD_METHOD(entity, ""); - PY_ADD_METHOD(interface, ""); - PY_ADD_METHOD(component, ""); - PY_ADD_METHOD(fixed_dict, ""); - PY_ADD_METHOD(fixed_array, ""); - PY_ADD_METHOD(fixed_item, ""); - - return true; -} - -//------------------------------------------------------------------------------------- -bool uninstallModule() -{ - while (!g_callContexts.empty()) g_callContexts.pop(); - DefContext::allScriptDefContextMaps.clear(); - DefContext::allScriptDefContextLineMaps.clear(); - return true; -} - -//------------------------------------------------------------------------------------- -static bool loadAllScriptForComponentType(COMPONENT_TYPE loadComponentType) -{ - std::string rootPath = Resmgr::getSingleton().getPyUserComponentScriptsPath(loadComponentType); - - if (rootPath.size() == 0) - { - ERROR_MSG(fmt::format("PyEntityDef::loadAllScriptForComponentType(): get scripts path error! loadComponentType={}\n", - COMPONENT_NAME_EX(loadComponentType))); - - return false; - } - - while (rootPath[rootPath.size() - 1] == '/' || rootPath[rootPath.size() - 1] == '\\') rootPath.pop_back(); - - wchar_t* wpath = strutil::char2wchar((rootPath).c_str()); - std::vector results; - Resmgr::getSingleton().listPathRes(wpath, L"py|pyc", results); - - std::vector::iterator iter = results.begin(); - for (; iter != results.end(); ++iter) - { - std::wstring wstrpath = (*iter); - - if (wstrpath.find(L"__pycache__") != std::wstring::npos) - continue; - - if (wstrpath.find(L"__init__.") != std::wstring::npos) - continue; - - std::pair pathPair = script::PyPlatform::splitPath(wstrpath); - std::pair filePair = script::PyPlatform::splitText(pathPair.second); - - if (filePair.first.size() == 0) - continue; - - char* cpacketPath = strutil::wchar2char(pathPair.first.c_str()); - std::string packetPath = cpacketPath; - free(cpacketPath); - - strutil::kbe_replace(packetPath, rootPath, ""); - while (packetPath.size() > 0 && (packetPath[0] == '/' || packetPath[0] == '\\')) packetPath.erase(0, 1); - strutil::kbe_replace(packetPath, "/", "."); - strutil::kbe_replace(packetPath, "\\", "."); - - char* moduleName = strutil::wchar2char(filePair.first.c_str()); - - // ڽűڲܻimportظimport ǹѾimportģ - if (DefContext::allScriptDefContextMaps.find(moduleName) == DefContext::allScriptDefContextMaps.end()) - { - PyObject* pyModule = NULL; - - if (packetPath.size() == 0 || packetPath == "components" || packetPath == "interfaces") - { - pyModule = PyImport_ImportModule(const_cast(moduleName)); - } - else - { - pyModule = PyImport_ImportModule(const_cast((packetPath + "." + moduleName).c_str())); - } - - if (!pyModule) - { - SCRIPT_ERROR_CHECK(); - return false; - } - else - { - Py_DECREF(pyModule); - } - } - - free(moduleName); - } - - free(wpath); - return true; -} - -//------------------------------------------------------------------------------------- -PyObject* __py_getAppPublish(PyObject* self, PyObject* args) -{ - return PyLong_FromLong(g_appPublish); -} - -//------------------------------------------------------------------------------------- -static bool execPython(COMPONENT_TYPE componentType) -{ - std::pair pyPaths = getComponentPythonPaths(componentType); - if (pyPaths.first.size() == 0) - { - ERROR_MSG(fmt::format("PyEntityDef::execPython(): PythonApp({}) paths error!\n", COMPONENT_NAME_EX(componentType))); - return false; - } - - APPEND_PYSYSPATH(pyPaths.second); - - PyObject* modulesOld = PySys_GetObject("modules"); - - PyThreadState* pCurInterpreter = PyThreadState_Get(); - PyThreadState* pNewInterpreter = Py_NewInterpreter(); - - if (!pNewInterpreter) - { - ERROR_MSG(fmt::format("PyEntityDef::execPython(): Py_NewInterpreter()!\n")); - SCRIPT_ERROR_CHECK(); - return false; - } - -#if KBE_PLATFORM != PLATFORM_WIN32 - strutil::kbe_replace(pyPaths.second, L";", L":"); -#endif - - PySys_SetPath(pyPaths.second.c_str()); - - PyObject* modulesNew = PySys_GetObject("modules"); - PyDict_Merge(modulesNew, Script::getSingleton().getSysInitModules(), 0); - - { - PyObject *key, *value; - Py_ssize_t pos = 0; - - while (PyDict_Next(modulesOld, &pos, &key, &value)) - { - const char* typeName = PyUnicode_AsUTF8AndSize(key, NULL); - - if (std::string(typeName) == "KBEngine") - continue; - - PyObject* pyDoc = PyObject_GetAttrString(value, "__doc__"); - - if (pyDoc) - { - const char* doc = PyUnicode_AsUTF8AndSize(pyDoc, NULL); - - if (doc && std::string(doc).find("KBEngine") != std::string::npos) - PyDict_SetItemString(modulesNew, typeName, value); - - if (PyErr_Occurred()) - PyErr_Clear(); - - Py_XDECREF(pyDoc); - } - else - { - SCRIPT_ERROR_CHECK(); - } - } - } - - PyObject *m = PyImport_AddModule("__main__"); - - // һűģ - PyObject* kbeModule = PyImport_AddModule("KBEngine"); - KBE_ASSERT(kbeModule); - - Entity::registerScript(kbeModule); - Space::registerScript(kbeModule); - EntityComponent::registerScript(kbeModule); - - if (componentType == BASEAPP_TYPE) - Proxy::registerScript(kbeModule); - - const char* componentName = COMPONENT_NAME_EX(componentType); - if (PyModule_AddStringConstant(kbeModule, "component", componentName)) - { - ERROR_MSG(fmt::format("PyEntityDef::execPython(): Unable to set KBEngine.component to {}\n", - componentName)); - - return false; - } - - APPEND_SCRIPT_MODULE_METHOD(kbeModule, publish, __py_getAppPublish, METH_VARARGS, 0); - - // ģmain - PyObject_SetAttrString(m, "KBEngine", kbeModule); - - if (pNewInterpreter != PyThreadState_Swap(pCurInterpreter)) - { - KBE_ASSERT(false); - return false; - } - - PyThreadState_Swap(pNewInterpreter); - - bool otherPartSuccess = loadAllScriptForComponentType(componentType); - - Entity::unregisterScript(); - Space::unregisterScript(); - EntityComponent::unregisterScript(); - - if (componentType == BASEAPP_TYPE) - Proxy::unregisterScript(); - - if (pNewInterpreter != PyThreadState_Swap(pCurInterpreter)) - { - KBE_ASSERT(false); - return false; - } - - // ˴ʹPy_EndInterpreterᵼMathDefģ - PyInterpreterState_Clear(pNewInterpreter->interp); - PyInterpreterState_Delete(pNewInterpreter->interp); - return otherPartSuccess; -} - -//------------------------------------------------------------------------------------- -static bool loadAllScripts() -{ - std::vector< COMPONENT_TYPE > loadOtherComponentTypes; - - if (g_componentType == CELLAPP_TYPE || g_componentType == BASEAPP_TYPE) - { - bool otherPartSuccess = loadAllScriptForComponentType(g_componentType); - if (!otherPartSuccess) - return false; - - loadOtherComponentTypes.push_back((g_componentType == BASEAPP_TYPE) ? CELLAPP_TYPE : BASEAPP_TYPE); - } - else - { - loadOtherComponentTypes.push_back(BASEAPP_TYPE); - loadOtherComponentTypes.push_back(CELLAPP_TYPE); - } - - for (std::vector< COMPONENT_TYPE >::iterator iter = loadOtherComponentTypes.begin(); iter != loadOtherComponentTypes.end(); ++iter) - { - COMPONENT_TYPE componentType = (*iter); - - if (!execPython(componentType)) - return false; - } - - return true; -} - -//------------------------------------------------------------------------------------- -static bool registerDefTypes() -{ - DefContext::DEF_CONTEXT_MAP::iterator iter = DefContext::allScriptDefContextMaps.begin(); - for (; iter != DefContext::allScriptDefContextMaps.end(); ++iter) - { - DefContext& defContext = iter->second; - - if (defContext.type == DefContext::DC_TYPE_FIXED_ARRAY) - { - FixedArrayType* fixedArray = new FixedArrayType; - - if (fixedArray->initialize(&defContext, defContext.moduleName)) - { - if (!DataTypes::addDataType(defContext.moduleName, fixedArray)) - return false; - } - else - { - ERROR_MSG(fmt::format("PyEntityDef::registerDefTypes: parse ARRAY [{}] error! file: \"{}\"!\n", - defContext.moduleName.c_str(), defContext.pyObjectSourceFile)); - - delete fixedArray; - return false; - } - } - else if (defContext.type == DefContext::DC_TYPE_FIXED_DICT) - { - FixedDictType* fixedDict = new FixedDictType; - - if (fixedDict->initialize(&defContext, defContext.moduleName)) - { - if (!DataTypes::addDataType(defContext.moduleName, fixedDict)) - return false; - } - else - { - ERROR_MSG(fmt::format("PyEntityDef::registerDefTypes: parse FIXED_DICT [{}] error! file: \"{}\"!\n", - defContext.moduleName.c_str(), defContext.pyObjectSourceFile)); - - delete fixedDict; - return false; - } - } - else if (defContext.type == DefContext::DC_TYPE_RENAME) - { - DataType* dataType = DataTypes::getDataType(defContext.returnType, false); - if (dataType == NULL) - { - ERROR_MSG(fmt::format("PyEntityDef::registerDefTypes: cannot fount type \'{}\', by alias[{}], file: \"{}\"!\n", - defContext.returnType, defContext.moduleName.c_str(), defContext.pyObjectSourceFile)); - - return false; - } - - if (!DataTypes::addDataType(defContext.moduleName, dataType)) - { - ERROR_MSG(fmt::format("PyEntityDef::registerDefTypes: addDataType \"{}\" error! file: \"{}\"!\n", - defContext.moduleName.c_str(), defContext.pyObjectSourceFile)); - - return false; - } - } - } - - return true; -} - -//------------------------------------------------------------------------------------- -static bool registerDetailLevelInfo(ScriptDefModule* pScriptModule, DefContext& defContext) -{ - return true; -} - -//------------------------------------------------------------------------------------- -static bool registerVolatileInfo(ScriptDefModule* pScriptModule, DefContext& defContext) -{ - return true; -} - -//------------------------------------------------------------------------------------- -static bool registerDefPropertys(ScriptDefModule* pScriptModule, DefContext& defContext) -{ - DefContext::DEF_CONTEXTS& propertys = defContext.propertys; - - DefContext::DEF_CONTEXTS::iterator iter = propertys.begin(); - for (; iter != propertys.end(); ++iter) - { - DefContext& defPropContext = (*iter); - - ENTITY_PROPERTY_UID futype = 0; - EntityDataFlags flags = stringToEntityDataFlags(defPropContext.propertyFlags); - int32 hasBaseFlags = 0; - int32 hasCellFlags = 0; - int32 hasClientFlags = 0; - DataType* dataType = NULL; - bool isPersistent = (defPropContext.persistent != 0); - bool isIdentifier = false; // Ƿһ - uint32 databaseLength = defPropContext.databaseLength; // ݿеij - std::string indexType = defPropContext.propertyIndex; - DETAIL_TYPE detailLevel = DETAIL_LEVEL_FAR; - std::string name = defPropContext.attrName; - - if (!EntityDef::validDefPropertyName(name)) - { - ERROR_MSG(fmt::format("PyEntityDef::registerDefPropertys: '{}' is limited, in module({}), file: \"{}\"!\n", - name, pScriptModule->getName(), defPropContext.pyObjectSourceFile)); - - return false; - } - - if (defPropContext.detailLevel == "FAR") - detailLevel = DETAIL_LEVEL_FAR; - else if (defPropContext.detailLevel == "MEDIUM") - detailLevel = DETAIL_LEVEL_MEDIUM; - else if (defPropContext.detailLevel == "NEAR") - detailLevel = DETAIL_LEVEL_NEAR; - else - detailLevel = DETAIL_LEVEL_FAR; - - if (!EntityDef::calcDefPropertyUType(pScriptModule->getName(), name, defPropContext.utype > 0 ? defPropContext.utype : -1, pScriptModule, futype)) - return false; - - hasBaseFlags = ((uint32)flags) & ENTITY_BASE_DATA_FLAGS; - if (hasBaseFlags > 0) - pScriptModule->setBase(true); - - hasCellFlags = ((uint32)flags) & ENTITY_CELL_DATA_FLAGS; - if (hasCellFlags > 0) - pScriptModule->setCell(true); - - hasClientFlags = ((uint32)flags) & ENTITY_CLIENT_DATA_FLAGS; - if (hasClientFlags > 0) - pScriptModule->setClient(true); - - if (hasBaseFlags <= 0 && hasCellFlags <= 0) - { - ERROR_MSG(fmt::format("PyEntityDef::registerDefPropertys: not fount flags[{}], is {}.{}, file: \"{}\"!\n", - defPropContext.propertyFlags, pScriptModule->getName(), defPropContext.pyObjectSourceFile)); - - return false; - } - - dataType = DataTypes::getDataType(defPropContext.returnType, false); - - if (!dataType) - { - DefContext* pDefPropTypeContext = DefContext::findDefContext(defPropContext.returnType); - if (!pDefPropTypeContext) - { - ERROR_MSG(fmt::format("PyEntityDef::registerDefPropertys: not fount type[{}], is {}.{}, file: \"{}\"!\n", - defPropContext.returnType, pScriptModule->getName(), name.c_str(), defPropContext.pyObjectSourceFile)); - - return false; - } - - // ŵд - if (pDefPropTypeContext->type == DefContext::DC_TYPE_COMPONENT) - continue; - - if (pDefPropTypeContext->type == DefContext::DC_TYPE_FIXED_ARRAY) - { - FixedArrayType* dataType1 = new FixedArrayType(); - if (dataType1->initialize(pDefPropTypeContext, std::string(pScriptModule->getName()) + "_" + name)) - dataType = dataType1; - else - return false; - } - else - { - dataType = DataTypes::getDataType(defPropContext.returnType, false); - } - } - - if (dataType == NULL) - { - ERROR_MSG(fmt::format("PyEntityDef::registerDefPropertys: not fount type[{}], is {}.{}, file: \"{}\"!\n", - defPropContext.returnType, pScriptModule->getName(), name.c_str(), defPropContext.pyObjectSourceFile)); - - return false; - } - - // һʵ - PropertyDescription* propertyDescription = PropertyDescription::createDescription(futype, defPropContext.returnType, - name, flags, isPersistent, - dataType, isIdentifier, indexType, - databaseLength, defPropContext.propertyDefaultVal, - detailLevel); - - bool ret = true; - - // ӵģ - if (hasCellFlags > 0) - ret = pScriptModule->addPropertyDescription(name.c_str(), - propertyDescription, CELLAPP_TYPE); - - if (hasBaseFlags > 0) - ret = pScriptModule->addPropertyDescription(name.c_str(), - propertyDescription, BASEAPP_TYPE); - - if (hasClientFlags > 0) - ret = pScriptModule->addPropertyDescription(name.c_str(), - propertyDescription, CLIENT_TYPE); - - if (!ret) - { - ERROR_MSG(fmt::format("PyEntityDef::registerDefPropertys: error, is {}.{}, file: \"{}\"!\n", - pScriptModule->getName(), name.c_str(), defPropContext.pyObjectSourceFile)); - - return false; - } - } - - return true; -} - -//------------------------------------------------------------------------------------- -static bool registerDefComponents(ScriptDefModule* pScriptModule, DefContext& defContext) -{ - DefContext::DEF_CONTEXTS& propertys = defContext.propertys; - - DefContext::DEF_CONTEXTS::iterator iter = propertys.begin(); - for (; iter != propertys.end(); ++iter) - { - DefContext& defPropContext = (*iter); - - std::string componentName = defPropContext.attrName; - std::string componentTypeName = defPropContext.returnType; - bool isPersistent = (defPropContext.persistent != 0); - ENTITY_PROPERTY_UID futype = 0; - - DefContext* pDefPropTypeContext = DefContext::findDefContext(componentTypeName); - if (!pDefPropTypeContext) - { - continue; - } - - if (pDefPropTypeContext->type != DefContext::DC_TYPE_COMPONENT) - continue; - - if (!EntityDef::calcDefPropertyUType(pScriptModule->getName(), componentName, defPropContext.utype > 0 ? defPropContext.utype : -1, pScriptModule, futype)) - return false; - - // һʵ - uint32 flags = ED_FLAG_BASE | ED_FLAG_CELL_PUBLIC | ENTITY_CLIENT_DATA_FLAGS; - bool isIdentifier = false; // Ƿһ - uint32 databaseLength = 0; // ݿеij - std::string indexType = ""; - DETAIL_TYPE detailLevel = DETAIL_LEVEL_FAR; - std::string detailLevelStr = ""; - std::string strisPersistent; - std::string defaultStr = ""; - - if (!EntityDef::validDefPropertyName(componentName)) - { - ERROR_MSG(fmt::format("PyEntityDef::registerDefComponents: '{}' is limited, in module({}), file: \"{}\"!\n", - componentName, pScriptModule->getName(), defPropContext.pyObjectSourceFile)); - - return false; - } - - // Ƿģ飬˵Ѿعٴμ - ScriptDefModule* pCompScriptDefModule = EntityDef::findScriptModule(componentTypeName.c_str(), false); - - if (!pCompScriptDefModule) - { - pCompScriptDefModule = EntityDef::registerNewScriptDefModule(componentTypeName); - pCompScriptDefModule->isPersistent(false); - pCompScriptDefModule->isComponentModule(true); - } - else - { - flags = ED_FLAG_UNKOWN; - - if (pCompScriptDefModule->hasBase()) - flags |= ED_FLAG_BASE; - - if (pCompScriptDefModule->hasCell()) - flags |= ED_FLAG_CELL_PUBLIC; - - if (pCompScriptDefModule->hasClient()) - { - if (pCompScriptDefModule->hasBase()) - flags |= ED_FLAG_BASE_AND_CLIENT; - else - flags |= (ED_FLAG_ALL_CLIENTS | ED_FLAG_CELL_PUBLIC_AND_OWN | ED_FLAG_OTHER_CLIENTS | ED_FLAG_OWN_CLIENT); - } - - EntityDef::addComponentProperty(futype, componentTypeName, componentName, flags, isPersistent, isIdentifier, - indexType, databaseLength, defaultStr, detailLevel, pScriptModule, pCompScriptDefModule); - - pScriptModule->addComponentDescription(componentName.c_str(), pCompScriptDefModule); - continue; - } - - // - if (!registerDefPropertys(pScriptModule, defContext)) - { - ERROR_MSG(fmt::format("PyEntityDef::registerDefComponents: failed to loadDefPropertys(), entity:{}\n", - pScriptModule->getName())); - - return false; - } - - // ԼdetailLevelInfo - if (!registerDetailLevelInfo(pScriptModule, defContext)) - { - ERROR_MSG(fmt::format("PyEntityDef::registerDefComponents: failed to register component:{} detailLevelInfo.\n", - pScriptModule->getName())); - - return false; - } - - // ԼvolatileInfo - if (!registerVolatileInfo(pScriptModule, defContext)) - { - ERROR_MSG(fmt::format("PyEntityDef::registerDefComponents: failed to register component:{} volatileInfo.\n", - pScriptModule->getName())); - - return false; - } - - pCompScriptDefModule->autoMatchCompOwn(); - - flags = ED_FLAG_UNKOWN; - - if (pCompScriptDefModule->hasBase()) - flags |= ED_FLAG_BASE; - - if (pCompScriptDefModule->hasCell()) - flags |= ED_FLAG_CELL_PUBLIC; - - if (pCompScriptDefModule->hasClient()) - { - if (pCompScriptDefModule->hasBase()) - flags |= ED_FLAG_BASE_AND_CLIENT; - - if (pCompScriptDefModule->hasCell()) - flags |= (ED_FLAG_ALL_CLIENTS | ED_FLAG_CELL_PUBLIC_AND_OWN | ED_FLAG_OTHER_CLIENTS | ED_FLAG_OWN_CLIENT); - } - - EntityDef::addComponentProperty(futype, componentTypeName, componentName, flags, isPersistent, isIdentifier, - indexType, databaseLength, defaultStr, detailLevel, pScriptModule, pCompScriptDefModule); - - pScriptModule->addComponentDescription(componentName.c_str(), pCompScriptDefModule); - } - - return true; -} - -//------------------------------------------------------------------------------------- -static bool registerEntityDef(ScriptDefModule* pScriptModule, DefContext& defContext) -{ - // - if (!registerDefPropertys(pScriptModule, defContext)) - { - ERROR_MSG(fmt::format("PyEntityDef::registerEntityDef: failed to registerDefPropertys(), entity:{}\n", - pScriptModule->getName())); - - return false; - } - - // ǵķԼ뵽ģ - if (!registerDefComponents(pScriptModule, defContext)) - { - ERROR_MSG(fmt::format("PyEntityDef::registerEntityDef: failed to registerDefComponents(), component:{}\n", - pScriptModule->getName())); - - return false; - } - - // ԼdetailLevelInfo - if (!registerDetailLevelInfo(pScriptModule, defContext)) - { - ERROR_MSG(fmt::format("PyEntityDef::registerEntityDef: failed to register entity:{} detailLevelInfo.\n", - pScriptModule->getName())); - - return false; - } - - // ԼvolatileInfo - if (!registerVolatileInfo(pScriptModule, defContext)) - { - ERROR_MSG(fmt::format("PyEntityDef::registerEntityDef: failed to register entity:{} volatileInfo.\n", - pScriptModule->getName())); - - return false; - } - - pScriptModule->autoMatchCompOwn(); - return true; -} - -//------------------------------------------------------------------------------------- -static bool registerEntityDefs() -{ - if (!registerDefTypes()) - return false; - - DefContext::DEF_CONTEXT_MAP::iterator iter = DefContext::allScriptDefContextMaps.begin(); - for (; iter != DefContext::allScriptDefContextMaps.end(); ++iter) - { - DefContext& defContext = iter->second; - - if (defContext.type != DefContext::DC_TYPE_ENTITY) - continue; - - ScriptDefModule* pScriptModule = EntityDef::registerNewScriptDefModule(defContext.moduleName); - - if (!registerEntityDef(pScriptModule, defContext)) - return false; - - pScriptModule->onLoaded(); - } - - return true; -} - -//------------------------------------------------------------------------------------- -bool initialize() -{ - if (g_inited) - return false; - - g_inited = true; - - PyObject *entitydefModule = PyImport_AddModule(pyDefModuleName.c_str()); - - ENTITYFLAGMAP::iterator iter = g_entityFlagMapping.begin(); - for (; iter != g_entityFlagMapping.end(); ++iter) - { - if (PyModule_AddStringConstant(entitydefModule, iter->first.c_str(), iter->first.c_str())) - { - ERROR_MSG(fmt::format("PyEntityDef::initialize(): Unable to set EntityDef.{} to {}\n", - iter->first, iter->first)); - - return false; - } - } - - static const char* UNIQUE = "UNIQUE"; - if (PyModule_AddStringConstant(entitydefModule, UNIQUE, UNIQUE)) - { - ERROR_MSG(fmt::format("PyEntityDef::initialize(): Unable to set EntityDef.{} to {}\n", - UNIQUE, UNIQUE)); - - return false; - } - - static const char* INDEX = "INDEX"; - if (PyModule_AddStringConstant(entitydefModule, INDEX, INDEX)) - { - ERROR_MSG(fmt::format("PyEntityDef::initialize(): Unable to set EntityDef.{} to {}\n", - INDEX, INDEX)); - - return false; - } - - static const char* thisClass = "thisClass"; - if (PyModule_AddStringConstant(entitydefModule, thisClass, thisClass)) - { - ERROR_MSG(fmt::format("PyEntityDef::initialize(): Unable to set EntityDef.{} to {}\n", - thisClass, thisClass)); - - return false; - } - - APPEND_SCRIPT_MODULE_METHOD(entitydefModule, ARRAY, __py_array, METH_VARARGS, 0); - - static char allBaseTypeNames[64][MAX_BUF]; - std::vector< std::string > baseTypeNames = DataTypes::getBaseTypeNames(); - KBE_ASSERT(baseTypeNames.size() < 64); - - for (size_t idx = 0; idx < baseTypeNames.size(); idx++) - { - memset(allBaseTypeNames[idx], 0, MAX_BUF); - kbe_snprintf(allBaseTypeNames[idx], MAX_BUF, "%s", baseTypeNames[idx].c_str()); - if (PyModule_AddStringConstant(entitydefModule, allBaseTypeNames[idx], allBaseTypeNames[idx])) - { - ERROR_MSG(fmt::format("PyEntityDef::initialize(): Unable to set EntityDef.{} to {}\n", - allBaseTypeNames[idx], allBaseTypeNames[idx])); - - return false; - } - } - - if (!loadAllScripts()) - { - SCRIPT_ERROR_CHECK(); - return false; - } - - while (!g_callContexts.empty()) - g_callContexts.pop(); - - DefContext::allScriptDefContextLineMaps.clear(); - - if (!assemblyContexts(true)) - { - SCRIPT_ERROR_CHECK(); - DefContext::allScriptDefContextMaps.clear(); - return false; - } - - if (!registerEntityDefs()) - { - DefContext::allScriptDefContextMaps.clear(); - return false; - } - - return true; -} - -//------------------------------------------------------------------------------------- -bool finalise(bool isReload) -{ - return true; -} - -//------------------------------------------------------------------------------------- -void reload(bool fullReload) -{ - g_inited = false; -} - -//------------------------------------------------------------------------------------- -bool initializeWatcher() -{ - return true; -} - -//------------------------------------------------------------------------------------- -bool addToStream(MemoryStream* pMemoryStream) -{ - int size = DefContext::allScriptDefContextMaps.size(); - (*pMemoryStream) << size; - - DefContext::DEF_CONTEXT_MAP::iterator iter = DefContext::allScriptDefContextMaps.begin(); - for (; iter != DefContext::allScriptDefContextMaps.end(); ++iter) - { - const std::string& name = iter->first; - DefContext& defContext = iter->second; - - (*pMemoryStream) << name; - - defContext.addToStream(pMemoryStream); - } - - return true; -} - -//------------------------------------------------------------------------------------- -bool createFromStream(MemoryStream* pMemoryStream) -{ - int size = 0; - (*pMemoryStream) >> size; - - for (int i = 0; i < size; ++i) - { - std::string name = ""; - (*pMemoryStream) >> name; - - DefContext defContext; - defContext.createFromStream(pMemoryStream); - DefContext::allScriptDefContextLineMaps[defContext.pyObjectSourceFile] = defContext; - } - - return true; -} - -//------------------------------------------------------------------------------------- - -} -} -} +// Copyright 2008-2018 Yolo Technologies, Inc. All Rights Reserved. https://www.comblockengine.com + +#include +#include +#include + +#include "common.h" +#include "entitydef.h" +#include "datatypes.h" +#include "py_entitydef.h" +#include "scriptdef_module.h" +#include "pyscript/py_platform.h" +#include "pyscript/script.h" +#include "pyscript/copy.h" +#include "resmgr/resmgr.h" +#include "server/serverconfig.h" +#include "server/components.h" + +namespace KBEngine{ namespace script{ namespace entitydef { + +struct CallContext +{ + PyObjectPtr pyArgs; + PyObjectPtr pyKwargs; + std::string optionName; +}; + +static std::stack g_callContexts; +static std::string pyDefModuleName = ""; + +DefContext::DEF_CONTEXT_MAP DefContext::allScriptDefContextMaps; +DefContext::DEF_CONTEXT_MAP DefContext::allScriptDefContextLineMaps; + +static bool g_inited = false; +static int g_order = 1; + +//------------------------------------------------------------------------------------- +/* +static PyObject* __py_array(PyObject* self, PyObject* args) +{ + if (PyTuple_GET_SIZE(args) == 0) + { + PyErr_Format(PyExc_AssertionError, "EntityDef.ARRAY: does not set itemType! should be like this \"EntityDef.ARRAY(itemType)\"\n"); + return NULL; + } + + PyObject *entitydefModule = PyImport_AddModule(pyDefModuleName.c_str()); + PyObject* pyARRAY = PyObject_GetAttrString(entitydefModule, "ARRAY"); + + PyObject* pyArrayItemType = PyTuple_GET_ITEM(args, 0); + Py_INCREF(pyArrayItemType); + + PyObject* ret = PyTuple_New(2); + PyTuple_SET_ITEM(ret, 0, pyARRAY); + PyTuple_SET_ITEM(ret, 1, pyArrayItemType); + + return ret; +} +*/ + +//------------------------------------------------------------------------------------- +class Entity : public script::ScriptObject +{ + BASE_SCRIPT_HREADER(Entity, ScriptObject) +public: + Entity(PyTypeObject* pyType = getScriptType(), bool isInitialised = true) : + ScriptObject(pyType, isInitialised) {} + ~Entity() {} +}; + +SCRIPT_METHOD_DECLARE_BEGIN(Entity) +SCRIPT_METHOD_DECLARE_END() + +SCRIPT_MEMBER_DECLARE_BEGIN(Entity) +SCRIPT_MEMBER_DECLARE_END() + +SCRIPT_GETSET_DECLARE_BEGIN(Entity) +SCRIPT_GETSET_DECLARE_END() +BASE_SCRIPT_INIT(Entity, 0, 0, 0, 0, 0) + +//------------------------------------------------------------------------------------- +class Space : public script::ScriptObject +{ + BASE_SCRIPT_HREADER(Space, ScriptObject) +public: + Space(PyTypeObject* pyType = getScriptType(), bool isInitialised = true) : + ScriptObject(pyType, isInitialised) {} + ~Space() {} +}; + +SCRIPT_METHOD_DECLARE_BEGIN(Space) +SCRIPT_METHOD_DECLARE_END() + +SCRIPT_MEMBER_DECLARE_BEGIN(Space) +SCRIPT_MEMBER_DECLARE_END() + +SCRIPT_GETSET_DECLARE_BEGIN(Space) +SCRIPT_GETSET_DECLARE_END() +BASE_SCRIPT_INIT(Space, 0, 0, 0, 0, 0) + +//------------------------------------------------------------------------------------- +class Proxy : public script::ScriptObject +{ + BASE_SCRIPT_HREADER(Proxy, ScriptObject) +public: + Proxy(PyTypeObject* pyType = getScriptType(), bool isInitialised = true) : + ScriptObject(pyType, isInitialised) {} + ~Proxy() {} +}; + +SCRIPT_METHOD_DECLARE_BEGIN(Proxy) +SCRIPT_METHOD_DECLARE_END() + +SCRIPT_MEMBER_DECLARE_BEGIN(Proxy) +SCRIPT_MEMBER_DECLARE_END() + +SCRIPT_GETSET_DECLARE_BEGIN(Proxy) +SCRIPT_GETSET_DECLARE_END() +BASE_SCRIPT_INIT(Proxy, 0, 0, 0, 0, 0) + +//------------------------------------------------------------------------------------- +class EntityComponent : public script::ScriptObject +{ + BASE_SCRIPT_HREADER(EntityComponent, ScriptObject) +public: + EntityComponent(PyTypeObject* pyType = getScriptType(), bool isInitialised = true) : + ScriptObject(pyType, isInitialised) {} + ~EntityComponent() {} +}; + +SCRIPT_METHOD_DECLARE_BEGIN(EntityComponent) +SCRIPT_METHOD_DECLARE_END() + +SCRIPT_MEMBER_DECLARE_BEGIN(EntityComponent) +SCRIPT_MEMBER_DECLARE_END() + +SCRIPT_GETSET_DECLARE_BEGIN(EntityComponent) +SCRIPT_GETSET_DECLARE_END() +BASE_SCRIPT_INIT(EntityComponent, 0, 0, 0, 0, 0) + +//------------------------------------------------------------------------------------- +DefContext::DefContext() +{ + order = g_order++; + + optionName = ""; + + moduleName = ""; + attrName = ""; + returnType = ""; + + isModuleScope = false; + + exposed = false; + hasClient = false; + persistent = -1; + databaseLength = 0; + utype = -1; + detailLevel = ""; + + propertyFlags = ""; + propertyIndex = ""; + propertyDefaultVal = ""; + + implementedByModuleName = ""; + implementedByModuleFile = ""; + pyObjectSourceFile = ""; + + inheritEngineModuleType = DC_TYPE_UNKNOWN; + type = DC_TYPE_UNKNOWN; + + componentType = UNKNOWN_COMPONENT_TYPE; +} + +//------------------------------------------------------------------------------------- +bool DefContext::addToStream(MemoryStream* pMemoryStream) +{ + (*pMemoryStream) << order; + (*pMemoryStream) << optionName; + (*pMemoryStream) << moduleName; + (*pMemoryStream) << attrName; + (*pMemoryStream) << returnType; + + (*pMemoryStream) << (int)argsvecs.size(); + std::vector< std::string >::iterator argsvecsIter = argsvecs.begin(); + for(; argsvecsIter != argsvecs.end(); ++argsvecsIter) + (*pMemoryStream) << (*argsvecsIter); + + (*pMemoryStream) << (int)annotationsMaps.size(); + std::map< std::string, std::string >::iterator annotationsMapsIter = annotationsMaps.begin(); + for (; annotationsMapsIter != annotationsMaps.end(); ++annotationsMapsIter) + (*pMemoryStream) << annotationsMapsIter->first << annotationsMapsIter->second; + + (*pMemoryStream) << isModuleScope; + (*pMemoryStream) << exposed; + (*pMemoryStream) << hasClient; + + (*pMemoryStream) << persistent; + (*pMemoryStream) << databaseLength; + (*pMemoryStream) << utype; + (*pMemoryStream) << detailLevel; + + (*pMemoryStream) << propertyFlags; + (*pMemoryStream) << propertyIndex; + (*pMemoryStream) << propertyDefaultVal; + + (*pMemoryStream) << implementedByModuleName; + (*pMemoryStream) << implementedByModuleFile; + (*pMemoryStream) << pyObjectSourceFile; + + (*pMemoryStream) << (int)baseClasses.size(); + std::vector< std::string >::iterator baseClassesIter = baseClasses.begin(); + for (; baseClassesIter != baseClasses.end(); ++baseClassesIter) + (*pMemoryStream) << (*baseClassesIter); + + (*pMemoryStream) << (int)inheritEngineModuleType; + (*pMemoryStream) << (int)type; + + (*pMemoryStream) << (int)base_methods.size(); + std::vector< DefContext >::iterator base_methodsIter = base_methods.begin(); + for (; base_methodsIter != base_methods.end(); ++base_methodsIter) + (*base_methodsIter).addToStream(pMemoryStream); + + (*pMemoryStream) << (int)cell_methods.size(); + std::vector< DefContext >::iterator cell_methodsIter = cell_methods.begin(); + for (; cell_methodsIter != cell_methods.end(); ++cell_methodsIter) + (*cell_methodsIter).addToStream(pMemoryStream); + + (*pMemoryStream) << (int)client_methods.size(); + std::vector< DefContext >::iterator client_methodsIter = client_methods.begin(); + for (; client_methodsIter != client_methods.end(); ++client_methodsIter) + (*client_methodsIter).addToStream(pMemoryStream); + + (*pMemoryStream) << (int)propertys.size(); + std::vector< DefContext >::iterator propertysIter = propertys.begin(); + for (; propertysIter != propertys.end(); ++propertysIter) + (*propertysIter).addToStream(pMemoryStream); + + (*pMemoryStream) << (int)components.size(); + std::vector< std::string >::iterator componentsIter = components.begin(); + for (; componentsIter != components.end(); ++componentsIter) + (*pMemoryStream) << (*componentsIter); + + return true; +} + +//------------------------------------------------------------------------------------- +bool DefContext::createFromStream(MemoryStream* pMemoryStream) +{ + (*pMemoryStream) >> order; + (*pMemoryStream) >> optionName; + (*pMemoryStream) >> moduleName; + (*pMemoryStream) >> attrName; + (*pMemoryStream) >> returnType; + + int size = 0; + + (*pMemoryStream) >> size; + for (int i = 0; i < size; ++i) + { + std::string str; + (*pMemoryStream) >> str; + + argsvecs.push_back(str); + } + + (*pMemoryStream) >> size; + for (int i = 0; i < size; ++i) + { + std::string key, val; + (*pMemoryStream) >> key >> val; + + annotationsMaps[key] = val; + } + + (*pMemoryStream) >> isModuleScope; + (*pMemoryStream) >> exposed; + (*pMemoryStream) >> hasClient; + + (*pMemoryStream) >> persistent; + (*pMemoryStream) >> databaseLength; + (*pMemoryStream) >> utype; + (*pMemoryStream) >> detailLevel; + + (*pMemoryStream) >> propertyFlags; + (*pMemoryStream) >> propertyIndex; + (*pMemoryStream) >> propertyDefaultVal; + + (*pMemoryStream) >> implementedByModuleName; + (*pMemoryStream) >> implementedByModuleFile; + (*pMemoryStream) >> pyObjectSourceFile; + + (*pMemoryStream) >> size; + for (int i = 0; i < size; ++i) + { + std::string str; + (*pMemoryStream) >> str; + + baseClasses.push_back(str); + } + + int t_inheritEngineModuleType; + (*pMemoryStream) >> t_inheritEngineModuleType; + inheritEngineModuleType = (DCType)t_inheritEngineModuleType; + + int t_type; + (*pMemoryStream) >> t_type; + type = (DCType)t_type; + + (*pMemoryStream) >> size; + for (int i = 0; i < size; ++i) + { + DefContext dc; + dc.createFromStream(pMemoryStream); + + base_methods.push_back(dc); + } + + (*pMemoryStream) >> size; + for (int i = 0; i < size; ++i) + { + DefContext dc; + dc.createFromStream(pMemoryStream); + + cell_methods.push_back(dc); + } + + (*pMemoryStream) >> size; + for (int i = 0; i < size; ++i) + { + DefContext dc; + dc.createFromStream(pMemoryStream); + + client_methods.push_back(dc); + } + + (*pMemoryStream) >> size; + for (int i = 0; i < size; ++i) + { + DefContext dc; + dc.createFromStream(pMemoryStream); + + propertys.push_back(dc); + } + + (*pMemoryStream) >> size; + for (int i = 0; i < size; ++i) + { + std::string str; + (*pMemoryStream) >> str; + + components.push_back(str); + } + + return true; +} + +//------------------------------------------------------------------------------------- +bool DefContext::addChildContext(DefContext& defContext) +{ + std::vector< DefContext >* pContexts = NULL; + + if (defContext.type == DefContext::DC_TYPE_PROPERTY) + { + pContexts = &propertys; + } + else if (defContext.type == DefContext::DC_TYPE_METHOD) + { + if(defContext.componentType == BASEAPP_TYPE) + pContexts = &base_methods; + else + pContexts = &cell_methods; + } + else if (defContext.type == DefContext::DC_TYPE_CLIENT_METHOD) + { + pContexts = &client_methods; + } + else if (defContext.type == DefContext::DC_TYPE_FIXED_ITEM) + { + pContexts = &propertys; + } + else + { + KBE_ASSERT(false); + } + + std::vector< DefContext >::iterator iter = pContexts->begin(); + for (; iter != pContexts->end(); ++iter) + { + if ((*iter).attrName == defContext.attrName) + { + // assemblyContextsʱܻᷢ + // + if (moduleName != defContext.moduleName || (*iter).pyObjectSourceFile == defContext.pyObjectSourceFile) + return true; + + return false; + } + } + + pContexts->push_back(defContext); + return true; +} + +//------------------------------------------------------------------------------------- +static bool assemblyContexts(bool notfoundModuleError = false) +{ + std::vector< std::string > dels; + + DefContext::DEF_CONTEXT_MAP::iterator iter = DefContext::allScriptDefContextMaps.begin(); + for (; iter != DefContext::allScriptDefContextMaps.end(); ++iter) + { + DefContext& defContext = iter->second; + + if (defContext.type == DefContext::DC_TYPE_PROPERTY || + defContext.type == DefContext::DC_TYPE_METHOD || + defContext.type == DefContext::DC_TYPE_CLIENT_METHOD || + defContext.type == DefContext::DC_TYPE_FIXED_ITEM) + { + DefContext::DEF_CONTEXT_MAP::iterator fiter = DefContext::allScriptDefContextMaps.find(defContext.moduleName); + if (fiter == DefContext::allScriptDefContextMaps.end()) + { + if (notfoundModuleError) + { + PyErr_Format(PyExc_AssertionError, "PyEntityDef::process(): No \'%s\' module defined!\n", defContext.moduleName.c_str()); + return false; + } + + return true; + } + + if (!fiter->second.addChildContext(defContext)) + { + PyErr_Format(PyExc_AssertionError, "\'%s.%s\' already exists!\n", + fiter->second.moduleName.c_str(), defContext.attrName.c_str()); + + return false; + } + + dels.push_back(iter->first); + } + else + { + } + } + + std::vector< std::string >::iterator diter = dels.begin(); + for (; diter != dels.end(); ++diter) + { + DefContext::allScriptDefContextMaps.erase((*diter)); + } + + // ԽϢ䵽 + iter = DefContext::allScriptDefContextMaps.begin(); + for (; iter != DefContext::allScriptDefContextMaps.end(); ++iter) + { + DefContext& defContext = iter->second; + if (defContext.baseClasses.size() > 0) + { + for (size_t i = 0; i < defContext.baseClasses.size(); ++i) + { + std::string parentClass = defContext.baseClasses[i]; + + DefContext::DEF_CONTEXT_MAP::iterator fiter = DefContext::allScriptDefContextMaps.find(parentClass); + if (fiter == DefContext::allScriptDefContextMaps.end()) + { + //PyErr_Format(PyExc_AssertionError, "not found parentClass(\'%s\')!\n", parentClass.c_str()); + //return false; + continue; + } + + DefContext& parentDefContext = fiter->second; + std::vector< DefContext > childContexts; + + childContexts.insert(childContexts.end(), parentDefContext.base_methods.begin(), parentDefContext.base_methods.end()); + childContexts.insert(childContexts.end(), parentDefContext.cell_methods.begin(), parentDefContext.cell_methods.end()); + childContexts.insert(childContexts.end(), parentDefContext.client_methods.begin(), parentDefContext.client_methods.end()); + childContexts.insert(childContexts.end(), parentDefContext.propertys.begin(), parentDefContext.propertys.end()); + + std::vector< DefContext >::iterator itemIter = childContexts.begin(); + for (; itemIter != childContexts.end(); ++itemIter) + { + DefContext& parentDefContext = (*itemIter); + if (!defContext.addChildContext(parentDefContext)) + { + PyErr_Format(PyExc_AssertionError, "\'%s.%s\' already exists(%s.%s)!\n", + defContext.moduleName.c_str(), parentDefContext.attrName.c_str(), parentDefContext.moduleName.c_str(), parentDefContext.attrName.c_str()); + + return false; + } + } + } + } + } + + return true; +} + +//------------------------------------------------------------------------------------- +static bool registerDefContext(DefContext& defContext) +{ + DefContext::allScriptDefContextLineMaps[defContext.pyObjectSourceFile] = defContext; + + std::string name = defContext.moduleName; + + if (defContext.type == DefContext::DC_TYPE_PROPERTY) + { + if(!EntityDef::validDefPropertyName(name)) + { + PyErr_Format(PyExc_AssertionError, "EntityDef.%s: '%s' is limited!\n\n", + defContext.optionName.c_str(), name.c_str()); + + return false; + } + + // Ƿڸý + bool flagsGood = true; + + if (defContext.componentType == BASEAPP_TYPE) + flagsGood = (stringToEntityDataFlags(defContext.propertyFlags) & ENTITY_BASE_DATA_FLAGS) != 0; + else if (defContext.componentType == CELLAPP_TYPE) + flagsGood = (stringToEntityDataFlags(defContext.propertyFlags) & ENTITY_CELL_DATA_FLAGS) != 0; + else if (defContext.componentType == CLIENT_TYPE) + flagsGood = (stringToEntityDataFlags(defContext.propertyFlags) & ENTITY_CLIENT_DATA_FLAGS) != 0; + + name += "." + defContext.attrName; + + if (!flagsGood) + { + PyErr_Format(PyExc_AssertionError, "EntityDef.%s: '%s'(%s) not a valid %s property flags!\n\n", + defContext.optionName.c_str(), name.c_str(), defContext.propertyFlags.c_str(), COMPONENT_NAME_EX(defContext.componentType)); + + return false; + } + } + else if(defContext.type == DefContext::DC_TYPE_METHOD || + defContext.type == DefContext::DC_TYPE_CLIENT_METHOD) + { + name += "." + defContext.attrName; + } + else if (defContext.type == DefContext::DC_TYPE_FIXED_ITEM) + { + name += "." + defContext.attrName; + } + else if (defContext.type == DefContext::DC_TYPE_FIXED_ARRAY || + defContext.type == DefContext::DC_TYPE_FIXED_DICT || + defContext.type == DefContext::DC_TYPE_RENAME) + { + if (!DataTypes::validTypeName(name)) + { + PyErr_Format(PyExc_AssertionError, "EntityDef.%s: Not allowed to use the prefix \"_\"! typeName=%s\n", + defContext.optionName.c_str(), name.c_str()); + + return false; + } + } + + DefContext::DEF_CONTEXT_MAP::iterator iter = DefContext::allScriptDefContextMaps.find(name); + if (iter != DefContext::allScriptDefContextMaps.end()) + { + if (iter->second.pyObjectSourceFile != defContext.pyObjectSourceFile) + { + // Dz̵ͬĽű֣ôҪкϲע + if (iter->second.componentType != defContext.componentType && DefContext::allScriptDefContextLineMaps.find(defContext.pyObjectSourceFile) != DefContext::allScriptDefContextLineMaps.end() && + (defContext.type == DefContext::DC_TYPE_ENTITY || defContext.type == DefContext::DC_TYPE_COMPONENT || defContext.type == DefContext::DC_TYPE_INTERFACE) && + iter->second.type == defContext.type) + { + std::vector< std::string >::iterator bciter = defContext.baseClasses.begin(); + for (; bciter != defContext.baseClasses.end(); ++bciter) + { + if (std::find(iter->second.baseClasses.begin(), iter->second.baseClasses.end(), (*bciter)) == iter->second.baseClasses.end()) + { + iter->second.baseClasses.push_back((*bciter)); + } + + KBE_ASSERT(defContext.base_methods.size() == 0 && + defContext.cell_methods.size() == 0 && + defContext.client_methods.size() == 0 && + defContext.components.size() == 0 && + defContext.propertys.size() == 0); + + if (defContext.hasClient) + iter->second.hasClient = true; + } + + // űǿΪǰ̵Ķ + if (g_componentType == defContext.componentType) + { + iter->second.pyObjectPtr = defContext.pyObjectPtr; + iter->second.pyObjectSourceFile = defContext.pyObjectSourceFile; + } + + return true; + } + + PyErr_Format(PyExc_AssertionError, "EntityDef.%s: \'%s\' already exists!\n", + defContext.optionName.c_str(), name.c_str()); + + return false; + } + + return true; + } + + DefContext::allScriptDefContextMaps[name] = defContext; + return assemblyContexts(); +} + +//------------------------------------------------------------------------------------- + +static bool onDefRename(DefContext& defContext) +{ + defContext.type = DefContext::DC_TYPE_RENAME; + return registerDefContext(defContext); +} + +//------------------------------------------------------------------------------------- +static bool onDefFixedDict(DefContext& defContext) +{ + defContext.type = DefContext::DC_TYPE_FIXED_DICT; + return registerDefContext(defContext); +} + +//------------------------------------------------------------------------------------- +static bool onDefFixedArray(DefContext& defContext) +{ + defContext.type = DefContext::DC_TYPE_FIXED_ARRAY; + return registerDefContext(defContext); +} + +//------------------------------------------------------------------------------------- +static bool onDefFixedItem(DefContext& defContext) +{ + defContext.type = DefContext::DC_TYPE_FIXED_ITEM; + return registerDefContext(defContext); +} + +//------------------------------------------------------------------------------------- +static bool onDefProperty(DefContext& defContext) +{ + defContext.type = DefContext::DC_TYPE_PROPERTY; + return registerDefContext(defContext); +} + +//------------------------------------------------------------------------------------- +static bool onDefMethod(DefContext& defContext) +{ + defContext.type = DefContext::DC_TYPE_METHOD; + return registerDefContext(defContext); +} + +//------------------------------------------------------------------------------------- +static bool onDefClientMethod(DefContext& defContext) +{ + defContext.type = DefContext::DC_TYPE_CLIENT_METHOD; + return registerDefContext(defContext); +} + +//------------------------------------------------------------------------------------- +static bool onDefEntity(DefContext& defContext) +{ + defContext.type = DefContext::DC_TYPE_ENTITY; + return registerDefContext(defContext); +} + +//------------------------------------------------------------------------------------- +static bool onDefInterface(DefContext& defContext) +{ + defContext.type = DefContext::DC_TYPE_INTERFACE; + return registerDefContext(defContext); +} + +//------------------------------------------------------------------------------------- +static bool onDefComponent(DefContext& defContext) +{ + defContext.type = DefContext::DC_TYPE_COMPONENT; + return registerDefContext(defContext); +} + +//------------------------------------------------------------------------------------- +static bool isRefEntityDefModule(PyObject *pyObj) +{ + if(!pyObj) + return true; + + PyObject *entitydefModule = PyImport_AddModule(pyDefModuleName.c_str()); + PyObject* pydict = PyObject_GetAttrString(entitydefModule, "__dict__"); + + PyObject *key, *value; + Py_ssize_t pos = 0; + + while (PyDict_Next(pydict, &pos, &key, &value)) { + if (value == pyObj) + { + Py_DECREF(pydict); + return true; + } + } + + Py_DECREF(pydict); + return false; +} + +//------------------------------------------------------------------------------------- +#define PY_RETURN_ERROR { DefContext::allScriptDefContextLineMaps.clear(); DefContext::allScriptDefContextMaps.clear(); while(!g_callContexts.empty()) g_callContexts.pop(); return NULL; } + +#define PYOBJECT_SOURCEFILE(PYOBJ, OUT) \ +{ \ + PyObject* pyInspectModule = \ + PyImport_ImportModule(const_cast("inspect")); \ + \ + PyObject* pyGetsourcefile = NULL; \ + PyObject* pyGetLineno = NULL; \ + PyObject* pyGetCurrentFrame = NULL; \ + \ + if (pyInspectModule) \ + { \ + pyGetsourcefile = \ + PyObject_GetAttrString(pyInspectModule, const_cast("getsourcefile")); \ + pyGetLineno = \ + PyObject_GetAttrString(pyInspectModule, const_cast("getlineno")); \ + pyGetCurrentFrame = \ + PyObject_GetAttrString(pyInspectModule, const_cast("currentframe")); \ + \ + Py_DECREF(pyInspectModule); \ + } \ + else \ + { \ + PY_RETURN_ERROR; \ + } \ + \ + if (pyGetsourcefile) \ + { \ + PyObject* pyFile = PyObject_CallFunction(pyGetsourcefile, \ + const_cast("(O)"), PYOBJ); \ + \ + Py_DECREF(pyGetsourcefile); \ + \ + if (!pyFile) \ + { \ + Py_XDECREF(pyGetLineno); \ + Py_XDECREF(pyGetCurrentFrame); \ + PY_RETURN_ERROR; \ + } \ + else \ + { \ + /* ֹͬϵͳɵ·һ£޳ϵͳ· */ \ + OUT = PyUnicode_AsUTF8AndSize(pyFile, NULL); \ + strutil::kbe_replace(OUT, "\\\\", "/"); \ + strutil::kbe_replace(OUT, "\\", "/"); \ + strutil::kbe_replace(OUT, "//", "/"); \ + std::string kbe_root = Resmgr::getSingleton().getPyUserScriptsPath(); \ + strutil::kbe_replace(kbe_root, "\\\\", "/"); \ + strutil::kbe_replace(kbe_root, "\\", "/"); \ + strutil::kbe_replace(kbe_root, "/", "/"); \ + strutil::kbe_replace(OUT, kbe_root, ""); \ + Py_DECREF(pyFile); \ + } \ + } \ + else \ + { \ + Py_XDECREF(pyGetLineno); \ + Py_XDECREF(pyGetCurrentFrame); \ + PY_RETURN_ERROR; \ + } \ + \ + if (pyGetLineno) \ + { \ + if(!pyGetCurrentFrame) \ + { \ + Py_DECREF(pyGetLineno); \ + } \ + \ + PyObject* pyCurrentFrame = PyObject_CallFunction(pyGetCurrentFrame, \ + const_cast("()")); \ + \ + PyObject* pyLine = PyObject_CallFunction(pyGetLineno, \ + const_cast("(O)"), pyCurrentFrame); \ + \ + Py_DECREF(pyGetLineno); \ + Py_DECREF(pyGetCurrentFrame); \ + Py_DECREF(pyCurrentFrame); \ + \ + if (!pyLine) \ + { \ + PY_RETURN_ERROR; \ + } \ + else \ + { \ + /* кţͬļжζ */ \ + OUT += fmt::format("#{}", PyLong_AsLong(pyLine)); \ + Py_DECREF(pyLine); \ + } \ + } \ + else \ + { \ + Py_XDECREF(pyGetCurrentFrame); \ + PY_RETURN_ERROR; \ + } \ +} + +static PyObject* __py_def_parse(PyObject *self, PyObject* args) +{ + CallContext cc = g_callContexts.top(); + g_callContexts.pop(); + + DefContext defContext; + defContext.optionName = cc.optionName; + + PyObject* kbeModule = PyImport_AddModule("KBEngine"); + KBE_ASSERT(kbeModule); + + PyObject* pyComponentName = PyObject_GetAttrString(kbeModule, "component"); + if (!pyComponentName) + { + PyErr_Format(PyExc_AssertionError, "EntityDef.__py_def_call(): get KBEngine.component error!\n"); + PY_RETURN_ERROR; + } + + defContext.componentType = ComponentName2ComponentType(PyUnicode_AsUTF8AndSize(pyComponentName, NULL)); + Py_DECREF(pyComponentName); + + if (!args || PyTuple_Size(args) < 1) + { + PyErr_Format(PyExc_AssertionError, "EntityDef.__py_def_call(EntityDef.%s): error!\n", defContext.optionName.c_str()); + PY_RETURN_ERROR; + } + + PyObject* pyFunc = PyTuple_GET_ITEM(args, 0); + + PyObject* pyModuleQualname = PyObject_GetAttrString(pyFunc, "__qualname__"); + if (!pyModuleQualname) + { + PY_RETURN_ERROR; + } + + const char* moduleQualname = PyUnicode_AsUTF8AndSize(pyModuleQualname, NULL); + Py_DECREF(pyModuleQualname); + + defContext.pyObjectPtr = PyObjectPtr(pyFunc); + PYOBJECT_SOURCEFILE(defContext.pyObjectPtr.get(), defContext.pyObjectSourceFile); + + if (defContext.optionName == "method" || defContext.optionName == "clientmethod") + { + static char * keywords[] = + { + const_cast ("exposed"), + const_cast ("utype"), + NULL + }; + + PyObject* pyExposed = NULL; + PyObject* pyUtype = NULL; + + if (!PyArg_ParseTupleAndKeywords(cc.pyArgs.get(), cc.pyKwargs.get(), "|OO", + keywords, &pyExposed, &pyUtype)) + { + PY_RETURN_ERROR; + } + + if (pyExposed && !PyBool_Check(pyExposed)) + { + PyErr_Format(PyExc_AssertionError, "EntityDef.%s: \'exposed\' error! not a bool type.\n", defContext.optionName.c_str()); + PY_RETURN_ERROR; + } + + if (pyUtype && !PyLong_Check(pyUtype)) + { + PyErr_Format(PyExc_AssertionError, "EntityDef.%s: \'utype\' error! not a number type.\n", defContext.optionName.c_str()); + PY_RETURN_ERROR; + } + + defContext.exposed = pyExposed == Py_True; + + if (pyUtype) + { + defContext.utype = (int)PyLong_AsLong(pyUtype); + + if (defContext.utype > 0) + { + ENTITY_METHOD_UID muid = defContext.utype; + + if (defContext.utype != int(muid)) + { + PyErr_Format(PyExc_AssertionError, "EntityDef.%s: 'Utype' has overflowed({} > 65535), is {}.{}!\n", defContext.optionName.c_str(), defContext.utype); + PY_RETURN_ERROR; + } + } + } + } + else if (defContext.optionName == "property" || defContext.optionName == "fixed_item") + { + if (defContext.optionName != "fixed_item") + { + static char * keywords[] = + { + const_cast ("flags"), + const_cast ("persistent"), + const_cast ("index"), + const_cast ("databaseLength"), + const_cast ("utype"), + NULL + }; + + PyObject* pyFlags = NULL; + PyObject* pyPersistent = NULL; + PyObject* pyIndex = NULL; + PyObject* pyDatabaseLength = NULL; + PyObject* pyUtype = NULL; + + if (!PyArg_ParseTupleAndKeywords(cc.pyArgs.get(), cc.pyKwargs.get(), "|OOOOO", + keywords, &pyFlags, &pyPersistent, &pyIndex, &pyDatabaseLength, &pyUtype)) + { + PY_RETURN_ERROR; + } + + if (!pyFlags || !isRefEntityDefModule(pyFlags)) + { + PyErr_Format(PyExc_AssertionError, "EntityDef.%s: \'flags\' must be referenced from the [EntityDef.ALL_CLIENTS, EntityDef.*] module!\n", defContext.optionName.c_str()); + PY_RETURN_ERROR; + } + + if (!isRefEntityDefModule(pyIndex)) + { + PyErr_Format(PyExc_AssertionError, "EntityDef.%s: \'index\' must be referenced from the [EntityDef.UNIQUE, EntityDef.INDEX] module!\n", defContext.optionName.c_str()); + PY_RETURN_ERROR; + } + + if (pyDatabaseLength && !PyLong_Check(pyDatabaseLength)) + { + PyErr_Format(PyExc_AssertionError, "EntityDef.%s: \'databaseLength\' error! not a number type.\n", defContext.optionName.c_str()); + PY_RETURN_ERROR; + } + + if (pyPersistent && !PyBool_Check(pyPersistent)) + { + PyErr_Format(PyExc_AssertionError, "EntityDef.%s: \'persistent\' error! not a bool type.\n", defContext.optionName.c_str()); + PY_RETURN_ERROR; + } + + if(pyUtype && !PyLong_Check(pyUtype)) + { + PyErr_Format(PyExc_AssertionError, "EntityDef.%s: \'utype\' error! not a number type.\n", defContext.optionName.c_str()); + PY_RETURN_ERROR; + } + + defContext.propertyFlags = PyUnicode_AsUTF8AndSize(pyFlags, NULL); + + if (pyPersistent) + defContext.persistent = pyPersistent == Py_True; + + if (pyIndex) + defContext.propertyIndex = PyUnicode_AsUTF8AndSize(pyIndex, NULL); + + if (pyDatabaseLength) + defContext.databaseLength = (int)PyLong_AsLong(pyDatabaseLength); + + if (pyUtype) + { + defContext.utype = (int)PyLong_AsLong(pyUtype); + + if (defContext.utype > 0) + { + ENTITY_PROPERTY_UID muid = defContext.utype; + + if (defContext.utype != int(muid)) + { + PyErr_Format(PyExc_AssertionError, "EntityDef.%s: 'Utype' has overflowed({} > 65535), is {}.{}!\n", defContext.optionName.c_str(), defContext.utype); + PY_RETURN_ERROR; + } + } + } + } + else + { + static char * keywords[] = + { + const_cast ("persistent"), + const_cast ("databaseLength"), + NULL + }; + + PyObject* pyPersistent = NULL; + PyObject* pyDatabaseLength = NULL; + + if (!PyArg_ParseTupleAndKeywords(cc.pyArgs.get(), cc.pyKwargs.get(), "|OO", + keywords, &pyPersistent, &pyDatabaseLength)) + { + PY_RETURN_ERROR; + } + + if (pyDatabaseLength && !PyLong_Check(pyDatabaseLength)) + { + PyErr_Format(PyExc_AssertionError, "EntityDef.%s: \'databaseLength\' error! not a number type.\n", defContext.optionName.c_str()); + PY_RETURN_ERROR; + } + + if (pyPersistent && !PyBool_Check(pyPersistent)) + { + PyErr_Format(PyExc_AssertionError, "EntityDef.%s: \'persistent\' error! not a bool type.\n", defContext.optionName.c_str()); + PY_RETURN_ERROR; + } + + // Ͷ壬ĬӦǴ洢 + // fixed_itemֻжfixed_dictʱõ + defContext.persistent = true; + + if (pyPersistent) + defContext.persistent = pyPersistent == Py_True; + + if (pyDatabaseLength) + defContext.databaseLength = (int)PyLong_AsLong(pyDatabaseLength); + } + + // ԣ Ҫ÷ֵΪĬֵ + PyObject* pyRet = PyObject_CallFunction(pyFunc, + const_cast("(O)"), Py_None); + + if (!pyRet) + return NULL; + + if (pyRet != Py_None) + { + PyObject* pyStrResult = PyObject_Str(pyRet); + Py_DECREF(pyRet); + + defContext.propertyDefaultVal = PyUnicode_AsUTF8AndSize(pyStrResult, NULL); + Py_DECREF(pyStrResult); + + // ַ֤ǷԻԭɶ + if (defContext.propertyDefaultVal.size() > 0) + { + PyObject* module = PyImport_AddModule("__main__"); + if (module == NULL) + return NULL; + + PyObject* mdict = PyModule_GetDict(module); // Borrowed reference. + PyObject* result = PyRun_String(const_cast(defContext.propertyDefaultVal.c_str()), + Py_eval_input, mdict, mdict); + + if (result == NULL) + return NULL; + + Py_DECREF(result); + } + } + else + { + Py_DECREF(pyRet); + } + } + else if (defContext.optionName == "entity") + { + defContext.isModuleScope = true; + + static char * keywords[] = + { + const_cast ("hasClient"), + NULL + }; + + PyObject* pyHasClient = NULL; + + if (!PyArg_ParseTupleAndKeywords(cc.pyArgs.get(), cc.pyKwargs.get(), "|O", + keywords, &pyHasClient)) + { + PY_RETURN_ERROR; + } + + defContext.hasClient = pyHasClient == Py_True; + } + else if (defContext.optionName == "interface") + { + defContext.isModuleScope = true; + defContext.inheritEngineModuleType = DefContext::DC_TYPE_INTERFACE; + } + else if (defContext.optionName == "component") + { + defContext.isModuleScope = true; + } + else if (defContext.optionName == "fixed_dict") + { + defContext.isModuleScope = true; + + if (PyObject_HasAttrString(pyFunc, "createObjFromDict") && PyObject_HasAttrString(pyFunc, "getDictFromObj") && PyObject_HasAttrString(pyFunc, "isSameType")) + { + defContext.implementedBy = pyFunc; + + PyObject* pyQualname = PyObject_GetAttrString(defContext.implementedBy.get(), "__qualname__"); + if (!pyQualname) + { + PY_RETURN_ERROR; + } + + defContext.implementedByModuleName = PyUnicode_AsUTF8AndSize(pyQualname, NULL); + Py_DECREF(pyQualname); + + PYOBJECT_SOURCEFILE(defContext.implementedBy.get(), defContext.implementedByModuleFile); + } + } + else if (defContext.optionName == "fixed_array") + { + // @Def.fixed_array() + // def ENTITYID_LIST()->ENTITY_ID: pass + defContext.isModuleScope = false; + } + else if (defContext.optionName == "fixed_item") + { + } + else if (defContext.optionName == "rename") + { + } + else + { + PyErr_Format(PyExc_AssertionError, "EntityDef.%s: not support!\n", defContext.optionName.c_str()); + PY_RETURN_ERROR; + } + + if (!defContext.isModuleScope) + { + std::vector outs; + + if (moduleQualname) + strutil::kbe_splits(moduleQualname, ".", outs); + + if (defContext.optionName != "rename" && defContext.optionName != "fixed_array") + { + if (outs.size() != 2) + { + if(PyFunction_Check(pyFunc)) + PyErr_Format(PyExc_AssertionError, "EntityDef.%s: \'%s\' must be defined in the entity module!\n", + defContext.optionName.c_str(), moduleQualname); + else + PyErr_Format(PyExc_AssertionError, "EntityDef.%s: please check the command format is: EntityDef.%s(..)\n", + defContext.optionName.c_str(), defContext.optionName.c_str()); + + PY_RETURN_ERROR; + } + + defContext.moduleName = outs[0]; + defContext.attrName = outs[1]; + } + else + { + if (outs.size() != 1) + { + PyErr_Format(PyExc_AssertionError, "EntityDef.%s: error! such as: @EntityDef.rename()\n\tdef ENTITY_ID() -> EntityDef.INT32: pass\n", defContext.optionName.c_str()); + PY_RETURN_ERROR; + } + + defContext.moduleName = outs[0]; + } + + PyObject* pyInspectModule = + PyImport_ImportModule(const_cast("inspect")); + + PyObject* pyGetfullargspec = NULL; + if (pyInspectModule) + { + pyGetfullargspec = + PyObject_GetAttrString(pyInspectModule, const_cast("getfullargspec")); + + Py_DECREF(pyInspectModule); + } + else + { + PY_RETURN_ERROR; + } + + if (pyGetfullargspec) + { + PyObject* pyGetMethodArgs = PyObject_CallFunction(pyGetfullargspec, + const_cast("(O)"), pyFunc); + + if (!pyGetMethodArgs) + { + PY_RETURN_ERROR; + } + else + { + PyObject* pyGetMethodArgsResult = PyObject_GetAttrString(pyGetMethodArgs, const_cast("args")); + PyObject* pyGetMethodAnnotationsResult = PyObject_GetAttrString(pyGetMethodArgs, const_cast("annotations")); + + Py_DECREF(pyGetMethodArgs); + + if (!pyGetMethodArgsResult || !pyGetMethodAnnotationsResult) + { + Py_XDECREF(pyGetMethodArgsResult); + Py_XDECREF(pyGetMethodAnnotationsResult); + PY_RETURN_ERROR; + } + + PyObjectPtr pyGetMethodArgsResultPtr = pyGetMethodArgsResult; + PyObjectPtr pyGetMethodAnnotationsResultPtr = pyGetMethodAnnotationsResult; + Py_DECREF(pyGetMethodArgsResult); + Py_DECREF(pyGetMethodAnnotationsResult); + + if (defContext.optionName != "rename" && defContext.optionName != "fixed_array") + { + Py_ssize_t argsSize = PyList_Size(pyGetMethodArgsResult); + if (argsSize == 0) + { + PyErr_Format(PyExc_AssertionError, "EntityDef.%s: \'%s\' did not find \'self\' parameter!\n", defContext.optionName.c_str(), moduleQualname); + PY_RETURN_ERROR; + } + + for (Py_ssize_t i = 1; i < argsSize; ++i) + { + PyObject* pyItem = PyList_GetItem(pyGetMethodArgsResult, i); + + const char* ccattr = PyUnicode_AsUTF8AndSize(pyItem, NULL); + if (!ccattr) + { + PY_RETURN_ERROR; + } + + defContext.argsvecs.push_back(ccattr); + } + } + + PyObject *key, *value; + Py_ssize_t pos = 0; + + while (PyDict_Next(pyGetMethodAnnotationsResult, &pos, &key, &value)) { + const char* skey = PyUnicode_AsUTF8AndSize(key, NULL); + if (!skey) + { + PY_RETURN_ERROR; + } + + std::string svalue = ""; + + // EntityDef.Array˴һtupleο__py_array + if (PyTuple_Check(value) && PyTuple_Size(value) == 2) + { + PyObject *entitydefModule = PyImport_AddModule(pyDefModuleName.c_str()); + PyObject* pyARRAY = PyObject_GetAttrString(entitydefModule, "ARRAY"); + PyObject* item0 = PyTuple_GET_ITEM(value, 0); + + if (pyARRAY == item0) + { + value = PyTuple_GET_ITEM(value, 1); + defContext.optionName = "anonymous_fixed_array"; + + if (std::string(skey) != "return") + { + PyErr_Format(PyExc_AssertionError, "EntityDef.%s: \'EntityDef.ARRAY\' Can only be used to define property types!\n", defContext.optionName.c_str()); + PY_RETURN_ERROR; + } + } + else + { + Py_DECREF(pyARRAY); + PY_RETURN_ERROR; + } + + Py_DECREF(pyARRAY); + } + + if (PyUnicode_Check(value)) + { + svalue = PyUnicode_AsUTF8AndSize(value, NULL); + } + else + { + PyObject* pyQualname = PyObject_GetAttrString(value, "__qualname__"); + if (!pyQualname) + { + PY_RETURN_ERROR; + } + + svalue = PyUnicode_AsUTF8AndSize(pyQualname, NULL); + Py_DECREF(pyQualname); + } + + if (svalue.size() == 0) + { + PY_RETURN_ERROR; + } + + if (std::string(skey) == "return") + defContext.returnType = svalue; + else + defContext.annotationsMaps[skey] = svalue; + } + } + } + else + { + PY_RETURN_ERROR; + } + } + else + { + defContext.moduleName = moduleQualname; + + PyObject* pyBases = PyObject_GetAttrString(pyFunc, "__bases__"); + if (!pyBases) + { + PY_RETURN_ERROR; + } + + Py_ssize_t basesSize = PyTuple_Size(pyBases); + if (basesSize == 0) + { + PyErr_Format(PyExc_AssertionError, "EntityDef.%s: \'%s\' does not inherit the KBEngine.Entity class!\n", defContext.optionName.c_str(), moduleQualname); + Py_XDECREF(pyBases); + PY_RETURN_ERROR; + } + + for (Py_ssize_t i = 0; i < basesSize; ++i) + { + PyObject* pyClass = PyTuple_GetItem(pyBases, i); + + PyObject* pyQualname = PyObject_GetAttrString(pyClass, "__qualname__"); + if (!pyQualname) + { + Py_XDECREF(pyBases); + PY_RETURN_ERROR; + } + + std::string parentClass = PyUnicode_AsUTF8AndSize(pyQualname, NULL); + Py_DECREF(pyQualname); + + if (parentClass == "object") + { + continue; + } + else if (parentClass == "Entity") + { + defContext.inheritEngineModuleType = DefContext::DC_TYPE_ENTITY; + continue; + } + else if (parentClass == "Proxy") + { + if (defContext.componentType != BASEAPP_TYPE) + { + PyErr_Format(PyExc_AssertionError, "EntityDef.%s: \'%s\' Only BASE can inherit KBEngine.Proxy!\n", defContext.optionName.c_str(), moduleQualname); + PY_RETURN_ERROR; + } + + defContext.inheritEngineModuleType = DefContext::DC_TYPE_ENTITY; + continue; + } + else if (parentClass == "EntityComponent") + { + defContext.inheritEngineModuleType = DefContext::DC_TYPE_COMPONENT; + continue; + } + + defContext.baseClasses.push_back(parentClass); + } + + Py_XDECREF(pyBases); + } + + bool noerror = true; + + if (defContext.optionName == "method" || defContext.optionName == "clientmethod") + { + if (defContext.annotationsMaps.size() != defContext.argsvecs.size()) + { + PyErr_Format(PyExc_AssertionError, "EntityDef.%s: \'%s\' all parameters must have annotations!\n", defContext.optionName.c_str(), moduleQualname); + PY_RETURN_ERROR; + } + + if (defContext.optionName == "method") + noerror = onDefMethod(defContext); + else + noerror = onDefClientMethod(defContext); + } + else if (defContext.optionName == "rename") + { + noerror = onDefRename(defContext); + } + else if (defContext.optionName == "property") + { + noerror = onDefProperty(defContext); + } + else if (defContext.optionName == "entity") + { + noerror = onDefEntity(defContext); + } + else if (defContext.optionName == "interface") + { + noerror = onDefInterface(defContext); + } + else if (defContext.optionName == "component") + { + noerror = onDefComponent(defContext); + } + else if (defContext.optionName == "fixed_dict") + { + noerror = onDefFixedDict(defContext); + } + else if (defContext.optionName == "fixed_array") + { + noerror = onDefFixedArray(defContext); + } + else if (defContext.optionName == "anonymous_fixed_array") + { + // һ + DefContext arrayType; + arrayType = defContext; + arrayType.moduleName += ".anonymous_fixed_array"; + + // ֹ2Դļһ£ײΪظ + arrayType.pyObjectSourceFile += "(array)"; + + // һitem + DefContext itemType; + itemType = arrayType; + itemType.optionName = "fixed_item"; + + // ֹ2Դļһ£ײΪظ + itemType.pyObjectSourceFile += "(array_item)"; + + noerror = onDefFixedItem(itemType); + + arrayType.returnType = ""; + + if(noerror) + noerror = onDefFixedArray(arrayType); + + defContext.returnType = arrayType.moduleName; + + if (noerror) + noerror = onDefFixedItem(defContext); + } + else if (defContext.optionName == "fixed_item") + { + noerror = onDefFixedItem(defContext); + } + + if (!noerror) + { + PY_RETURN_ERROR; + } + + Py_INCREF(pyFunc); + return pyFunc; +} + +//------------------------------------------------------------------------------------- +static PyMethodDef __call_def_parse = { "_PyEntityDefParse", (PyCFunction)&__py_def_parse, METH_VARARGS, 0 }; + +#define PY_DEF_HOOK(NAME) \ + static PyObject* __py_def_##NAME(PyObject* self, PyObject* args, PyObject* kwargs) \ + { \ + CallContext cc; \ + cc.pyArgs = PyObjectPtr(Copy::deepcopy(args)); \ + cc.pyKwargs = kwargs ? PyObjectPtr(Copy::deepcopy(kwargs)) : PyObjectPtr(NULL); \ + cc.optionName = #NAME; \ + g_callContexts.push(cc); \ + Py_XDECREF(cc.pyArgs.get()); \ + Py_XDECREF(cc.pyKwargs.get()); \ + \ + return PyCFunction_New(&__call_def_parse, self); \ + } + +static PyObject* __py_def_rename(PyObject* self, PyObject* args, PyObject* kwargs) +{ + CallContext cc; + cc.pyArgs = PyObjectPtr(Copy::deepcopy(args)); + cc.pyKwargs = kwargs ? PyObjectPtr(Copy::deepcopy(kwargs)) : PyObjectPtr(NULL); + cc.optionName = "rename"; + + Py_XDECREF(cc.pyArgs.get()); + Py_XDECREF(cc.pyKwargs.get()); + + // ֶ巽ʽ EntityDef.rename(ENTITY_ID=EntityDef.INT32) + if (kwargs) + { + PyObject *key, *value; + Py_ssize_t pos = 0; + + while (PyDict_Next(kwargs, &pos, &key, &value)) + { + std::string typeName = ""; + if (!PyUnicode_Check(value)) + { + if (PyType_Check(value)) + { + PyObject* pyQualname = PyObject_GetAttrString(value, "__qualname__"); + if (!pyQualname) + { + PyErr_Format(PyExc_AssertionError, "EntityDef.%s: arg2 get __qualname__ error! such as: EntityDef.rename(ENTITY_ID=EntityDef.INT32)\n", cc.optionName.c_str()); + return NULL; + } + + typeName = PyUnicode_AsUTF8AndSize(pyQualname, NULL); + Py_DECREF(pyQualname); + } + else + { + PyObject* pyName = PyObject_GetAttrString(value, "__name__"); + if (!pyName) + { + PyErr_Format(PyExc_AssertionError, "EntityDef.%s: arg2 get __qualname__ error! such as: EntityDef.rename(ENTITY_ID=EntityDef.INT32)\n", cc.optionName.c_str()); + return NULL; + } + + typeName = PyUnicode_AsUTF8AndSize(pyName, NULL); + Py_DECREF(pyName); + } + } + else + { + typeName = PyUnicode_AsUTF8AndSize(value, NULL); + } + + if (!PyUnicode_Check(key)) + { + PyErr_Format(PyExc_AssertionError, "EntityDef.%s: arg1 must be a string! such as: EntityDef.rename(ENTITY_ID=EntityDef.INT32)\n", cc.optionName.c_str()); + return NULL; + } + + if (!isRefEntityDefModule(value)) + { + DefContext::DEF_CONTEXT_MAP::iterator iter = DefContext::allScriptDefContextMaps.find(typeName); + if (iter == DefContext::allScriptDefContextMaps.end()) + { + PyErr_Format(PyExc_AssertionError, "EntityDef.%s: arg2 not legal type! such as: EntityDef.rename(ENTITY_ID=EntityDef.INT32)\n", cc.optionName.c_str()); + return NULL; + } + } + + DefContext defContext; + defContext.optionName = cc.optionName; + defContext.moduleName = PyUnicode_AsUTF8AndSize(key, NULL); + defContext.returnType = typeName; + + PyObject* kbeModule = PyImport_AddModule("KBEngine"); + KBE_ASSERT(kbeModule); + + PyObject* pyComponentName = PyObject_GetAttrString(kbeModule, "component"); + if (!pyComponentName) + { + PyErr_Format(PyExc_AssertionError, "EntityDef.rename(): get KBEngine.component error!\n"); + PY_RETURN_ERROR; + } + + defContext.componentType = ComponentName2ComponentType(PyUnicode_AsUTF8AndSize(pyComponentName, NULL)); + Py_DECREF(pyComponentName); + + if (!onDefRename(defContext)) + return NULL; + } + + S_Return; + } + + g_callContexts.push(cc); + + // @EntityDef.rename() + // def ENTITY_ID() -> int: pass + return PyCFunction_New(&__call_def_parse, self); +} + +static PyObject* __py_def_fixed_array(PyObject* self, PyObject* args, PyObject* kwargs) +{ + CallContext cc; + cc.pyArgs = PyObjectPtr(Copy::deepcopy(args)); + cc.pyKwargs = kwargs ? PyObjectPtr(Copy::deepcopy(kwargs)) : PyObjectPtr(NULL); + cc.optionName = "fixed_array"; + + Py_XDECREF(cc.pyArgs.get()); + Py_XDECREF(cc.pyKwargs.get()); + + // ֶ巽ʽ EntityDef.fixed_array(XXArray=EntityDef.INT32) + if (kwargs) + { + PyObject *key, *value; + Py_ssize_t pos = 0; + + while (PyDict_Next(kwargs, &pos, &key, &value)) + { + std::string typeName = ""; + if (!PyUnicode_Check(value)) + { + if (PyType_Check(value)) + { + PyObject* pyQualname = PyObject_GetAttrString(value, "__qualname__"); + if (!pyQualname) + { + PyErr_Format(PyExc_AssertionError, "EntityDef.%s: arg2 get __qualname__ error! such as: EntityDef.fixed_array(XXArray=EntityDef.INT32)\n", cc.optionName.c_str()); + return NULL; + } + + typeName = PyUnicode_AsUTF8AndSize(pyQualname, NULL); + Py_DECREF(pyQualname); + } + else + { + PyObject* pyName = PyObject_GetAttrString(value, "__name__"); + if (!pyName) + { + PyErr_Format(PyExc_AssertionError, "EntityDef.%s: arg2 get __qualname__ error! such as: EntityDef.fixed_array(XXArray=EntityDef.INT32)\n", cc.optionName.c_str()); + return NULL; + } + + typeName = PyUnicode_AsUTF8AndSize(pyName, NULL); + Py_DECREF(pyName); + } + } + else + { + typeName = PyUnicode_AsUTF8AndSize(value, NULL); + } + + if (!PyUnicode_Check(key)) + { + PyErr_Format(PyExc_AssertionError, "EntityDef.%s: arg1 must be a string! such as: EntityDef.fixed_array(XXArray=EntityDef.INT32)\n", cc.optionName.c_str()); + return NULL; + } + + if (!isRefEntityDefModule(value)) + { + DefContext::DEF_CONTEXT_MAP::iterator iter = DefContext::allScriptDefContextMaps.find(typeName); + if (iter == DefContext::allScriptDefContextMaps.end()) + { + PyErr_Format(PyExc_AssertionError, "EntityDef.%s: arg2 not legal type! such as: EntityDef.fixed_array(XXArray=EntityDef.INT32)\n", cc.optionName.c_str()); + return NULL; + } + } + + DefContext defContext; + defContext.optionName = cc.optionName; + defContext.moduleName = PyUnicode_AsUTF8AndSize(key, NULL); + defContext.returnType = typeName; + + PyObject* kbeModule = PyImport_AddModule("KBEngine"); + KBE_ASSERT(kbeModule); + + PyObject* pyComponentName = PyObject_GetAttrString(kbeModule, "component"); + if (!pyComponentName) + { + PyErr_Format(PyExc_AssertionError, "EntityDef.fixed_array(): get KBEngine.component error!\n"); + PY_RETURN_ERROR; + } + + defContext.componentType = ComponentName2ComponentType(PyUnicode_AsUTF8AndSize(pyComponentName, NULL)); + Py_DECREF(pyComponentName); + + if (!onDefRename(defContext)) + return NULL; + } + + S_Return; + } + + g_callContexts.push(cc); + + // @EntityDef.fixed_array() + // class XXXArray + return PyCFunction_New(&__call_def_parse, self); +} + +#define PY_ADD_METHOD(NAME, DOCS) APPEND_SCRIPT_MODULE_METHOD(entitydefModule, NAME, __py_def_##NAME, METH_VARARGS | METH_KEYWORDS, 0); + +#ifdef interface +#undef interface +#endif + +#ifdef property +#undef property +#endif + +PY_DEF_HOOK(method) +PY_DEF_HOOK(clientmethod) +PY_DEF_HOOK(property) +PY_DEF_HOOK(entity) +PY_DEF_HOOK(interface) +PY_DEF_HOOK(component) +PY_DEF_HOOK(fixed_dict) +PY_DEF_HOOK(fixed_item) + +static PyObject* _tp_entitydef_getattro(PyObject* self, PyObject* name) +{ + if (!isRefEntityDefModule(name)) + { + const char* typeName = PyUnicode_AsUTF8AndSize(name, NULL); + DefContext::DEF_CONTEXT_MAP::iterator iter = DefContext::allScriptDefContextMaps.find(typeName); + if (iter != DefContext::allScriptDefContextMaps.end()) + { + Py_INCREF(name); + return name; + } + } + + return PyObject_GenericGetAttr(self, name); +} + +//------------------------------------------------------------------------------------- +bool installModule(const char* moduleName) +{ + pyDefModuleName = moduleName; + + PyObject *entitydefModule = PyImport_AddModule(pyDefModuleName.c_str()); + PyObject* pyDoc = PyUnicode_FromString("This module is created by KBEngine!"); + PyObject_SetAttrString(entitydefModule, "__doc__", pyDoc); + Py_DECREF(pyDoc); + + static PyMethodDef entitydef_getattro_methods[] = + { + { "__getattr__", (PyCFunction)&_tp_entitydef_getattro, METH_O, 0 }, + { NULL, NULL, 0, NULL } + }; + + PyModule_AddFunctions(entitydefModule, entitydef_getattro_methods); + + PY_ADD_METHOD(rename, ""); + PY_ADD_METHOD(method, ""); + PY_ADD_METHOD(clientmethod, ""); + PY_ADD_METHOD(property, ""); + PY_ADD_METHOD(entity, ""); + PY_ADD_METHOD(interface, ""); + PY_ADD_METHOD(component, ""); + PY_ADD_METHOD(fixed_dict, ""); + PY_ADD_METHOD(fixed_array, ""); + PY_ADD_METHOD(fixed_item, ""); + + return true; +} + +//------------------------------------------------------------------------------------- +bool uninstallModule() +{ + while (!g_callContexts.empty()) g_callContexts.pop(); + DefContext::allScriptDefContextMaps.clear(); + DefContext::allScriptDefContextLineMaps.clear(); + return true; +} + +//------------------------------------------------------------------------------------- +static bool loadAllScriptForComponentType(COMPONENT_TYPE loadComponentType) +{ + std::string entryScriptFileName = ""; + if (loadComponentType == BASEAPP_TYPE) + { + ENGINE_COMPONENT_INFO& info = g_kbeSrvConfig.getBaseApp(); + entryScriptFileName = info.entryScriptFile; + } + else if (loadComponentType == CELLAPP_TYPE) + { + ENGINE_COMPONENT_INFO& info = g_kbeSrvConfig.getCellApp(); + entryScriptFileName = info.entryScriptFile; + } + else + { + KBE_ASSERT(false); + } + + std::string rootPath = Resmgr::getSingleton().getPyUserComponentScriptsPath(loadComponentType); + + if (rootPath.size() == 0) + { + ERROR_MSG(fmt::format("PyEntityDef::loadAllScriptForComponentType(): get scripts path error! loadComponentType={}\n", + COMPONENT_NAME_EX(loadComponentType))); + + return false; + } + + while (rootPath[rootPath.size() - 1] == '/' || rootPath[rootPath.size() - 1] == '\\') rootPath.pop_back(); + + wchar_t* wpath = strutil::char2wchar((rootPath).c_str()); + + wchar_t* _wentryScriptFileName = strutil::char2wchar((entryScriptFileName).c_str()); + std::wstring wentryScriptFileName = _wentryScriptFileName; + free(_wentryScriptFileName); + + std::vector results; + Resmgr::getSingleton().listPathRes(wpath, L"py|pyc", results); + + // ִڽű + std::vector::iterator iter = results.begin(); + for (; iter != results.end(); ) + { + std::wstring wstrpath = (*iter); + if (wstrpath.find(wentryScriptFileName + L".py") == std::wstring::npos && wstrpath.find(wentryScriptFileName + L".pyc") == std::wstring::npos) + { + ++iter; + continue; + } + + iter = results.erase(iter); + } + + results.insert(results.begin(), std::wstring(wpath) + L"/" + wentryScriptFileName + L".py"); + + iter = results.begin(); + for (; iter != results.end(); ++iter) + { + std::wstring wstrpath = (*iter); + + if (wstrpath.find(L"__pycache__") != std::wstring::npos) + continue; + + if (wstrpath.find(L"__init__.") != std::wstring::npos) + continue; + + std::pair pathPair = script::PyPlatform::splitPath(wstrpath); + std::pair filePair = script::PyPlatform::splitText(pathPair.second); + + if (filePair.first.size() == 0) + continue; + + char* cpacketPath = strutil::wchar2char(pathPair.first.c_str()); + std::string packetPath = cpacketPath; + free(cpacketPath); + + strutil::kbe_replace(packetPath, rootPath, ""); + while (packetPath.size() > 0 && (packetPath[0] == '/' || packetPath[0] == '\\')) packetPath.erase(0, 1); + strutil::kbe_replace(packetPath, "/", "."); + strutil::kbe_replace(packetPath, "\\", "."); + + char* moduleName = strutil::wchar2char(filePair.first.c_str()); + + { + PyObject* pyModule = NULL; + + if (packetPath.size() == 0 || packetPath == "components" || packetPath == "interfaces") + { + pyModule = PyImport_ImportModule(const_cast(moduleName)); + } + else + { + pyModule = PyImport_ImportModule(const_cast((packetPath + "." + moduleName).c_str())); + } + + if (!pyModule) + { + SCRIPT_ERROR_CHECK(); + free(wpath); + return false; + } + else + { + Py_DECREF(pyModule); + } + } + + free(moduleName); + } + + free(wpath); + + return true; +} + +//------------------------------------------------------------------------------------- +PyObject* __py_getAppPublish(PyObject* self, PyObject* args) +{ + return PyLong_FromLong(g_appPublish); +} + +//------------------------------------------------------------------------------------- +static bool execPython(COMPONENT_TYPE componentType) +{ + std::pair pyPaths = getComponentPythonPaths(componentType); + if (pyPaths.first.size() == 0) + { + ERROR_MSG(fmt::format("PyEntityDef::execPython(): PythonApp({}) paths error!\n", COMPONENT_NAME_EX(componentType))); + return false; + } + + APPEND_PYSYSPATH(pyPaths.second); + + PyObject* modulesOld = PySys_GetObject("modules"); + + PyThreadState* pCurInterpreter = PyThreadState_Get(); + PyThreadState* pNewInterpreter = Py_NewInterpreter(); + + if (!pNewInterpreter) + { + ERROR_MSG(fmt::format("PyEntityDef::execPython(): Py_NewInterpreter()!\n")); + SCRIPT_ERROR_CHECK(); + return false; + } + +#if KBE_PLATFORM != PLATFORM_WIN32 + strutil::kbe_replace(pyPaths.second, L";", L":"); +#endif + + PySys_SetPath(pyPaths.second.c_str()); + + PyObject* modulesNew = PySys_GetObject("modules"); + PyDict_Merge(modulesNew, Script::getSingleton().getSysInitModules(), 0); + + { + PyObject *key, *value; + Py_ssize_t pos = 0; + + while (PyDict_Next(modulesOld, &pos, &key, &value)) + { + const char* typeName = PyUnicode_AsUTF8AndSize(key, NULL); + + if (std::string(typeName) == "KBEngine") + continue; + + PyObject* pyDoc = PyObject_GetAttrString(value, "__doc__"); + + if (pyDoc) + { + const char* doc = PyUnicode_AsUTF8AndSize(pyDoc, NULL); + + if (doc && std::string(doc).find("KBEngine") != std::string::npos) + PyDict_SetItemString(modulesNew, typeName, value); + + if (PyErr_Occurred()) + PyErr_Clear(); + + Py_XDECREF(pyDoc); + } + else + { + SCRIPT_ERROR_CHECK(); + } + } + } + + PyObject *m = PyImport_AddModule("__main__"); + + // һűģ + PyObject* kbeModule = PyImport_AddModule("KBEngine"); + KBE_ASSERT(kbeModule); + + Entity::registerScript(kbeModule); + Space::registerScript(kbeModule); + EntityComponent::registerScript(kbeModule); + + if (componentType == BASEAPP_TYPE) + Proxy::registerScript(kbeModule); + + const char* componentName = COMPONENT_NAME_EX(componentType); + if (PyModule_AddStringConstant(kbeModule, "component", componentName)) + { + ERROR_MSG(fmt::format("PyEntityDef::execPython(): Unable to set KBEngine.component to {}\n", + componentName)); + + return false; + } + + APPEND_SCRIPT_MODULE_METHOD(kbeModule, publish, __py_getAppPublish, METH_VARARGS, 0); + + // ģmain + PyObject_SetAttrString(m, "KBEngine", kbeModule); + + if (pNewInterpreter != PyThreadState_Swap(pCurInterpreter)) + { + KBE_ASSERT(false); + return false; + } + + PyThreadState_Swap(pNewInterpreter); + + bool otherPartSuccess = loadAllScriptForComponentType(componentType); + + Entity::unregisterScript(); + Space::unregisterScript(); + EntityComponent::unregisterScript(); + + if (componentType == BASEAPP_TYPE) + Proxy::unregisterScript(); + + if (pNewInterpreter != PyThreadState_Swap(pCurInterpreter)) + { + KBE_ASSERT(false); + return false; + } + + // ˴ʹPy_EndInterpreterᵼMathDefģ + PyInterpreterState_Clear(pNewInterpreter->interp); + PyInterpreterState_Delete(pNewInterpreter->interp); + return otherPartSuccess; +} + +//------------------------------------------------------------------------------------- +static bool loadAllScripts() +{ + std::vector< COMPONENT_TYPE > loadOtherComponentTypes; + loadOtherComponentTypes.push_back(BASEAPP_TYPE); + loadOtherComponentTypes.push_back(CELLAPP_TYPE); + + for (std::vector< COMPONENT_TYPE >::iterator iter = loadOtherComponentTypes.begin(); iter != loadOtherComponentTypes.end(); ++iter) + { + COMPONENT_TYPE componentType = (*iter); + + if (g_componentType == componentType) + { + if (!loadAllScriptForComponentType(g_componentType)) + return false; + } + else + { + if (!execPython(componentType)) + return false; + } + } + + return true; +} + +//------------------------------------------------------------------------------------- +static bool sortFun(const DefContext* def1, const DefContext* def2) +{ + return def1->order < def2->order; +} + +static bool registerDefTypes() +{ + std::vector< DefContext* > defContexts; + + { + // עȺϵٺ + DefContext::DEF_CONTEXT_MAP::iterator iter = DefContext::allScriptDefContextMaps.begin(); + for (; iter != DefContext::allScriptDefContextMaps.end(); ++iter) + { + DefContext& defContext = iter->second; + + defContexts.push_back(&defContext); + } + + std::sort(defContexts.begin(), defContexts.end(), sortFun); + } + + { + std::vector< DefContext* >::iterator iter = defContexts.begin(); + for (; iter != defContexts.end(); ++iter) + { + DefContext& defContext = *(*iter); + + if (defContext.type == DefContext::DC_TYPE_FIXED_ARRAY) + { + FixedArrayType* fixedArray = new FixedArrayType(); + + if (fixedArray->initialize(&defContext, defContext.moduleName)) + { + if (!DataTypes::addDataType(defContext.moduleName, fixedArray)) + return false; + } + else + { + ERROR_MSG(fmt::format("PyEntityDef::registerDefTypes: parse ARRAY [{}] error! file: \"{}\"!\n", + defContext.moduleName.c_str(), defContext.pyObjectSourceFile)); + + delete fixedArray; + return false; + } + } + else if (defContext.type == DefContext::DC_TYPE_FIXED_DICT) + { + FixedDictType* fixedDict = new FixedDictType(); + + if (fixedDict->initialize(&defContext, defContext.moduleName)) + { + if (!DataTypes::addDataType(defContext.moduleName, fixedDict)) + return false; + } + else + { + ERROR_MSG(fmt::format("PyEntityDef::registerDefTypes: parse FIXED_DICT [{}] error! file: \"{}\"!\n", + defContext.moduleName.c_str(), defContext.pyObjectSourceFile)); + + delete fixedDict; + return false; + } + } + else if (defContext.type == DefContext::DC_TYPE_RENAME) + { + DataType* dataType = DataTypes::getDataType(defContext.returnType, false); + if (dataType == NULL) + { + ERROR_MSG(fmt::format("PyEntityDef::registerDefTypes: cannot fount type \'{}\', by alias[{}], file: \"{}\"!\n", + defContext.returnType, defContext.moduleName.c_str(), defContext.pyObjectSourceFile)); + + return false; + } + + if (!DataTypes::addDataType(defContext.moduleName, dataType)) + { + ERROR_MSG(fmt::format("PyEntityDef::registerDefTypes: addDataType \"{}\" error! file: \"{}\"!\n", + defContext.moduleName.c_str(), defContext.pyObjectSourceFile)); + + return false; + } + } + } + } + + return true; +} + +//------------------------------------------------------------------------------------- +static bool registerDetailLevelInfo(ScriptDefModule* pScriptModule, DefContext& defContext) +{ + return true; +} + +//------------------------------------------------------------------------------------- +static bool registerVolatileInfo(ScriptDefModule* pScriptModule, DefContext& defContext) +{ + return true; +} + +//------------------------------------------------------------------------------------- +static bool registerDefMethods(ScriptDefModule* pScriptModule, DefContext& defContext) +{ + DefContext::DEF_CONTEXTS methods; + methods.insert(methods.end(), defContext.base_methods.begin(), defContext.base_methods.end()); + methods.insert(methods.end(), defContext.cell_methods.begin(), defContext.cell_methods.end()); + methods.insert(methods.end(), defContext.client_methods.begin(), defContext.client_methods.end()); + + DefContext::DEF_CONTEXTS::iterator iter = methods.begin(); + for (; iter != methods.end(); ++iter) + { + DefContext& defMethodContext = (*iter); + + MethodDescription::EXPOSED_TYPE exposedType = MethodDescription::NO_EXPOSED; + + size_t argIdx = 0; + + // һǷexposed callerID + if (defMethodContext.argsvecs.size() > 0) + { + std::string argName = defMethodContext.argsvecs[0]; + std::string argType = defMethodContext.annotationsMaps[argName]; + + // exposedcallerIDͺ + if (argType == "CALLER_ID") + { + if (!defMethodContext.exposed) + { + ERROR_MSG(fmt::format("PyEntityDef::registerDefMethods: arg1 is Def.CallerID, but the method is not exposed! is {}.{}(arg={}), file: \"{}\"!\n", + pScriptModule->getName(), defMethodContext.attrName.c_str(), argName, defMethodContext.pyObjectSourceFile)); + + return false; + } + + if (defMethodContext.componentType != CELLAPP_TYPE) + { + ERROR_MSG(fmt::format("PyEntityDef::registerDefMethods: arg1 is Def.CallerID, only the cell method supports this parameter.! is {}.{}(arg={}), file: \"{}\"!\n", + pScriptModule->getName(), defMethodContext.attrName.c_str(), argName, defMethodContext.pyObjectSourceFile)); + + return false; + } + + exposedType = MethodDescription::EXPOSED_AND_CALLER_CHECK; + argIdx = 1; + } + } + + if (defMethodContext.exposed && exposedType == MethodDescription::NO_EXPOSED) + exposedType = MethodDescription::EXPOSED; + + MethodDescription* methodDescription = EntityDef::createMethodDescription(pScriptModule, defMethodContext.utype > 0 ? defMethodContext.utype : 0, + defMethodContext.componentType, defMethodContext.attrName, exposedType); + + if (!methodDescription) + return false; + + for (; argIdx < defMethodContext.argsvecs.size(); ++argIdx) + { + std::string argName = defMethodContext.argsvecs[argIdx]; + std::string argType = defMethodContext.annotationsMaps[argName]; + + // exposedcallerIDͺ + if (argType == "CALLER_ID") + { + ERROR_MSG(fmt::format("PyEntityDef::registerDefMethods: Def.CallerID must be the first parameter! is {}.{}(arg={}), file: \"{}\"!\n", + pScriptModule->getName(), defMethodContext.attrName.c_str(), argName, defMethodContext.pyObjectSourceFile)); + + return false; + } + + DataType* dataType = DataTypes::getDataType(argType, false); + + if (!dataType) + { + DefContext* pDefMethodArgContext = DefContext::findDefContext(argType); + if (!pDefMethodArgContext) + { + ERROR_MSG(fmt::format("PyEntityDef::registerDefMethods: not fount type[{}], is {}.{}(arg={}), file: \"{}\"!\n", + argType, pScriptModule->getName(), defMethodContext.attrName.c_str(), argName, defMethodContext.pyObjectSourceFile)); + + return false; + } + + if (pDefMethodArgContext->type == DefContext::DC_TYPE_FIXED_ARRAY) + { + FixedArrayType* dataType1 = new FixedArrayType(); + std::string parentName = defContext.moduleName + "_" + defMethodContext.attrName; + + if (!dataType1->initialize(pDefMethodArgContext, parentName)) + { + ERROR_MSG(fmt::format("PyEntityDef::registerDefMethods: parse ARRAY [{}.{}(arg={})] error! file: \"{}\"!\n", + pScriptModule->getName(), defMethodContext.attrName.c_str(), argName, defMethodContext.pyObjectSourceFile)); + + delete dataType1; + return false; + } + + dataType = dataType1; + } + } + + if (dataType == NULL) + { + ERROR_MSG(fmt::format("PyEntityDef::registerDefMethods: dataType[{}] not found, in {}.{}(arg={}), file: \"{}\"!\n", + argType, pScriptModule->getName(), defMethodContext.attrName.c_str(), argName, defMethodContext.pyObjectSourceFile)); + + return false; + } + + methodDescription->pushArgType(dataType); + } + + if (defMethodContext.componentType == CELLAPP_TYPE) + { + if (!pScriptModule->addCellMethodDescription(defMethodContext.attrName.c_str(), methodDescription)) + return false; + } + else if(defMethodContext.componentType == BASEAPP_TYPE) + { + if (!pScriptModule->addBaseMethodDescription(defMethodContext.attrName.c_str(), methodDescription)) + return false; + } + else + { + if (!pScriptModule->addClientMethodDescription(defMethodContext.attrName.c_str(), methodDescription)) + return false; + } + } + + // ⼸⣬̲Ҫݷűȷ + if (g_componentType == BASEAPP_TYPE || g_componentType == CELLAPP_TYPE || g_componentType == CLIENT_TYPE) + { + if (!EntityDef::checkDefMethod(pScriptModule, defContext.pyObjectPtr.get(), defContext.moduleName)) + { + ERROR_MSG(fmt::format("PyEntityDef::registerDefMethods: EntityClass[{}] checkDefMethod is failed!\n", + defContext.moduleName.c_str())); + + return false; + } + } + + return true; +} + +//------------------------------------------------------------------------------------- +static bool registerDefPropertys(ScriptDefModule* pScriptModule, DefContext& defContext) +{ + DefContext::DEF_CONTEXTS& propertys = defContext.propertys; + + DefContext::DEF_CONTEXTS::iterator iter = propertys.begin(); + for (; iter != propertys.end(); ++iter) + { + DefContext& defPropContext = (*iter); + + ENTITY_PROPERTY_UID futype = 0; + EntityDataFlags flags = stringToEntityDataFlags(defPropContext.propertyFlags); + int32 hasBaseFlags = 0; + int32 hasCellFlags = 0; + int32 hasClientFlags = 0; + DataType* dataType = NULL; + bool isPersistent = (defPropContext.persistent != 0); + bool isIdentifier = false; // Ƿһ + uint32 databaseLength = defPropContext.databaseLength; // ݿеij + std::string indexType = defPropContext.propertyIndex; + DETAIL_TYPE detailLevel = DETAIL_LEVEL_FAR; + std::string name = defPropContext.attrName; + + if (!EntityDef::validDefPropertyName(name)) + { + ERROR_MSG(fmt::format("PyEntityDef::registerDefPropertys: '{}' is limited, in module({}), file: \"{}\"!\n", + name, pScriptModule->getName(), defPropContext.pyObjectSourceFile)); + + return false; + } + + if (defPropContext.detailLevel == "FAR") + detailLevel = DETAIL_LEVEL_FAR; + else if (defPropContext.detailLevel == "MEDIUM") + detailLevel = DETAIL_LEVEL_MEDIUM; + else if (defPropContext.detailLevel == "NEAR") + detailLevel = DETAIL_LEVEL_NEAR; + else + detailLevel = DETAIL_LEVEL_FAR; + + if (!EntityDef::calcDefPropertyUType(pScriptModule->getName(), name, defPropContext.utype > 0 ? defPropContext.utype : -1, pScriptModule, futype)) + return false; + + hasBaseFlags = ((uint32)flags) & ENTITY_BASE_DATA_FLAGS; + if (hasBaseFlags > 0) + pScriptModule->setBase(true); + + hasCellFlags = ((uint32)flags) & ENTITY_CELL_DATA_FLAGS; + if (hasCellFlags > 0) + pScriptModule->setCell(true); + + hasClientFlags = ((uint32)flags) & ENTITY_CLIENT_DATA_FLAGS; + if (hasClientFlags > 0) + pScriptModule->setClient(true); + + if (hasBaseFlags <= 0 && hasCellFlags <= 0) + { + ERROR_MSG(fmt::format("PyEntityDef::registerDefPropertys: not fount flags[{}], is {}.{}, file: \"{}\"!\n", + defPropContext.propertyFlags, pScriptModule->getName(), name.c_str(), defPropContext.pyObjectSourceFile)); + + return false; + } + + dataType = DataTypes::getDataType(defPropContext.returnType, false); + + if (!dataType) + { + DefContext* pDefPropTypeContext = DefContext::findDefContext(defPropContext.returnType); + if (!pDefPropTypeContext) + { + ERROR_MSG(fmt::format("PyEntityDef::registerDefPropertys: not fount type[{}], is {}.{}, file: \"{}\"!\n", + defPropContext.returnType, pScriptModule->getName(), name.c_str(), defPropContext.pyObjectSourceFile)); + + return false; + } + + // ŵд + if (pDefPropTypeContext->type == DefContext::DC_TYPE_COMPONENT) + continue; + + if (pDefPropTypeContext->type == DefContext::DC_TYPE_FIXED_ARRAY) + { + FixedArrayType* dataType1 = new FixedArrayType(); + if (dataType1->initialize(pDefPropTypeContext, std::string(pScriptModule->getName()) + "_" + name)) + dataType = dataType1; + else + return false; + } + else + { + dataType = DataTypes::getDataType(defPropContext.returnType, false); + } + } + + if (dataType == NULL) + { + ERROR_MSG(fmt::format("PyEntityDef::registerDefPropertys: not fount type[{}], is {}.{}, file: \"{}\"!\n", + defPropContext.returnType, pScriptModule->getName(), name.c_str(), defPropContext.pyObjectSourceFile)); + + return false; + } + + // һʵ + PropertyDescription* propertyDescription = PropertyDescription::createDescription(futype, defPropContext.returnType, + name, flags, isPersistent, + dataType, isIdentifier, indexType, + databaseLength, defPropContext.propertyDefaultVal, + detailLevel); + + bool ret = true; + + // ӵģ + if (hasCellFlags > 0) + ret = pScriptModule->addPropertyDescription(name.c_str(), + propertyDescription, CELLAPP_TYPE); + + if (hasBaseFlags > 0) + ret = pScriptModule->addPropertyDescription(name.c_str(), + propertyDescription, BASEAPP_TYPE); + + if (hasClientFlags > 0) + ret = pScriptModule->addPropertyDescription(name.c_str(), + propertyDescription, CLIENT_TYPE); + + if (!ret) + { + ERROR_MSG(fmt::format("PyEntityDef::registerDefPropertys: error, is {}.{}, file: \"{}\"!\n", + pScriptModule->getName(), name.c_str(), defPropContext.pyObjectSourceFile)); + + return false; + } + } + + return true; +} + +//------------------------------------------------------------------------------------- +static bool registerDefComponents(ScriptDefModule* pScriptModule, DefContext& defContext) +{ + DefContext::DEF_CONTEXTS& propertys = defContext.propertys; + + DefContext::DEF_CONTEXTS::iterator iter = propertys.begin(); + for (; iter != propertys.end(); ++iter) + { + DefContext& defPropContext = (*iter); + + std::string componentName = defPropContext.attrName; + std::string componentTypeName = defPropContext.returnType; + bool isPersistent = (defPropContext.persistent != 0); + ENTITY_PROPERTY_UID futype = 0; + + DefContext* pDefPropTypeContext = DefContext::findDefContext(componentTypeName); + if (!pDefPropTypeContext) + { + continue; + } + + if (pDefPropTypeContext->type != DefContext::DC_TYPE_COMPONENT) + continue; + + if (!EntityDef::calcDefPropertyUType(pScriptModule->getName(), componentName, defPropContext.utype > 0 ? defPropContext.utype : -1, pScriptModule, futype)) + return false; + + // һʵ + uint32 flags = ED_FLAG_BASE | ED_FLAG_CELL_PUBLIC | ENTITY_CLIENT_DATA_FLAGS; + bool isIdentifier = false; // Ƿһ + uint32 databaseLength = 0; // ݿеij + std::string indexType = ""; + DETAIL_TYPE detailLevel = DETAIL_LEVEL_FAR; + std::string detailLevelStr = ""; + std::string strisPersistent; + std::string defaultStr = ""; + + if (!EntityDef::validDefPropertyName(componentName)) + { + ERROR_MSG(fmt::format("PyEntityDef::registerDefComponents: '{}' is limited, in module({}), file: \"{}\"!\n", + componentName, pScriptModule->getName(), defPropContext.pyObjectSourceFile)); + + return false; + } + + // Ƿģ飬˵Ѿعٴμ + ScriptDefModule* pCompScriptDefModule = EntityDef::findScriptModule(componentTypeName.c_str(), false); + + if (!pCompScriptDefModule) + { + pCompScriptDefModule = EntityDef::registerNewScriptDefModule(componentTypeName); + pCompScriptDefModule->isPersistent(false); + pCompScriptDefModule->isComponentModule(true); + } + else + { + flags = ED_FLAG_UNKOWN; + + if (pCompScriptDefModule->hasBase()) + flags |= ED_FLAG_BASE; + + if (pCompScriptDefModule->hasCell()) + flags |= ED_FLAG_CELL_PUBLIC; + + if (pCompScriptDefModule->hasClient()) + { + if (pCompScriptDefModule->hasBase()) + flags |= ED_FLAG_BASE_AND_CLIENT; + else + flags |= (ED_FLAG_ALL_CLIENTS | ED_FLAG_CELL_PUBLIC_AND_OWN | ED_FLAG_OTHER_CLIENTS | ED_FLAG_OWN_CLIENT); + } + + EntityDef::addComponentProperty(futype, componentTypeName, componentName, flags, isPersistent, isIdentifier, + indexType, databaseLength, defaultStr, detailLevel, pScriptModule, pCompScriptDefModule); + + pScriptModule->addComponentDescription(componentName.c_str(), pCompScriptDefModule); + continue; + } + + // ע + if (!registerDefPropertys(pScriptModule, defContext)) + { + ERROR_MSG(fmt::format("PyEntityDef::registerDefComponents: failed to registerDefPropertys(), entity:{}\n", + pScriptModule->getName())); + + return false; + } + + // ע᷽ + if(!registerDefMethods(pScriptModule, defContext)) + { + ERROR_MSG(fmt::format("PyEntityDef::registerDefComponents: failed to registerDefMethods(), entity:{}\n", + pScriptModule->getName())); + + return false; + } + + // ԼdetailLevelInfo + if (!registerDetailLevelInfo(pScriptModule, defContext)) + { + ERROR_MSG(fmt::format("PyEntityDef::registerDefComponents: failed to register component:{} detailLevelInfo.\n", + pScriptModule->getName())); + + return false; + } + + // ԼvolatileInfo + if (!registerVolatileInfo(pScriptModule, defContext)) + { + ERROR_MSG(fmt::format("PyEntityDef::registerDefComponents: failed to register component:{} volatileInfo.\n", + pScriptModule->getName())); + + return false; + } + + pCompScriptDefModule->autoMatchCompOwn(); + + flags = ED_FLAG_UNKOWN; + + if (pCompScriptDefModule->hasBase()) + flags |= ED_FLAG_BASE; + + if (pCompScriptDefModule->hasCell()) + flags |= ED_FLAG_CELL_PUBLIC; + + if (pCompScriptDefModule->hasClient()) + { + if (pCompScriptDefModule->hasBase()) + flags |= ED_FLAG_BASE_AND_CLIENT; + + if (pCompScriptDefModule->hasCell()) + flags |= (ED_FLAG_ALL_CLIENTS | ED_FLAG_CELL_PUBLIC_AND_OWN | ED_FLAG_OTHER_CLIENTS | ED_FLAG_OWN_CLIENT); + } + + EntityDef::addComponentProperty(futype, componentTypeName, componentName, flags, isPersistent, isIdentifier, + indexType, databaseLength, defaultStr, detailLevel, pScriptModule, pCompScriptDefModule); + + pScriptModule->addComponentDescription(componentName.c_str(), pCompScriptDefModule); + } + + return true; +} + +//------------------------------------------------------------------------------------- +static bool registerEntityDef(ScriptDefModule* pScriptModule, DefContext& defContext) +{ + // ע + if (!registerDefPropertys(pScriptModule, defContext)) + { + ERROR_MSG(fmt::format("PyEntityDef::registerEntityDef: failed to registerDefPropertys(), entity:{}\n", + pScriptModule->getName())); + + return false; + } + + // ע᷽ + if (!registerDefMethods(pScriptModule, defContext)) + { + ERROR_MSG(fmt::format("PyEntityDef::registerDefComponents: failed to registerDefMethods(), entity:{}\n", + pScriptModule->getName())); + + return false; + } + + // ǵķԼ뵽ģ + if (!registerDefComponents(pScriptModule, defContext)) + { + ERROR_MSG(fmt::format("PyEntityDef::registerEntityDef: failed to registerDefComponents(), component:{}\n", + pScriptModule->getName())); + + return false; + } + + // ԼdetailLevelInfo + if (!registerDetailLevelInfo(pScriptModule, defContext)) + { + ERROR_MSG(fmt::format("PyEntityDef::registerEntityDef: failed to register entity:{} detailLevelInfo.\n", + pScriptModule->getName())); + + return false; + } + + // ԼvolatileInfo + if (!registerVolatileInfo(pScriptModule, defContext)) + { + ERROR_MSG(fmt::format("PyEntityDef::registerEntityDef: failed to register entity:{} volatileInfo.\n", + pScriptModule->getName())); + + return false; + } + + // ⼸⣬̲Ҫʽű + if (g_componentType == BASEAPP_TYPE || g_componentType == CELLAPP_TYPE || g_componentType == CLIENT_TYPE) + { + PyObject* pyClass = defContext.pyObjectPtr.get(); + if (pyClass) + { + if (!PyType_Check(pyClass)) + { + ERROR_MSG(fmt::format("PyEntityDef::registerEntityDef: EntityClass[{}] is valid!\n", + defContext.moduleName.c_str())); + + return false; + } + + Py_INCREF((PyTypeObject *)pyClass); + pScriptModule->setScriptType((PyTypeObject *)pyClass); + } + } + + pScriptModule->autoMatchCompOwn(); + return true; +} + +//------------------------------------------------------------------------------------- +static bool registerEntityDefs() +{ + if (!registerDefTypes()) + return false; + + DefContext::DEF_CONTEXT_MAP::iterator iter = DefContext::allScriptDefContextMaps.begin(); + for (; iter != DefContext::allScriptDefContextMaps.end(); ++iter) + { + DefContext& defContext = iter->second; + + if (defContext.type != DefContext::DC_TYPE_ENTITY) + continue; + + ScriptDefModule* pScriptModule = EntityDef::registerNewScriptDefModule(defContext.moduleName); + + if (!registerEntityDef(pScriptModule, defContext)) + return false; + + pScriptModule->onLoaded(); + } + + return true; +} + +//------------------------------------------------------------------------------------- +bool initialize() +{ + if (g_inited) + return false; + + g_inited = true; + + KBE_ASSERT(pyDefModuleName.size() > 0); + PyObject *entitydefModule = PyImport_AddModule(pyDefModuleName.c_str()); + + ENTITYFLAGMAP::iterator iter = g_entityFlagMapping.begin(); + for (; iter != g_entityFlagMapping.end(); ++iter) + { + if (PyModule_AddStringConstant(entitydefModule, iter->first.c_str(), iter->first.c_str())) + { + ERROR_MSG(fmt::format("PyEntityDef::initialize(): Unable to set EntityDef.{} to {}\n", + iter->first, iter->first)); + + return false; + } + } + + static const char* UNIQUE = "UNIQUE"; + if (PyModule_AddStringConstant(entitydefModule, UNIQUE, UNIQUE)) + { + ERROR_MSG(fmt::format("PyEntityDef::initialize(): Unable to set EntityDef.{} to {}\n", + UNIQUE, UNIQUE)); + + return false; + } + + static const char* INDEX = "INDEX"; + if (PyModule_AddStringConstant(entitydefModule, INDEX, INDEX)) + { + ERROR_MSG(fmt::format("PyEntityDef::initialize(): Unable to set EntityDef.{} to {}\n", + INDEX, INDEX)); + + return false; + } + + static const char* CALLER_ID = "CALLER_ID"; + if (PyModule_AddStringConstant(entitydefModule, CALLER_ID, CALLER_ID)) + { + ERROR_MSG(fmt::format("PyEntityDef::initialize(): Unable to set EntityDef.{} to {}\n", + CALLER_ID, CALLER_ID)); + + return false; + } + + //APPEND_SCRIPT_MODULE_METHOD(entitydefModule, ARRAY, __py_array, METH_VARARGS, 0); + + static char allBaseTypeNames[64][MAX_BUF]; + std::vector< std::string > baseTypeNames = DataTypes::getBaseTypeNames(); + KBE_ASSERT(baseTypeNames.size() < 64); + + for (size_t idx = 0; idx < baseTypeNames.size(); idx++) + { + memset(allBaseTypeNames[idx], 0, MAX_BUF); + kbe_snprintf(allBaseTypeNames[idx], MAX_BUF, "%s", baseTypeNames[idx].c_str()); + if (PyModule_AddStringConstant(entitydefModule, allBaseTypeNames[idx], allBaseTypeNames[idx])) + { + ERROR_MSG(fmt::format("PyEntityDef::initialize(): Unable to set EntityDef.{} to {}\n", + allBaseTypeNames[idx], allBaseTypeNames[idx])); + + return false; + } + } + + if (!loadAllScripts()) + { + SCRIPT_ERROR_CHECK(); + return false; + } + + while (!g_callContexts.empty()) + g_callContexts.pop(); + + DefContext::allScriptDefContextLineMaps.clear(); + + if (!assemblyContexts(true)) + { + SCRIPT_ERROR_CHECK(); + DefContext::allScriptDefContextMaps.clear(); + return false; + } + + if (!registerEntityDefs()) + { + DefContext::allScriptDefContextMaps.clear(); + return false; + } + + return true; +} + +//------------------------------------------------------------------------------------- +bool finalise(bool isReload) +{ + return true; +} + +//------------------------------------------------------------------------------------- +void reload(bool fullReload) +{ + g_inited = false; +} + +//------------------------------------------------------------------------------------- +bool initializeWatcher() +{ + return true; +} + +//------------------------------------------------------------------------------------- +bool addToStream(MemoryStream* pMemoryStream) +{ + int size = DefContext::allScriptDefContextMaps.size(); + (*pMemoryStream) << size; + + DefContext::DEF_CONTEXT_MAP::iterator iter = DefContext::allScriptDefContextMaps.begin(); + for (; iter != DefContext::allScriptDefContextMaps.end(); ++iter) + { + const std::string& name = iter->first; + DefContext& defContext = iter->second; + + (*pMemoryStream) << name; + + defContext.addToStream(pMemoryStream); + } + + return true; +} + +//------------------------------------------------------------------------------------- +bool createFromStream(MemoryStream* pMemoryStream) +{ + int size = 0; + (*pMemoryStream) >> size; + + for (int i = 0; i < size; ++i) + { + std::string name = ""; + (*pMemoryStream) >> name; + + DefContext defContext; + defContext.createFromStream(pMemoryStream); + DefContext::allScriptDefContextLineMaps[defContext.pyObjectSourceFile] = defContext; + } + + return true; +} + +//------------------------------------------------------------------------------------- + +} +} +} diff --git a/kbe/src/lib/entitydef/py_entitydef.h b/kbe/src/lib/entitydef/py_entitydef.h index b24cdca74f..321ca90f53 100644 --- a/kbe/src/lib/entitydef/py_entitydef.h +++ b/kbe/src/lib/entitydef/py_entitydef.h @@ -1,125 +1,127 @@ -// Copyright 2008-2018 Yolo Technologies, Inc. All Rights Reserved. https://www.comblockengine.com - -#ifndef KBE_SCRIPT_ENTITYDEF_H -#define KBE_SCRIPT_ENTITYDEF_H - -#include "common.h" -#include "pyscript/pyobject_pointer.h" - -namespace KBEngine{ - -class MemoryStream; - -namespace script { namespace entitydef { - -class DefContext -{ -public: - enum DCType - { - DC_TYPE_UNKNOWN = 0, - DC_TYPE_ENTITY = 1, - DC_TYPE_COMPONENT = 2, - DC_TYPE_INTERFACE = 3, - - DC_TYPE_PROPERTY = 4, - DC_TYPE_METHOD = 5, - DC_TYPE_CLIENT_METHOD = 6, - - DC_TYPE_FIXED_DICT = 7, - DC_TYPE_FIXED_ARRAY = 8, - DC_TYPE_FIXED_ITEM = 9, - - DC_TYPE_RENAME = 10, - }; - - DefContext(); - - bool addToStream(MemoryStream* pMemoryStream); - bool createFromStream(MemoryStream* pMemoryStream); - - bool addChildContext(DefContext& defContext); - - static DefContext* findDefContext(const std::string& name) - { - DEF_CONTEXT_MAP::iterator iter = allScriptDefContextMaps.find(name); - if (iter != script::entitydef::DefContext::allScriptDefContextMaps.end()) - { - return &iter->second; - } - - return NULL; - } - - std::string optionName; - - std::string moduleName; - std::string attrName; - std::string methodArgs; - std::string returnType; - - std::vector< std::string > argsvecs; - std::map< std::string, std::string > annotationsMaps; - - bool isModuleScope; - - bool exposed; - bool hasClient; - - // -1δã 0false 1true - int persistent; - - int databaseLength; - int utype; - - std::string detailLevel; - - std::string propertyFlags; - std::string propertyIndex; - std::string propertyDefaultVal; - - PyObjectPtr implementedBy; - std::string implementedByModuleName; - std::string implementedByModuleFile; - - PyObjectPtr pyObjectPtr; - std::string pyObjectSourceFile; - - std::vector< std::string > baseClasses; - - DCType inheritEngineModuleType; - DCType type; - - typedef std::vector< DefContext > DEF_CONTEXTS; - - DEF_CONTEXTS methods; - DEF_CONTEXTS client_methods; - DEF_CONTEXTS propertys; - std::vector< std::string > components; - - COMPONENT_TYPE componentType; - - typedef std::map DEF_CONTEXT_MAP; - static DEF_CONTEXT_MAP allScriptDefContextMaps; - static DEF_CONTEXT_MAP allScriptDefContextLineMaps; -}; - -/** װentitydefģ */ -bool installModule(const char* moduleName); -bool uninstallModule(); - -bool initialize(); -bool finalise(bool isReload = false); - -void reload(bool fullReload); - -bool initializeWatcher(); - -bool addToStream(MemoryStream* pMemoryStream); -bool createFromStream(MemoryStream* pMemoryStream); - -} -} -} - -#endif // KBE_SCRIPT_ENTITYDEF_H +// Copyright 2008-2018 Yolo Technologies, Inc. All Rights Reserved. https://www.comblockengine.com + +#ifndef KBE_SCRIPT_ENTITYDEF_H +#define KBE_SCRIPT_ENTITYDEF_H + +#include "common.h" +#include "pyscript/pyobject_pointer.h" + +namespace KBEngine{ + +class MemoryStream; + +namespace script { namespace entitydef { + +class DefContext +{ +public: + enum DCType + { + DC_TYPE_UNKNOWN = 0, + DC_TYPE_ENTITY = 1, + DC_TYPE_COMPONENT = 2, + DC_TYPE_INTERFACE = 3, + + DC_TYPE_PROPERTY = 4, + DC_TYPE_METHOD = 5, + DC_TYPE_CLIENT_METHOD = 6, + + DC_TYPE_FIXED_DICT = 7, + DC_TYPE_FIXED_ARRAY = 8, + DC_TYPE_FIXED_ITEM = 9, + + DC_TYPE_RENAME = 10, + }; + + DefContext(); + + bool addToStream(MemoryStream* pMemoryStream); + bool createFromStream(MemoryStream* pMemoryStream); + + bool addChildContext(DefContext& defContext); + + static DefContext* findDefContext(const std::string& name) + { + DEF_CONTEXT_MAP::iterator iter = allScriptDefContextMaps.find(name); + if (iter != script::entitydef::DefContext::allScriptDefContextMaps.end()) + { + return &iter->second; + } + + return NULL; + } + + int order; + + std::string optionName; + + std::string moduleName; + std::string attrName; + std::string returnType; + + std::vector< std::string > argsvecs; + std::map< std::string, std::string > annotationsMaps; + + bool isModuleScope; + + bool exposed; + bool hasClient; + + // -1δã 0false 1true + int persistent; + + int databaseLength; + int utype; + + std::string detailLevel; + + std::string propertyFlags; + std::string propertyIndex; + std::string propertyDefaultVal; + + PyObjectPtr implementedBy; + std::string implementedByModuleName; + std::string implementedByModuleFile; + + PyObjectPtr pyObjectPtr; + std::string pyObjectSourceFile; + + std::vector< std::string > baseClasses; + + DCType inheritEngineModuleType; + DCType type; + + typedef std::vector< DefContext > DEF_CONTEXTS; + + DEF_CONTEXTS base_methods; + DEF_CONTEXTS cell_methods; + DEF_CONTEXTS client_methods; + DEF_CONTEXTS propertys; + std::vector< std::string > components; + + COMPONENT_TYPE componentType; + + typedef std::map DEF_CONTEXT_MAP; + static DEF_CONTEXT_MAP allScriptDefContextMaps; + static DEF_CONTEXT_MAP allScriptDefContextLineMaps; +}; + +/** װentitydefģ */ +bool installModule(const char* moduleName); +bool uninstallModule(); + +bool initialize(); +bool finalise(bool isReload = false); + +void reload(bool fullReload); + +bool initializeWatcher(); + +bool addToStream(MemoryStream* pMemoryStream); +bool createFromStream(MemoryStream* pMemoryStream); + +} +} +} + +#endif // KBE_SCRIPT_ENTITYDEF_H diff --git a/kbe/src/lib/navigation/navigation_mesh_handle.cpp b/kbe/src/lib/navigation/navigation_mesh_handle.cpp index c030a8b14b..4b0f3ead6e 100644 --- a/kbe/src/lib/navigation/navigation_mesh_handle.cpp +++ b/kbe/src/lib/navigation/navigation_mesh_handle.cpp @@ -1,4 +1,4 @@ -// Copyright 2008-2018 Yolo Technologies, Inc. All Rights Reserved. https://www.comblockengine.com +// Copyright 2008-2018 Yolo Technologies, Inc. All Rights Reserved. https://www.comblockengine.com #include "navigation_mesh_handle.h" #include "navigation/navigation.h" @@ -116,18 +116,18 @@ int NavMeshHandle::findStraightPath(int layer, const Position3D& start, const Po } //------------------------------------------------------------------------------------- -int NavMeshHandle::findRandomPointAroundCircle(int layer, const Position3D& centerPos, +int NavMeshHandle::findRandomPointAroundCircle(int layer, const Position3D& centerPos, std::vector& points, uint32 max_points, float maxRadius) { std::map::iterator iter = navmeshLayer.find(layer); - if(iter == navmeshLayer.end()) + if (iter == navmeshLayer.end()) { - ERROR_MSG(fmt::format("NavMeshHandle::findRandomPointAroundCircle: not found layer({})\n", layer)); + ERROR_MSG(fmt::format("NavMeshHandle::findRandomPointAroundCircle: not found layer({})\n", layer)); return NAV_ERROR; } dtNavMeshQuery* navmeshQuery = iter->second.pNavmeshQuery; - + dtQueryFilter filter; filter.setIncludeFlags(0xffff); filter.setExcludeFlags(0); @@ -135,12 +135,13 @@ int NavMeshHandle::findRandomPointAroundCircle(int layer, const Position3D& cent if (maxRadius <= 0.0001f) { Position3D currpos; - + for (uint32 i = 0; i < max_points; i++) { float pt[3]; dtPolyRef ref; dtStatus status = navmeshQuery->findRandomPoint(&filter, frand, &ref, pt); + if (dtStatusSucceed(status)) { currpos.x = pt[0]; @@ -153,8 +154,8 @@ int NavMeshHandle::findRandomPointAroundCircle(int layer, const Position3D& cent return (int)points.size(); } - - const float extents[3] = {2.f, 4.f, 2.f}; + + const float extents[3] = { 2.f, 4.f, 2.f }; dtPolyRef startRef = INVALID_NAVMESH_POLYREF; @@ -171,41 +172,162 @@ int NavMeshHandle::findRandomPointAroundCircle(int layer, const Position3D& cent ERROR_MSG(fmt::format("NavMeshHandle::findRandomPointAroundCircle({1}): Could not find any nearby poly's ({0})\n", startRef, resPath)); return NAV_ERROR_NEARESTPOLY; } - - Position3D currpos; - bool done = false; - int itry = 0; - while (itry++ < 3 && points.size() == 0) + const float squareSize = (float)maxRadius; + float squareVerts[12] = { + spos[0] - squareSize, spos[1], spos[2] + squareSize,\ + spos[0] + squareSize, spos[1], spos[2] + squareSize,\ + spos[0] + squareSize, spos[1], spos[2] - squareSize,\ + spos[0] - squareSize, spos[1], spos[2] - squareSize,\ + }; + + static const int maxResult = 32; + dtPolyRef polyRefs[maxResult]; + dtPolyRef parentPolyRefs[maxResult]; + int polyCount = 0; + float cost[maxResult]; + + navmeshQuery->findPolysAroundShape(startRef, squareVerts, 4, &filter, polyRefs, parentPolyRefs, cost, &polyCount, maxResult); + + if (polyCount == 0) + { + return (int)points.size(); + } + + float* allPolyAreas = new float[polyCount]; + Position3D currpos; + for (uint32 time = 0; time < max_points; time++) { - max_points -= points.size(); + const dtMeshTile* randomTile = 0; + const dtPoly* randomPoly = 0; + dtPolyRef randomPolyRef = 0; + float areaSum = 0.0f; - for (uint32 i = 0; i < max_points; i++) + for (int i = 0; i < polyCount; i++) { - float pt[3]; - dtPolyRef ref; - dtStatus status = navmeshQuery->findRandomPointAroundCircle(startRef, spos, maxRadius, &filter, frand, &ref, pt); + float polyArea = 0.0f; - if (dtStatusSucceed(status)) + if (time == 0) { - done = true; - currpos.x = pt[0]; - currpos.y = pt[1]; - currpos.z = pt[2]; + const dtMeshTile* tile = 0; + const dtPoly* poly = 0; + dtPolyRef ref = polyRefs[i]; + navmeshQuery->getAttachedNavMesh()->getTileAndPolyByRefUnsafe(ref, &tile, &poly); + + if (poly->getType() != DT_POLYTYPE_GROUND) continue; + + // Place random locations on on ground. + if (poly->getType() == DT_POLYTYPE_GROUND) + { + // Calc area of the polygon. + for (int j = 2; j < poly->vertCount; ++j) + { + const float* va = &tile->verts[poly->verts[0] * 3]; + const float* vb = &tile->verts[poly->verts[j - 1] * 3]; + const float* vc = &tile->verts[poly->verts[j] * 3]; + polyArea += dtTriArea2D(va, vb, vc); + } + + allPolyAreas[i] = polyArea; + areaSum += polyArea; + const float u = frand(); + + if (u*areaSum <= polyArea) + { + randomTile = tile; + randomPoly = poly; + randomPolyRef = ref; + } + } + } + else + { + // Choose random polygon weighted by area, using reservoi sampling. + areaSum += allPolyAreas[i]; + const float u = frand(); + + if (u*areaSum <= allPolyAreas[i]) + { + const dtMeshTile* tile = 0; + const dtPoly* poly = 0; + dtPolyRef ref = polyRefs[i]; + navmeshQuery->getAttachedNavMesh()->getTileAndPolyByRefUnsafe(ref, &tile, &poly); + + if (poly->getType() != DT_POLYTYPE_GROUND) continue; + + randomTile = tile; + randomPoly = poly; + randomPolyRef = ref; + } + } + } - Position3D v = centerPos - currpos; - float dist_len = KBEVec3Length(&v); - if (dist_len > maxRadius) - continue; + // Randomly pick point on polygon. + dtPolyRef randomRef = INVALID_NAVMESH_POLYREF; + const float* v = &randomTile->verts[randomPoly->verts[0] * 3]; + float verts[3 * DT_VERTS_PER_POLYGON]; + float areas[DT_VERTS_PER_POLYGON + 4]; + dtVcopy(&verts[0 * 3], v); - points.push_back(currpos); - } + for (int j = 1; j < randomPoly->vertCount; ++j) + { + v = &randomTile->verts[randomPoly->verts[j] * 3]; + dtVcopy(&verts[j * 3], v); } - if (!done) - break; + float overlapPolyVerts[(DT_VERTS_PER_POLYGON + 4) * 3]; + int nOverlapPolyVerts = 0; + + getOverlapPolyPoly2D(squareVerts, 4, verts, randomPoly->vertCount, overlapPolyVerts, &nOverlapPolyVerts); + + if (nOverlapPolyVerts <= 0) + { + delete[] allPolyAreas; + return (int)points.size(); + } + + const float s = frand(); + const float t = frand(); + float pt[3]; + dtRandomPointInConvexPoly(overlapPolyVerts, nOverlapPolyVerts, areas, s, t, pt); + + float h = 0.0f; + dtStatus status = navmeshQuery->getPolyHeight(randomPolyRef, pt, &h); + + if (dtStatusFailed(status)) + { + delete[] allPolyAreas; + return (int)points.size(); + } + + pt[1] = h; + randomRef = randomPolyRef; + + if (randomRef) + { + currpos.x = pt[0]; + currpos.y = pt[1]; + currpos.z = pt[2]; + + float src_len = sqrt(2) * squareSize; + float xx = centerPos.x - currpos.x; + float yy = centerPos.y - currpos.y; + float dist_len = sqrt(xx * xx + yy * yy); + + if (dist_len > src_len) + { + ERROR_MSG(fmt::format("NavMeshHandle::findRandomPointAroundCircle::(Out of range)::centerPos({},{},{}), currpos({},{},{}), errLen({}), {}, {}\n", + centerPos.x, centerPos.y, centerPos.z, currpos.x, currpos.y, currpos.z, (dist_len - src_len), dist_len, src_len)); + + continue; + } + + points.push_back(currpos); + } } + delete[] allPolyAreas; + return (int)points.size(); } @@ -495,7 +617,7 @@ bool NavMeshHandle::_create(int layer, const std::string& resPath, const std::st dtNavMesh* mesh = tryReadNavmesh(data, readsize, res, false); - // ʧԼչʽ + // 如果加载失败则尝试加载扩展格式 if(!mesh) mesh = tryReadNavmesh(data, readsize, res, true); @@ -553,5 +675,161 @@ bool NavMeshHandle::_create(int layer, const std::string& resPath, const std::st } //------------------------------------------------------------------------------------- +inline float calAtan(float* srcPoint, float* point) +{ + return atan2(point[2] - srcPoint[2], point[0] - srcPoint[0]); +} + +inline void swapPoint(float* a, float* b) +{ + float tmp[3] = { a[0],a[1],a[2] }; + a[0] = b[0]; + a[1] = b[1]; + a[2] = b[2]; + b[0] = tmp[0]; + b[1] = tmp[1]; + b[2] = tmp[2]; +} + +void NavMeshHandle::getOverlapPolyPoly2D(const float* polyVertsA, const int nPolyVertsA, const float* polyVertsB, const int nPolyVertsB, float* intsectPt, int* intsectPtCount) +{ + *intsectPtCount = 0; + + ///Find polyA's verts which in polyB. + for (int i = 0; i < nPolyVertsA; ++i) + { + const float* va = &polyVertsA[i * 3]; + if (dtPointInPolygon(va, polyVertsB, nPolyVertsB)) + { + intsectPt[*intsectPtCount * 3] = va[0]; + intsectPt[*intsectPtCount * 3 + 1] = va[1]; + intsectPt[*intsectPtCount * 3 + 2] = va[2]; + *intsectPtCount += 1; + } + } + + ///Find polyB's verts which in polyA. + for (int i = 0; i < nPolyVertsB; ++i) + { + const float* va = &polyVertsB[i * 3]; + if (dtPointInPolygon(va, polyVertsA, nPolyVertsA)) + { + intsectPt[*intsectPtCount * 3] = va[0]; + intsectPt[*intsectPtCount * 3 + 1] = va[1]; + intsectPt[*intsectPtCount * 3 + 2] = va[2]; + *intsectPtCount += 1; + } + } + + ///Find edge intersection of polyA and polyB. + for (int i = 0; i < nPolyVertsA; ++i) + { + const float* p1 = &polyVertsA[i * 3]; + int p2_idx = (i + 1) % nPolyVertsA; + const float* p2 = &polyVertsA[p2_idx * 3]; + + for (int j = 0; j < nPolyVertsB; ++j) + { + const float* q1 = &polyVertsB[j * 3]; + int q2_idx = (j + 1) % nPolyVertsB; + const float* q2 = &polyVertsB[q2_idx * 3]; + + if (isSegSegCross2D(p1, p2, q1, q2)) ///If two segment is cross + { + float s, t; + if (dtIntersectSegSeg2D(p1, p2, q1, q2, s, t)) ///Caculate intersection point + { + float pt[3]; + dtVlerp(pt, q1, q2, t); + intsectPt[*intsectPtCount * 3] = pt[0]; + intsectPt[*intsectPtCount * 3 + 1] = pt[1]; + intsectPt[*intsectPtCount * 3 + 2] = pt[2]; + *intsectPtCount += 1; + } + } + } + } + + ///sort intersection to clockwise. + if (*intsectPtCount > 0) + { + clockwiseSortPoints(intsectPt, *intsectPtCount); + } + +} + +void NavMeshHandle::clockwiseSortPoints(float* verts, const int nVerts) +{ + float x = 0.0; + float z = 0.0; + for (int i = 0; i < nVerts; ++i) + { + x += verts[i * 3]; + z += verts[i * 3 + 2]; + } + + //Put most left point in first position. + for (int i = 0; i < nVerts; i++) + { + if (verts[i * 3] < verts[0]) + { + swapPoint(&verts[i * 3], &verts[0]); + } + else if (verts[i * 3] == verts[0] && verts[i * 3 + 2] < verts[2]) + { + swapPoint(&verts[i * 3], &verts[0]); + } + } + + //Sort points by slope. + for (int i = 1; i < nVerts; i++) + { + for (int j = 1; j < nVerts - i; j++) + { + int index = j * 3; + int n_index = (j + 1) * 3; + float angle = calAtan(&verts[0], &verts[index]); + float n_angle = calAtan(&verts[0], &verts[n_index]); + if (angle < n_angle) + { + swapPoint(&verts[index], &verts[n_index]); + } + } + } +} + +bool NavMeshHandle::isSegSegCross2D(const float* p1, const float *p2, const float* q1, const float* q2) +{ + bool ret = dtMin(p1[0], p2[0]) <= dtMax(q1[0], q2[0]) && + dtMin(q1[0], q2[0]) <= dtMax(p1[0], p2[0]) && + dtMin(p1[2], p2[2]) <= dtMax(q1[2], q2[2]) && + dtMin(q1[2], q2[2]) <= dtMax(p1[2], p2[2]); + + if (!ret) + { + return false; + } + + long line1, line2; + line1 = (long)(p1[0] * (q1[2] - p2[2]) + p2[0] * (p1[2] - q1[2]) + q1[0] * (p2[2] - p1[2])); + line2 = (long)(p1[0] * (q2[2] - p2[2]) + p2[0] * (p1[2] - q2[2]) + q2[0] * (p2[2] - p1[2])); + if (((line1 ^ line2) >= 0) && !(line1 == 0 && line2 == 0)) + { + return false; + } + + line1 = (long)(q1[0] * (p1[2] - q2[2]) + q2[0] * (q1[2] - p1[2]) + p1[0] * (q2[2] - q1[2])); + line2 = (long)(q1[0] * (p2[2] - q2[2]) + q2[0] * (q1[2] - p2[2]) + p2[0] * (q2[2] - q1[2])); + if (((line1 ^ line2) >= 0) && !(line1 == 0 && line2 == 0)) + { + return false; + } + + return true; +} + +//------------------------------------------------------------------------------------- + } + diff --git a/kbe/src/lib/navigation/navigation_mesh_handle.h b/kbe/src/lib/navigation/navigation_mesh_handle.h index d99ceb375c..7474d276c4 100644 --- a/kbe/src/lib/navigation/navigation_mesh_handle.h +++ b/kbe/src/lib/navigation/navigation_mesh_handle.h @@ -1,4 +1,4 @@ -// Copyright 2008-2018 Yolo Technologies, Inc. All Rights Reserved. https://www.comblockengine.com +// Copyright 2008-2018 Yolo Technologies, Inc. All Rights Reserved. https://www.comblockengine.com #ifndef KBE_NAVIGATEMESHHANDLE_H #define KBE_NAVIGATEMESHHANDLE_H @@ -10,64 +10,80 @@ #include "DetourCommon.h" #include "DetourNavMesh.h" -namespace KBEngine{ +namespace KBEngine { -struct NavMeshSetHeader -{ - int version; - int tileCount; - dtNavMeshParams params; -}; - -struct NavMeshSetHeaderEx -{ - int magic; - int version; - int tileCount; - dtNavMeshParams params; -}; - -struct NavMeshTileHeader -{ - dtTileRef tileRef; - int dataSize; -}; + struct NavMeshSetHeader + { + int version; + int tileCount; + dtNavMeshParams params; + }; -class NavMeshHandle : public NavigationHandle -{ -public: - static const int MAX_POLYS = 256; - static const int NAV_ERROR_NEARESTPOLY = -2; + struct NavMeshSetHeaderEx + { + int magic; + int version; + int tileCount; + dtNavMeshParams params; + }; - static const long RCN_NAVMESH_VERSION = 1; - static const int INVALID_NAVMESH_POLYREF = 0; - - struct NavmeshLayer + struct NavMeshTileHeader { - dtNavMesh* pNavmesh; - dtNavMeshQuery* pNavmeshQuery; + dtTileRef tileRef; + int dataSize; }; -public: - NavMeshHandle(); - virtual ~NavMeshHandle(); + class NavMeshHandle : public NavigationHandle + { + public: + static const int MAX_POLYS = 256; + static const int NAV_ERROR_NEARESTPOLY = -2; - int findStraightPath(int layer, const Position3D& start, const Position3D& end, std::vector& paths); + static const long RCN_NAVMESH_VERSION = 1; + static const int INVALID_NAVMESH_POLYREF = 0; - int findRandomPointAroundCircle(int layer, const Position3D& centerPos, std::vector& points, - uint32 max_points, float maxRadius); + struct NavmeshLayer + { + dtNavMesh* pNavmesh; + dtNavMeshQuery* pNavmeshQuery; + }; - int raycast(int layer, const Position3D& start, const Position3D& end, std::vector& hitPointVec); + public: + NavMeshHandle(); + virtual ~NavMeshHandle(); - virtual NavigationHandle::NAV_TYPE type() const{ return NAV_MESH; } + int findStraightPath(int layer, const Position3D& start, const Position3D& end, std::vector& paths); - static NavigationHandle* create(std::string resPath, const std::map< int, std::string >& params); - static bool _create(int layer, const std::string& resPath, const std::string& res, NavMeshHandle* pNavMeshHandle); - - std::map navmeshLayer; -}; + int findRandomPointAroundCircle(int layer, const Position3D& centerPos, std::vector& points, + uint32 max_points, float maxRadius); + + int raycast(int layer, const Position3D& start, const Position3D& end, std::vector& hitPointVec); + + virtual NavigationHandle::NAV_TYPE type() const { return NAV_MESH; } + + static NavigationHandle* create(std::string resPath, const std::map< int, std::string >& params); + static bool _create(int layer, const std::string& resPath, const std::string& res, NavMeshHandle* pNavMeshHandle); + std::map navmeshLayer; + + private: + /* Derives overlap polygon of two polygon on the xz-plane. + @param[in] polyVertsA Vertices of polygon A. + @param[in] nPolyVertsA Vertices number of polygon A. + @param[in] polyVertsB Vertices of polygon B. + @param[in] nPolyVertsB Vertices number of polygon B. + @param[out] intsectPt Vertices of overlap polygon. + @param[out] intsectPtCount Vertices number of overlap polygon. + */ + void getOverlapPolyPoly2D(const float* polyVertsA, const int nPolyVertsA, const float* polyVertsB, const int nPolyVertsB, float* intsectPt, int* intsectPtCount); + + /* Sort vertices to clockwise. */ + void clockwiseSortPoints(float* verts, const int nVerts); + + /* Determines if two segment cross on xz-plane. */ + bool isSegSegCross2D(const float* p1, const float *p2, const float* q1, const float* q2); +}; } -#endif // KBE_NAVIGATEMESHHANDLE_H +#endif // KBE_NAVIGATEMESHHANDLE_H \ No newline at end of file diff --git a/kbe/src/lib/network/bundle_broadcast.cpp b/kbe/src/lib/network/bundle_broadcast.cpp index a00c9690f1..38c42a84e0 100644 --- a/kbe/src/lib/network/bundle_broadcast.cpp +++ b/kbe/src/lib/network/bundle_broadcast.cpp @@ -61,7 +61,7 @@ BundleBroadcast::BundleBroadcast(NetworkInterface & networkInterface, epListen_.addr(htons(bindPort), htonl(INADDR_ANY)); good_ = true; - DEBUG_MSG(fmt::format("BundleBroadcast::BundleBroadcast: epListen {}\n", epListen_.c_str())); + // DEBUG_MSG(fmt::format("BundleBroadcast::BundleBroadcast: epListen {}\n", epListen_.c_str())); break; } } diff --git a/kbe/src/lib/network/channel.cpp b/kbe/src/lib/network/channel.cpp index 71eefd6d13..5a300bdbc6 100644 --- a/kbe/src/lib/network/channel.cpp +++ b/kbe/src/lib/network/channel.cpp @@ -663,6 +663,7 @@ void Channel::handleTimeout(TimerHandle, void * arg) { this->networkInterface().onChannelTimeOut(this); } + break; } case KCP_UPDATE: diff --git a/kbe/src/lib/network/channel.h b/kbe/src/lib/network/channel.h index 99b5fe4251..48da3e2819 100644 --- a/kbe/src/lib/network/channel.h +++ b/kbe/src/lib/network/channel.h @@ -155,16 +155,18 @@ class Channel : public TimerHandler, public PoolObject ChannelID id() const { return id_; } void id(ChannelID v) { id_ = v; } - uint32 numPacketsSent() const { return numPacketsSent_; } - uint32 numPacketsReceived() const { return numPacketsReceived_; } - uint32 numBytesSent() const { return numBytesSent_; } - uint32 numBytesReceived() const { return numBytesReceived_; } - - uint64 lastReceivedTime() const { return lastReceivedTime_; } - void updateLastReceivedTime() { lastReceivedTime_ = timestamp(); } - + uint32 numPacketsSent() const { return numPacketsSent_; } + uint32 numPacketsReceived() const { return numPacketsReceived_; } + uint32 numBytesSent() const { return numBytesSent_; } + uint32 numBytesReceived() const { return numBytesReceived_; } + + uint64 lastReceivedTime() const { return lastReceivedTime_; } + void updateLastReceivedTime() { lastReceivedTime_ = timestamp(); } + void addReceiveWindow(Packet* pPacket); + uint64 inactivityExceptionPeriod() const { return inactivityExceptionPeriod_; } + void updateTick(KBEngine::Network::MessageHandlers* pMsgHandlers); void processPackets(KBEngine::Network::MessageHandlers* pMsgHandlers, Packet* pPacket); diff --git a/kbe/src/lib/pyscript/math.cpp b/kbe/src/lib/pyscript/math.cpp index af9d889715..cef44918f6 100644 --- a/kbe/src/lib/pyscript/math.cpp +++ b/kbe/src/lib/pyscript/math.cpp @@ -1,34 +1,36 @@ -// Copyright 2008-2018 Yolo Technologies, Inc. All Rights Reserved. https://www.comblockengine.com - - -#include "math.h" -namespace KBEngine{ namespace script{ namespace math { - -//------------------------------------------------------------------------------------- -bool installModule(const char* moduleName) -{ - // ʼһѧصģ - PyObject *mathModule = PyImport_AddModule(moduleName); - PyObject_SetAttrString(mathModule, "__doc__", PyUnicode_FromString("This module is created by KBEngine!")); - - // ʼScriptVector2 - script::ScriptVector2::installScript(mathModule, "Vector2"); - // ʼScriptVector3 - script::ScriptVector3::installScript(mathModule, "Vector3"); - // ʼScriptVector4 - script::ScriptVector4::installScript(mathModule, "Vector4"); - return true; -} - -//------------------------------------------------------------------------------------- -bool uninstallModule() -{ - script::ScriptVector2::uninstallScript(); - script::ScriptVector3::uninstallScript(); - script::ScriptVector4::uninstallScript(); - return true; -} - -} -} -} +// Copyright 2008-2018 Yolo Technologies, Inc. All Rights Reserved. https://www.comblockengine.com + + +#include "math.h" +namespace KBEngine{ namespace script{ namespace math { + +//------------------------------------------------------------------------------------- +bool installModule(const char* moduleName) +{ + // ʼһѧصģ + PyObject *mathModule = PyImport_AddModule(moduleName); + PyObject* pyDoc = PyUnicode_FromString("This module is created by KBEngine!"); + PyObject_SetAttrString(mathModule, "__doc__", pyDoc); + Py_DECREF(pyDoc); + + // ʼScriptVector2 + script::ScriptVector2::installScript(mathModule, "Vector2"); + // ʼScriptVector3 + script::ScriptVector3::installScript(mathModule, "Vector3"); + // ʼScriptVector4 + script::ScriptVector4::installScript(mathModule, "Vector4"); + return true; +} + +//------------------------------------------------------------------------------------- +bool uninstallModule() +{ + script::ScriptVector2::uninstallScript(); + script::ScriptVector3::uninstallScript(); + script::ScriptVector4::uninstallScript(); + return true; +} + +} +} +} diff --git a/kbe/src/lib/pyscript/script.cpp b/kbe/src/lib/pyscript/script.cpp index 8bbd588276..f0d63864c7 100644 --- a/kbe/src/lib/pyscript/script.cpp +++ b/kbe/src/lib/pyscript/script.cpp @@ -1,376 +1,378 @@ -// Copyright 2008-2018 Yolo Technologies, Inc. All Rights Reserved. https://www.comblockengine.com - - -#include "script.h" -#include "math.h" -#include "pickler.h" -#include "pyprofile.h" -#include "copy.h" -#include "pystruct.h" -#include "py_gc.h" -#include "pyurl.h" -#include "py_compression.h" -#include "py_platform.h" -#include "resmgr/resmgr.h" -#include "thread/concurrency.h" - -#ifndef CODE_INLINE -#include "script.inl" -#endif - -namespace KBEngine{ - -KBE_SINGLETON_INIT(script::Script); -namespace script{ - -//------------------------------------------------------------------------------------- -static PyObject* __py_genUUID64(PyObject *self, void *closure) -{ - static int8 check = -1; - - if(check < 0) - { - if(g_componentGlobalOrder <= 0 || g_componentGlobalOrder > 65535) - { - WARNING_MSG(fmt::format("globalOrder({}) is not in the range of 0~65535, genUUID64 is not safe, " - "in the multi process may be repeated.\n", g_componentGlobalOrder)); - } - - check = 1; - } - - return PyLong_FromUnsignedLongLong(genUUID64()); -} - -//------------------------------------------------------------------------------------- -PyObject * PyTuple_FromStringVector(const std::vector< std::string > & v) -{ - int sz = v.size(); - PyObject * t = PyTuple_New( sz ); - for (int i = 0; i < sz; ++i) - { - PyTuple_SetItem( t, i, PyUnicode_FromString( v[i].c_str() ) ); - } - - return t; -} - -//------------------------------------------------------------------------------------- -Script::Script(): -module_(NULL), -extraModule_(NULL), -sysInitModules_(NULL), -pyStdouterr_(NULL) -{ -} - -//------------------------------------------------------------------------------------- -Script::~Script() -{ -} - -//------------------------------------------------------------------------------------- -int Script::run_simpleString(const char* command, std::string* retBufferPtr) -{ - if(command == NULL) - { - ERROR_MSG("Script::Run_SimpleString: command is NULL!\n"); - return 0; - } - - ScriptStdOutErrHook* pStdouterrHook = new ScriptStdOutErrHook(); - - if(retBufferPtr != NULL) - { - DebugHelper::getSingleton().resetScriptMsgType(); - if(!pStdouterrHook->install()){ - ERROR_MSG("Script::Run_SimpleString: pyStdouterrHook_->install() is failed!\n"); - SCRIPT_ERROR_CHECK(); - delete pStdouterrHook; - return -1; - } - - pStdouterrHook->setHookBuffer(retBufferPtr); - //PyRun_SimpleString(command); - - PyObject *m, *d, *v; - m = PyImport_AddModule("__main__"); - if (m == NULL) - { - SCRIPT_ERROR_CHECK(); - pStdouterrHook->uninstall(); - delete pStdouterrHook; - return -1; - } - - d = PyModule_GetDict(m); - - v = PyRun_String(command, Py_single_input, d, d); - if (v == NULL) - { - PyErr_Print(); - pStdouterrHook->uninstall(); - delete pStdouterrHook; - return -1; - } - - Py_DECREF(v); - SCRIPT_ERROR_CHECK(); - - pStdouterrHook->uninstall(); - delete pStdouterrHook; - return 0; - } - - PyRun_SimpleString(command); - - SCRIPT_ERROR_CHECK(); - delete pStdouterrHook; - return 0; -} - -//------------------------------------------------------------------------------------- -bool Script::install(const wchar_t* pythonHomeDir, std::wstring pyPaths, - const char* moduleName, COMPONENT_TYPE componentType) -{ - APPEND_PYSYSPATH(pyPaths); - - // pythonĻ - Py_SetPythonHome(const_cast(pythonHomeDir)); - -#if KBE_PLATFORM != PLATFORM_WIN32 - strutil::kbe_replace(pyPaths, L";", L":"); - - char* tmpchar = strutil::wchar2char(const_cast(pyPaths.c_str())); - DEBUG_MSG(fmt::format("Script::install(): paths={}.\n", tmpchar)); - free(tmpchar); -#endif - - // Initialise python - // Py_VerboseFlag = 2; - Py_FrozenFlag = 1; - - // Warn if tab and spaces are mixed in indentation. - // Py_TabcheckFlag = 1; - Py_NoSiteFlag = 1; - Py_IgnoreEnvironmentFlag = 1; - - Py_SetPath(pyPaths.c_str()); - - // pythonijʼ - Py_Initialize(); - if (!Py_IsInitialized()) - { - ERROR_MSG("Script::install(): Py_Initialize is failed!\n"); - return false; - } - - sysInitModules_ = PyDict_Copy(PySys_GetObject("modules")); - - PySys_SetArgvEx(0, NULL, 0); - PyObject *m = PyImport_AddModule("__main__"); - - // һűģ - module_ = PyImport_AddModule(moduleName); - if (module_ == NULL) - return false; - - const char* componentName = COMPONENT_NAME_EX(componentType); - if (PyModule_AddStringConstant(module_, "component", componentName)) - { - ERROR_MSG(fmt::format("Script::install(): Unable to set KBEngine.component to {}\n", - componentName)); - return false; - } - - PyEval_InitThreads(); - - // עuuidpy - APPEND_SCRIPT_MODULE_METHOD(module_, genUUID64, __py_genUUID64, METH_VARARGS, 0); - - // װpyضģ - ScriptStdOut::installScript(NULL); - ScriptStdErr::installScript(NULL); - - // ģmain - PyObject_SetAttrString(m, moduleName, module_); - PyObject_SetAttrString(module_, "__doc__", PyUnicode_FromString("This module is created by KBEngine!")); - - // ضpython - pyStdouterr_ = new ScriptStdOutErr(); - - // װpyضűģ - if(!pyStdouterr_->install()){ - ERROR_MSG("Script::install::pyStdouterr_->install() is failed!\n"); - delete pyStdouterr_; - SCRIPT_ERROR_CHECK(); - return false; - } - - PyGC::initialize(); - Pickler::initialize(); - PyProfile::initialize(this); - PyStruct::initialize(); - Copy::initialize(); - PyUrl::initialize(this); - PyCompression::initialize(); - PyPlatform::initialize(); - SCRIPT_ERROR_CHECK(); - - math::installModule("Math"); - INFO_MSG(fmt::format("Script::install(): is successfully, Python=({})!\n", Py_GetVersion())); - return installExtraModule("KBExtra"); -} - -//------------------------------------------------------------------------------------- -bool Script::uninstall() -{ - math::uninstallModule(); - Pickler::finalise(); - PyProfile::finalise(); - PyStruct::finalise(); - Copy::finalise(); - PyUrl::finalise(); - PyCompression::finalise(); - PyPlatform::finalise(); - SCRIPT_ERROR_CHECK(); - - if(pyStdouterr_) - { - if(pyStdouterr_->isInstall() && !pyStdouterr_->uninstall()) { - ERROR_MSG("Script::uninstall(): pyStdouterr_->uninstall() is failed!\n"); - } - - delete pyStdouterr_; - } - - ScriptStdOut::uninstallScript(); - ScriptStdErr::uninstallScript(); - - PyGC::finalise(); - - if (sysInitModules_) - { - Py_DECREF(sysInitModules_); - sysInitModules_ = NULL; - } - - // жpython - Py_Finalize(); - - INFO_MSG("Script::uninstall(): is successfully!\n"); - return true; -} - -//------------------------------------------------------------------------------------- -bool Script::installExtraModule(const char* moduleName) -{ - PyObject *m = PyImport_AddModule("__main__"); - - // һűչģ - extraModule_ = PyImport_AddModule(moduleName); - if (extraModule_ == NULL) - return false; - - // չģmain - PyObject_SetAttrString(m, moduleName, extraModule_); - - INFO_MSG(fmt::format("Script::install(): {} is successfully!\n", moduleName)); - return true; -} - -//------------------------------------------------------------------------------------- -bool Script::registerExtraMethod(const char* attrName, PyMethodDef* pyFunc) -{ - return PyModule_AddObject(extraModule_, attrName, PyCFunction_New(pyFunc, NULL)) != -1; -} - -//------------------------------------------------------------------------------------- -bool Script::registerExtraObject(const char* attrName, PyObject* pyObj) -{ - return PyObject_SetAttrString(extraModule_, attrName, pyObj) != -1; -} - -//------------------------------------------------------------------------------------- -int Script::registerToModule(const char* attrName, PyObject* pyObj) -{ - return PyObject_SetAttrString(module_, attrName, pyObj); -} - -//------------------------------------------------------------------------------------- -int Script::unregisterToModule(const char* attrName) -{ - if(module_ == NULL || attrName == NULL) - return 0; - - return PyObject_DelAttrString(module_, attrName); -} - -//------------------------------------------------------------------------------------- -void Script::setenv(const std::string& name, const std::string& value) -{ - PyObject* osModule = PyImport_ImportModule("os"); - - if(osModule) - { - PyObject* py_environ = NULL; - PyObject* py_name = NULL; - PyObject* py_value = NULL; - - PyObject* supports_bytes_environ = PyObject_GetAttrString(osModule, "supports_bytes_environ"); - if(Py_True == supports_bytes_environ) - { - py_environ = PyObject_GetAttrString(osModule, "environb"); - py_name = PyBytes_FromString(name.c_str()); - py_value = PyBytes_FromString(value.c_str()); - } - else - { - py_environ = PyObject_GetAttrString(osModule, "environ"); - py_name = PyUnicode_FromString(name.c_str()); - py_value = PyUnicode_FromString(value.c_str()); - } - - Py_DECREF(supports_bytes_environ); - Py_DECREF(osModule); - - if (!py_environ) - { - ERROR_MSG("Script::setenv: get os.environ error!\n"); - PyErr_PrintEx(0); - Py_DECREF(py_value); - Py_DECREF(py_name); - return; - } - - PyObject* environData = PyObject_GetAttrString(py_environ, "_data"); - if (!environData) - { - ERROR_MSG("Script::setenv: os.environ._data not exist!\n"); - PyErr_PrintEx(0); - Py_DECREF(py_value); - Py_DECREF(py_name); - Py_DECREF(py_environ); - return; - } - - int ret = PyDict_SetItem(environData, py_name, py_value); - - Py_DECREF(environData); - Py_DECREF(py_environ); - Py_DECREF(py_value); - Py_DECREF(py_name); - - if(ret == -1) - { - ERROR_MSG("Script::setenv: get os.environ error!\n"); - PyErr_PrintEx(0); - return; - } - } -} - -//------------------------------------------------------------------------------------- - -} -} +// Copyright 2008-2018 Yolo Technologies, Inc. All Rights Reserved. https://www.comblockengine.com + + +#include "script.h" +#include "math.h" +#include "pickler.h" +#include "pyprofile.h" +#include "copy.h" +#include "pystruct.h" +#include "py_gc.h" +#include "pyurl.h" +#include "py_compression.h" +#include "py_platform.h" +#include "resmgr/resmgr.h" +#include "thread/concurrency.h" + +#ifndef CODE_INLINE +#include "script.inl" +#endif + +namespace KBEngine{ + +KBE_SINGLETON_INIT(script::Script); +namespace script{ + +//------------------------------------------------------------------------------------- +static PyObject* __py_genUUID64(PyObject *self, void *closure) +{ + static int8 check = -1; + + if(check < 0) + { + if(g_componentGlobalOrder <= 0 || g_componentGlobalOrder > 65535) + { + WARNING_MSG(fmt::format("globalOrder({}) is not in the range of 0~65535, genUUID64 is not safe, " + "in the multi process may be repeated.\n", g_componentGlobalOrder)); + } + + check = 1; + } + + return PyLong_FromUnsignedLongLong(genUUID64()); +} + +//------------------------------------------------------------------------------------- +PyObject * PyTuple_FromStringVector(const std::vector< std::string > & v) +{ + int sz = v.size(); + PyObject * t = PyTuple_New( sz ); + for (int i = 0; i < sz; ++i) + { + PyTuple_SetItem( t, i, PyUnicode_FromString( v[i].c_str() ) ); + } + + return t; +} + +//------------------------------------------------------------------------------------- +Script::Script(): +module_(NULL), +extraModule_(NULL), +sysInitModules_(NULL), +pyStdouterr_(NULL) +{ +} + +//------------------------------------------------------------------------------------- +Script::~Script() +{ +} + +//------------------------------------------------------------------------------------- +int Script::run_simpleString(const char* command, std::string* retBufferPtr) +{ + if(command == NULL) + { + ERROR_MSG("Script::Run_SimpleString: command is NULL!\n"); + return 0; + } + + ScriptStdOutErrHook* pStdouterrHook = new ScriptStdOutErrHook(); + + if(retBufferPtr != NULL) + { + DebugHelper::getSingleton().resetScriptMsgType(); + if(!pStdouterrHook->install()){ + ERROR_MSG("Script::Run_SimpleString: pyStdouterrHook_->install() is failed!\n"); + SCRIPT_ERROR_CHECK(); + delete pStdouterrHook; + return -1; + } + + pStdouterrHook->setHookBuffer(retBufferPtr); + //PyRun_SimpleString(command); + + PyObject *m, *d, *v; + m = PyImport_AddModule("__main__"); + if (m == NULL) + { + SCRIPT_ERROR_CHECK(); + pStdouterrHook->uninstall(); + delete pStdouterrHook; + return -1; + } + + d = PyModule_GetDict(m); + + v = PyRun_String(command, Py_single_input, d, d); + if (v == NULL) + { + PyErr_Print(); + pStdouterrHook->uninstall(); + delete pStdouterrHook; + return -1; + } + + Py_DECREF(v); + SCRIPT_ERROR_CHECK(); + + pStdouterrHook->uninstall(); + delete pStdouterrHook; + return 0; + } + + PyRun_SimpleString(command); + + SCRIPT_ERROR_CHECK(); + delete pStdouterrHook; + return 0; +} + +//------------------------------------------------------------------------------------- +bool Script::install(const wchar_t* pythonHomeDir, std::wstring pyPaths, + const char* moduleName, COMPONENT_TYPE componentType) +{ + APPEND_PYSYSPATH(pyPaths); + + // pythonĻ + Py_SetPythonHome(const_cast(pythonHomeDir)); + +#if KBE_PLATFORM != PLATFORM_WIN32 + strutil::kbe_replace(pyPaths, L";", L":"); + + char* tmpchar = strutil::wchar2char(const_cast(pyPaths.c_str())); + DEBUG_MSG(fmt::format("Script::install(): paths={}.\n", tmpchar)); + free(tmpchar); +#endif + + // Initialise python + // Py_VerboseFlag = 2; + Py_FrozenFlag = 1; + + // Warn if tab and spaces are mixed in indentation. + // Py_TabcheckFlag = 1; + Py_NoSiteFlag = 1; + Py_IgnoreEnvironmentFlag = 1; + + Py_SetPath(pyPaths.c_str()); + + // pythonijʼ + Py_Initialize(); + if (!Py_IsInitialized()) + { + ERROR_MSG("Script::install(): Py_Initialize is failed!\n"); + return false; + } + + sysInitModules_ = PyDict_Copy(PySys_GetObject("modules")); + + PySys_SetArgvEx(0, NULL, 0); + PyObject *m = PyImport_AddModule("__main__"); + + // һűģ + module_ = PyImport_AddModule(moduleName); + if (module_ == NULL) + return false; + + const char* componentName = COMPONENT_NAME_EX(componentType); + if (PyModule_AddStringConstant(module_, "component", componentName)) + { + ERROR_MSG(fmt::format("Script::install(): Unable to set KBEngine.component to {}\n", + componentName)); + return false; + } + + PyEval_InitThreads(); + + // עuuidpy + APPEND_SCRIPT_MODULE_METHOD(module_, genUUID64, __py_genUUID64, METH_VARARGS, 0); + + // װpyضģ + ScriptStdOut::installScript(NULL); + ScriptStdErr::installScript(NULL); + + // ģmain + PyObject_SetAttrString(m, moduleName, module_); + PyObject* pyDoc = PyUnicode_FromString("This module is created by KBEngine!"); + PyObject_SetAttrString(module_, "__doc__", pyDoc); + Py_DECREF(pyDoc); + + // ضpython + pyStdouterr_ = new ScriptStdOutErr(); + + // װpyضűģ + if(!pyStdouterr_->install()){ + ERROR_MSG("Script::install::pyStdouterr_->install() is failed!\n"); + delete pyStdouterr_; + SCRIPT_ERROR_CHECK(); + return false; + } + + PyGC::initialize(); + Pickler::initialize(); + PyProfile::initialize(this); + PyStruct::initialize(); + Copy::initialize(); + PyUrl::initialize(this); + PyCompression::initialize(); + PyPlatform::initialize(); + SCRIPT_ERROR_CHECK(); + + math::installModule("Math"); + INFO_MSG(fmt::format("Script::install(): is successfully, Python=({})!\n", Py_GetVersion())); + return installExtraModule("KBExtra"); +} + +//------------------------------------------------------------------------------------- +bool Script::uninstall() +{ + math::uninstallModule(); + Pickler::finalise(); + PyProfile::finalise(); + PyStruct::finalise(); + Copy::finalise(); + PyUrl::finalise(); + PyCompression::finalise(); + PyPlatform::finalise(); + SCRIPT_ERROR_CHECK(); + + if(pyStdouterr_) + { + if(pyStdouterr_->isInstall() && !pyStdouterr_->uninstall()) { + ERROR_MSG("Script::uninstall(): pyStdouterr_->uninstall() is failed!\n"); + } + + delete pyStdouterr_; + } + + ScriptStdOut::uninstallScript(); + ScriptStdErr::uninstallScript(); + + PyGC::finalise(); + + if (sysInitModules_) + { + Py_DECREF(sysInitModules_); + sysInitModules_ = NULL; + } + + // жpython + Py_Finalize(); + + INFO_MSG("Script::uninstall(): is successfully!\n"); + return true; +} + +//------------------------------------------------------------------------------------- +bool Script::installExtraModule(const char* moduleName) +{ + PyObject *m = PyImport_AddModule("__main__"); + + // һűչģ + extraModule_ = PyImport_AddModule(moduleName); + if (extraModule_ == NULL) + return false; + + // չģmain + PyObject_SetAttrString(m, moduleName, extraModule_); + + INFO_MSG(fmt::format("Script::install(): {} is successfully!\n", moduleName)); + return true; +} + +//------------------------------------------------------------------------------------- +bool Script::registerExtraMethod(const char* attrName, PyMethodDef* pyFunc) +{ + return PyModule_AddObject(extraModule_, attrName, PyCFunction_New(pyFunc, NULL)) != -1; +} + +//------------------------------------------------------------------------------------- +bool Script::registerExtraObject(const char* attrName, PyObject* pyObj) +{ + return PyObject_SetAttrString(extraModule_, attrName, pyObj) != -1; +} + +//------------------------------------------------------------------------------------- +int Script::registerToModule(const char* attrName, PyObject* pyObj) +{ + return PyObject_SetAttrString(module_, attrName, pyObj); +} + +//------------------------------------------------------------------------------------- +int Script::unregisterToModule(const char* attrName) +{ + if(module_ == NULL || attrName == NULL) + return 0; + + return PyObject_DelAttrString(module_, attrName); +} + +//------------------------------------------------------------------------------------- +void Script::setenv(const std::string& name, const std::string& value) +{ + PyObject* osModule = PyImport_ImportModule("os"); + + if(osModule) + { + PyObject* py_environ = NULL; + PyObject* py_name = NULL; + PyObject* py_value = NULL; + + PyObject* supports_bytes_environ = PyObject_GetAttrString(osModule, "supports_bytes_environ"); + if(Py_True == supports_bytes_environ) + { + py_environ = PyObject_GetAttrString(osModule, "environb"); + py_name = PyBytes_FromString(name.c_str()); + py_value = PyBytes_FromString(value.c_str()); + } + else + { + py_environ = PyObject_GetAttrString(osModule, "environ"); + py_name = PyUnicode_FromString(name.c_str()); + py_value = PyUnicode_FromString(value.c_str()); + } + + Py_DECREF(supports_bytes_environ); + Py_DECREF(osModule); + + if (!py_environ) + { + ERROR_MSG("Script::setenv: get os.environ error!\n"); + PyErr_PrintEx(0); + Py_DECREF(py_value); + Py_DECREF(py_name); + return; + } + + PyObject* environData = PyObject_GetAttrString(py_environ, "_data"); + if (!environData) + { + ERROR_MSG("Script::setenv: os.environ._data not exist!\n"); + PyErr_PrintEx(0); + Py_DECREF(py_value); + Py_DECREF(py_name); + Py_DECREF(py_environ); + return; + } + + int ret = PyDict_SetItem(environData, py_name, py_value); + + Py_DECREF(environData); + Py_DECREF(py_environ); + Py_DECREF(py_value); + Py_DECREF(py_name); + + if(ret == -1) + { + ERROR_MSG("Script::setenv: get os.environ error!\n"); + PyErr_PrintEx(0); + return; + } + } +} + +//------------------------------------------------------------------------------------- + +} +} diff --git a/kbe/src/lib/python/.azure-pipelines/ci.yml b/kbe/src/lib/python/.azure-pipelines/ci.yml index 49a7bb6232..15a83dd037 100644 --- a/kbe/src/lib/python/.azure-pipelines/ci.yml +++ b/kbe/src/lib/python/.azure-pipelines/ci.yml @@ -2,6 +2,11 @@ variables: manylinux: false coverage: false +resources: + containers: + - container: manylinux1 + image: pyca/cryptography-manylinux1:x86_64 + jobs: - job: Prebuild displayName: Pre-build checks @@ -54,10 +59,12 @@ jobs: variables: testRunTitle: '$(build.sourceBranchName)-linux' testRunPlatform: linux - openssl_version: 1.1.0g + openssl_version: 1.1.0j steps: - template: ./posix-steps.yml + parameters: + dependencies: apt - job: ManyLinux1_CI_Tests @@ -75,13 +82,20 @@ jobs: pool: vmImage: ubuntu-16.04 + container: manylinux1 + variables: testRunTitle: '$(build.sourceBranchName)-manylinux1' testRunPlatform: manylinux1 - imageName: 'dockcross/manylinux-x64' + openssl_version: '' steps: - - template: ./docker-steps.yml + - template: ./posix-steps.yml + parameters: + dependencies: yum + sudo_dependencies: '' + xvfb: false + patchcheck: false - job: Ubuntu_Coverage_CI_Tests @@ -102,11 +116,12 @@ jobs: variables: testRunTitle: '$(Build.SourceBranchName)-linux-coverage' testRunPlatform: linux-coverage - openssl_version: 1.1.0g + openssl_version: 1.1.0j steps: - template: ./posix-steps.yml parameters: + dependencies: apt coverage: true @@ -134,3 +149,14 @@ jobs: steps: - template: ./windows-steps.yml + + - template: ./windows-layout-steps.yml + parameters: + kind: nuget + - template: ./windows-layout-steps.yml + parameters: + kind: embed + - template: ./windows-layout-steps.yml + parameters: + kind: appx + fulltest: true diff --git a/kbe/src/lib/python/.azure-pipelines/docker-steps.yml b/kbe/src/lib/python/.azure-pipelines/docker-steps.yml deleted file mode 100644 index ba4dfd72dd..0000000000 --- a/kbe/src/lib/python/.azure-pipelines/docker-steps.yml +++ /dev/null @@ -1,76 +0,0 @@ -steps: -- checkout: self - clean: true - fetchDepth: 5 - -- ${{ if ne(parameters.targetBranch, '') }}: - - script: | - git fetch -q origin ${{ parameters.targetbranch }} - if ! git diff --name-only HEAD $(git merge-base HEAD FETCH_HEAD) | grep -qvE '(\.rst$|^Doc|^Misc)' - then - echo "Only docs were updated, stopping build process." - echo "##vso[task.setvariable variable=DocOnly]true" - exit - fi - displayName: Detect doc-only changes - -- task: docker@0 - displayName: 'Configure CPython (debug)' - inputs: - action: 'Run an image' - imageName: $(imageName) - volumes: | - $(build.sourcesDirectory):/src - $(build.binariesDirectory):/build - workDir: '/src' - containerCommand: './configure --with-pydebug' - detached: false - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - -- task: docker@0 - displayName: 'Build CPython' - inputs: - action: 'Run an image' - imageName: $(imageName) - volumes: | - $(build.sourcesDirectory):/src - $(build.binariesDirectory):/build - workDir: '/src' - containerCommand: 'make -s -j4' - detached: false - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - -- task: docker@0 - displayName: 'Display build info' - inputs: - action: 'Run an image' - imageName: $(imageName) - volumes: | - $(build.sourcesDirectory):/src - $(build.binariesDirectory):/build - workDir: '/src' - containerCommand: 'make pythoninfo' - detached: false - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - -- task: docker@0 - displayName: 'Tests' - inputs: - action: 'Run an image' - imageName: $(imageName) - volumes: | - $(build.sourcesDirectory):/src - $(build.binariesDirectory):/build - workDir: '/src' - containerCommand: 'make buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=/build/test-results.xml"' - detached: false - condition: and(succeeded(), ne(variables['DocOnly'], 'true')) - -- task: PublishTestResults@2 - displayName: 'Publish Test Results' - inputs: - testResultsFiles: '$(build.binariesDirectory)/test-results.xml' - mergeTestResults: true - testRunTitle: $(testRunTitle) - platform: $(testRunPlatform) - condition: and(succeededOrFailed(), ne(variables['DocOnly'], 'true')) diff --git a/kbe/src/lib/python/.azure-pipelines/posix-deps.sh b/kbe/src/lib/python/.azure-pipelines/posix-deps-apt.sh old mode 100755 new mode 100644 similarity index 92% rename from kbe/src/lib/python/.azure-pipelines/posix-deps.sh rename to kbe/src/lib/python/.azure-pipelines/posix-deps-apt.sh index a572107566..4f489903ab --- a/kbe/src/lib/python/.azure-pipelines/posix-deps.sh +++ b/kbe/src/lib/python/.azure-pipelines/posix-deps-apt.sh @@ -1,6 +1,6 @@ -sudo apt-get update +apt-get update -sudo apt-get -yq install \ +apt-get -yq install \ build-essential \ zlib1g-dev \ libbz2-dev \ diff --git a/kbe/src/lib/python/.azure-pipelines/posix-steps.yml b/kbe/src/lib/python/.azure-pipelines/posix-steps.yml index 9fec9be801..a4160e5a1b 100644 --- a/kbe/src/lib/python/.azure-pipelines/posix-steps.yml +++ b/kbe/src/lib/python/.azure-pipelines/posix-steps.yml @@ -1,12 +1,16 @@ parameters: coverage: false + sudo_dependencies: sudo + dependencies: apt + patchcheck: true + xvfb: true steps: - checkout: self clean: true fetchDepth: 5 -- script: ./.azure-pipelines/posix-deps.sh $(openssl_version) +- script: ${{ parameters.sudo_dependencies }} ./.azure-pipelines/posix-deps-${{ parameters.dependencies }}.sh $(openssl_version) displayName: 'Install dependencies' - script: ./configure --with-pydebug @@ -23,7 +27,7 @@ steps: displayName: 'Display build info' - script: | - xvfb-run ./venv/bin/python -m coverage run --pylib -m test \ + $COMMAND -m coverage run --pylib -m test \ --fail-env-changed \ -uall,-cpu \ --junit-xml=$(build.binariesDirectory)/test-results.xml" \ @@ -32,6 +36,11 @@ steps: -x test_multiprocessing_spawn \ -x test_concurrent_futures displayName: 'Tests with coverage' + env: + ${{ if eq(parameters.xvfb, 'true') }}: + COMMAND: xvfb-run ./venv/bin/python + ${{ if ne(parameters.xvfb, 'true') }}: + COMMAND: ./venv/bin/python - script: ./venv/bin/python -m coverage xml displayName: 'Generate coverage.xml' @@ -44,13 +53,18 @@ steps: - script: make pythoninfo displayName: 'Display build info' - - script: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=$(build.binariesDirectory)/test-results.xml" + - script: $COMMAND buildbottest TESTOPTS="-j4 -uall,-cpu --junit-xml=$(build.binariesDirectory)/test-results.xml" displayName: 'Tests' - - -- script: ./python Tools/scripts/patchcheck.py --travis true - displayName: 'Run patchcheck.py' - condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest')) + env: + ${{ if eq(parameters.xvfb, 'true') }}: + COMMAND: xvfb-run make + ${{ if ne(parameters.xvfb, 'true') }}: + COMMAND: make + +- ${{ if eq(parameters.patchcheck, 'true') }}: + - script: ./python Tools/scripts/patchcheck.py --travis true + displayName: 'Run patchcheck.py' + condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest')) - task: PublishTestResults@2 diff --git a/kbe/src/lib/python/.azure-pipelines/pr.yml b/kbe/src/lib/python/.azure-pipelines/pr.yml index 2d7fba9cf3..0bd7921bcb 100644 --- a/kbe/src/lib/python/.azure-pipelines/pr.yml +++ b/kbe/src/lib/python/.azure-pipelines/pr.yml @@ -1,3 +1,12 @@ +variables: + manylinux: false + coverage: false + +resources: + containers: + - container: manylinux1 + image: pyca/cryptography-manylinux1:x86_64 + jobs: - job: Prebuild displayName: Pre-build checks @@ -50,12 +59,70 @@ jobs: variables: testRunTitle: '$(system.pullRequest.TargetBranch)-linux' testRunPlatform: linux - openssl_version: 1.1.0g + openssl_version: 1.1.0j steps: - template: ./posix-steps.yml parameters: - targetBranch: $(System.PullRequest.TargetBranch) + dependencies: apt + + +- job: ManyLinux1_PR_Tests + displayName: ManyLinux1 PR Tests + dependsOn: Prebuild + condition: | + and( + and( + succeeded(), + eq(variables['manylinux'], 'true') + ), + eq(dependencies.Prebuild.outputs['tests.run'], 'true') + ) + + pool: + vmImage: ubuntu-16.04 + + container: manylinux1 + + variables: + testRunTitle: '$(system.pullRequest.TargetBranch)-manylinux1' + testRunPlatform: manylinux1 + openssl_version: '' + + steps: + - template: ./posix-steps.yml + parameters: + dependencies: yum + sudo_dependencies: '' + xvfb: false + patchcheck: false + + +- job: Ubuntu_Coverage_PR_Tests + displayName: Ubuntu PR Tests (coverage) + dependsOn: Prebuild + condition: | + and( + and( + succeeded(), + eq(variables['coverage'], 'true') + ), + eq(dependencies.Prebuild.outputs['tests.run'], 'true') + ) + + pool: + vmImage: ubuntu-16.04 + + variables: + testRunTitle: '$(Build.SourceBranchName)-linux-coverage' + testRunPlatform: linux-coverage + openssl_version: 1.1.0j + + steps: + - template: ./posix-steps.yml + parameters: + dependencies: apt + coverage: true - job: Windows_PR_Tests diff --git a/kbe/src/lib/python/.azure-pipelines/windows-appx-test.yml b/kbe/src/lib/python/.azure-pipelines/windows-appx-test.yml deleted file mode 100644 index 5f3fe6c945..0000000000 --- a/kbe/src/lib/python/.azure-pipelines/windows-appx-test.yml +++ /dev/null @@ -1,67 +0,0 @@ -jobs: -- job: Prebuild - displayName: Pre-build checks - - pool: - vmImage: ubuntu-16.04 - - steps: - - template: ./prebuild-checks.yml - - -- job: Windows_Appx_Tests - displayName: Windows Appx Tests - dependsOn: Prebuild - condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) - - pool: - vmImage: vs2017-win2016 - - strategy: - matrix: - win64: - arch: amd64 - buildOpt: '-p x64' - testRunTitle: '$(Build.SourceBranchName)-win64-appx' - testRunPlatform: win64 - maxParallel: 2 - - steps: - - checkout: self - clean: true - fetchDepth: 5 - - - powershell: | - # Relocate build outputs outside of source directory to make cleaning faster - Write-Host '##vso[task.setvariable variable=Py_IntDir]$(Build.BinariesDirectory)\obj' - # UNDONE: Do not build to a different directory because of broken tests - Write-Host '##vso[task.setvariable variable=Py_OutDir]$(Build.SourcesDirectory)\PCbuild' - Write-Host '##vso[task.setvariable variable=EXTERNAL_DIR]$(Build.BinariesDirectory)\externals' - displayName: Update build locations - - - script: PCbuild\build.bat -e $(buildOpt) - displayName: 'Build CPython' - env: - IncludeUwp: true - - - script: python.bat PC\layout -vv -s "$(Build.SourcesDirectory)" -b "$(Py_OutDir)\$(arch)" -t "$(Py_IntDir)\layout-tmp-$(arch)" --copy "$(Py_IntDir)\layout-$(arch)" --precompile --preset-appx --include-tests - displayName: 'Create APPX layout' - - - script: .\python.exe -m test.pythoninfo - workingDirectory: $(Py_IntDir)\layout-$(arch) - displayName: 'Display build info' - - - script: .\python.exe -m test -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 --junit-xml="$(Build.BinariesDirectory)\test-results.xml" --tempdir "$(Py_IntDir)\tmp-$(arch)" - workingDirectory: $(Py_IntDir)\layout-$(arch) - displayName: 'Tests' - env: - PREFIX: $(Py_IntDir)\layout-$(arch) - - - task: PublishTestResults@2 - displayName: 'Publish Test Results' - inputs: - testResultsFiles: '$(Build.BinariesDirectory)\test-results.xml' - mergeTestResults: true - testRunTitle: $(testRunTitle) - platform: $(testRunPlatform) - condition: succeededOrFailed() diff --git a/kbe/src/lib/python/.azure-pipelines/windows-layout-steps.yml b/kbe/src/lib/python/.azure-pipelines/windows-layout-steps.yml new file mode 100644 index 0000000000..e15729fac3 --- /dev/null +++ b/kbe/src/lib/python/.azure-pipelines/windows-layout-steps.yml @@ -0,0 +1,28 @@ +parameters: + kind: nuget + extraOpts: --precompile + fulltest: false + +steps: +- script: .\python.bat PC\layout -vv -s "$(Build.SourcesDirectory)" -b "$(Py_OutDir)\$(arch)" -t "$(Build.BinariesDirectory)\layout-tmp-${{ parameters.kind }}-$(arch)" --copy "$(Build.BinariesDirectory)\layout-${{ parameters.kind }}-$(arch)" ${{ parameters.extraOpts }} --preset-${{ parameters.kind }} --include-tests + displayName: Create ${{ parameters.kind }} layout + +- script: .\python.exe -m test.pythoninfo + workingDirectory: $(Build.BinariesDirectory)\layout-${{ parameters.kind }}-$(arch) + displayName: Show layout info (${{ parameters.kind }}) + +- ${{ if eq(parameters.fulltest, 'true') }}: + - script: .\python.exe -m test -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 --junit-xml="$(Build.BinariesDirectory)\test-results-${{ parameters.kind }}.xml" --tempdir "$(Build.BinariesDirectory)\tmp-${{ parameters.kind }}-$(arch)" + workingDirectory: $(Build.BinariesDirectory)\layout-${{ parameters.kind }}-$(arch) + displayName: ${{ parameters.kind }} Tests + env: + PREFIX: $(Build.BinariesDirectory)\layout-${{ parameters.kind }}-$(arch) + + - task: PublishTestResults@2 + displayName: Publish ${{ parameters.kind }} Test Results + inputs: + testResultsFiles: $(Build.BinariesDirectory)\test-results-${{ parameters.kind }}.xml + mergeTestResults: true + testRunTitle: ${{ parameters.kind }}-$(testRunTitle) + platform: $(testRunPlatform) + condition: succeededOrFailed() diff --git a/kbe/src/lib/python/.azure-pipelines/windows-steps.yml b/kbe/src/lib/python/.azure-pipelines/windows-steps.yml index cba00158ad..794a23a5d7 100644 --- a/kbe/src/lib/python/.azure-pipelines/windows-steps.yml +++ b/kbe/src/lib/python/.azure-pipelines/windows-steps.yml @@ -1,6 +1,6 @@ steps: - checkout: self - clean: true + clean: false fetchDepth: 5 - powershell: | @@ -8,7 +8,8 @@ steps: Write-Host '##vso[task.setvariable variable=Py_IntDir]$(Build.BinariesDirectory)\obj' # UNDONE: Do not build to a different directory because of broken tests Write-Host '##vso[task.setvariable variable=Py_OutDir]$(Build.SourcesDirectory)\PCbuild' - Write-Host '##vso[task.setvariable variable=EXTERNAL_DIR]$(Build.BinariesDirectory)\externals' + #Write-Host '##vso[task.setvariable variable=Py_OutDir]$(Build.BinariesDirectory)\bin' + Write-Host '##vso[task.setvariable variable=EXTERNALS_DIR]$(Build.BinariesDirectory)\externals' displayName: Update build locations - script: PCbuild\build.bat -e $(buildOpt) diff --git a/kbe/src/lib/python/Doc/bugs.rst b/kbe/src/lib/python/Doc/bugs.rst index 109e9eb202..c449ba2e71 100644 --- a/kbe/src/lib/python/Doc/bugs.rst +++ b/kbe/src/lib/python/Doc/bugs.rst @@ -17,7 +17,7 @@ Documentation bugs If you find a bug in this documentation or would like to propose an improvement, please submit a bug report on the :ref:`tracker `. If you -have a suggestion how to fix it, include that as well. +have a suggestion on how to fix it, include that as well. If you're short on time, you can also email documentation bug reports to docs@python.org (behavioral bugs can be sent to python-list@python.org). @@ -89,4 +89,4 @@ any and all questions pertaining to the process of fixing issues in Python. .. _Documentation bugs: https://bugs.python.org/issue?@filter=status&@filter=components&components=4&status=1&@columns=id,activity,title,status&@sort=-activity .. _Python Developer's Guide: https://devguide.python.org/ -.. _core-mentorship mailing list: https://mail.python.org/mailman/listinfo/core-mentorship/ +.. _core-mentorship mailing list: https://mail.python.org/mailman3/lists/core-mentorship.python.org/ diff --git a/kbe/src/lib/python/Doc/c-api/buffer.rst b/kbe/src/lib/python/Doc/c-api/buffer.rst index 33abb5bb94..c7c1e3cc74 100644 --- a/kbe/src/lib/python/Doc/c-api/buffer.rst +++ b/kbe/src/lib/python/Doc/c-api/buffer.rst @@ -429,7 +429,7 @@ Buffer-related functions Return ``1`` if *obj* supports the buffer interface otherwise ``0``. When ``1`` is returned, it doesn't guarantee that :c:func:`PyObject_GetBuffer` will - succeed. + succeed. This function always succeeds. .. c:function:: int PyObject_GetBuffer(PyObject *exporter, Py_buffer *view, int flags) @@ -470,7 +470,7 @@ Buffer-related functions Return ``1`` if the memory defined by the *view* is C-style (*order* is ``'C'``) or Fortran-style (*order* is ``'F'``) :term:`contiguous` or either one - (*order* is ``'A'``). Return ``0`` otherwise. + (*order* is ``'A'``). Return ``0`` otherwise. This function always succeeds. .. c:function:: int PyBuffer_ToContiguous(void *buf, Py_buffer *src, Py_ssize_t len, char order) diff --git a/kbe/src/lib/python/Doc/c-api/codec.rst b/kbe/src/lib/python/Doc/c-api/codec.rst index dfe3d436e5..c55f19970e 100644 --- a/kbe/src/lib/python/Doc/c-api/codec.rst +++ b/kbe/src/lib/python/Doc/c-api/codec.rst @@ -13,7 +13,7 @@ Codec registry and support functions .. c:function:: int PyCodec_KnownEncoding(const char *encoding) Return ``1`` or ``0`` depending on whether there is a registered codec for - the given *encoding*. + the given *encoding*. This function always succeeds. .. c:function:: PyObject* PyCodec_Encode(PyObject *object, const char *encoding, const char *errors) diff --git a/kbe/src/lib/python/Doc/c-api/dict.rst b/kbe/src/lib/python/Doc/c-api/dict.rst index b7225faf40..0ced5a5fd0 100644 --- a/kbe/src/lib/python/Doc/c-api/dict.rst +++ b/kbe/src/lib/python/Doc/c-api/dict.rst @@ -95,6 +95,10 @@ Dictionary Objects Return the object from dictionary *p* which has a key *key*. Return *NULL* if the key *key* is not present, but *without* setting an exception. + Note that exceptions which occur while calling :meth:`__hash__` and + :meth:`__eq__` methods will get suppressed. + To get error reporting use :c:func:`PyDict_GetItemWithError()` instead. + .. c:function:: PyObject* PyDict_GetItemWithError(PyObject *p, PyObject *key) @@ -109,8 +113,13 @@ Dictionary Objects This is the same as :c:func:`PyDict_GetItem`, but *key* is specified as a :c:type:`const char\*`, rather than a :c:type:`PyObject\*`. + Note that exceptions which occur while calling :meth:`__hash__` and + :meth:`__eq__` methods and creating a temporary string object + will get suppressed. + To get error reporting use :c:func:`PyDict_GetItemWithError()` instead. + -.. c:function:: PyObject* PyDict_SetDefault(PyObject *p, PyObject *key, PyObject *default) +.. c:function:: PyObject* PyDict_SetDefault(PyObject *p, PyObject *key, PyObject *defaultobj) This is the same as the Python-level :meth:`dict.setdefault`. If present, it returns the value corresponding to *key* from the dictionary *p*. If the key diff --git a/kbe/src/lib/python/Doc/c-api/exceptions.rst b/kbe/src/lib/python/Doc/c-api/exceptions.rst index dd1e026cb0..cd06096ef7 100644 --- a/kbe/src/lib/python/Doc/c-api/exceptions.rst +++ b/kbe/src/lib/python/Doc/c-api/exceptions.rst @@ -53,8 +53,12 @@ Printing and clearing .. c:function:: void PyErr_PrintEx(int set_sys_last_vars) Print a standard traceback to ``sys.stderr`` and clear the error indicator. - Call this function only when the error indicator is set. (Otherwise it will - cause a fatal error!) + **Unless** the error is a ``SystemExit``. In that case the no traceback + is printed and Python process will exit with the error code specified by + the ``SystemExit`` instance. + + Call this function **only** when the error indicator is set. Otherwise it + will cause a fatal error! If *set_sys_last_vars* is nonzero, the variables :data:`sys.last_type`, :data:`sys.last_value` and :data:`sys.last_traceback` will be set to the diff --git a/kbe/src/lib/python/Doc/c-api/mapping.rst b/kbe/src/lib/python/Doc/c-api/mapping.rst index b8eaadbd70..e37dec9949 100644 --- a/kbe/src/lib/python/Doc/c-api/mapping.rst +++ b/kbe/src/lib/python/Doc/c-api/mapping.rst @@ -60,6 +60,10 @@ See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and This is equivalent to the Python expression ``key in o``. This function always succeeds. + Note that exceptions which occur while calling the :meth:`__getitem__` + method will get suppressed. + To get error reporting use :c:func:`PyObject_GetItem()` instead. + .. c:function:: int PyMapping_HasKeyString(PyObject *o, const char *key) @@ -67,6 +71,10 @@ See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and This is equivalent to the Python expression ``key in o``. This function always succeeds. + Note that exceptions which occur while calling the :meth:`__getitem__` + method and creating a temporary string object will get suppressed. + To get error reporting use :c:func:`PyMapping_GetItemString()` instead. + .. c:function:: PyObject* PyMapping_Keys(PyObject *o) diff --git a/kbe/src/lib/python/Doc/c-api/number.rst b/kbe/src/lib/python/Doc/c-api/number.rst index 3c7605a67f..296b21c132 100644 --- a/kbe/src/lib/python/Doc/c-api/number.rst +++ b/kbe/src/lib/python/Doc/c-api/number.rst @@ -280,3 +280,4 @@ Number Protocol Returns ``1`` if *o* is an index integer (has the nb_index slot of the tp_as_number structure filled in), and ``0`` otherwise. + This function always succeeds. diff --git a/kbe/src/lib/python/Doc/c-api/objbuffer.rst b/kbe/src/lib/python/Doc/c-api/objbuffer.rst index e7f4fde002..3572564b13 100644 --- a/kbe/src/lib/python/Doc/c-api/objbuffer.rst +++ b/kbe/src/lib/python/Doc/c-api/objbuffer.rst @@ -39,7 +39,11 @@ an object, and :c:func:`PyBuffer_Release` when the buffer view can be released. .. c:function:: int PyObject_CheckReadBuffer(PyObject *o) Returns ``1`` if *o* supports the single-segment readable buffer interface. - Otherwise returns ``0``. + Otherwise returns ``0``. This function always succeeds. + + Note that this function tries to get and release a buffer, and exceptions + which occur while calling corresponding functions will get suppressed. + To get error reporting use :c:func:`PyObject_GetBuffer()` instead. .. c:function:: int PyObject_AsWriteBuffer(PyObject *obj, void **buffer, Py_ssize_t *buffer_len) diff --git a/kbe/src/lib/python/Doc/c-api/object.rst b/kbe/src/lib/python/Doc/c-api/object.rst index f0b2005f2d..a64ff2e6b5 100644 --- a/kbe/src/lib/python/Doc/c-api/object.rst +++ b/kbe/src/lib/python/Doc/c-api/object.rst @@ -33,6 +33,10 @@ Object Protocol is equivalent to the Python expression ``hasattr(o, attr_name)``. This function always succeeds. + Note that exceptions which occur while calling :meth:`__getattr__` and + :meth:`__getattribute__` methods will get suppressed. + To get error reporting use :c:func:`PyObject_GetAttr()` instead. + .. c:function:: int PyObject_HasAttrString(PyObject *o, const char *attr_name) @@ -40,6 +44,11 @@ Object Protocol is equivalent to the Python expression ``hasattr(o, attr_name)``. This function always succeeds. + Note that exceptions which occur while calling :meth:`__getattr__` and + :meth:`__getattribute__` methods and creating a temporary string object + will get suppressed. + To get error reporting use :c:func:`PyObject_GetAttrString()` instead. + .. c:function:: PyObject* PyObject_GetAttr(PyObject *o, PyObject *attr_name) diff --git a/kbe/src/lib/python/Doc/c-api/tuple.rst b/kbe/src/lib/python/Doc/c-api/tuple.rst index a66832cfa4..20bf9f0b08 100644 --- a/kbe/src/lib/python/Doc/c-api/tuple.rst +++ b/kbe/src/lib/python/Doc/c-api/tuple.rst @@ -209,7 +209,7 @@ type. This function "steals" a reference to *o*. -.. c:function:: PyObject* PyStructSequence_SET_ITEM(PyObject *p, Py_ssize_t *pos, PyObject *o) +.. c:function:: void PyStructSequence_SET_ITEM(PyObject *p, Py_ssize_t *pos, PyObject *o) Macro equivalent of :c:func:`PyStructSequence_SetItem`. diff --git a/kbe/src/lib/python/Doc/c-api/unicode.rst b/kbe/src/lib/python/Doc/c-api/unicode.rst index 92e22b16a4..39c067d439 100644 --- a/kbe/src/lib/python/Doc/c-api/unicode.rst +++ b/kbe/src/lib/python/Doc/c-api/unicode.rst @@ -935,7 +935,7 @@ wchar_t Support Return *NULL* on failure. -.. c:function:: Py_ssize_t PyUnicode_AsWideChar(PyUnicodeObject *unicode, wchar_t *w, Py_ssize_t size) +.. c:function:: Py_ssize_t PyUnicode_AsWideChar(PyObject *unicode, wchar_t *w, Py_ssize_t size) Copy the Unicode object contents into the :c:type:`wchar_t` buffer *w*. At most *size* :c:type:`wchar_t` characters are copied (excluding a possibly trailing @@ -1346,7 +1346,7 @@ These are the "Raw Unicode Escape" codec APIs: .. c:function:: PyObject* PyUnicode_EncodeRawUnicodeEscape(const Py_UNICODE *s, \ - Py_ssize_t size, const char *errors) + Py_ssize_t size) Encode the :c:type:`Py_UNICODE` buffer of the given *size* using Raw-Unicode-Escape and return a bytes object. Return *NULL* if an exception was raised by the codec. @@ -1515,8 +1515,8 @@ the user settings on the machine running the codec. Return *NULL* if an exception was raised by the codec. -.. c:function:: PyObject* PyUnicode_DecodeMBCSStateful(const char *s, int size, \ - const char *errors, int *consumed) +.. c:function:: PyObject* PyUnicode_DecodeMBCSStateful(const char *s, Py_ssize_t size, \ + const char *errors, Py_ssize_t *consumed) If *consumed* is *NULL*, behave like :c:func:`PyUnicode_DecodeMBCS`. If *consumed* is not *NULL*, :c:func:`PyUnicode_DecodeMBCSStateful` will not decode diff --git a/kbe/src/lib/python/Doc/copyright.rst b/kbe/src/lib/python/Doc/copyright.rst index 540ff5ef05..393a1f0375 100644 --- a/kbe/src/lib/python/Doc/copyright.rst +++ b/kbe/src/lib/python/Doc/copyright.rst @@ -4,7 +4,7 @@ Copyright Python and this documentation is: -Copyright © 2001-2018 Python Software Foundation. All rights reserved. +Copyright © 2001-2019 Python Software Foundation. All rights reserved. Copyright © 2000 BeOpen.com. All rights reserved. diff --git a/kbe/src/lib/python/Doc/data/refcounts.dat b/kbe/src/lib/python/Doc/data/refcounts.dat index 62cc938327..35527c179f 100644 --- a/kbe/src/lib/python/Doc/data/refcounts.dat +++ b/kbe/src/lib/python/Doc/data/refcounts.dat @@ -31,29 +31,130 @@ # The parameter names are as they appear in the API manual, not the source # code. +PyAnySet_Check:int::: +PyAnySet_Check:PyObject*:p:0: + +PyAnySet_CheckExact:int::: +PyAnySet_CheckExact:PyObject*:p:0: + +PyBool_Check:int::: +PyBool_Check:PyObject*:o:0: + PyBool_FromLong:PyObject*::+1: -PyBool_FromLong:long:v:0: +PyBool_FromLong:long:v:: + +PyBuffer_FillContiguousStrides:void::: +PyBuffer_FillContiguousStrides:int:ndims:: +PyBuffer_FillContiguousStrides:Py_ssize_t*:shape:: +PyBuffer_FillContiguousStrides:Py_ssize_t*:strides:: +PyBuffer_FillContiguousStrides:int:itemsize:: +PyBuffer_FillContiguousStrides:char:order:: + +PyBuffer_FillInfo:int::: +PyBuffer_FillInfo:Py_buffer*:view:: +PyBuffer_FillInfo:PyObject*:exporter:0: +PyBuffer_FillInfo:void*:buf:: +PyBuffer_FillInfo:Py_ssize_t:len:: +PyBuffer_FillInfo:int:readonly:: +PyBuffer_FillInfo:int:flags:: + +PyBuffer_IsContiguous:int::: +PyBuffer_IsContiguous:Py_buffer*:view:: +PyBuffer_IsContiguous:char:order:: + +PyBuffer_Release:void::: +PyBuffer_Release:Py_buffer*:view:: + +PyBuffer_ToContiguous:int::: +PyBuffer_ToContiguous:void*:buf:: +PyBuffer_ToContiguous:Py_buffer*:src:: +PyBuffer_ToContiguous:Py_ssize_t:len:: +PyBuffer_ToContiguous:char:order:: + +PyByteArray_AS_STRING:char*::: +PyByteArray_AS_STRING:PyObject*:bytearray:0: + +PyByteArray_AsString:char*::: +PyByteArray_AsString:PyObject*:bytearray:0: + +PyByteArray_Check:int::: +PyByteArray_Check:PyObject*:o:0: + +PyByteArray_CheckExact:int::: +PyByteArray_CheckExact:PyObject*:o:0: + +PyByteArray_Concat:PyObject*::+1: +PyByteArray_Concat:PyObject*:a:0: +PyByteArray_Concat:PyObject*:b:0: + +PyByteArray_FromObject:PyObject*::+1: +PyByteArray_FromObject:PyObject*:o:0: + +PyByteArray_FromStringAndSize:PyObject*::+1: +PyByteArray_FromStringAndSize:const char*:string:: +PyByteArray_FromStringAndSize:Py_ssize_t:len:: + +PyByteArray_GET_SIZE:Py_ssize_t::: +PyByteArray_GET_SIZE:PyObject*:bytearray:0: + +PyByteArray_Resize:int::: +PyByteArray_Resize:PyObject*:bytearray:0: +PyByteArray_Resize:Py_ssize_t:len:: + +PyByteArray_Size:Py_ssize_t::: +PyByteArray_Size:PyObject*:bytearray:0: + +PyBytes_AS_STRING:char*::: +PyBytes_AS_STRING:PyObject*:string:0: -PyBuffer_FromObject:PyObject*::+1: -PyBuffer_FromObject:PyObject*:base:+1: -PyBuffer_FromObject:int:offset:: -PyBuffer_FromObject:int:size:: +PyBytes_AsString:char*::: +PyBytes_AsString:PyObject*:o:0: -PyBuffer_FromReadWriteObject:PyObject*::+1: -PyBuffer_FromReadWriteObject:PyObject*:base:+1: -PyBuffer_FromReadWriteObject:int:offset:: -PyBuffer_FromReadWriteObject:int:size:: +PyBytes_AsStringAndSize:int::: +PyBytes_AsStringAndSize:PyObject*:obj:0: +PyBytes_AsStringAndSize:char**:buffer:: +PyBytes_AsStringAndSize:Py_ssize_t*:length:: -PyBuffer_FromMemory:PyObject*::+1: -PyBuffer_FromMemory:void*:ptr:: -PyBuffer_FromMemory:int:size:: +PyBytes_Check:int::: +PyBytes_Check:PyObject*:o:0: -PyBuffer_FromReadWriteMemory:PyObject*::+1: -PyBuffer_FromReadWriteMemory:void*:ptr:: -PyBuffer_FromReadWriteMemory:int:size:: +PyBytes_CheckExact:int::: +PyBytes_CheckExact:PyObject*:o:0: -PyBuffer_New:PyObject*::+1: -PyBuffer_New:int:size:: +PyBytes_Concat:void::: +PyBytes_Concat:PyObject**:bytes:0: +PyBytes_Concat:PyObject*:newpart:0: + +PyBytes_ConcatAndDel:void::: +PyBytes_ConcatAndDel:PyObject**:bytes:0: +PyBytes_ConcatAndDel:PyObject*:newpart:-1: + +PyBytes_FromString:PyObject*::+1: +PyBytes_FromString:const char*:v:: + +PyBytes_FromStringAndSize:PyObject*::+1: +PyBytes_FromStringAndSize:const char*:v:: +PyBytes_FromStringAndSize:Py_ssize_t:len:: + +PyBytes_FromFormat:PyObject*::+1: +PyBytes_FromFormat:const char*:format:: +PyBytes_FromFormat::...:: + +PyBytes_FromFormatV:PyObject*::+1: +PyBytes_FromFormatV:const char*:format:: +PyBytes_FromFormatV:va_list:vargs:: + +PyBytes_FromObject:PyObject*::+1: +PyBytes_FromObject:PyObject*:o:0: + +PyBytes_GET_SIZE:Py_ssize_t::: +PyBytes_GET_SIZE:PyObject*:o:0: + +PyBytes_Size:Py_ssize_t::: +PyBytes_Size:PyObject*:o:0: + +PyCapsule_CheckExact:int::: +PyCapsule_CheckExact:PyObject*:p:0: PyCapsule_GetContext:void *::: PyCapsule_GetContext:PyObject*:self:0: @@ -72,6 +173,10 @@ PyCapsule_Import:void *::: PyCapsule_Import:const char *:name:: PyCapsule_Import:int:no_block:: +PyCapsule_IsValid:int::: +PyCapsule_IsValid:PyObject*:capsule:0: +PyCapsule_IsValid:const char*:name:: + PyCapsule_New:PyObject*::+1: PyCapsule_New:void*:pointer:: PyCapsule_New:const char *:name:: @@ -93,21 +198,8 @@ PyCapsule_SetPointer:int::: PyCapsule_SetPointer:PyObject*:self:0: PyCapsule_SetPointer:void*:pointer:: - -PyCObject_AsVoidPtr:void*::: -PyCObject_AsVoidPtr:PyObject*:self:0: - -PyCObject_FromVoidPtr:PyObject*::+1: -PyCObject_FromVoidPtr:void*:cobj:: -PyCObject_FromVoidPtr::void (* destr)(void* ):: - -PyCObject_FromVoidPtrAndDesc:PyObject*::+1: -PyCObject_FromVoidPtrAndDesc:void*:cobj:: -PyCObject_FromVoidPtrAndDesc:void*:desc:: -PyCObject_FromVoidPtrAndDesc:void(*)(void*,void*):destr:: - -PyCObject_GetDesc:void*::: -PyCObject_GetDesc:PyObject*:self:0: +PyCell_Check:int::: +PyCell_Check::ob:: PyCell_New:PyObject*::+1: PyCell_New:PyObject*:ob:0: @@ -126,19 +218,118 @@ PyCell_Set:int::: PyCell_Set:PyObject*:cell:0: PyCell_Set:PyObject*:value:0: +PyCallIter_Check:int::: +PyCallIter_Check::op:: + PyCallIter_New:PyObject*::+1: -PyCallIter_New:PyObject*:callable:: -PyCallIter_New:PyObject*:sentinel:: +PyCallIter_New:PyObject*:callable:+1: +PyCallIter_New:PyObject*:sentinel:+1: PyCallable_Check:int::: PyCallable_Check:PyObject*:o:0: +PyCode_Check:int::: +PyCode_Check:PyObject*:co:0: + +PyCode_GetNumFree:int::: +PyCode_GetNumFree:PyCodeObject*:co:0: + +PyCode_New:PyCodeObject*::+1: +PyCode_New:int:argcount:: +PyCode_New:int:kwonlyargcount:: +PyCode_New:int:nlocals:: +PyCode_New:int:stacksize:: +PyCode_New:int:flags:: +PyCode_New:PyObject*:code:0: +PyCode_New:PyObject*:consts:0: +PyCode_New:PyObject*:names:0: +PyCode_New:PyObject*:varnames:0: +PyCode_New:PyObject*:freevars:0: +PyCode_New:PyObject*:cellvars:0: +PyCode_New:PyObject*:filename:0: +PyCode_New:PyObject*:name:0: +PyCode_New:int:firstlineno:: +PyCode_New:PyObject*:lnotab:0: + +PyCode_NewEmpty:PyCodeObject*::+1: +PyCode_NewEmpty:const char*:filename:: +PyCode_NewEmpty:const char*:funcname:: +PyCode_NewEmpty:int:firstlineno:: + +PyCodec_Register:int::: +PyCodec_Register:PyObject*:search_function:+1: + +PyCodec_KnownEncoding:int::: +PyCodec_KnownEncoding:const char*:encoding:: + +PyCodec_Encode:PyObject*::+1: +PyCodec_Encode:PyObject*:object:0: +PyCodec_Encode:const char*:encoding:: +PyCodec_Encode:const char*:errors:: + +PyCodec_Decode:PyObject*::+1: +PyCodec_Decode:PyObject*:object:0: +PyCodec_Decode:const char*:encoding:: +PyCodec_Decode:const char*:errors:: + +PyCodec_Encoder:PyObject*::+1: +PyCodec_Encoder:const char*:encoding:: + +PyCodec_Decoder:PyObject*::+1: +PyCodec_Decoder:const char*:encoding:: + +PyCodec_IncrementalEncoder:PyObject*::+1: +PyCodec_IncrementalEncoder:const char*:encoding:: +PyCodec_IncrementalEncoder:const char*:errors:: + +PyCodec_IncrementalDecoder:PyObject*::+1: +PyCodec_IncrementalDecoder:const char*:encoding:: +PyCodec_IncrementalDecoder:const char*:errors:: + +PyCodec_StreamReader:PyObject*::+1: +PyCodec_StreamReader:const char*:encoding:: +PyCodec_StreamReader:PyObject*:stream:0: +PyCodec_StreamReader:const char*:errors:: + +PyCodec_StreamWriter:PyObject*::+1: +PyCodec_StreamWriter:const char*:encoding:: +PyCodec_StreamWriter:PyObject*:stream:0: +PyCodec_StreamWriter:const char*:errors:: + +PyCodec_RegisterError:int::: +PyCodec_RegisterError:const char*:name:: +PyCodec_RegisterError:PyObject*:error:+1: + +PyCodec_LookupError:PyObject*::+1: +PyCodec_LookupError:const char*:name:: + +PyCodec_StrictErrors:PyObject*::null: +PyCodec_StrictErrors:PyObject*:exc:0: + +PyCodec_IgnoreErrors:PyObject*::+1: +PyCodec_IgnoreErrors:PyObject*:exc:0: + +PyCodec_ReplaceErrors:PyObject*::+1: +PyCodec_ReplaceErrors:PyObject*:exc:0: + +PyCodec_XMLCharRefReplaceErrors:PyObject*::+1: +PyCodec_XMLCharRefReplaceErrors:PyObject*:exc:0: + +PyCodec_BackslashReplaceErrors:PyObject*::+1: +PyCodec_BackslashReplaceErrors:PyObject*:exc:0: + +PyCodec_NameReplaceErrors:PyObject*::+1: +PyCodec_NameReplaceErrors:PyObject*:exc:0: + PyComplex_AsCComplex:Py_complex::: PyComplex_AsCComplex:PyObject*:op:0: PyComplex_Check:int::: PyComplex_Check:PyObject*:p:0: +PyComplex_CheckExact:int::: +PyComplex_CheckExact:PyObject*:p:0: + PyComplex_FromCComplex:PyObject*::+1: PyComplex_FromCComplex::Py_complex v:: @@ -193,6 +384,12 @@ PyContextVar_Reset:int::: PyContextVar_Reset:PyObject*:var:0: PyContextVar_Reset:PyObject*:token:-1: +PyDate_Check:int::: +PyDate_Check:PyObject*:ob:0: + +PyDate_CheckExact:int::: +PyDate_CheckExact:PyObject*:ob:0: + PyDate_FromDate:PyObject*::+1: PyDate_FromDate:int:year:: PyDate_FromDate:int:month:: @@ -201,6 +398,12 @@ PyDate_FromDate:int:day:: PyDate_FromTimestamp:PyObject*::+1: PyDate_FromTimestamp:PyObject*:args:0: +PyDateTime_Check:int::: +PyDateTime_Check:PyObject*:ob:0: + +PyDateTime_CheckExact:int::: +PyDateTime_CheckExact:PyObject*:ob:0: + PyDateTime_FromDateAndTime:PyObject*::+1: PyDateTime_FromDateAndTime:int:year:: PyDateTime_FromDateAndTime:int:month:: @@ -213,46 +416,62 @@ PyDateTime_FromDateAndTime:int:usecond:: PyDateTime_FromTimestamp:PyObject*::+1: PyDateTime_FromTimestamp:PyObject*:args:0: +PyDelta_Check:int::: +PyDelta_Check:PyObject*:ob:0: + +PyDelta_CheckExact:int::: +PyDelta_CheckExact:PyObject*:ob:0: + PyDelta_FromDSU:PyObject*::+1: PyDelta_FromDSU:int:days:: PyDelta_FromDSU:int:seconds:: PyDelta_FromDSU:int:useconds:: PyTimeZone_FromOffset:PyObject*::+1: -PyTimeZone_FromOffset:PyDateTime_DeltaType*:offset:+1:Reference count not increased if offset is +00:00 +PyTimeZone_FromOffset:PyObject*:offset:+1:Reference count not increased if offset is +00:00 PyTimeZone_FromOffsetAndName:PyObject*::+1: -PyTimeZone_FromOffsetAndName:PyDateTime_DeltaType*:offset:+1:Reference count not increased if offset is +00:00 and name == NULL -PyTimeZone_FromOffsetAndName:PyUnicode*:name:+1: +PyTimeZone_FromOffsetAndName:PyObject*:offset:+1:Reference count not increased if offset is +00:00 and name == NULL +PyTimeZone_FromOffsetAndName:PyObject*:name:+1: +PyDescr_IsData:int::: +PyDescr_IsData:PyObject*:descr:0: + PyDescr_NewClassMethod:PyObject*::+1: -PyDescr_NewClassMethod:PyTypeObject*:type:: +PyDescr_NewClassMethod:PyTypeObject*:type:+1: PyDescr_NewClassMethod:PyMethodDef*:method:: PyDescr_NewGetSet:PyObject*::+1: -PyDescr_NewGetSet:PyTypeObject*:type:: +PyDescr_NewGetSet:PyTypeObject*:type:+1: PyDescr_NewGetSet:PyGetSetDef*:getset:: PyDescr_NewMember:PyObject*::+1: -PyDescr_NewMember:PyTypeObject*:type:: +PyDescr_NewMember:PyTypeObject*:type:+1: PyDescr_NewMember:PyMemberDef*:member:: PyDescr_NewMethod:PyObject*::+1: -PyDescr_NewMethod:PyTypeObject*:type:: +PyDescr_NewMethod:PyTypeObject*:type:+1: PyDescr_NewMethod:PyMethodDef*:meth:: PyDescr_NewWrapper:PyObject*::+1: -PyDescr_NewWrapper:PyTypeObject*:type:: +PyDescr_NewWrapper:PyTypeObject*:type:+1: PyDescr_NewWrapper:struct wrapperbase*:base:: PyDescr_NewWrapper:void*:wrapped:: PyDict_Check:int::: PyDict_Check:PyObject*:p:0: +PyDict_CheckExact:int::: +PyDict_CheckExact:PyObject*:p:0: + PyDict_Clear:void::: PyDict_Clear:PyObject*:p:0: +PyDict_Contains:int::: +PyDict_Contains:PyObject*:p:0: +PyDict_Contains:PyObject*:key:0: + PyDict_DelItem:int::: PyDict_DelItem:PyObject*:p:0: PyDict_DelItem:PyObject*:key:0: @@ -261,7 +480,7 @@ PyDict_DelItemString:int::: PyDict_DelItemString:PyObject*:p:0: PyDict_DelItemString:const char*:key:: -PyDict_GetItem:PyObject*::0:0 +PyDict_GetItem:PyObject*::0: PyDict_GetItem:PyObject*:p:0: PyDict_GetItem:PyObject*:key:0: @@ -289,9 +508,19 @@ PyDict_New:PyObject*::+1: PyDict_Copy:PyObject*::+1: PyDict_Copy:PyObject*:p:0: +PyDict_Merge:int::: +PyDict_Merge:PyObject*:a:0: +PyDict_Merge:PyObject*:b:0: +PyDict_Merge:int:override:: + +PyDict_MergeFromSeq2:int::: +PyDict_MergeFromSeq2:PyObject*:a:0: +PyDict_MergeFromSeq2:PyObject*:seq2:0: +PyDict_MergeFromSeq2:int:override:: + PyDict_Next:int::: PyDict_Next:PyObject*:p:0: -PyDict_Next:int:ppos:: +PyDict_Next:Py_ssize_t:ppos:: PyDict_Next:PyObject**:pkey:0: PyDict_Next:PyObject**:pvalue:0: @@ -305,8 +534,12 @@ PyDict_SetItemString:PyObject*:p:0: PyDict_SetItemString:const char*:key:: PyDict_SetItemString:PyObject*:val:+1: -PyDict_Size:int::: -PyDict_Size:PyObject*:p:: +PyDict_Size:Py_ssize_t::: +PyDict_Size:PyObject*:p:0: + +PyDict_Update:int::: +PyDict_Update:PyObject*:a:0: +PyDict_Update:PyObject*:b:0: PyDict_Values:PyObject*::+1: PyDict_Values:PyObject*:p:0: @@ -330,6 +563,21 @@ PyErr_Fetch:PyObject**:ptype:0: PyErr_Fetch:PyObject**:pvalue:0: PyErr_Fetch:PyObject**:ptraceback:0: +PyErr_Format:PyObject*::null: +PyErr_Format:PyObject*:exception:+1: +PyErr_Format:const char*:format:: +PyErr_Format::...:: + +PyErr_FormatV:PyObject*::null: +PyErr_FormatV:PyObject*:exception:+1: +PyErr_FormatV:const char*:format:: +PyErr_FormatV:va_list:vargs:: + +PyErr_GetExcInfo:void::: +PyErr_GetExcInfo:PyObject**:ptype:+1: +PyErr_GetExcInfo:PyObject**:pvalue:+1: +PyErr_GetExcInfo:PyObject**:ptraceback:+1: + PyErr_GivenExceptionMatches:int::: PyErr_GivenExceptionMatches:PyObject*:given:0: PyErr_GivenExceptionMatches:PyObject*:exc:0: @@ -356,27 +604,61 @@ PyErr_Occurred:PyObject*::0: PyErr_Print:void::: +PyErr_PrintEx:void::: +PyErr_PrintEx:int:set_sys_last_vars:: + +PyErr_ResourceWarning:int::: +PyErr_ResourceWarning:PyObject*:source:0: +PyErr_ResourceWarning:Py_ssize_t:stack_level:: +PyErr_ResourceWarning:const char*:format:: +PyErr_ResourceWarning::...:: + PyErr_Restore:void::: PyErr_Restore:PyObject*:type:-1: PyErr_Restore:PyObject*:value:-1: PyErr_Restore:PyObject*:traceback:-1: PyErr_SetExcFromWindowsErr:PyObject*::null: -PyErr_SetExcFromWindowsErr:PyObject*:type:0: +PyErr_SetExcFromWindowsErr:PyObject*:type:+1: PyErr_SetExcFromWindowsErr:int:ierr:: PyErr_SetExcFromWindowsErrWithFilename:PyObject*::null: -PyErr_SetExcFromWindowsErrWithFilename:PyObject*:type:0: +PyErr_SetExcFromWindowsErrWithFilename:PyObject*:type:+1: PyErr_SetExcFromWindowsErrWithFilename:int:ierr:: PyErr_SetExcFromWindowsErrWithFilename:const char*:filename:: +PyErr_SetExcFromWindowsErrWithFilenameObject:PyObject*::null: +PyErr_SetExcFromWindowsErrWithFilenameObject:PyObject*:type:+1: +PyErr_SetExcFromWindowsErrWithFilenameObject:int:ierr:: +PyErr_SetExcFromWindowsErrWithFilenameObject:PyObject*:filename:+1: + +PyErr_SetExcFromWindowsErrWithFilenameObjects:PyObject*::null: +PyErr_SetExcFromWindowsErrWithFilenameObjects:PyObject*:type:+1: +PyErr_SetExcFromWindowsErrWithFilenameObjects:int:ierr:: +PyErr_SetExcFromWindowsErrWithFilenameObjects:PyObject*:filename:+1: +PyErr_SetExcFromWindowsErrWithFilenameObjects:PyObject*:filename2:+1: + +PyErr_SetExcInfo:void::: +PyErr_SetExcInfo:PyObject*:type:0: +PyErr_SetExcInfo:PyObject*:value:0: +PyErr_SetExcInfo:PyObject*:traceback:0: + PyErr_SetFromErrno:PyObject*::null: -PyErr_SetFromErrno:PyObject*:type:0: +PyErr_SetFromErrno:PyObject*:type:+1: PyErr_SetFromErrnoWithFilename:PyObject*::null: -PyErr_SetFromErrnoWithFilename:PyObject*:type:0: +PyErr_SetFromErrnoWithFilename:PyObject*:type:+1: PyErr_SetFromErrnoWithFilename:const char*:filename:: +PyErr_SetFromErrnoWithFilenameObject:PyObject*::null: +PyErr_SetFromErrnoWithFilenameObject:PyObject*:type:+1: +PyErr_SetFromErrnoWithFilenameObject:PyObject*:filenameObject:+1: + +PyErr_SetFromErrnoWithFilenameObjects:PyObject*::null: +PyErr_SetFromErrnoWithFilenameObjects:PyObject*:type:+1: +PyErr_SetFromErrnoWithFilenameObjects:PyObject*:filenameObject:+1: +PyErr_SetFromErrnoWithFilenameObjects:PyObject*:filenameObject2:+1: + PyErr_SetFromWindowsErr:PyObject*::null: PyErr_SetFromWindowsErr:int:ierr:: @@ -384,6 +666,16 @@ PyErr_SetFromWindowsErrWithFilename:PyObject*::null: PyErr_SetFromWindowsErrWithFilename:int:ierr:: PyErr_SetFromWindowsErrWithFilename:const char*:filename:: +PyErr_SetImportError:PyObject*::null: +PyErr_SetImportError:PyObject*:msg:+1: +PyErr_SetImportError:PyObject*:name:+1: +PyErr_SetImportError:PyObject*:path:+1: + +PyErr_SetImportErrorSubclass:PyObject*::null: +PyErr_SetImportErrorSubclass:PyObject*:msg:+1: +PyErr_SetImportErrorSubclass:PyObject*:name:+1: +PyErr_SetImportErrorSubclass:PyObject*:path:+1: + PyErr_SetInterrupt:void::: PyErr_SetNone:void::: @@ -397,31 +689,69 @@ PyErr_SetString:void::: PyErr_SetString:PyObject*:type:+1: PyErr_SetString:const char*:message:: -PyErr_Format:PyObject*::null: -PyErr_Format:PyObject*:exception:+1: -PyErr_Format:const char*:format:: -PyErr_Format::...:: +PyErr_SyntaxLocation:void::: +PyErr_SyntaxLocation:const char*:filename:: +PyErr_SyntaxLocation:int:lineno:: -PyErr_FormatV:PyObject*::null: -PyErr_FormatV:PyObject*:exception:+1: -PyErr_FormatV:const char*:format:: -PyErr_FormatV:va_list:vargs:: +PyErr_SyntaxLocationEx:void::: +PyErr_SyntaxLocationEx:const char*:filename:: +PyErr_SyntaxLocationEx:int:lineno:: +PyErr_SyntaxLocationEx:int:col_offset:: + +PyErr_SyntaxLocationObject:void::: +PyErr_SyntaxLocationObject:PyObject*:filename:+1: +PyErr_SyntaxLocationObject:int:lineno:: +PyErr_SyntaxLocationObject:int:col_offset:: PyErr_WarnEx:int::: PyErr_WarnEx:PyObject*:category:0: PyErr_WarnEx:const char*:message:: PyErr_WarnEx:Py_ssize_t:stack_level:: +PyErr_WarnExplicit:int::: +PyErr_WarnExplicit:PyObject*:category:0: +PyErr_WarnExplicit:const char*:message:: +PyErr_WarnExplicit:const char*:filename:: +PyErr_WarnExplicit:int:lineno:: +PyErr_WarnExplicit:const char*:module:: +PyErr_WarnExplicit:PyObject*:registry:0: + +PyErr_WarnExplicitObject:int::: +PyErr_WarnExplicitObject:PyObject*:category:0: +PyErr_WarnExplicitObject:PyObject*:message:0: +PyErr_WarnExplicitObject:PyObject*:filename:0: +PyErr_WarnExplicitObject:int:lineno:: +PyErr_WarnExplicitObject:PyObject*:module:0: +PyErr_WarnExplicitObject:PyObject*:registry:0: + +PyErr_WarnFormat:int::: +PyErr_WarnFormat:PyObject*:category:0: +PyErr_WarnFormat:Py_ssize_t:stack_level:: +PyErr_WarnFormat:const char*:format:: +PyErr_WarnFormat::...:: + +PyErr_WriteUnraisable:void::: +PyErr_WriteUnraisable:PyObject*:obj:0: + PyEval_AcquireLock:void::: PyEval_AcquireThread:void::: PyEval_AcquireThread:PyThreadState*:tstate:: PyEval_GetBuiltins:PyObject*::0: + PyEval_GetLocals:PyObject*::0: + PyEval_GetGlobals:PyObject*::0: + PyEval_GetFrame:PyObject*::0: +PyEval_GetFuncDesc:const char*::: +PyEval_GetFuncDesc:PyObject*:func:0: + +PyEval_GetFuncName:const char*::: +PyEval_GetFuncName:PyObject*:func:0: + PyEval_InitThreads:void::: PyEval_ReleaseLock:void::: @@ -434,61 +764,85 @@ PyEval_RestoreThread:PyThreadState*:tstate:: PyEval_SaveThread:PyThreadState*::: +PyEval_SetProfile:void::: +PyEval_SetProfile:Py_tracefunc:func:: +PyEval_SetProfile:PyObject*:obj:+1: + +PyEval_SetTrace:void::: +PyEval_SetTrace:Py_tracefunc:func:: +PyEval_SetTrace:PyObject*:obj:+1: + PyEval_EvalCode:PyObject*::+1: -PyEval_EvalCode:PyCodeObject*:co:0: +PyEval_EvalCode:PyObject*:co:0: PyEval_EvalCode:PyObject*:globals:0: PyEval_EvalCode:PyObject*:locals:0: -PyException_GetTraceback:PyObject*::+1: +PyEval_EvalCodeEx:PyObject*::+1: +PyEval_EvalCodeEx:PyObject*:co:0: +PyEval_EvalCodeEx:PyObject*:globals:0: +PyEval_EvalCodeEx:PyObject*:locals:0: +PyEval_EvalCodeEx:PyObject*const*:args:: +PyEval_EvalCodeEx:int:argcount:: +PyEval_EvalCodeEx:PyObject*const*:kws:: +PyEval_EvalCodeEx:int:kwcount:: +PyEval_EvalCodeEx:PyObject*const*:defs:: +PyEval_EvalCodeEx:int:defcount:: +PyEval_EvalCodeEx:PyObject*:kwdefs:0: +PyEval_EvalCodeEx:PyObject*:closure:0: -PyFile_AsFile:FILE*::: -PyFile_AsFile:PyFileObject*:p:0: +PyEval_EvalFrame:PyObject*::+1: +PyEval_EvalFrame:PyFrameObject*:f:0: -PyFile_Check:int::: -PyFile_Check:PyObject*:p:0: +PyEval_EvalFrameEx:PyObject*::+1: +PyEval_EvalFrameEx:PyFrameObject*:f:0: +PyEval_EvalFrameEx:int:throwflag:: -PyFile_FromFile:PyObject*::+1: -PyFile_FromFile:FILE*:fp:: -PyFile_FromFile:const char*:name:: -PyFile_FromFile:const char*:mode:: -PyFile_FromFile:int(*:close):: +PyEval_MergeCompilerFlags:int::: +PyEval_MergeCompilerFlags:PyCompilerFlags*:cf:: -PyFile_FromFileEx:PyObject*::+1: -PyFile_FromFileEx:FILE*:fp:: -PyFile_FromFileEx:const char*:name:: -PyFile_FromFileEx:const char*:mode:: -PyFile_FromFileEx:int(*:close):: -PyFile_FromFileEx:int:buffering:: -PyFile_FromFileEx:const char*:encoding:: -PyFile_FromFileEx:const char*:newline:: +PyException_GetCause:PyObject*::+1: +PyException_GetCause:PyObject*:ex:0: -PyFile_FromString:PyObject*::+1: -PyFile_FromString:const char*:name:: -PyFile_FromString:const char*:mode:: +PyException_GetContext:PyObject*::+1: +PyException_GetContext:PyObject*:ex:0: + +PyException_GetTraceback:PyObject*::+1: +PyException_GetTraceback:PyObject*:ex:0: + +PyException_SetCause:void::: +PyException_SetCause:PyObject*:ex:0: +PyException_SetCause:PyObject*:cause:+1: + +PyException_SetContext:void::: +PyException_SetContext:PyObject*:ex:0: +PyException_SetContext:PyObject*:ctx:+1: + +PyException_SetTraceback:int::: +PyException_SetTraceback:PyObject*:ex:0: +PyException_SetTraceback:PyObject*:tb:+1: + +PyFile_FromFd:PyObject*::+1: +PyFile_FromFd:int:fd:: +PyFile_FromFd:const char*:name:: +PyFile_FromFd:const char*:mode:: +PyFile_FromFd:int:buffering:: +PyFile_FromFd:const char*:encoding:: +PyFile_FromFd:const char*:errors:: +PyFile_FromFd:const char*:newline:: +PyFile_FromFd:int:closefd:: PyFile_GetLine:PyObject*::+1: -PyFile_GetLine:PyObject*:p:: +PyFile_GetLine:PyObject*:p:0: PyFile_GetLine:int:n:: -PyFile_Name:PyObject*::0: -PyFile_Name:PyObject*:p:0: - -PyFile_SetBufSize:void::: -PyFile_SetBufSize:PyFileObject*:p:0: -PyFile_SetBufSize:int:n:: - -PyFile_SoftSpace:int::: -PyFile_SoftSpace:PyFileObject*:p:0: -PyFile_SoftSpace:int:newflag:: - PyFile_WriteObject:int::: PyFile_WriteObject:PyObject*:obj:0: -PyFile_WriteObject:PyFileObject*:p:0: +PyFile_WriteObject:PyObject*:p:0: PyFile_WriteObject:int:flags:: PyFile_WriteString:int::: PyFile_WriteString:const char*:s:: -PyFile_WriteString:PyFileObject*:p:0: +PyFile_WriteString:PyObject*:p:0: PyFile_WriteString:int:flags:: PyFloat_AS_DOUBLE:double::: @@ -500,15 +854,33 @@ PyFloat_AsDouble:PyObject*:pyfloat:0: PyFloat_Check:int::: PyFloat_Check:PyObject*:p:0: +PyFloat_CheckExact:int::: +PyFloat_CheckExact:PyObject*:p:0: + PyFloat_FromDouble:PyObject*::+1: PyFloat_FromDouble:double:v:: PyFloat_FromString:PyObject*::+1: PyFloat_FromString:PyObject*:str:0: +PyFloat_GetInfo:PyObject*::+1: +PyFloat_GetInfo::void:: + +PyFrozenSet_Check:int::: +PyFrozenSet_Check:PyObject*:p:0: + +PyFrozenSet_CheckExact:int::: +PyFrozenSet_CheckExact:PyObject*:p:0: + PyFrozenSet_New:PyObject*::+1: PyFrozenSet_New:PyObject*:iterable:0: +PyFunction_Check:int::: +PyFunction_Check:PyObject*:o:0: + +PyFunction_GetAnnotations:PyObject*::0: +PyFunction_GetAnnotations:PyObject*:op:0: + PyFunction_GetClosure:PyObject*::0: PyFunction_GetClosure:PyObject*:op:0: @@ -533,6 +905,10 @@ PyFunction_NewWithQualName:PyObject*:code:+1: PyFunction_NewWithQualName:PyObject*:globals:+1: PyFunction_NewWithQualName:PyObject*:qualname:+1: +PyFunction_SetAnnotations:int::: +PyFunction_SetAnnotations:PyObject*:op:0: +PyFunction_SetAnnotations:PyObject*:annotations:+1: + PyFunction_SetClosure:int::: PyFunction_SetClosure:PyObject*:op:0: PyFunction_SetClosure:PyObject*:closure:+1: @@ -541,34 +917,34 @@ PyFunction_SetDefaults:int::: PyFunction_SetDefaults:PyObject*:op:0: PyFunction_SetDefaults:PyObject*:defaults:+1: +PyGen_Check:int::: +PyGen_Check:PyObject*:ob:0: + +PyGen_CheckExact:int::: +PyGen_CheckExact:PyObject*:ob:0: + PyGen_New:PyObject*::+1: PyGen_New:PyFrameObject*:frame:0: PyGen_NewWithQualName:PyObject*::+1: PyGen_NewWithQualName:PyFrameObject*:frame:0: +PyGen_NewWithQualName:PyObject*:name:0: +PyGen_NewWithQualName:PyObject*:qualname:0: + +PyCoro_CheckExact:int::: +PyCoro_CheckExact:PyObject*:ob:0: PyCoro_New:PyObject*::+1: PyCoro_New:PyFrameObject*:frame:0: - -Py_InitModule:PyObject*::0: -Py_InitModule:const char*:name:: -Py_InitModule:PyMethodDef[]:methods:: - -Py_InitModule3:PyObject*::0: -Py_InitModule3:const char*:name:: -Py_InitModule3:PyMethodDef[]:methods:: -Py_InitModule3:const char*:doc:: - -Py_InitModule4:PyObject*::0: -Py_InitModule4:const char*:name:: -Py_InitModule4:PyMethodDef[]:methods:: -Py_InitModule4:const char*:doc:: -Py_InitModule4:PyObject*:self:: -Py_InitModule4:int:apiver::usually provided by Py_InitModule or Py_InitModule3 +PyCoro_New:PyObject*:name:0: +PyCoro_New:PyObject*:qualname:0: PyImport_AddModule:PyObject*::0:reference borrowed from sys.modules PyImport_AddModule:const char*:name:: +PyImport_AddModuleObject:PyObject*::0:reference borrowed from sys.modules +PyImport_AddModuleObject:PyObject*:name:0: + PyImport_Cleanup:void::: PyImport_ExecCodeModule:PyObject*::+1: @@ -580,6 +956,21 @@ PyImport_ExecCodeModuleEx:const char*:name:: PyImport_ExecCodeModuleEx:PyObject*:co:0: PyImport_ExecCodeModuleEx:const char*:pathname:: +PyImport_ExecCodeModuleObject:PyObject*::+1: +PyImport_ExecCodeModuleObject:const char*:name:: +PyImport_ExecCodeModuleObject:PyObject*:co:0: +PyImport_ExecCodeModuleObject:PyObject*:pathname:0: +PyImport_ExecCodeModuleObject:PyObject*:cpathname:0: + +PyImport_ExecCodeModuleWithPathnames:PyObject*::+1: +PyImport_ExecCodeModuleWithPathnames:const char*:name:: +PyImport_ExecCodeModuleWithPathnames:PyObject*:co:0: +PyImport_ExecCodeModuleWithPathnames:const char*:pathname:: +PyImport_ExecCodeModuleWithPathnames:const char*:cpathname:: + +PyImport_GetImporter:PyObject*::+1: +PyImport_GetImporter:PyObject*:path:0: + PyImport_GetMagicNumber:long::: PyImport_GetModule:PyObject*::+1: @@ -593,6 +984,9 @@ PyImport_Import:PyObject*:name:0: PyImport_ImportFrozenModule:int::: PyImport_ImportFrozenModule:const char*::: +PyImport_ImportFrozenModuleObject:int::: +PyImport_ImportFrozenModuleObject:PyObject*::+1: + PyImport_ImportModule:PyObject*::+1: PyImport_ImportModule:const char*:name:: @@ -609,39 +1003,33 @@ PyImport_ImportModuleLevel:PyObject*:locals:0:??? PyImport_ImportModuleLevel:PyObject*:fromlist:0:??? PyImport_ImportModuleLevel:int:level:: -PyImport_ReloadModule:PyObject*::+1: -PyImport_ReloadModule:PyObject*:m:0: - -PyInstance_New:PyObject*::+1: -PyInstance_New:PyObject*:klass:+1: -PyInstance_New:PyObject*:arg:0: -PyInstance_New:PyObject*:kw:0: - -PyInstance_NewRaw:PyObject*::+1: -PyInstance_NewRaw:PyObject*:klass:+1: -PyInstance_NewRaw:PyObject*:dict:+1: +PyImport_ImportModuleLevelObject:PyObject*::+1: +PyImport_ImportModuleLevelObject:PyObject*:name:0: +PyImport_ImportModuleLevelObject:PyObject*:globals:0:??? +PyImport_ImportModuleLevelObject:PyObject*:locals:0:??? +PyImport_ImportModuleLevelObject:PyObject*:fromlist:0:??? +PyImport_ImportModuleLevelObject:int:level:: -PyInt_AS_LONG:long::: -PyInt_AS_LONG:PyIntObject*:io:0: +PyImport_ImportModuleNoBlock:PyObject*::+1: +PyImport_ImportModuleNoBlock:const char*:name:: -PyInt_AsLong:long::: -PyInt_AsLong:PyObject*:io:0: +PyImport_ReloadModule:PyObject*::+1: +PyImport_ReloadModule:PyObject*:m:0: -PyInt_Check:int::: -PyInt_Check:PyObject*:op:0: +PyIndex_Check:int::: +PyIndex_Check:PyObject*:o:0: -PyInt_FromLong:PyObject*::+1: -PyInt_FromLong:long:ival:: +PyInstanceMethod_Check:int::: +PyInstanceMethod_Check:PyObject*:o:0: -PyInt_FromString:PyObject*::+1: -PyInt_FromString:char*:str:0: -PyInt_FromString:char**:pend:0: -PyInt_FromString:int:base:0: +PyInstanceMethod_Function:PyObject*::0: +PyInstanceMethod_Function:PyObject*:im:0: -PyInt_FromSsize_t:PyObject*::+1: -PyInt_FromSsize_t:Py_ssize_t:ival:: +PyInstanceMethod_GET_FUNCTION:PyObject*::0: +PyInstanceMethod_GET_FUNCTION:PyObject*:im:0: -PyInt_GetMax:long::: +PyInstanceMethod_New:PyObject*::+1: +PyInstanceMethod_New:PyObject*:func:0: PyInterpreterState_Clear:void::: PyInterpreterState_Clear:PyInterpreterState*:interp:: @@ -654,7 +1042,8 @@ PyInterpreterState_GetID:PyInterpreterState*:interp:: PyInterpreterState_New:PyInterpreterState*::: -PyIter_Check:int:o:0: +PyIter_Check:int::: +PyIter_Check:PyObject*:o:0: PyIter_Next:PyObject*::+1: PyIter_Next:PyObject*:o:0: @@ -669,50 +1058,53 @@ PyList_AsTuple:PyObject*:list:0: PyList_Check:int::: PyList_Check:PyObject*:p:0: +PyList_CheckExact:int::: +PyList_CheckExact:PyObject*:p:0: + PyList_GET_ITEM:PyObject*::0: PyList_GET_ITEM:PyObject*:list:0: -PyList_GET_ITEM:int:i:0: +PyList_GET_ITEM:Py_ssize_t:i:: -PyList_GET_SIZE:int::: +PyList_GET_SIZE:Py_ssize_t::: PyList_GET_SIZE:PyObject*:list:0: PyList_GetItem:PyObject*::0: PyList_GetItem:PyObject*:list:0: -PyList_GetItem:int:index:: +PyList_GetItem:Py_ssize_t:index:: PyList_GetSlice:PyObject*::+1: PyList_GetSlice:PyObject*:list:0: -PyList_GetSlice:int:low:: -PyList_GetSlice:int:high:: +PyList_GetSlice:Py_ssize_t:low:: +PyList_GetSlice:Py_ssize_t:high:: PyList_Insert:int::: PyList_Insert:PyObject*:list:0: -PyList_Insert:int:index:: +PyList_Insert:Py_ssize_t:index:: PyList_Insert:PyObject*:item:+1: PyList_New:PyObject*::+1: -PyList_New:int:len:: +PyList_New:Py_ssize_t:len:: PyList_Reverse:int::: PyList_Reverse:PyObject*:list:0: PyList_SET_ITEM:void::: PyList_SET_ITEM:PyObject*:list:0: -PyList_SET_ITEM:int:i:: +PyList_SET_ITEM:Py_ssize_t:i:: PyList_SET_ITEM:PyObject*:o:0: PyList_SetItem:int::: PyList_SetItem:PyObject*:list:0: -PyList_SetItem:int:index:: +PyList_SetItem:Py_ssize_t:index:: PyList_SetItem:PyObject*:item:0: PyList_SetSlice:int::: PyList_SetSlice:PyObject*:list:0: -PyList_SetSlice:int:low:: -PyList_SetSlice:int:high:: +PyList_SetSlice:Py_ssize_t:low:: +PyList_SetSlice:Py_ssize_t:high:: PyList_SetSlice:PyObject*:itemlist:0:but increfs its elements? -PyList_Size:int::: +PyList_Size:Py_ssize_t::: PyList_Size:PyObject*:list:0: PyList_Sort:int::: @@ -724,12 +1116,44 @@ PyLong_AsDouble:PyObject*:pylong:0: PyLong_AsLong:long::: PyLong_AsLong:PyObject*:pylong:0: +PyLong_AsLongAndOverflow:long::: +PyLong_AsLongAndOverflow:PyObject*:obj:0: +PyLong_AsLongAndOverflow:int*:overflow:: + +PyLong_AsLongLong:long long::: +PyLong_AsLongLong:PyObject*:obj:0: + +PyLong_AsLongLongAndOverflow:long long::: +PyLong_AsLongLongAndOverflow:PyObject*:obj:0: +PyLong_AsLongLongAndOverflow:int*:overflow:: + +PyLong_AsSize_t:size_t::: +PyLong_AsSize_t:PyObject*:pylong:0: + +PyLong_AsSsize_t:Py_ssize_t::: +PyLong_AsSsize_t:PyObject*:pylong:0: + PyLong_AsUnsignedLong:unsigned long::: PyLong_AsUnsignedLong:PyObject*:pylong:0: +PyLong_AsUnsignedLongLong:unsigned long long::: +PyLong_AsUnsignedLongLong:PyObject*:pylong:0: + +PyLong_AsUnsignedLongMask:unsigned long::: +PyLong_AsUnsignedLongMask:PyObject*:obj:0: + +PyLong_AsUnsignedLongLongMask:unsigned long long::: +PyLong_AsUnsignedLongLongMask:PyObject*:obj:0: + +PyLong_AsVoidPtr:void*::: +PyLong_AsVoidPtr:PyObject*:pylong:0: + PyLong_Check:int::: PyLong_Check:PyObject*:p:0: +PyLong_CheckExact:int::: +PyLong_CheckExact:PyObject*:p:0: + PyLong_FromDouble:PyObject*::+1: PyLong_FromDouble:double:v:: @@ -742,16 +1166,26 @@ PyLong_FromLongLong:long long:v:: PyLong_FromUnsignedLongLong:PyObject*::+1: PyLong_FromUnsignedLongLong:unsigned long long:v:: +PyLong_FromSize_t:PyObject*::+1: +PyLong_FromSize_t:size_t:v:: + +PyLong_FromSsize_t:PyObject*::+1: +PyLong_FromSsize_t:Py_ssize_t:v:: + PyLong_FromString:PyObject*::+1: PyLong_FromString:const char*:str:: PyLong_FromString:char**:pend:: PyLong_FromString:int:base:: PyLong_FromUnicode:PyObject*::+1: -PyLong_FromUnicode:Py_UNICODE:u:: -PyLong_FromUnicode:int:length:: +PyLong_FromUnicode:Py_UNICODE*:u:: +PyLong_FromUnicode:Py_ssize_t:length:: PyLong_FromUnicode:int:base:: +PyLong_FromUnicodeObject:PyObject*::+1: +PyLong_FromUnicodeObject:PyObject*:u:0: +PyLong_FromUnicodeObject:int:base:: + PyLong_FromUnsignedLong:PyObject*::+1: PyLong_FromUnsignedLong:unsignedlong:v:: @@ -787,7 +1221,7 @@ PyMapping_Items:PyObject*:o:0: PyMapping_Keys:PyObject*::+1: PyMapping_Keys:PyObject*:o:0: -PyMapping_Length:int::: +PyMapping_Length:Py_ssize_t::: PyMapping_Length:PyObject*:o:0: PyMapping_SetItemString:int::: @@ -795,6 +1229,9 @@ PyMapping_SetItemString:PyObject*:o:0: PyMapping_SetItemString:const char*:key:: PyMapping_SetItemString:PyObject*:v:+1: +PyMapping_Size:Py_ssize_t::: +PyMapping_Size:PyObject*:o:0: + PyMapping_Values:PyObject*::+1: PyMapping_Values:PyObject*:o:0: @@ -806,20 +1243,43 @@ PyMarshal_ReadObjectFromFile:FILE*:file:: PyMarshal_ReadObjectFromString:PyObject*::+1: PyMarshal_ReadObjectFromString:const char*:string:: -PyMarshal_ReadObjectFromString:int:len:: +PyMarshal_ReadObjectFromString:Py_ssize_t:len:: PyMarshal_WriteObjectToString:PyObject*::+1: PyMarshal_WriteObjectToString:PyObject*:value:0: +PyMarshal_WriteObjectToString:int:version:: + +PyMemoryView_Check:int::: +PyMemoryView_Check:PyObject*:obj:0: + +PyMemoryView_FromBuffer:PyObject*::+1: +PyMemoryView_FromBuffer:Py_buffer*:view:: + +PyMemoryView_FromMemory:PyObject*::+1: +PyMemoryView_FromMemory:char*:mem:: +PyMemoryView_FromMemory:Py_ssize_t:size:: +PyMemoryView_FromMemory:int:flags:: + +PyMemoryView_FromObject:PyObject*::+1: +PyMemoryView_FromObject:PyObject*:obj:0: + +PyMemoryView_GET_BASE:Py_buffer*::: +PyMemoryView_GET_BASE:PyObject*:mview:0: + +PyMemoryView_GET_BUFFER:Py_buffer*::: +PyMemoryView_GET_BUFFER:PyObject*:mview:0: -PyMethod_Class:PyObject*::0: -PyMethod_Class:PyObject*:im:0: +PyMemoryView_GetContiguous:PyObject*::+1: +PyMemoryView_GetContiguous:PyObject*:obj:0: +PyMemoryView_GetContiguous:int:buffertype:: +PyMemoryView_GetContiguous:char:order:: + +PyMethod_Check:int::: +PyMethod_Check:PyObject*:o:0: PyMethod_Function:PyObject*::0: PyMethod_Function:PyObject*:im:0: -PyMethod_GET_CLASS:PyObject*::0: -PyMethod_GET_CLASS:PyObject*:im:0: - PyMethod_GET_FUNCTION:PyObject*::0: PyMethod_GET_FUNCTION:PyObject*:im:0: @@ -834,18 +1294,93 @@ PyMethod_New:PyObject*:class:0: PyMethod_Self:PyObject*::0: PyMethod_Self:PyObject*:im:0: +PyModule_AddFunctions:int::: +PyModule_AddFunctions:PyObject*:module:0: +PyModule_AddFunctions:PyMethodDef*:functions:: + +PyModule_AddIntConstant:int::: +PyModule_AddIntConstant:PyObject*:module:0: +PyModule_AddIntConstant:const char*:name:: +PyModule_AddIntConstant:long:value:: + +PyModule_AddIntMacro:int::: +PyModule_AddIntMacro:PyObject*:module:0: +PyModule_AddIntMacro::macro:: + +PyModule_AddObject:int::: +PyModule_AddObject:PyObject*:module:0: +PyModule_AddObject:const char*:name:: +PyModule_AddObject:PyObject*:value:+1: + +PyModule_AddStringConstant:int::: +PyModule_AddStringConstant:PyObject*:module:0: +PyModule_AddStringConstant:const char*:name:: +PyModule_AddStringConstant:const char*:value:: + +PyModule_AddStringMacro:int::: +PyModule_AddStringMacro:PyObject*:module:0: +PyModule_AddStringMacro::macro:: + +PyModule_Check:int::: +PyModule_Check:PyObject*:p:0: + +PyModule_CheckExact:int::: +PyModule_CheckExact:PyObject*:p:0: + +PyModule_Create:PyObject*::+1: +PyModule_Create:PyModuleDef*:def:: + +PyModule_Create2:PyObject*::+1: +PyModule_Create2:PyModuleDef*:def:: +PyModule_Create2:int:module_api_version:: + +PyModule_ExecDef:int::: +PyModule_ExecDef:PyObject*:module:0: +PyModule_ExecDef:PyModuleDef*:def:: + +PyModule_FromDefAndSpec:PyObject*::+1: +PyModule_FromDefAndSpec:PyModuleDef*:def:: +PyModule_FromDefAndSpec:PyObject*:spec:0: + +PyModule_FromDefAndSpec2:PyObject*::+1: +PyModule_FromDefAndSpec2:PyModuleDef*:def:: +PyModule_FromDefAndSpec2:PyObject*:spec:0: +PyModule_FromDefAndSpec2:int:module_api_version:: + +PyModule_GetDef:PyModuleDef*::0: +PyModule_GetDef:PyObject*:module:0: + PyModule_GetDict:PyObject*::0: -PyModule_GetDict::PyObject* module:0: +PyModule_GetDict:PyObject*:module:0: PyModule_GetFilename:const char*::: PyModule_GetFilename:PyObject*:module:0: +PyModule_GetFilenameObject:PyObject*::+1: +PyModule_GetFilenameObject:PyObject*:module:0: + PyModule_GetName:const char*::: PyModule_GetName:PyObject*:module:0: +PyModule_GetNameObject:PyObject*::+1: +PyModule_GetNameObject:PyObject*:module:0: + +PyModule_GetState:void*::: +PyModule_GetState:PyObject*:module:0: + PyModule_New:PyObject*::+1: PyModule_New::char* name:: +PyModule_NewObject:PyObject*::+1: +PyModule_NewObject:PyObject*:name:+1: + +PyModule_SetDocString:int::: +PyModule_SetDocString:PyObject*:module:0: +PyModule_SetDocString:const char*:docstring:: + +PyModuleDef_Init:PyObject*::0: +PyModuleDef_Init:PyModuleDef*:def:0: + PyNumber_Absolute:PyObject*::+1: PyNumber_Absolute:PyObject*:o:0: @@ -857,12 +1392,12 @@ PyNumber_And:PyObject*::+1: PyNumber_And:PyObject*:o1:0: PyNumber_And:PyObject*:o2:0: -PyNumber_Check:PyObject*:o:0: -PyNumber_Check:int::: +PyNumber_AsSsize_t:Py_ssize_t::: +PyNumber_AsSsize_t:PyObject*:o:0: +PyNumber_AsSsize_t:PyObject*:exc:0: -PyNumber_Divide:PyObject*::+1: -PyNumber_Divide:PyObject*:o1:0: -PyNumber_Divide:PyObject*:o2:0: +PyNumber_Check:int::: +PyNumber_Check:PyObject*:o:0: PyNumber_Divmod:PyObject*::+1: PyNumber_Divmod:PyObject*:o1:0: @@ -875,6 +1410,9 @@ PyNumber_FloorDivide:PyObject*::+1: PyNumber_FloorDivide:PyObject*:v:0: PyNumber_FloorDivide:PyObject*:w:0: +PyNumber_Index:PyObject*::+1: +PyNumber_Index:PyObject*:o:0: + PyNumber_InPlaceAdd:PyObject*::+1: PyNumber_InPlaceAdd:PyObject*:v:0: PyNumber_InPlaceAdd:PyObject*:w:0: @@ -883,10 +1421,6 @@ PyNumber_InPlaceAnd:PyObject*::+1: PyNumber_InPlaceAnd:PyObject*:v:0: PyNumber_InPlaceAnd:PyObject*:w:0: -PyNumber_InPlaceDivide:PyObject*::+1: -PyNumber_InPlaceDivide:PyObject*:v:0: -PyNumber_InPlaceDivide:PyObject*:w:0: - PyNumber_InPlaceFloorDivide:PyObject*::+1: PyNumber_InPlaceFloorDivide:PyObject*:v:0: PyNumber_InPlaceFloorDivide:PyObject*:w:0: @@ -895,6 +1429,10 @@ PyNumber_InPlaceLshift:PyObject*::+1: PyNumber_InPlaceLshift:PyObject*:v:0: PyNumber_InPlaceLshift:PyObject*:w:0: +PyNumber_InPlaceMatrixMultiply:PyObject*::+1: +PyNumber_InPlaceMatrixMultiply:PyObject*:o1:0: +PyNumber_InPlaceMatrixMultiply:PyObject*:o2:0: + PyNumber_InPlaceMultiply:PyObject*::+1: PyNumber_InPlaceMultiply:PyObject*:v:0: PyNumber_InPlaceMultiply:PyObject*:w:0: @@ -938,6 +1476,10 @@ PyNumber_Lshift:PyObject*::+1: PyNumber_Lshift:PyObject*:o1:0: PyNumber_Lshift:PyObject*:o2:0: +PyNumber_MatrixMultiply:PyObject*::+1: +PyNumber_MatrixMultiply:PyObject*:o1:0: +PyNumber_MatrixMultiply:PyObject*:o2:0: + PyNumber_Multiply:PyObject*::+1: PyNumber_Multiply:PyObject*:o1:0: PyNumber_Multiply:PyObject*:o2:0: @@ -969,6 +1511,10 @@ PyNumber_Subtract:PyObject*::+1: PyNumber_Subtract:PyObject*:o1:0: PyNumber_Subtract:PyObject*:o2:0: +PyNumber_ToBase:PyObject*::+1: +PyNumber_ToBase:PyObject*:n:0: +PyNumber_ToBase:int:base:: + PyNumber_TrueDivide:PyObject*::+1: PyNumber_TrueDivide:PyObject*:v:0: PyNumber_TrueDivide:PyObject*:w:0: @@ -991,6 +1537,27 @@ PyOS_BeforeFork:void::: PyOS_FSPath:PyObject*::+1: PyOS_FSPath:PyObject*:path:0: +PyObject_ASCII:PyObject*::+1: +PyObject_ASCII:PyObject*:o:0: + +PyObject_AsCharBuffer:int::: +PyObject_AsCharBuffer:PyObject*:obj:0: +PyObject_AsCharBuffer:const char**:buffer:: +PyObject_AsCharBuffer:Py_ssize_t*:buffer_len:: + +PyObject_AsReadBuffer:int::: +PyObject_AsReadBuffer:PyObject*:obj:0: +PyObject_AsReadBuffer:const void**:buffer:: +PyObject_AsReadBuffer:Py_ssize_t*:buffer_len:: + +PyObject_AsWriteBuffer:int::: +PyObject_AsWriteBuffer:PyObject*:obj:0: +PyObject_AsWriteBuffer:void**:buffer:: +PyObject_AsWriteBuffer:Py_ssize_t*:buffer_len:: + +PyObject_Bytes:PyObject*::+1: +PyObject_Bytes:PyObject*:o:0: + PyObject_Call:PyObject*::+1: PyObject_Call:PyObject*:callable_object:0: PyObject_Call:PyObject*:args:0: @@ -1020,14 +1587,11 @@ PyObject_CallObject:PyObject*::+1: PyObject_CallObject:PyObject*:callable_object:0: PyObject_CallObject:PyObject*:args:0: -PyObject_Cmp:int::: -PyObject_Cmp:PyObject*:o1:0: -PyObject_Cmp:PyObject*:o2:0: -PyObject_Cmp:int*:result:: +PyObject_CheckBuffer:int::: +PyObject_CheckBuffer:PyObject*:obj:0: -PyObject_Compare:int::: -PyObject_Compare:PyObject*:o1:0: -PyObject_Compare:PyObject*:o2:0: +PyObject_CheckReadBuffer:int::: +PyObject_CheckReadBuffer:PyObject*:o:0: PyObject_DelAttr:int::: PyObject_DelAttr:PyObject*:o:0: @@ -1044,6 +1608,46 @@ PyObject_DelItem:PyObject*:key:0: PyObject_Dir:PyObject*::+1: PyObject_Dir:PyObject*:o:0: +PyObject_GC_Del:void::: +PyObject_GC_Del:void*:op:: + +PyObject_GC_New:TYPE*::+1: +PyObject_GC_New::TYPE:: +PyObject_GC_New:PyTypeObject*:type:0: + +PyObject_GC_NewVar:TYPE*::+1: +PyObject_GC_NewVar::TYPE:: +PyObject_GC_NewVar:PyTypeObject*:type:0: +PyObject_GC_NewVar:Py_ssize_t:size:: + +PyObject_GC_Resize:TYPE*::0: +PyObject_GC_Resize::TYPE:: +PyObject_GC_Resize:PyVarObject*:op:0: +PyObject_GC_Resize:Py_ssize_t:newsize:: + +PyObject_GC_Track:void::: +PyObject_GC_Track:PyObject*:op:0: + +PyObject_GC_UnTrack:void::: +PyObject_GC_UnTrack:void*:op:: + +PyObject_GenericGetAttr:PyObject*::+1: +PyObject_GenericGetAttr:PyObject*:o:0: +PyObject_GenericGetAttr:PyObject*:name:0: + +PyObject_GenericGetDict:PyObject*::+1: +PyObject_GenericGetDict:PyObject*:o:0: +PyObject_GenericGetDict:void*:context:: + +PyObject_GenericSetAttr:int::: +PyObject_GenericSetAttr:PyObject*:o:0: +PyObject_GenericSetAttr:PyObject*:name:0: +PyObject_GenericSetAttr:PyObject*:value:+1: + +PyObject_GenericSetDict:int::: +PyObject_GenericSetDict:PyObject*:o:+1: +PyObject_GenericSetDict:void*:context:: + PyObject_GetAttr:PyObject*::+1: PyObject_GetAttr:PyObject*:o:0: PyObject_GetAttr:PyObject*:attr_name:0: @@ -1052,6 +1656,11 @@ PyObject_GetAttrString:PyObject*::+1: PyObject_GetAttrString:PyObject*:o:0: PyObject_GetAttrString:const char*:attr_name:: +PyObject_GetBuffer:int::: +PyObject_GetBuffer:PyObject*:exporter:0: +PyObject_GetBuffer:Py_buffer*:view:: +PyObject_GetBuffer:int:flags:: + PyObject_GetItem:PyObject*::+1: PyObject_GetItem:PyObject*:o:0: PyObject_GetItem:PyObject*:key:0: @@ -1065,30 +1674,59 @@ PyObject_HasAttr:PyObject*:attr_name:0: PyObject_HasAttrString:int::: PyObject_HasAttrString:PyObject*:o:0: -PyObject_HasAttrString:const char*:attr_name:0: +PyObject_HasAttrString:const char*:attr_name:: PyObject_Hash:int::: PyObject_Hash:PyObject*:o:0: +PyObject_HashNotImplemented:Py_hash_t::: +PyObject_HashNotImplemented:PyObject*:o:0: + +PyObject_IsInstance:int::: +PyObject_IsInstance:PyObject*:inst:0: +PyObject_IsInstance:PyObject*:cls:0: + +PyObject_IsSubclass:int::: +PyObject_IsSubclass:PyObject*:derived:0: +PyObject_IsSubclass:PyObject*:cls:0: + PyObject_IsTrue:int::: PyObject_IsTrue:PyObject*:o:0: PyObject_Init:PyObject*::0: PyObject_Init:PyObject*:op:0: +PyObject_Init:PyTypeObject*:type:0: PyObject_InitVar:PyVarObject*::0: PyObject_InitVar:PyVarObject*:op:0: -PyObject_Length:int::: +PyObject_Length:Py_ssize_t::: PyObject_Length:PyObject*:o:0: +PyObject_LengthHint:Py_ssize_t::: +PyObject_LengthHint:PyObject*:o:0: +PyObject_LengthHint:Py_ssize_t:default:: + PyObject_NEW:PyObject*::+1: +PyObject_NEW::TYPE:: +PyObject_NEW:PyTypeObject*:type:0: PyObject_New:PyObject*::+1: +PyObject_New::TYPE:: +PyObject_New:PyTypeObject*:type:0: PyObject_NEW_VAR:PyObject*::+1: +PyObject_NEW_VAR::TYPE:: +PyObject_NEW_VAR:PyTypeObject*:type:0: +PyObject_NEW_VAR:Py_ssize_t:size:: PyObject_NewVar:PyObject*::+1: +PyObject_NewVar::TYPE:: +PyObject_NewVar:PyTypeObject*:type:0: +PyObject_NewVar:Py_ssize_t:size:: + +PyObject_Not:int::: +PyObject_Not:PyObject*:o:0: PyObject_Print:int::: PyObject_Print:PyObject*:o:0: @@ -1123,28 +1761,65 @@ PyObject_SetItem:PyObject*:o:0: PyObject_SetItem:PyObject*:key:0: PyObject_SetItem:PyObject*:v:+1: +PyObject_Size:Py_ssize_t::: +PyObject_Size:PyObject*:o:0: + PyObject_Str:PyObject*::+1: PyObject_Str:PyObject*:o:0: PyObject_Type:PyObject*::+1: PyObject_Type:PyObject*:o:0: -PyObject_Unicode:PyObject*::+1: -PyObject_Unicode:PyObject*:o:0: +PyObject_TypeCheck:int::: +PyObject_TypeCheck:PyObject*:o:0: +PyObject_TypeCheck:PyTypeObject*:type:0: PyParser_SimpleParseFile:struct _node*::: PyParser_SimpleParseFile:FILE*:fp:: PyParser_SimpleParseFile:const char*:filename:: PyParser_SimpleParseFile:int:start:: +PyParser_SimpleParseFileFlags:struct _node*::: +PyParser_SimpleParseFileFlags:FILE*:fp:: +PyParser_SimpleParseFileFlags:const char*:filename:: +PyParser_SimpleParseFileFlags:int:start:: +PyParser_SimpleParseFileFlags:int:flags:: + PyParser_SimpleParseString:struct _node*::: PyParser_SimpleParseString:const char*:str:: PyParser_SimpleParseString:int:start:: +PyParser_SimpleParseStringFlags:struct _node*::: +PyParser_SimpleParseStringFlags:const char*:str:: +PyParser_SimpleParseStringFlags:int:start:: +PyParser_SimpleParseStringFlags:int:flags:: + +PyParser_SimpleParseStringFlagsFilename:struct _node*::: +PyParser_SimpleParseStringFlagsFilename:const char*:str:: +PyParser_SimpleParseStringFlagsFilename:const char*:filename:: +PyParser_SimpleParseStringFlagsFilename:int:start:: +PyParser_SimpleParseStringFlagsFilename:int:flags:: + PyRun_AnyFile:int::: PyRun_AnyFile:FILE*:fp:: PyRun_AnyFile:const char*:filename:: +PyRun_AnyFileFlags:int::: +PyRun_AnyFileFlags:FILE*:fp:: +PyRun_AnyFileFlags:const char*:filename:: +PyRun_AnyFileFlags:PyCompilerFlags*:flags:: + +PyRun_AnyFileEx:int::: +PyRun_AnyFileEx:FILE*:fp:: +PyRun_AnyFileEx:const char*:filename:: +PyRun_AnyFileEx:int:closeit:: + +PyRun_AnyFileExFlags:int::: +PyRun_AnyFileExFlags:FILE*:fp:: +PyRun_AnyFileExFlags:const char*:filename:: +PyRun_AnyFileExFlags:int:closeit:: +PyRun_AnyFileExFlags:PyCompilerFlags*:flags:: + PyRun_File:PyObject*::+1:??? -- same as eval_code2() PyRun_File:FILE*:fp:: PyRun_File:const char*:filename:: @@ -1181,17 +1856,42 @@ PyRun_InteractiveLoop:int::: PyRun_InteractiveLoop:FILE*:fp:: PyRun_InteractiveLoop:const char*:filename:: +PyRun_InteractiveLoopFlags:int::: +PyRun_InteractiveLoopFlags:FILE*:fp:: +PyRun_InteractiveLoopFlags:const char*:filename:: +PyRun_InteractiveLoopFlags:PyCompilerFlags*:flags:: + PyRun_InteractiveOne:int::: PyRun_InteractiveOne:FILE*:fp:: PyRun_InteractiveOne:const char*:filename:: +PyRun_InteractiveOneFlags:int::: +PyRun_InteractiveOneFlags:FILE*:fp:: +PyRun_InteractiveOneFlags:const char*:filename:: +PyRun_InteractiveOneFlags:PyCompilerFlags*:flags:: + PyRun_SimpleFile:int::: PyRun_SimpleFile:FILE*:fp:: PyRun_SimpleFile:const char*:filename:: +PyRun_SimpleFileEx:int::: +PyRun_SimpleFileEx:FILE*:fp:: +PyRun_SimpleFileEx:const char*:filename:: +PyRun_SimpleFileEx:int:closeit:: + +PyRun_SimpleFileExFlags:int::: +PyRun_SimpleFileExFlags:FILE*:fp:: +PyRun_SimpleFileExFlags:const char*:filename:: +PyRun_SimpleFileExFlags:int:closeit:: +PyRun_SimpleFileExFlags:PyCompilerFlags*:flags:: + PyRun_SimpleString:int::: PyRun_SimpleString:const char*:command:: +PyRun_SimpleStringFlags:int::: +PyRun_SimpleStringFlags:const char*:command:: +PyRun_SimpleStringFlags:PyCompilerFlags*:flags:: + PyRun_String:PyObject*::+1:??? -- same as eval_code2() PyRun_String:const char*:str:: PyRun_String:int:start:: @@ -1205,6 +1905,9 @@ PyRun_StringFlags:PyObject*:globals:0: PyRun_StringFlags:PyObject*:locals:0: PyRun_StringFlags:PyCompilerFlags*:flags:: +PySeqIter_Check:int::: +PySeqIter_Check::op:: + PySeqIter_New:PyObject*::+1: PySeqIter_New:PyObject*:seq:: @@ -1215,18 +1918,22 @@ PySequence_Concat:PyObject*::+1: PySequence_Concat:PyObject*:o1:0: PySequence_Concat:PyObject*:o2:0: -PySequence_Count:int::: +PySequence_Contains:int::: +PySequence_Contains:PyObject*:o:0: +PySequence_Contains:PyObject*:value:0: + +PySequence_Count:Py_ssize_t::: PySequence_Count:PyObject*:o:0: PySequence_Count:PyObject*:value:0: PySequence_DelItem:int::: PySequence_DelItem:PyObject*:o:0: -PySequence_DelItem:int:i:: +PySequence_DelItem:Py_ssize_t:i:: PySequence_DelSlice:int::: PySequence_DelSlice:PyObject*:o:0: -PySequence_DelSlice:int:i1:: -PySequence_DelSlice:int:i2:: +PySequence_DelSlice:Py_ssize_t:i1:: +PySequence_DelSlice:Py_ssize_t:i2:: PySequence_Fast:PyObject*::+1: PySequence_Fast:PyObject*:v:0: @@ -1234,22 +1941,28 @@ PySequence_Fast:const char*:m:: PySequence_Fast_GET_ITEM:PyObject*::0: PySequence_Fast_GET_ITEM:PyObject*:o:0: -PySequence_Fast_GET_ITEM:int:i:: +PySequence_Fast_GET_ITEM:Py_ssize_t:i:: + +PySequence_Fast_GET_SIZE:Py_ssize_t::: +PySequence_Fast_GET_SIZE:PyObject*:o:0: + +PySequence_Fast_ITEMS:PyObject**::: +PySequence_Fast_ITEMS:PyObject*:o:0: PySequence_GetItem:PyObject*::+1: PySequence_GetItem:PyObject*:o:0: -PySequence_GetItem:int:i:: +PySequence_GetItem:Py_ssize_t:i:: PySequence_GetSlice:PyObject*::+1: PySequence_GetSlice:PyObject*:o:0: -PySequence_GetSlice:int:i1:: -PySequence_GetSlice:int:i2:: +PySequence_GetSlice:Py_ssize_t:i1:: +PySequence_GetSlice:Py_ssize_t:i2:: PySequence_In:int::: PySequence_In:PyObject*:o:0: PySequence_In:PyObject*:value:0: -PySequence_Index:int::: +PySequence_Index:Py_ssize_t::: PySequence_Index:PyObject*:o:0: PySequence_Index:PyObject*:value:0: @@ -1263,22 +1976,25 @@ PySequence_InPlaceRepeat:PyObject*:o:0: PySequence_ITEM:PyObject*::+1: PySequence_ITEM:PyObject*:o:0: -PySequence_ITEM:int:i:: +PySequence_ITEM:Py_ssize_t:i:: PySequence_Repeat:PyObject*::+1: PySequence_Repeat:PyObject*:o:0: -PySequence_Repeat:int:count:: +PySequence_Repeat:Py_ssize_t:count:: PySequence_SetItem:int::: PySequence_SetItem:PyObject*:o:0: -PySequence_SetItem:int:i:: +PySequence_SetItem:Py_ssize_t:i:: PySequence_SetItem:PyObject*:v:+1: PySequence_SetSlice:int::: PySequence_SetSlice:PyObject*:o:0: -PySequence_SetSlice:int:i1:: -PySequence_SetSlice:int:i2:: -PySequence_SetSlice:PyObject*:v:+1: +PySequence_SetSlice:Py_ssize_t:i1:: +PySequence_SetSlice:Py_ssize_t:i2:: +PySequence_SetSlice:PyObject*:v:0: + +PySequence_Size:Py_ssize_t::: +PySequence_Size:PyObject*:o:0: PySequence_List:PyObject*::+1: PySequence_List:PyObject*:o:0: @@ -1286,9 +2002,15 @@ PySequence_List:PyObject*:o:0: PySequence_Tuple:PyObject*::+1: PySequence_Tuple:PyObject*:o:0: -PySet_Append:int::: -PySet_Append:PyObject*:set:0: -PySet_Append:PyObject*:key:+1: +PySet_Add:int::: +PySet_Add:PyObject*:set:0: +PySet_Add:PyObject*:key:+1: + +PySet_Check:int::: +PySet_Check:PyObject*:p:0: + +PySet_Clear:int::: +PySet_Clear:PyObject*:set:0: PySet_Contains:int::: PySet_Contains:PyObject*:anyset:0: @@ -1298,15 +2020,21 @@ PySet_Discard:int::: PySet_Discard:PyObject*:set:0: PySet_Discard:PyObject*:key:-1:no effect if key not found +PySet_GET_SIZE:Py_ssize_t::: +PySet_GET_SIZE:PyObject*:anyset:0: + PySet_New:PyObject*::+1: PySet_New:PyObject*:iterable:0: PySet_Pop:PyObject*::+1:or returns NULL and raises KeyError if set is empty PySet_Pop:PyObject*:set:0: -PySet_Size:int::: +PySet_Size:Py_ssize_t::: PySet_Size:PyObject*:anyset:0: +PySignal_SetWakeupFd:int::: +PySignal_SetWakeupFd:int:fd:: + PySlice_AdjustIndices:Py_ssize_t::: PySlice_AdjustIndices:Py_ssize_t:length:: PySlice_AdjustIndices:Py_ssize_t*:start:: @@ -1316,6 +2044,21 @@ PySlice_AdjustIndices:Py_ssize_t*:step:: PySlice_Check:int::: PySlice_Check:PyObject*:ob:0: +PySlice_GetIndices:int::: +PySlice_GetIndices:PyObject*:slice:0: +PySlice_GetIndices:Py_ssize_t:length:: +PySlice_GetIndices:Py_ssize_t*:start:: +PySlice_GetIndices:Py_ssize_t*:stop:: +PySlice_GetIndices:Py_ssize_t*:step:: + +PySlice_GetIndicesEx:int::: +PySlice_GetIndicesEx:PyObject*:slice:0: +PySlice_GetIndicesEx:Py_ssize_t:length:: +PySlice_GetIndicesEx:Py_ssize_t*:start:: +PySlice_GetIndicesEx:Py_ssize_t*:stop:: +PySlice_GetIndicesEx:Py_ssize_t*:step:: +PySlice_GetIndicesEx:Py_ssize_t*:slicelength:: + PySlice_New:PyObject*::+1: PySlice_New:PyObject*:start:0: PySlice_New:PyObject*:stop:0: @@ -1327,100 +2070,78 @@ PySlice_Unpack:Py_ssize_t*:start:: PySlice_Unpack:Py_ssize_t*:stop:: PySlice_Unpack:Py_ssize_t*:step:: -PyString_AS_STRING:const char*::: -PyString_AS_STRING:PyObject*:string:0: - -PyString_AsDecodedObject:PyObject*::+1: -PyString_AsDecodedObject:PyObject*:str:0: -PyString_AsDecodedObject:const char*:encoding:: -PyString_AsDecodedObject:const char*:errors:: - -PyString_AsEncodedObject:PyObject*::+1: -PyString_AsEncodedObject:PyObject*:str:0: -PyString_AsEncodedObject:const char*:encoding:: -PyString_AsEncodedObject:const char*:errors:: - -PyString_AsString:const char*::: -PyString_AsString:PyObject*:string:0: - -PyString_AsStringAndSize:int::: -PyString_AsStringAndSize:PyObject*:obj:0: -PyString_AsStringAndSize:char**:buffer:: -PyString_AsStringAndSize:int*:length:: - -PyString_Check:int::: -PyString_Check:PyObject*:o:0: - -PyString_Concat:void::: -PyString_Concat:PyObject**:string:0:??? -- replaces w/ new string or NULL -PyString_Concat:PyObject*:newpart:0: - -PyString_ConcatAndDel:void::: -PyString_ConcatAndDel:PyObject**:string:0:??? -- replaces w/ new string or NULL -PyString_ConcatAndDel:PyObject*:newpart:-1: - -PyString_Format:PyObject*::+1: -PyString_Format:PyObject*:format:0: -PyString_Format:PyObject*:args:0: - -PyString_FromString:PyObject*::+1: -PyString_FromString:const char*:v:: +PyState_AddModule:int::: +PyState_AddModule:PyObject*:module:+1: +PyState_AddModule:PyModuleDef*:def:: -PyString_FromStringAndSize:PyObject*::+1: -PyString_FromStringAndSize:const char*:v:: -PyString_FromStringAndSize:int:len:: +PyState_FindModule:PyObject*::0: +PyState_FindModule:PyModuleDef*:def:: -PyString_FromFormat:PyObject*::+1: -PyString_FromFormat:const char*:format:: -PyString_FromFormat::...:: +PyState_RemoveModule:int::: +PyState_RemoveModule:PyModuleDef*:def:: -PyString_FromFormatV:PyObject*::+1: -PyString_FromFormatV:const char*:format:: -PyString_FromFormatV:va_list:vargs:: +PyStructSequence_GET_ITEM:PyObject*::0: +PyStructSequence_GET_ITEM:PyObject*:p:0: +PyStructSequence_GET_ITEM:Py_ssize_t:pos:: -PyString_GET_SIZE:int::: -PyString_GET_SIZE:PyObject*:string:0: +PyStructSequence_GetItem:PyObject*::0: +PyStructSequence_GetItem:PyObject*:p:0: +PyStructSequence_GetItem:Py_ssize_t:pos:: -PyString_InternFromString:PyObject*::+1: -PyString_InternFromString:const char*:v:: +PyStructSequence_InitType:void::: +PyStructSequence_InitType:PyTypeObject*:type:+1: +PyStructSequence_InitType:PyStructSequence_Desc*:desc:: -PyString_InternInPlace:void::: -PyString_InternInPlace:PyObject**:string:+1:??? +PyStructSequence_InitType2:int::: +PyStructSequence_InitType2:PyTypeObject*:type:+1: +PyStructSequence_InitType2:PyStructSequence_Desc*:desc:: -PyString_Size:int::: -PyString_Size:PyObject*:string:0: +PyStructSequence_New:PyObject*::+1: +PyStructSequence_New:PyTypeObject*:type:0: -PyString_Decode:PyObject*::+1: -PyString_Decode:const char*:s:: -PyString_Decode:int:size:: -PyString_Decode:const char*:encoding:: -PyString_Decode:const char*:errors:: +PyStructSequence_NewType:PyTypeObject*::+1: +PyStructSequence_NewType:PyStructSequence_Desc*:desc:: -PyString_Encode:PyObject*::+1: -PyString_Encode:const char*:s:: -PyString_Encode:int:size:: -PyString_Encode:const char*:encoding:: -PyString_Encode:const char*:errors:: +PyStructSequence_SET_ITEM:void::: +PyStructSequence_SET_ITEM:PyObject*:p:0: +PyStructSequence_SET_ITEM:Py_ssize_t*:pos:: +PyStructSequence_SET_ITEM:PyObject*:o:0: -PyString_AsEncodedString:PyObject*::+1: -PyString_AsEncodedString:PyObject*:str:: -PyString_AsEncodedString:const char*:encoding:: -PyString_AsEncodedString:const char*:errors:: +PyStructSequence_SetItem:void::: +PyStructSequence_SetItem:PyObject*:p:0: +PyStructSequence_SetItem:Py_ssize_t:pos:: +PyStructSequence_SetItem:PyObject*:o:0: PySys_AddWarnOption:void::: -PySys_AddWarnOption:const char*:s:: +PySys_AddWarnOption:const wchar_t*:s:: + +PySys_AddWarnOptionUnicode:void::: +PySys_AddWarnOptionUnicode:PyObject*:unicode:0: PySys_AddXOption:void::: PySys_AddXOption:const wchar_t*:s:: +PySys_FormatStderr:void::: +PySys_FormatStderr:const char*:format:: +PySys_FormatStderr::...:: + +PySys_FormatStdout:void::: +PySys_FormatStdout:const char*:format:: +PySys_FormatStdout::...:: + PySys_GetObject:PyObject*::0: PySys_GetObject:const char*:name:: PySys_GetXOptions:PyObject*::0: -PySys_SetArgv:int::: +PySys_SetArgv:void::: PySys_SetArgv:int:argc:: -PySys_SetArgv:char**:argv:: +PySys_SetArgv:wchar_t**:argv:: + +PySys_SetArgvEx:void::: +PySys_SetArgvEx:int:argc:: +PySys_SetArgvEx:wchar_t**:argv:: +PySys_SetArgvEx:int:updatepath:: PySys_SetObject:int::: PySys_SetObject:const char*:name:: @@ -1430,9 +2151,11 @@ PySys_ResetWarnOptions:void::: PySys_WriteStdout:void::: PySys_WriteStdout:const char*:format:: +PySys_WriteStdout::...:: PySys_WriteStderr:void::: PySys_WriteStderr:const char*:format:: +PySys_WriteStderr::...:: PyThreadState_Clear:void::: PyThreadState_Clear:PyThreadState*:tstate:: @@ -1447,6 +2170,10 @@ PyThreadState_GetDict:PyObject*::0: PyThreadState_New:PyThreadState*::: PyThreadState_New:PyInterpreterState*:interp:: +PyThreadState_SetAsyncExc:int::: +PyThreadState_SetAsyncExc:unsigned long:id:: +PyThreadState_SetAsyncExc:PyObject*:exc:+1: + PyThreadState_Swap:PyThreadState*::: PyThreadState_Swap:PyThreadState*:tstate:: @@ -1471,6 +2198,12 @@ PyThread_tss_set:int::: PyThread_tss_set:Py_tss_t*:key:: PyThread_tss_set:void*:value:: +PyTime_Check:int::: +PyTime_Check:PyObject*:ob:0: + +PyTime_CheckExact:int::: +PyTime_CheckExact:PyObject*:ob:0: + PyTime_FromTime:PyObject*::+1: PyTime_FromTime:int:hour:: PyTime_FromTime:int:minute:: @@ -1489,63 +2222,130 @@ PyTraceMalloc_Untrack:uintptr_t:ptr:: PyTuple_Check:int::: PyTuple_Check:PyObject*:p:0: +PyTuple_CheckExact:int::: +PyTuple_CheckExact:PyObject*:p:0: + PyTuple_GET_ITEM:PyObject*::0: -PyTuple_GET_ITEM:PyTupleObject*:p:0: -PyTuple_GET_ITEM:int:pos:: +PyTuple_GET_ITEM:PyObject*:p:0: +PyTuple_GET_ITEM:Py_ssize_t:pos:: PyTuple_GetItem:PyObject*::0: -PyTuple_GetItem:PyTupleObject*:p:0: -PyTuple_GetItem:int:pos:: +PyTuple_GetItem:PyObject*:p:0: +PyTuple_GetItem:Py_ssize_t:pos:: + +PyTuple_GET_SIZE:Py_ssize_t::: +PyTuple_GET_SIZE:PyObject*:p:0: PyTuple_GetSlice:PyObject*::+1: -PyTuple_GetSlice:PyTupleObject*:p:0: -PyTuple_GetSlice:int:low:: -PyTuple_GetSlice:int:high:: +PyTuple_GetSlice:PyObject*:p:0: +PyTuple_GetSlice:Py_ssize_t:low:: +PyTuple_GetSlice:Py_ssize_t:high:: PyTuple_New:PyObject*::+1: -PyTuple_New:int:len:: +PyTuple_New:Py_ssize_t:len:: PyTuple_Pack:PyObject*::+1: -PyTuple_Pack:int:len:: +PyTuple_Pack:Py_ssize_t:len:: PyTuple_Pack:PyObject*:...:0: PyTuple_SET_ITEM:void::: -PyTuple_SET_ITEM:PyTupleObject*:p:0: -PyTuple_SET_ITEM:int:pos:: +PyTuple_SET_ITEM:PyObject*:p:0: +PyTuple_SET_ITEM:Py_ssize_t:pos:: PyTuple_SET_ITEM:PyObject*:o:0: PyTuple_SetItem:int::: -PyTuple_SetItem:PyTupleObject*:p:0: -PyTuple_SetItem:int:pos:: +PyTuple_SetItem:PyObject*:p:0: +PyTuple_SetItem:Py_ssize_t:pos:: PyTuple_SetItem:PyObject*:o:0: -PyTuple_Size:int::: -PyTuple_Size:PyTupleObject*:p:0: +PyTuple_Size:Py_ssize_t::: +PyTuple_Size:PyObject*:p:0: + +PyType_Check:int::: +PyType_Check:PyObject*:o:0: + +PyType_CheckExact:int::: +PyType_CheckExact:PyObject*:o:0: + +PyType_FromSpec:PyObject*::+1: +PyType_FromSpec:PyType_Spec*:spec:: + +PyType_FromSpecWithBases:PyObject*::+1: +PyType_FromSpecWithBases:PyType_Spec*:spec:: +PyType_FromSpecWithBases:PyObject*:bases:0: PyType_GenericAlloc:PyObject*::+1: PyType_GenericAlloc:PyObject*:type:0: -PyType_GenericAlloc:int:nitems:0: +PyType_GenericAlloc:Py_ssize_t:nitems:: PyType_GenericNew:PyObject*::+1: PyType_GenericNew:PyObject*:type:0: PyType_GenericNew:PyObject*:args:0: PyType_GenericNew:PyObject*:kwds:0: +PyType_GetFlags:unsigned long::: +PyType_GetFlags:PyTypeObject*:type:0: + +PyType_GetSlot:void*::: +PyType_GetSlot:PyTypeObject*:type:0: +PyType_GetSlot:int:slot:: + +PyType_HasFeature:int::: +PyType_HasFeature:PyTypeObject*:o:0: +PyType_HasFeature:int:feature:: + +PyType_IS_GC:int::: +PyType_IS_GC:PyTypeObject*:o:0: + +PyType_IsSubtype:int::: +PyType_IsSubtype:PyTypeObject*:a:0: +PyType_IsSubtype:PyTypeObject*:b:0: + +PyType_Modified:void::: +PyType_Modified:PyTypeObject*:type:0: + +PyType_Ready:int::: +PyType_Ready:PyTypeObject*:type:0: + +PyUnicode_1BYTE_DATA:Py_UCS1*::: +PyUnicode_1BYTE_DATA:PyObject*:o:0: + PyUnicode_Check:int::: PyUnicode_Check:PyObject*:o:0: -PyUnicode_GET_SIZE:int::: +PyUnicode_CheckExact:int::: +PyUnicode_CheckExact:PyObject*:o:0: + +PyUnicode_DATA:void*::: +PyUnicode_DATA:PyObject*:o:0: + +PyUnicode_GET_LENGTH:Py_ssize_t::: +PyUnicode_GET_LENGTH:PyObject*:o:0: + +PyUnicode_GET_SIZE:Py_ssize_t::: PyUnicode_GET_SIZE:PyObject*:o:0: -PyUnicode_GET_DATA_SIZE:int::: +PyUnicode_GET_DATA_SIZE:Py_ssize_t::: PyUnicode_GET_DATA_SIZE:PyObject*:o:0: +PyUnicode_KIND:int::: +PyUnicode_KIND:PyObject*:o:0: + +PyUnicode_MAX_CHAR_VALUE:::: +PyUnicode_MAX_CHAR_VALUE:PyObject*:o:0: + PyUnicode_AS_UNICODE:Py_UNICODE*::: PyUnicode_AS_UNICODE:PyObject*:o:0: PyUnicode_AS_DATA:const char*::: PyUnicode_AS_DATA:PyObject*:o:0: +Py_UNICODE_ISALNUM:int::: +Py_UNICODE_ISALNUM:Py_UNICODE:ch:: + +Py_UNICODE_ISALPHA:int::: +Py_UNICODE_ISALPHA:Py_UNICODE:ch:: + Py_UNICODE_ISSPACE:int::: Py_UNICODE_ISSPACE:Py_UNICODE:ch:: @@ -1570,6 +2370,9 @@ Py_UNICODE_ISDIGIT:Py_UNICODE:ch:: Py_UNICODE_ISNUMERIC:int::: Py_UNICODE_ISNUMERIC:Py_UNICODE:ch:: +Py_UNICODE_ISPRINTABLE:int::: +Py_UNICODE_ISPRINTABLE:Py_UNICODE:ch:: + Py_UNICODE_TOLOWER:Py_UNICODE::: Py_UNICODE_TOLOWER:Py_UNICODE:ch:: @@ -1590,150 +2393,210 @@ Py_UNICODE_TONUMERIC:Py_UNICODE:ch:: PyUnicode_FromUnicode:PyObject*::+1: PyUnicode_FromUnicode:const Py_UNICODE*:u:: -PyUnicode_FromUnicode:int:size:: +PyUnicode_FromUnicode:Py_ssize_t:size:: PyUnicode_AsUnicode:Py_UNICODE*::: -PyUnicode_AsUnicode:PyObject :*unicode:0: +PyUnicode_AsUnicode:PyObject*:unicode:0: + +PyUnicode_TransformDecimalToASCII:PyObject*::+1: +PyUnicode_TransformDecimalToASCII:Py_UNICODE*:s:: +PyUnicode_TransformDecimalToASCII:Py_ssize_t:size:: + +PyUnicode_AsUnicodeAndSize:Py_UNICODE*::: +PyUnicode_AsUnicodeAndSize:PyObject*:unicode:0: +PyUnicode_AsUnicodeAndSize:Py_ssize_t*:size:: + +PyUnicode_AsUnicodeCopy:Py_UNICODE*::: +PyUnicode_AsUnicodeCopy:PyObject*:unicode:0: -PyUnicode_GetSize:int::: -PyUnicode_GetSize:PyObject :*unicode:0: +PyUnicode_GetSize:Py_ssize_t::: +PyUnicode_GetSize:PyObject*:unicode:0: PyUnicode_FromObject:PyObject*::+1: -PyUnicode_FromObject:PyObject*:*obj:0: +PyUnicode_FromObject:PyObject*:obj:0: PyUnicode_FromEncodedObject:PyObject*::+1: -PyUnicode_FromEncodedObject:PyObject*:*obj:0: +PyUnicode_FromEncodedObject:PyObject*:obj:0: PyUnicode_FromEncodedObject:const char*:encoding:: PyUnicode_FromEncodedObject:const char*:errors:: PyUnicode_FromWideChar:PyObject*::+1: PyUnicode_FromWideChar:const wchar_t*:w:: -PyUnicode_FromWideChar:int:size:: +PyUnicode_FromWideChar:Py_ssize_t:size:: -PyUnicode_AsWideChar:int::: +PyUnicode_AsWideChar:Py_ssize_t::: PyUnicode_AsWideChar:PyObject*:*unicode:0: PyUnicode_AsWideChar:wchar_t*:w:: -PyUnicode_AsWideChar:int:size:: +PyUnicode_AsWideChar:Pyssize_t:size:: + +PyUnicode_AsWideCharString:wchar_t*::: +PyUnicode_AsWideCharString:PyObject*:unicode:0: +PyUnicode_AsWideCharString:Py_ssize_t*:size:: PyUnicode_Decode:PyObject*::+1: PyUnicode_Decode:const char*:s:: -PyUnicode_Decode:int:size:: +PyUnicode_Decode:Py_ssize_t:size:: PyUnicode_Decode:const char*:encoding:: PyUnicode_Decode:const char*:errors:: PyUnicode_DecodeUTF16Stateful:PyObject*::+1: PyUnicode_DecodeUTF16Stateful:const char*:s:: -PyUnicode_DecodeUTF16Stateful:int:size:: +PyUnicode_DecodeUTF16Stateful:Py_ssize_t:size:: PyUnicode_DecodeUTF16Stateful:const char*:errors:: PyUnicode_DecodeUTF16Stateful:int*:byteorder:: -PyUnicode_DecodeUTF16Stateful:int*:consumed:: +PyUnicode_DecodeUTF16Stateful:Py_ssize_t*:consumed:: PyUnicode_DecodeUTF8Stateful:PyObject*::+1: PyUnicode_DecodeUTF8Stateful:const char*:s:: -PyUnicode_DecodeUTF8Stateful:int:size:: +PyUnicode_DecodeUTF8Stateful:Py_ssize_t:size:: PyUnicode_DecodeUTF8Stateful:const char*:errors:: -PyUnicode_DecodeUTF8Stateful:int*:consumed:: +PyUnicode_DecodeUTF8Stateful:Py_ssize_t*:consumed:: PyUnicode_Encode:PyObject*::+1: PyUnicode_Encode:const Py_UNICODE*:s:: -PyUnicode_Encode:int:size:: +PyUnicode_Encode:Py_ssize_t:size:: PyUnicode_Encode:const char*:encoding:: PyUnicode_Encode:const char*:errors:: PyUnicode_AsEncodedString:PyObject*::+1: -PyUnicode_AsEncodedString:PyObject*:unicode:: +PyUnicode_AsEncodedString:PyObject*:unicode:0: PyUnicode_AsEncodedString:const char*:encoding:: PyUnicode_AsEncodedString:const char*:errors:: +PyUnicode_DecodeUTF7:PyObject*::+1: +PyUnicode_DecodeUTF7:const char*:s:: +PyUnicode_DecodeUTF7:Py_ssize_t:size:: +PyUnicode_DecodeUTF7:const char*:errors:: + +PyUnicode_DecodeUTF7Stateful:PyObject*::+1: +PyUnicode_DecodeUTF7Stateful:const char*:s:: +PyUnicode_DecodeUTF7Stateful:Py_ssize_t:size:: +PyUnicode_DecodeUTF7Stateful:const char*:errors:: +PyUnicode_DecodeUTF7Stateful:Py_ssize_t*:consumed:: + +PyUnicode_EncodeUTF7:PyObject*::+1: +PyUnicode_EncodeUTF7:const Py_UNICODE*:s:: +PyUnicode_EncodeUTF7:Py_ssize_t:size:: +PyUnicode_EncodeUTF7:int:base64SetO:: +PyUnicode_EncodeUTF7:int:base64WhiteSpace:: +PyUnicode_EncodeUTF7:const char*:errors:: + PyUnicode_DecodeUTF8:PyObject*::+1: PyUnicode_DecodeUTF8:const char*:s:: -PyUnicode_DecodeUTF8:int:size:: +PyUnicode_DecodeUTF8:Py_ssize_t:size:: PyUnicode_DecodeUTF8:const char*:errors:: PyUnicode_EncodeUTF8:PyObject*::+1: PyUnicode_EncodeUTF8:const Py_UNICODE*:s:: -PyUnicode_EncodeUTF8:int:size:: +PyUnicode_EncodeUTF8:Py_ssize_t:size:: PyUnicode_EncodeUTF8:const char*:errors:: PyUnicode_AsUTF8String:PyObject*::+1: -PyUnicode_AsUTF8String:PyObject*:unicode:: +PyUnicode_AsUTF8String:PyObject*:unicode:0: + +PyUnicode_AsUTF8AndSize:const char*::: +PyUnicode_AsUTF8AndSize:PyObject*:unicode:0: +PyUnicode_AsUTF8AndSize:Py_ssize_t*:size:0: + +PyUnicode_AsUTF8:const char*::: +PyUnicode_AsUTF8:PyObject*:unicode:0: PyUnicode_DecodeUTF16:PyObject*::+1: PyUnicode_DecodeUTF16:const char*:s:: -PyUnicode_DecodeUTF16:int:size:: +PyUnicode_DecodeUTF16:Py_ssize_t:size:: PyUnicode_DecodeUTF16:const char*:errors:: PyUnicode_DecodeUTF16:int*:byteorder:: PyUnicode_EncodeUTF16:PyObject*::+1: PyUnicode_EncodeUTF16:const Py_UNICODE*:s:: -PyUnicode_EncodeUTF16:int:size:: +PyUnicode_EncodeUTF16:Py_ssize_t:size:: PyUnicode_EncodeUTF16:const char*:errors:: PyUnicode_EncodeUTF16:int:byteorder:: PyUnicode_AsUTF16String:PyObject*::+1: -PyUnicode_AsUTF16String:PyObject*:unicode:: +PyUnicode_AsUTF16String:PyObject*:unicode:0: + +PyUnicode_DecodeUTF32:PyObject*::+1: +PyUnicode_DecodeUTF32:const char*:s:: +PyUnicode_DecodeUTF32:Py_ssize_t:size:: +PyUnicode_DecodeUTF32:const char*:errors:: +PyUnicode_DecodeUTF32:int*:byteorder:: + +PyUnicode_DecodeUTF32Stateful:PyObject*::+1: +PyUnicode_DecodeUTF32Stateful:const char*:s:: +PyUnicode_DecodeUTF32Stateful:Py_ssize_t:size:: +PyUnicode_DecodeUTF32Stateful:const char*:errors:: +PyUnicode_DecodeUTF32Stateful:int*:byteorder:: +PyUnicode_DecodeUTF32Stateful:Py_ssize_t*:consumed:: + +PyUnicode_AsUTF32String:PyObject*::+1: +PyUnicode_AsUTF32String:PyObject*:unicode:0: + +PyUnicode_EncodeUTF32:PyObject*::+1: +PyUnicode_EncodeUTF32:const Py_UNICODE*:s:: +PyUnicode_EncodeUTF32:Py_ssize_t:size:: +PyUnicode_EncodeUTF32:const char*:errors:: +PyUnicode_EncodeUTF32:int:byteorder:: PyUnicode_DecodeUnicodeEscape:PyObject*::+1: PyUnicode_DecodeUnicodeEscape:const char*:s:: -PyUnicode_DecodeUnicodeEscape:int:size:: +PyUnicode_DecodeUnicodeEscape:Py_ssize_t:size:: PyUnicode_DecodeUnicodeEscape:const char*:errors:: PyUnicode_EncodeUnicodeEscape:PyObject*::+1: PyUnicode_EncodeUnicodeEscape:const Py_UNICODE*:s:: -PyUnicode_EncodeUnicodeEscape:int:size:: -PyUnicode_EncodeUnicodeEscape:const char*:errors:: +PyUnicode_EncodeUnicodeEscape:Py_ssize_t:size:: PyUnicode_AsUnicodeEscapeString:PyObject*::+1: -PyUnicode_AsUnicodeEscapeString:PyObject*:unicode:: +PyUnicode_AsUnicodeEscapeString:PyObject*:unicode:0: PyUnicode_DecodeRawUnicodeEscape:PyObject*::+1: PyUnicode_DecodeRawUnicodeEscape:const char*:s:: -PyUnicode_DecodeRawUnicodeEscape:int:size:: +PyUnicode_DecodeRawUnicodeEscape:Py_ssize_t:size:: PyUnicode_DecodeRawUnicodeEscape:const char*:errors:: PyUnicode_EncodeRawUnicodeEscape:PyObject*::+1: PyUnicode_EncodeRawUnicodeEscape:const Py_UNICODE*:s:: -PyUnicode_EncodeRawUnicodeEscape:int:size:: -PyUnicode_EncodeRawUnicodeEscape:const char*:errors:: +PyUnicode_EncodeRawUnicodeEscape:Py_ssize_t:size:: PyUnicode_AsRawUnicodeEscapeString:PyObject*::+1: -PyUnicode_AsRawUnicodeEscapeString:PyObject*:unicode:: +PyUnicode_AsRawUnicodeEscapeString:PyObject*:unicode:0: PyUnicode_DecodeLatin1:PyObject*::+1: PyUnicode_DecodeLatin1:const char*:s:: -PyUnicode_DecodeLatin1:int:size:: +PyUnicode_DecodeLatin1:Py_ssize_t:size:: PyUnicode_DecodeLatin1:const char*:errors:: PyUnicode_EncodeLatin1:PyObject*::+1: PyUnicode_EncodeLatin1:const Py_UNICODE*:s:: -PyUnicode_EncodeLatin1:int:size:: +PyUnicode_EncodeLatin1:Py_ssize_t:size:: PyUnicode_EncodeLatin1:const char*:errors:: PyUnicode_AsLatin1String:PyObject*::+1: -PyUnicode_AsLatin1String:PyObject*:unicode:: +PyUnicode_AsLatin1String:PyObject*:unicode:0: PyUnicode_DecodeASCII:PyObject*::+1: PyUnicode_DecodeASCII:const char*:s:: -PyUnicode_DecodeASCII:int:size:: +PyUnicode_DecodeASCII:Py_ssize_t:size:: PyUnicode_DecodeASCII:const char*:errors:: PyUnicode_EncodeASCII:PyObject*::+1: PyUnicode_EncodeASCII:const Py_UNICODE*:s:: -PyUnicode_EncodeASCII:int:size:: +PyUnicode_EncodeASCII:Py_ssize_t:size:: PyUnicode_EncodeASCII:const char*:errors:: PyUnicode_AsASCIIString:PyObject*::+1: -PyUnicode_AsASCIIString:PyObject*:unicode:: +PyUnicode_AsASCIIString:PyObject*:unicode:0: PyUnicode_DecodeCharmap:PyObject*::+1: PyUnicode_DecodeCharmap:const char*:s:: -PyUnicode_DecodeCharmap:int:size:: +PyUnicode_DecodeCharmap:Py_ssize_t:size:: PyUnicode_DecodeCharmap:PyObject*:mapping:0: PyUnicode_DecodeCharmap:const char*:errors:: PyUnicode_EncodeCharmap:PyObject*::+1: PyUnicode_EncodeCharmap:const Py_UNICODE*:s:: -PyUnicode_EncodeCharmap:int:size:: +PyUnicode_EncodeCharmap:Py_ssize_t:size:: PyUnicode_EncodeCharmap:PyObject*:mapping:0: PyUnicode_EncodeCharmap:const char*:errors:: @@ -1743,22 +2606,33 @@ PyUnicode_AsCharmapString:PyObject*:mapping:0: PyUnicode_TranslateCharmap:PyObject*::+1: PyUnicode_TranslateCharmap:const Py_UNICODE*:s:: -PyUnicode_TranslateCharmap:int:size:: -PyUnicode_TranslateCharmap:PyObject*:table:0: +PyUnicode_TranslateCharmap:Py_ssize_t:size:: +PyUnicode_TranslateCharmap:PyObject*:mapping:0: PyUnicode_TranslateCharmap:const char*:errors:: PyUnicode_DecodeMBCS:PyObject*::+1: PyUnicode_DecodeMBCS:const char*:s:: -PyUnicode_DecodeMBCS:int:size:: +PyUnicode_DecodeMBCS:Py_ssize_t:size:: PyUnicode_DecodeMBCS:const char*:errors:: +PyUnicode_DecodeMBCSStateful:PyObject*::+1: +PyUnicode_DecodeMBCSStateful:const char*:s:: +PyUnicode_DecodeMBCSStateful:Py_ssize_t:size:: +PyUnicode_DecodeMBCSStateful:const char*:errors:: +PyUnicode_DecodeMBCSStateful:Py_ssize_t*:consumed:: + +PyUnicode_EncodeCodePage:PyObject*::+1: +PyUnicode_EncodeCodePage:int:code_page:: +PyUnicode_EncodeCodePage:PyObject*:unicode:0: +PyUnicode_EncodeCodePage:const char*:errors:: + PyUnicode_EncodeMBCS:PyObject*::+1: PyUnicode_EncodeMBCS:const Py_UNICODE*:s:: -PyUnicode_EncodeMBCS:int:size:: +PyUnicode_EncodeMBCS:Py_ssize_t:size:: PyUnicode_EncodeMBCS:const char*:errors:: PyUnicode_AsMBCSString:PyObject*::+1: -PyUnicode_AsMBCSString:PyObject*:unicode:: +PyUnicode_AsMBCSString:PyObject*:unicode:0: PyUnicode_Concat:PyObject*::+1: PyUnicode_Concat:PyObject*:left:0: @@ -1767,11 +2641,11 @@ PyUnicode_Concat:PyObject*:right:0: PyUnicode_Split:PyObject*::+1: PyUnicode_Split:PyObject*:left:0: PyUnicode_Split:PyObject*:right:0: -PyUnicode_Split:int:maxsplit:: +PyUnicode_Split:Py_ssize_t:maxsplit:: PyUnicode_Splitlines:PyObject*::+1: PyUnicode_Splitlines:PyObject*:s:0: -PyUnicode_Splitlines:int:maxsplit:: +PyUnicode_Splitlines:int:keepend:: PyUnicode_Translate:PyObject*::+1: PyUnicode_Translate:PyObject*:str:0: @@ -1782,36 +2656,52 @@ PyUnicode_Join:PyObject*::+1: PyUnicode_Join:PyObject*:separator:0: PyUnicode_Join:PyObject*:seq:0: -PyUnicode_Tailmatch:int::: +PyUnicode_Tailmatch:Py_ssize_t::: PyUnicode_Tailmatch:PyObject*:str:0: PyUnicode_Tailmatch:PyObject*:substr:0: -PyUnicode_Tailmatch:int:start:: -PyUnicode_Tailmatch:int:end:: +PyUnicode_Tailmatch:Py_ssize_t:start:: +PyUnicode_Tailmatch:Py_ssize_t:end:: PyUnicode_Tailmatch:int:direction:: -PyUnicode_Find:int::: +PyUnicode_Find:Py_ssize_t::: PyUnicode_Find:PyObject*:str:0: PyUnicode_Find:PyObject*:substr:0: -PyUnicode_Find:int:start:: -PyUnicode_Find:int:end:: +PyUnicode_Find:Py_ssize_t:start:: +PyUnicode_Find:Py_ssize_t:end:: PyUnicode_Find:int:direction:: -PyUnicode_Count:int::: +PyUnicode_FindChar:Py_ssize_t::: +PyUnicode_FindChar:PyObject*:str:0: +PyUnicode_FindChar:Py_UCS4:ch:: +PyUnicode_FindChar:Py_ssize_t:start:: +PyUnicode_FindChar:Py_ssize_t:end:: +PyUnicode_FindChar:int:direction:: + +PyUnicode_Count:Py_ssize_t::: PyUnicode_Count:PyObject*:str:0: PyUnicode_Count:PyObject*:substr:0: -PyUnicode_Count:int:start:: -PyUnicode_Count:int:end:: +PyUnicode_Count:Py_ssize_t:start:: +PyUnicode_Count:Py_ssize_t:end:: PyUnicode_Replace:PyObject*::+1: PyUnicode_Replace:PyObject*:str:0: PyUnicode_Replace:PyObject*:substr:0: PyUnicode_Replace:PyObject*:replstr:0: -PyUnicode_Replace:int:maxcount:: +PyUnicode_Replace:Py_ssize_t:maxcount:: PyUnicode_Compare:int::: PyUnicode_Compare:PyObject*:left:0: PyUnicode_Compare:PyObject*:right:0: +PyUnicode_CompareWithASCIIString:int::: +PyUnicode_CompareWithASCIIString:PyObject*:uni:0: +PyUnicode_CompareWithASCIIString:const char*:string:: + +PyUnicode_RichCompare:PyObject*::+1: +PyUnicode_RichCompare:PyObject*:left:0: +PyUnicode_RichCompare:PyObject*:right:0: +PyUnicode_RichCompare:int:op:: + PyUnicode_Format:PyObject*::+1: PyUnicode_Format:PyObject*:format:0: PyUnicode_Format:PyObject*:args:0: @@ -1820,6 +2710,185 @@ PyUnicode_Contains:int::: PyUnicode_Contains:PyObject*:container:0: PyUnicode_Contains:PyObject*:element:0: +PyUnicode_InternInPlace:void::: +PyUnicode_InternInPlace:PyObject**:string:+1: + +PyUnicode_InternFromString:PyObject*::+1: +PyUnicode_InternFromString:const char*:v:: + +PyUnicode_New:PyObject*::+1: +PyUnicode_New:Py_ssize_t:size:: +PyUnicode_New:Py_UCS4:maxchar:: + +PyUnicode_FromKindAndData:PyObject*::+1: +PyUnicode_FromKindAndData:int:kind:: +PyUnicode_FromKindAndData:const void*:buffer:: +PyUnicode_FromKindAndData:Py_ssize_t:size:: + +PyUnicode_FromStringAndSize:PyObject*::+1: +PyUnicode_FromStringAndSize:const char*:u:: +PyUnicode_FromStringAndSize:Py_ssize_t:size:: + +PyUnicode_FromString:PyObject*::+1: +PyUnicode_FromString:const char*:u:: + +PyUnicode_FromFormat:PyObject*::+1: +PyUnicode_FromFormat:const char*:format:: +PyUnicode_FromFormat::...:: + +PyUnicode_FromFormatV:PyObject*::+1: +PyUnicode_FromFormatV:const char*:format:: +PyUnicode_FromFormatV:va_list:args:: + +PyUnicode_GetLength:Py_ssize_t::: +PyUnicode_GetLength:PyObject*:unicode:0: + +PyUnicode_CopyCharacters:Py_ssize_t::: +PyUnicode_CopyCharacters:PyObject*:to:0: +PyUnicode_CopyCharacters:Py_ssize_t:to_start:: +PyUnicode_CopyCharacters:PyObject*:from:0: +PyUnicode_CopyCharacters:Py_ssize_t:from_start:: +PyUnicode_CopyCharacters:Py_ssize_t:how_many:: + +PyUnicode_Fill:Py_ssize_t::: +PyUnicode_Fill:PyObject*:unicode:0: +PyUnicode_Fill:Py_ssize_t:start:: +PyUnicode_Fill:Py_ssize_t:length:: +PyUnicode_Fill:Py_UCS4:fill_char:: + +PyUnicode_READ:Py_UCS4::: +PyUnicode_READ:int:kind:: +PyUnicode_READ:void*:data:: +PyUnicode_READ:Py_ssize_t:index:: + +PyUnicode_READ_CHAR:Py_UCS4::: +PyUnicode_READ_CHAR:PyObject*:o:0: +PyUnicode_READ_CHAR:Py_ssize_t:index:: + +PyUnicode_ReadChar:Py_UCS4::: +PyUnicode_ReadChar:PyObject*:unicode:0: +PyUnicode_ReadChar:Py_ssize_t:index:: + +PyUnicode_WRITE:void::: +PyUnicode_WRITE:int:kind:: +PyUnicode_WRITE:void*:data:: +PyUnicode_WRITE:Py_ssize_t:index:: +PyUnicode_WRITE:Py_UCS4:value:: + +PyUnicode_WriteChar:int::: +PyUnicode_WriteChar:PyObject*:unicode:0: +PyUnicode_WriteChar:Py_ssize_t:index:: +PyUnicode_WriteChar:Py_UCS4:character:: + +PyUnicode_READY:int::: +PyUnicode_READY:PyObject*:o:0: + +PyUnicode_Substring:PyObject*::+1: +PyUnicode_Substring:PyObject*:str:0: +PyUnicode_Substring:Py_ssize_t:start:: +PyUnicode_Substring:Py_ssize_t:end:: + +PyUnicode_AsUCS4:Py_UCS4*::: +PyUnicode_AsUCS4:PyObject*:u:0: +PyUnicode_AsUCS4:Py_UCS4*:buffer:: +PyUnicode_AsUCS4:Py_ssize_t:buflen:: +PyUnicode_AsUCS4:int:copy_null:: + +PyUnicode_AsUCS4Copy:Py_UCS4*::: +PyUnicode_AsUCS4Copy:PyObject*:u:0: + +PyUnicode_DecodeLocaleAndSize:PyObject*::+1: +PyUnicode_DecodeLocaleAndSize:const char*:str:: +PyUnicode_DecodeLocaleAndSize:Py_ssize_t:len:: +PyUnicode_DecodeLocaleAndSize:const char*:errors:: + +PyUnicode_DecodeLocale:PyObject*::+1: +PyUnicode_DecodeLocale:const char*:str:: +PyUnicode_DecodeLocale:const char*:errors:: + +PyUnicode_EncodeLocale:PyObject*::+1: +PyUnicode_EncodeLocale:PyObject*:unicode:0: +PyUnicode_EncodeLocale:const char*:errors:: + +PyUnicode_FSConverter:int::: +PyUnicode_FSConverter:PyObject*:obj:0: +PyUnicode_FSConverter:void*:result:: + +PyUnicode_FSDecoder:int::: +PyUnicode_FSDecoder:PyObject*:obj:0: +PyUnicode_FSDecoder:void*:result:: + +PyUnicode_DecodeFSDefaultAndSize:PyObject*::+1: +PyUnicode_DecodeFSDefaultAndSize:const char*:s:: +PyUnicode_DecodeFSDefaultAndSize:Py_ssize_t:size:: + +PyUnicode_DecodeFSDefault:PyObject*::+1: +PyUnicode_DecodeFSDefault:const char*:s:: + +PyUnicode_EncodeFSDefault:PyObject*::+1: +PyUnicode_EncodeFSDefault:PyObject*:unicode:0: + +PyUnicodeDecodeError_Create:PyObject*::+1: +PyUnicodeDecodeError_Create:const char*:encoding:: +PyUnicodeDecodeError_Create:const char*:object:: +PyUnicodeDecodeError_Create:Py_ssize_t:length:: +PyUnicodeDecodeError_Create:Py_ssize_t:start:: +PyUnicodeDecodeError_Create:Py_ssize_t:end:: +PyUnicodeDecodeError_Create:const char*:reason:: + +PyUnicodeDecodeError_GetEncoding:PyObject*::+1: +PyUnicodeDecodeError_GetEncoding:PyObject*:exc:0: + +PyUnicodeDecodeError_GetEnd:Py_ssize_t::: +PyUnicodeDecodeError_GetEnd:PyObject*:exc:0: +PyUnicodeDecodeError_GetEnd:Py_ssize_t*:end:: + +PyUnicodeDecodeError_GetObject:PyObject*::+1: +PyUnicodeDecodeError_GetObject:PyObject*:exc:0: + +PyUnicodeDecodeError_GetReason:PyObject*::+1: +PyUnicodeDecodeError_GetReason:PyObject*:exc:0: + +PyUnicodeDecodeError_GetStart:Py_ssize_t::: +PyUnicodeDecodeError_GetStart:PyObject*:exc:0: +PyUnicodeDecodeError_GetStart:Py_ssize_t*:start:: + +PyUnicodeDecodeError_SetEnd:int::: +PyUnicodeDecodeError_SetEnd:PyObject*:exc:0: +PyUnicodeDecodeError_SetEnd:Py_ssize_t:end:: + +PyUnicodeDecodeError_SetReason:int::: +PyUnicodeDecodeError_SetReason:PyObject*:exc:0: +PyUnicodeDecodeError_SetReason:const char*:reason:: + +PyUnicodeDecodeError_SetStart:int::: +PyUnicodeDecodeError_SetStart:PyObject*:exc:0: +PyUnicodeDecodeError_SetStart:Py_ssize_t:start:: + +PyUnicodeEncodeError_Create:PyObject*::+1: +PyUnicodeEncodeError_Create:const char*:encoding:: +PyUnicodeEncodeError_Create:const Py_UNICODE*:object:: +PyUnicodeEncodeError_Create:Py_ssize_t:length:: +PyUnicodeEncodeError_Create:Py_ssize_t:start:: +PyUnicodeEncodeError_Create:Py_ssize_t:end:: +PyUnicodeEncodeError_Create:const char*:reason:: + +PyUnicodeTranslateError_Create:PyObject*::+1: +PyUnicodeTranslateError_Create:const Py_UNICODE*:object:: +PyUnicodeTranslateError_Create:Py_ssize_t:length:: +PyUnicodeTranslateError_Create:Py_ssize_t:start:: +PyUnicodeTranslateError_Create:Py_ssize_t:end:: +PyUnicodeTranslateError_Create:const char*:reason:: + +PyWeakref_Check:int::: +PyWeakref_Check:PyObject*:ob:: + +PyWeakref_CheckProxy:int::: +PyWeakref_CheckProxy:PyObject*:ob:: + +PyWeakref_CheckRef:int::: +PyWeakref_CheckRef:PyObject*:ob:: + PyWeakref_GET_OBJECT:PyObject*::0: PyWeakref_GET_OBJECT:PyObject*:ref:0: @@ -1843,18 +2912,40 @@ Py_AtExit:void (*)():func:: Py_BuildValue:PyObject*::+1: Py_BuildValue:const char*:format:: +Py_BuildValue::...:: + +Py_VaBuildValue:PyObject*::+1: +Py_VaBuildValue:const char*:format:: +Py_VaBuildValue:va_list:vargs:: + +Py_CLEAR:void::: +Py_CLEAR:PyObject*:o:-1: Py_CompileString:PyObject*::+1: Py_CompileString:const char*:str:: Py_CompileString:const char*:filename:: Py_CompileString:int:start:: +Py_CompileStringExFlags:PyObject*::+1: +Py_CompileStringExFlags:const char*:str:: +Py_CompileStringExFlags:const char*:filename:: +Py_CompileStringExFlags:int:start:: +Py_CompileStringExFlags:PyCompilerFlags*:flags:: +Py_CompileStringExFlags:int:optimize:: + Py_CompileStringFlags:PyObject*::+1: Py_CompileStringFlags:const char*:str:: Py_CompileStringFlags:const char*:filename:: Py_CompileStringFlags:int:start:: Py_CompileStringFlags:PyCompilerFlags*:flags:: +Py_CompileStringObject:PyObject*::+1: +Py_CompileStringObject:const char*:str:: +Py_CompileStringObject:PyObject*:filename:0: +Py_CompileStringObject:int:start:: +Py_CompileStringObject:PyCompilerFlags*:flags:: +Py_CompileStringObject:int:optimize:: + Py_DECREF:void::: Py_DECREF:PyObject*:o:-1: @@ -1873,25 +2964,25 @@ Py_FdIsInteractive:const char*:filename:: Py_Finalize:void::: -Py_GetBuildInfoconst:const char*::: +Py_GetBuildInfo:const char*::: -Py_GetCompilerconst:const char*::: +Py_GetCompiler:const char*::: -Py_GetCopyrightconst:const char*::: +Py_GetCopyright:const char*::: -Py_GetExecPrefix:const char*::: +Py_GetExecPrefix:wchar_t*::: -Py_GetPath:const char*::: +Py_GetPath:wchar_t*::: -Py_GetPlatformconst:const char*::: +Py_GetPlatform:const char*::: -Py_GetPrefix:const char*::: +Py_GetPrefix:wchar_t*::: -Py_GetProgramFullPath:const char*::: +Py_GetProgramFullPath:wchar_t*::: -Py_GetProgramName:const char*::: +Py_GetProgramName:wchar_t*::: -Py_GetVersionconst:const char*::: +Py_GetVersion:const char*::: Py_INCREF:void::: Py_INCREF:PyObject*:o:+1: @@ -1902,8 +2993,14 @@ Py_IsInitialized:int::: Py_NewInterpreter:PyThreadState*::: +Py_ReprEnter:int::: +Py_ReprEnter:PyObject*:object:+1: + +Py_ReprLeave:void::: +Py_ReprLeave:PyObject*:object:-1: + Py_SetProgramName:void::: -Py_SetProgramName:const char*:name:: +Py_SetProgramName:const wchar_t*:name:: Py_XDECREF:void::: Py_XDECREF:PyObject*:o:-1:if o is not NULL @@ -1911,32 +3008,24 @@ Py_XDECREF:PyObject*:o:-1:if o is not NULL Py_XINCREF:void::: Py_XINCREF:PyObject*:o:+1:if o is not NULL -_PyImport_FindExtension:PyObject*::0:??? see PyImport_AddModule -_PyImport_FindExtension:const char*::: -_PyImport_FindExtension:const char*::: - _PyImport_Fini:void::: -_PyImport_FixupExtension:PyObject*:::??? -_PyImport_FixupExtension:const char*::: -_PyImport_FixupExtension:const char*::: - _PyImport_Init:void::: _PyObject_New:PyObject*::+1: _PyObject_New:PyTypeObject*:type:0: -_PyObject_NewVar:PyObject*::+1: +_PyObject_NewVar:PyVarObject*::+1: _PyObject_NewVar:PyTypeObject*:type:0: -_PyObject_NewVar:int:size:: +_PyObject_NewVar:Py_ssize_t:size:: -_PyString_Resize:int::: -_PyString_Resize:PyObject**:string:+1: -_PyString_Resize:int:newsize:: +_PyBytes_Resize:int::: +_PyBytes_Resize:PyObject**:bytes:0: +_PyBytes_Resize:Py_ssize_t:newsize:: _PyTuple_Resize:int::: -_PyTuple_Resize:PyTupleObject**:p:+1: -_PyTuple_Resize:int:new:: +_PyTuple_Resize:PyObject**:p:0: +_PyTuple_Resize:Py_ssize_t:new:: _Py_c_diff:Py_complex::: _Py_c_diff:Py_complex:left:: diff --git a/kbe/src/lib/python/Doc/distutils/setupscript.rst b/kbe/src/lib/python/Doc/distutils/setupscript.rst index c1051d2e80..54ed1aebc2 100644 --- a/kbe/src/lib/python/Doc/distutils/setupscript.rst +++ b/kbe/src/lib/python/Doc/distutils/setupscript.rst @@ -524,20 +524,23 @@ following way:: setup(..., data_files=[('bitmaps', ['bm/b1.gif', 'bm/b2.gif']), ('config', ['cfg/data.cfg']), - ('/etc/init.d', ['init-script'])] ) -Note that you can specify the directory names where the data files will be -installed, but you cannot rename the data files themselves. - Each (*directory*, *files*) pair in the sequence specifies the installation -directory and the files to install there. If *directory* is a relative path, it -is interpreted relative to the installation prefix (Python's ``sys.prefix`` for -pure-Python packages, ``sys.exec_prefix`` for packages that contain extension -modules). Each file name in *files* is interpreted relative to the -:file:`setup.py` script at the top of the package source distribution. No -directory information from *files* is used to determine the final location of -the installed file; only the name of the file is used. +directory and the files to install there. + +Each file name in *files* is interpreted relative to the :file:`setup.py` +script at the top of the package source distribution. Note that you can +specify the directory where the data files will be installed, but you cannot +rename the data files themselves. + +The *directory* should be a relative path. It is interpreted relative to the +installation prefix (Python's ``sys.prefix`` for system installations; +``site.USER_BASE`` for user installations). Distutils allows *directory* to be +an absolute installation path, but this is discouraged since it is +incompatible with the wheel packaging format. No directory information from +*files* is used to determine the final location of the installed file; only +the name of the file is used. You can specify the ``data_files`` options as a simple sequence of files without specifying a target directory, but this is not recommended, and the diff --git a/kbe/src/lib/python/Doc/faq/general.rst b/kbe/src/lib/python/Doc/faq/general.rst index 90fd69e729..7ee340d795 100644 --- a/kbe/src/lib/python/Doc/faq/general.rst +++ b/kbe/src/lib/python/Doc/faq/general.rst @@ -268,14 +268,8 @@ Python references; or perhaps search for "Python" and "language". Where in the world is www.python.org located? --------------------------------------------- -The Python project's infrastructure is located all over the world. -`www.python.org `_ is graciously hosted by `Rackspace -`_, with CDN caching provided by `Fastly -`_. `Upfront Systems -`_ hosts `bugs.python.org -`_. Many other Python services like `the Wiki -`_ are hosted by `Oregon State -University Open Source Lab `_. +The Python project's infrastructure is located all over the world and is managed +by the Python Infrastructure Team. Details `here `__. Why is it called Python? diff --git a/kbe/src/lib/python/Doc/faq/programming.rst b/kbe/src/lib/python/Doc/faq/programming.rst index fd720c1a30..31614189a6 100644 --- a/kbe/src/lib/python/Doc/faq/programming.rst +++ b/kbe/src/lib/python/Doc/faq/programming.rst @@ -738,7 +738,7 @@ Is it possible to write obfuscated one-liners in Python? -------------------------------------------------------- Yes. Usually this is done by nesting :keyword:`lambda` within -:keyword:`lambda`. See the following three examples, due to Ulf Bartelt:: +:keyword:`!lambda`. See the following three examples, due to Ulf Bartelt:: from functools import reduce @@ -767,6 +767,41 @@ Yes. Usually this is done by nesting :keyword:`lambda` within Don't try this at home, kids! +.. _faq-positional-only-arguments: + +What does the slash(/) in the parameter list of a function mean? +---------------------------------------------------------------- + +A slash in the argument list of a function denotes that the parameters prior to +it are positional-only. Positional-only parameters are the ones without an +externally-usable name. Upon calling a function that accepts positional-only +parameters, arguments are mapped to parameters based solely on their position. +For example, :func:`pow` is a function that accepts positional-only parameters. +Its documentation looks like this:: + + >>> help(pow) + Help on built-in function pow in module builtins: + + pow(x, y, z=None, /) + Equivalent to x**y (with two arguments) or x**y % z (with three arguments) + + Some types, such as ints, are able to use a more efficient algorithm when + invoked using the three argument form. + +The slash at the end of the parameter list means that all three parameters are +positional-only. Thus, calling :func:`pow` with keyword aguments would lead to +an error:: + + >>> pow(x=3, y=4) + Traceback (most recent call last): + File "", line 1, in + TypeError: pow() takes no keyword arguments + +Note that as of this writing this is only documentational and no valid syntax +in Python, although there is :pep:`570`, which proposes a syntax for +position-only parameters in Python. + + Numbers and strings =================== @@ -1317,9 +1352,6 @@ The ``__iadd__`` succeeds, and thus the list is extended, but even though that final assignment still results in an error, because tuples are immutable. -Dictionaries -============ - I want to do a complicated sort: can you do a Schwartzian Transform in Python? ------------------------------------------------------------------------------ diff --git a/kbe/src/lib/python/Doc/glossary.rst b/kbe/src/lib/python/Doc/glossary.rst index b8d98dd34c..fb8ff2a7c6 100644 --- a/kbe/src/lib/python/Doc/glossary.rst +++ b/kbe/src/lib/python/Doc/glossary.rst @@ -332,7 +332,7 @@ Glossary names, attribute access, operators or function calls which all return a value. In contrast to many other languages, not all language constructs are expressions. There are also :term:`statement`\s which cannot be used - as expressions, such as :keyword:`if`. Assignments are also statements, + as expressions, such as :keyword:`while`. Assignments are also statements, not expressions. extension module @@ -444,8 +444,8 @@ Glossary generator expression An expression that returns an iterator. It looks like a normal expression - followed by a :keyword:`for` expression defining a loop variable, range, - and an optional :keyword:`if` expression. The combined expression + followed by a :keyword:`!for` clause defining a loop variable, range, + and an optional :keyword:`!if` clause. The combined expression generates values for an enclosing function:: >>> sum(i*i for i in range(10)) # sum of squares 0, 1, 4, ... 81 diff --git a/kbe/src/lib/python/Doc/howto/functional.rst b/kbe/src/lib/python/Doc/howto/functional.rst index 2efe4537e2..f8f2aac70f 100644 --- a/kbe/src/lib/python/Doc/howto/functional.rst +++ b/kbe/src/lib/python/Doc/howto/functional.rst @@ -1108,7 +1108,7 @@ need to define a new function at all:: existing_files = filter(os.path.exists, file_list) If the function you need doesn't exist, you need to write it. One way to write -small functions is to use the :keyword:`lambda` statement. ``lambda`` takes a +small functions is to use the :keyword:`lambda` expression. ``lambda`` takes a number of parameters and an expression combining these parameters, and creates an anonymous function that returns the value of the expression:: diff --git a/kbe/src/lib/python/Doc/howto/logging-cookbook.rst b/kbe/src/lib/python/Doc/howto/logging-cookbook.rst index b1930a791f..e391506ce2 100644 --- a/kbe/src/lib/python/Doc/howto/logging-cookbook.rst +++ b/kbe/src/lib/python/Doc/howto/logging-cookbook.rst @@ -186,7 +186,7 @@ previous simple module-based configuration example:: # 'application' code logger.debug('debug message') logger.info('info message') - logger.warn('warn message') + logger.warning('warn message') logger.error('error message') logger.critical('critical message') @@ -295,7 +295,7 @@ Here is an example of a module using the logging configuration server:: while True: logger.debug('debug message') logger.info('info message') - logger.warn('warn message') + logger.warning('warn message') logger.error('error message') logger.critical('critical message') time.sleep(5) diff --git a/kbe/src/lib/python/Doc/howto/logging.rst b/kbe/src/lib/python/Doc/howto/logging.rst index 2a2282e9ec..7a68ca8919 100644 --- a/kbe/src/lib/python/Doc/howto/logging.rst +++ b/kbe/src/lib/python/Doc/howto/logging.rst @@ -610,7 +610,7 @@ logger, a console handler, and a simple formatter using Python code:: # 'application' code logger.debug('debug message') logger.info('info message') - logger.warn('warn message') + logger.warning('warn message') logger.error('error message') logger.critical('critical message') @@ -640,7 +640,7 @@ the names of the objects:: # 'application' code logger.debug('debug message') logger.info('info message') - logger.warn('warn message') + logger.warning('warn message') logger.error('error message') logger.critical('critical message') @@ -695,15 +695,15 @@ noncoders to easily modify the logging properties. .. warning:: The :func:`fileConfig` function takes a default parameter, ``disable_existing_loggers``, which defaults to ``True`` for reasons of backward compatibility. This may or may not be what you want, since it - will cause any loggers existing before the :func:`fileConfig` call to - be disabled unless they (or an ancestor) are explicitly named in the - configuration. Please refer to the reference documentation for more + will cause any non-root loggers existing before the :func:`fileConfig` + call to be disabled unless they (or an ancestor) are explicitly named in + the configuration. Please refer to the reference documentation for more information, and specify ``False`` for this parameter if you wish. The dictionary passed to :func:`dictConfig` can also specify a Boolean value with key ``disable_existing_loggers``, which if not specified explicitly in the dictionary also defaults to being interpreted as - ``True``. This leads to the logger-disabling behaviour described above, + ``True``. This leads to the logger-disabling behaviour described above, which may not be what you want - in which case, provide the key explicitly with a value of ``False``. @@ -802,7 +802,7 @@ the best default behaviour. If for some reason you *don't* want these messages printed in the absence of any logging configuration, you can attach a do-nothing handler to the top-level logger for your library. This avoids the message being printed, since a handler -will be always be found for the library's events: it just doesn't produce any +will always be found for the library's events: it just doesn't produce any output. If the library user configures logging for application use, presumably that configuration will add some handlers, and if levels are suitably configured then logging calls made in library code will send output to those diff --git a/kbe/src/lib/python/Doc/howto/regex.rst b/kbe/src/lib/python/Doc/howto/regex.rst index b09f748a92..d385d99134 100644 --- a/kbe/src/lib/python/Doc/howto/regex.rst +++ b/kbe/src/lib/python/Doc/howto/regex.rst @@ -96,8 +96,9 @@ special nature. You can match the characters not listed within the class by :dfn:`complementing` the set. This is indicated by including a ``'^'`` as the first character of the -class; ``'^'`` outside a character class will simply match the ``'^'`` -character. For example, ``[^5]`` will match any character except ``'5'``. +class. For example, ``[^5]`` will match any character except ``'5'``. If the +caret appears elsewhere in a character class, it does not have special meaning. +For example: ``[5^]`` will match either a ``'5'`` or a ``'^'``. Perhaps the most important metacharacter is the backslash, ``\``. As in Python string literals, the backslash can be followed by various characters to signal diff --git a/kbe/src/lib/python/Doc/howto/unicode.rst b/kbe/src/lib/python/Doc/howto/unicode.rst index be1fefb35a..5339bf45bf 100644 --- a/kbe/src/lib/python/Doc/howto/unicode.rst +++ b/kbe/src/lib/python/Doc/howto/unicode.rst @@ -6,95 +6,48 @@ :Release: 1.12 -This HOWTO discusses Python support for Unicode, and explains -various problems that people commonly encounter when trying to work -with Unicode. +This HOWTO discusses Python's support for the Unicode specification +for representing textual data, and explains various problems that +people commonly encounter when trying to work with Unicode. + Introduction to Unicode ======================= -History of Character Codes --------------------------- - -In 1968, the American Standard Code for Information Interchange, better known by -its acronym ASCII, was standardized. ASCII defined numeric codes for various -characters, with the numeric values running from 0 to 127. For example, the -lowercase letter 'a' is assigned 97 as its code value. - -ASCII was an American-developed standard, so it only defined unaccented -characters. There was an 'e', but no 'é' or 'Í'. This meant that languages -which required accented characters couldn't be faithfully represented in ASCII. -(Actually the missing accents matter for English, too, which contains words such -as 'naïve' and 'café', and some publications have house styles which require -spellings such as 'coöperate'.) - -For a while people just wrote programs that didn't display accents. -In the mid-1980s an Apple II BASIC program written by a French speaker -might have lines like these: - -.. code-block:: basic - - PRINT "MISE A JOUR TERMINEE" - PRINT "PARAMETRES ENREGISTRES" - -Those messages should contain accents (terminée, paramètre, enregistrés) and -they just look wrong to someone who can read French. - -In the 1980s, almost all personal computers were 8-bit, meaning that bytes could -hold values ranging from 0 to 255. ASCII codes only went up to 127, so some -machines assigned values between 128 and 255 to accented characters. Different -machines had different codes, however, which led to problems exchanging files. -Eventually various commonly used sets of values for the 128--255 range emerged. -Some were true standards, defined by the International Organization for -Standardization, and some were *de facto* conventions that were invented by one -company or another and managed to catch on. - -255 characters aren't very many. For example, you can't fit both the accented -characters used in Western Europe and the Cyrillic alphabet used for Russian -into the 128--255 range because there are more than 128 such characters. - -You could write files using different codes (all your Russian files in a coding -system called KOI8, all your French files in a different coding system called -Latin1), but what if you wanted to write a French document that quotes some -Russian text? In the 1980s people began to want to solve this problem, and the -Unicode standardization effort began. - -Unicode started out using 16-bit characters instead of 8-bit characters. 16 -bits means you have 2^16 = 65,536 distinct values available, making it possible -to represent many different characters from many different alphabets; an initial -goal was to have Unicode contain the alphabets for every single human language. -It turns out that even 16 bits isn't enough to meet that goal, and the modern -Unicode specification uses a wider range of codes, 0 through 1,114,111 ( -``0x10FFFF`` in base 16). - -There's a related ISO standard, ISO 10646. Unicode and ISO 10646 were -originally separate efforts, but the specifications were merged with the 1.1 -revision of Unicode. - -(This discussion of Unicode's history is highly simplified. The -precise historical details aren't necessary for understanding how to -use Unicode effectively, but if you're curious, consult the Unicode -consortium site listed in the References or -the `Wikipedia entry for Unicode `_ -for more information.) - - Definitions ----------- +Today's programs need to be able to handle a wide variety of +characters. Applications are often internationalized to display +messages and output in a variety of user-selectable languages; the +same program might need to output an error message in English, French, +Japanese, Hebrew, or Russian. Web content can be written in any of +these languages and can also include a variety of emoji symbols. +Python's string type uses the Unicode Standard for representing +characters, which lets Python programs work with all these different +possible characters. + +Unicode (https://www.unicode.org/) is a specification that aims to +list every character used by human languages and give each character +its own unique code. The Unicode specifications are continually +revised and updated to add new languages and symbols. + A **character** is the smallest possible component of a text. 'A', 'B', 'C', -etc., are all different characters. So are 'È' and 'Í'. Characters are -abstractions, and vary depending on the language or context you're talking -about. For example, the symbol for ohms (Ω) is usually drawn much like the -capital letter omega (Ω) in the Greek alphabet (they may even be the same in -some fonts), but these are two different characters that have different -meanings. - -The Unicode standard describes how characters are represented by **code -points**. A code point is an integer value, usually denoted in base 16. In the -standard, a code point is written using the notation ``U+12CA`` to mean the -character with value ``0x12ca`` (4,810 decimal). The Unicode standard contains -a lot of tables listing characters and their corresponding code points: +etc., are all different characters. So are 'È' and 'Í'. Characters vary +depending on the language or context you're talking +about. For example, there's a character for "Roman Numeral One", 'Ⅰ', that's +separate from the uppercase letter 'I'. They'll usually look the same, +but these are two different characters that have different meanings. + +The Unicode standard describes how characters are represented by +**code points**. A code point value is an integer in the range 0 to +0x10FFFF (about 1.1 million values, with some 110 thousand assigned so +far). In the standard and in this document, a code point is written +using the notation ``U+265E`` to mean the character with value +``0x265e`` (9,822 in decimal). + +The Unicode standard contains a lot of tables listing characters and +their corresponding code points: .. code-block:: none @@ -103,10 +56,21 @@ a lot of tables listing characters and their corresponding code points: 0063 'c'; LATIN SMALL LETTER C ... 007B '{'; LEFT CURLY BRACKET + ... + 2167 'Ⅶ': ROMAN NUMERAL EIGHT + 2168 'Ⅸ': ROMAN NUMERAL NINE + ... + 265E '♞': BLACK CHESS KNIGHT + 265F '♟': BLACK CHESS PAWN + ... + 1F600 '😀': GRINNING FACE + 1F609 '😉': WINKING FACE + ... Strictly, these definitions imply that it's meaningless to say 'this is -character ``U+12CA``'. ``U+12CA`` is a code point, which represents some particular -character; in this case, it represents the character 'ETHIOPIC SYLLABLE WI'. In +character ``U+265E``'. ``U+265E`` is a code point, which represents some particular +character; in this case, it represents the character 'BLACK CHESS KNIGHT', +'♞'. In informal contexts, this distinction between code points and characters will sometimes be forgotten. @@ -121,14 +85,17 @@ toolkit or a terminal's font renderer. Encodings --------- -To summarize the previous section: a Unicode string is a sequence of code -points, which are numbers from 0 through ``0x10FFFF`` (1,114,111 decimal). This -sequence needs to be represented as a set of bytes (meaning, values -from 0 through 255) in memory. The rules for translating a Unicode string -into a sequence of bytes are called an **encoding**. +To summarize the previous section: a Unicode string is a sequence of +code points, which are numbers from 0 through ``0x10FFFF`` (1,114,111 +decimal). This sequence of code points needs to be represented in +memory as a set of **code units**, and **code units** are then mapped +to 8-bit bytes. The rules for translating a Unicode string into a +sequence of bytes are called a **character encoding**, or just +an **encoding**. -The first encoding you might think of is an array of 32-bit integers. In this -representation, the string "Python" would look like this: +The first encoding you might think of is using 32-bit integers as the +code unit, and then using the CPU's representation of 32-bit integers. +In this representation, the string "Python" might look like this: .. code-block:: none @@ -152,40 +119,14 @@ problems. 3. It's not compatible with existing C functions such as ``strlen()``, so a new family of wide string functions would need to be used. -4. Many Internet standards are defined in terms of textual data, and can't - handle content with embedded zero bytes. - -Generally people don't use this encoding, instead choosing other -encodings that are more efficient and convenient. UTF-8 is probably -the most commonly supported encoding; it will be discussed below. - -Encodings don't have to handle every possible Unicode character, and most -encodings don't. The rules for converting a Unicode string into the ASCII -encoding, for example, are simple; for each code point: - -1. If the code point is < 128, each byte is the same as the value of the code - point. +Therefore this encoding isn't used very much, and people instead choose other +encodings that are more efficient and convenient, such as UTF-8. -2. If the code point is 128 or greater, the Unicode string can't be represented - in this encoding. (Python raises a :exc:`UnicodeEncodeError` exception in this - case.) - -Latin-1, also known as ISO-8859-1, is a similar encoding. Unicode code points -0--255 are identical to the Latin-1 values, so converting to this encoding simply -requires converting code points to byte values; if a code point larger than 255 -is encountered, the string can't be encoded into Latin-1. - -Encodings don't have to be simple one-to-one mappings like Latin-1. Consider -IBM's EBCDIC, which was used on IBM mainframes. Letter values weren't in one -block: 'a' through 'i' had values from 129 to 137, but 'j' through 'r' were 145 -through 153. If you wanted to use EBCDIC as an encoding, you'd probably use -some sort of lookup table to perform the conversion, but this is largely an -internal detail. - -UTF-8 is one of the most commonly used encodings. UTF stands for "Unicode -Transformation Format", and the '8' means that 8-bit numbers are used in the -encoding. (There are also a UTF-16 and UTF-32 encodings, but they are less -frequently used than UTF-8.) UTF-8 uses the following rules: +UTF-8 is one of the most commonly used encodings, and Python often +defaults to using it. UTF stands for "Unicode Transformation Format", +and the '8' means that 8-bit values are used in the encoding. (There +are also UTF-16 and UTF-32 encodings, but they are less frequently +used than UTF-8.) UTF-8 uses the following rules: 1. If the code point is < 128, it's represented by the corresponding byte value. 2. If the code point is >= 128, it's turned into a sequence of two, three, or @@ -215,6 +156,10 @@ glossary, and PDF versions of the Unicode specification. Be prepared for some difficult reading. `A chronology `_ of the origin and development of Unicode is also available on the site. +On the Computerphile Youtube channel, Tom Scott briefly +`discusses the history of Unicode and UTF-8 ` +(9 minutes 36 seconds). + To help understand the standard, Jukka Korpela has written `an introductory guide `_ to reading the Unicode character tables. @@ -238,7 +183,7 @@ Unicode features. The String Type --------------- -Since Python 3.0, the language features a :class:`str` type that contain Unicode +Since Python 3.0, the language's :class:`str` type contains Unicode characters, meaning any string created using ``"unicode rocks!"``, ``'unicode rocks!'``, or the triple-quoted string syntax is stored as Unicode. @@ -252,11 +197,6 @@ include a Unicode character in a string literal:: # 'File not found' error message. print("Fichier non trouvé") -You can use a different encoding from UTF-8 by putting a specially-formatted -comment as the first or second line of the source code:: - - # -*- coding: -*- - Side note: Python 3 also supports using Unicode characters in identifiers:: répertoire = "/tmp/records.log" @@ -299,7 +239,7 @@ The following examples show the differences:: >>> b'\x80abc'.decode("utf-8", "ignore") 'abc' -Encodings are specified as strings containing the encoding's name. Python 3.2 +Encodings are specified as strings containing the encoding's name. Python comes with roughly 100 different encodings; see the Python Library Reference at :ref:`standard-encodings` for a list. Some encodings have multiple names; for example, ``'latin-1'``, ``'iso_8859_1'`` and ``'8859``' are all synonyms for @@ -409,12 +349,13 @@ already mentioned. See also :pep:`263` for more information. Unicode Properties ------------------ -The Unicode specification includes a database of information about code points. -For each defined code point, the information includes the character's -name, its category, the numeric value if applicable (Unicode has characters -representing the Roman numerals and fractions such as one-third and -four-fifths). There are also properties related to the code point's use in -bidirectional text and other display-related properties. +The Unicode specification includes a database of information about +code points. For each defined code point, the information includes +the character's name, its category, the numeric value if applicable +(for characters representing numeric concepts such as the Roman +numerals, fractions such as one-third and four-fifths, etc.). There +are also display-related properties, such as how to use the code point +in bidirectional text. The following program displays some information about several characters, and prints the numeric value of one particular character:: @@ -451,6 +392,88 @@ other". See list of category codes. +Comparing Strings +----------------- + +Unicode adds some complication to comparing strings, because the same +set of characters can be represented by different sequences of code +points. For example, a letter like 'ê' can be represented as a single +code point U+00EA, or as U+0065 U+0302, which is the code point for +'e' followed by a code point for 'COMBINING CIRCUMFLEX ACCENT'. These +will produce the same output when printed, but one is a string of +length 1 and the other is of length 2. + +One tool for a case-insensitive comparison is the +:meth:`~str.casefold` string method that converts a string to a +case-insensitive form following an algorithm described by the Unicode +Standard. This algorithm has special handling for characters such as +the German letter 'ß' (code point U+00DF), which becomes the pair of +lowercase letters 'ss'. + +:: + + >>> street = 'Gürzenichstraße' + >>> street.casefold() + 'gürzenichstrasse' + +A second tool is the :mod:`unicodedata` module's +:func:`~unicodedata.normalize` function that converts strings to one +of several normal forms, where letters followed by a combining +character are replaced with single characters. :func:`normalize` can +be used to perform string comparisons that won't falsely report +inequality if two strings use combining characters differently: + +:: + + import unicodedata + + def compare_strs(s1, s2): + def NFD(s): + return unicodedata.normalize('NFD', s) + + return NFD(s1) == NFD(s2) + + single_char = 'ê' + multiple_chars = '\N{LATIN SMALL LETTER E}\N{COMBINING CIRCUMFLEX ACCENT}' + print('length of first string=', len(single_char)) + print('length of second string=', len(multiple_chars)) + print(compare_strs(single_char, multiple_chars)) + +When run, this outputs: + +.. code-block:: shell-session + + $ python3 compare-strs.py + length of first string= 1 + length of second string= 2 + True + +The first argument to the :func:`~unicodedata.normalize` function is a +string giving the desired normalization form, which can be one of +'NFC', 'NFKC', 'NFD', and 'NFKD'. + +The Unicode Standard also specifies how to do caseless comparisons:: + + import unicodedata + + def compare_caseless(s1, s2): + def NFD(s): + return unicodedata.normalize('NFD', s) + + return NFD(NFD(s1).casefold()) == NFD(NFD(s2).casefold()) + + # Example usage + single_char = 'ê' + multiple_chars = '\N{LATIN CAPITAL LETTER E}\N{COMBINING CIRCUMFLEX ACCENT}' + + print(compare_caseless(single_char, multiple_chars)) + +This will print ``True``. (Why is :func:`NFD` invoked twice? Because +there are a few characters that make :meth:`casefold` return a +non-normalized string, so the result needs to be normalized again. See +section 3.13 of the Unicode Standard for a discussion and an example.) + + Unicode Regular Expressions --------------------------- @@ -567,22 +590,22 @@ particular byte ordering and don't skip the BOM. In some areas, it is also convention to use a "BOM" at the start of UTF-8 encoded files; the name is misleading since UTF-8 is not byte-order dependent. -The mark simply announces that the file is encoded in UTF-8. Use the -'utf-8-sig' codec to automatically skip the mark if present for reading such -files. +The mark simply announces that the file is encoded in UTF-8. For reading such +files, use the 'utf-8-sig' codec to automatically skip the mark if present. Unicode filenames ----------------- -Most of the operating systems in common use today support filenames that contain -arbitrary Unicode characters. Usually this is implemented by converting the -Unicode string into some encoding that varies depending on the system. For -example, Mac OS X uses UTF-8 while Windows uses a configurable encoding; on -Windows, Python uses the name "mbcs" to refer to whatever the currently -configured encoding is. On Unix systems, there will only be a filesystem -encoding if you've set the ``LANG`` or ``LC_CTYPE`` environment variables; if -you haven't, the default encoding is UTF-8. +Most of the operating systems in common use today support filenames +that contain arbitrary Unicode characters. Usually this is +implemented by converting the Unicode string into some encoding that +varies depending on the system. Today Python is converging on using +UTF-8: Python on MacOS has used UTF-8 for several versions, and Python +3.6 switched to using UTF-8 on Windows as well. On Unix systems, +there will only be a filesystem encoding if you've set the ``LANG`` or +``LC_CTYPE`` environment variables; if you haven't, the default +encoding is again UTF-8. The :func:`sys.getfilesystemencoding` function returns the encoding to use on your current system, in case you want to do the encoding manually, but there's @@ -597,9 +620,9 @@ automatically converted to the right encoding for you:: Functions in the :mod:`os` module such as :func:`os.stat` will also accept Unicode filenames. -The :func:`os.listdir` function returns filenames and raises an issue: should it return +The :func:`os.listdir` function returns filenames, which raises an issue: should it return the Unicode version of filenames, or should it return bytes containing -the encoded versions? :func:`os.listdir` will do both, depending on whether you +the encoded versions? :func:`os.listdir` can do both, depending on whether you provided the directory path as bytes or a Unicode string. If you pass a Unicode string as the path, filenames will be decoded using the filesystem's encoding and a list of Unicode strings will be returned, while passing a byte @@ -619,16 +642,17 @@ will produce the following output: .. code-block:: shell-session - amk:~$ python t.py + $ python listdir-test.py [b'filename\xe4\x94\x80abc', ...] ['filename\u4500abc', ...] The first list contains UTF-8-encoded filenames, and the second list contains the Unicode versions. -Note that on most occasions, the Unicode APIs should be used. The bytes APIs -should only be used on systems where undecodable file names can be present, -i.e. Unix systems. +Note that on most occasions, you should can just stick with using +Unicode with these APIs. The bytes APIs should only be used on +systems where undecodable file names can be present; that's +pretty much only Unix systems now. Tips for Writing Unicode-aware Programs @@ -695,10 +719,10 @@ with the ``surrogateescape`` error handler:: f.write(data) The ``surrogateescape`` error handler will decode any non-ASCII bytes -as code points in the Unicode Private Use Area ranging from U+DC80 to -U+DCFF. These private code points will then be turned back into the -same bytes when the ``surrogateescape`` error handler is used when -encoding the data and writing it back out. +as code points in a special range running from U+DC80 to +U+DCFF. These code points will then turn back into the +same bytes when the ``surrogateescape`` error handler is used to +encode the data and write it back out. References @@ -730,4 +754,5 @@ Andrew Kuchling, and Ezio Melotti. Thanks to the following people who have noted errors or offered suggestions on this article: Éric Araujo, Nicholas Bastin, Nick Coghlan, Marius Gedminas, Kent Johnson, Ken Krugler, Marc-André -Lemburg, Martin von Löwis, Terry J. Reedy, Chad Whitacre. +Lemburg, Martin von Löwis, Terry J. Reedy, Serhiy Storchaka, +Eryk Sun, Chad Whitacre, Graham Wideman. diff --git a/kbe/src/lib/python/Doc/library/aifc.rst b/kbe/src/lib/python/Doc/library/aifc.rst index 970a7aeb98..7328907730 100644 --- a/kbe/src/lib/python/Doc/library/aifc.rst +++ b/kbe/src/lib/python/Doc/library/aifc.rst @@ -45,7 +45,7 @@ Module :mod:`aifc` defines the following function: time how many samples you are going to write in total and use :meth:`writeframesraw` and :meth:`setnframes`. The :func:`.open` function may be used in a :keyword:`with` statement. When - the :keyword:`with` block completes, the :meth:`~aifc.close` method is called. + the :keyword:`!with` block completes, the :meth:`~aifc.close` method is called. .. versionchanged:: 3.4 Support for the :keyword:`with` statement was added. diff --git a/kbe/src/lib/python/Doc/library/asyncio-eventloop.rst b/kbe/src/lib/python/Doc/library/asyncio-eventloop.rst index 647b7fc5e7..d59cf055b6 100644 --- a/kbe/src/lib/python/Doc/library/asyncio-eventloop.rst +++ b/kbe/src/lib/python/Doc/library/asyncio-eventloop.rst @@ -960,12 +960,20 @@ Unix signals Set *callback* as the handler for the *signum* signal. + The callback will be invoked by *loop*, along with other queued callbacks + and runnable coroutines of that event loop. Unlike signal handlers + registered using :func:`signal.signal`, a callback registered with this + function is allowed to interact with the event loop. + Raise :exc:`ValueError` if the signal number is invalid or uncatchable. Raise :exc:`RuntimeError` if there is a problem setting up the handler. Use :func:`functools.partial` :ref:`to pass keyword arguments ` to *callback*. + Like :func:`signal.signal`, this function must be invoked in the main + thread. + .. method:: loop.remove_signal_handler(sig) Remove the handler for the *sig* signal. diff --git a/kbe/src/lib/python/Doc/library/asyncio-exceptions.rst b/kbe/src/lib/python/Doc/library/asyncio-exceptions.rst index dbd5df7208..e49577a203 100644 --- a/kbe/src/lib/python/Doc/library/asyncio-exceptions.rst +++ b/kbe/src/lib/python/Doc/library/asyncio-exceptions.rst @@ -65,11 +65,11 @@ Exceptions .. exception:: IncompleteReadError - The requested read operation did not complete fully. + The requested read operation did not complete fully. - Raised by the :ref:`asyncio stream APIs`. + Raised by the :ref:`asyncio stream APIs`. - This exception is a subclass of :exc:`EOFError`. + This exception is a subclass of :exc:`EOFError`. .. attribute:: expected diff --git a/kbe/src/lib/python/Doc/library/asyncio-policy.rst b/kbe/src/lib/python/Doc/library/asyncio-policy.rst index cab25934d7..07842daa28 100644 --- a/kbe/src/lib/python/Doc/library/asyncio-policy.rst +++ b/kbe/src/lib/python/Doc/library/asyncio-policy.rst @@ -81,7 +81,7 @@ The abstract event loop policy base class is defined as follows: .. method:: set_child_watcher(watcher) - Get the current child process watcher to *watcher*. + Set the current child process watcher to *watcher*. This function is Unix specific. diff --git a/kbe/src/lib/python/Doc/library/asyncio-queue.rst b/kbe/src/lib/python/Doc/library/asyncio-queue.rst index bd0e70c0d9..7be1023c80 100644 --- a/kbe/src/lib/python/Doc/library/asyncio-queue.rst +++ b/kbe/src/lib/python/Doc/library/asyncio-queue.rst @@ -64,7 +64,7 @@ Queue Block until all items in the queue have been received and processed. The count of unfinished tasks goes up whenever an item is added - to the queue. The count goes down whenever a consumer thread calls + to the queue. The count goes down whenever a consumer coroutine calls :meth:`task_done` to indicate that the item was retrieved and all work on it is complete. When the count of unfinished tasks drops to zero, :meth:`join` unblocks. diff --git a/kbe/src/lib/python/Doc/library/asyncio-task.rst b/kbe/src/lib/python/Doc/library/asyncio-task.rst index 376de8f404..9e685b1764 100644 --- a/kbe/src/lib/python/Doc/library/asyncio-task.rst +++ b/kbe/src/lib/python/Doc/library/asyncio-task.rst @@ -604,7 +604,7 @@ Scheduling From Other Threads See the :ref:`concurrency and multithreading ` section of the documentation. - Unlike other asyncio functions this functions requires the *loop* + Unlike other asyncio functions this function requires the *loop* argument to be passed explicitly. .. versionadded:: 3.5.1 diff --git a/kbe/src/lib/python/Doc/library/binascii.rst b/kbe/src/lib/python/Doc/library/binascii.rst index a4efef8465..89ecddc778 100644 --- a/kbe/src/lib/python/Doc/library/binascii.rst +++ b/kbe/src/lib/python/Doc/library/binascii.rst @@ -152,6 +152,8 @@ The :mod:`binascii` module defines the following functions: *data* is converted into the corresponding 2-digit hex representation. The returned bytes object is therefore twice as long as the length of *data*. + Similar functionality (but returning a text string) is also conveniently + accessible using the :meth:`bytes.hex` method. .. function:: a2b_hex(hexstr) unhexlify(hexstr) @@ -161,6 +163,9 @@ The :mod:`binascii` module defines the following functions: of hexadecimal digits (which can be upper or lower case), otherwise an :exc:`Error` exception is raised. + Similar functionality (accepting only text string arguments, but more + liberal towards whitespace) is also accessible using the + :meth:`bytes.fromhex` class method. .. exception:: Error diff --git a/kbe/src/lib/python/Doc/library/collections.rst b/kbe/src/lib/python/Doc/library/collections.rst index 016e56c4bd..0413f46922 100644 --- a/kbe/src/lib/python/Doc/library/collections.rst +++ b/kbe/src/lib/python/Doc/library/collections.rst @@ -100,6 +100,21 @@ The class can be used to simulate nested scopes and is useful in templating. :func:`super` function. A reference to ``d.parents`` is equivalent to: ``ChainMap(*d.maps[1:])``. + Note, the iteration order of a :class:`ChainMap()` is determined by + scanning the mappings last to first:: + + >>> baseline = {'music': 'bach', 'art': 'rembrandt'} + >>> adjustments = {'art': 'van gogh', 'opera': 'carmen'} + >>> list(ChainMap(adjustments, baseline)) + ['music', 'art', 'opera'] + + This gives the same ordering as a series of :meth:`dict.update` calls + starting with the last mapping:: + + >>> combined = baseline.copy() + >>> combined.update(adjustments) + >>> list(combined) + ['music', 'art', 'opera'] .. seealso:: @@ -163,8 +178,8 @@ contexts:: e.maps[-1] # Root context -- like Python's globals() e.parents # Enclosing context chain -- like Python's nonlocals - d['x'] # Get first key in the chain of contexts d['x'] = 1 # Set value in current context + d['x'] # Get first key in the chain of contexts del d['x'] # Delete from current context list(d) # All nested values k in d # Check all nested values @@ -887,7 +902,7 @@ field names, the method and attribute names start with an underscore. .. method:: somenamedtuple._asdict() - Return a new :class:`OrderedDict` which maps field names to their corresponding + Return a new :class:`dict` which maps field names to their corresponding values: .. doctest:: @@ -1017,17 +1032,41 @@ customize a prototype instance: :class:`OrderedDict` objects ---------------------------- -Ordered dictionaries are just like regular dictionaries but they remember the -order that items were inserted. When iterating over an ordered dictionary, -the items are returned in the order their keys were first added. +Ordered dictionaries are just like regular dictionaries but have some extra +capabilities relating to ordering operations. They have become less +important now that the built-in :class:`dict` class gained the ability +to remember insertion order (this new behavior became guaranteed in +Python 3.7). + +Some differences from :class:`dict` still remain: + +* The regular :class:`dict` was designed to be very good at mapping + operations. Tracking insertion order was secondary. + +* The :class:`OrderedDict` was designed to be good at reordering operations. + Space efficiency, iteration speed, and the performance of update + operations were secondary. + +* Algorithmically, :class:`OrderedDict` can handle frequent reordering + operations better than :class:`dict`. This makes it suitable for tracking + recent accesses (for example in an `LRU cache + `_). + +* The equality operation for :class:`OrderedDict` checks for matching order. + +* The :meth:`popitem` method of :class:`OrderedDict` has a different + signature. It accepts an optional argument to specify which item is popped. + +* :class:`OrderedDict` has a :meth:`move_to_end` method to + efficiently reposition an element to an endpoint. + +* Until Python 3.8, :class:`dict` lacked a :meth:`__reversed__` method. + .. class:: OrderedDict([items]) - Return an instance of a dict subclass, supporting the usual :class:`dict` - methods. An *OrderedDict* is a dict that remembers the order that keys - were first inserted. If a new entry overwrites an existing entry, the - original insertion position is left unchanged. Deleting an entry and - reinserting it will move it to the end. + Return an instance of a :class:`dict` subclass that has methods + specialized for rearranging dictionary order. .. versionadded:: 3.1 @@ -1077,29 +1116,7 @@ anywhere a regular dictionary is used. :class:`OrderedDict` Examples and Recipes ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Since an ordered dictionary remembers its insertion order, it can be used -in conjunction with sorting to make a sorted dictionary:: - - >>> # regular unsorted dictionary - >>> d = {'banana': 3, 'apple': 4, 'pear': 1, 'orange': 2} - - >>> # dictionary sorted by key - >>> OrderedDict(sorted(d.items(), key=lambda t: t[0])) - OrderedDict([('apple', 4), ('banana', 3), ('orange', 2), ('pear', 1)]) - - >>> # dictionary sorted by value - >>> OrderedDict(sorted(d.items(), key=lambda t: t[1])) - OrderedDict([('pear', 1), ('orange', 2), ('banana', 3), ('apple', 4)]) - - >>> # dictionary sorted by length of the key string - >>> OrderedDict(sorted(d.items(), key=lambda t: len(t[0]))) - OrderedDict([('pear', 1), ('apple', 4), ('orange', 2), ('banana', 3)]) - -The new sorted dictionaries maintain their sort order when entries -are deleted. But when new keys are added, the keys are appended -to the end and the sort is not maintained. - -It is also straight-forward to create an ordered dictionary variant +It is straightforward to create an ordered dictionary variant that remembers the order the keys were *last* inserted. If a new entry overwrites an existing entry, the original insertion position is changed and moved to the end:: @@ -1108,21 +1125,29 @@ original insertion position is changed and moved to the end:: 'Store items in the order the keys were last added' def __setitem__(self, key, value): - if key in self: - del self[key] - OrderedDict.__setitem__(self, key, value) + super().__setitem__(key, value) + super().move_to_end(key) -An ordered dictionary can be combined with the :class:`Counter` class -so that the counter remembers the order elements are first encountered:: +An :class:`OrderedDict` would also be useful for implementing +variants of :func:`functools.lru_cache`:: - class OrderedCounter(Counter, OrderedDict): - 'Counter that remembers the order elements are first encountered' + class LRU(OrderedDict): + 'Limit size, evicting the least recently looked-up key when full' - def __repr__(self): - return '%s(%r)' % (self.__class__.__name__, OrderedDict(self)) + def __init__(self, maxsize=128, *args, **kwds): + self.maxsize = maxsize + super().__init__(*args, **kwds) - def __reduce__(self): - return self.__class__, (OrderedDict(self),) + def __getitem__(self, key): + value = super().__getitem__(key) + self.move_to_end(key) + return value + + def __setitem__(self, key, value): + super().__setitem__(key, value) + if len(self) > self.maxsize: + oldest = next(iter(self)) + del self[oldest] :class:`UserDict` objects diff --git a/kbe/src/lib/python/Doc/library/compileall.rst b/kbe/src/lib/python/Doc/library/compileall.rst index 5151f3a523..258de28e3b 100644 --- a/kbe/src/lib/python/Doc/library/compileall.rst +++ b/kbe/src/lib/python/Doc/library/compileall.rst @@ -105,7 +105,7 @@ compile Python sources. byte-code file ending in ``.pyc``, never ``.pyo``. .. versionchanged:: 3.7 - Added the ``--invalidation-mode`` parameter. + Added the ``--invalidation-mode`` option. There is no command-line option to control the optimization level used by the diff --git a/kbe/src/lib/python/Doc/library/configparser.rst b/kbe/src/lib/python/Doc/library/configparser.rst index 321770242b..95cc352010 100644 --- a/kbe/src/lib/python/Doc/library/configparser.rst +++ b/kbe/src/lib/python/Doc/library/configparser.rst @@ -462,7 +462,8 @@ the :meth:`__init__` options: Please note: there are ways to add a set of key-value pairs in a single operation. When you use a regular dictionary in those operations, the order - of the keys may be random. For example: + of the keys will be ordered because dict preserves order from Python 3.7. + For example: .. doctest:: @@ -477,41 +478,10 @@ the :meth:`__init__` options: ... 'bar': 'y', ... 'baz': 'z'} ... }) - >>> parser.sections() # doctest: +SKIP - ['section3', 'section2', 'section1'] - >>> [option for option in parser['section3']] # doctest: +SKIP - ['baz', 'foo', 'bar'] - - In these operations you need to use an ordered dictionary as well: - - .. doctest:: - - >>> from collections import OrderedDict - >>> parser = configparser.ConfigParser() - >>> parser.read_dict( - ... OrderedDict(( - ... ('s1', - ... OrderedDict(( - ... ('1', '2'), - ... ('3', '4'), - ... ('5', '6'), - ... )) - ... ), - ... ('s2', - ... OrderedDict(( - ... ('a', 'b'), - ... ('c', 'd'), - ... ('e', 'f'), - ... )) - ... ), - ... )) - ... ) - >>> parser.sections() # doctest: +SKIP - ['s1', 's2'] - >>> [option for option in parser['s1']] # doctest: +SKIP - ['1', '3', '5'] - >>> [option for option in parser['s2'].values()] # doctest: +SKIP - ['b', 'd', 'f'] + >>> parser.sections() + ['section1', 'section2', 'section3'] + >>> [option for option in parser['section3']] + ['foo', 'bar', 'baz'] * *allow_no_value*, default value: ``False`` @@ -891,7 +861,7 @@ interpolation if an option used is not defined elsewhere. :: ConfigParser Objects -------------------- -.. class:: ConfigParser(defaults=None, dict_type=dict, allow_no_value=False, delimiters=('=', ':'), comment_prefixes=('#', ';'), inline_comment_prefixes=None, strict=True, empty_lines_in_values=True, default_section=configparser.DEFAULTSECT, interpolation=BasicInterpolation(), converters={}) +.. class:: ConfigParser(defaults=None, dict_type=collections.OrderedDict, allow_no_value=False, delimiters=('=', ':'), comment_prefixes=('#', ';'), inline_comment_prefixes=None, strict=True, empty_lines_in_values=True, default_section=configparser.DEFAULTSECT, interpolation=BasicInterpolation(), converters={}) The main configuration parser. When *defaults* is given, it is initialized into the dictionary of intrinsic defaults. When *dict_type* is given, it @@ -953,10 +923,6 @@ ConfigParser Objects providing consistent behavior across the parser: non-string keys and values are implicitly converted to strings. - .. versionchanged:: 3.7 - The default *dict_type* is :class:`dict`, since it now preserves - insertion order. - .. method:: defaults() Return a dictionary containing the instance-wide defaults. @@ -1213,7 +1179,7 @@ ConfigParser Objects RawConfigParser Objects ----------------------- -.. class:: RawConfigParser(defaults=None, dict_type=dict, \ +.. class:: RawConfigParser(defaults=None, dict_type=collections.OrderedDict, \ allow_no_value=False, *, delimiters=('=', ':'), \ comment_prefixes=('#', ';'), \ inline_comment_prefixes=None, strict=True, \ @@ -1226,10 +1192,6 @@ RawConfigParser Objects names, and values via its unsafe ``add_section`` and ``set`` methods, as well as the legacy ``defaults=`` keyword argument handling. - .. versionchanged:: 3.7 - The default *dict_type* is :class:`dict`, since it now preserves - insertion order. - .. note:: Consider using :class:`ConfigParser` instead which checks types of the values to be stored internally. If you don't want interpolation, you diff --git a/kbe/src/lib/python/Doc/library/contextlib.rst b/kbe/src/lib/python/Doc/library/contextlib.rst index 930c97358e..017a87a564 100644 --- a/kbe/src/lib/python/Doc/library/contextlib.rst +++ b/kbe/src/lib/python/Doc/library/contextlib.rst @@ -1,5 +1,5 @@ -:mod:`contextlib` --- Utilities for :keyword:`with`\ -statement contexts -======================================================================== +:mod:`!contextlib` --- Utilities for :keyword:`!with`\ -statement contexts +========================================================================== .. module:: contextlib :synopsis: Utilities for with-statement contexts. @@ -72,7 +72,7 @@ Functions and classes provided: The function being decorated must return a :term:`generator`-iterator when called. This iterator must yield exactly one value, which will be bound to - the targets in the :keyword:`with` statement's :keyword:`as` clause, if any. + the targets in the :keyword:`with` statement's :keyword:`!as` clause, if any. At the point where the generator yields, the block nested in the :keyword:`with` statement is executed. The generator is then resumed after the block is exited. @@ -82,9 +82,9 @@ Functions and classes provided: the error (if any), or ensure that some cleanup takes place. If an exception is trapped merely in order to log it or to perform some action (rather than to suppress it entirely), the generator must reraise that exception. Otherwise the - generator context manager will indicate to the :keyword:`with` statement that + generator context manager will indicate to the :keyword:`!with` statement that the exception has been handled, and execution will resume with the statement - immediately following the :keyword:`with` statement. + immediately following the :keyword:`!with` statement. :func:`contextmanager` uses :class:`ContextDecorator` so the context managers it creates can be used as decorators as well as in :keyword:`with` statements. @@ -346,7 +346,7 @@ Functions and classes provided: As the decorated function must be able to be called multiple times, the underlying context manager must support use in multiple :keyword:`with` statements. If this is not the case, then the original construct with the - explicit :keyword:`with` statement inside the function should be used. + explicit :keyword:`!with` statement inside the function should be used. .. versionadded:: 3.2 @@ -771,7 +771,7 @@ Reentrant context managers More sophisticated context managers may be "reentrant". These context managers can not only be used in multiple :keyword:`with` statements, -but may also be used *inside* a :keyword:`with` statement that is already +but may also be used *inside* a :keyword:`!with` statement that is already using the same context manager. :class:`threading.RLock` is an example of a reentrant context manager, as are diff --git a/kbe/src/lib/python/Doc/library/dataclasses.rst b/kbe/src/lib/python/Doc/library/dataclasses.rst index fe0feeda8b..6af60b6e26 100644 --- a/kbe/src/lib/python/Doc/library/dataclasses.rst +++ b/kbe/src/lib/python/Doc/library/dataclasses.rst @@ -51,9 +51,9 @@ Module-level decorators, classes, and functions The :func:`dataclass` decorator examines the class to find ``field``\s. A ``field`` is defined as class variable that has a - type annotation. With two exceptions described below, nothing in - :func:`dataclass` examines the type specified in the variable - annotation. + :term:`type annotation `. With two + exceptions described below, nothing in :func:`dataclass` + examines the type specified in the variable annotation. The order of the fields in all of the generated methods is the order in which they appear in the class definition. diff --git a/kbe/src/lib/python/Doc/library/datetime.rst b/kbe/src/lib/python/Doc/library/datetime.rst index db3a6522c2..121f73bbe8 100644 --- a/kbe/src/lib/python/Doc/library/datetime.rst +++ b/kbe/src/lib/python/Doc/library/datetime.rst @@ -2034,6 +2034,12 @@ calls the platform C library's :func:`strftime` function, and platform variations are common. To see the full set of format codes supported on your platform, consult the :manpage:`strftime(3)` documentation. +For the same reason, handling of format strings containing Unicode code points +that can't be represented in the charset of the current locale is also +platform-dependent. On some platforms such code points are preserved intact in +the output, while on others ``strftime`` may raise :exc:`UnicodeError` or return +an empty string instead. + The following is a list of all the format codes that the C standard (1989 version) requires, and these work on all platforms with a standard C implementation. Note that the 1999 version of the C standard added additional diff --git a/kbe/src/lib/python/Doc/library/decimal.rst b/kbe/src/lib/python/Doc/library/decimal.rst index f2a677e693..bcae55eb82 100644 --- a/kbe/src/lib/python/Doc/library/decimal.rst +++ b/kbe/src/lib/python/Doc/library/decimal.rst @@ -2115,3 +2115,23 @@ Alternatively, inputs can be rounded upon creation using the >>> Context(prec=5, rounding=ROUND_DOWN).create_decimal('1.2345678') Decimal('1.2345') + +Q. Is the CPython implementation fast for large numbers? + +A. Yes. In the CPython and PyPy3 implementations, the C/CFFI versions of +the decimal module integrate the high speed `libmpdec +`_ library for +arbitrary precision correctly-rounded decimal floating point arithmetic. +``libmpdec`` uses `Karatsuba multiplication +`_ +for medium-sized numbers and the `Number Theoretic Transform +`_ +for very large numbers. However, to realize this performance gain, the +context needs to be set for unrounded calculations. + + >>> c = getcontext() + >>> c.prec = MAX_PREC + >>> c.Emax = MAX_EMAX + >>> c.Emin = MIN_EMIN + +.. versionadded:: 3.3 \ No newline at end of file diff --git a/kbe/src/lib/python/Doc/library/enum.rst b/kbe/src/lib/python/Doc/library/enum.rst index 81e9766e42..a6285ffaf1 100644 --- a/kbe/src/lib/python/Doc/library/enum.rst +++ b/kbe/src/lib/python/Doc/library/enum.rst @@ -394,7 +394,7 @@ A new :class:`Enum` class must have one base Enum class, up to one concrete data type, and as many :class:`object`-based mixin classes as needed. The order of these base classes is:: - def EnumName([mix-in, ...,] [data-type,] base-enum): + class EnumName([mix-in, ...,] [data-type,] base-enum): pass Also, subclassing an enumeration is allowed only if the enumeration does not define diff --git a/kbe/src/lib/python/Doc/library/fileinput.rst b/kbe/src/lib/python/Doc/library/fileinput.rst index 5881fefe29..1fc11ffce2 100644 --- a/kbe/src/lib/python/Doc/library/fileinput.rst +++ b/kbe/src/lib/python/Doc/library/fileinput.rst @@ -63,7 +63,7 @@ The following function is the primary interface of this module: The :class:`FileInput` instance can be used as a context manager in the :keyword:`with` statement. In this example, *input* is closed after the - :keyword:`with` statement is exited, even if an exception occurs:: + :keyword:`!with` statement is exited, even if an exception occurs:: with fileinput.input(files=('spam.txt', 'eggs.txt')) as f: for line in f: @@ -155,7 +155,7 @@ available for subclassing as well: A :class:`FileInput` instance can be used as a context manager in the :keyword:`with` statement. In this example, *input* is closed after the - :keyword:`with` statement is exited, even if an exception occurs:: + :keyword:`!with` statement is exited, even if an exception occurs:: with FileInput(files=('spam.txt', 'eggs.txt')) as input: process(input) diff --git a/kbe/src/lib/python/Doc/library/functions.rst b/kbe/src/lib/python/Doc/library/functions.rst index c0f4ffd2cb..9326b8d0fe 100644 --- a/kbe/src/lib/python/Doc/library/functions.rst +++ b/kbe/src/lib/python/Doc/library/functions.rst @@ -668,6 +668,11 @@ are always available. They are listed here in alphabetical order. topic, and a help page is printed on the console. If the argument is any other kind of object, a help page on the object is generated. + Note that if a slash(/) appears in the parameter list of a function, when + invoking :func:`help`, it means that the parameters prior to the slash are + positional-only. For more info, see + :ref:`the FAQ entry on positional-only parameters `. + This function is added to the built-in namespace by the :mod:`site` module. .. versionchanged:: 3.4 @@ -809,13 +814,14 @@ are always available. They are listed here in alphabetical order. See also :ref:`typeiter`. - One useful application of the second form of :func:`iter` is to read lines of - a file until a certain line is reached. The following example reads a file - until the :meth:`~io.TextIOBase.readline` method returns an empty string:: + One useful application of the second form of :func:`iter` is to build a + block-reader. For example, reading fixed-width blocks from a binary + database file until the end of file is reached:: - with open('mydata.txt') as fp: - for line in iter(fp.readline, ''): - process_line(line) + from functools import partial + with open('mydata.db', 'rb') as f: + for block in iter(partial(f.read, 64), b''): + process_block(block) .. function:: len(s) @@ -996,7 +1002,6 @@ are always available. They are listed here in alphabetical order. ``'b'`` binary mode ``'t'`` text mode (default) ``'+'`` open a disk file for updating (reading and writing) - ``'U'`` :term:`universal newlines` mode (deprecated) ========= =============================================================== The default mode is ``'r'`` (open for reading text, synonym of ``'rt'``). @@ -1011,6 +1016,12 @@ are always available. They are listed here in alphabetical order. first decoded using a platform-dependent encoding or using the specified *encoding* if given. + There is an additional mode character permitted, ``'U'``, which no longer + has any effect, and is considered deprecated. It previously enabled + :term:`universal newlines` in text mode, which became the default behaviour + in Python 3.0. Refer to the documentation of the + :ref:`newline ` parameter for further details. + .. note:: Python doesn't depend on the underlying operating system's notion of text @@ -1077,6 +1088,8 @@ are always available. They are listed here in alphabetical order. .. index:: single: universal newlines; open() built-in function + .. _open-newline-parameter: + *newline* controls how :term:`universal newlines` mode works (it only applies to text mode). It can be ``None``, ``''``, ``'\n'``, ``'\r'``, and ``'\r\n'``. It works as follows: @@ -1655,7 +1668,7 @@ are always available. They are listed here in alphabetical order. This function is invoked by the :keyword:`import` statement. It can be replaced (by importing the :mod:`builtins` module and assigning to ``builtins.__import__``) in order to change semantics of the - :keyword:`import` statement, but doing so is **strongly** discouraged as it + :keyword:`!import` statement, but doing so is **strongly** discouraged as it is usually simpler to use import hooks (see :pep:`302`) to attain the same goals and does not cause issues with code which assumes the default import implementation is in use. Direct use of :func:`__import__` is also diff --git a/kbe/src/lib/python/Doc/library/http.client.rst b/kbe/src/lib/python/Doc/library/http.client.rst index c4b7c79730..3408c103e2 100644 --- a/kbe/src/lib/python/Doc/library/http.client.rst +++ b/kbe/src/lib/python/Doc/library/http.client.rst @@ -497,6 +497,7 @@ Here is an example session that uses the ``GET`` method:: b'\n one new dict self.assertEqual(ChainMap({1:2}).maps, [{1:2}]) # 1 arg --> list diff --git a/kbe/src/lib/python/Lib/test/test_dataclasses.py b/kbe/src/lib/python/Lib/test/test_dataclasses.py index ff6060c6d2..9c83459f09 100755 --- a/kbe/src/lib/python/Lib/test/test_dataclasses.py +++ b/kbe/src/lib/python/Lib/test/test_dataclasses.py @@ -1737,23 +1737,33 @@ class C: i: int = field(metadata=0) # Make sure an empty dict works. + d = {} @dataclass class C: - i: int = field(metadata={}) + i: int = field(metadata=d) self.assertFalse(fields(C)[0].metadata) self.assertEqual(len(fields(C)[0].metadata), 0) + # Update should work (see bpo-35960). + d['foo'] = 1 + self.assertEqual(len(fields(C)[0].metadata), 1) + self.assertEqual(fields(C)[0].metadata['foo'], 1) with self.assertRaisesRegex(TypeError, 'does not support item assignment'): fields(C)[0].metadata['test'] = 3 # Make sure a non-empty dict works. + d = {'test': 10, 'bar': '42', 3: 'three'} @dataclass class C: - i: int = field(metadata={'test': 10, 'bar': '42', 3: 'three'}) + i: int = field(metadata=d) self.assertEqual(len(fields(C)[0].metadata), 3) self.assertEqual(fields(C)[0].metadata['test'], 10) self.assertEqual(fields(C)[0].metadata['bar'], '42') self.assertEqual(fields(C)[0].metadata[3], 'three') + # Update should work. + d['foo'] = 1 + self.assertEqual(len(fields(C)[0].metadata), 4) + self.assertEqual(fields(C)[0].metadata['foo'], 1) with self.assertRaises(KeyError): # Non-existent key. fields(C)[0].metadata['baz'] diff --git a/kbe/src/lib/python/Lib/test/test_doctest.py b/kbe/src/lib/python/Lib/test/test_doctest.py index 83941c129f..4a3c488738 100644 --- a/kbe/src/lib/python/Lib/test/test_doctest.py +++ b/kbe/src/lib/python/Lib/test/test_doctest.py @@ -2450,6 +2450,10 @@ def test_unittest_reportflags(): Then the default eporting options are ignored: >>> result = suite.run(unittest.TestResult()) + + *NOTE*: These doctest are intentionally not placed in raw string to depict + the trailing whitespace using `\x20` in the diff below. + >>> print(result.failures[0][1]) # doctest: +ELLIPSIS Traceback ... Failed example: @@ -2463,7 +2467,7 @@ def test_unittest_reportflags(): Differences (ndiff with -expected +actual): a - - + + +\x20 b @@ -2952,6 +2956,47 @@ def test_CLI(): r""" """ +def test_no_trailing_whitespace_stripping(): + r""" + The fancy reports had a bug for a long time where any trailing whitespace on + the reported diff lines was stripped, making it impossible to see the + differences in line reported as different that differed only in the amount of + trailing whitespace. The whitespace still isn't particularly visible unless + you use NDIFF, but at least it is now there to be found. + + *NOTE*: This snippet was intentionally put inside a raw string to get rid of + leading whitespace error in executing the example below + + >>> def f(x): + ... r''' + ... >>> print('\n'.join(['a ', 'b'])) + ... a + ... b + ... ''' + """ + """ + *NOTE*: These doctest are not placed in raw string to depict the trailing whitespace + using `\x20` + + >>> test = doctest.DocTestFinder().find(f)[0] + >>> flags = doctest.REPORT_NDIFF + >>> doctest.DocTestRunner(verbose=False, optionflags=flags).run(test) + ... # doctest: +ELLIPSIS + ********************************************************************** + File ..., line 3, in f + Failed example: + print('\n'.join(['a ', 'b'])) + Differences (ndiff with -expected +actual): + - a + + a + b + TestResults(failed=1, attempted=1) + + *NOTE*: `\x20` is for checking the trailing whitespace on the +a line above. + We cannot use actual spaces there, as a commit hook prevents from committing + patches that contain trailing whitespace. More info on Issue 24746. + """ + ###################################################################### ## Main ###################################################################### diff --git a/kbe/src/lib/python/Lib/test/test_dummy_thread.py b/kbe/src/lib/python/Lib/test/test_dummy_thread.py index 0840be67d5..da51216783 100644 --- a/kbe/src/lib/python/Lib/test/test_dummy_thread.py +++ b/kbe/src/lib/python/Lib/test/test_dummy_thread.py @@ -70,14 +70,14 @@ def delay_unlock(to_unlock, delay): to_unlock.release() self.lock.acquire() - start_time = int(time.time()) + start_time = int(time.monotonic()) _thread.start_new_thread(delay_unlock,(self.lock, DELAY)) if support.verbose: print() print("*** Waiting for thread to release the lock "\ "(approx. %s sec.) ***" % DELAY) self.lock.acquire() - end_time = int(time.time()) + end_time = int(time.monotonic()) if support.verbose: print("done") self.assertGreaterEqual(end_time - start_time, DELAY, diff --git a/kbe/src/lib/python/Lib/test/test_enum.py b/kbe/src/lib/python/Lib/test/test_enum.py index b221045328..29a429ccd9 100644 --- a/kbe/src/lib/python/Lib/test/test_enum.py +++ b/kbe/src/lib/python/Lib/test/test_enum.py @@ -1866,6 +1866,15 @@ class Decision2(MyEnum): REVERT_ALL = "REVERT_ALL" RETRY = "RETRY" + def test_empty_globals(self): + # bpo-35717: sys._getframe(2).f_globals['__name__'] fails with KeyError + # when using compile and exec because f_globals is empty + code = "from enum import Enum; Enum('Animal', 'ANT BEE CAT DOG')" + code = compile(code, "", "exec") + global_ns = {} + local_ls = {} + exec(code, global_ns, local_ls) + class TestOrder(unittest.TestCase): @@ -2729,6 +2738,23 @@ def cycle_enum(): self.assertEqual(256, len(seen), 'too many composite members created') +class TestEmptyAndNonLatinStrings(unittest.TestCase): + + def test_empty_string(self): + with self.assertRaises(ValueError): + empty_abc = Enum('empty_abc', ('', 'B', 'C')) + + def test_non_latin_character_string(self): + greek_abc = Enum('greek_abc', ('\u03B1', 'B', 'C')) + item = getattr(greek_abc, '\u03B1') + self.assertEqual(item.value, 1) + + def test_non_latin_number_string(self): + hebrew_123 = Enum('hebrew_123', ('\u05D0', '2', '3')) + item = getattr(hebrew_123, '\u05D0') + self.assertEqual(item.value, 1) + + class TestUnique(unittest.TestCase): def test_unique_clean(self): diff --git a/kbe/src/lib/python/Lib/test/test_faulthandler.py b/kbe/src/lib/python/Lib/test/test_faulthandler.py index d6dc4ba55d..700b7ad6b8 100644 --- a/kbe/src/lib/python/Lib/test/test_faulthandler.py +++ b/kbe/src/lib/python/Lib/test/test_faulthandler.py @@ -5,6 +5,7 @@ import signal import subprocess import sys +import sysconfig from test import support from test.support import script_helper, is_android import tempfile @@ -19,6 +20,17 @@ TIMEOUT = 0.5 MS_WINDOWS = (os.name == 'nt') +_cflags = sysconfig.get_config_var('CFLAGS') or '' +_config_args = sysconfig.get_config_var('CONFIG_ARGS') or '' +UB_SANITIZER = ( + '-fsanitize=undefined' in _cflags or + '--with-undefined-behavior-sanitizer' in _config_args +) +MEMORY_SANITIZER = ( + '-fsanitize=memory' in _cflags or + '--with-memory-sanitizer' in _config_args +) + def expected_traceback(lineno1, lineno2, header, min_count=1): regex = header @@ -94,7 +106,7 @@ def check_error(self, code, line_number, fatal_error, *, else: header = 'Stack' regex = r""" - ^{fatal_error} + (?m)^{fatal_error} {header} \(most recent call first\): File "", line {lineno} in @@ -252,6 +264,8 @@ def test_gil_released(self): 3, 'Segmentation fault') + @unittest.skipIf(UB_SANITIZER or MEMORY_SANITIZER, + "sanitizer builds change crashing process output.") @skip_segfault_on_android def test_enable_file(self): with temporary_filename() as filename: @@ -267,6 +281,8 @@ def test_enable_file(self): @unittest.skipIf(sys.platform == "win32", "subprocess doesn't support pass_fds on Windows") + @unittest.skipIf(UB_SANITIZER or MEMORY_SANITIZER, + "sanitizer builds change crashing process output.") @skip_segfault_on_android def test_enable_fd(self): with tempfile.TemporaryFile('wb+') as fp: diff --git a/kbe/src/lib/python/Lib/test/test_float.py b/kbe/src/lib/python/Lib/test/test_float.py index 06ea90c207..49c1fbcd4e 100644 --- a/kbe/src/lib/python/Lib/test/test_float.py +++ b/kbe/src/lib/python/Lib/test/test_float.py @@ -701,6 +701,25 @@ def test_issue5864(self): self.assertEqual(format(1234.56, '.4'), '1.235e+03') self.assertEqual(format(12345.6, '.4'), '1.235e+04') + def test_issue35560(self): + self.assertEqual(format(123.0, '00'), '123.0') + self.assertEqual(format(123.34, '00f'), '123.340000') + self.assertEqual(format(123.34, '00e'), '1.233400e+02') + self.assertEqual(format(123.34, '00g'), '123.34') + self.assertEqual(format(123.34, '00.10f'), '123.3400000000') + self.assertEqual(format(123.34, '00.10e'), '1.2334000000e+02') + self.assertEqual(format(123.34, '00.10g'), '123.34') + self.assertEqual(format(123.34, '01f'), '123.340000') + + self.assertEqual(format(-123.0, '00'), '-123.0') + self.assertEqual(format(-123.34, '00f'), '-123.340000') + self.assertEqual(format(-123.34, '00e'), '-1.233400e+02') + self.assertEqual(format(-123.34, '00g'), '-123.34') + self.assertEqual(format(-123.34, '00.10f'), '-123.3400000000') + self.assertEqual(format(-123.34, '00.10f'), '-123.3400000000') + self.assertEqual(format(-123.34, '00.10e'), '-1.2334000000e+02') + self.assertEqual(format(-123.34, '00.10g'), '-123.34') + class ReprTestCase(unittest.TestCase): def test_repr(self): floats_file = open(os.path.join(os.path.split(__file__)[0], diff --git a/kbe/src/lib/python/Lib/test/test_frame.py b/kbe/src/lib/python/Lib/test/test_frame.py index fd795085a5..d6aa2834cb 100644 --- a/kbe/src/lib/python/Lib/test/test_frame.py +++ b/kbe/src/lib/python/Lib/test/test_frame.py @@ -109,10 +109,7 @@ class C: self.assertIs(None, wr()) -class FrameLocalsTest(unittest.TestCase): - """ - Tests for the .f_locals attribute. - """ +class FrameAttrsTest(unittest.TestCase): def make_frames(self): def outer(): @@ -159,6 +156,11 @@ def test_locals_clear_locals(self): self.assertEqual(outer.f_locals, {}) self.assertEqual(inner.f_locals, {}) + def test_f_lineno_del_segfault(self): + f, _, _ = self.make_frames() + with self.assertRaises(AttributeError): + del f.f_lineno + class ReprTest(unittest.TestCase): """ diff --git a/kbe/src/lib/python/Lib/test/test_functools.py b/kbe/src/lib/python/Lib/test/test_functools.py index 7a0757f7e1..a91c6348e7 100644 --- a/kbe/src/lib/python/Lib/test/test_functools.py +++ b/kbe/src/lib/python/Lib/test/test_functools.py @@ -1226,6 +1226,33 @@ def f(x): self.assertEqual(misses, 4) self.assertEqual(currsize, 2) + def test_lru_bug_35780(self): + # C version of the lru_cache was not checking to see if + # the user function call has already modified the cache + # (this arises in recursive calls and in multi-threading). + # This cause the cache to have orphan links not referenced + # by the cache dictionary. + + once = True # Modified by f(x) below + + @self.module.lru_cache(maxsize=10) + def f(x): + nonlocal once + rv = f'.{x}.' + if x == 20 and once: + once = False + rv = f(x) + return rv + + # Fill the cache + for x in range(15): + self.assertEqual(f(x), f'.{x}.') + self.assertEqual(f.cache_info().currsize, 10) + + # Make a recursive call and make sure the cache remains full + self.assertEqual(f(20), '.20.') + self.assertEqual(f.cache_info().currsize, 10) + def test_lru_hash_only_once(self): # To protect against weird reentrancy bugs and to improve # efficiency when faced with slow __hash__ methods, the @@ -1322,7 +1349,7 @@ def eq(n): for i in (0, 1): self.assertEqual([eq(n) for n in range(150)], list(range(150))) self.assertEqual(eq.cache_info(), - self.module._CacheInfo(hits=0, misses=300, maxsize=-10, currsize=1)) + self.module._CacheInfo(hits=0, misses=300, maxsize=0, currsize=0)) def test_lru_with_exceptions(self): # Verify that user_function exceptions get passed through without diff --git a/kbe/src/lib/python/Lib/test/test_future4.py b/kbe/src/lib/python/Lib/test/test_future4.py index 413dd4d96b..b27ca40d2e 100644 --- a/kbe/src/lib/python/Lib/test/test_future4.py +++ b/kbe/src/lib/python/Lib/test/test_future4.py @@ -1,6 +1,11 @@ from __future__ import unicode_literals - import unittest + +class Tests(unittest.TestCase): + def test_unicode_literals(self): + self.assertIsInstance("literal", str) + + if __name__ == "__main__": unittest.main() diff --git a/kbe/src/lib/python/Lib/test/test_genericclass.py b/kbe/src/lib/python/Lib/test/test_genericclass.py index 37e755b116..37a87bc681 100644 --- a/kbe/src/lib/python/Lib/test/test_genericclass.py +++ b/kbe/src/lib/python/Lib/test/test_genericclass.py @@ -248,7 +248,14 @@ def __class_getitem__(cls, item): return f'{cls.__name__}[{item.__name__}]' self.assertEqual(Meta[int], 'Meta[int]') - def test_class_getitem_metaclass_2(self): + def test_class_getitem_with_metaclass(self): + class Meta(type): pass + class C(metaclass=Meta): + def __class_getitem__(cls, item): + return f'{cls.__name__}[{item.__name__}]' + self.assertEqual(C[int], 'C[int]') + + def test_class_getitem_metaclass_first(self): class Meta(type): def __getitem__(cls, item): return 'from metaclass' diff --git a/kbe/src/lib/python/Lib/test/test_http_cookiejar.py b/kbe/src/lib/python/Lib/test/test_http_cookiejar.py index abc625d672..16edf34a99 100644 --- a/kbe/src/lib/python/Lib/test/test_http_cookiejar.py +++ b/kbe/src/lib/python/Lib/test/test_http_cookiejar.py @@ -415,6 +415,7 @@ def test_domain_return_ok(self): ("http://foo.bar.com/", ".foo.bar.com", True), ("http://foo.bar.com/", "foo.bar.com", True), ("http://foo.bar.com/", ".bar.com", True), + ("http://foo.bar.com/", "bar.com", True), ("http://foo.bar.com/", "com", True), ("http://foo.com/", "rhubarb.foo.com", False), ("http://foo.com/", ".foo.com", True), @@ -425,6 +426,8 @@ def test_domain_return_ok(self): ("http://foo/", "foo", True), ("http://foo/", "foo.local", True), ("http://foo/", ".local", True), + ("http://barfoo.com", ".foo.com", False), + ("http://barfoo.com", "foo.com", False), ]: request = urllib.request.Request(url) r = pol.domain_return_ok(domain, request) @@ -692,6 +695,30 @@ def test_request_path(self): req = urllib.request.Request("http://www.example.com") self.assertEqual(request_path(req), "/") + def test_path_prefix_match(self): + pol = DefaultCookiePolicy() + strict_ns_path_pol = DefaultCookiePolicy(strict_ns_set_path=True) + + c = CookieJar(pol) + base_url = "http://bar.com" + interact_netscape(c, base_url, 'spam=eggs; Path=/foo') + cookie = c._cookies['bar.com']['/foo']['spam'] + + for path, ok in [('/foo', True), + ('/foo/', True), + ('/foo/bar', True), + ('/', False), + ('/foobad/foo', False)]: + url = f'{base_url}{path}' + req = urllib.request.Request(url) + h = interact_netscape(c, url) + if ok: + self.assertIn('spam=eggs', h, f"cookie not set for {path}") + self.assertTrue(strict_ns_path_pol.set_ok_path(cookie, req)) + else: + self.assertNotIn('spam=eggs', h, f"cookie set for {path}") + self.assertFalse(strict_ns_path_pol.set_ok_path(cookie, req)) + def test_request_port(self): req = urllib.request.Request("http://www.acme.com:1234/", headers={"Host": "www.acme.com:4321"}) @@ -959,6 +986,33 @@ def test_domain_block(self): c.add_cookie_header(req) self.assertFalse(req.has_header("Cookie")) + c.clear() + + pol.set_blocked_domains([]) + req = urllib.request.Request("http://acme.com/") + res = FakeResponse(headers, "http://acme.com/") + cookies = c.make_cookies(res, req) + c.extract_cookies(res, req) + self.assertEqual(len(c), 1) + + req = urllib.request.Request("http://acme.com/") + c.add_cookie_header(req) + self.assertTrue(req.has_header("Cookie")) + + req = urllib.request.Request("http://badacme.com/") + c.add_cookie_header(req) + self.assertFalse(pol.return_ok(cookies[0], req)) + self.assertFalse(req.has_header("Cookie")) + + p = pol.set_blocked_domains(["acme.com"]) + req = urllib.request.Request("http://acme.com/") + c.add_cookie_header(req) + self.assertFalse(req.has_header("Cookie")) + + req = urllib.request.Request("http://badacme.com/") + c.add_cookie_header(req) + self.assertFalse(req.has_header("Cookie")) + def test_secure(self): for ns in True, False: for whitespace in " ", "": diff --git a/kbe/src/lib/python/Lib/test/test_imaplib.py b/kbe/src/lib/python/Lib/test/test_imaplib.py index a0b598d0c7..a060143e1f 100644 --- a/kbe/src/lib/python/Lib/test/test_imaplib.py +++ b/kbe/src/lib/python/Lib/test/test_imaplib.py @@ -8,6 +8,7 @@ import time import calendar import threading +import socket from test.support import (reap_threads, verbose, transient_internet, run_with_tz, run_with_locale, cpython_only) @@ -71,6 +72,15 @@ def test_that_Time2Internaldate_returns_a_result(self): imaplib.Time2Internaldate(t) def test_imap4_host_default_value(self): + # Check whether the IMAP4_PORT is truly unavailable. + with socket.socket() as s: + try: + s.connect(('', imaplib.IMAP4_PORT)) + self.skipTest( + "Cannot run the test with local IMAP server running.") + except socket.error: + pass + expected_errnos = [ # This is the exception that should be raised. errno.ECONNREFUSED, diff --git a/kbe/src/lib/python/Lib/test/test_io.py b/kbe/src/lib/python/Lib/test/test_io.py index b41141e28a..6655576b6d 100644 --- a/kbe/src/lib/python/Lib/test/test_io.py +++ b/kbe/src/lib/python/Lib/test/test_io.py @@ -28,6 +28,7 @@ import random import signal import sys +import sysconfig import threading import time import unittest @@ -59,6 +60,13 @@ def byteslike(*pos, **kw): class EmptyStruct(ctypes.Structure): pass +_cflags = sysconfig.get_config_var('CFLAGS') or '' +_config_args = sysconfig.get_config_var('CONFIG_ARGS') or '' +MEMORY_SANITIZER = ( + '-fsanitize=memory' in _cflags or + '--with-memory-sanitizer' in _config_args +) + def _default_chunk_size(): """Get the default TextIOWrapper chunk size""" with open(__file__, "r", encoding="latin-1") as f: @@ -1496,6 +1504,8 @@ def test_read_on_closed(self): class CBufferedReaderTest(BufferedReaderTest, SizeofTest): tp = io.BufferedReader + @unittest.skipIf(MEMORY_SANITIZER, "MSan defaults to crashing " + "instead of returning NULL for malloc failure.") def test_constructor(self): BufferedReaderTest.test_constructor(self) # The allocation can succeed on 32-bit builds, e.g. with more @@ -1840,6 +1850,8 @@ def test_slow_close_from_thread(self): class CBufferedWriterTest(BufferedWriterTest, SizeofTest): tp = io.BufferedWriter + @unittest.skipIf(MEMORY_SANITIZER, "MSan defaults to crashing " + "instead of returning NULL for malloc failure.") def test_constructor(self): BufferedWriterTest.test_constructor(self) # The allocation can succeed on 32-bit builds, e.g. with more @@ -2314,6 +2326,8 @@ def test_interleaved_readline_write(self): class CBufferedRandomTest(BufferedRandomTest, SizeofTest): tp = io.BufferedRandom + @unittest.skipIf(MEMORY_SANITIZER, "MSan defaults to crashing " + "instead of returning NULL for malloc failure.") def test_constructor(self): BufferedRandomTest.test_constructor(self) # The allocation can succeed on 32-bit builds, e.g. with more @@ -3635,6 +3649,11 @@ def test_rwpair_cleared_before_textio(self): t2.buddy = t1 support.gc_collect() + def test_del__CHUNK_SIZE_SystemError(self): + t = self.TextIOWrapper(self.BytesIO(), encoding='ascii') + with self.assertRaises(AttributeError): + del t._CHUNK_SIZE + class PyTextIOWrapperTest(TextIOWrapperTest): io = pyio @@ -4116,10 +4135,9 @@ def check_interrupted_write(self, item, bytes, **fdopen_kwargs): in the latter.""" read_results = [] def _read(): - if hasattr(signal, 'pthread_sigmask'): - signal.pthread_sigmask(signal.SIG_BLOCK, [signal.SIGALRM]) s = os.read(r, 1) read_results.append(s) + t = threading.Thread(target=_read) t.daemon = True r, w = os.pipe() @@ -4127,7 +4145,14 @@ def _read(): large_data = item * (support.PIPE_MAX_SIZE // len(item) + 1) try: wio = self.io.open(w, **fdopen_kwargs) - t.start() + if hasattr(signal, 'pthread_sigmask'): + # create the thread with SIGALRM signal blocked + signal.pthread_sigmask(signal.SIG_BLOCK, [signal.SIGALRM]) + t.start() + signal.pthread_sigmask(signal.SIG_UNBLOCK, [signal.SIGALRM]) + else: + t.start() + # Fill the pipe enough that the write will be blocking. # It will be interrupted by the timer armed above. Since the # other thread has read one byte, the low-level write will diff --git a/kbe/src/lib/python/Lib/test/test_msilib.py b/kbe/src/lib/python/Lib/test/test_msilib.py index 6d89ca4b77..4aa4753fc2 100644 --- a/kbe/src/lib/python/Lib/test/test_msilib.py +++ b/kbe/src/lib/python/Lib/test/test_msilib.py @@ -1,5 +1,5 @@ """ Test suite for the code in msilib """ -import os.path +import os import unittest from test.support import TESTFN, import_module, unlink msilib = import_module('msilib') @@ -42,6 +42,29 @@ def test_view_fetch_returns_none(self): ) self.addCleanup(unlink, db_path) + def test_summaryinfo_getproperty_issue1104(self): + db, db_path = init_database() + try: + sum_info = db.GetSummaryInformation(99) + title = sum_info.GetProperty(msilib.PID_TITLE) + self.assertEqual(title, b"Installation Database") + + sum_info.SetProperty(msilib.PID_TITLE, "a" * 999) + title = sum_info.GetProperty(msilib.PID_TITLE) + self.assertEqual(title, b"a" * 999) + + sum_info.SetProperty(msilib.PID_TITLE, "a" * 1000) + title = sum_info.GetProperty(msilib.PID_TITLE) + self.assertEqual(title, b"a" * 1000) + + sum_info.SetProperty(msilib.PID_TITLE, "a" * 1001) + title = sum_info.GetProperty(msilib.PID_TITLE) + self.assertEqual(title, b"a" * 1001) + finally: + db = None + sum_info = None + os.unlink(db_path) + def test_database_open_failed(self): with self.assertRaises(msilib.MSIError) as cm: msilib.OpenDatabase('non-existent.msi', msilib.MSIDBOPEN_READONLY) @@ -92,7 +115,7 @@ def test_invalid_first_char(self): def test_invalid_any_char(self): self.assertEqual( msilib.make_id(".s\x82ort"), "_.s_ort") - self.assertEqual ( + self.assertEqual( msilib.make_id(".s\x82o?*+rt"), "_.s_o___rt") diff --git a/kbe/src/lib/python/Lib/test/test_multiprocessing_main_handling.py b/kbe/src/lib/python/Lib/test/test_multiprocessing_main_handling.py index 9fd5c9fcd9..b6abfcc7e2 100644 --- a/kbe/src/lib/python/Lib/test/test_multiprocessing_main_handling.py +++ b/kbe/src/lib/python/Lib/test/test_multiprocessing_main_handling.py @@ -54,18 +54,21 @@ def f(x): if __name__ == '__main__': start_method = sys.argv[1] set_start_method(start_method) - p = Pool(5) results = [] - p.map_async(f, [1, 2, 3], callback=results.extend) - start_time = time.monotonic() - while not results: - time.sleep(0.05) - # up to 1 min to report the results - dt = time.monotonic() - start_time - if dt > 60.0: - raise RuntimeError("Timed out waiting for results (%.1f sec)" % dt) + with Pool(5) as pool: + pool.map_async(f, [1, 2, 3], callback=results.extend) + start_time = time.monotonic() + while not results: + time.sleep(0.05) + # up to 1 min to report the results + dt = time.monotonic() - start_time + if dt > 60.0: + raise RuntimeError("Timed out waiting for results (%.1f sec)" % dt) + results.sort() print(start_method, "->", results) + + pool.join() """ test_source_main_skipped_in_children = """\ @@ -84,18 +87,21 @@ def f(x): start_method = sys.argv[1] set_start_method(start_method) -p = Pool(5) results = [] -p.map_async(int, [1, 4, 9], callback=results.extend) -start_time = time.monotonic() -while not results: - time.sleep(0.05) - # up to 1 min to report the results - dt = time.monotonic() - start_time - if dt > 60.0: - raise RuntimeError("Timed out waiting for results (%.1f sec)" % dt) +with Pool(5) as pool: + pool.map_async(int, [1, 4, 9], callback=results.extend) + start_time = time.monotonic() + while not results: + time.sleep(0.05) + # up to 1 min to report the results + dt = time.monotonic() - start_time + if dt > 60.0: + raise RuntimeError("Timed out waiting for results (%.1f sec)" % dt) + results.sort() print(start_method, "->", results) + +pool.join() """ # These helpers were copied from test_cmd_line_script & tweaked a bit... diff --git a/kbe/src/lib/python/Lib/test/test_os.py b/kbe/src/lib/python/Lib/test/test_os.py index fe7261dd76..fd9f70e30d 100644 --- a/kbe/src/lib/python/Lib/test/test_os.py +++ b/kbe/src/lib/python/Lib/test/test_os.py @@ -923,7 +923,7 @@ def setUp(self): ["broken_link", "broken_link2", "broken_link3", "tmp3"]) else: - self.sub2_tree = (sub2_path, [], ["tmp3"]) + self.sub2_tree = (sub2_path, ["SUB21"], ["tmp3"]) os.chmod(sub21_path, 0) try: @@ -1798,36 +1798,46 @@ def test_unicode_name(self): @unittest.skipIf(sys.platform == "win32", "Posix specific tests") class PosixUidGidTests(unittest.TestCase): + # uid_t and gid_t are 32-bit unsigned integers on Linux + UID_OVERFLOW = (1 << 32) + GID_OVERFLOW = (1 << 32) + @unittest.skipUnless(hasattr(os, 'setuid'), 'test needs os.setuid()') def test_setuid(self): if os.getuid() != 0: self.assertRaises(OSError, os.setuid, 0) - self.assertRaises(OverflowError, os.setuid, 1<<32) + self.assertRaises(TypeError, os.setuid, 'not an int') + self.assertRaises(OverflowError, os.setuid, self.UID_OVERFLOW) @unittest.skipUnless(hasattr(os, 'setgid'), 'test needs os.setgid()') def test_setgid(self): if os.getuid() != 0 and not HAVE_WHEEL_GROUP: self.assertRaises(OSError, os.setgid, 0) - self.assertRaises(OverflowError, os.setgid, 1<<32) + self.assertRaises(TypeError, os.setgid, 'not an int') + self.assertRaises(OverflowError, os.setgid, self.GID_OVERFLOW) @unittest.skipUnless(hasattr(os, 'seteuid'), 'test needs os.seteuid()') def test_seteuid(self): if os.getuid() != 0: self.assertRaises(OSError, os.seteuid, 0) - self.assertRaises(OverflowError, os.seteuid, 1<<32) + self.assertRaises(TypeError, os.setegid, 'not an int') + self.assertRaises(OverflowError, os.seteuid, self.UID_OVERFLOW) @unittest.skipUnless(hasattr(os, 'setegid'), 'test needs os.setegid()') def test_setegid(self): if os.getuid() != 0 and not HAVE_WHEEL_GROUP: self.assertRaises(OSError, os.setegid, 0) - self.assertRaises(OverflowError, os.setegid, 1<<32) + self.assertRaises(TypeError, os.setegid, 'not an int') + self.assertRaises(OverflowError, os.setegid, self.GID_OVERFLOW) @unittest.skipUnless(hasattr(os, 'setreuid'), 'test needs os.setreuid()') def test_setreuid(self): if os.getuid() != 0: self.assertRaises(OSError, os.setreuid, 0, 0) - self.assertRaises(OverflowError, os.setreuid, 1<<32, 0) - self.assertRaises(OverflowError, os.setreuid, 0, 1<<32) + self.assertRaises(TypeError, os.setreuid, 'not an int', 0) + self.assertRaises(TypeError, os.setreuid, 0, 'not an int') + self.assertRaises(OverflowError, os.setreuid, self.UID_OVERFLOW, 0) + self.assertRaises(OverflowError, os.setreuid, 0, self.UID_OVERFLOW) @unittest.skipUnless(hasattr(os, 'setreuid'), 'test needs os.setreuid()') def test_setreuid_neg1(self): @@ -1841,8 +1851,10 @@ def test_setreuid_neg1(self): def test_setregid(self): if os.getuid() != 0 and not HAVE_WHEEL_GROUP: self.assertRaises(OSError, os.setregid, 0, 0) - self.assertRaises(OverflowError, os.setregid, 1<<32, 0) - self.assertRaises(OverflowError, os.setregid, 0, 1<<32) + self.assertRaises(TypeError, os.setregid, 'not an int', 0) + self.assertRaises(TypeError, os.setregid, 0, 'not an int') + self.assertRaises(OverflowError, os.setregid, self.GID_OVERFLOW, 0) + self.assertRaises(OverflowError, os.setregid, 0, self.GID_OVERFLOW) @unittest.skipUnless(hasattr(os, 'setregid'), 'test needs os.setregid()') def test_setregid_neg1(self): @@ -2278,6 +2290,62 @@ def test_unlink_removes_junction(self): os.unlink(self.junction) self.assertFalse(os.path.exists(self.junction)) +@unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") +class Win32NtTests(unittest.TestCase): + def setUp(self): + from test import support + self.nt = support.import_module('nt') + pass + + def tearDown(self): + pass + + def test_getfinalpathname_handles(self): + try: + import ctypes, ctypes.wintypes + except ImportError: + raise unittest.SkipTest('ctypes module is required for this test') + + kernel = ctypes.WinDLL('Kernel32.dll', use_last_error=True) + kernel.GetCurrentProcess.restype = ctypes.wintypes.HANDLE + + kernel.GetProcessHandleCount.restype = ctypes.wintypes.BOOL + kernel.GetProcessHandleCount.argtypes = (ctypes.wintypes.HANDLE, + ctypes.wintypes.LPDWORD) + + # This is a pseudo-handle that doesn't need to be closed + hproc = kernel.GetCurrentProcess() + + handle_count = ctypes.wintypes.DWORD() + ok = kernel.GetProcessHandleCount(hproc, ctypes.byref(handle_count)) + self.assertEqual(1, ok) + + before_count = handle_count.value + + # The first two test the error path, __file__ tests the success path + filenames = [ r'\\?\C:', + r'\\?\NUL', + r'\\?\CONIN', + __file__ ] + + for i in range(10): + for name in filenames: + try: + tmp = self.nt._getfinalpathname(name) + except: + # Failure is expected + pass + try: + tmp = os.stat(name) + except: + pass + + ok = kernel.GetProcessHandleCount(hproc, ctypes.byref(handle_count)) + self.assertEqual(1, ok) + + handle_delta = handle_count.value - before_count + + self.assertEqual(0, handle_delta) @support.skip_unless_symlink class NonLocalSymlinkTests(unittest.TestCase): @@ -3241,7 +3309,7 @@ def test_path_t_converter(self): cleanup_fn(result) with self.assertRaisesRegex( - TypeError, 'should be string, bytes'): + TypeError, 'to return str or bytes'): fn(int_fspath, *extra_args) if allow_fd: @@ -3254,6 +3322,15 @@ def test_path_t_converter(self): 'os.PathLike'): fn(fd, *extra_args) + def test_path_t_converter_and_custom_class(self): + msg = r'__fspath__\(\) to return str or bytes, not %s' + with self.assertRaisesRegex(TypeError, msg % r'int'): + os.stat(FakePath(2)) + with self.assertRaisesRegex(TypeError, msg % r'float'): + os.stat(FakePath(2.34)) + with self.assertRaisesRegex(TypeError, msg % r'object'): + os.stat(FakePath(object())) + @unittest.skipUnless(hasattr(os, 'get_blocking'), 'needs os.get_blocking() and os.set_blocking()') diff --git a/kbe/src/lib/python/Lib/test/test_ossaudiodev.py b/kbe/src/lib/python/Lib/test/test_ossaudiodev.py index c9e2a24767..624fbf21ba 100644 --- a/kbe/src/lib/python/Lib/test/test_ossaudiodev.py +++ b/kbe/src/lib/python/Lib/test/test_ossaudiodev.py @@ -77,10 +77,10 @@ def play_sound_file(self, data, rate, ssize, nchannels): # set parameters based on .au file headers dsp.setparameters(AFMT_S16_NE, nchannels, rate) self.assertTrue(abs(expected_time - 3.51) < 1e-2, expected_time) - t1 = time.time() + t1 = time.monotonic() dsp.write(data) dsp.close() - t2 = time.time() + t2 = time.monotonic() elapsed_time = t2 - t1 percent_diff = (abs(elapsed_time - expected_time) / expected_time) * 100 diff --git a/kbe/src/lib/python/Lib/test/test_pdb.py b/kbe/src/lib/python/Lib/test/test_pdb.py index 9aa38e08dd..f573f5f54b 100644 --- a/kbe/src/lib/python/Lib/test/test_pdb.py +++ b/kbe/src/lib/python/Lib/test/test_pdb.py @@ -1482,6 +1482,28 @@ def test_relative_imports_on_plain_module(self): stdout, _ = self._run_pdb(['-m', self.module_name + '.runme'], commands) self.assertTrue(any("VAR from module" in l for l in stdout.splitlines()), stdout) + def test_errors_in_command(self): + commands = "\n".join([ + 'print(', + 'debug print(', + 'debug doesnotexist', + 'c', + ]) + stdout, _ = self.run_pdb_script('', commands + '\n') + + self.assertEqual(stdout.splitlines()[1:], [ + '(Pdb) *** SyntaxError: unexpected EOF while parsing', + + '(Pdb) ENTERING RECURSIVE DEBUGGER', + '*** SyntaxError: unexpected EOF while parsing', + 'LEAVING RECURSIVE DEBUGGER', + + '(Pdb) ENTERING RECURSIVE DEBUGGER', + '> (1)()', + "((Pdb)) *** NameError: name 'doesnotexist' is not defined", + 'LEAVING RECURSIVE DEBUGGER', + '(Pdb) ', + ]) def load_tests(*args): from test import test_pdb diff --git a/kbe/src/lib/python/Lib/test/test_pydoc.py b/kbe/src/lib/python/Lib/test/test_pydoc.py index 9bf365ed4b..198cea93eb 100644 --- a/kbe/src/lib/python/Lib/test/test_pydoc.py +++ b/kbe/src/lib/python/Lib/test/test_pydoc.py @@ -1013,12 +1013,12 @@ def my_url_handler(url, content_type): serverthread = pydoc._start_server(my_url_handler, hostname='0.0.0.0', port=0) self.assertIn('0.0.0.0', serverthread.docserver.address) - starttime = time.time() + starttime = time.monotonic() timeout = 1 #seconds while serverthread.serving: time.sleep(.01) - if serverthread.serving and time.time() - starttime > timeout: + if serverthread.serving and time.monotonic() - starttime > timeout: serverthread.stop() break diff --git a/kbe/src/lib/python/Lib/test/test_re.py b/kbe/src/lib/python/Lib/test/test_re.py index 9fed4bef88..5ef6d7b12c 100644 --- a/kbe/src/lib/python/Lib/test/test_re.py +++ b/kbe/src/lib/python/Lib/test/test_re.py @@ -1516,8 +1516,18 @@ def test_ascii_and_unicode_flag(self): self.assertRaises(re.error, re.compile, r'(?au)\w') def test_locale_flag(self): - import locale - _, enc = locale.getlocale(locale.LC_CTYPE) + # On Windows, Python 3.7 doesn't call setlocale(LC_CTYPE, "") at + # startup and so the LC_CTYPE locale uses Latin1 encoding by default, + # whereas getpreferredencoding() returns the ANSI code page. Set + # temporarily the LC_CTYPE locale to the user preferred encoding to + # ensure that it uses the ANSI code page. + oldloc = locale.setlocale(locale.LC_CTYPE, None) + locale.setlocale(locale.LC_CTYPE, "") + self.addCleanup(locale.setlocale, locale.LC_CTYPE, oldloc) + + # Get the current locale encoding + enc = locale.getpreferredencoding(False) + # Search non-ASCII letter for i in range(128, 256): try: @@ -2031,6 +2041,40 @@ def test_bug_29444(self): self.assertEqual(m.group(), b'xyz') self.assertEqual(m2.group(), b'') + def test_bug_34294(self): + # Issue 34294: wrong capturing groups + + # exists since Python 2 + s = "a\tx" + p = r"\b(?=(\t)|(x))x" + self.assertEqual(re.search(p, s).groups(), (None, 'x')) + + # introduced in Python 3.7.0 + s = "ab" + p = r"(?=(.)(.)?)" + self.assertEqual(re.findall(p, s), + [('a', 'b'), ('b', '')]) + self.assertEqual([m.groups() for m in re.finditer(p, s)], + [('a', 'b'), ('b', None)]) + + # test-cases provided by issue34294, introduced in Python 3.7.0 + p = r"(?=<(?P\w+)/?>(?:(?P.+?))?)" + s = "" + self.assertEqual(re.findall(p, s), + [('test', ''), ('foo2', '')]) + self.assertEqual([m.groupdict() for m in re.finditer(p, s)], + [{'tag': 'test', 'text': ''}, + {'tag': 'foo2', 'text': None}]) + s = "Hello" + self.assertEqual([m.groupdict() for m in re.finditer(p, s)], + [{'tag': 'test', 'text': 'Hello'}, + {'tag': 'foo', 'text': None}]) + s = "Hello" + self.assertEqual([m.groupdict() for m in re.finditer(p, s)], + [{'tag': 'test', 'text': 'Hello'}, + {'tag': 'foo', 'text': None}, + {'tag': 'foo', 'text': None}]) + class PatternReprTests(unittest.TestCase): def check(self, pattern, expected): diff --git a/kbe/src/lib/python/Lib/test/test_regrtest.py b/kbe/src/lib/python/Lib/test/test_regrtest.py index db9bd6dfc0..a67458313a 100644 --- a/kbe/src/lib/python/Lib/test/test_regrtest.py +++ b/kbe/src/lib/python/Lib/test/test_regrtest.py @@ -1004,6 +1004,7 @@ def test_bug(self): output = self.run_tests("-w", testname, exitcode=2) self.check_executed_tests(output, [testname], failed=testname, rerun=testname) + def test_no_tests_ran(self): code = textwrap.dedent(""" import unittest @@ -1017,6 +1018,19 @@ def test_bug(self): output = self.run_tests(testname, "-m", "nosuchtest", exitcode=0) self.check_executed_tests(output, [testname], no_test_ran=testname) + def test_no_tests_ran_skip(self): + code = textwrap.dedent(""" + import unittest + + class Tests(unittest.TestCase): + def test_skipped(self): + self.skipTest("because") + """) + testname = self.create_test(code=code) + + output = self.run_tests(testname, exitcode=0) + self.check_executed_tests(output, [testname]) + def test_no_tests_ran_multiple_tests_nonexistent(self): code = textwrap.dedent(""" import unittest diff --git a/kbe/src/lib/python/Lib/test/test_signal.py b/kbe/src/lib/python/Lib/test/test_signal.py index 24cab0f89e..406684bdbe 100644 --- a/kbe/src/lib/python/Lib/test/test_signal.py +++ b/kbe/src/lib/python/Lib/test/test_signal.py @@ -1122,18 +1122,18 @@ def second_handler(signum=None, frame=None): self.setsig(signal.SIGALRM, second_handler) # for ITIMER_REAL expected_sigs = 0 - deadline = time.time() + 15.0 + deadline = time.monotonic() + 15.0 while expected_sigs < N: os.kill(os.getpid(), signal.SIGPROF) expected_sigs += 1 # Wait for handlers to run to avoid signal coalescing - while len(sigs) < expected_sigs and time.time() < deadline: + while len(sigs) < expected_sigs and time.monotonic() < deadline: time.sleep(1e-5) os.kill(os.getpid(), signal.SIGUSR1) expected_sigs += 1 - while len(sigs) < expected_sigs and time.time() < deadline: + while len(sigs) < expected_sigs and time.monotonic() < deadline: time.sleep(1e-5) # All ITIMER_REAL signals should have been delivered to the @@ -1156,7 +1156,7 @@ def handler(signum, frame): self.setsig(signal.SIGALRM, handler) # for ITIMER_REAL expected_sigs = 0 - deadline = time.time() + 15.0 + deadline = time.monotonic() + 15.0 while expected_sigs < N: # Hopefully the SIGALRM will be received somewhere during @@ -1166,7 +1166,7 @@ def handler(signum, frame): expected_sigs += 2 # Wait for handlers to run to avoid signal coalescing - while len(sigs) < expected_sigs and time.time() < deadline: + while len(sigs) < expected_sigs and time.monotonic() < deadline: time.sleep(1e-5) # All ITIMER_REAL signals should have been delivered to the diff --git a/kbe/src/lib/python/Lib/test/test_site.py b/kbe/src/lib/python/Lib/test/test_site.py index a30bd2f006..655a12ddd1 100644 --- a/kbe/src/lib/python/Lib/test/test_site.py +++ b/kbe/src/lib/python/Lib/test/test_site.py @@ -183,7 +183,9 @@ def test_addsitedir(self): finally: pth_file.cleanup() - def test_getuserbase(self): + # This tests _getuserbase, hence the double underline + # to distinguish from a test for getuserbase + def test__getuserbase(self): self.assertEqual(site._getuserbase(), sysconfig._getuserbase()) def test_get_path(self): diff --git a/kbe/src/lib/python/Lib/test/test_socket.py b/kbe/src/lib/python/Lib/test/test_socket.py index 9b9d11368c..3819652896 100644 --- a/kbe/src/lib/python/Lib/test/test_socket.py +++ b/kbe/src/lib/python/Lib/test/test_socket.py @@ -5533,7 +5533,7 @@ def tearDownClass(cls): support.unlink(support.TESTFN) def accept_conn(self): - self.serv.settimeout(self.TIMEOUT) + self.serv.settimeout(MAIN_TIMEOUT) conn, addr = self.serv.accept() conn.settimeout(self.TIMEOUT) self.addCleanup(conn.close) @@ -5717,11 +5717,11 @@ def testWithTimeout(self): def _testWithTimeoutTriggeredSend(self): address = self.serv.getsockname() - file = open(support.TESTFN, 'rb') - with socket.create_connection(address, timeout=0.01) as sock, \ - file as file: - meth = self.meth_from_sock(sock) - self.assertRaises(socket.timeout, meth, file) + with open(support.TESTFN, 'rb') as file: + with socket.create_connection(address) as sock: + sock.settimeout(0.01) + meth = self.meth_from_sock(sock) + self.assertRaises(socket.timeout, meth, file) def testWithTimeoutTriggeredSend(self): conn = self.accept_conn() diff --git a/kbe/src/lib/python/Lib/test/test_ssl.py b/kbe/src/lib/python/Lib/test/test_ssl.py index f1b9565c8d..8eab447415 100644 --- a/kbe/src/lib/python/Lib/test/test_ssl.py +++ b/kbe/src/lib/python/Lib/test/test_ssl.py @@ -34,6 +34,19 @@ IS_OPENSSL_1_1_1 = not IS_LIBRESSL and ssl.OPENSSL_VERSION_INFO >= (1, 1, 1) PY_SSL_DEFAULT_CIPHERS = sysconfig.get_config_var('PY_SSL_DEFAULT_CIPHERS') +PROTOCOL_TO_TLS_VERSION = {} +for proto, ver in ( + ("PROTOCOL_SSLv23", "SSLv3"), + ("PROTOCOL_TLSv1", "TLSv1"), + ("PROTOCOL_TLSv1_1", "TLSv1_1"), +): + try: + proto = getattr(ssl, proto) + ver = getattr(ssl.TLSVersion, ver) + except AttributeError: + continue + PROTOCOL_TO_TLS_VERSION[proto] = ver + def data_file(*name): return os.path.join(os.path.dirname(__file__), *name) @@ -116,6 +129,7 @@ def data_file(*name): BADKEY = data_file("badkey.pem") NOKIACERT = data_file("nokia.pem") NULLBYTECERT = data_file("nullbytecert.pem") +TALOS_INVALID_CRLDP = data_file("talos-2019-0758.pem") DHFILE = data_file("ffdh3072.pem") BYTES_DHFILE = os.fsencode(DHFILE) @@ -365,6 +379,27 @@ def test_parse_cert(self): self.assertEqual(p['crlDistributionPoints'], ('http://SVRIntl-G3-crl.verisign.com/SVRIntlG3.crl',)) + def test_parse_cert_CVE_2019_5010(self): + p = ssl._ssl._test_decode_cert(TALOS_INVALID_CRLDP) + if support.verbose: + sys.stdout.write("\n" + pprint.pformat(p) + "\n") + self.assertEqual( + p, + { + 'issuer': ( + (('countryName', 'UK'),), (('commonName', 'cody-ca'),)), + 'notAfter': 'Jun 14 18:00:58 2028 GMT', + 'notBefore': 'Jun 18 18:00:58 2018 GMT', + 'serialNumber': '02', + 'subject': ((('countryName', 'UK'),), + (('commonName', + 'codenomicon-vm-2.test.lal.cisco.com'),)), + 'subjectAltName': ( + ('DNS', 'codenomicon-vm-2.test.lal.cisco.com'),), + 'version': 3 + } + ) + def test_parse_cert_CVE_2013_4238(self): p = ssl._ssl._test_decode_cert(NULLBYTECERT) if support.verbose: @@ -1086,8 +1121,15 @@ def test_hostname_checks_common_name(self): "required OpenSSL 1.1.0g") def test_min_max_version(self): ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) - self.assertEqual( - ctx.minimum_version, ssl.TLSVersion.MINIMUM_SUPPORTED + # OpenSSL default is MINIMUM_SUPPORTED, however some vendors like + # Fedora override the setting to TLS 1.0. + self.assertIn( + ctx.minimum_version, + {ssl.TLSVersion.MINIMUM_SUPPORTED, + # Fedora 29 uses TLS 1.0 by default + ssl.TLSVersion.TLSv1, + # RHEL 8 uses TLS 1.2 by default + ssl.TLSVersion.TLSv1_2} ) self.assertEqual( ctx.maximum_version, ssl.TLSVersion.MAXIMUM_SUPPORTED @@ -2605,6 +2647,17 @@ def try_protocol_combo(server_protocol, client_protocol, expect_success, server_context = ssl.SSLContext(server_protocol) server_context.options |= server_options + min_version = PROTOCOL_TO_TLS_VERSION.get(client_protocol, None) + if (min_version is not None + # SSLContext.minimum_version is only available on recent OpenSSL + # (setter added in OpenSSL 1.1.0, getter added in OpenSSL 1.1.1) + and hasattr(server_context, 'minimum_version') + and server_protocol == ssl.PROTOCOL_TLS + and server_context.minimum_version > min_version): + # If OpenSSL configuration is strict and requires more recent TLS + # version, we have to change the minimum to test old TLS versions. + server_context.minimum_version = min_version + # NOTE: we must enable "ALL" ciphers on the client, otherwise an # SSLv23 client will send an SSLv3 hello (rather than SSLv2) # starting from OpenSSL 1.0.0 (see issue #8322). diff --git a/kbe/src/lib/python/Lib/test/test_symbol.py b/kbe/src/lib/python/Lib/test/test_symbol.py index c1306f5432..32564f3df2 100644 --- a/kbe/src/lib/python/Lib/test/test_symbol.py +++ b/kbe/src/lib/python/Lib/test/test_symbol.py @@ -2,6 +2,7 @@ from test import support import os import sys +import sysconfig import subprocess @@ -35,8 +36,8 @@ def compare_files(self, file1, file2): lines2 = fp.readlines() self.assertEqual(lines1, lines2) - @unittest.skipIf(not os.path.exists(GRAMMAR_FILE), - 'test only works from source build directory') + @unittest.skipUnless(sysconfig.is_python_build(), + 'test only works from source build directory') def test_real_grammar_and_symbol_file(self): output = support.TESTFN self.addCleanup(support.unlink, output) diff --git a/kbe/src/lib/python/Lib/test/test_tarfile.py b/kbe/src/lib/python/Lib/test/test_tarfile.py index 7d2eec8a7c..5e4d75ecfc 100644 --- a/kbe/src/lib/python/Lib/test/test_tarfile.py +++ b/kbe/src/lib/python/Lib/test/test_tarfile.py @@ -973,16 +973,21 @@ def test_sparse_file_10(self): def _fs_supports_holes(): # Return True if the platform knows the st_blocks stat attribute and # uses st_blocks units of 512 bytes, and if the filesystem is able to - # store holes in files. + # store holes of 4 KiB in files. + # + # The function returns False if page size is larger than 4 KiB. + # For example, ppc64 uses pages of 64 KiB. if sys.platform.startswith("linux"): # Linux evidentially has 512 byte st_blocks units. name = os.path.join(TEMPDIR, "sparse-test") with open(name, "wb") as fobj: + # Seek to "punch a hole" of 4 KiB fobj.seek(4096) + fobj.write(b'x' * 4096) fobj.truncate() s = os.stat(name) support.unlink(name) - return s.st_blocks == 0 + return (s.st_blocks * 512 < s.st_size) else: return False diff --git a/kbe/src/lib/python/Lib/test/test_threading.py b/kbe/src/lib/python/Lib/test/test_threading.py index 8160a5af00..27f328dbe6 100644 --- a/kbe/src/lib/python/Lib/test/test_threading.py +++ b/kbe/src/lib/python/Lib/test/test_threading.py @@ -415,7 +415,8 @@ def test_old_threading_api(self): t.setDaemon(True) t.getName() t.setName("name") - t.isAlive() + with self.assertWarnsRegex(PendingDeprecationWarning, 'use is_alive()'): + t.isAlive() e = threading.Event() e.isSet() threading.activeCount() diff --git a/kbe/src/lib/python/Lib/test/test_threading_local.py b/kbe/src/lib/python/Lib/test/test_threading_local.py index 984f8dda37..2fd14ae2e1 100644 --- a/kbe/src/lib/python/Lib/test/test_threading_local.py +++ b/kbe/src/lib/python/Lib/test/test_threading_local.py @@ -1,3 +1,4 @@ +import sys import unittest from doctest import DocTestSuite from test import support diff --git a/kbe/src/lib/python/Lib/test/test_threadsignals.py b/kbe/src/lib/python/Lib/test/test_threadsignals.py index 7e13b1720f..eeacd3698c 100644 --- a/kbe/src/lib/python/Lib/test/test_threadsignals.py +++ b/kbe/src/lib/python/Lib/test/test_threadsignals.py @@ -95,9 +95,9 @@ def test_lock_acquire_interruption(self): lock = thread.allocate_lock() lock.acquire() signal.alarm(1) - t1 = time.time() + t1 = time.monotonic() self.assertRaises(KeyboardInterrupt, lock.acquire, timeout=5) - dt = time.time() - t1 + dt = time.monotonic() - t1 # Checking that KeyboardInterrupt was raised is not sufficient. # We want to assert that lock.acquire() was interrupted because # of the signal, not that the signal handler was called immediately @@ -136,9 +136,9 @@ def other_thread(): rlock.release() time.sleep(0.01) signal.alarm(1) - t1 = time.time() + t1 = time.monotonic() self.assertRaises(KeyboardInterrupt, rlock.acquire, timeout=5) - dt = time.time() - t1 + dt = time.monotonic() - t1 # See rationale above in test_lock_acquire_interruption self.assertLess(dt, 3.0) finally: @@ -203,9 +203,9 @@ def my_handler(signum, frame): old_handler = signal.signal(signal.SIGUSR1, my_handler) try: def timed_acquire(): - self.start = time.time() + self.start = time.monotonic() lock.acquire(timeout=0.5) - self.end = time.time() + self.end = time.monotonic() def send_signals(): for _ in range(40): time.sleep(0.02) diff --git a/kbe/src/lib/python/Lib/test/test_timeout.py b/kbe/src/lib/python/Lib/test/test_timeout.py index 3c75dcc6f9..b54fc826ae 100644 --- a/kbe/src/lib/python/Lib/test/test_timeout.py +++ b/kbe/src/lib/python/Lib/test/test_timeout.py @@ -127,11 +127,11 @@ def _sock_operation(self, count, timeout, method, *args): self.sock.settimeout(timeout) method = getattr(self.sock, method) for i in range(count): - t1 = time.time() + t1 = time.monotonic() try: method(*args) except socket.timeout as e: - delta = time.time() - t1 + delta = time.monotonic() - t1 break else: self.fail('socket.timeout was not raised') diff --git a/kbe/src/lib/python/Lib/test/test_unicode.py b/kbe/src/lib/python/Lib/test/test_unicode.py index 3cc018c0cc..1aad933407 100644 --- a/kbe/src/lib/python/Lib/test/test_unicode.py +++ b/kbe/src/lib/python/Lib/test/test_unicode.py @@ -2676,6 +2676,12 @@ def check_format(expected, format, *args): check_format('%.%s', b'%.%s', b'abc') + # Issue #33817: empty strings + check_format('', + b'') + check_format('', + b'%s', b'') + # Test PyUnicode_AsWideChar() @support.cpython_only def test_aswidechar(self): diff --git a/kbe/src/lib/python/Lib/test/test_urllib.py b/kbe/src/lib/python/Lib/test/test_urllib.py index c292d74f84..2ac73b58d8 100644 --- a/kbe/src/lib/python/Lib/test/test_urllib.py +++ b/kbe/src/lib/python/Lib/test/test_urllib.py @@ -712,7 +712,7 @@ def _reporthook(par1, par2, par3): with self.assertRaises(urllib.error.ContentTooShortError): try: - urllib.request.urlretrieve('http://example.com/', + urllib.request.urlretrieve(support.TEST_HTTP_URL, reporthook=_reporthook) finally: self.unfakehttp() @@ -729,7 +729,7 @@ def test_short_content_raises_ContentTooShortError_without_reporthook(self): ''') with self.assertRaises(urllib.error.ContentTooShortError): try: - urllib.request.urlretrieve('http://example.com/') + urllib.request.urlretrieve(support.TEST_HTTP_URL) finally: self.unfakehttp() diff --git a/kbe/src/lib/python/Lib/test/test_urllib2net.py b/kbe/src/lib/python/Lib/test/test_urllib2net.py index 1aa64cbee1..0f43d71ea6 100644 --- a/kbe/src/lib/python/Lib/test/test_urllib2net.py +++ b/kbe/src/lib/python/Lib/test/test_urllib2net.py @@ -84,7 +84,7 @@ class CloseSocketTest(unittest.TestCase): def test_close(self): # calling .close() on urllib2's response objects should close the # underlying socket - url = "http://www.example.com/" + url = support.TEST_HTTP_URL with support.transient_internet(url): response = _urlopen_with_retry(url) sock = response.fp @@ -173,7 +173,7 @@ def test_redirect_url_withfrag(self): "http://www.pythontest.net/elsewhere/#frag") def test_custom_headers(self): - url = "http://www.example.com" + url = support.TEST_HTTP_URL with support.transient_internet(url): opener = urllib.request.build_opener() request = urllib.request.Request(url) @@ -259,7 +259,7 @@ def _extra_handlers(self): class TimeoutTest(unittest.TestCase): def test_http_basic(self): self.assertIsNone(socket.getdefaulttimeout()) - url = "http://www.example.com" + url = support.TEST_HTTP_URL with support.transient_internet(url, timeout=None): u = _urlopen_with_retry(url) self.addCleanup(u.close) @@ -267,7 +267,7 @@ def test_http_basic(self): def test_http_default_timeout(self): self.assertIsNone(socket.getdefaulttimeout()) - url = "http://www.example.com" + url = support.TEST_HTTP_URL with support.transient_internet(url): socket.setdefaulttimeout(60) try: @@ -279,7 +279,7 @@ def test_http_default_timeout(self): def test_http_no_timeout(self): self.assertIsNone(socket.getdefaulttimeout()) - url = "http://www.example.com" + url = support.TEST_HTTP_URL with support.transient_internet(url): socket.setdefaulttimeout(60) try: @@ -290,7 +290,7 @@ def test_http_no_timeout(self): self.assertIsNone(u.fp.raw._sock.gettimeout()) def test_http_timeout(self): - url = "http://www.example.com" + url = support.TEST_HTTP_URL with support.transient_internet(url): u = _urlopen_with_retry(url, timeout=120) self.addCleanup(u.close) diff --git a/kbe/src/lib/python/Lib/test/test_urllibnet.py b/kbe/src/lib/python/Lib/test/test_urllibnet.py index 4103b6c075..d394ceddd0 100644 --- a/kbe/src/lib/python/Lib/test/test_urllibnet.py +++ b/kbe/src/lib/python/Lib/test/test_urllibnet.py @@ -3,6 +3,7 @@ import contextlib import socket +import urllib.parse import urllib.request import os import email.message @@ -24,8 +25,9 @@ def tearDown(self): socket.setdefaulttimeout(None) def testURLread(self): - with support.transient_internet("www.example.com"): - f = urllib.request.urlopen("http://www.example.com/") + domain = urllib.parse.urlparse(support.TEST_HTTP_URL).netloc + with support.transient_internet(domain): + f = urllib.request.urlopen(support.TEST_HTTP_URL) f.read() diff --git a/kbe/src/lib/python/Lib/test/test_urlparse.py b/kbe/src/lib/python/Lib/test/test_urlparse.py index be50b47603..e6638aee22 100644 --- a/kbe/src/lib/python/Lib/test/test_urlparse.py +++ b/kbe/src/lib/python/Lib/test/test_urlparse.py @@ -1,3 +1,5 @@ +import sys +import unicodedata import unittest import urllib.parse @@ -984,6 +986,27 @@ def test_all(self): expected.append(name) self.assertCountEqual(urllib.parse.__all__, expected) + def test_urlsplit_normalization(self): + # Certain characters should never occur in the netloc, + # including under normalization. + # Ensure that ALL of them are detected and cause an error + illegal_chars = '/:#?@' + hex_chars = {'{:04X}'.format(ord(c)) for c in illegal_chars} + denorm_chars = [ + c for c in map(chr, range(128, sys.maxunicode)) + if (hex_chars & set(unicodedata.decomposition(c).split())) + and c not in illegal_chars + ] + # Sanity check that we found at least one such character + self.assertIn('\u2100', denorm_chars) + self.assertIn('\uFF03', denorm_chars) + + for scheme in ["http", "https", "ftp"]: + for c in denorm_chars: + url = "{}://netloc{}false.netloc/path".format(scheme, c) + with self.subTest(url=url, char='{:04X}'.format(ord(c))): + with self.assertRaises(ValueError): + urllib.parse.urlsplit(url) class Utility_Tests(unittest.TestCase): """Testcase to test the various utility functions in the urllib.""" diff --git a/kbe/src/lib/python/Lib/test/test_uu.py b/kbe/src/lib/python/Lib/test/test_uu.py index 1147205a3b..c9f05e5b76 100644 --- a/kbe/src/lib/python/Lib/test/test_uu.py +++ b/kbe/src/lib/python/Lib/test/test_uu.py @@ -6,6 +6,8 @@ import unittest from test import support +import os +import stat import sys import uu import io @@ -218,6 +220,23 @@ def test_decodetwice(self): with open(self.tmpin, 'rb') as f: self.assertRaises(uu.Error, uu.decode, f) + def test_decode_mode(self): + # Verify that decode() will set the given mode for the out_file + expected_mode = 0o444 + with open(self.tmpin, 'wb') as f: + f.write(encodedtextwrapped(expected_mode, self.tmpout)) + + # make file writable again, so it can be removed (Windows only) + self.addCleanup(os.chmod, self.tmpout, expected_mode | stat.S_IWRITE) + + with open(self.tmpin, 'rb') as f: + uu.decode(f) + + self.assertEqual( + stat.S_IMODE(os.stat(self.tmpout).st_mode), + expected_mode + ) + if __name__=="__main__": unittest.main() diff --git a/kbe/src/lib/python/Lib/test/test_venv.py b/kbe/src/lib/python/Lib/test/test_venv.py index 22a3b78852..347544a677 100644 --- a/kbe/src/lib/python/Lib/test/test_venv.py +++ b/kbe/src/lib/python/Lib/test/test_venv.py @@ -52,10 +52,7 @@ def setUp(self): self.bindir = 'bin' self.lib = ('lib', 'python%d.%d' % sys.version_info[:2]) self.include = 'include' - if sys.platform == 'darwin' and '__PYVENV_LAUNCHER__' in os.environ: - executable = os.environ['__PYVENV_LAUNCHER__'] - else: - executable = sys.executable + executable = getattr(sys, '_base_executable', sys.executable) self.exe = os.path.split(executable)[-1] def tearDown(self): @@ -100,11 +97,7 @@ def test_defaults(self): else: self.assertFalse(os.path.exists(p)) data = self.get_text_file_contents('pyvenv.cfg') - if sys.platform == 'darwin' and ('__PYVENV_LAUNCHER__' - in os.environ): - executable = os.environ['__PYVENV_LAUNCHER__'] - else: - executable = sys.executable + executable = getattr(sys, '_base_executable', sys.executable) path = os.path.dirname(executable) self.assertIn('home = %s' % path, data) fn = self.get_env_file(self.bindir, self.exe) @@ -243,7 +236,6 @@ def test_isolation(self): self.assertIn('include-system-site-packages = %s\n' % s, data) @unittest.skipUnless(can_symlink(), 'Needs symlinks') - @unittest.skipIf(os.name == 'nt', 'Symlinks are never used on Windows') def test_symlinking(self): """ Test symlinking works as expected @@ -306,6 +298,19 @@ def test_unicode_in_batch_file(self): ) self.assertEqual(out.strip(), '0') + def test_multiprocessing(self): + """ + Test that the multiprocessing is able to spawn. + """ + rmtree(self.env_dir) + self.run_with_capture(venv.create, self.env_dir) + envpy = os.path.join(os.path.realpath(self.env_dir), + self.bindir, self.exe) + out, err = check_output([envpy, '-c', + 'from multiprocessing import Pool; ' + + 'print(Pool(1).apply_async("Python".lower).get(3))']) + self.assertEqual(out.strip(), "python".encode()) + @skipInVenv class EnsurePipTest(BaseTest): """Test venv module installation of pip.""" diff --git a/kbe/src/lib/python/Lib/test/test_warnings/__init__.py b/kbe/src/lib/python/Lib/test/test_warnings/__init__.py index a40a9a297c..87cc3a7e36 100644 --- a/kbe/src/lib/python/Lib/test/test_warnings/__init__.py +++ b/kbe/src/lib/python/Lib/test/test_warnings/__init__.py @@ -940,6 +940,25 @@ def test_showwarning(self): file_object, expected_file_line) self.assertEqual(expect, file_object.getvalue()) + def test_formatwarning_override(self): + # bpo-35178: Test that a custom formatwarning function gets the 'line' + # argument as a positional argument, and not only as a keyword argument + def myformatwarning(message, category, filename, lineno, text): + return f'm={message}:c={category}:f={filename}:l={lineno}:t={text}' + + file_name = os.path.splitext(warning_tests.__file__)[0] + '.py' + line_num = 3 + file_line = linecache.getline(file_name, line_num).strip() + message = 'msg' + category = Warning + file_object = StringIO() + expected = f'm={message}:c={category}:f={file_name}:l={line_num}' + \ + f':t={file_line}' + with support.swap_attr(self.module, 'formatwarning', myformatwarning): + self.module.showwarning(message, category, file_name, line_num, + file_object, file_line) + self.assertEqual(file_object.getvalue(), expected) + class CWarningsDisplayTests(WarningsDisplayTests, unittest.TestCase): module = c_warnings diff --git a/kbe/src/lib/python/Lib/test/test_weakref.py b/kbe/src/lib/python/Lib/test/test_weakref.py index 9fa0bbd780..1fac08dafc 100644 --- a/kbe/src/lib/python/Lib/test/test_weakref.py +++ b/kbe/src/lib/python/Lib/test/test_weakref.py @@ -8,6 +8,7 @@ import copy import threading import time +import random from test import support from test.support import script_helper @@ -1688,6 +1689,87 @@ def test_threaded_weak_valued_consistency(self): self.assertEqual(len(d), 1) o = None # lose ref + def check_threaded_weak_dict_copy(self, type_, deepcopy): + # `type_` should be either WeakKeyDictionary or WeakValueDictionary. + # `deepcopy` should be either True or False. + exc = [] + + class DummyKey: + def __init__(self, ctr): + self.ctr = ctr + + class DummyValue: + def __init__(self, ctr): + self.ctr = ctr + + def dict_copy(d, exc): + try: + if deepcopy is True: + _ = copy.deepcopy(d) + else: + _ = d.copy() + except Exception as ex: + exc.append(ex) + + def pop_and_collect(lst): + gc_ctr = 0 + while lst: + i = random.randint(0, len(lst) - 1) + gc_ctr += 1 + lst.pop(i) + if gc_ctr % 10000 == 0: + gc.collect() # just in case + + self.assertIn(type_, (weakref.WeakKeyDictionary, weakref.WeakValueDictionary)) + + d = type_() + keys = [] + values = [] + # Initialize d with many entries + for i in range(70000): + k, v = DummyKey(i), DummyValue(i) + keys.append(k) + values.append(v) + d[k] = v + del k + del v + + t_copy = threading.Thread(target=dict_copy, args=(d, exc,)) + if type_ is weakref.WeakKeyDictionary: + t_collect = threading.Thread(target=pop_and_collect, args=(keys,)) + else: # weakref.WeakValueDictionary + t_collect = threading.Thread(target=pop_and_collect, args=(values,)) + + t_copy.start() + t_collect.start() + + t_copy.join() + t_collect.join() + + # Test exceptions + if exc: + raise exc[0] + + def test_threaded_weak_key_dict_copy(self): + # Issue #35615: Weakref keys or values getting GC'ed during dict + # copying should not result in a crash. + self.check_threaded_weak_dict_copy(weakref.WeakKeyDictionary, False) + + def test_threaded_weak_key_dict_deepcopy(self): + # Issue #35615: Weakref keys or values getting GC'ed during dict + # copying should not result in a crash. + self.check_threaded_weak_dict_copy(weakref.WeakKeyDictionary, True) + + def test_threaded_weak_value_dict_copy(self): + # Issue #35615: Weakref keys or values getting GC'ed during dict + # copying should not result in a crash. + self.check_threaded_weak_dict_copy(weakref.WeakValueDictionary, False) + + def test_threaded_weak_value_dict_deepcopy(self): + # Issue #35615: Weakref keys or values getting GC'ed during dict + # copying should not result in a crash. + self.check_threaded_weak_dict_copy(weakref.WeakValueDictionary, True) + from test import mapping_tests diff --git a/kbe/src/lib/python/Lib/test/test_xml_etree_c.py b/kbe/src/lib/python/Lib/test/test_xml_etree_c.py index e87de60944..2144d203e1 100644 --- a/kbe/src/lib/python/Lib/test/test_xml_etree_c.py +++ b/kbe/src/lib/python/Lib/test/test_xml_etree_c.py @@ -1,4 +1,5 @@ # xml.etree test for cElementTree +import io import struct from test import support from test.support import import_fresh_module @@ -133,6 +134,26 @@ def test_setstate_leaks(self): self.assertEqual(len(elem), 1) self.assertEqual(elem[0].tag, 'child') + def test_iterparse_leaks(self): + # Test reference leaks in TreeBuilder (issue #35502). + # The test is written to be executed in the hunting reference leaks + # mode. + XML = '' + parser = cET.iterparse(io.StringIO(XML)) + next(parser) + del parser + support.gc_collect() + + def test_xmlpullparser_leaks(self): + # Test reference leaks in TreeBuilder (issue #35502). + # The test is written to be executed in the hunting reference leaks + # mode. + XML = '' + parser = cET.XMLPullParser() + parser.feed(XML) + del parser + support.gc_collect() + @unittest.skipUnless(cET, 'requires _elementtree') class TestAliasWorking(unittest.TestCase): diff --git a/kbe/src/lib/python/Lib/test/test_xmlrpc.py b/kbe/src/lib/python/Lib/test/test_xmlrpc.py index 5f780d8800..32263f7f0b 100644 --- a/kbe/src/lib/python/Lib/test/test_xmlrpc.py +++ b/kbe/src/lib/python/Lib/test/test_xmlrpc.py @@ -818,11 +818,10 @@ def test_nonascii_methodname(self): # protocol error; provide additional information in test output self.fail("%s\n%s" % (e, getattr(e, "headers", ""))) - # [ch] The test 404 is causing lots of false alarms. - def XXXtest_404(self): + def test_404(self): # send POST with http.client, it should return 404 header and # 'Not Found' message. - conn = httplib.client.HTTPConnection(ADDR, PORT) + conn = http.client.HTTPConnection(ADDR, PORT) conn.request('POST', '/this-is-not-valid') response = conn.getresponse() conn.close() diff --git a/kbe/src/lib/python/Lib/test/test_zipfile64.py b/kbe/src/lib/python/Lib/test/test_zipfile64.py index cba909f6bc..524dcf0213 100644 --- a/kbe/src/lib/python/Lib/test/test_zipfile64.py +++ b/kbe/src/lib/python/Lib/test/test_zipfile64.py @@ -22,7 +22,7 @@ TESTFN2 = TESTFN + "2" # How much time in seconds can pass before we print a 'Still working' message. -_PRINT_WORKING_MSG_INTERVAL = 5 * 60 +_PRINT_WORKING_MSG_INTERVAL = 60 class TestsWithSourceFile(unittest.TestCase): def setUp(self): @@ -43,12 +43,12 @@ def zipTest(self, f, compression): # raw data to store. filecount = 6*1024**3 // len(self.data) - next_time = time.time() + _PRINT_WORKING_MSG_INTERVAL + next_time = time.monotonic() + _PRINT_WORKING_MSG_INTERVAL for num in range(filecount): zipfp.writestr("testfn%d" % num, self.data) # Print still working message since this test can be really slow - if next_time <= time.time(): - next_time = time.time() + _PRINT_WORKING_MSG_INTERVAL + if next_time <= time.monotonic(): + next_time = time.monotonic() + _PRINT_WORKING_MSG_INTERVAL print(( ' zipTest still writing %d of %d, be patient...' % (num, filecount)), file=sys.__stdout__) @@ -60,8 +60,8 @@ def zipTest(self, f, compression): for num in range(filecount): self.assertEqual(zipfp.read("testfn%d" % num), self.data) # Print still working message since this test can be really slow - if next_time <= time.time(): - next_time = time.time() + _PRINT_WORKING_MSG_INTERVAL + if next_time <= time.monotonic(): + next_time = time.monotonic() + _PRINT_WORKING_MSG_INTERVAL print(( ' zipTest still reading %d of %d, be patient...' % (num, filecount)), file=sys.__stdout__) diff --git a/kbe/src/lib/python/Lib/test/time_hashlib.py b/kbe/src/lib/python/Lib/test/time_hashlib.py index 2585ecb782..55ebac6291 100644 --- a/kbe/src/lib/python/Lib/test/time_hashlib.py +++ b/kbe/src/lib/python/Lib/test/time_hashlib.py @@ -14,26 +14,26 @@ def test_scaled_msg(scale, name): longStr = b'Z'*scale localCF = creatorFunc - start = time.time() + start = time.perf_counter() for f in range(iterations): x = localCF(longStr).digest() - end = time.time() + end = time.perf_counter() print(('%2.2f' % (end-start)), "seconds", iterations, "x", len(longStr), "bytes", name) def test_create(): - start = time.time() + start = time.perf_counter() for f in range(20000): d = creatorFunc() - end = time.time() + end = time.perf_counter() print(('%2.2f' % (end-start)), "seconds", '[20000 creations]') def test_zero(): - start = time.time() + start = time.perf_counter() for f in range(20000): x = creatorFunc().digest() - end = time.time() + end = time.perf_counter() print(('%2.2f' % (end-start)), "seconds", '[20000 "" digests]') diff --git a/kbe/src/lib/python/Lib/threading.py b/kbe/src/lib/python/Lib/threading.py index bb41456fb1..318b330112 100644 --- a/kbe/src/lib/python/Lib/threading.py +++ b/kbe/src/lib/python/Lib/threading.py @@ -568,8 +568,8 @@ class Barrier: """Implements a Barrier. Useful for synchronizing a fixed number of threads at known synchronization - points. Threads block on 'wait()' and are simultaneously once they have all - made that call. + points. Threads block on 'wait()' and are simultaneously awoken once they + have all made that call. """ @@ -578,7 +578,7 @@ def __init__(self, parties, action=None, timeout=None): 'action' is a callable which, when supplied, will be called by one of the threads after they have all entered the barrier and just prior to - releasing them all. If a 'timeout' is provided, it is uses as the + releasing them all. If a 'timeout' is provided, it is used as the default for all subsequent 'wait()' calls. """ @@ -1007,7 +1007,7 @@ def join(self, timeout=None): When the timeout argument is present and not None, it should be a floating point number specifying a timeout for the operation in seconds (or fractions thereof). As join() always returns None, you must call - isAlive() after join() to decide whether a timeout happened -- if the + is_alive() after join() to decide whether a timeout happened -- if the thread is still alive, the join() call timed out. When the timeout argument is not present or None, the operation will @@ -1091,7 +1091,15 @@ def is_alive(self): self._wait_for_tstate_lock(False) return not self._is_stopped - isAlive = is_alive + def isAlive(self): + """Return whether the thread is alive. + + This method is deprecated, use is_alive() instead. + """ + import warnings + warnings.warn('isAlive() is deprecated, use is_alive() instead', + PendingDeprecationWarning, stacklevel=2) + return self.is_alive() @property def daemon(self): diff --git a/kbe/src/lib/python/Lib/tkinter/test/test_ttk/test_widgets.py b/kbe/src/lib/python/Lib/tkinter/test/test_ttk/test_widgets.py index 06e3dfe70d..ba9e3b54f7 100644 --- a/kbe/src/lib/python/Lib/tkinter/test/test_ttk/test_widgets.py +++ b/kbe/src/lib/python/Lib/tkinter/test/test_ttk/test_widgets.py @@ -329,7 +329,12 @@ def test_identify(self): self.entry.wait_visibility() self.entry.update_idletasks() - self.assertEqual(self.entry.identify(5, 5), "textarea") + # bpo-27313: macOS Cocoa widget differs from X, allow either + if sys.platform == 'darwin': + self.assertIn(self.entry.identify(5, 5), + ("textarea", "Combobox.button") ) + else: + self.assertEqual(self.entry.identify(5, 5), "textarea") self.assertEqual(self.entry.identify(-1, -1), "") self.assertRaises(tkinter.TclError, self.entry.identify, None, 5) diff --git a/kbe/src/lib/python/Lib/types.py b/kbe/src/lib/python/Lib/types.py index ce4652f371..2e0513ca15 100644 --- a/kbe/src/lib/python/Lib/types.py +++ b/kbe/src/lib/python/Lib/types.py @@ -55,7 +55,7 @@ def _m(self): pass GetSetDescriptorType = type(FunctionType.__code__) MemberDescriptorType = type(FunctionType.__globals__) -del sys, _f, _g, _C, _c, # Not for export +del sys, _f, _g, _C, _c, _ag # Not for export # Provide a PEP 3115 compliant mechanism for class creation diff --git a/kbe/src/lib/python/Lib/unittest/loader.py b/kbe/src/lib/python/Lib/unittest/loader.py index d936a96e73..ba7105e1ad 100644 --- a/kbe/src/lib/python/Lib/unittest/loader.py +++ b/kbe/src/lib/python/Lib/unittest/loader.py @@ -229,7 +229,9 @@ def shouldIncludeMethod(attrname): testFunc = getattr(testCaseClass, attrname) if not callable(testFunc): return False - fullName = '%s.%s' % (testCaseClass.__module__, testFunc.__qualname__) + fullName = f'%s.%s.%s' % ( + testCaseClass.__module__, testCaseClass.__qualname__, attrname + ) return self.testNamePatterns is None or \ any(fnmatchcase(fullName, pattern) for pattern in self.testNamePatterns) testFnNames = list(filter(shouldIncludeMethod, dir(testCaseClass))) diff --git a/kbe/src/lib/python/Lib/unittest/mock.py b/kbe/src/lib/python/Lib/unittest/mock.py index f641e38dc5..5b8e744140 100644 --- a/kbe/src/lib/python/Lib/unittest/mock.py +++ b/kbe/src/lib/python/Lib/unittest/mock.py @@ -102,6 +102,7 @@ def checksig(_mock_self, *args, **kwargs): sig.bind(*args, **kwargs) _copy_func_details(func, checksig) type(mock)._mock_check_sig = checksig + type(mock).__signature__ = sig def _copy_func_details(func, funcopy): @@ -171,11 +172,11 @@ def checksig(*args, **kwargs): return mock(*args, **kwargs)""" % name exec (src, context) funcopy = context[name] - _setup_func(funcopy, mock) + _setup_func(funcopy, mock, sig) return funcopy -def _setup_func(funcopy, mock): +def _setup_func(funcopy, mock, sig): funcopy.mock = mock # can't use isinstance with mocks @@ -223,6 +224,7 @@ def reset_mock(): funcopy.assert_called = assert_called funcopy.assert_not_called = assert_not_called funcopy.assert_called_once = assert_called_once + funcopy.__signature__ = sig mock._mock_delegate = funcopy @@ -318,6 +320,14 @@ def __repr__(self): def _check_and_set_parent(parent, value, name, new_name): + # function passed to create_autospec will have mock + # attribute attached to which parent must be set + if isinstance(value, FunctionTypes): + try: + value = value.mock + except AttributeError: + pass + if not _is_instance_mock(value): return False if ((value._mock_name or value._mock_new_name) or @@ -726,11 +736,10 @@ def __delattr__(self, name): # not set on the instance itself return - if name in self.__dict__: - object.__delattr__(self, name) - obj = self._mock_children.get(name, _missing) - if obj is _deleted: + if name in self.__dict__: + super().__delattr__(name) + elif obj is _deleted: raise AttributeError(name) if obj is not _missing: del self._mock_children[name] @@ -1594,8 +1603,6 @@ class _patch_dict(object): """ def __init__(self, in_dict, values=(), clear=False, **kwargs): - if isinstance(in_dict, str): - in_dict = _importer(in_dict) self.in_dict = in_dict # support any argument supported by dict(...) constructor self.values = dict(values) @@ -1636,6 +1643,8 @@ def __enter__(self): def _patch_dict(self): values = self.values + if isinstance(self.in_dict, str): + self.in_dict = _importer(self.in_dict) in_dict = self.in_dict clear = self.clear @@ -2350,7 +2359,7 @@ def mock_open(mock=None, read_data=''): default) then a `MagicMock` will be created for you, with the API limited to methods or attributes available on standard file handles. - `read_data` is a string for the `read` methoddline`, and `readlines` of the + `read_data` is a string for the `read`, `readline` and `readlines` of the file handle to return. This is an empty string by default. """ def _readlines_side_effect(*args, **kwargs): diff --git a/kbe/src/lib/python/Lib/unittest/runner.py b/kbe/src/lib/python/Lib/unittest/runner.py index 2c5ea4ab07..45e7e4c045 100644 --- a/kbe/src/lib/python/Lib/unittest/runner.py +++ b/kbe/src/lib/python/Lib/unittest/runner.py @@ -168,7 +168,7 @@ def run(self, test): warnings.filterwarnings('module', category=DeprecationWarning, message=r'Please use assert\w+ instead.') - startTime = time.time() + startTime = time.perf_counter() startTestRun = getattr(result, 'startTestRun', None) if startTestRun is not None: startTestRun() @@ -178,7 +178,7 @@ def run(self, test): stopTestRun = getattr(result, 'stopTestRun', None) if stopTestRun is not None: stopTestRun() - stopTime = time.time() + stopTime = time.perf_counter() timeTaken = stopTime - startTime result.printErrors() if hasattr(result, 'separator2'): diff --git a/kbe/src/lib/python/Lib/unittest/test/test_loader.py b/kbe/src/lib/python/Lib/unittest/test/test_loader.py index bfd722940b..bc54bf0553 100644 --- a/kbe/src/lib/python/Lib/unittest/test/test_loader.py +++ b/kbe/src/lib/python/Lib/unittest/test/test_loader.py @@ -1,3 +1,4 @@ +import functools import sys import types import warnings @@ -1575,5 +1576,20 @@ def test_suiteClass__default_value(self): self.assertIs(loader.suiteClass, unittest.TestSuite) + def test_partial_functions(self): + def noop(arg): + pass + + class Foo(unittest.TestCase): + pass + + setattr(Foo, 'test_partial', functools.partial(noop, None)) + + loader = unittest.TestLoader() + + test_names = ['test_partial'] + self.assertEqual(loader.getTestCaseNames(Foo), test_names) + + if __name__ == "__main__": unittest.main() diff --git a/kbe/src/lib/python/Lib/unittest/test/testmock/support.py b/kbe/src/lib/python/Lib/unittest/test/testmock/support.py index 205431adca..73e406f0a8 100644 --- a/kbe/src/lib/python/Lib/unittest/test/testmock/support.py +++ b/kbe/src/lib/python/Lib/unittest/test/testmock/support.py @@ -1,3 +1,6 @@ +target = {'foo': 'FOO'} + + def is_instance(obj, klass): """Version of is_instance that doesn't access __class__""" return issubclass(type(obj), klass) diff --git a/kbe/src/lib/python/Lib/unittest/test/testmock/testhelpers.py b/kbe/src/lib/python/Lib/unittest/test/testmock/testhelpers.py index 9388311e3e..745580ef79 100644 --- a/kbe/src/lib/python/Lib/unittest/test/testmock/testhelpers.py +++ b/kbe/src/lib/python/Lib/unittest/test/testmock/testhelpers.py @@ -1,3 +1,4 @@ +import inspect import time import types import unittest @@ -901,6 +902,35 @@ def __getattr__(self, attribute): self.assertFalse(hasattr(autospec, '__name__')) + def test_spec_inspect_signature(self): + + def myfunc(x, y): + pass + + mock = create_autospec(myfunc) + mock(1, 2) + mock(x=1, y=2) + + self.assertEqual(inspect.getfullargspec(mock), inspect.getfullargspec(myfunc)) + self.assertEqual(mock.mock_calls, [call(1, 2), call(x=1, y=2)]) + self.assertRaises(TypeError, mock, 1) + + + def test_spec_inspect_signature_annotations(self): + + def foo(a: int, b: int=10, *, c:int) -> int: + return a + b + c + + mock = create_autospec(foo) + mock(1, 2, c=3) + mock(1, c=3) + + self.assertEqual(inspect.getfullargspec(mock), inspect.getfullargspec(foo)) + self.assertEqual(mock.mock_calls, [call(1, 2, c=3), call(1, c=3)]) + self.assertRaises(TypeError, mock, 1) + self.assertRaises(TypeError, mock, 1, 2, 3, c=4) + + class TestCallList(unittest.TestCase): def test_args_list_contains_call_list(self): diff --git a/kbe/src/lib/python/Lib/unittest/test/testmock/testmock.py b/kbe/src/lib/python/Lib/unittest/test/testmock/testmock.py index 49ecbb4466..447a502b57 100644 --- a/kbe/src/lib/python/Lib/unittest/test/testmock/testmock.py +++ b/kbe/src/lib/python/Lib/unittest/test/testmock/testmock.py @@ -1739,6 +1739,33 @@ def test_attribute_deletion(self): self.assertRaises(AttributeError, getattr, mock, 'f') + def test_mock_does_not_raise_on_repeated_attribute_deletion(self): + # bpo-20239: Assigning and deleting twice an attribute raises. + for mock in (Mock(), MagicMock(), NonCallableMagicMock(), + NonCallableMock()): + mock.foo = 3 + self.assertTrue(hasattr(mock, 'foo')) + self.assertEqual(mock.foo, 3) + + del mock.foo + self.assertFalse(hasattr(mock, 'foo')) + + mock.foo = 4 + self.assertTrue(hasattr(mock, 'foo')) + self.assertEqual(mock.foo, 4) + + del mock.foo + self.assertFalse(hasattr(mock, 'foo')) + + + def test_mock_raises_when_deleting_nonexistent_attribute(self): + for mock in (Mock(), MagicMock(), NonCallableMagicMock(), + NonCallableMock()): + del mock.foo + with self.assertRaises(AttributeError): + del mock.foo + + def test_reset_mock_does_not_raise_on_attr_deletion(self): # bpo-31177: reset_mock should not raise AttributeError when attributes # were deleted in a mock instance @@ -1772,5 +1799,18 @@ def test_parent_attribute_of_call(self): self.assertEqual(type(call.parent().parent), _Call) + def test_parent_propagation_with_create_autospec(self): + + def foo(a, b): + pass + + mock = Mock() + mock.child = create_autospec(foo) + mock.child(1, 2) + + self.assertRaises(TypeError, mock.child, 1) + self.assertEqual(mock.mock_calls, [call.child(1, 2)]) + + if __name__ == '__main__': unittest.main() diff --git a/kbe/src/lib/python/Lib/unittest/test/testmock/testpatch.py b/kbe/src/lib/python/Lib/unittest/test/testmock/testpatch.py index f05225730d..c484adb605 100644 --- a/kbe/src/lib/python/Lib/unittest/test/testmock/testpatch.py +++ b/kbe/src/lib/python/Lib/unittest/test/testmock/testpatch.py @@ -664,6 +664,23 @@ def test(): test() + def test_patch_dict_decorator_resolution(self): + # bpo-35512: Ensure that patch with a string target resolves to + # the new dictionary during function call + original = support.target.copy() + + @patch.dict('unittest.test.testmock.support.target', {'bar': 'BAR'}) + def test(): + self.assertEqual(support.target, {'foo': 'BAZ', 'bar': 'BAR'}) + + try: + support.target = {'foo': 'BAZ'} + test() + self.assertEqual(support.target, {'foo': 'BAZ'}) + finally: + support.target = original + + def test_patch_descriptor(self): # would be some effort to fix this - we could special case the # builtin descriptors: classmethod, property, staticmethod diff --git a/kbe/src/lib/python/Lib/unittest/test/testmock/testwith.py b/kbe/src/lib/python/Lib/unittest/test/testmock/testwith.py index 43b36a1199..ec4e540dcf 100644 --- a/kbe/src/lib/python/Lib/unittest/test/testmock/testwith.py +++ b/kbe/src/lib/python/Lib/unittest/test/testmock/testwith.py @@ -126,6 +126,20 @@ def test_dict_context_manager(self): self.assertEqual(foo, {}) + def test_double_patch_instance_method(self): + class C: + def f(self): + pass + + c = C() + + with patch.object(c, 'f', autospec=True) as patch1: + with patch.object(c, 'f', autospec=True) as patch2: + c.f() + self.assertEqual(patch2.call_count, 1) + self.assertEqual(patch1.call_count, 0) + c.f() + self.assertEqual(patch1.call_count, 1) class TestMockOpen(unittest.TestCase): diff --git a/kbe/src/lib/python/Lib/urllib/parse.py b/kbe/src/lib/python/Lib/urllib/parse.py index f691ab74f8..39c5d6a808 100644 --- a/kbe/src/lib/python/Lib/urllib/parse.py +++ b/kbe/src/lib/python/Lib/urllib/parse.py @@ -391,6 +391,21 @@ def _splitnetloc(url, start=0): delim = min(delim, wdelim) # use earliest delim position return url[start:delim], url[delim:] # return (domain, rest) +def _checknetloc(netloc): + if not netloc or netloc.isascii(): + return + # looking for characters like \u2100 that expand to 'a/c' + # IDNA uses NFKC equivalence, so normalize for this check + import unicodedata + netloc2 = unicodedata.normalize('NFKC', netloc) + if netloc == netloc2: + return + _, _, netloc = netloc.rpartition('@') # anything to the left of '@' is okay + for c in '/?#@:': + if c in netloc2: + raise ValueError("netloc '" + netloc2 + "' contains invalid " + + "characters under NFKC normalization") + def urlsplit(url, scheme='', allow_fragments=True): """Parse a URL into 5 components: :///?# @@ -419,6 +434,7 @@ def urlsplit(url, scheme='', allow_fragments=True): url, fragment = url.split('#', 1) if '?' in url: url, query = url.split('?', 1) + _checknetloc(netloc) v = SplitResult('http', netloc, url, query, fragment) _parse_cache[key] = v return _coerce_result(v) @@ -442,6 +458,7 @@ def urlsplit(url, scheme='', allow_fragments=True): url, fragment = url.split('#', 1) if '?' in url: url, query = url.split('?', 1) + _checknetloc(netloc) v = SplitResult(scheme, netloc, url, query, fragment) _parse_cache[key] = v return _coerce_result(v) diff --git a/kbe/src/lib/python/Lib/uu.py b/kbe/src/lib/python/Lib/uu.py index 8333e864d8..9b1e5e6072 100755 --- a/kbe/src/lib/python/Lib/uu.py +++ b/kbe/src/lib/python/Lib/uu.py @@ -133,10 +133,7 @@ def decode(in_file, out_file=None, mode=None, quiet=False): out_file = sys.stdout.buffer elif isinstance(out_file, str): fp = open(out_file, 'wb') - try: - os.path.chmod(out_file, mode) - except AttributeError: - pass + os.chmod(out_file, mode) out_file = fp opened_files.append(out_file) # diff --git a/kbe/src/lib/python/Lib/venv/__init__.py b/kbe/src/lib/python/Lib/venv/__init__.py index 5438b0d4e5..d5ab38958b 100644 --- a/kbe/src/lib/python/Lib/venv/__init__.py +++ b/kbe/src/lib/python/Lib/venv/__init__.py @@ -64,11 +64,10 @@ def create(self, env_dir): self.system_site_packages = False self.create_configuration(context) self.setup_python(context) - if not self.upgrade: - self.setup_scripts(context) if self.with_pip: self._setup_pip(context) if not self.upgrade: + self.setup_scripts(context) self.post_setup(context) if true_system_site_packages: # We had set it to False before, now @@ -107,10 +106,7 @@ def create_if_needed(d): context.prompt = '(%s) ' % prompt create_if_needed(env_dir) env = os.environ - if sys.platform == 'darwin' and '__PYVENV_LAUNCHER__' in env: - executable = os.environ['__PYVENV_LAUNCHER__'] - else: - executable = sys.executable + executable = getattr(sys, '_base_executable', sys.executable) dirname, exename = os.path.split(os.path.abspath(executable)) context.executable = executable context.python_dir = dirname @@ -176,6 +172,23 @@ def symlink_or_copy(self, src, dst, relative_symlinks_ok=False): logger.warning('Unable to symlink %r to %r', src, dst) force_copy = True if force_copy: + if os.name == 'nt': + # On Windows, we rewrite symlinks to our base python.exe into + # copies of venvlauncher.exe + basename, ext = os.path.splitext(os.path.basename(src)) + if basename.endswith('_d'): + ext = '_d' + ext + basename = basename[:-2] + if sysconfig.is_python_build(True): + if basename == 'python': + basename = 'venvlauncher' + elif basename == 'pythonw': + basename = 'venvwlauncher' + scripts = os.path.dirname(src) + else: + scripts = os.path.join(os.path.dirname(__file__), "scripts", "nt") + src = os.path.join(scripts, basename + ext) + shutil.copyfile(src, dst) def setup_python(self, context): @@ -202,23 +215,31 @@ def setup_python(self, context): if not os.path.islink(path): os.chmod(path, 0o755) else: - # For normal cases, the venvlauncher will be copied from - # our scripts folder. For builds, we need to copy it - # manually. - if sysconfig.is_python_build(True): - suffix = '.exe' - if context.python_exe.lower().endswith('_d.exe'): - suffix = '_d.exe' - - src = os.path.join(dirname, "venvlauncher" + suffix) - dst = os.path.join(binpath, context.python_exe) - copier(src, dst) + if self.symlinks: + # For symlinking, we need a complete copy of the root directory + # If symlinks fail, you'll get unnecessary copies of files, but + # we assume that if you've opted into symlinks on Windows then + # you know what you're doing. + suffixes = [ + f for f in os.listdir(dirname) if + os.path.normcase(os.path.splitext(f)[1]) in ('.exe', '.dll') + ] + if sysconfig.is_python_build(True): + suffixes = [ + f for f in suffixes if + os.path.normcase(f).startswith(('python', 'vcruntime')) + ] + else: + suffixes = ['python.exe', 'python_d.exe', 'pythonw.exe', + 'pythonw_d.exe'] - src = os.path.join(dirname, "venvwlauncher" + suffix) - dst = os.path.join(binpath, "pythonw" + suffix) - copier(src, dst) + for suffix in suffixes: + src = os.path.join(dirname, suffix) + if os.path.exists(src): + copier(src, os.path.join(binpath, suffix)) - # copy init.tcl over + if sysconfig.is_python_build(True): + # copy init.tcl for root, dirs, files in os.walk(context.python_dir): if 'init.tcl' in files: tcldir = os.path.basename(root) @@ -304,6 +325,9 @@ def install_scripts(self, context, path): dirs.remove(d) continue # ignore files in top level for f in files: + if (os.name == 'nt' and f.startswith('python') + and f.endswith(('.exe', '.pdb'))): + continue srcfile = os.path.join(root, f) suffix = root[plen:].split(os.sep)[2:] if not suffix: diff --git a/kbe/src/lib/python/Lib/warnings.py b/kbe/src/lib/python/Lib/warnings.py index ae4295e120..9064f56827 100644 --- a/kbe/src/lib/python/Lib/warnings.py +++ b/kbe/src/lib/python/Lib/warnings.py @@ -124,7 +124,7 @@ def _formatwarnmsg(msg): if fw is not _formatwarning_orig: # warnings.formatwarning() was replaced return fw(msg.message, msg.category, - msg.filename, msg.lineno, line=msg.line) + msg.filename, msg.lineno, msg.line) return _formatwarnmsg_impl(msg) def filterwarnings(action, message="", category=Warning, module="", lineno=0, diff --git a/kbe/src/lib/python/Lib/weakref.py b/kbe/src/lib/python/Lib/weakref.py index 99de2eab74..753f07291e 100644 --- a/kbe/src/lib/python/Lib/weakref.py +++ b/kbe/src/lib/python/Lib/weakref.py @@ -171,10 +171,11 @@ def copy(self): if self._pending_removals: self._commit_removals() new = WeakValueDictionary() - for key, wr in self.data.items(): - o = wr() - if o is not None: - new[key] = o + with _IterationGuard(self): + for key, wr in self.data.items(): + o = wr() + if o is not None: + new[key] = o return new __copy__ = copy @@ -184,10 +185,11 @@ def __deepcopy__(self, memo): if self._pending_removals: self._commit_removals() new = self.__class__() - for key, wr in self.data.items(): - o = wr() - if o is not None: - new[deepcopy(key, memo)] = o + with _IterationGuard(self): + for key, wr in self.data.items(): + o = wr() + if o is not None: + new[deepcopy(key, memo)] = o return new def get(self, key, default=None): @@ -408,10 +410,11 @@ def __setitem__(self, key, value): def copy(self): new = WeakKeyDictionary() - for key, value in self.data.items(): - o = key() - if o is not None: - new[o] = value + with _IterationGuard(self): + for key, value in self.data.items(): + o = key() + if o is not None: + new[o] = value return new __copy__ = copy @@ -419,10 +422,11 @@ def copy(self): def __deepcopy__(self, memo): from copy import deepcopy new = self.__class__() - for key, value in self.data.items(): - o = key() - if o is not None: - new[o] = deepcopy(value, memo) + with _IterationGuard(self): + for key, value in self.data.items(): + o = key() + if o is not None: + new[o] = deepcopy(value, memo) return new def get(self, key, default=None): diff --git a/kbe/src/lib/python/Mac/BuildScript/build-installer.py b/kbe/src/lib/python/Mac/BuildScript/build-installer.py index 5d11bbb10e..2e3a61ec71 100755 --- a/kbe/src/lib/python/Mac/BuildScript/build-installer.py +++ b/kbe/src/lib/python/Mac/BuildScript/build-installer.py @@ -227,9 +227,9 @@ def library_recipes(): if internalTk(): result.extend([ dict( - name="Tcl 8.6.9", - url="ftp://ftp.tcl.tk/pub/tcl//tcl8_6/tcl8.6.9-src.tar.gz", - checksum='aa0a121d95a0e7b73a036f26028538d4', + name="Tcl 8.6.8", + url="ftp://ftp.tcl.tk/pub/tcl//tcl8_6/tcl8.6.8-src.tar.gz", + checksum='81656d3367af032e0ae6157eff134f89', buildDir="unix", configure_pre=[ '--enable-shared', @@ -243,9 +243,12 @@ def library_recipes(): }, ), dict( - name="Tk 8.6.9.1", - url="ftp://ftp.tcl.tk/pub/tcl//tcl8_6/tk8.6.9.1-src.tar.gz", - checksum='9efe3976468352dc894dae0c4e785a8e', + name="Tk 8.6.8", + url="ftp://ftp.tcl.tk/pub/tcl//tcl8_6/tk8.6.8-src.tar.gz", + checksum='5e0faecba458ee1386078fb228d008ba', + patches=[ + "tk868_on_10_8_10_9.patch", + ], buildDir="unix", configure_pre=[ '--enable-aqua', @@ -706,7 +709,6 @@ def extractArchive(builddir, archiveName): work for current Tcl and Tk source releases where the basename of the archive ends with "-src" but the uncompressed directory does not. For now, just special case Tcl and Tk tar.gz downloads. - Another special case: the tk8.6.9.1 tarball extracts to tk8.6.9. """ curdir = os.getcwd() try: @@ -716,8 +718,6 @@ def extractArchive(builddir, archiveName): if ((retval.startswith('tcl') or retval.startswith('tk')) and retval.endswith('-src')): retval = retval[:-4] - if retval == 'tk8.6.9.1': - retval = 'tk8.6.9' if os.path.exists(retval): shutil.rmtree(retval) fp = os.popen("tar zxf %s 2>&1"%(shellQuote(archiveName),), 'r') diff --git a/kbe/src/lib/python/Mac/BuildScript/tk868_on_10_8_10_9.patch b/kbe/src/lib/python/Mac/BuildScript/tk868_on_10_8_10_9.patch new file mode 100644 index 0000000000..8fe10604a6 --- /dev/null +++ b/kbe/src/lib/python/Mac/BuildScript/tk868_on_10_8_10_9.patch @@ -0,0 +1,18 @@ +Fix build failure with +quartz variant on OS X 10.8 and 10.9. +Even though Gestalt was deprecated in OS X 10.8, it should work fine +through OS X 10.9, and its replacement NSOperatingSystemVersion was +not introduced until OS X 10.10. + +Patch from MacPorts project and reported upstream: +https://trac.macports.org/ticket/55649 +--- tk8.6.8/macosx/tkMacOSXXStubs.c.orig 2017-12-06 09:25:08.000000000 -0600 ++++ tk8.6.8-patched/macosx/tkMacOSXXStubs.c 2018-01-06 19:34:17.000000000 -0600 +@@ -175,7 +175,7 @@ + { + int major, minor, patch; + +-#if MAC_OS_X_VERSION_MIN_REQUIRED < 1080 ++#if MAC_OS_X_VERSION_MIN_REQUIRED < 101000 + Gestalt(gestaltSystemVersionMajor, (SInt32*)&major); + Gestalt(gestaltSystemVersionMinor, (SInt32*)&minor); + Gestalt(gestaltSystemVersionBugFix, (SInt32*)&patch); diff --git a/kbe/src/lib/python/Mac/IDLE/IDLE.app/Contents/Info.plist b/kbe/src/lib/python/Mac/IDLE/IDLE.app/Contents/Info.plist index 826d3793f7..04a0a08c83 100644 --- a/kbe/src/lib/python/Mac/IDLE/IDLE.app/Contents/Info.plist +++ b/kbe/src/lib/python/Mac/IDLE/IDLE.app/Contents/Info.plist @@ -36,7 +36,7 @@ CFBundleExecutable IDLE CFBundleGetInfoString - %version%, © 2001-2018 Python Software Foundation + %version%, © 2001-2019 Python Software Foundation CFBundleIconFile IDLE.icns CFBundleIdentifier diff --git a/kbe/src/lib/python/Mac/PythonLauncher/Info.plist.in b/kbe/src/lib/python/Mac/PythonLauncher/Info.plist.in index 5fa346ed4d..9fb4e0affd 100644 --- a/kbe/src/lib/python/Mac/PythonLauncher/Info.plist.in +++ b/kbe/src/lib/python/Mac/PythonLauncher/Info.plist.in @@ -40,7 +40,7 @@ CFBundleExecutable Python Launcher CFBundleGetInfoString - %VERSION%, © 2001-2018 Python Software Foundation + %VERSION%, © 2001-2019 Python Software Foundation CFBundleIconFile PythonLauncher.icns CFBundleIdentifier diff --git a/kbe/src/lib/python/Mac/Resources/app/Info.plist.in b/kbe/src/lib/python/Mac/Resources/app/Info.plist.in index abe9ae23e3..b7581984dd 100644 --- a/kbe/src/lib/python/Mac/Resources/app/Info.plist.in +++ b/kbe/src/lib/python/Mac/Resources/app/Info.plist.in @@ -37,7 +37,7 @@ CFBundleInfoDictionaryVersion 6.0 CFBundleLongVersionString - %version%, (c) 2001-2018 Python Software Foundation. + %version%, (c) 2001-2019 Python Software Foundation. CFBundleName Python CFBundlePackageType diff --git a/kbe/src/lib/python/Mac/Resources/framework/Info.plist.in b/kbe/src/lib/python/Mac/Resources/framework/Info.plist.in index c1ea9f6889..0dc2e17156 100644 --- a/kbe/src/lib/python/Mac/Resources/framework/Info.plist.in +++ b/kbe/src/lib/python/Mac/Resources/framework/Info.plist.in @@ -17,9 +17,9 @@ CFBundlePackageType FMWK CFBundleShortVersionString - %VERSION%, (c) 2001-2018 Python Software Foundation. + %VERSION%, (c) 2001-2019 Python Software Foundation. CFBundleLongVersionString - %VERSION%, (c) 2001-2018 Python Software Foundation. + %VERSION%, (c) 2001-2019 Python Software Foundation. CFBundleSignature ???? CFBundleVersion diff --git a/kbe/src/lib/python/Misc/ACKS b/kbe/src/lib/python/Misc/ACKS index c35ef1c0fa..193592290f 100644 --- a/kbe/src/lib/python/Misc/ACKS +++ b/kbe/src/lib/python/Misc/ACKS @@ -60,6 +60,7 @@ Heidi Annexstad Ramchandra Apte Éric Araujo Alexandru Ardelean +Emmanuel Arias Alicia Arlen Jeffrey Armstrong Jason Asbahr @@ -894,6 +895,7 @@ Glenn Langford Andrew Langmead Wolfgang Langner Detlef Lannert +Rémi Lapeyre Soren Larsen Amos Latteier Piers Lauder @@ -1821,3 +1823,4 @@ Jelle Zijlstra Gennadiy Zlobin Doug Zongker Peter Åstrand +Zheao Li diff --git a/kbe/src/lib/python/Misc/NEWS b/kbe/src/lib/python/Misc/NEWS index bce9073bee..66cd43897c 100644 --- a/kbe/src/lib/python/Misc/NEWS +++ b/kbe/src/lib/python/Misc/NEWS @@ -2,6 +2,443 @@ Python News +++++++++++ +What's New in Python 3.7.3 final? +================================= + +*Release date: 2019-03-25* + +There were no new changes in version 3.7.3. + + + +What's New in Python 3.7.3 release candidate 1? +=============================================== + +*Release date: 2019-03-12* + +Security +-------- + +- bpo-36216: Changes urlsplit() to raise ValueError when the URL contains + characters that decompose under IDNA encoding (NFKC-normalization) into + characters that affect how the URL is parsed. + +- bpo-35746: [CVE-2019-5010] Fix a NULL pointer deref in ssl module. The + cert parser did not handle CRL distribution points with empty DP or URI + correctly. A malicious or buggy certificate can result into segfault. + Vulnerability (TALOS-2018-0758) reported by Colin Read and Nicolas Edet of + Cisco. + +- bpo-35121: Don't send cookies of domain A without Domain attribute to + domain B when domain A is a suffix match of domain B while using a + cookiejar with :class:`http.cookiejar.DefaultCookiePolicy` policy. Patch + by Karthikeyan Singaravelan. + +Core and Builtins +----------------- + +- bpo-35942: The error message emitted when returning invalid types from + ``__fspath__`` in interfaces that allow passing :class:`~os.PathLike` + objects has been improved and now it does explain the origin of the error. + +- bpo-35992: Fix ``__class_getitem__()`` not being called on a class with a + custom non-subscriptable metaclass. + +- bpo-35991: Fix a potential double free in Modules/_randommodule.c. + +- bpo-35961: Fix a crash in slice_richcompare(): use strong references + rather than stolen references for the two temporary internal tuples. + +- bpo-31506: Clarify the errors reported when ``object.__new__`` and + ``object.__init__`` receive more than one argument. Contributed by Sanyam + Khurana. + +- bpo-35720: Fixed a minor memory leak in pymain_parse_cmdline_impl function + in Modules/main.c + +- bpo-35623: Fix a crash when sorting very long lists. Patch by Stephan + Hohe. + +- bpo-35214: clang Memory Sanitizer build instrumentation was added to work + around false positives from posix, socket, time, test_io, and + test_faulthandler. + +- bpo-35560: Fix an assertion error in :func:`format` in debug build for + floating point formatting with "n" format, zero padding and small width. + Release build is not impacted. Patch by Karthikeyan Singaravelan. + +- bpo-35552: Format characters ``%s`` and ``%V`` in + :c:func:`PyUnicode_FromFormat` and ``%s`` in :c:func:`PyBytes_FromFormat` + no longer read memory past the limit if *precision* is specified. + +- bpo-35504: Fix segfaults and :exc:`SystemError`\ s when deleting certain + attributes. Patch by Zackery Spytz. + +- bpo-33989: Fix a possible crash in :meth:`list.sort` when sorting objects + with ``ob_type->tp_richcompare == NULL``. Patch by Zackery Spytz. + +Library +------- + +- bpo-35931: The :mod:`pdb` ``debug`` command now gracefully handles all + exceptions. + +- bpo-36251: Fix format strings used for stderrprinter and re.Match reprs. + Patch by Stephan Hohe. + +- bpo-35807: Update ensurepip to install pip 19.0.3 and setuptools 40.8.0. + +- bpo-36179: Fix two unlikely reference leaks in _hashopenssl. The leaks + only occur in out-of-memory cases. + +- bpo-35178: Ensure custom :func:`warnings.formatwarning` function can + receive `line` as positional argument. Based on patch by Tashrif Billah. + +- bpo-36106: Resolve potential name clash with libm's sinpi(). Patch by + Dmitrii Pasechnik. + +- bpo-35512: :func:`unittest.mock.patch.dict` used as a decorator with + string target resolves the target during function call instead of during + decorator construction. Patch by Karthikeyan Singaravelan. + +- bpo-36091: Clean up reference to async generator in Lib/types. Patch by + Henry Chen. + +- bpo-35899: Enum has been fixed to correctly handle empty strings and + strings with non-Latin characters (ie. 'α', 'א') without crashing. + Original patch contributed by Maxwell. Assisted by Stéphane Wirtel. + +- bpo-35918: Removed broken ``has_key`` method from + multiprocessing.managers.SyncManager.dict. Contributed by Rémi Lapeyre. + +- bpo-35960: Fix :func:`dataclasses.field` throwing away empty mapping + objects passed as metadata. + +- bpo-35847: RISC-V needed the CTYPES_PASS_BY_REF_HACK. Fixes ctypes + Structure test_pass_by_value. + +- bpo-35780: Fix lru_cache() errors arising in recursive, reentrant, or + multi-threaded code. These errors could result in orphan links and in the + cache being trapped in a state with fewer than the specified maximum + number of links. Fix handling of negative maxsize which should have been + treated as zero. Fix errors in toggling the "full" status flag. Fix + misordering of links when errors are encountered. Sync-up the C code and + pure Python code for the space saving path in functions with a single + positional argument. In this common case, the space overhead of an lru + cache entry is reduced by almost half. Fix counting of cache misses. In + error cases, the miss count was out of sync with the actual number of + times the underlying user function was called. + +- bpo-23846: :class:`asyncio.ProactorEventLoop` now catches and logs send + errors when the self-pipe is full. + +- bpo-34323: :mod:`asyncio`: Enhance ``IocpProactor.close()`` log: wait 1 + second before the first log, then log every second. Log also the number of + seconds since ``close()`` was called. + +- bpo-34294: re module, fix wrong capturing groups in rare cases. + :func:`re.search`, :func:`re.findall`, :func:`re.sub` and other functions + that scan through string looking for a match, should reset capturing + groups between two match attempts. Patch by Ma Lin. + +- bpo-35717: Fix KeyError exception raised when using enums and compile. + Patch contributed by Rémi Lapeyre. + +- bpo-35699: Fixed detection of Visual Studio Build Tools 2017 in distutils + +- bpo-32710: Fix memory leaks in asyncio ProactorEventLoop on overlapped + operation failure. + +- bpo-32710: Fix a memory leak in asyncio in the ProactorEventLoop when + ``ReadFile()`` or ``WSASend()`` overlapped operation fail immediately: + release the internal buffer. + +- bpo-35682: Fix ``asyncio.ProactorEventLoop.sendfile()``: don't attempt to + set the result of an internal future if it's already done. + +- bpo-35283: Add a pending deprecated warning for the + :meth:`threading.Thread.isAlive` method. Patch by Dong-hee Na. + +- bpo-35643: Fixed a SyntaxWarning: invalid escape sequence in + Modules/_sha3/cleanup.py. Patch by Mickaël Schoentgen. + +- bpo-35615: :mod:`weakref`: Fix a RuntimeError when copying a + WeakKeyDictionary or a WeakValueDictionary, due to some keys or values + disappearing while iterating. + +- bpo-28503: The `crypt` module now internally uses the `crypt_r()` library + function instead of `crypt()` when available. + +- bpo-35121: Don't set cookie for a request when the request path is a + prefix match of the cookie's path attribute but doesn't end with "/". + Patch by Karthikeyan Singaravelan. + +- bpo-35585: Speed-up building enums by value, e.g. http.HTTPStatus(200). + +- bpo-21478: Calls to a child function created with + :func:`unittest.mock.create_autospec` should propagate to the parent. + Patch by Karthikeyan Singaravelan. + +- bpo-35513: :class:`~unittest.runner.TextTestRunner` of + :mod:`unittest.runner` now uses :func:`time.perf_counter` rather than + :func:`time.time` to measure the execution time of a test: + :func:`time.time` can go backwards, whereas :func:`time.perf_counter` is + monotonic. + +- bpo-35502: Fixed reference leaks in + :class:`xml.etree.ElementTree.TreeBuilder` in case of unfinished building + of the tree (in particular when an error was raised during parsing XML). + +- bpo-31446: Copy command line that was passed to CreateProcessW since this + function can change the content of the input buffer. + +- bpo-20239: Allow repeated assignment deletion of + :class:`unittest.mock.Mock` attributes. Patch by Pablo Galindo. + +- bpo-17185: Set ``__signature__`` on mock for :mod:`inspect` to get + signature. Patch by Karthikeyan Singaravelan. + +- bpo-10496: :func:`~distutils.utils.check_environ` of + :mod:`distutils.utils` now catches :exc:`KeyError` on calling + :func:`pwd.getpwuid`: don't create the ``HOME`` environment variable in + this case. + +- bpo-35066: Previously, calling the strftime() method on a datetime object + with a trailing '%' in the format string would result in an exception. + However, this only occured when the datetime C module was being used; the + python implementation did not match this behavior. Datetime is now PEP-399 + compliant, and will not throw an exception on a trailing '%'. + +- bpo-24746: Avoid stripping trailing whitespace in doctest fancy diff. + Orignial patch by R. David Murray & Jairo Trad. Enhanced by Sanyam + Khurana. + +- bpo-35198: Fix C++ extension compilation on AIX + +- bpo-28441: On Cygwin and MinGW, ensure that ``sys.executable`` always + includes the full filename in the path, including the ``.exe`` suffix + (unless it is a symbolic link). + +- bpo-34572: Fix C implementation of pickle.loads to use importlib's locking + mechanisms, and thereby avoid using partially-loaded modules. Patch by Tim + Burgess. + +- bpo-33687: Fix the call to ``os.chmod()`` for ``uu.decode()`` if a mode is + given or decoded. Patch by Timo Furrer. + +- bpo-32146: Document the interaction between frozen executables and the + spawn and forkserver start methods in multiprocessing. + +Documentation +------------- + +- bpo-36083: Fix formatting of --check-hash-based-pycs options in the + manpage Synopsis. + +- bpo-34764: Improve example of iter() with 2nd sentinel argument. + +- bpo-21314: A new entry was added to the Core Language Section of the + Programming FAQ, which explaines the usage of slash(/) in the signature of + a function. Patch by Lysandros Nikolaou + +- bpo-22062: Update documentation and docstrings for pathlib. Original patch + by Mike Short. + +Tests +----- + +- bpo-36234: test_posix.PosixUidGidTests: add tests for invalid uid/gid type + (str). Initial patch written by David Malcolm. + +- bpo-29571: Fix ``test_re.test_locale_flag()``: use + ``locale.getpreferredencoding()`` rather than ``locale.getlocale()`` to + get the locale encoding. With some locales, ``locale.getlocale()`` returns + the wrong encoding. On Windows, set temporarily the ``LC_CTYPE`` locale to + the user preferred encoding to ensure that it uses the ANSI code page, to + be consistent with ``locale.getpreferredencoding()``. + +- bpo-36123: Fix race condition in test_socket. + +- bpo-27313: Avoid test_ttk_guionly ComboboxTest failure with macOS Cocoa + Tk. + +- bpo-36019: Add test.support.TEST_HTTP_URL and replace references of + http://www.example.com by this new constant. Contributed by Stéphane + Wirtel. + +- bpo-36037: Fix test_ssl for strict OpenSSL configuration like RHEL8 strict + crypto policy. Use older TLS version for minimum TLS version of the server + SSL context if needed, to test TLS version older than default minimum TLS + version. + +- bpo-35505: Make test_imap4_host_default_value independent on whether the + local IMAP server is running. + +- bpo-35917: multiprocessing: provide unit tests for SyncManager and + SharedMemoryManager classes + all the shareable types which are supposed + to be supported by them. (patch by Giampaolo Rodola) + +- bpo-35772: Fix sparse file tests of test_tarfile on ppc64 with the tmpfs + filesystem. Fix the function testing if the filesystem supports sparse + files: create a file which contains data and "holes", instead of creating + a file which contains no data. tmpfs effective block size is a page size + (tmpfs lives in the page cache). RHEL uses 64 KiB pages on aarch64, ppc64, + ppc64le, only s390x and x86_64 use 4 KiB pages, whereas the test punch + holes of 4 KiB. + +- bpo-35045: Make ssl tests less strict and also accept TLSv1 as system + default. The changes unbreaks test_min_max_version on Fedora 29. + +- bpo-31731: Fix a race condition in ``check_interrupted_write()`` of + test_io: create directly the thread with SIGALRM signal blocked, rather + than blocking the signal later from the thread. Previously, it was + possible that the thread gets the signal before the signal is blocked. + +- bpo-35424: Fix test_multiprocessing_main_handling: use + :class:`multiprocessing.Pool` with a context manager and then explicitly + join the pool. + +- bpo-35519: Rename :mod:`test.bisect` module to :mod:`test.bisect_cmd` to + avoid conflict with :mod:`bisect` module when running directly a test like + ``./python Lib/test/test_xmlrpc.py``. + +- bpo-35513: Replace :func:`time.time` with :func:`time.monotonic` in tests + to measure time delta. + +- bpo-34279: :func:`test.support.run_unittest` no longer raise + :exc:`TestDidNotRun` if the test result contains skipped tests. The + exception is now only raised if no test have been run and no test have + been skipped. + +- bpo-35412: Add testcase to ``test_future4``: check unicode literal. + +- bpo-26704: Added test demonstrating double-patching of an instance method. + Patch by Anthony Sottile. + +Build +----- + +- bpo-34691: The _contextvars module is now built into the core Python + library on Windows. + +- bpo-35683: Improved Azure Pipelines build steps and now verifying layouts + correctly + +- bpo-35642: Remove asynciomodule.c from pythoncore.vcxproj + +- bpo-35550: Fix incorrect Solaris #ifdef checks to look for __sun && __SVR4 + instead of sun when compiling. + +Windows +------- + +- bpo-24643: Fix name collisions due to ``#define timezone _timezone`` in + PC/pyconfig.h. + +- bpo-35692: ``pathlib`` no longer raises when checking file and directory + existence on drives that are not ready + +- bpo-35872: Uses the base Python executable when invoking venv in a virtual + environment + +- bpo-35873: Prevents venv paths being inherited by child processes + +- bpo-35299: Fix sysconfig detection of the source directory and distutils + handling of pyconfig.h during PGO profiling + +- bpo-32560: The ``py`` launcher now forwards its ``STARTUPINFO`` structure + to child processes. + +- bpo-35854: Fix EnvBuilder and --symlinks in venv on Windows + +- bpo-35811: Avoid propagating venv settings when launching via py.exe + +- bpo-35797: Fix default executable used by the multiprocessing module + +- bpo-29734: Fix handle leaks in os.stat on Windows. + +- bpo-35596: Use unchecked PYCs for the embeddable distro to avoid zipimport + restrictions. + +- bpo-35596: Fix vcruntime140.dll being added to embeddable distro multiple + times. + +- bpo-35402: Update Windows build to use Tcl and Tk 8.6.9 + +- bpo-33316: PyThread_release_lock always fails + +- bpo-1104: Correctly handle string length in + ``msilib.SummaryInfo.GetProperty()`` to prevent it from truncating the + last character. + +IDLE +---- + +- bpo-36176: Fix IDLE autocomplete & calltip popup colors. Prevent conflicts + with Linux dark themes (and slightly darken calltip background). + +- bpo-36152: Remove colorizer.ColorDelegator.close_when_done and the + corresponding argument of .close(). In IDLE, both have always been None + or False since 2007. + +- bpo-32129: Avoid blurry IDLE application icon on macOS with Tk 8.6. Patch + by Kevin Walzer. + +- bpo-24310: IDLE -- Document settings dialog font tab sample. + +- bpo-36096: Refactor class variables to instance variables in colorizer. + +- bpo-35833: Revise IDLE doc for control codes sent to Shell. Add a code + example block. + +- bpo-35770: IDLE macosx deletes Options => Configure IDLE. It previously + deleted Window => Zoom Height by mistake. (Zoom Height is now on the + Options menu). On Mac, the settings dialog is accessed via Preferences on + the IDLE menu. + +- bpo-35769: Change IDLE's new file name from 'Untitled' to 'untitled' + +- bpo-35689: Add docstrings and unittests for colorizer.py. + +- bpo-35660: Fix imports in idlelib.window. + +- bpo-35641: Proper format `calltip` when the function has no docstring. + +- bpo-33987: Use ttk Frame for ttk widgets. + +- bpo-34055: Fix erroneous 'smart' indents and newlines in IDLE Shell. + +- bpo-35591: Find Selection now works when selection not found. + +- bpo-35196: Speed up squeezer line counting. + +- bpo-35598: Update config_key: use PEP 8 names and ttk widgets, make some + objects global, and add tests. + +- bpo-28097: Add Previous/Next History entries to Shell menu. + +- bpo-35208: Squeezer now properly counts wrapped lines before newlines. + +- bpo-35555: Gray out Code Context menu entry when it's not applicable. + +- bpo-35521: Document the IDLE editor code context feature. Add some + internal references within the IDLE doc. + +- bpo-22703: The Code Context menu label now toggles between Show/Hide Code + Context. The Zoom Height menu now toggles between Zoom/Restore Height. + Zoom Height has moved from the Window menu to the Options menu. + +Tools/Demos +----------- + +- bpo-35132: Fix py-list and py-bt commands of python-gdb.py on gdb7. + +C API +----- + +- bpo-33817: Fixed :c:func:`_PyBytes_Resize` for empty bytes objects. + + What's New in Python 3.7.2 final? ================================= @@ -344,7 +781,10 @@ Windows macOS ----- -- bpo-35402: Update macOS installer to use Tcl/Tk 8.6.9.1. +- bpo-35402: Update macOS installer to use Tcl/Tk 8.6.9.1. [NOTE: This + change was reverted for the released python.org 3.7.2 macOS installers due + to regressions found in Tk 8.6.9.1. For now, the installers provide + Tcl/Tk 8.6.8.] - bpo-35401: Update macOS installer to use OpenSSL 1.1.0j. @@ -1008,11 +1448,11 @@ IDLE Tools/Demos ----------- -- bpo-32962: python-gdb now catchs ``UnicodeDecodeError`` exceptions when +- bpo-32962: python-gdb now catches ``UnicodeDecodeError`` exceptions when calling ``string()``. -- bpo-32962: python-gdb now catchs ValueError on read_var(): when Python has - no debug symbols for example. +- bpo-32962: python-gdb now catches ValueError on read_var(): when Python + has no debug symbols for example. C API ----- diff --git a/kbe/src/lib/python/Misc/python.man b/kbe/src/lib/python/Misc/python.man index 3cdf8840d4..8d5ad8cd6c 100644 --- a/kbe/src/lib/python/Misc/python.man +++ b/kbe/src/lib/python/Misc/python.man @@ -75,7 +75,11 @@ python \- an interpreted, interactive, object-oriented programming language .br [ .B \--check-hash-based-pycs -\'default\'|\'always\'|\'never\' +.I default +| +.I always +| +.I never ] .br [ diff --git a/kbe/src/lib/python/Modules/_abc.c b/kbe/src/lib/python/Modules/_abc.c index 36c1757b5f..1fbf3a8310 100644 --- a/kbe/src/lib/python/Modules/_abc.c +++ b/kbe/src/lib/python/Modules/_abc.c @@ -66,7 +66,7 @@ PyDoc_STRVAR(abc_data_doc, "Internal state held by ABC machinery."); static PyTypeObject _abc_data_type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) + PyVarObject_HEAD_INIT(NULL, 0) "_abc_data", /*tp_name*/ sizeof(_abc_data), /*tp_basicsize*/ .tp_dealloc = (destructor)abc_data_dealloc, diff --git a/kbe/src/lib/python/Modules/_asynciomodule.c b/kbe/src/lib/python/Modules/_asynciomodule.c index 5816a6748f..35264f5815 100644 --- a/kbe/src/lib/python/Modules/_asynciomodule.c +++ b/kbe/src/lib/python/Modules/_asynciomodule.c @@ -1110,6 +1110,10 @@ FutureObj_set_blocking(FutureObj *fut, PyObject *val, void *Py_UNUSED(ignored)) if (future_ensure_alive(fut)) { return -1; } + if (val == NULL) { + PyErr_SetString(PyExc_AttributeError, "cannot delete attribute"); + return -1; + } int is_true = PyObject_IsTrue(val); if (is_true < 0) { @@ -1134,6 +1138,10 @@ FutureObj_get_log_traceback(FutureObj *fut, void *Py_UNUSED(ignored)) static int FutureObj_set_log_traceback(FutureObj *fut, PyObject *val, void *Py_UNUSED(ignored)) { + if (val == NULL) { + PyErr_SetString(PyExc_AttributeError, "cannot delete attribute"); + return -1; + } int is_true = PyObject_IsTrue(val); if (is_true < 0) { return -1; @@ -2008,6 +2016,10 @@ TaskObj_get_log_destroy_pending(TaskObj *task, void *Py_UNUSED(ignored)) static int TaskObj_set_log_destroy_pending(TaskObj *task, PyObject *val, void *Py_UNUSED(ignored)) { + if (val == NULL) { + PyErr_SetString(PyExc_AttributeError, "cannot delete attribute"); + return -1; + } int is_true = PyObject_IsTrue(val); if (is_true < 0) { return -1; diff --git a/kbe/src/lib/python/Modules/_cryptmodule.c b/kbe/src/lib/python/Modules/_cryptmodule.c index 58d179e6a3..5d03f45f64 100644 --- a/kbe/src/lib/python/Modules/_cryptmodule.c +++ b/kbe/src/lib/python/Modules/_cryptmodule.c @@ -34,7 +34,15 @@ static PyObject * crypt_crypt_impl(PyObject *module, const char *word, const char *salt) /*[clinic end generated code: output=0512284a03d2803c input=0e8edec9c364352b]*/ { - return Py_BuildValue("s", crypt(word, salt)); + char *crypt_result; +#ifdef HAVE_CRYPT_R + struct crypt_data data; + memset(&data, 0, sizeof(data)); + crypt_result = crypt_r(word, salt, &data); +#else + crypt_result = crypt(word, salt); +#endif + return Py_BuildValue("s", crypt_result); } diff --git a/kbe/src/lib/python/Modules/_ctypes/_ctypes.c b/kbe/src/lib/python/Modules/_ctypes/_ctypes.c index c5fc811ad9..f98eabbb37 100644 --- a/kbe/src/lib/python/Modules/_ctypes/_ctypes.c +++ b/kbe/src/lib/python/Modules/_ctypes/_ctypes.c @@ -1171,6 +1171,10 @@ CharArray_set_raw(CDataObject *self, PyObject *value, void *Py_UNUSED(ignored)) Py_ssize_t size; Py_buffer view; + if (value == NULL) { + PyErr_SetString(PyExc_AttributeError, "cannot delete attribute"); + return -1; + } if (PyObject_GetBuffer(value, &view, PyBUF_SIMPLE) < 0) return -1; size = view.len; @@ -3405,20 +3409,23 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } #endif - Py_INCREF(dll); /* for KeepRef */ - Py_DECREF(ftuple); - if (!_validate_paramflags(type, paramflags)) + if (!_validate_paramflags(type, paramflags)) { + Py_DECREF(ftuple); return NULL; + } self = (PyCFuncPtrObject *)GenericPyCData_new(type, args, kwds); - if (!self) + if (!self) { + Py_DECREF(ftuple); return NULL; + } Py_XINCREF(paramflags); self->paramflags = paramflags; *(void **)self->b_ptr = address; - + Py_INCREF(dll); + Py_DECREF(ftuple); if (-1 == KeepRef((CDataObject *)self, 0, dll)) { Py_DECREF((PyObject *)self); return NULL; diff --git a/kbe/src/lib/python/Modules/_ctypes/callproc.c b/kbe/src/lib/python/Modules/_ctypes/callproc.c index ec596b4de3..e971388f69 100644 --- a/kbe/src/lib/python/Modules/_ctypes/callproc.c +++ b/kbe/src/lib/python/Modules/_ctypes/callproc.c @@ -1052,7 +1052,7 @@ GetComError(HRESULT errcode, GUID *riid, IUnknown *pIunk) #endif #if (defined(__x86_64__) && (defined(__MINGW64__) || defined(__CYGWIN__))) || \ - defined(__aarch64__) + defined(__aarch64__) || defined(__riscv) #define CTYPES_PASS_BY_REF_HACK #define POW2(x) (((x & ~(x - 1)) == x) ? x : 0) #define IS_PASS_BY_REF(x) (x > 8 || !POW2(x)) diff --git a/kbe/src/lib/python/Modules/_datetimemodule.c b/kbe/src/lib/python/Modules/_datetimemodule.c index 5afeeea488..9405b4610d 100644 --- a/kbe/src/lib/python/Modules/_datetimemodule.c +++ b/kbe/src/lib/python/Modules/_datetimemodule.c @@ -1514,10 +1514,13 @@ wrap_strftime(PyObject *object, PyObject *format, PyObject *timetuple, ntoappend = 1; } else if ((ch = *pin++) == '\0') { - /* There's a lone trailing %; doesn't make sense. */ - PyErr_SetString(PyExc_ValueError, "strftime format " - "ends with raw %"); - goto Done; + /* Null byte follows %, copy only '%'. + * + * Back the pin up one char so that we catch the null check + * the next time through the loop.*/ + pin--; + ptoappend = pin - 1; + ntoappend = 1; } /* A % has been seen and ch is the character after it. */ else if (ch == 'z') { @@ -1602,7 +1605,7 @@ wrap_strftime(PyObject *object, PyObject *format, PyObject *timetuple, usednew += ntoappend; assert(usednew <= totalnew); } /* end while() */ - + if (_PyBytes_Resize(&newfmt, usednew) < 0) goto Done; { diff --git a/kbe/src/lib/python/Modules/_elementtree.c b/kbe/src/lib/python/Modules/_elementtree.c index ff3a240f2e..79f1ccd685 100644 --- a/kbe/src/lib/python/Modules/_elementtree.c +++ b/kbe/src/lib/python/Modules/_elementtree.c @@ -2429,6 +2429,11 @@ _elementtree_TreeBuilder___init___impl(TreeBuilderObject *self, static int treebuilder_gc_traverse(TreeBuilderObject *self, visitproc visit, void *arg) { + Py_VISIT(self->end_ns_event_obj); + Py_VISIT(self->start_ns_event_obj); + Py_VISIT(self->end_event_obj); + Py_VISIT(self->start_event_obj); + Py_VISIT(self->events_append); Py_VISIT(self->root); Py_VISIT(self->this); Py_VISIT(self->last); diff --git a/kbe/src/lib/python/Modules/_functoolsmodule.c b/kbe/src/lib/python/Modules/_functoolsmodule.c index ff4172d663..c6a92ed169 100644 --- a/kbe/src/lib/python/Modules/_functoolsmodule.c +++ b/kbe/src/lib/python/Modules/_functoolsmodule.c @@ -661,6 +661,26 @@ sequence is empty."); /* lru_cache object **********************************************************/ +/* There are four principal algorithmic differences from the pure python version: + + 1). The C version relies on the GIL instead of having its own reentrant lock. + + 2). The prev/next link fields use borrowed references. + + 3). For a full cache, the pure python version rotates the location of the + root entry so that it never has to move individual links and it can + limit updates to just the key and result fields. However, in the C + version, links are temporarily removed while the cache dict updates are + occurring. Afterwards, they are appended or prepended back into the + doubly-linked lists. + + 4) In the Python version, the _HashSeq class is used to prevent __hash__ + from being called more than once. In the C version, the "known hash" + variants of dictionary calls as used to the same effect. + +*/ + + /* this object is used delimit args and keywords in the cache keys */ static PyObject *kwd_mark = NULL; @@ -711,16 +731,15 @@ typedef PyObject *(*lru_cache_ternaryfunc)(struct lru_cache_object *, PyObject * typedef struct lru_cache_object { lru_list_elem root; /* includes PyObject_HEAD */ - Py_ssize_t maxsize; - PyObject *maxsize_O; - PyObject *func; lru_cache_ternaryfunc wrapper; + int typed; PyObject *cache; + Py_ssize_t hits; + PyObject *func; + Py_ssize_t maxsize; + Py_ssize_t misses; PyObject *cache_info_type; - Py_ssize_t misses, hits; - int typed; PyObject *dict; - int full; } lru_cache_object; static PyTypeObject lru_cache_type; @@ -733,6 +752,15 @@ lru_cache_make_key(PyObject *args, PyObject *kwds, int typed) /* short path, key will match args anyway, which is a tuple */ if (!typed && !kwds) { + if (PyTuple_GET_SIZE(args) == 1) { + key = PyTuple_GET_ITEM(args, 0); + if (PyUnicode_CheckExact(key) || PyLong_CheckExact(key)) { + /* For common scalar keys, save space by + dropping the enclosing args tuple */ + Py_INCREF(key); + return key; + } + } Py_INCREF(args); return args; } @@ -788,10 +816,12 @@ lru_cache_make_key(PyObject *args, PyObject *kwds, int typed) static PyObject * uncached_lru_cache_wrapper(lru_cache_object *self, PyObject *args, PyObject *kwds) { - PyObject *result = PyObject_Call(self->func, args, kwds); + PyObject *result; + + self->misses++; + result = PyObject_Call(self->func, args, kwds); if (!result) return NULL; - self->misses++; return result; } @@ -819,6 +849,7 @@ infinite_lru_cache_wrapper(lru_cache_object *self, PyObject *args, PyObject *kwd Py_DECREF(key); return NULL; } + self->misses++; result = PyObject_Call(self->func, args, kwds); if (!result) { Py_DECREF(key); @@ -830,15 +861,16 @@ infinite_lru_cache_wrapper(lru_cache_object *self, PyObject *args, PyObject *kwd return NULL; } Py_DECREF(key); - self->misses++; return result; } static void -lru_cache_extricate_link(lru_list_elem *link) +lru_cache_extract_link(lru_list_elem *link) { - link->prev->next = link->next; - link->next->prev = link->prev; + lru_list_elem *link_prev = link->prev; + lru_list_elem *link_next = link->next; + link_prev->next = link->next; + link_next->prev = link->prev; } static void @@ -851,11 +883,52 @@ lru_cache_append_link(lru_cache_object *self, lru_list_elem *link) link->next = root; } +static void +lru_cache_prepend_link(lru_cache_object *self, lru_list_elem *link) +{ + lru_list_elem *root = &self->root; + lru_list_elem *first = root->next; + first->prev = root->next = link; + link->prev = root; + link->next = first; +} + +/* General note on reentrancy: + + There are four dictionary calls in the bounded_lru_cache_wrapper(): + 1) The initial check for a cache match. 2) The post user-function + check for a cache match. 3) The deletion of the oldest entry. + 4) The addition of the newest entry. + + In all four calls, we have a known hash which lets use avoid a call + to __hash__(). That leaves only __eq__ as a possible source of a + reentrant call. + + The __eq__ method call is always made for a cache hit (dict access #1). + Accordingly, we have make sure not modify the cache state prior to + this call. + + The __eq__ method call is never made for the deletion (dict access #3) + because it is an identity match. + + For the other two accesses (#2 and #4), calls to __eq__ only occur + when some other entry happens to have an exactly matching hash (all + 64-bits). Though rare, this can happen, so we have to make sure to + either call it at the top of its code path before any cache + state modifications (dict access #2) or be prepared to restore + invariants at the end of the code path (dict access #4). + + Another possible source of reentrancy is a decref which can trigger + arbitrary code execution. To make the code easier to reason about, + the decrefs are deferred to the end of the each possible code path + so that we know the cache is a consistent state. + */ + static PyObject * bounded_lru_cache_wrapper(lru_cache_object *self, PyObject *args, PyObject *kwds) { lru_list_elem *link; - PyObject *key, *result; + PyObject *key, *result, *testresult; Py_hash_t hash; key = lru_cache_make_key(args, kwds, self->typed); @@ -867,11 +940,11 @@ bounded_lru_cache_wrapper(lru_cache_object *self, PyObject *args, PyObject *kwds return NULL; } link = (lru_list_elem *)_PyDict_GetItem_KnownHash(self->cache, key, hash); - if (link) { - lru_cache_extricate_link(link); + if (link != NULL) { + lru_cache_extract_link(link); lru_cache_append_link(self, link); - self->hits++; result = link->result; + self->hits++; Py_INCREF(result); Py_DECREF(key); return result; @@ -880,65 +953,38 @@ bounded_lru_cache_wrapper(lru_cache_object *self, PyObject *args, PyObject *kwds Py_DECREF(key); return NULL; } + self->misses++; result = PyObject_Call(self->func, args, kwds); if (!result) { Py_DECREF(key); return NULL; } - if (self->full && self->root.next != &self->root) { - /* Use the oldest item to store the new key and result. */ - PyObject *oldkey, *oldresult, *popresult; - /* Extricate the oldest item. */ - link = self->root.next; - lru_cache_extricate_link(link); - /* Remove it from the cache. - The cache dict holds one reference to the link, - and the linked list holds yet one reference to it. */ - popresult = _PyDict_Pop_KnownHash(self->cache, - link->key, link->hash, - Py_None); - if (popresult == Py_None) { - /* Getting here means that this same key was added to the - cache while the lock was released. Since the link - update is already done, we need only return the - computed result and update the count of misses. */ - Py_DECREF(popresult); - Py_DECREF(link); - Py_DECREF(key); - } - else if (popresult == NULL) { - lru_cache_append_link(self, link); - Py_DECREF(key); - Py_DECREF(result); - return NULL; - } - else { - Py_DECREF(popresult); - /* Keep a reference to the old key and old result to - prevent their ref counts from going to zero during the - update. That will prevent potentially arbitrary object - clean-up code (i.e. __del__) from running while we're - still adjusting the links. */ - oldkey = link->key; - oldresult = link->result; - - link->hash = hash; - link->key = key; - link->result = result; - if (_PyDict_SetItem_KnownHash(self->cache, key, (PyObject *)link, - hash) < 0) { - Py_DECREF(link); - Py_DECREF(oldkey); - Py_DECREF(oldresult); - return NULL; - } - lru_cache_append_link(self, link); - Py_INCREF(result); /* for return */ - Py_DECREF(oldkey); - Py_DECREF(oldresult); - } - } else { - /* Put result in a new link at the front of the queue. */ + testresult = _PyDict_GetItem_KnownHash(self->cache, key, hash); + if (testresult != NULL) { + /* Getting here means that this same key was added to the cache + during the PyObject_Call(). Since the link update is already + done, we need only return the computed result. */ + Py_DECREF(key); + return result; + } + if (PyErr_Occurred()) { + /* This is an unusual case since this same lookup + did not previously trigger an error during lookup. + Treat it the same as an error in user function + and return with the error set. */ + Py_DECREF(key); + Py_DECREF(result); + return NULL; + } + /* This is the normal case. The new key wasn't found before + user function call and it is still not there. So we + proceed normally and update the cache with the new result. */ + + assert(self->maxsize > 0); + if (PyDict_GET_SIZE(self->cache) < self->maxsize || + self->root.next == &self->root) + { + /* Cache is not full, so put the result in a new link */ link = (lru_list_elem *)PyObject_New(lru_list_elem, &lru_list_elem_type); if (link == NULL) { @@ -950,6 +996,11 @@ bounded_lru_cache_wrapper(lru_cache_object *self, PyObject *args, PyObject *kwds link->hash = hash; link->key = key; link->result = result; + /* What is really needed here is a SetItem variant with a "no clobber" + option. If the __eq__ call triggers a reentrant call that adds + this same key, then this setitem call will update the cache dict + with this new link, leaving the old link as an orphan (i.e. not + having a cache dict entry that refers to it). */ if (_PyDict_SetItem_KnownHash(self->cache, key, (PyObject *)link, hash) < 0) { Py_DECREF(link); @@ -957,9 +1008,84 @@ bounded_lru_cache_wrapper(lru_cache_object *self, PyObject *args, PyObject *kwds } lru_cache_append_link(self, link); Py_INCREF(result); /* for return */ - self->full = (PyDict_GET_SIZE(self->cache) >= self->maxsize); + return result; } - self->misses++; + /* Since the cache is full, we need to evict an old key and add + a new key. Rather than free the old link and allocate a new + one, we reuse the link for the new key and result and move it + to front of the cache to mark it as recently used. + + We try to assure all code paths (including errors) leave all + of the links in place. Either the link is successfully + updated and moved or it is restored to its old position. + However if an unrecoverable error is found, it doesn't + make sense to reinsert the link, so we leave it out + and the cache will no longer register as full. + */ + PyObject *oldkey, *oldresult, *popresult; + + /* Extract the oldest item. */ + assert(self->root.next != &self->root); + link = self->root.next; + lru_cache_extract_link(link); + /* Remove it from the cache. + The cache dict holds one reference to the link. + We created one other reference when the link was created. + The linked list only has borrowed references. */ + popresult = _PyDict_Pop_KnownHash(self->cache, link->key, + link->hash, Py_None); + if (popresult == Py_None) { + /* Getting here means that the user function call or another + thread has already removed the old key from the dictionary. + This link is now an orphan. Since we don't want to leave the + cache in an inconsistent state, we don't restore the link. */ + Py_DECREF(popresult); + Py_DECREF(link); + Py_DECREF(key); + return result; + } + if (popresult == NULL) { + /* An error arose while trying to remove the oldest key (the one + being evicted) from the cache. We restore the link to its + original position as the oldest link. Then we allow the + error propagate upward; treating it the same as an error + arising in the user function. */ + lru_cache_prepend_link(self, link); + Py_DECREF(key); + Py_DECREF(result); + return NULL; + } + /* Keep a reference to the old key and old result to prevent their + ref counts from going to zero during the update. That will + prevent potentially arbitrary object clean-up code (i.e. __del__) + from running while we're still adjusting the links. */ + oldkey = link->key; + oldresult = link->result; + + link->hash = hash; + link->key = key; + link->result = result; + /* Note: The link is being added to the cache dict without the + prev and next fields set to valid values. We have to wait + for successful insertion in the cache dict before adding the + link to the linked list. Otherwise, the potentially reentrant + __eq__ call could cause the then orphan link to be visited. */ + if (_PyDict_SetItem_KnownHash(self->cache, key, (PyObject *)link, + hash) < 0) { + /* Somehow the cache dict update failed. We no longer can + restore the old link. Let the error propagate upward and + leave the cache short one link. */ + Py_DECREF(popresult); + Py_DECREF(link); + Py_DECREF(oldkey); + Py_DECREF(oldresult); + return NULL; + } + lru_cache_append_link(self, link); + Py_INCREF(result); /* for return */ + Py_DECREF(popresult); + Py_DECREF(oldkey); + Py_DECREF(oldresult); return result; } @@ -995,6 +1121,9 @@ lru_cache_new(PyTypeObject *type, PyObject *args, PyObject *kw) maxsize = PyNumber_AsSsize_t(maxsize_O, PyExc_OverflowError); if (maxsize == -1 && PyErr_Occurred()) return NULL; + if (maxsize < 0) { + maxsize = 0; + } if (maxsize == 0) wrapper = uncached_lru_cache_wrapper; else @@ -1013,20 +1142,17 @@ lru_cache_new(PyTypeObject *type, PyObject *args, PyObject *kw) return NULL; } - obj->cache = cachedict; obj->root.prev = &obj->root; obj->root.next = &obj->root; - obj->maxsize = maxsize; - Py_INCREF(maxsize_O); - obj->maxsize_O = maxsize_O; + obj->wrapper = wrapper; + obj->typed = typed; + obj->cache = cachedict; Py_INCREF(func); obj->func = func; - obj->wrapper = wrapper; obj->misses = obj->hits = 0; - obj->typed = typed; + obj->maxsize = maxsize; Py_INCREF(cache_info_type); obj->cache_info_type = cache_info_type; - return (PyObject *)obj; } @@ -1060,11 +1186,10 @@ lru_cache_dealloc(lru_cache_object *obj) PyObject_GC_UnTrack(obj); list = lru_cache_unlink_list(obj); - Py_XDECREF(obj->maxsize_O); - Py_XDECREF(obj->func); Py_XDECREF(obj->cache); - Py_XDECREF(obj->dict); + Py_XDECREF(obj->func); Py_XDECREF(obj->cache_info_type); + Py_XDECREF(obj->dict); lru_cache_clear_list(list); Py_TYPE(obj)->tp_free(obj); } @@ -1088,8 +1213,13 @@ lru_cache_descr_get(PyObject *self, PyObject *obj, PyObject *type) static PyObject * lru_cache_cache_info(lru_cache_object *self, PyObject *unused) { - return PyObject_CallFunction(self->cache_info_type, "nnOn", - self->hits, self->misses, self->maxsize_O, + if (self->maxsize == -1) { + return PyObject_CallFunction(self->cache_info_type, "nnOn", + self->hits, self->misses, Py_None, + PyDict_GET_SIZE(self->cache)); + } + return PyObject_CallFunction(self->cache_info_type, "nnnn", + self->hits, self->misses, self->maxsize, PyDict_GET_SIZE(self->cache)); } @@ -1098,7 +1228,6 @@ lru_cache_cache_clear(lru_cache_object *self, PyObject *unused) { lru_list_elem *list = lru_cache_unlink_list(self); self->hits = self->misses = 0; - self->full = 0; PyDict_Clear(self->cache); lru_cache_clear_list(list); Py_RETURN_NONE; @@ -1134,7 +1263,6 @@ lru_cache_tp_traverse(lru_cache_object *self, visitproc visit, void *arg) Py_VISIT(link->result); link = next; } - Py_VISIT(self->maxsize_O); Py_VISIT(self->func); Py_VISIT(self->cache); Py_VISIT(self->cache_info_type); @@ -1146,7 +1274,6 @@ static int lru_cache_tp_clear(lru_cache_object *self) { lru_list_elem *list = lru_cache_unlink_list(self); - Py_CLEAR(self->maxsize_O); Py_CLEAR(self->func); Py_CLEAR(self->cache); Py_CLEAR(self->cache_info_type); diff --git a/kbe/src/lib/python/Modules/_hashopenssl.c b/kbe/src/lib/python/Modules/_hashopenssl.c index 31d05409f2..b69f16c61a 100644 --- a/kbe/src/lib/python/Modules/_hashopenssl.c +++ b/kbe/src/lib/python/Modules/_hashopenssl.c @@ -112,17 +112,18 @@ newEVPobject(PyObject *name) return NULL; } + /* save the name for .name to return */ + Py_INCREF(name); + retval->name = name; + retval->lock = NULL; + retval->ctx = EVP_MD_CTX_new(); if (retval->ctx == NULL) { + Py_DECREF(retval); PyErr_NoMemory(); return NULL; } - /* save the name for .name to return */ - Py_INCREF(name); - retval->name = name; - retval->lock = NULL; - return retval; } @@ -181,6 +182,7 @@ EVP_copy(EVPobject *self, PyObject *unused) return NULL; if (!locked_EVP_MD_CTX_copy(newobj->ctx, self)) { + Py_DECREF(newobj); return _setException(PyExc_ValueError); } return (PyObject *)newobj; @@ -824,7 +826,7 @@ _hashlib_scrypt_impl(PyObject *module, Py_buffer *password, Py_buffer *salt, if (!retval) { /* sorry, can't do much better */ PyErr_SetString(PyExc_ValueError, - "Invalid paramemter combination for n, r, p, maxmem."); + "Invalid parameter combination for n, r, p, maxmem."); return NULL; } diff --git a/kbe/src/lib/python/Modules/_io/textio.c b/kbe/src/lib/python/Modules/_io/textio.c index 9d0d9cac40..49b545c478 100644 --- a/kbe/src/lib/python/Modules/_io/textio.c +++ b/kbe/src/lib/python/Modules/_io/textio.c @@ -3049,6 +3049,10 @@ textiowrapper_chunk_size_set(textio *self, PyObject *arg, void *context) { Py_ssize_t n; CHECK_ATTACHED_INT(self); + if (arg == NULL) { + PyErr_SetString(PyExc_AttributeError, "cannot delete attribute"); + return -1; + } n = PyNumber_AsSsize_t(arg, PyExc_ValueError); if (n == -1 && PyErr_Occurred()) return -1; diff --git a/kbe/src/lib/python/Modules/_pickle.c b/kbe/src/lib/python/Modules/_pickle.c index 60ef921521..15e15cdf45 100644 --- a/kbe/src/lib/python/Modules/_pickle.c +++ b/kbe/src/lib/python/Modules/_pickle.c @@ -6636,13 +6636,13 @@ _pickle_Unpickler_find_class_impl(UnpicklerObject *self, } } - module = PyImport_GetModule(module_name); + /* + * we don't use PyImport_GetModule here, because it can return partially- + * initialised modules, which then cause the getattribute to fail. + */ + module = PyImport_Import(module_name); if (module == NULL) { - if (PyErr_Occurred()) - return NULL; - module = PyImport_Import(module_name); - if (module == NULL) - return NULL; + return NULL; } global = getattribute(module, global_name, self->proto >= 4); Py_DECREF(module); diff --git a/kbe/src/lib/python/Modules/_posixsubprocess.c b/kbe/src/lib/python/Modules/_posixsubprocess.c index 851aa97771..3cf0683ad9 100644 --- a/kbe/src/lib/python/Modules/_posixsubprocess.c +++ b/kbe/src/lib/python/Modules/_posixsubprocess.c @@ -30,7 +30,7 @@ # define SYS_getdents64 __NR_getdents64 #endif -#if defined(sun) +#if defined(__sun) && defined(__SVR4) /* readdir64 is used to work around Solaris 9 bug 6395699. */ # define readdir readdir64 # define dirent dirent64 diff --git a/kbe/src/lib/python/Modules/_randommodule.c b/kbe/src/lib/python/Modules/_randommodule.c index 51677f8b00..1a76ba99ab 100644 --- a/kbe/src/lib/python/Modules/_randommodule.c +++ b/kbe/src/lib/python/Modules/_randommodule.c @@ -292,7 +292,6 @@ random_seed(RandomObject *self, PyObject *args) PY_LITTLE_ENDIAN, 0); /* unsigned */ if (res == -1) { - PyMem_Free(key); goto Done; } diff --git a/kbe/src/lib/python/Modules/_sha3/cleanup.py b/kbe/src/lib/python/Modules/_sha3/cleanup.py index 17c56b34b2..4f53681b49 100755 --- a/kbe/src/lib/python/Modules/_sha3/cleanup.py +++ b/kbe/src/lib/python/Modules/_sha3/cleanup.py @@ -8,7 +8,7 @@ import re CPP1 = re.compile("^//(.*)") -CPP2 = re.compile("\ //(.*)") +CPP2 = re.compile(r"\ //(.*)") STATICS = ("void ", "int ", "HashReturn ", "const UINT64 ", "UINT16 ", " int prefix##") diff --git a/kbe/src/lib/python/Modules/_sqlite/connection.c b/kbe/src/lib/python/Modules/_sqlite/connection.c index 351317e78d..d43286a2d3 100644 --- a/kbe/src/lib/python/Modules/_sqlite/connection.c +++ b/kbe/src/lib/python/Modules/_sqlite/connection.c @@ -1138,6 +1138,10 @@ static PyObject* pysqlite_connection_get_in_transaction(pysqlite_Connection* sel static int pysqlite_connection_set_isolation_level(pysqlite_Connection* self, PyObject* isolation_level, void *Py_UNUSED(ignored)) { + if (isolation_level == NULL) { + PyErr_SetString(PyExc_AttributeError, "cannot delete attribute"); + return -1; + } if (isolation_level == Py_None) { PyObject *res = pysqlite_connection_commit(self, NULL); if (!res) { diff --git a/kbe/src/lib/python/Modules/_sre.c b/kbe/src/lib/python/Modules/_sre.c index d2ea62d55a..4d2bdcc209 100644 --- a/kbe/src/lib/python/Modules/_sre.c +++ b/kbe/src/lib/python/Modules/_sre.c @@ -347,7 +347,7 @@ _sre_unicode_tolower_impl(PyObject *module, int character) LOCAL(void) state_reset(SRE_STATE* state) { - /* FIXME: dynamic! */ + /* state->mark will be set to 0 in SRE_OP_MARK dynamically. */ /*memset(state->mark, 0, sizeof(*state->mark) * SRE_MARK_SIZE);*/ state->lastmark = -1; @@ -2319,7 +2319,7 @@ match_repr(MatchObject *self) if (group0 == NULL) return NULL; result = PyUnicode_FromFormat( - "<%s object; span=(%d, %d), match=%.50R>", + "<%s object; span=(%zd, %zd), match=%.50R>", Py_TYPE(self)->tp_name, self->mark[0], self->mark[1], group0); Py_DECREF(group0); diff --git a/kbe/src/lib/python/Modules/_ssl.c b/kbe/src/lib/python/Modules/_ssl.c index 310b38bf11..9baec8a9bc 100644 --- a/kbe/src/lib/python/Modules/_ssl.c +++ b/kbe/src/lib/python/Modules/_ssl.c @@ -1516,6 +1516,10 @@ _get_crl_dp(X509 *certificate) { STACK_OF(GENERAL_NAME) *gns; dp = sk_DIST_POINT_value(dps, i); + if (dp->distpoint == NULL) { + /* Ignore empty DP value, CVE-2019-5010 */ + continue; + } gns = dp->distpoint->name.fullname; for (j=0; j < sk_GENERAL_NAME_num(gns); j++) { @@ -3626,6 +3630,10 @@ static int set_post_handshake_auth(PySSLContext *self, PyObject *arg, void *c) { int (*verify_cb)(int, X509_STORE_CTX *) = NULL; int mode = SSL_CTX_get_verify_mode(self->ctx); + if (arg == NULL) { + PyErr_SetString(PyExc_AttributeError, "cannot delete attribute"); + return -1; + } int pha = PyObject_IsTrue(arg); if (pha == -1) { diff --git a/kbe/src/lib/python/Modules/_winapi.c b/kbe/src/lib/python/Modules/_winapi.c index 9ea5a92662..036464d9ab 100644 --- a/kbe/src/lib/python/Modules/_winapi.c +++ b/kbe/src/lib/python/Modules/_winapi.c @@ -974,7 +974,8 @@ getattributelist(PyObject *obj, const char *name, AttributeList *attribute_list) _winapi.CreateProcess application_name: Py_UNICODE(accept={str, NoneType}) - command_line: Py_UNICODE(accept={str, NoneType}) + command_line: object + Can be str or None proc_attrs: object Ignored internally, can be None. thread_attrs: object @@ -993,13 +994,14 @@ process ID, and thread ID. [clinic start generated code]*/ static PyObject * -_winapi_CreateProcess_impl(PyObject *module, Py_UNICODE *application_name, - Py_UNICODE *command_line, PyObject *proc_attrs, +_winapi_CreateProcess_impl(PyObject *module, + const Py_UNICODE *application_name, + PyObject *command_line, PyObject *proc_attrs, PyObject *thread_attrs, BOOL inherit_handles, DWORD creation_flags, PyObject *env_mapping, - Py_UNICODE *current_directory, + const Py_UNICODE *current_directory, PyObject *startup_info) -/*[clinic end generated code: output=4652a33aff4b0ae1 input=4a43b05038d639bb]*/ +/*[clinic end generated code: output=9b2423a609230132 input=42ac293eaea03fc4]*/ { PyObject *ret = NULL; BOOL result; @@ -1007,6 +1009,7 @@ _winapi_CreateProcess_impl(PyObject *module, Py_UNICODE *application_name, STARTUPINFOEXW si; PyObject *environment = NULL; wchar_t *wenvironment; + wchar_t *command_line_copy = NULL; AttributeList attribute_list = {0}; ZeroMemory(&si, sizeof(si)); @@ -1041,10 +1044,23 @@ _winapi_CreateProcess_impl(PyObject *module, Py_UNICODE *application_name, goto cleanup; si.lpAttributeList = attribute_list.attribute_list; + if (PyUnicode_Check(command_line)) { + command_line_copy = PyUnicode_AsWideCharString(command_line, NULL); + if (command_line_copy == NULL) { + goto cleanup; + } + } + else if (command_line != Py_None) { + PyErr_Format(PyExc_TypeError, + "CreateProcess() argument 2 must be str or None, not %s", + Py_TYPE(command_line)->tp_name); + goto cleanup; + } + Py_BEGIN_ALLOW_THREADS result = CreateProcessW(application_name, - command_line, + command_line_copy, NULL, NULL, inherit_handles, @@ -1068,6 +1084,7 @@ _winapi_CreateProcess_impl(PyObject *module, Py_UNICODE *application_name, pi.dwThreadId); cleanup: + PyMem_Free(command_line_copy); Py_XDECREF(environment); freeattributelist(&attribute_list); diff --git a/kbe/src/lib/python/Modules/arraymodule.c b/kbe/src/lib/python/Modules/arraymodule.c index 7b4a4a3ad2..ee7ae54661 100644 --- a/kbe/src/lib/python/Modules/arraymodule.c +++ b/kbe/src/lib/python/Modules/arraymodule.c @@ -1712,9 +1712,9 @@ some other type. [clinic start generated code]*/ static PyObject * -array_array_fromunicode_impl(arrayobject *self, Py_UNICODE *ustr, +array_array_fromunicode_impl(arrayobject *self, const Py_UNICODE *ustr, Py_ssize_clean_t ustr_length) -/*[clinic end generated code: output=ebb72fc16975e06d input=150f00566ffbca6e]*/ +/*[clinic end generated code: output=cf2f662908e2befc input=150f00566ffbca6e]*/ { char typecode; diff --git a/kbe/src/lib/python/Modules/cjkcodecs/multibytecodec.c b/kbe/src/lib/python/Modules/cjkcodecs/multibytecodec.c index 4d0aaf3a33..5c91ada103 100644 --- a/kbe/src/lib/python/Modules/cjkcodecs/multibytecodec.c +++ b/kbe/src/lib/python/Modules/cjkcodecs/multibytecodec.c @@ -133,6 +133,10 @@ codecctx_errors_set(MultibyteStatefulCodecContext *self, PyObject *value, PyObject *cb; const char *str; + if (value == NULL) { + PyErr_SetString(PyExc_AttributeError, "cannot delete attribute"); + return -1; + } if (!PyUnicode_Check(value)) { PyErr_SetString(PyExc_TypeError, "errors must be a string"); return -1; diff --git a/kbe/src/lib/python/Modules/clinic/_winapi.c.h b/kbe/src/lib/python/Modules/clinic/_winapi.c.h index c66522ebab..f4d884237b 100644 --- a/kbe/src/lib/python/Modules/clinic/_winapi.c.h +++ b/kbe/src/lib/python/Modules/clinic/_winapi.c.h @@ -286,6 +286,8 @@ PyDoc_STRVAR(_winapi_CreateProcess__doc__, "\n" "Create a new process and its primary thread.\n" "\n" +" command_line\n" +" Can be str or None\n" " proc_attrs\n" " Ignored internally, can be None.\n" " thread_attrs\n" @@ -298,28 +300,29 @@ PyDoc_STRVAR(_winapi_CreateProcess__doc__, {"CreateProcess", (PyCFunction)_winapi_CreateProcess, METH_FASTCALL, _winapi_CreateProcess__doc__}, static PyObject * -_winapi_CreateProcess_impl(PyObject *module, Py_UNICODE *application_name, - Py_UNICODE *command_line, PyObject *proc_attrs, +_winapi_CreateProcess_impl(PyObject *module, + const Py_UNICODE *application_name, + PyObject *command_line, PyObject *proc_attrs, PyObject *thread_attrs, BOOL inherit_handles, DWORD creation_flags, PyObject *env_mapping, - Py_UNICODE *current_directory, + const Py_UNICODE *current_directory, PyObject *startup_info); static PyObject * _winapi_CreateProcess(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - Py_UNICODE *application_name; - Py_UNICODE *command_line; + const Py_UNICODE *application_name; + PyObject *command_line; PyObject *proc_attrs; PyObject *thread_attrs; BOOL inherit_handles; DWORD creation_flags; PyObject *env_mapping; - Py_UNICODE *current_directory; + const Py_UNICODE *current_directory; PyObject *startup_info; - if (!_PyArg_ParseStack(args, nargs, "ZZOOikOZO:CreateProcess", + if (!_PyArg_ParseStack(args, nargs, "ZOOOikOZO:CreateProcess", &application_name, &command_line, &proc_attrs, &thread_attrs, &inherit_handles, &creation_flags, &env_mapping, ¤t_directory, &startup_info)) { goto exit; } @@ -941,4 +944,4 @@ _winapi_GetFileType(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P exit: return return_value; } -/*[clinic end generated code: output=baaf3d379b91be0a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=896d06ce2290aa86 input=a9049054013a1b77]*/ diff --git a/kbe/src/lib/python/Modules/clinic/arraymodule.c.h b/kbe/src/lib/python/Modules/clinic/arraymodule.c.h index b03a507914..c85d0c6262 100644 --- a/kbe/src/lib/python/Modules/clinic/arraymodule.c.h +++ b/kbe/src/lib/python/Modules/clinic/arraymodule.c.h @@ -376,14 +376,14 @@ PyDoc_STRVAR(array_array_fromunicode__doc__, {"fromunicode", (PyCFunction)array_array_fromunicode, METH_O, array_array_fromunicode__doc__}, static PyObject * -array_array_fromunicode_impl(arrayobject *self, Py_UNICODE *ustr, +array_array_fromunicode_impl(arrayobject *self, const Py_UNICODE *ustr, Py_ssize_clean_t ustr_length); static PyObject * array_array_fromunicode(arrayobject *self, PyObject *arg) { PyObject *return_value = NULL; - Py_UNICODE *ustr; + const Py_UNICODE *ustr; Py_ssize_clean_t ustr_length; if (!PyArg_Parse(arg, "u#:fromunicode", &ustr, &ustr_length)) { @@ -505,4 +505,4 @@ PyDoc_STRVAR(array_arrayiterator___setstate____doc__, #define ARRAY_ARRAYITERATOR___SETSTATE___METHODDEF \ {"__setstate__", (PyCFunction)array_arrayiterator___setstate__, METH_O, array_arrayiterator___setstate____doc__}, -/*[clinic end generated code: output=1289bde2a095a712 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=2d0fb1937dea02c2 input=a9049054013a1b77]*/ diff --git a/kbe/src/lib/python/Modules/clinic/posixmodule.c.h b/kbe/src/lib/python/Modules/clinic/posixmodule.c.h index 3230cd7009..caa1cacd3f 100644 --- a/kbe/src/lib/python/Modules/clinic/posixmodule.c.h +++ b/kbe/src/lib/python/Modules/clinic/posixmodule.c.h @@ -1264,7 +1264,7 @@ PyDoc_STRVAR(os_replace__doc__, " descriptor open to a directory, and the respective path string (src or dst)\n" " should be relative; the path will then be relative to that directory.\n" "src_dir_fd and dst_dir_fd, may not be implemented on your platform.\n" -" If they are unavailable, using them will raise a NotImplementedError.\""); +" If they are unavailable, using them will raise a NotImplementedError."); #define OS_REPLACE_METHODDEF \ {"replace", (PyCFunction)os_replace, METH_FASTCALL|METH_KEYWORDS, os_replace__doc__}, @@ -1350,7 +1350,7 @@ PyDoc_STRVAR(os_system__doc__, {"system", (PyCFunction)os_system, METH_FASTCALL|METH_KEYWORDS, os_system__doc__}, static long -os_system_impl(PyObject *module, Py_UNICODE *command); +os_system_impl(PyObject *module, const Py_UNICODE *command); static PyObject * os_system(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -1358,7 +1358,7 @@ os_system(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *k PyObject *return_value = NULL; static const char * const _keywords[] = {"command", NULL}; static _PyArg_Parser _parser = {"u:system", _keywords, 0}; - Py_UNICODE *command; + const Py_UNICODE *command; long _return_value; if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, @@ -5201,7 +5201,8 @@ PyDoc_STRVAR(os_startfile__doc__, {"startfile", (PyCFunction)os_startfile, METH_FASTCALL|METH_KEYWORDS, os_startfile__doc__}, static PyObject * -os_startfile_impl(PyObject *module, path_t *filepath, Py_UNICODE *operation); +os_startfile_impl(PyObject *module, path_t *filepath, + const Py_UNICODE *operation); static PyObject * os_startfile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -5210,7 +5211,7 @@ os_startfile(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject static const char * const _keywords[] = {"filepath", "operation", NULL}; static _PyArg_Parser _parser = {"O&|u:startfile", _keywords, 0}; path_t filepath = PATH_T_INITIALIZE("startfile", "filepath", 0, 0); - Py_UNICODE *operation = NULL; + const Py_UNICODE *operation = NULL; if (!_PyArg_ParseStackAndKeywords(args, nargs, kwnames, &_parser, path_converter, &filepath, &operation)) { @@ -6537,4 +6538,4 @@ os_getrandom(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject #ifndef OS_GETRANDOM_METHODDEF #define OS_GETRANDOM_METHODDEF #endif /* !defined(OS_GETRANDOM_METHODDEF) */ -/*[clinic end generated code: output=c6ca6ad4afa64454 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=32c935671ee020d5 input=a9049054013a1b77]*/ diff --git a/kbe/src/lib/python/Modules/getpath.c b/kbe/src/lib/python/Modules/getpath.c index e6a3e8e78c..bfb33316d9 100644 --- a/kbe/src/lib/python/Modules/getpath.c +++ b/kbe/src/lib/python/Modules/getpath.c @@ -296,6 +296,41 @@ absolutize(wchar_t *path) } +#if defined(__CYGWIN__) || defined(__MINGW32__) +/* add_exe_suffix requires that progpath be allocated at least + MAXPATHLEN + 1 bytes. +*/ + +#ifndef EXE_SUFFIX +#define EXE_SUFFIX L".exe" +#endif + +static void +add_exe_suffix(wchar_t *progpath) +{ + /* Check for already have an executable suffix */ + size_t n = wcslen(progpath); + size_t s = wcslen(EXE_SUFFIX); + if (wcsncasecmp(EXE_SUFFIX, progpath+n-s, s) != 0) { + if (n + s > MAXPATHLEN) { + Py_FatalError("progpath overflow in getpath.c's add_exe_suffix()"); + } + /* Save original path for revert */ + wchar_t orig[MAXPATHLEN+1]; + wcsncpy(orig, progpath, MAXPATHLEN); + + wcsncpy(progpath+n, EXE_SUFFIX, s); + progpath[n+s] = '\0'; + + if (!isxfile(progpath)) { + /* Path that added suffix is invalid */ + wcsncpy(progpath, orig, MAXPATHLEN); + } + } +} +#endif + + /* search_for_prefix requires that argv0_path be no more than MAXPATHLEN bytes long. */ @@ -605,6 +640,16 @@ calculate_program_full_path(const _PyCoreConfig *core_config, if (program_full_path[0] != SEP && program_full_path[0] != '\0') { absolutize(program_full_path); } +#if defined(__CYGWIN__) || defined(__MINGW32__) + /* For these platforms it is necessary to ensure that the .exe suffix + * is appended to the filename, otherwise there is potential for + * sys.executable to return the name of a directory under the same + * path (bpo-28441). + */ + if (program_full_path[0] != '\0') { + add_exe_suffix(program_full_path); + } +#endif config->program_full_path = _PyMem_RawWcsdup(program_full_path); if (config->program_full_path == NULL) { diff --git a/kbe/src/lib/python/Modules/main.c b/kbe/src/lib/python/Modules/main.c index af2c191b9b..a745381109 100644 --- a/kbe/src/lib/python/Modules/main.c +++ b/kbe/src/lib/python/Modules/main.c @@ -2165,6 +2165,7 @@ pymain_read_conf(_PyMain *pymain, _PyCoreConfig *config, _PyCmdline *cmdline) goto done; } pymain_clear_cmdline(pymain, cmdline); + pymain_clear_pymain(pymain); memset(cmdline, 0, sizeof(*cmdline)); cmdline_get_global_config(cmdline); diff --git a/kbe/src/lib/python/Modules/mathmodule.c b/kbe/src/lib/python/Modules/mathmodule.c index 5d9fe5aa71..9eaeff1159 100644 --- a/kbe/src/lib/python/Modules/mathmodule.c +++ b/kbe/src/lib/python/Modules/mathmodule.c @@ -77,7 +77,7 @@ static const double sqrtpi = 1.772453850905516027298167483341145182798; #endif /* !defined(HAVE_ERF) || !defined(HAVE_ERFC) */ static double -sinpi(double x) +m_sinpi(double x) { double y, r; int n; @@ -305,7 +305,7 @@ m_tgamma(double x) integer. */ if (absx > 200.0) { if (x < 0.0) { - return 0.0/sinpi(x); + return 0.0/m_sinpi(x); } else { errno = ERANGE; @@ -329,7 +329,7 @@ m_tgamma(double x) } z = z * lanczos_g / y; if (x < 0.0) { - r = -pi / sinpi(absx) / absx * exp(y) / lanczos_sum(absx); + r = -pi / m_sinpi(absx) / absx * exp(y) / lanczos_sum(absx); r -= z * r; if (absx < 140.0) { r /= pow(y, absx - 0.5); @@ -400,7 +400,7 @@ m_lgamma(double x) r += (absx - 0.5) * (log(absx + lanczos_g - 0.5) - 1); if (x < 0.0) /* Use reflection formula to get value for negative x. */ - r = logpi - log(fabs(sinpi(absx))) - log(absx) - r; + r = logpi - log(fabs(m_sinpi(absx))) - log(absx) - r; if (Py_IS_INFINITY(r)) errno = ERANGE; return r; diff --git a/kbe/src/lib/python/Modules/overlapped.c b/kbe/src/lib/python/Modules/overlapped.c index ae7cddadd0..7798856bee 100644 --- a/kbe/src/lib/python/Modules/overlapped.c +++ b/kbe/src/lib/python/Modules/overlapped.c @@ -561,6 +561,28 @@ Overlapped_new(PyTypeObject *type, PyObject *args, PyObject *kwds) return (PyObject *)self; } + +/* Note (bpo-32710): OverlappedType.tp_clear is not defined to not release + buffers while overlapped are still running, to prevent a crash. */ +static int +Overlapped_clear(OverlappedObject *self) +{ + switch (self->type) { + case TYPE_READ: + case TYPE_ACCEPT: + Py_CLEAR(self->allocated_buffer); + break; + case TYPE_WRITE: + case TYPE_READINTO: + if (self->user_buffer.obj) { + PyBuffer_Release(&self->user_buffer); + } + break; + } + self->type = TYPE_NOT_STARTED; + return 0; +} + static void Overlapped_dealloc(OverlappedObject *self) { @@ -594,20 +616,11 @@ Overlapped_dealloc(OverlappedObject *self) } } - if (self->overlapped.hEvent != NULL) + if (self->overlapped.hEvent != NULL) { CloseHandle(self->overlapped.hEvent); - - switch (self->type) { - case TYPE_READ: - case TYPE_ACCEPT: - Py_CLEAR(self->allocated_buffer); - break; - case TYPE_WRITE: - case TYPE_READINTO: - if (self->user_buffer.obj) - PyBuffer_Release(&self->user_buffer); - break; } + + Overlapped_clear(self); PyObject_Del(self); SetLastError(olderr); } @@ -723,7 +736,7 @@ do_ReadFile(OverlappedObject *self, HANDLE handle, case ERROR_IO_PENDING: Py_RETURN_NONE; default: - self->type = TYPE_NOT_STARTED; + Overlapped_clear(self); return SetFromWindowsErr(err); } } @@ -826,7 +839,7 @@ do_WSARecv(OverlappedObject *self, HANDLE handle, case ERROR_IO_PENDING: Py_RETURN_NONE; default: - self->type = TYPE_NOT_STARTED; + Overlapped_clear(self); return SetFromWindowsErr(err); } } @@ -954,7 +967,7 @@ Overlapped_WriteFile(OverlappedObject *self, PyObject *args) case ERROR_IO_PENDING: Py_RETURN_NONE; default: - self->type = TYPE_NOT_STARTED; + Overlapped_clear(self); return SetFromWindowsErr(err); } } @@ -1011,7 +1024,7 @@ Overlapped_WSASend(OverlappedObject *self, PyObject *args) case ERROR_IO_PENDING: Py_RETURN_NONE; default: - self->type = TYPE_NOT_STARTED; + Overlapped_clear(self); return SetFromWindowsErr(err); } } @@ -1061,7 +1074,7 @@ Overlapped_AcceptEx(OverlappedObject *self, PyObject *args) case ERROR_IO_PENDING: Py_RETURN_NONE; default: - self->type = TYPE_NOT_STARTED; + Overlapped_clear(self); return SetFromWindowsErr(err); } } @@ -1153,7 +1166,7 @@ Overlapped_ConnectEx(OverlappedObject *self, PyObject *args) case ERROR_IO_PENDING: Py_RETURN_NONE; default: - self->type = TYPE_NOT_STARTED; + Overlapped_clear(self); return SetFromWindowsErr(err); } } @@ -1192,7 +1205,7 @@ Overlapped_DisconnectEx(OverlappedObject *self, PyObject *args) case ERROR_IO_PENDING: Py_RETURN_NONE; default: - self->type = TYPE_NOT_STARTED; + Overlapped_clear(self); return SetFromWindowsErr(err); } } @@ -1247,7 +1260,7 @@ Overlapped_TransmitFile(OverlappedObject *self, PyObject *args) case ERROR_IO_PENDING: Py_RETURN_NONE; default: - self->type = TYPE_NOT_STARTED; + Overlapped_clear(self); return SetFromWindowsErr(err); } } @@ -1288,7 +1301,7 @@ Overlapped_ConnectNamedPipe(OverlappedObject *self, PyObject *args) case ERROR_IO_PENDING: Py_RETURN_FALSE; default: - self->type = TYPE_NOT_STARTED; + Overlapped_clear(self); return SetFromWindowsErr(err); } } @@ -1338,6 +1351,25 @@ Overlapped_getpending(OverlappedObject *self) self->type != TYPE_NOT_STARTED); } +static int +Overlapped_traverse(OverlappedObject *self, visitproc visit, void *arg) +{ + switch (self->type) { + case TYPE_READ: + case TYPE_ACCEPT: + Py_VISIT(self->allocated_buffer); + break; + case TYPE_WRITE: + case TYPE_READINTO: + if (self->user_buffer.obj) { + Py_VISIT(&self->user_buffer.obj); + } + break; + } + return 0; +} + + static PyMethodDef Overlapped_methods[] = { {"getresult", (PyCFunction) Overlapped_getresult, METH_VARARGS, Overlapped_getresult_doc}, @@ -1408,7 +1440,7 @@ PyTypeObject OverlappedType = { /* tp_as_buffer */ 0, /* tp_flags */ Py_TPFLAGS_DEFAULT, /* tp_doc */ "OVERLAPPED structure wrapper", - /* tp_traverse */ 0, + /* tp_traverse */ (traverseproc)Overlapped_traverse, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, diff --git a/kbe/src/lib/python/Modules/posixmodule.c b/kbe/src/lib/python/Modules/posixmodule.c index 5403660ba4..e7a1f987de 100644 --- a/kbe/src/lib/python/Modules/posixmodule.c +++ b/kbe/src/lib/python/Modules/posixmodule.c @@ -393,6 +393,10 @@ static int win32_can_symlink = 0; #define HAVE_STRUCT_STAT_ST_FSTYPE 1 #endif +#ifdef _Py_MEMORY_SANITIZER +# include +#endif + #ifdef HAVE_FORK static void run_at_forkers(PyObject *lst, int reverse) @@ -975,28 +979,35 @@ path_converter(PyObject *o, void *p) if (!is_index && !is_buffer && !is_unicode && !is_bytes) { /* Inline PyOS_FSPath() for better error messages. */ _Py_IDENTIFIER(__fspath__); - PyObject *func = NULL; + PyObject *func, *res; func = _PyObject_LookupSpecial(o, &PyId___fspath__); if (NULL == func) { goto error_format; } - /* still owns a reference to the original object */ - Py_DECREF(o); - o = _PyObject_CallNoArg(func); + res = _PyObject_CallNoArg(func); Py_DECREF(func); - if (NULL == o) { + if (NULL == res) { goto error_exit; } - else if (PyUnicode_Check(o)) { + else if (PyUnicode_Check(res)) { is_unicode = 1; } - else if (PyBytes_Check(o)) { + else if (PyBytes_Check(res)) { is_bytes = 1; } else { - goto error_format; + PyErr_Format(PyExc_TypeError, + "expected %.200s.__fspath__() to return str or bytes, " + "not %.200s", Py_TYPE(o)->tp_name, + Py_TYPE(res)->tp_name); + Py_DECREF(res); + goto error_exit; } + + /* still owns a reference to the original object */ + Py_DECREF(o); + o = res; } if (is_unicode) { @@ -1612,11 +1623,6 @@ get_target_path(HANDLE hdl, wchar_t **target_path) return FALSE; } - if(!CloseHandle(hdl)) { - PyMem_RawFree(buf); - return FALSE; - } - buf[result_length] = 0; *target_path = buf; @@ -1674,9 +1680,10 @@ win32_xstat_impl(const wchar_t *path, struct _Py_stat_struct *result, return -1; } if (info.dwFileAttributes & FILE_ATTRIBUTE_REPARSE_POINT) { - if (!win32_get_reparse_tag(hFile, &reparse_tag)) + if (!win32_get_reparse_tag(hFile, &reparse_tag)) { + CloseHandle(hFile); return -1; - + } /* Close the outer open file handle now that we're about to reopen it with different flags. */ if (!CloseHandle(hFile)) @@ -1693,8 +1700,14 @@ win32_xstat_impl(const wchar_t *path, struct _Py_stat_struct *result, if (hFile2 == INVALID_HANDLE_VALUE) return -1; - if (!get_target_path(hFile2, &target_path)) + if (!get_target_path(hFile2, &target_path)) { + CloseHandle(hFile2); + return -1; + } + + if (!CloseHandle(hFile2)) { return -1; + } code = win32_xstat_impl(target_path, result, FALSE); PyMem_RawFree(target_path); @@ -4116,13 +4129,13 @@ If either src_dir_fd or dst_dir_fd is not None, it should be a file descriptor open to a directory, and the respective path string (src or dst) should be relative; the path will then be relative to that directory. src_dir_fd and dst_dir_fd, may not be implemented on your platform. - If they are unavailable, using them will raise a NotImplementedError." + If they are unavailable, using them will raise a NotImplementedError. [clinic start generated code]*/ static PyObject * os_replace_impl(PyObject *module, path_t *src, path_t *dst, int src_dir_fd, int dst_dir_fd) -/*[clinic end generated code: output=1968c02e7857422b input=25515dfb107c8421]*/ +/*[clinic end generated code: output=1968c02e7857422b input=c003f0def43378ef]*/ { return internal_rename(src, dst, src_dir_fd, dst_dir_fd, 1); } @@ -4181,8 +4194,8 @@ Execute the command in a subshell. [clinic start generated code]*/ static long -os_system_impl(PyObject *module, Py_UNICODE *command) -/*[clinic end generated code: output=96c4dffee36dfb48 input=303f5ce97df606b0]*/ +os_system_impl(PyObject *module, const Py_UNICODE *command) +/*[clinic end generated code: output=5b7c3599c068ca42 input=303f5ce97df606b0]*/ { long result; Py_BEGIN_ALLOW_THREADS @@ -5689,6 +5702,9 @@ os_sched_rr_get_interval_impl(PyObject *module, pid_t pid) posix_error(); return -1.0; } +#ifdef _Py_MEMORY_SANITIZER + __msan_unpoison(&interval, sizeof(interval)); +#endif return (double)interval.tv_sec + 1e-9*interval.tv_nsec; } #endif /* HAVE_SCHED_RR_GET_INTERVAL */ @@ -5926,7 +5942,7 @@ os_openpty_impl(PyObject *module) #endif #if defined(HAVE_DEV_PTMX) && !defined(HAVE_OPENPTY) && !defined(HAVE__GETPTY) PyOS_sighandler_t sig_saved; -#ifdef sun +#if defined(__sun) && defined(__SVR4) extern char *ptsname(int fildes); #endif #endif @@ -6155,6 +6171,12 @@ posix_getgrouplist(PyObject *self, PyObject *args) return posix_error(); } +#ifdef _Py_MEMORY_SANITIZER + /* Clang memory sanitizer libc intercepts don't know getgrouplist. */ + __msan_unpoison(&ngroups, sizeof(ngroups)); + __msan_unpoison(groups, ngroups*sizeof(*groups)); +#endif + list = PyList_New(ngroups); if (list == NULL) { PyMem_Del(groups); @@ -10792,8 +10814,9 @@ the underlying Win32 ShellExecute function doesn't work if it is. [clinic start generated code]*/ static PyObject * -os_startfile_impl(PyObject *module, path_t *filepath, Py_UNICODE *operation) -/*[clinic end generated code: output=912ceba79acfa1c9 input=63950bf2986380d0]*/ +os_startfile_impl(PyObject *module, path_t *filepath, + const Py_UNICODE *operation) +/*[clinic end generated code: output=66dc311c94d50797 input=63950bf2986380d0]*/ { HINSTANCE rc; diff --git a/kbe/src/lib/python/Modules/socketmodule.c b/kbe/src/lib/python/Modules/socketmodule.c index e6d3f8be6e..988471e15f 100644 --- a/kbe/src/lib/python/Modules/socketmodule.c +++ b/kbe/src/lib/python/Modules/socketmodule.c @@ -100,6 +100,10 @@ Local naming conventions: #include "Python.h" #include "structmember.h" +#ifdef _Py_MEMORY_SANITIZER +# include +#endif + /* Socket object documentation */ PyDoc_STRVAR(sock_doc, "socket(family=AF_INET, type=SOCK_STREAM, proto=0) -> socket object\n\ @@ -261,7 +265,7 @@ if_indextoname(index) -- return the corresponding interface name\n\ #endif /* Solaris fails to define this variable at all. */ -#if defined(sun) && !defined(INET_ADDRSTRLEN) +#if (defined(__sun) && defined(__SVR4)) && !defined(INET_ADDRSTRLEN) #define INET_ADDRSTRLEN 16 #endif @@ -6463,7 +6467,23 @@ socket_if_nameindex(PyObject *self, PyObject *arg) return NULL; } +#ifdef _Py_MEMORY_SANITIZER + __msan_unpoison(ni, sizeof(ni)); + __msan_unpoison(&ni[0], sizeof(ni[0])); +#endif for (i = 0; ni[i].if_index != 0 && i < INT_MAX; i++) { +#ifdef _Py_MEMORY_SANITIZER + /* This one isn't the end sentinel, the next one must exist. */ + __msan_unpoison(&ni[i+1], sizeof(ni[0])); + /* Otherwise Py_BuildValue internals are flagged by MSan when + they access the not-msan-tracked if_name string data. */ + { + char *to_sanitize = ni[i].if_name; + do { + __msan_unpoison(to_sanitize, 1); + } while (*to_sanitize++ != '\0'); + } +#endif PyObject *ni_tuple = Py_BuildValue("IO&", ni[i].if_index, PyUnicode_DecodeFSDefault, ni[i].if_name); diff --git a/kbe/src/lib/python/Modules/sre_lib.h b/kbe/src/lib/python/Modules/sre_lib.h index 44948e21ad..437ab43f43 100644 --- a/kbe/src/lib/python/Modules/sre_lib.h +++ b/kbe/src/lib/python/Modules/sre_lib.h @@ -1363,6 +1363,10 @@ SRE(match)(SRE_STATE* state, SRE_CODE* pattern, int toplevel) return ret; /* should never get here */ } +/* need to reset capturing groups between two SRE(match) callings in loops */ +#define RESET_CAPTURE_GROUP() \ + do { state->lastmark = state->lastindex = -1; } while (0) + LOCAL(Py_ssize_t) SRE(search)(SRE_STATE* state, SRE_CODE* pattern) { @@ -1440,6 +1444,7 @@ SRE(search)(SRE_STATE* state, SRE_CODE* pattern) if (status != 0) return status; ++ptr; + RESET_CAPTURE_GROUP(); } return 0; } @@ -1487,6 +1492,7 @@ SRE(search)(SRE_STATE* state, SRE_CODE* pattern) /* close but no cigar -- try again */ if (++ptr >= end) return 0; + RESET_CAPTURE_GROUP(); } i = overlap[i]; } while (i != 0); @@ -1510,6 +1516,7 @@ SRE(search)(SRE_STATE* state, SRE_CODE* pattern) if (status != 0) break; ptr++; + RESET_CAPTURE_GROUP(); } } else { /* general case */ @@ -1520,6 +1527,7 @@ SRE(search)(SRE_STATE* state, SRE_CODE* pattern) state->must_advance = 0; while (status == 0 && ptr < end) { ptr++; + RESET_CAPTURE_GROUP(); TRACE(("|%p|%p|SEARCH\n", pattern, ptr)); state->start = state->ptr = ptr; status = SRE(match)(state, pattern, 0); diff --git a/kbe/src/lib/python/Modules/timemodule.c b/kbe/src/lib/python/Modules/timemodule.c index 13a174a4ea..ae7de5b2c7 100644 --- a/kbe/src/lib/python/Modules/timemodule.c +++ b/kbe/src/lib/python/Modules/timemodule.c @@ -34,6 +34,20 @@ #endif /* MS_WINDOWS */ #endif /* !__WATCOMC__ || __QNX__ */ +#ifdef _Py_MEMORY_SANITIZER +# include +#endif + +#ifdef _MSC_VER +#define _Py_timezone _timezone +#define _Py_daylight _daylight +#define _Py_tzname _tzname +#else +#define _Py_timezone timezone +#define _Py_daylight daylight +#define _Py_tzname tzname +#endif + #define SEC_TO_NS (1000 * 1000 * 1000) /* Forward declarations */ @@ -331,6 +345,9 @@ time_pthread_getcpuclockid(PyObject *self, PyObject *args) PyErr_SetFromErrno(PyExc_OSError); return NULL; } +#ifdef _Py_MEMORY_SANITIZER + __msan_unpoison(&clk_id, sizeof(clk_id)); +#endif return PyLong_FromLong(clk_id); } @@ -718,7 +735,7 @@ time_strftime(PyObject *self, PyObject *args) return NULL; } -#if defined(_MSC_VER) || defined(sun) || defined(_AIX) +#if defined(_MSC_VER) || (defined(__sun) && defined(__SVR4)) || defined(_AIX) if (buf.tm_year + 1900 < 1 || 9999 < buf.tm_year + 1900) { PyErr_SetString(PyExc_ValueError, "strftime() requires year in [1; 9999]"); @@ -764,7 +781,7 @@ time_strftime(PyObject *self, PyObject *args) return NULL; } } -#elif (defined(_AIX) || defined(sun)) && defined(HAVE_WCSFTIME) +#elif (defined(_AIX) || (defined(__sun) && defined(__SVR4))) && defined(HAVE_WCSFTIME) for (outbuf = wcschr(fmt, '%'); outbuf != NULL; outbuf = wcschr(outbuf+2, '%')) @@ -1547,18 +1564,18 @@ init_timezone(PyObject *m) #if defined(HAVE_TZNAME) && !defined(__GLIBC__) && !defined(__CYGWIN__) PyObject *otz0, *otz1; tzset(); - PyModule_AddIntConstant(m, "timezone", timezone); + PyModule_AddIntConstant(m, "timezone", _Py_timezone); #ifdef HAVE_ALTZONE PyModule_AddIntConstant(m, "altzone", altzone); #else - PyModule_AddIntConstant(m, "altzone", timezone-3600); + PyModule_AddIntConstant(m, "altzone", _Py_timezone-3600); #endif - PyModule_AddIntConstant(m, "daylight", daylight); - otz0 = PyUnicode_DecodeLocale(tzname[0], "surrogateescape"); + PyModule_AddIntConstant(m, "daylight", _Py_daylight); + otz0 = PyUnicode_DecodeLocale(_Py_tzname[0], "surrogateescape"); if (otz0 == NULL) { return -1; } - otz1 = PyUnicode_DecodeLocale(tzname[1], "surrogateescape"); + otz1 = PyUnicode_DecodeLocale(_Py_tzname[1], "surrogateescape"); if (otz1 == NULL) { Py_DECREF(otz0); return -1; diff --git a/kbe/src/lib/python/Objects/abstract.c b/kbe/src/lib/python/Objects/abstract.c index 2d48a112aa..9416df4321 100644 --- a/kbe/src/lib/python/Objects/abstract.c +++ b/kbe/src/lib/python/Objects/abstract.c @@ -143,6 +143,7 @@ PyObject * PyObject_GetItem(PyObject *o, PyObject *key) { PyMappingMethods *m; + PySequenceMethods *ms; if (o == NULL || key == NULL) { return null_error(); @@ -155,7 +156,8 @@ PyObject_GetItem(PyObject *o, PyObject *key) return item; } - if (o->ob_type->tp_as_sequence) { + ms = o->ob_type->tp_as_sequence; + if (ms && ms->sq_item) { if (PyIndex_Check(key)) { Py_ssize_t key_value; key_value = PyNumber_AsSsize_t(key, PyExc_IndexError); @@ -163,9 +165,10 @@ PyObject_GetItem(PyObject *o, PyObject *key) return NULL; return PySequence_GetItem(o, key_value); } - else if (o->ob_type->tp_as_sequence->sq_item) + else { return type_error("sequence index must " "be integer, not '%.200s'", key); + } } if (PyType_Check(o)) { diff --git a/kbe/src/lib/python/Objects/bytesobject.c b/kbe/src/lib/python/Objects/bytesobject.c index 711faba645..172c7f38b9 100644 --- a/kbe/src/lib/python/Objects/bytesobject.c +++ b/kbe/src/lib/python/Objects/bytesobject.c @@ -311,9 +311,15 @@ PyBytes_FromFormatV(const char *format, va_list vargs) Py_ssize_t i; p = va_arg(vargs, const char*); - i = strlen(p); - if (prec > 0 && i > prec) - i = prec; + if (prec <= 0) { + i = strlen(p); + } + else { + i = 0; + while (i < prec && p[i]) { + i++; + } + } s = _PyBytesWriter_WriteBytes(&writer, s, p, i); if (s == NULL) goto error; @@ -2990,9 +2996,22 @@ _PyBytes_Resize(PyObject **pv, Py_ssize_t newsize) /* return early if newsize equals to v->ob_size */ return 0; } + if (Py_SIZE(v) == 0) { + if (newsize == 0) { + return 0; + } + *pv = _PyBytes_FromSize(newsize, 0); + Py_DECREF(v); + return (*pv == NULL) ? -1 : 0; + } if (Py_REFCNT(v) != 1) { goto error; } + if (newsize == 0) { + *pv = _PyBytes_FromSize(0, 0); + Py_DECREF(v); + return (*pv == NULL) ? -1 : 0; + } /* XXX UNREF/NEWREF interface should be more symmetrical */ _Py_DEC_REFTOTAL; _Py_ForgetReference(v); diff --git a/kbe/src/lib/python/Objects/fileobject.c b/kbe/src/lib/python/Objects/fileobject.c index ed4e12ba8a..d886e96e0f 100644 --- a/kbe/src/lib/python/Objects/fileobject.c +++ b/kbe/src/lib/python/Objects/fileobject.c @@ -407,7 +407,7 @@ stdprinter_fileno(PyStdPrinter_Object *self) static PyObject * stdprinter_repr(PyStdPrinter_Object *self) { - return PyUnicode_FromFormat("", + return PyUnicode_FromFormat("", self->fd, self); } diff --git a/kbe/src/lib/python/Objects/frameobject.c b/kbe/src/lib/python/Objects/frameobject.c index 4ef10d0eb7..4362615cb0 100644 --- a/kbe/src/lib/python/Objects/frameobject.c +++ b/kbe/src/lib/python/Objects/frameobject.c @@ -90,6 +90,10 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore int blockstack_top = 0; /* (ditto) */ unsigned char setup_op = 0; /* (ditto) */ + if (p_new_lineno == NULL) { + PyErr_SetString(PyExc_AttributeError, "cannot delete attribute"); + return -1; + } /* f_lineno must be an integer. */ if (!PyLong_CheckExact(p_new_lineno)) { PyErr_SetString(PyExc_ValueError, diff --git a/kbe/src/lib/python/Objects/listobject.c b/kbe/src/lib/python/Objects/listobject.c index c8ffeff093..7dc68a73bd 100644 --- a/kbe/src/lib/python/Objects/listobject.c +++ b/kbe/src/lib/python/Objects/listobject.c @@ -2232,7 +2232,6 @@ list_sort_impl(PyListObject *self, PyObject *keyfunc, int reverse) int ints_are_bounded = 1; /* Prove that assumption by checking every key. */ - int i; for (i=0; i < saved_ob_size; i++) { if (keys_are_in_tuples && @@ -2280,6 +2279,9 @@ list_sort_impl(PyListObject *self, PyObject *keyfunc, int reverse) else if ((ms.key_richcompare = key_type->tp_richcompare) != NULL) { ms.key_compare = unsafe_object_compare; } + else { + ms.key_compare = safe_object_compare; + } } else { ms.key_compare = safe_object_compare; diff --git a/kbe/src/lib/python/Objects/sliceobject.c b/kbe/src/lib/python/Objects/sliceobject.c index 59f084d1a6..4d5212f6f0 100644 --- a/kbe/src/lib/python/Objects/sliceobject.c +++ b/kbe/src/lib/python/Objects/sliceobject.c @@ -564,14 +564,11 @@ static PyMethodDef slice_methods[] = { static PyObject * slice_richcompare(PyObject *v, PyObject *w, int op) { - PyObject *t1; - PyObject *t2; - PyObject *res; - if (!PySlice_Check(v) || !PySlice_Check(w)) Py_RETURN_NOTIMPLEMENTED; if (v == w) { + PyObject *res; /* XXX Do we really need this shortcut? There's a unit test for it, but is that fair? */ switch (op) { @@ -588,34 +585,27 @@ slice_richcompare(PyObject *v, PyObject *w, int op) return res; } - t1 = PyTuple_New(3); - if (t1 == NULL) + + PyObject *t1 = PyTuple_Pack(3, + ((PySliceObject *)v)->start, + ((PySliceObject *)v)->stop, + ((PySliceObject *)v)->step); + if (t1 == NULL) { return NULL; - t2 = PyTuple_New(3); + } + + PyObject *t2 = PyTuple_Pack(3, + ((PySliceObject *)w)->start, + ((PySliceObject *)w)->stop, + ((PySliceObject *)w)->step); if (t2 == NULL) { Py_DECREF(t1); return NULL; } - PyTuple_SET_ITEM(t1, 0, ((PySliceObject *)v)->start); - PyTuple_SET_ITEM(t1, 1, ((PySliceObject *)v)->stop); - PyTuple_SET_ITEM(t1, 2, ((PySliceObject *)v)->step); - PyTuple_SET_ITEM(t2, 0, ((PySliceObject *)w)->start); - PyTuple_SET_ITEM(t2, 1, ((PySliceObject *)w)->stop); - PyTuple_SET_ITEM(t2, 2, ((PySliceObject *)w)->step); - - res = PyObject_RichCompare(t1, t2, op); - - PyTuple_SET_ITEM(t1, 0, NULL); - PyTuple_SET_ITEM(t1, 1, NULL); - PyTuple_SET_ITEM(t1, 2, NULL); - PyTuple_SET_ITEM(t2, 0, NULL); - PyTuple_SET_ITEM(t2, 1, NULL); - PyTuple_SET_ITEM(t2, 2, NULL); - + PyObject *res = PyObject_RichCompare(t1, t2, op); Py_DECREF(t1); Py_DECREF(t2); - return res; } diff --git a/kbe/src/lib/python/Objects/typeobject.c b/kbe/src/lib/python/Objects/typeobject.c index 4682105741..c578d6e935 100644 --- a/kbe/src/lib/python/Objects/typeobject.c +++ b/kbe/src/lib/python/Objects/typeobject.c @@ -3654,11 +3654,13 @@ object_init(PyObject *self, PyObject *args, PyObject *kwds) PyTypeObject *type = Py_TYPE(self); if (excess_args(args, kwds)) { if (type->tp_init != object_init) { - PyErr_SetString(PyExc_TypeError, "object.__init__() takes no arguments"); + PyErr_SetString(PyExc_TypeError, + "object.__init__() takes exactly one argument (the instance to initialize)"); return -1; } if (type->tp_new == object_new) { - PyErr_Format(PyExc_TypeError, "%.200s().__init__() takes no arguments", + PyErr_Format(PyExc_TypeError, + "%.200s.__init__() takes exactly one argument (the instance to initialize)", type->tp_name); return -1; } @@ -3671,7 +3673,8 @@ object_new(PyTypeObject *type, PyObject *args, PyObject *kwds) { if (excess_args(args, kwds)) { if (type->tp_new != object_new) { - PyErr_SetString(PyExc_TypeError, "object.__new__() takes no arguments"); + PyErr_SetString(PyExc_TypeError, + "object.__new__() takes exactly one argument (the type to instantiate)"); return NULL; } if (type->tp_init == object_init) { diff --git a/kbe/src/lib/python/Objects/unicodeobject.c b/kbe/src/lib/python/Objects/unicodeobject.c index d46ab2a1e2..b67ffac4e9 100644 --- a/kbe/src/lib/python/Objects/unicodeobject.c +++ b/kbe/src/lib/python/Objects/unicodeobject.c @@ -2579,9 +2579,15 @@ unicode_fromformat_write_cstr(_PyUnicodeWriter *writer, const char *str, PyObject *unicode; int res; - length = strlen(str); - if (precision != -1) - length = Py_MIN(length, precision); + if (precision == -1) { + length = strlen(str); + } + else { + length = 0; + while (length < precision && str[length]) { + length++; + } + } unicode = PyUnicode_DecodeUTF8Stateful(str, length, "replace", NULL); if (unicode == NULL) return -1; @@ -9356,6 +9362,7 @@ _PyUnicode_InsertThousandsGrouping( PyObject *thousands_sep, Py_UCS4 *maxchar) { + min_width = Py_MAX(0, min_width); if (writer) { assert(digits != NULL); assert(maxchar == NULL); @@ -9366,7 +9373,6 @@ _PyUnicode_InsertThousandsGrouping( } assert(0 <= d_pos); assert(0 <= n_digits); - assert(0 <= min_width); assert(grouping != NULL); if (digits != NULL) { diff --git a/kbe/src/lib/python/PC/_msi.c b/kbe/src/lib/python/PC/_msi.c index 024b2d3c9f..ae30acbc9b 100644 --- a/kbe/src/lib/python/PC/_msi.c +++ b/kbe/src/lib/python/PC/_msi.c @@ -555,7 +555,7 @@ summary_getproperty(msiobj* si, PyObject *args) FILETIME fval; char sbuf[1000]; char *sval = sbuf; - DWORD ssize = sizeof(sval); + DWORD ssize = sizeof(sbuf); if (!PyArg_ParseTuple(args, "i:GetProperty", &field)) return NULL; @@ -563,6 +563,7 @@ summary_getproperty(msiobj* si, PyObject *args) status = MsiSummaryInfoGetProperty(si->h, field, &type, &ival, &fval, sval, &ssize); if (status == ERROR_MORE_DATA) { + ssize++; sval = malloc(ssize); if (sval == NULL) { return PyErr_NoMemory(); @@ -572,21 +573,29 @@ summary_getproperty(msiobj* si, PyObject *args) } switch(type) { - case VT_I2: case VT_I4: - return PyLong_FromLong(ival); + case VT_I2: + case VT_I4: + result = PyLong_FromLong(ival); + break; case VT_FILETIME: PyErr_SetString(PyExc_NotImplementedError, "FILETIME result"); - return NULL; + result = NULL; + break; case VT_LPSTR: result = PyBytes_FromStringAndSize(sval, ssize); - if (sval != sbuf) - free(sval); - return result; + break; case VT_EMPTY: - Py_RETURN_NONE; + Py_INCREF(Py_None); + result = Py_None; + break; + default: + PyErr_Format(PyExc_NotImplementedError, "result of type %d", type); + result = NULL; + break; } - PyErr_Format(PyExc_NotImplementedError, "result of type %d", type); - return NULL; + if (sval != sbuf) + free(sval); + return result; } static PyObject* @@ -907,7 +916,7 @@ msidb_getsummaryinformation(msiobj *db, PyObject *args) return msierror(status); oresult = PyObject_NEW(struct msiobj, &summary_Type); - if (!result) { + if (!oresult) { MsiCloseHandle(result); return NULL; } diff --git a/kbe/src/lib/python/PC/clinic/winreg.c.h b/kbe/src/lib/python/PC/clinic/winreg.c.h index 69781a9692..6a5f613808 100644 --- a/kbe/src/lib/python/PC/clinic/winreg.c.h +++ b/kbe/src/lib/python/PC/clinic/winreg.c.h @@ -137,14 +137,14 @@ PyDoc_STRVAR(winreg_ConnectRegistry__doc__, {"ConnectRegistry", (PyCFunction)winreg_ConnectRegistry, METH_FASTCALL, winreg_ConnectRegistry__doc__}, static HKEY -winreg_ConnectRegistry_impl(PyObject *module, Py_UNICODE *computer_name, - HKEY key); +winreg_ConnectRegistry_impl(PyObject *module, + const Py_UNICODE *computer_name, HKEY key); static PyObject * winreg_ConnectRegistry(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - Py_UNICODE *computer_name; + const Py_UNICODE *computer_name; HKEY key; HKEY _return_value; @@ -185,14 +185,14 @@ PyDoc_STRVAR(winreg_CreateKey__doc__, {"CreateKey", (PyCFunction)winreg_CreateKey, METH_FASTCALL, winreg_CreateKey__doc__}, static HKEY -winreg_CreateKey_impl(PyObject *module, HKEY key, Py_UNICODE *sub_key); +winreg_CreateKey_impl(PyObject *module, HKEY key, const Py_UNICODE *sub_key); static PyObject * winreg_CreateKey(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; HKEY key; - Py_UNICODE *sub_key; + const Py_UNICODE *sub_key; HKEY _return_value; if (!_PyArg_ParseStack(args, nargs, "O&Z:CreateKey", @@ -238,8 +238,9 @@ PyDoc_STRVAR(winreg_CreateKeyEx__doc__, {"CreateKeyEx", (PyCFunction)winreg_CreateKeyEx, METH_FASTCALL|METH_KEYWORDS, winreg_CreateKeyEx__doc__}, static HKEY -winreg_CreateKeyEx_impl(PyObject *module, HKEY key, Py_UNICODE *sub_key, - int reserved, REGSAM access); +winreg_CreateKeyEx_impl(PyObject *module, HKEY key, + const Py_UNICODE *sub_key, int reserved, + REGSAM access); static PyObject * winreg_CreateKeyEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -248,7 +249,7 @@ winreg_CreateKeyEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py static const char * const _keywords[] = {"key", "sub_key", "reserved", "access", NULL}; static _PyArg_Parser _parser = {"O&Z|ii:CreateKeyEx", _keywords, 0}; HKEY key; - Py_UNICODE *sub_key; + const Py_UNICODE *sub_key; int reserved = 0; REGSAM access = KEY_WRITE; HKEY _return_value; @@ -289,14 +290,14 @@ PyDoc_STRVAR(winreg_DeleteKey__doc__, {"DeleteKey", (PyCFunction)winreg_DeleteKey, METH_FASTCALL, winreg_DeleteKey__doc__}, static PyObject * -winreg_DeleteKey_impl(PyObject *module, HKEY key, Py_UNICODE *sub_key); +winreg_DeleteKey_impl(PyObject *module, HKEY key, const Py_UNICODE *sub_key); static PyObject * winreg_DeleteKey(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; HKEY key; - Py_UNICODE *sub_key; + const Py_UNICODE *sub_key; if (!_PyArg_ParseStack(args, nargs, "O&u:DeleteKey", clinic_HKEY_converter, &key, &sub_key)) { @@ -337,8 +338,9 @@ PyDoc_STRVAR(winreg_DeleteKeyEx__doc__, {"DeleteKeyEx", (PyCFunction)winreg_DeleteKeyEx, METH_FASTCALL|METH_KEYWORDS, winreg_DeleteKeyEx__doc__}, static PyObject * -winreg_DeleteKeyEx_impl(PyObject *module, HKEY key, Py_UNICODE *sub_key, - REGSAM access, int reserved); +winreg_DeleteKeyEx_impl(PyObject *module, HKEY key, + const Py_UNICODE *sub_key, REGSAM access, + int reserved); static PyObject * winreg_DeleteKeyEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -347,7 +349,7 @@ winreg_DeleteKeyEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, Py static const char * const _keywords[] = {"key", "sub_key", "access", "reserved", NULL}; static _PyArg_Parser _parser = {"O&u|ii:DeleteKeyEx", _keywords, 0}; HKEY key; - Py_UNICODE *sub_key; + const Py_UNICODE *sub_key; REGSAM access = KEY_WOW64_64KEY; int reserved = 0; @@ -376,14 +378,14 @@ PyDoc_STRVAR(winreg_DeleteValue__doc__, {"DeleteValue", (PyCFunction)winreg_DeleteValue, METH_FASTCALL, winreg_DeleteValue__doc__}, static PyObject * -winreg_DeleteValue_impl(PyObject *module, HKEY key, Py_UNICODE *value); +winreg_DeleteValue_impl(PyObject *module, HKEY key, const Py_UNICODE *value); static PyObject * winreg_DeleteValue(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; HKEY key; - Py_UNICODE *value; + const Py_UNICODE *value; if (!_PyArg_ParseStack(args, nargs, "O&Z:DeleteValue", clinic_HKEY_converter, &key, &value)) { @@ -490,13 +492,14 @@ PyDoc_STRVAR(winreg_ExpandEnvironmentStrings__doc__, {"ExpandEnvironmentStrings", (PyCFunction)winreg_ExpandEnvironmentStrings, METH_O, winreg_ExpandEnvironmentStrings__doc__}, static PyObject * -winreg_ExpandEnvironmentStrings_impl(PyObject *module, Py_UNICODE *string); +winreg_ExpandEnvironmentStrings_impl(PyObject *module, + const Py_UNICODE *string); static PyObject * winreg_ExpandEnvironmentStrings(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; - Py_UNICODE *string; + const Py_UNICODE *string; if (!PyArg_Parse(arg, "u:ExpandEnvironmentStrings", &string)) { goto exit; @@ -579,16 +582,16 @@ PyDoc_STRVAR(winreg_LoadKey__doc__, {"LoadKey", (PyCFunction)winreg_LoadKey, METH_FASTCALL, winreg_LoadKey__doc__}, static PyObject * -winreg_LoadKey_impl(PyObject *module, HKEY key, Py_UNICODE *sub_key, - Py_UNICODE *file_name); +winreg_LoadKey_impl(PyObject *module, HKEY key, const Py_UNICODE *sub_key, + const Py_UNICODE *file_name); static PyObject * winreg_LoadKey(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; HKEY key; - Py_UNICODE *sub_key; - Py_UNICODE *file_name; + const Py_UNICODE *sub_key; + const Py_UNICODE *file_name; if (!_PyArg_ParseStack(args, nargs, "O&uu:LoadKey", clinic_HKEY_converter, &key, &sub_key, &file_name)) { @@ -623,7 +626,7 @@ PyDoc_STRVAR(winreg_OpenKey__doc__, {"OpenKey", (PyCFunction)winreg_OpenKey, METH_FASTCALL|METH_KEYWORDS, winreg_OpenKey__doc__}, static HKEY -winreg_OpenKey_impl(PyObject *module, HKEY key, Py_UNICODE *sub_key, +winreg_OpenKey_impl(PyObject *module, HKEY key, const Py_UNICODE *sub_key, int reserved, REGSAM access); static PyObject * @@ -633,7 +636,7 @@ winreg_OpenKey(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObje static const char * const _keywords[] = {"key", "sub_key", "reserved", "access", NULL}; static _PyArg_Parser _parser = {"O&Z|ii:OpenKey", _keywords, 0}; HKEY key; - Py_UNICODE *sub_key; + const Py_UNICODE *sub_key; int reserved = 0; REGSAM access = KEY_READ; HKEY _return_value; @@ -675,7 +678,7 @@ PyDoc_STRVAR(winreg_OpenKeyEx__doc__, {"OpenKeyEx", (PyCFunction)winreg_OpenKeyEx, METH_FASTCALL|METH_KEYWORDS, winreg_OpenKeyEx__doc__}, static HKEY -winreg_OpenKeyEx_impl(PyObject *module, HKEY key, Py_UNICODE *sub_key, +winreg_OpenKeyEx_impl(PyObject *module, HKEY key, const Py_UNICODE *sub_key, int reserved, REGSAM access); static PyObject * @@ -685,7 +688,7 @@ winreg_OpenKeyEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb static const char * const _keywords[] = {"key", "sub_key", "reserved", "access", NULL}; static _PyArg_Parser _parser = {"O&Z|ii:OpenKeyEx", _keywords, 0}; HKEY key; - Py_UNICODE *sub_key; + const Py_UNICODE *sub_key; int reserved = 0; REGSAM access = KEY_READ; HKEY _return_value; @@ -764,14 +767,14 @@ PyDoc_STRVAR(winreg_QueryValue__doc__, {"QueryValue", (PyCFunction)winreg_QueryValue, METH_FASTCALL, winreg_QueryValue__doc__}, static PyObject * -winreg_QueryValue_impl(PyObject *module, HKEY key, Py_UNICODE *sub_key); +winreg_QueryValue_impl(PyObject *module, HKEY key, const Py_UNICODE *sub_key); static PyObject * winreg_QueryValue(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; HKEY key; - Py_UNICODE *sub_key; + const Py_UNICODE *sub_key; if (!_PyArg_ParseStack(args, nargs, "O&Z:QueryValue", clinic_HKEY_converter, &key, &sub_key)) { @@ -803,14 +806,14 @@ PyDoc_STRVAR(winreg_QueryValueEx__doc__, {"QueryValueEx", (PyCFunction)winreg_QueryValueEx, METH_FASTCALL, winreg_QueryValueEx__doc__}, static PyObject * -winreg_QueryValueEx_impl(PyObject *module, HKEY key, Py_UNICODE *name); +winreg_QueryValueEx_impl(PyObject *module, HKEY key, const Py_UNICODE *name); static PyObject * winreg_QueryValueEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; HKEY key; - Py_UNICODE *name; + const Py_UNICODE *name; if (!_PyArg_ParseStack(args, nargs, "O&Z:QueryValueEx", clinic_HKEY_converter, &key, &name)) { @@ -847,14 +850,14 @@ PyDoc_STRVAR(winreg_SaveKey__doc__, {"SaveKey", (PyCFunction)winreg_SaveKey, METH_FASTCALL, winreg_SaveKey__doc__}, static PyObject * -winreg_SaveKey_impl(PyObject *module, HKEY key, Py_UNICODE *file_name); +winreg_SaveKey_impl(PyObject *module, HKEY key, const Py_UNICODE *file_name); static PyObject * winreg_SaveKey(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; HKEY key; - Py_UNICODE *file_name; + const Py_UNICODE *file_name; if (!_PyArg_ParseStack(args, nargs, "O&u:SaveKey", clinic_HKEY_converter, &key, &file_name)) { @@ -896,8 +899,8 @@ PyDoc_STRVAR(winreg_SetValue__doc__, {"SetValue", (PyCFunction)winreg_SetValue, METH_FASTCALL, winreg_SetValue__doc__}, static PyObject * -winreg_SetValue_impl(PyObject *module, HKEY key, Py_UNICODE *sub_key, - DWORD type, Py_UNICODE *value, +winreg_SetValue_impl(PyObject *module, HKEY key, const Py_UNICODE *sub_key, + DWORD type, const Py_UNICODE *value, Py_ssize_clean_t value_length); static PyObject * @@ -905,9 +908,9 @@ winreg_SetValue(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; HKEY key; - Py_UNICODE *sub_key; + const Py_UNICODE *sub_key; DWORD type; - Py_UNICODE *value; + const Py_UNICODE *value; Py_ssize_clean_t value_length; if (!_PyArg_ParseStack(args, nargs, "O&Zku#:SetValue", @@ -967,15 +970,16 @@ PyDoc_STRVAR(winreg_SetValueEx__doc__, {"SetValueEx", (PyCFunction)winreg_SetValueEx, METH_FASTCALL, winreg_SetValueEx__doc__}, static PyObject * -winreg_SetValueEx_impl(PyObject *module, HKEY key, Py_UNICODE *value_name, - PyObject *reserved, DWORD type, PyObject *value); +winreg_SetValueEx_impl(PyObject *module, HKEY key, + const Py_UNICODE *value_name, PyObject *reserved, + DWORD type, PyObject *value); static PyObject * winreg_SetValueEx(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; HKEY key; - Py_UNICODE *value_name; + const Py_UNICODE *value_name; PyObject *reserved; DWORD type; PyObject *value; @@ -1091,4 +1095,4 @@ winreg_QueryReflectionKey(PyObject *module, PyObject *arg) exit: return return_value; } -/*[clinic end generated code: output=d1c8e2678015dd7d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=60c92ffc7438f8cf input=a9049054013a1b77]*/ diff --git a/kbe/src/lib/python/PC/config.c b/kbe/src/lib/python/PC/config.c index 2037b3db64..c4922ccd62 100644 --- a/kbe/src/lib/python/PC/config.c +++ b/kbe/src/lib/python/PC/config.c @@ -72,6 +72,8 @@ extern PyObject* PyInit__string(void); extern PyObject* PyInit__stat(void); extern PyObject* PyInit__opcode(void); +extern PyObject* PyInit__contextvars(void); + /* tools/freeze/makeconfig.py marker for additional "extern" */ /* -- ADDMODULE MARKER 1 -- */ @@ -164,6 +166,8 @@ struct _inittab _PyImport_Inittab[] = { {"_stat", PyInit__stat}, {"_opcode", PyInit__opcode}, + {"_contextvars", PyInit__contextvars}, + /* Sentinel */ {0, 0} }; diff --git a/kbe/src/lib/python/PC/launcher.c b/kbe/src/lib/python/PC/launcher.c index 4c620dab7c..a4e678115f 100644 --- a/kbe/src/lib/python/PC/launcher.c +++ b/kbe/src/lib/python/PC/launcher.c @@ -666,7 +666,7 @@ run_child(wchar_t * cmdline) if (!ok) error(RC_CREATE_PROCESS, L"Job information setting failed"); memset(&si, 0, sizeof(si)); - si.cb = sizeof(si); + GetStartupInfoW(&si); ok = safe_duplicate_handle(GetStdHandle(STD_INPUT_HANDLE), &si.hStdInput); if (!ok) error(RC_NO_STD_HANDLES, L"stdin duplication failed"); @@ -1707,6 +1707,17 @@ process(int argc, wchar_t ** argv) command = skip_me(GetCommandLineW()); debug(L"Called with command line: %ls\n", command); +#if !defined(VENV_REDIRECT) + /* bpo-35811: The __PYVENV_LAUNCHER__ variable is used to + * override sys.executable and locate the original prefix path. + * However, if it is silently inherited by a non-venv Python + * process, that process will believe it is running in the venv + * still. This is the only place where *we* can clear it (that is, + * when py.exe is being used to launch Python), so we do. + */ + SetEnvironmentVariableW(L"__PYVENV_LAUNCHER__", NULL); +#endif + #if defined(SCRIPT_WRAPPER) /* The launcher is being used in "script wrapper" mode. * There should therefore be a Python script named -script.py in diff --git a/kbe/src/lib/python/PC/layout/main.py b/kbe/src/lib/python/PC/layout/main.py index 217b2b096e..910085c01b 100644 --- a/kbe/src/lib/python/PC/layout/main.py +++ b/kbe/src/lib/python/PC/layout/main.py @@ -46,7 +46,7 @@ VENV_DIRS_ONLY = FileNameSet("venv", "ensurepip") -EXCLUDE_FROM_PYDS = FileStemSet("python*", "pyshellext") +EXCLUDE_FROM_PYDS = FileStemSet("python*", "pyshellext", "vcruntime*") EXCLUDE_FROM_LIB = FileNameSet("*.pyc", "__pycache__", "*.pickle") EXCLUDE_FROM_PACKAGED_LIB = FileNameSet("readme.txt") EXCLUDE_FROM_COMPILE = FileNameSet("badsyntax_*", "bad_*") @@ -156,6 +156,8 @@ def in_build(f, dest="", new_name=None): for dest, src in rglob(ns.build, "vcruntime*.dll"): yield dest, src + yield "LICENSE.txt", ns.source / "LICENSE" + for dest, src in rglob(ns.build, ("*.pyd", "*.dll")): if src.stem.endswith("_d") != bool(ns.debug) and src not in REQUIRED_DLLS: continue @@ -240,12 +242,18 @@ def _c(d): yield "DLLs/{}".format(ns.include_cat.name), ns.include_cat -def _compile_one_py(src, dest, name, optimize): +def _compile_one_py(src, dest, name, optimize, checked=True): import py_compile if dest is not None: dest = str(dest) + mode = ( + py_compile.PycInvalidationMode.CHECKED_HASH + if checked + else py_compile.PycInvalidationMode.UNCHECKED_HASH + ) + try: return Path( py_compile.compile( @@ -254,7 +262,7 @@ def _compile_one_py(src, dest, name, optimize): str(name), doraise=True, optimize=optimize, - invalidation_mode=py_compile.PycInvalidationMode.CHECKED_HASH, + invalidation_mode=mode, ) ) except py_compile.PyCompileError: @@ -262,16 +270,16 @@ def _compile_one_py(src, dest, name, optimize): return None -def _py_temp_compile(src, ns, dest_dir=None): +def _py_temp_compile(src, ns, dest_dir=None, checked=True): if not ns.precompile or src not in PY_FILES or src.parent in DATA_DIRS: return None dest = (dest_dir or ns.temp) / (src.stem + ".py") - return _compile_one_py(src, dest.with_suffix(".pyc"), dest, optimize=2) + return _compile_one_py(src, dest.with_suffix(".pyc"), dest, optimize=2, checked=checked) -def _write_to_zip(zf, dest, src, ns): - pyc = _py_temp_compile(src, ns) +def _write_to_zip(zf, dest, src, ns, checked=True): + pyc = _py_temp_compile(src, ns, checked=checked) if pyc: try: zf.write(str(pyc), dest.with_suffix(".pyc")) @@ -321,7 +329,7 @@ def generate_source_files(ns): ns.temp.mkdir(parents=True, exist_ok=True) with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf: for dest, src in get_lib_layout(ns): - _write_to_zip(zf, dest, src, ns) + _write_to_zip(zf, dest, src, ns, checked=False) if ns.include_underpth: log_info("Generating {} in {}", PYTHON_PTH_NAME, ns.temp) diff --git a/kbe/src/lib/python/PC/pyconfig.h b/kbe/src/lib/python/PC/pyconfig.h index d2a3f5dd39..46fc84e92f 100644 --- a/kbe/src/lib/python/PC/pyconfig.h +++ b/kbe/src/lib/python/PC/pyconfig.h @@ -192,18 +192,6 @@ typedef int pid_t; #define Py_IS_FINITE(X) _finite(X) #define copysign _copysign -/* VS 2010 and above already defines hypot as _hypot */ -#if _MSC_VER < 1600 -#define hypot _hypot -#endif - -/* VS 2015 defines these names with a leading underscore */ -#if _MSC_VER >= 1900 -#define timezone _timezone -#define daylight _daylight -#define tzname _tzname -#endif - /* Side by Side assemblies supported in VS 2005 and VS 2008 but not 2010*/ #if _MSC_VER >= 1400 && _MSC_VER < 1600 #define HAVE_SXS 1 @@ -231,7 +219,6 @@ typedef int pid_t; #endif #define COMPILER "[gcc]" -#define hypot _hypot #define PY_LONG_LONG long long #define PY_LLONG_MIN LLONG_MIN #define PY_LLONG_MAX LLONG_MAX diff --git a/kbe/src/lib/python/PC/winreg.c b/kbe/src/lib/python/PC/winreg.c index 6f3106644b..e3801b257b 100644 --- a/kbe/src/lib/python/PC/winreg.c +++ b/kbe/src/lib/python/PC/winreg.c @@ -826,9 +826,9 @@ If the function fails, an OSError exception is raised. [clinic start generated code]*/ static HKEY -winreg_ConnectRegistry_impl(PyObject *module, Py_UNICODE *computer_name, - HKEY key) -/*[clinic end generated code: output=5ab79d02aa3167b4 input=5f98a891a347e68e]*/ +winreg_ConnectRegistry_impl(PyObject *module, + const Py_UNICODE *computer_name, HKEY key) +/*[clinic end generated code: output=cd4f70fb9ec901fb input=5f98a891a347e68e]*/ { HKEY retKey; long rc; @@ -863,8 +863,8 @@ If the function fails, an OSError exception is raised. [clinic start generated code]*/ static HKEY -winreg_CreateKey_impl(PyObject *module, HKEY key, Py_UNICODE *sub_key) -/*[clinic end generated code: output=9c81d4095527c927 input=3cdd1622488acea2]*/ +winreg_CreateKey_impl(PyObject *module, HKEY key, const Py_UNICODE *sub_key) +/*[clinic end generated code: output=2af13910d56eae26 input=3cdd1622488acea2]*/ { HKEY retKey; long rc; @@ -902,9 +902,10 @@ If the function fails, an OSError exception is raised. [clinic start generated code]*/ static HKEY -winreg_CreateKeyEx_impl(PyObject *module, HKEY key, Py_UNICODE *sub_key, - int reserved, REGSAM access) -/*[clinic end generated code: output=b9fce6dc5c4e39b1 input=42c2b03f98406b66]*/ +winreg_CreateKeyEx_impl(PyObject *module, HKEY key, + const Py_UNICODE *sub_key, int reserved, + REGSAM access) +/*[clinic end generated code: output=643a70ad6a361a97 input=42c2b03f98406b66]*/ { HKEY retKey; long rc; @@ -937,8 +938,8 @@ is removed. If the function fails, an OSError exception is raised. [clinic start generated code]*/ static PyObject * -winreg_DeleteKey_impl(PyObject *module, HKEY key, Py_UNICODE *sub_key) -/*[clinic end generated code: output=7734b1e431991ae4 input=b31d225b935e4211]*/ +winreg_DeleteKey_impl(PyObject *module, HKEY key, const Py_UNICODE *sub_key) +/*[clinic end generated code: output=d2652a84f70e0862 input=b31d225b935e4211]*/ { long rc; rc = RegDeleteKeyW(key, sub_key ); @@ -972,9 +973,10 @@ On unsupported Windows versions, NotImplementedError is raised. [clinic start generated code]*/ static PyObject * -winreg_DeleteKeyEx_impl(PyObject *module, HKEY key, Py_UNICODE *sub_key, - REGSAM access, int reserved) -/*[clinic end generated code: output=01378d86ad3eb936 input=711d9d89e7ecbed7]*/ +winreg_DeleteKeyEx_impl(PyObject *module, HKEY key, + const Py_UNICODE *sub_key, REGSAM access, + int reserved) +/*[clinic end generated code: output=52a1c8b374ebc003 input=711d9d89e7ecbed7]*/ { HMODULE hMod; typedef LONG (WINAPI *RDKEFunc)(HKEY, const wchar_t*, REGSAM, int); @@ -1014,8 +1016,8 @@ Removes a named value from a registry key. [clinic start generated code]*/ static PyObject * -winreg_DeleteValue_impl(PyObject *module, HKEY key, Py_UNICODE *value) -/*[clinic end generated code: output=67e7e9a514f84951 input=a78d3407a4197b21]*/ +winreg_DeleteValue_impl(PyObject *module, HKEY key, const Py_UNICODE *value) +/*[clinic end generated code: output=56fa9d21f3a54371 input=a78d3407a4197b21]*/ { long rc; Py_BEGIN_ALLOW_THREADS @@ -1182,8 +1184,9 @@ Expand environment vars. [clinic start generated code]*/ static PyObject * -winreg_ExpandEnvironmentStrings_impl(PyObject *module, Py_UNICODE *string) -/*[clinic end generated code: output=cba46ac293a8af1a input=b2a9714d2b751aa6]*/ +winreg_ExpandEnvironmentStrings_impl(PyObject *module, + const Py_UNICODE *string) +/*[clinic end generated code: output=8fa4e959747a7312 input=b2a9714d2b751aa6]*/ { wchar_t *retValue = NULL; DWORD retValueSize; @@ -1275,9 +1278,9 @@ tree. [clinic start generated code]*/ static PyObject * -winreg_LoadKey_impl(PyObject *module, HKEY key, Py_UNICODE *sub_key, - Py_UNICODE *file_name) -/*[clinic end generated code: output=87344005c5905cde input=e3b5b45ade311582]*/ +winreg_LoadKey_impl(PyObject *module, HKEY key, const Py_UNICODE *sub_key, + const Py_UNICODE *file_name) +/*[clinic end generated code: output=65f89f2548cb27c7 input=e3b5b45ade311582]*/ { long rc; @@ -1309,9 +1312,9 @@ If the function fails, an OSError exception is raised. [clinic start generated code]*/ static HKEY -winreg_OpenKey_impl(PyObject *module, HKEY key, Py_UNICODE *sub_key, +winreg_OpenKey_impl(PyObject *module, HKEY key, const Py_UNICODE *sub_key, int reserved, REGSAM access) -/*[clinic end generated code: output=a905f1b947f3ce85 input=098505ac36a9ae28]*/ +/*[clinic end generated code: output=8849bff2c30104ad input=098505ac36a9ae28]*/ { HKEY retKey; long rc; @@ -1336,9 +1339,9 @@ If the function fails, an OSError exception is raised. [clinic start generated code]*/ static HKEY -winreg_OpenKeyEx_impl(PyObject *module, HKEY key, Py_UNICODE *sub_key, +winreg_OpenKeyEx_impl(PyObject *module, HKEY key, const Py_UNICODE *sub_key, int reserved, REGSAM access) -/*[clinic end generated code: output=226042593b37e940 input=c6c4972af8622959]*/ +/*[clinic end generated code: output=81bc2bd684bc77ae input=c6c4972af8622959]*/ { return winreg_OpenKey_impl(module, key, sub_key, reserved, access); } @@ -1406,8 +1409,8 @@ completeness. [clinic start generated code]*/ static PyObject * -winreg_QueryValue_impl(PyObject *module, HKEY key, Py_UNICODE *sub_key) -/*[clinic end generated code: output=2bb8d1e02c10d0b6 input=41cafbbf423b21d6]*/ +winreg_QueryValue_impl(PyObject *module, HKEY key, const Py_UNICODE *sub_key) +/*[clinic end generated code: output=c655810ae50c63a9 input=41cafbbf423b21d6]*/ { long rc; PyObject *retStr; @@ -1473,8 +1476,8 @@ The return value is a tuple of the value and the type_id. [clinic start generated code]*/ static PyObject * -winreg_QueryValueEx_impl(PyObject *module, HKEY key, Py_UNICODE *name) -/*[clinic end generated code: output=5b4fa3e33d6d3e8f input=cf366cada4836891]*/ +winreg_QueryValueEx_impl(PyObject *module, HKEY key, const Py_UNICODE *name) +/*[clinic end generated code: output=f1b85b1c3d887ec7 input=cf366cada4836891]*/ { long rc; BYTE *retBuf, *tmp; @@ -1546,8 +1549,8 @@ to the API. [clinic start generated code]*/ static PyObject * -winreg_SaveKey_impl(PyObject *module, HKEY key, Py_UNICODE *file_name) -/*[clinic end generated code: output=1dda1502bd4c30d8 input=da735241f91ac7a2]*/ +winreg_SaveKey_impl(PyObject *module, HKEY key, const Py_UNICODE *file_name) +/*[clinic end generated code: output=ca94b835c88f112b input=da735241f91ac7a2]*/ { LPSECURITY_ATTRIBUTES pSA = NULL; @@ -1592,10 +1595,10 @@ KEY_SET_VALUE access. [clinic start generated code]*/ static PyObject * -winreg_SetValue_impl(PyObject *module, HKEY key, Py_UNICODE *sub_key, - DWORD type, Py_UNICODE *value, +winreg_SetValue_impl(PyObject *module, HKEY key, const Py_UNICODE *sub_key, + DWORD type, const Py_UNICODE *value, Py_ssize_clean_t value_length) -/*[clinic end generated code: output=1e31931174820631 input=2cd2adab79339c53]*/ +/*[clinic end generated code: output=686bedb1cbb4367b input=2cd2adab79339c53]*/ { long rc; @@ -1658,9 +1661,10 @@ the configuration registry to help the registry perform efficiently. [clinic start generated code]*/ static PyObject * -winreg_SetValueEx_impl(PyObject *module, HKEY key, Py_UNICODE *value_name, - PyObject *reserved, DWORD type, PyObject *value) -/*[clinic end generated code: output=c88c8426b6c00ec7 input=900a9e3990bfb196]*/ +winreg_SetValueEx_impl(PyObject *module, HKEY key, + const Py_UNICODE *value_name, PyObject *reserved, + DWORD type, PyObject *value) +/*[clinic end generated code: output=811b769a66ae11b7 input=900a9e3990bfb196]*/ { BYTE *data; DWORD len; diff --git a/kbe/src/lib/python/PCbuild/_contextvars.vcxproj b/kbe/src/lib/python/PCbuild/_contextvars.vcxproj deleted file mode 100644 index ac4d54e1c2..0000000000 --- a/kbe/src/lib/python/PCbuild/_contextvars.vcxproj +++ /dev/null @@ -1,93 +0,0 @@ - - - - - Debug - Win32 - - - Debug - x64 - - - PGInstrument - Win32 - - - PGInstrument - x64 - - - PGUpdate - Win32 - - - PGUpdate - x64 - - - Release - Win32 - - - Release - x64 - - - - {B8BF1D81-09DC-42D4-B406-4F868B33A89E} - _contextvars - Win32Proj - - - - - DynamicLibrary - NotSet - - - - .pyd - - - - - - - - - - <_ProjectFileVersion>10.0.30319.1 - - - ../../../../res/scripts/common/DLLs/ - ../../../_objs\$(ProjectName)_d\ - - - ../../../../res/scripts/common/DLLs/ - ../../../_objs\$(ProjectName)_d\ - - - ../../../../res/scripts/common/DLLs/ - ../../../_objs\$(ProjectName)\ - - - ../../../../res/scripts/common/DLLs/ - ../../../_objs\$(ProjectName)\ - - - - - - - - - - {cf7ac3d1-e2df-41d2-bea6-1e2556cdea26} - false - - - - - - \ No newline at end of file diff --git a/kbe/src/lib/python/PCbuild/_contextvars.vcxproj.filters b/kbe/src/lib/python/PCbuild/_contextvars.vcxproj.filters deleted file mode 100644 index 06200c3478..0000000000 --- a/kbe/src/lib/python/PCbuild/_contextvars.vcxproj.filters +++ /dev/null @@ -1,16 +0,0 @@ - - - - - - - - {7CBD8910-233D-4E9A-9164-9BA66C1F0E6D} - - - - - Source Files - - - diff --git a/kbe/src/lib/python/PCbuild/_testcapi.vcxproj.filters b/kbe/src/lib/python/PCbuild/_testcapi.vcxproj.filters index 5a7edeac42..04735f7bc8 100644 --- a/kbe/src/lib/python/PCbuild/_testcapi.vcxproj.filters +++ b/kbe/src/lib/python/PCbuild/_testcapi.vcxproj.filters @@ -10,7 +10,4 @@ Source Files - - - \ No newline at end of file diff --git a/kbe/src/lib/python/PCbuild/find_python.bat b/kbe/src/lib/python/PCbuild/find_python.bat index df69a195b8..7f90fca267 100644 --- a/kbe/src/lib/python/PCbuild/find_python.bat +++ b/kbe/src/lib/python/PCbuild/find_python.bat @@ -24,7 +24,7 @@ :begin_search @set PYTHON= -@set _Py_EXTERNALS_DIR=%EXTERNAL_DIR% +@set _Py_EXTERNALS_DIR=%EXTERNALS_DIR% @if "%_Py_EXTERNALS_DIR%"=="" (set _Py_EXTERNALS_DIR=%~dp0\..\externals) @rem If we have Python in externals, use that one diff --git a/kbe/src/lib/python/PCbuild/get_externals.bat b/kbe/src/lib/python/PCbuild/get_externals.bat index dfbc924d3b..ddeb962699 100644 --- a/kbe/src/lib/python/PCbuild/get_externals.bat +++ b/kbe/src/lib/python/PCbuild/get_externals.bat @@ -51,8 +51,8 @@ set libraries= set libraries=%libraries% bzip2-1.0.6 if NOT "%IncludeSSLSrc%"=="false" set libraries=%libraries% openssl-1.1.0j set libraries=%libraries% sqlite-3.21.0.0 -if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.8.0 -if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.8.0 +if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tcl-core-8.6.9.0 +if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tk-8.6.9.0 if NOT "%IncludeTkinterSrc%"=="false" set libraries=%libraries% tix-8.4.3.6 set libraries=%libraries% xz-5.2.2 set libraries=%libraries% zlib-1.2.11 @@ -73,7 +73,7 @@ echo.Fetching external binaries... set binaries= if NOT "%IncludeSSL%"=="false" set binaries=%binaries% openssl-bin-1.1.0j -if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.8.0 +if NOT "%IncludeTkinter%"=="false" set binaries=%binaries% tcltk-8.6.9.0 if NOT "%IncludeSSLSrc%"=="false" set binaries=%binaries% nasm-2.11.06 for %%b in (%binaries%) do ( diff --git a/kbe/src/lib/python/PCbuild/openssl.props b/kbe/src/lib/python/PCbuild/openssl.props index 001dc45d84..4b8d5df8c6 100644 --- a/kbe/src/lib/python/PCbuild/openssl.props +++ b/kbe/src/lib/python/PCbuild/openssl.props @@ -23,6 +23,6 @@ - + \ No newline at end of file diff --git a/kbe/src/lib/python/PCbuild/pcbuild.proj b/kbe/src/lib/python/PCbuild/pcbuild.proj index 19d8d2f70a..b13abb7dc2 100644 --- a/kbe/src/lib/python/PCbuild/pcbuild.proj +++ b/kbe/src/lib/python/PCbuild/pcbuild.proj @@ -50,7 +50,7 @@ - + diff --git a/kbe/src/lib/python/PCbuild/pyexpat.vcxproj b/kbe/src/lib/python/PCbuild/pyexpat.vcxproj index cd786d83b0..041a881d02 100644 --- a/kbe/src/lib/python/PCbuild/pyexpat.vcxproj +++ b/kbe/src/lib/python/PCbuild/pyexpat.vcxproj @@ -38,7 +38,6 @@ <_ProjectFileVersion>10.0.30319.1 {D06B6426-4762-44CC-8BAD-D79052507F2F} pyexpat - Win32Proj @@ -102,4 +101,4 @@ - \ No newline at end of file + diff --git a/kbe/src/lib/python/PCbuild/pyproject.props b/kbe/src/lib/python/PCbuild/pyproject.props index 2b36835bd5..eaeb8b0d99 100644 --- a/kbe/src/lib/python/PCbuild/pyproject.props +++ b/kbe/src/lib/python/PCbuild/pyproject.props @@ -185,10 +185,11 @@ public override bool Execute() { $(registry:HKEY_LOCAL_MACHINE\Software\Microsoft\Windows Kits\Installed Roots@KitsRoot)\bin\x86 $(registry:HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Microsoft SDKs\Windows\v7.1A@InstallationFolder)\Bin\ <_SignCommand Condition="Exists($(SdkBinPath)) and '$(SigningCertificate)' != '' and $(SupportSigning)">"$(SdkBinPath)\signtool.exe" sign /q /a /n "$(SigningCertificate)" /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d "Python $(PythonVersion)" + <_SignCommand Condition="Exists($(SdkBinPath)) and '$(SigningCertificateSha1)' != '' and $(SupportSigning)">"$(SdkBinPath)\signtool.exe" sign /q /a /sha1 "$(SigningCertificateSha1)" /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d "Python $(PythonVersion)" <_MakeCatCommand Condition="Exists($(SdkBinPath))">"$(SdkBinPath)\makecat.exe" - - + + diff --git a/kbe/src/lib/python/PCbuild/python.props b/kbe/src/lib/python/PCbuild/python.props index 3e60cddff8..66533b6967 100644 --- a/kbe/src/lib/python/PCbuild/python.props +++ b/kbe/src/lib/python/PCbuild/python.props @@ -76,7 +76,7 @@ matter which WinSDK version we use. --> <_RegistryVersion>$(Registry:HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Microsoft SDKs\Windows\v10.0@ProductVersion) - <_RegistryVersion Condition="$(_RegistryVersion) == ''">$(Registry:HKEY_LOCAL_MACHINE\WOW6432Node\SOFTWARE\Microsoft\Microsoft SDKs\Windows\v10.0@ProductVersion) + <_RegistryVersion Condition="$(_RegistryVersion) == ''">$(Registry:HKEY_LOCAL_MACHINE\SOFTWARE\WOW6432Node\Microsoft\Microsoft SDKs\Windows\v10.0@ProductVersion) <_RegistryVersion Condition="$(_RegistryVersion) != '' and !$(_RegistryVersion.EndsWith('.0'))">$(_RegistryVersion).0 diff --git a/kbe/src/lib/python/PCbuild/pythoncore.vcxproj b/kbe/src/lib/python/PCbuild/pythoncore.vcxproj index 29d820bc54..e8eea0ad72 100644 --- a/kbe/src/lib/python/PCbuild/pythoncore.vcxproj +++ b/kbe/src/lib/python/PCbuild/pythoncore.vcxproj @@ -500,6 +500,7 @@ + $(VCInstallDir)\Redist\MSVC\$(VCToolsRedistVersion)\ $(VCRedistDir)x86\ @@ -514,4 +515,4 @@ - \ No newline at end of file + diff --git a/kbe/src/lib/python/PCbuild/pythoncore.vcxproj.filters b/kbe/src/lib/python/PCbuild/pythoncore.vcxproj.filters index f9c407d4a4..a0978e0f6b 100644 --- a/kbe/src/lib/python/PCbuild/pythoncore.vcxproj.filters +++ b/kbe/src/lib/python/PCbuild/pythoncore.vcxproj.filters @@ -438,6 +438,39 @@ Include + + Modules\zlib + + + Modules\zlib + + + Modules\zlib + + + Modules\zlib + + + Modules\zlib + + + Modules\zlib + + + Modules\zlib + + + Modules\zlib + + + Modules\zlib + + + Modules\zlib + + + Modules\zlib + @@ -485,6 +518,9 @@ Modules + + Modules + Modules @@ -821,6 +857,9 @@ Python + + Python + Python @@ -863,6 +902,9 @@ Python + + Python + Python @@ -971,18 +1013,49 @@ Objects - - Modules + + PC Modules - - + + Modules\zlib + + + Modules\zlib + + + Modules\zlib + + + Modules\zlib + + + Modules\zlib + + + Modules\zlib + + + Modules\zlib + + + Modules\zlib + + + Modules\zlib + + + Modules\zlib + + + Modules\zlib + Resource Files - \ No newline at end of file + diff --git a/kbe/src/lib/python/PCbuild/tcltk.props b/kbe/src/lib/python/PCbuild/tcltk.props index 6e0ec0e074..7dc31c393f 100644 --- a/kbe/src/lib/python/PCbuild/tcltk.props +++ b/kbe/src/lib/python/PCbuild/tcltk.props @@ -4,7 +4,7 @@ 8 6 - 8 + 9 0 $(TclMajorVersion) $(TclMinorVersion) diff --git a/kbe/src/lib/python/Python/bootstrap_hash.c b/kbe/src/lib/python/Python/bootstrap_hash.c index e2afba2b2e..58b0802f46 100644 --- a/kbe/src/lib/python/Python/bootstrap_hash.c +++ b/kbe/src/lib/python/Python/bootstrap_hash.c @@ -116,7 +116,7 @@ py_getrandom(void *buffer, Py_ssize_t size, int blocking, int raise) flags = blocking ? 0 : GRND_NONBLOCK; dest = buffer; while (0 < size) { -#ifdef sun +#if defined(__sun) && defined(__SVR4) /* Issue #26735: On Solaris, getrandom() is limited to returning up to 1024 bytes. Call it multiple times if more bytes are requested. */ @@ -266,7 +266,7 @@ py_getentropy(char *buffer, Py_ssize_t size, int raise) } return 1; } -#endif /* defined(HAVE_GETENTROPY) && !defined(sun) */ +#endif /* defined(HAVE_GETENTROPY) && !(defined(__sun) && defined(__SVR4)) */ static struct { diff --git a/kbe/src/lib/python/Python/ceval.c b/kbe/src/lib/python/Python/ceval.c index c389674704..9b6007b7ae 100644 --- a/kbe/src/lib/python/Python/ceval.c +++ b/kbe/src/lib/python/Python/ceval.c @@ -3340,9 +3340,13 @@ _PyEval_EvalFrameDefault(PyFrameObject *f, int throwflag) why = WHY_EXCEPTION; /* Double-check exception status. */ +#ifdef NDEBUG if (!PyErr_Occurred()) PyErr_SetString(PyExc_SystemError, "error return without exception set"); +#else + +#endif /* Log traceback info. */ PyTraceBack_Here(f); diff --git a/kbe/src/lib/python/Python/getcopyright.c b/kbe/src/lib/python/Python/getcopyright.c index 51772ecef3..27a1731f46 100644 --- a/kbe/src/lib/python/Python/getcopyright.c +++ b/kbe/src/lib/python/Python/getcopyright.c @@ -4,7 +4,7 @@ static const char cprt[] = "\ -Copyright (c) 2001-2018 Python Software Foundation.\n\ +Copyright (c) 2001-2019 Python Software Foundation.\n\ All Rights Reserved.\n\ \n\ Copyright (c) 2000 BeOpen.com.\n\ diff --git a/kbe/src/lib/python/Python/import.c b/kbe/src/lib/python/Python/import.c index 5f5d135c7a..ccdd599305 100644 --- a/kbe/src/lib/python/Python/import.c +++ b/kbe/src/lib/python/Python/import.c @@ -743,7 +743,6 @@ _PyImport_FindExtensionObjectEx(PyObject *name, PyObject *filename, } if (_PyState_AddModule(mod, def) < 0) { PyMapping_DelItem(modules, name); - Py_DECREF(mod); return NULL; } if (Py_VerboseFlag) diff --git a/kbe/src/lib/python/Python/sysmodule.c b/kbe/src/lib/python/Python/sysmodule.c index efe5b29ef3..cdc2edf038 100644 --- a/kbe/src/lib/python/Python/sysmodule.c +++ b/kbe/src/lib/python/Python/sysmodule.c @@ -134,11 +134,14 @@ sys_breakpointhook(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyOb modulepath = PyUnicode_FromString("builtins"); attrname = envar; } - else { + else if (last_dot != envar) { /* Split on the last dot; */ modulepath = PyUnicode_FromStringAndSize(envar, last_dot - envar); attrname = last_dot + 1; } + else { + goto warn; + } if (modulepath == NULL) { PyMem_RawFree(envar); return NULL; @@ -156,21 +159,29 @@ sys_breakpointhook(PyObject *self, PyObject *const *args, Py_ssize_t nargs, PyOb Py_DECREF(fromlist); if (module == NULL) { - goto error; + if (PyErr_ExceptionMatches(PyExc_ImportError)) { + goto warn; + } + PyMem_RawFree(envar); + return NULL; } PyObject *hook = PyObject_GetAttrString(module, attrname); Py_DECREF(module); if (hook == NULL) { - goto error; + if (PyErr_ExceptionMatches(PyExc_AttributeError)) { + goto warn; + } + PyMem_RawFree(envar); + return NULL; } PyMem_RawFree(envar); PyObject *retval = _PyObject_FastCallKeywords(hook, args, nargs, keywords); Py_DECREF(hook); return retval; - error: + warn: /* If any of the imports went wrong, then warn and ignore. */ PyErr_Clear(); int status = PyErr_WarnFormat( diff --git a/kbe/src/lib/python/Python/thread_nt.h b/kbe/src/lib/python/Python/thread_nt.h index 61fa8619bc..56295d0e8c 100644 --- a/kbe/src/lib/python/Python/thread_nt.h +++ b/kbe/src/lib/python/Python/thread_nt.h @@ -104,8 +104,9 @@ LeaveNonRecursiveMutex(PNRMUTEX mutex) if (PyMUTEX_LOCK(&mutex->cs)) return FALSE; mutex->locked = 0; - result = PyCOND_SIGNAL(&mutex->cv); - result &= PyMUTEX_UNLOCK(&mutex->cs); + /* condvar APIs return 0 on success. We need to return TRUE on success. */ + result = !PyCOND_SIGNAL(&mutex->cv); + PyMUTEX_UNLOCK(&mutex->cs); return result; } diff --git a/kbe/src/lib/python/README.rst b/kbe/src/lib/python/README.rst index 43ded3e4c7..96d05f931d 100644 --- a/kbe/src/lib/python/README.rst +++ b/kbe/src/lib/python/README.rst @@ -1,4 +1,4 @@ -This is Python version 3.7.2 +This is Python version 3.7.3 ============================ .. image:: https://travis-ci.org/python/cpython.svg?branch=master @@ -18,8 +18,8 @@ This is Python version 3.7.2 :target: https://codecov.io/gh/python/cpython Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, -2012, 2013, 2014, 2015, 2016, 2017, 2018 Python Software Foundation. All rights -reserved. +2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019 Python Software Foundation. All +rights reserved. See the end of this file for further copyright and license information. @@ -60,7 +60,7 @@ On Unix, Linux, BSD, macOS, and Cygwin:: make test sudo make install -This will install Python as python3. +This will install Python as ``python3``. You can pass many options to the configure script; run ``./configure --help`` to find out more. On macOS and Cygwin, the executable is called ``python.exe``; @@ -247,8 +247,8 @@ Copyright and License Information --------------------------------- Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, -2012, 2013, 2014, 2015, 2016, 2017, 2018 Python Software Foundation. All rights -reserved. +2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019 Python Software Foundation. All +rights reserved. Copyright (c) 2000 BeOpen.com. All rights reserved. diff --git a/kbe/src/lib/python/Tools/clinic/clinic.py b/kbe/src/lib/python/Tools/clinic/clinic.py index b704b5a669..0ba8caba9e 100755 --- a/kbe/src/lib/python/Tools/clinic/clinic.py +++ b/kbe/src/lib/python/Tools/clinic/clinic.py @@ -2824,7 +2824,7 @@ class unicode_converter(CConverter): @add_legacy_c_converter('Z', accept={str, NoneType}) @add_legacy_c_converter('Z#', accept={str, NoneType}, zeroes=True) class Py_UNICODE_converter(CConverter): - type = 'Py_UNICODE *' + type = 'const Py_UNICODE *' default_type = (str, Null, NoneType) format_unit = 'u' diff --git a/kbe/src/lib/python/Tools/gdb/libpython.py b/kbe/src/lib/python/Tools/gdb/libpython.py index bfaa9403b7..d744cab764 100755 --- a/kbe/src/lib/python/Tools/gdb/libpython.py +++ b/kbe/src/lib/python/Tools/gdb/libpython.py @@ -1178,7 +1178,7 @@ def char_width(self): def proxyval(self, visited): global _is_pep393 if _is_pep393 is None: - fields = gdb.lookup_type('PyUnicodeObject').target().fields() + fields = gdb.lookup_type('PyUnicodeObject').fields() _is_pep393 = 'data' in [f.name for f in fields] if _is_pep393: # Python 3.3 and newer diff --git a/kbe/src/lib/python/Tools/msi/dev/dev.wixproj b/kbe/src/lib/python/Tools/msi/dev/dev.wixproj index bc3a19ce33..4a56cec357 100644 --- a/kbe/src/lib/python/Tools/msi/dev/dev.wixproj +++ b/kbe/src/lib/python/Tools/msi/dev/dev.wixproj @@ -21,7 +21,8 @@ - + $(PySourcePath) !(bindpath.src) $(PySourcePath) diff --git a/kbe/src/lib/python/Tools/msi/lib/lib_files.wxs b/kbe/src/lib/python/Tools/msi/lib/lib_files.wxs index 4bd0c57e32..a9952bdac4 100644 --- a/kbe/src/lib/python/Tools/msi/lib/lib_files.wxs +++ b/kbe/src/lib/python/Tools/msi/lib/lib_files.wxs @@ -1,6 +1,6 @@  - + diff --git a/kbe/src/lib/python/Tools/msi/sdktools.psm1 b/kbe/src/lib/python/Tools/msi/sdktools.psm1 index 81a74d3679..61edb34117 100644 --- a/kbe/src/lib/python/Tools/msi/sdktools.psm1 +++ b/kbe/src/lib/python/Tools/msi/sdktools.psm1 @@ -21,6 +21,9 @@ function Sign-File { $description = "Python"; } } + if (-not $certsha1) { + $certsha1 = $env:SigningCertificateSha1; + } if (-not $certname) { $certname = $env:SigningCertificate; } @@ -32,7 +35,7 @@ function Sign-File { if ($certsha1) { SignTool sign /sha1 $certsha1 /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d $description $a } elseif ($certname) { - SignTool sign /n $certname /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d $description $a + SignTool sign /a /n $certname /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d $description $a } elseif ($certfile) { SignTool sign /f $certfile /fd sha256 /t http://timestamp.verisign.com/scripts/timestamp.dll /d $description $a } else { diff --git a/kbe/src/lib/python/Tools/nuget/python.nuspec b/kbe/src/lib/python/Tools/nuget/python.nuspec index d5f3e63242..8f98e80891 100644 --- a/kbe/src/lib/python/Tools/nuget/python.nuspec +++ b/kbe/src/lib/python/Tools/nuget/python.nuspec @@ -5,9 +5,8 @@ Python 0.0.0.0 Python Software Foundation - https://docs.python.org/3/license.html + tools\LICENSE.txt https://www.python.org/ - false Installs 64-bit Python for use in build scenarios. https://www.python.org/static/favicon.ico python diff --git a/kbe/src/lib/python/Tools/nuget/pythondaily.nuspec b/kbe/src/lib/python/Tools/nuget/pythondaily.nuspec index ee3343bbb7..5cf55806dd 100644 --- a/kbe/src/lib/python/Tools/nuget/pythondaily.nuspec +++ b/kbe/src/lib/python/Tools/nuget/pythondaily.nuspec @@ -3,11 +3,10 @@ pythondaily Python (Daily build) - 0.0.0.0 Python Software Foundation - https://docs.python.org/3/license.html + 0.0.0.0 + tools\LICENSE.txt https://www.python.org/ - false Installs an unsigned, untested build of Python for test purposes only. https://www.python.org/static/favicon.ico python diff --git a/kbe/src/lib/python/Tools/nuget/pythondaily.symbols.nuspec b/kbe/src/lib/python/Tools/nuget/pythondaily.symbols.nuspec index b89717a1af..608e15c50e 100644 --- a/kbe/src/lib/python/Tools/nuget/pythondaily.symbols.nuspec +++ b/kbe/src/lib/python/Tools/nuget/pythondaily.symbols.nuspec @@ -5,9 +5,7 @@ Python (Daily build) 0.0.0.0 Python Software Foundation - https://docs.python.org/3/license.html https://www.python.org/ - false Contains symbols for the daily build of Python. https://www.python.org/static/favicon.ico python diff --git a/kbe/src/lib/python/Tools/nuget/pythonx86.nuspec b/kbe/src/lib/python/Tools/nuget/pythonx86.nuspec index ebfcd6c92a..27ef67e7f5 100644 --- a/kbe/src/lib/python/Tools/nuget/pythonx86.nuspec +++ b/kbe/src/lib/python/Tools/nuget/pythonx86.nuspec @@ -5,9 +5,8 @@ Python (32-bit) Python Software Foundation 0.0.0.0 - https://docs.python.org/3/license.html + tools\LICENSE.txt https://www.python.org/ - false Installs 32-bit Python for use in build scenarios. https://www.python.org/static/favicon.ico python diff --git a/kbe/src/lib/python/aclocal.m4 b/kbe/src/lib/python/aclocal.m4 index 32828c24dd..85a0dbba27 100644 --- a/kbe/src/lib/python/aclocal.m4 +++ b/kbe/src/lib/python/aclocal.m4 @@ -1,6 +1,6 @@ -# generated automatically by aclocal 1.15.1 -*- Autoconf -*- +# generated automatically by aclocal 1.16.1 -*- Autoconf -*- -# Copyright (C) 1996-2017 Free Software Foundation, Inc. +# Copyright (C) 1996-2018 Free Software Foundation, Inc. # This file is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, @@ -13,7 +13,7 @@ m4_ifndef([AC_CONFIG_MACRO_DIRS], [m4_defun([_AM_CONFIG_MACRO_DIRS], [])m4_defun([AC_CONFIG_MACRO_DIRS], [_AM_CONFIG_MACRO_DIRS($@)])]) dnl pkg.m4 - Macros to locate and utilise pkg-config. -*- Autoconf -*- -dnl serial 11 (pkg-config-0.29) +dnl serial 11 (pkg-config-0.29.1) dnl dnl Copyright © 2004 Scott James Remnant . dnl Copyright © 2012-2015 Dan Nicholson @@ -55,7 +55,7 @@ dnl dnl See the "Since" comment for each macro you use to see what version dnl of the macros you require. m4_defun([PKG_PREREQ], -[m4_define([PKG_MACROS_VERSION], [0.29]) +[m4_define([PKG_MACROS_VERSION], [0.29.1]) m4_if(m4_version_compare(PKG_MACROS_VERSION, [$1]), -1, [m4_fatal([pkg.m4 version $1 or higher is required but ]PKG_MACROS_VERSION[ found])]) ])dnl PKG_PREREQ diff --git a/kbe/src/lib/python/configure b/kbe/src/lib/python/configure index bd3ed927a9..18047ced43 100755 --- a/kbe/src/lib/python/configure +++ b/kbe/src/lib/python/configure @@ -784,6 +784,7 @@ infodir docdir oldincludedir includedir +runstatedir localstatedir sharedstatedir sysconfdir @@ -897,6 +898,7 @@ datadir='${datarootdir}' sysconfdir='${prefix}/etc' sharedstatedir='${prefix}/com' localstatedir='${prefix}/var' +runstatedir='${localstatedir}/run' includedir='${prefix}/include' oldincludedir='/usr/include' docdir='${datarootdir}/doc/${PACKAGE_TARNAME}' @@ -1149,6 +1151,15 @@ do | -silent | --silent | --silen | --sile | --sil) silent=yes ;; + -runstatedir | --runstatedir | --runstatedi | --runstated \ + | --runstate | --runstat | --runsta | --runst | --runs \ + | --run | --ru | --r) + ac_prev=runstatedir ;; + -runstatedir=* | --runstatedir=* | --runstatedi=* | --runstated=* \ + | --runstate=* | --runstat=* | --runsta=* | --runst=* | --runs=* \ + | --run=* | --ru=* | --r=*) + runstatedir=$ac_optarg ;; + -sbindir | --sbindir | --sbindi | --sbind | --sbin | --sbi | --sb) ac_prev=sbindir ;; -sbindir=* | --sbindir=* | --sbindi=* | --sbind=* | --sbin=* \ @@ -1286,7 +1297,7 @@ fi for ac_var in exec_prefix prefix bindir sbindir libexecdir datarootdir \ datadir sysconfdir sharedstatedir localstatedir includedir \ oldincludedir docdir infodir htmldir dvidir pdfdir psdir \ - libdir localedir mandir + libdir localedir mandir runstatedir do eval ac_val=\$$ac_var # Remove trailing slashes. @@ -1439,6 +1450,7 @@ Fine tuning of the installation directories: --sysconfdir=DIR read-only single-machine data [PREFIX/etc] --sharedstatedir=DIR modifiable architecture-independent data [PREFIX/com] --localstatedir=DIR modifiable single-machine data [PREFIX/var] + --runstatedir=DIR modifiable per-process data [LOCALSTATEDIR/run] --libdir=DIR object code libraries [EPREFIX/lib] --includedir=DIR C header files [PREFIX/include] --oldincludedir=DIR C header files for non-gcc [/usr/include] @@ -12727,6 +12739,150 @@ fi done +# We search for both crypt and crypt_r as one or the other may be defined +# This gets us our -lcrypt in LIBS when required on the target platform. +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing crypt" >&5 +$as_echo_n "checking for library containing crypt... " >&6; } +if ${ac_cv_search_crypt+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_func_search_save_LIBS=$LIBS +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char crypt (); +int +main () +{ +return crypt (); + ; + return 0; +} +_ACEOF +for ac_lib in '' crypt; do + if test -z "$ac_lib"; then + ac_res="none required" + else + ac_res=-l$ac_lib + LIBS="-l$ac_lib $ac_func_search_save_LIBS" + fi + if ac_fn_c_try_link "$LINENO"; then : + ac_cv_search_crypt=$ac_res +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext + if ${ac_cv_search_crypt+:} false; then : + break +fi +done +if ${ac_cv_search_crypt+:} false; then : + +else + ac_cv_search_crypt=no +fi +rm conftest.$ac_ext +LIBS=$ac_func_search_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_crypt" >&5 +$as_echo "$ac_cv_search_crypt" >&6; } +ac_res=$ac_cv_search_crypt +if test "$ac_res" != no; then : + test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" + +fi + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing crypt_r" >&5 +$as_echo_n "checking for library containing crypt_r... " >&6; } +if ${ac_cv_search_crypt_r+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_func_search_save_LIBS=$LIBS +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char crypt_r (); +int +main () +{ +return crypt_r (); + ; + return 0; +} +_ACEOF +for ac_lib in '' crypt; do + if test -z "$ac_lib"; then + ac_res="none required" + else + ac_res=-l$ac_lib + LIBS="-l$ac_lib $ac_func_search_save_LIBS" + fi + if ac_fn_c_try_link "$LINENO"; then : + ac_cv_search_crypt_r=$ac_res +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext + if ${ac_cv_search_crypt_r+:} false; then : + break +fi +done +if ${ac_cv_search_crypt_r+:} false; then : + +else + ac_cv_search_crypt_r=no +fi +rm conftest.$ac_ext +LIBS=$ac_func_search_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_crypt_r" >&5 +$as_echo "$ac_cv_search_crypt_r" >&6; } +ac_res=$ac_cv_search_crypt_r +if test "$ac_res" != no; then : + test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" + +fi + + +ac_fn_c_check_func "$LINENO" "crypt_r" "ac_cv_func_crypt_r" +if test "x$ac_cv_func_crypt_r" = xyes; then : + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +#define _GNU_SOURCE /* Required for crypt_r()'s prototype in glibc. */ +#include + +int +main () +{ + +struct crypt_data d; +char *r = crypt_r("", "", &d); + + ; + return 0; +} +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + +$as_echo "#define HAVE_CRYPT_R 1" >>confdefs.h + +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext + +fi + + for ac_func in clock_gettime do : ac_fn_c_check_func "$LINENO" "clock_gettime" "ac_cv_func_clock_gettime" @@ -15598,6 +15754,8 @@ else cat confdefs.h - <<_ACEOF >conftest.$ac_ext /* end confdefs.h. */ +#include +#include int main() { int val1 = nice(1); @@ -17059,7 +17217,7 @@ fi $as_echo_n "checking for openssl/ssl.h in $ssldir... " >&6; } if test -f "$ssldir/include/openssl/ssl.h"; then OPENSSL_INCLUDES="-I$ssldir/include" - OPENSSL_LDFLAGS="-L${KBE_ROOT}/kbe/src/libs" + OPENSSL_LDFLAGS="-L$ssldir/lib" OPENSSL_LIBS="-lssl -lcrypto" found=true { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 diff --git a/kbe/src/lib/python/configure.ac b/kbe/src/lib/python/configure.ac index a7de901e08..ad0f4d42b4 100644 --- a/kbe/src/lib/python/configure.ac +++ b/kbe/src/lib/python/configure.ac @@ -3875,6 +3875,23 @@ AC_CHECK_FUNCS(gettimeofday, ]) ) +# We search for both crypt and crypt_r as one or the other may be defined +# This gets us our -lcrypt in LIBS when required on the target platform. +AC_SEARCH_LIBS(crypt, crypt) +AC_SEARCH_LIBS(crypt_r, crypt) + +AC_CHECK_FUNC(crypt_r, + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ +#define _GNU_SOURCE /* Required for crypt_r()'s prototype in glibc. */ +#include +]], [[ +struct crypt_data d; +char *r = crypt_r("", "", &d); +]])], + [AC_DEFINE(HAVE_CRYPT_R, 1, [Define if you have the crypt_r() function.])], + []) +) + AC_CHECK_FUNCS(clock_gettime, [], [ AC_CHECK_LIB(rt, clock_gettime, [ LIBS="$LIBS -lrt" @@ -4873,6 +4890,8 @@ LIBS=$LIBS_no_readline AC_MSG_CHECKING(for broken nice()) AC_CACHE_VAL(ac_cv_broken_nice, [ AC_RUN_IFELSE([AC_LANG_SOURCE([[ +#include +#include int main() { int val1 = nice(1); diff --git a/kbe/src/lib/python/pyconfig.h.in b/kbe/src/lib/python/pyconfig.h.in index a82e82cc9c..4032fa519f 100644 --- a/kbe/src/lib/python/pyconfig.h.in +++ b/kbe/src/lib/python/pyconfig.h.in @@ -143,6 +143,9 @@ /* Define to 1 if you have the header file. */ #undef HAVE_CRYPT_H +/* Define if you have the crypt_r() function. */ +#undef HAVE_CRYPT_R + /* Define to 1 if you have the `ctermid' function. */ #undef HAVE_CTERMID diff --git a/kbe/src/lib/server/components.cpp b/kbe/src/lib/server/components.cpp index 7a638935a2..9c9bbe93c6 100644 --- a/kbe/src/lib/server/components.cpp +++ b/kbe/src/lib/server/components.cpp @@ -326,11 +326,11 @@ void Components::removeComponentByChannel(Network::Channel * pChannel, bool isSh if (!isShutingdown && g_componentType != LOGGER_TYPE && g_componentType != INTERFACES_TYPE) { - ERROR_MSG(fmt::format("Components::removeComponentByChannel: {} : {}, Abnormal exit! {}\n", - COMPONENT_NAME_EX(componentType), (*iter).cid, pChannel->condemnReason())); + ERROR_MSG(fmt::format("Components::removeComponentByChannel: {} : {}, Abnormal exit(reason={})! Channel(timestamp={}, lastReceivedTime={}, inactivityExceptionPeriod={})\n", + COMPONENT_NAME_EX(componentType), (*iter).cid, pChannel->condemnReason(), timestamp(), pChannel->lastReceivedTime(), pChannel->inactivityExceptionPeriod())); #if KBE_PLATFORM == PLATFORM_WIN32 - printf("[ERROR]: %s.\n", (fmt::format("Components::removeComponentByChannel: {} : {}, Abnormal exit! {}\n", + printf("[ERROR]: %s.\n", (fmt::format("Components::removeComponentByChannel: {} : {}, Abnormal exit(reason={})!\n", COMPONENT_NAME_EX(componentType), (*iter).cid, pChannel->condemnReason())).c_str()); #endif } diff --git a/kbe/src/lib/server/id_component_querier.cpp b/kbe/src/lib/server/id_component_querier.cpp index 059c2c9f43..eb8644c34d 100644 --- a/kbe/src/lib/server/id_component_querier.cpp +++ b/kbe/src/lib/server/id_component_querier.cpp @@ -1,3 +1,5 @@ +// Copyright 2008-2018 Yolo Technologies, Inc. All Rights Reserved. https://www.comblockengine.com + #include "id_component_querier.h" #include "helper/sys_info.h" #include "common.h" diff --git a/kbe/src/lib/server/serverapp.cpp b/kbe/src/lib/server/serverapp.cpp index 3e3b1c221d..6baa798cee 100644 --- a/kbe/src/lib/server/serverapp.cpp +++ b/kbe/src/lib/server/serverapp.cpp @@ -441,7 +441,7 @@ void ServerApp::onAppActiveTick(Network::Channel* pChannel, COMPONENT_TYPE compo cinfos->pChannel->updateLastReceivedTime(); } - //DEBUG_MSG(fmt::format("ServerApp::onAppActiveTick[:p]: {}:{} lastReceivedTime:{} at {}.\n", + //DEBUG_MSG(fmt::format("ServerApp::onAppActiveTick[{:p}]: {}:{} lastReceivedTime:{} at {}.\n", // (void*)pChannel, COMPONENT_NAME_EX(componentType), componentID, pChannel->lastReceivedTime(), pChannel->c_str())); } diff --git a/kbe/src/server/baseapp/baseapp.cpp b/kbe/src/server/baseapp/baseapp.cpp index 6e24969810..0a15f20182 100644 --- a/kbe/src/server/baseapp/baseapp.cpp +++ b/kbe/src/server/baseapp/baseapp.cpp @@ -3821,15 +3821,6 @@ void Baseapp::loginBaseapp(Network::Channel* pChannel, return; } - // Ȼdbmgr룬loginappʱύpasswordӦøύƥ - // ױӹʽ̽½ - if (!ptinfos->needCheckPassword && ptinfos->password != password) - { - loginBaseappFailed(pChannel, accountName, SERVER_ERR_NAME_PASSWORD); - pendingLoginMgr_.removeNextTick(accountName); - return; - } - // entityID0˵entityǴ״̬¼ if(ptinfos->entityID > 0) { diff --git a/kbe/src/server/cellapp/entity.cpp b/kbe/src/server/cellapp/entity.cpp index 9815c5b150..4c8d018c1c 100644 --- a/kbe/src/server/cellapp/entity.cpp +++ b/kbe/src/server/cellapp/entity.cpp @@ -1487,7 +1487,13 @@ void Entity::addWitnessed(Entity* entity) //------------------------------------------------------------------------------------- void Entity::delWitnessed(Entity* entity) { - KBE_ASSERT(witnesses_count_ > 0); + if (witnesses_count_ == 0) + { + ERROR_MSG(fmt::format("{}::delWitnessed({}): witness is empty!\n", + scriptName(), id())); + + return; + } witnesses_.remove(entity->id()); --witnesses_count_; diff --git a/kbe/src/server/dbmgr/dbmgr.cpp b/kbe/src/server/dbmgr/dbmgr.cpp index 5f95de31fe..7cf1fdf3c3 100644 --- a/kbe/src/server/dbmgr/dbmgr.cpp +++ b/kbe/src/server/dbmgr/dbmgr.cpp @@ -1,1274 +1,1287 @@ -// Copyright 2008-2018 Yolo Technologies, Inc. All Rights Reserved. https://www.comblockengine.com - - -#include "dbmgr.h" -#include "dbmgr_interface.h" -#include "dbtasks.h" -#include "profile.h" -#include "interfaces_handler.h" -#include "sync_app_datas_handler.h" -#include "update_dblog_handler.h" -#include "db_mysql/kbe_table_mysql.h" -#include "network/common.h" -#include "network/tcp_packet.h" -#include "network/udp_packet.h" -#include "network/message_handler.h" -#include "thread/threadpool.h" -#include "server/components.h" -#include "server/telnet_server.h" -#include "db_interface/db_interface.h" -#include "db_mysql/db_interface_mysql.h" -#include "entitydef/scriptdef_module.h" - -#include "baseapp/baseapp_interface.h" -#include "cellapp/cellapp_interface.h" -#include "baseappmgr/baseappmgr_interface.h" -#include "cellappmgr/cellappmgr_interface.h" -#include "loginapp/loginapp_interface.h" - -namespace KBEngine{ - -ServerConfig g_serverConfig; -KBE_SINGLETON_INIT(Dbmgr); - -//------------------------------------------------------------------------------------- -Dbmgr::Dbmgr(Network::EventDispatcher& dispatcher, - Network::NetworkInterface& ninterface, - COMPONENT_TYPE componentType, - COMPONENT_ID componentID): - PythonApp(dispatcher, ninterface, componentType, componentID), - loopCheckTimerHandle_(), - mainProcessTimer_(), - idServer_(1, 1024), - pGlobalData_(NULL), - pBaseAppData_(NULL), - pCellAppData_(NULL), - bufferedDBTasksMaps_(), - numWrittenEntity_(0), - numRemovedEntity_(0), - numQueryEntity_(0), - numExecuteRawDatabaseCommand_(0), - numCreatedAccount_(0), - pInterfacesHandlers_(), - pSyncAppDatasHandler_(NULL), - pUpdateDBServerLogHandler_(NULL), - pTelnetServer_(NULL), - loseBaseappts_() -{ - KBEngine::Network::MessageHandlers::pMainMessageHandlers = &DbmgrInterface::messageHandlers; -} - -//------------------------------------------------------------------------------------- -Dbmgr::~Dbmgr() -{ - loopCheckTimerHandle_.cancel(); - mainProcessTimer_.cancel(); - KBEngine::sleep(300); - - for (std::vector::iterator iter = pInterfacesHandlers_.begin(); iter != pInterfacesHandlers_.end(); ++iter) - { - SAFE_RELEASE((*iter)); - } -} - -//------------------------------------------------------------------------------------- -ShutdownHandler::CAN_SHUTDOWN_STATE Dbmgr::canShutdown() -{ - if (getEntryScript().get() && PyObject_HasAttrString(getEntryScript().get(), "onReadyForShutDown") > 0) - { - // нű - PyObject* pyResult = PyObject_CallMethod(getEntryScript().get(), - const_cast("onReadyForShutDown"), - const_cast("")); - - if (pyResult != NULL) - { - bool isReady = (pyResult == Py_True); - Py_DECREF(pyResult); - - if (!isReady) - return ShutdownHandler::CAN_SHUTDOWN_STATE_USER_FALSE; - } - else - { - SCRIPT_ERROR_CHECK(); - return ShutdownHandler::CAN_SHUTDOWN_STATE_USER_FALSE; - } - } - - KBEUnordered_map::iterator bditer = bufferedDBTasksMaps_.begin(); - for (; bditer != bufferedDBTasksMaps_.end(); ++bditer) - { - if (bditer->second.size() > 0) - { - thread::ThreadPool* pThreadPool = DBUtil::pThreadPool(bditer->first); - KBE_ASSERT(pThreadPool); - - INFO_MSG(fmt::format("Dbmgr::canShutdown(): Wait for the task to complete, dbInterface={}, tasks{}=[{}], threads={}/{}, threadpoolDestroyed={}!\n", - bditer->first, bditer->second.size(), bditer->second.getTasksinfos(), (pThreadPool->currentThreadCount() - pThreadPool->currentFreeThreadCount()), - pThreadPool->currentThreadCount(), pThreadPool->isDestroyed())); - - return ShutdownHandler::CAN_SHUTDOWN_STATE_FALSE; - } - } - - Components::COMPONENTS& cellapp_components = Components::getSingleton().getComponents(CELLAPP_TYPE); - if (cellapp_components.size() > 0) - { - std::string s; - for (size_t i = 0; i 0) - { - std::string s; - for (size_t i = 0; i("onDBMgrShutDown"), false); -} - -//------------------------------------------------------------------------------------- -void Dbmgr::onShutdownEnd() -{ - PythonApp::onShutdownEnd(); -} - -//------------------------------------------------------------------------------------- -bool Dbmgr::initializeWatcher() -{ - WATCH_OBJECT("numWrittenEntity", numWrittenEntity_); - WATCH_OBJECT("numRemovedEntity", numRemovedEntity_); - WATCH_OBJECT("numQueryEntity", numQueryEntity_); - WATCH_OBJECT("numExecuteRawDatabaseCommand", numExecuteRawDatabaseCommand_); - WATCH_OBJECT("numCreatedAccount", numCreatedAccount_); - - KBEUnordered_map::iterator bditer = bufferedDBTasksMaps_.begin(); - for (; bditer != bufferedDBTasksMaps_.end(); ++bditer) - { - WATCH_OBJECT(fmt::format("DBThreadPool/{}/dbid_tasksSize", bditer->first).c_str(), &bditer->second, &Buffered_DBTasks::dbid_tasksSize); - WATCH_OBJECT(fmt::format("DBThreadPool/{}/entityid_tasksSize", bditer->first).c_str(), &bditer->second, &Buffered_DBTasks::entityid_tasksSize); - WATCH_OBJECT(fmt::format("DBThreadPool/{}/printBuffered_dbid", bditer->first).c_str(), &bditer->second, &Buffered_DBTasks::printBuffered_dbid); - WATCH_OBJECT(fmt::format("DBThreadPool/{}/printBuffered_entityID", bditer->first).c_str(), &bditer->second, &Buffered_DBTasks::printBuffered_entityID); - } - - return ServerApp::initializeWatcher() && DBUtil::initializeWatcher(); -} - -//------------------------------------------------------------------------------------- -bool Dbmgr::run() -{ - return PythonApp::run(); -} - -//------------------------------------------------------------------------------------- -void Dbmgr::handleTimeout(TimerHandle handle, void * arg) -{ - PythonApp::handleTimeout(handle, arg); - - switch (reinterpret_cast(arg)) - { - case TIMEOUT_TICK: - this->handleMainTick(); - break; - case TIMEOUT_CHECK_STATUS: - this->handleCheckStatusTick(); - break; - default: - break; - } -} - -//------------------------------------------------------------------------------------- -void Dbmgr::handleMainTick() -{ - AUTO_SCOPED_PROFILE("mainTick"); - - // time_t t = ::time(NULL); - // static int kbeTime = 0; - // DEBUG_MSG(fmt::format("Dbmgr::handleGameTick[{}]:{}\n", t, ++kbeTime)); - - threadPool_.onMainThreadTick(); - DBUtil::handleMainTick(); - networkInterface().processChannels(&DbmgrInterface::messageHandlers); -} - -//------------------------------------------------------------------------------------- -void Dbmgr::handleCheckStatusTick() -{ - // 鶪ʧ̣һʱ֮Ȼ޷֣Ҫݿentitylog - if (loseBaseappts_.size() > 0) - { - std::map::iterator iter = loseBaseappts_.begin(); - for (; iter != loseBaseappts_.end();) - { - if (timestamp() > iter->second) - { - Components::ComponentInfos* cinfo = Components::getSingleton().findComponent(iter->first); - if (!cinfo) - { - ENGINE_COMPONENT_INFO& dbcfg = g_kbeSrvConfig.getDBMgr(); - std::vector::iterator dbinfo_iter = dbcfg.dbInterfaceInfos.begin(); - for (; dbinfo_iter != dbcfg.dbInterfaceInfos.end(); ++dbinfo_iter) - { - std::string dbInterfaceName = dbinfo_iter->name; - - DBUtil::pThreadPool(dbInterfaceName)-> - addTask(new DBTaskEraseBaseappEntityLog(iter->first)); - } - } - - loseBaseappts_.erase(iter++); - } - else - { - ++iter; - } - } - } -} - -//------------------------------------------------------------------------------------- -bool Dbmgr::initializeBegin() -{ - idServer_.set_range_step(g_kbeSrvConfig.getDBMgr().ids_increasing_range); - return true; -} - -//------------------------------------------------------------------------------------- -bool Dbmgr::inInitialize() -{ - // ʼչģ - // assets/scripts/ - if (!PythonApp::inInitialize()) - return false; - - std::vector scriptBaseTypes; - if(!EntityDef::initialize(scriptBaseTypes, componentType_)){ - return false; - } - - return true; -} - -//------------------------------------------------------------------------------------- -bool Dbmgr::initializeEnd() -{ - PythonApp::initializeEnd(); - - // һtimer ÿһЩ״̬ - loopCheckTimerHandle_ = this->dispatcher().addTimer(1000000, this, - reinterpret_cast(TIMEOUT_CHECK_STATUS)); - - mainProcessTimer_ = this->dispatcher().addTimer(1000000 / 50, this, - reinterpret_cast(TIMEOUT_TICK)); - - // globalData, baseAppData, cellAppData֧ - pGlobalData_ = new GlobalDataServer(GlobalDataServer::GLOBAL_DATA); - pBaseAppData_ = new GlobalDataServer(GlobalDataServer::BASEAPP_DATA); - pCellAppData_ = new GlobalDataServer(GlobalDataServer::CELLAPP_DATA); - pGlobalData_->addConcernComponentType(CELLAPP_TYPE); - pGlobalData_->addConcernComponentType(BASEAPP_TYPE); - pBaseAppData_->addConcernComponentType(BASEAPP_TYPE); - pCellAppData_->addConcernComponentType(CELLAPP_TYPE); - - INFO_MSG(fmt::format("Dbmgr::initializeEnd: digest({})\n", - EntityDef::md5().getDigestStr())); - - SCOPED_PROFILE(SCRIPTCALL_PROFILE); - - // нű - PyObject* pyResult = PyObject_CallMethod(getEntryScript().get(), - const_cast("onDBMgrReady"), - const_cast("")); - - if(pyResult != NULL) - Py_DECREF(pyResult); - else - SCRIPT_ERROR_CHECK(); - - pTelnetServer_ = new TelnetServer(&this->dispatcher(), &this->networkInterface()); - pTelnetServer_->pScript(&this->getScript()); - - bool ret = pTelnetServer_->start(g_kbeSrvConfig.getDBMgr().telnet_passwd, - g_kbeSrvConfig.getDBMgr().telnet_deflayer, - g_kbeSrvConfig.getDBMgr().telnet_port); - - Components::getSingleton().extraData4(pTelnetServer_->port()); - - return ret && initInterfacesHandler() && initDB(); -} - -//------------------------------------------------------------------------------------- -void Dbmgr::onInstallPyModules() -{ - PyObject * module = getScript().getModule(); - - for (int i = 0; i < SERVER_ERR_MAX; i++) - { - if(PyModule_AddIntConstant(module, SERVER_ERR_STR[i], i)) - { - ERROR_MSG( fmt::format("Dbmgr::onInstallPyModules: Unable to set KBEngine.{}.\n", SERVER_ERR_STR[i])); - } - } - - APPEND_SCRIPT_MODULE_METHOD(module, executeRawDatabaseCommand, __py_executeRawDatabaseCommand, METH_VARARGS, 0); -} - -//------------------------------------------------------------------------------------- -bool Dbmgr::initInterfacesHandler() -{ - std::vector< Network::Address > addresses = g_kbeSrvConfig.interfacesAddrs(); - std::string type = addresses.size() == 0 ? "dbmgr" : "interfaces"; - - if (type == "dbmgr") - { - InterfacesHandler* pInterfacesHandler = InterfacesHandlerFactory::create(type); - - INFO_MSG(fmt::format("Dbmgr::initInterfacesHandler: interfaces addr({}), accountType:({}), chargeType:({}).\n", - Network::Address::NONE.c_str(), - type, - type)); - - if (!pInterfacesHandler->initialize()) - return false; - - pInterfacesHandlers_.push_back(pInterfacesHandler); - } - else - { - std::vector< Network::Address >::iterator iter = addresses.begin(); - for (; iter != addresses.end(); ++iter) - { - InterfacesHandler* pInterfacesHandler = InterfacesHandlerFactory::create(type); - - const Network::Address& addr = (*iter); - - INFO_MSG(fmt::format("Dbmgr::initInterfacesHandler: interfaces addr({}), accountType:({}), chargeType:({}).\n", - addr.c_str(), - type, - type)); - - ((InterfacesHandler_Interfaces*)pInterfacesHandler)->setAddr(addr); - - if (!pInterfacesHandler->initialize()) - return false; - - pInterfacesHandlers_.push_back(pInterfacesHandler); - } - } - - return pInterfacesHandlers_.size() > 0; -} - -//------------------------------------------------------------------------------------- -bool Dbmgr::initDB() -{ - ScriptDefModule* pModule = EntityDef::findScriptModule(DBUtil::accountScriptName()); - if(pModule == NULL) - { - ERROR_MSG(fmt::format("Dbmgr::initDB(): not found account script[{}]!\n", - DBUtil::accountScriptName())); - - return false; - } - - ENGINE_COMPONENT_INFO& dbcfg = g_kbeSrvConfig.getDBMgr(); - if (dbcfg.dbInterfaceInfos.size() == 0) - { - ERROR_MSG(fmt::format("DBUtil::initialize: not found dbInterface! (kbengine[_defs].xml->dbmgr->databaseInterfaces)\n")); - return false; - } - - if (!DBUtil::initialize()) - { - ERROR_MSG("Dbmgr::initDB(): can't initialize dbInterface!\n"); - return false; - } - - bool hasDefaultInterface = false; - - std::vector::iterator dbinfo_iter = dbcfg.dbInterfaceInfos.begin(); - for (; dbinfo_iter != dbcfg.dbInterfaceInfos.end(); ++dbinfo_iter) - { - Buffered_DBTasks buffered_DBTasks; - bufferedDBTasksMaps_.insert(std::make_pair((*dbinfo_iter).name, buffered_DBTasks)); - BUFFERED_DBTASKS_MAP::iterator buffered_DBTasks_iter = bufferedDBTasksMaps_.find((*dbinfo_iter).name); - buffered_DBTasks_iter->second.dbInterfaceName((*dbinfo_iter).name); - } - - for (dbinfo_iter = dbcfg.dbInterfaceInfos.begin(); dbinfo_iter != dbcfg.dbInterfaceInfos.end(); ++dbinfo_iter) - { - DBInterface* pDBInterface = DBUtil::createInterface((*dbinfo_iter).name); - if(pDBInterface == NULL) - { - ERROR_MSG("Dbmgr::initDB(): can't create dbInterface!\n"); - return false; - } - - bool ret = DBUtil::initInterface(pDBInterface); - pDBInterface->detach(); - SAFE_RELEASE(pDBInterface); - - if(!ret) - return false; - - if (std::string("default") == (*dbinfo_iter).name) - hasDefaultInterface = true; - } - - if (!hasDefaultInterface) - { - ERROR_MSG("Dbmgr::initDB(): \"default\" dbInterface was not found! (kbengine[_defs].xml->dbmgr->databaseInterfaces)\n"); - return false; - } - - if(pUpdateDBServerLogHandler_ == NULL) - pUpdateDBServerLogHandler_ = new UpdateDBServerLogHandler(); - - return true; -} - -//------------------------------------------------------------------------------------- -void Dbmgr::finalise() -{ - SAFE_RELEASE(pUpdateDBServerLogHandler_); - - SAFE_RELEASE(pGlobalData_); - SAFE_RELEASE(pBaseAppData_); - SAFE_RELEASE(pCellAppData_); - - if (pTelnetServer_) - { - pTelnetServer_->stop(); - SAFE_RELEASE(pTelnetServer_); - } - - DBUtil::finalise(); - PythonApp::finalise(); -} - -//------------------------------------------------------------------------------------- -InterfacesHandler* Dbmgr::findBestInterfacesHandler() -{ - if (pInterfacesHandlers_.size() == 0) - return NULL; - - static size_t i = 0; - - return pInterfacesHandlers_[i++ % pInterfacesHandlers_.size()]; -} - -//------------------------------------------------------------------------------------- -void Dbmgr::onReqAllocEntityID(Network::Channel* pChannel, COMPONENT_ORDER componentType, COMPONENT_ID componentID) -{ - KBEngine::COMPONENT_TYPE ct = static_cast(componentType); - - // ȡһid IDClient - std::pair idRange = idServer_.allocRange(); - Network::Bundle* pBundle = Network::Bundle::createPoolObject(OBJECTPOOL_POINT); - - if(ct == BASEAPP_TYPE) - (*pBundle).newMessage(BaseappInterface::onReqAllocEntityID); - else - (*pBundle).newMessage(CellappInterface::onReqAllocEntityID); - - (*pBundle) << idRange.first; - (*pBundle) << idRange.second; - pChannel->send(pBundle); -} - -//------------------------------------------------------------------------------------- -void Dbmgr::onRegisterNewApp(Network::Channel* pChannel, int32 uid, std::string& username, - COMPONENT_TYPE componentType, COMPONENT_ID componentID, COMPONENT_ORDER globalorderID, COMPONENT_ORDER grouporderID, - uint32 intaddr, uint16 intport, uint32 extaddr, uint16 extport, std::string& extaddrEx) -{ - if(pChannel->isExternal()) - return; - - ServerApp::onRegisterNewApp(pChannel, uid, username, componentType, componentID, globalorderID, grouporderID, - intaddr, intport, extaddr, extport, extaddrEx); - - KBEngine::COMPONENT_TYPE tcomponentType = (KBEngine::COMPONENT_TYPE)componentType; - - COMPONENT_ORDER startGroupOrder = 1; - COMPONENT_ORDER startGlobalOrder = Components::getSingleton().getGlobalOrderLog()[getUserUID()]; - - if(grouporderID > 0) - startGroupOrder = grouporderID; - - if(globalorderID > 0) - startGlobalOrder = globalorderID; - - if(pSyncAppDatasHandler_ == NULL) - pSyncAppDatasHandler_ = new SyncAppDatasHandler(this->networkInterface()); - - // һ: - // ӵdbmgrҪȴappʼϢ - // 磺ʼentityIDԼapp˳ϢǷһbaseapp - if(tcomponentType == BASEAPP_TYPE || - tcomponentType == CELLAPP_TYPE || - tcomponentType == LOGINAPP_TYPE) - { - switch(tcomponentType) - { - case BASEAPP_TYPE: - { - if(grouporderID <= 0) - startGroupOrder = Components::getSingleton().getBaseappGroupOrderLog()[getUserUID()]; - } - break; - case CELLAPP_TYPE: - { - if(grouporderID <= 0) - startGroupOrder = Components::getSingleton().getCellappGroupOrderLog()[getUserUID()]; - } - break; - case LOGINAPP_TYPE: - if(grouporderID <= 0) - startGroupOrder = Components::getSingleton().getLoginappGroupOrderLog()[getUserUID()]; - - break; - default: - break; - } - } - - pSyncAppDatasHandler_->pushApp(componentID, startGroupOrder, startGlobalOrder); - - // baseappcellappԼעᵽbaseappcellapp - if(tcomponentType == BASEAPP_TYPE || - tcomponentType == CELLAPP_TYPE) - { - KBEngine::COMPONENT_TYPE broadcastCpTypes[2] = {BASEAPP_TYPE, CELLAPP_TYPE}; - for(int idx = 0; idx < 2; ++idx) - { - Components::COMPONENTS& cts = Components::getSingleton().getComponents(broadcastCpTypes[idx]); - Components::COMPONENTS::iterator fiter = cts.begin(); - for(; fiter != cts.end(); ++fiter) - { - if((*fiter).cid == componentID) - continue; - - Network::Bundle* pBundle = Network::Bundle::createPoolObject(OBJECTPOOL_POINT); - ENTITTAPP_COMMON_NETWORK_MESSAGE(broadcastCpTypes[idx], (*pBundle), onGetEntityAppFromDbmgr); - - if(tcomponentType == BASEAPP_TYPE) - { - BaseappInterface::onGetEntityAppFromDbmgrArgs11::staticAddToBundle((*pBundle), - uid, username, componentType, componentID, startGlobalOrder, startGroupOrder, - intaddr, intport, extaddr, extport, g_kbeSrvConfig.getConfig().externalAddress); - } - else - { - CellappInterface::onGetEntityAppFromDbmgrArgs11::staticAddToBundle((*pBundle), - uid, username, componentType, componentID, startGlobalOrder, startGroupOrder, - intaddr, intport, extaddr, extport, g_kbeSrvConfig.getConfig().externalAddress); - } - - KBE_ASSERT((*fiter).pChannel != NULL); - (*fiter).pChannel->send(pBundle); - } - } - } -} - -//------------------------------------------------------------------------------------- -void Dbmgr::onGlobalDataClientLogon(Network::Channel* pChannel, COMPONENT_TYPE componentType) -{ - if(BASEAPP_TYPE == componentType) - { - pBaseAppData_->onGlobalDataClientLogon(pChannel, componentType); - pGlobalData_->onGlobalDataClientLogon(pChannel, componentType); - } - else if(CELLAPP_TYPE == componentType) - { - pGlobalData_->onGlobalDataClientLogon(pChannel, componentType); - pCellAppData_->onGlobalDataClientLogon(pChannel, componentType); - } - else - { - ERROR_MSG(fmt::format("Dbmgr::onGlobalDataClientLogon: nonsupport {}!\n", - COMPONENT_NAME_EX(componentType))); - } -} - -//------------------------------------------------------------------------------------- -void Dbmgr::onBroadcastGlobalDataChanged(Network::Channel* pChannel, KBEngine::MemoryStream& s) -{ - uint8 dataType; - std::string key, value; - bool isDelete; - COMPONENT_TYPE componentType; - - s >> dataType; - s >> isDelete; - - s.readBlob(key); - - if(!isDelete) - { - s.readBlob(value); - } - - s >> componentType; - - switch(dataType) - { - case GlobalDataServer::GLOBAL_DATA: - if(isDelete) - pGlobalData_->del(pChannel, componentType, key); - else - pGlobalData_->write(pChannel, componentType, key, value); - break; - case GlobalDataServer::BASEAPP_DATA: - if(isDelete) - pBaseAppData_->del(pChannel, componentType, key); - else - pBaseAppData_->write(pChannel, componentType, key, value); - break; - case GlobalDataServer::CELLAPP_DATA: - if(isDelete) - pCellAppData_->del(pChannel, componentType, key); - else - pCellAppData_->write(pChannel, componentType, key, value); - break; - default: - KBE_ASSERT(false && "dataType error!\n"); - break; - }; -} - -//------------------------------------------------------------------------------------- -void Dbmgr::reqCreateAccount(Network::Channel* pChannel, KBEngine::MemoryStream& s) -{ - std::string registerName, password, datas; - uint8 uatype = 0; - - s >> registerName >> password >> uatype; - s.readBlob(datas); - - if(registerName.size() == 0) - { - ERROR_MSG("Dbmgr::reqCreateAccount: registerName is empty.\n"); - return; - } - - findBestInterfacesHandler()->createAccount(pChannel, registerName, password, datas, ACCOUNT_TYPE(uatype)); - numCreatedAccount_++; -} - -//------------------------------------------------------------------------------------- -void Dbmgr::onCreateAccountCBFromInterfaces(Network::Channel* pChannel, KBEngine::MemoryStream& s) -{ - findBestInterfacesHandler()->onCreateAccountCB(s); -} - -//------------------------------------------------------------------------------------- -void Dbmgr::onAccountLogin(Network::Channel* pChannel, KBEngine::MemoryStream& s) -{ - std::string loginName, password, datas; - s >> loginName >> password; - s.readBlob(datas); - - if(loginName.size() == 0) - { - ERROR_MSG("Dbmgr::onAccountLogin: loginName is empty.\n"); - return; - } - - findBestInterfacesHandler()->loginAccount(pChannel, loginName, password, datas); -} - -//------------------------------------------------------------------------------------- -void Dbmgr::onLoginAccountCBBFromInterfaces(Network::Channel* pChannel, KBEngine::MemoryStream& s) -{ - findBestInterfacesHandler()->onLoginAccountCB(s); -} - -//------------------------------------------------------------------------------------- -void Dbmgr::queryAccount(Network::Channel* pChannel, - std::string& accountName, - std::string& password, - bool needCheckPassword, - COMPONENT_ID componentID, - ENTITY_ID entityID, - DBID entityDBID, - uint32 ip, - uint16 port) -{ - if(accountName.size() == 0) - { - ERROR_MSG("Dbmgr::queryAccount: accountName is empty.\n"); - return; - } - - Buffered_DBTasks* pBuffered_DBTasks = - findBufferedDBTask(Dbmgr::getSingleton().selectAccountDBInterfaceName(accountName)); - - if (!pBuffered_DBTasks) - { - ERROR_MSG(fmt::format("Dbmgr::queryAccount: not found dbInterface({})!\n", - Dbmgr::getSingleton().selectAccountDBInterfaceName(accountName))); - return; - } - - pBuffered_DBTasks->addTask(new DBTaskQueryAccount(pChannel->addr(), accountName, password, needCheckPassword, - componentID, entityID, entityDBID, ip, port)); - - numQueryEntity_++; -} - -//------------------------------------------------------------------------------------- -void Dbmgr::onAccountOnline(Network::Channel* pChannel, - std::string& accountName, - COMPONENT_ID componentID, - ENTITY_ID entityID) -{ - // bufferedDBTasks_.addTask(new DBTaskAccountOnline(pChannel->addr(), - // accountName, componentID, entityID)); -} - -//------------------------------------------------------------------------------------- -void Dbmgr::onEntityOffline(Network::Channel* pChannel, DBID dbid, ENTITY_SCRIPT_UID sid, uint16 dbInterfaceIndex) -{ - Buffered_DBTasks* pBuffered_DBTasks = findBufferedDBTask(g_kbeSrvConfig.dbInterfaceIndex2dbInterfaceName(dbInterfaceIndex)); - if (!pBuffered_DBTasks) - { - ERROR_MSG(fmt::format("Dbmgr::onEntityOffline: not found dbInterfaceIndex({})!\n", dbInterfaceIndex)); - return; - } - - pBuffered_DBTasks->addTask(new DBTaskEntityOffline(pChannel->addr(), dbid, sid)); -} - -//------------------------------------------------------------------------------------- -void Dbmgr::executeRawDatabaseCommand(Network::Channel* pChannel, - KBEngine::MemoryStream& s) -{ - ENTITY_ID entityID = -1; - s >> entityID; - - uint16 dbInterfaceIndex = 0; - s >> dbInterfaceIndex; - - std::string dbInterfaceName = g_kbeSrvConfig.dbInterfaceIndex2dbInterfaceName(dbInterfaceIndex); - if (dbInterfaceName.size() == 0) - { - ERROR_MSG(fmt::format("Dbmgr::executeRawDatabaseCommand: not found dbInterface({})!\n", dbInterfaceName)); - s.done(); - return; - } - - if (entityID == -1) - { - thread::ThreadPool* pThreadPool = DBUtil::pThreadPool(dbInterfaceName); - if (!pThreadPool) - { - ERROR_MSG(fmt::format("Dbmgr::executeRawDatabaseCommand: not found pThreadPool(dbInterface={})!\n", dbInterfaceName)); - s.done(); - return; - } - - pThreadPool->addTask(new DBTaskExecuteRawDatabaseCommand(pChannel ? pChannel->addr() : Network::Address::NONE, s)); - } - else - { - Buffered_DBTasks* pBuffered_DBTasks = findBufferedDBTask(dbInterfaceName); - if (!pBuffered_DBTasks) - { - ERROR_MSG(fmt::format("Dbmgr::executeRawDatabaseCommand: not found pBuffered_DBTasks(dbInterface={})!\n", dbInterfaceName)); - s.done(); - return; - } - - pBuffered_DBTasks->addTask(new DBTaskExecuteRawDatabaseCommandByEntity(pChannel ? pChannel->addr() : Network::Address::NONE, s, entityID)); - } - - s.done(); - - ++numExecuteRawDatabaseCommand_; -} - -//------------------------------------------------------------------------------------- -PyObject* Dbmgr::__py_executeRawDatabaseCommand(PyObject* self, PyObject* args) -{ - int argCount = (int)PyTuple_Size(args); - PyObject* pycallback = NULL; - PyObject* pyDBInterfaceName = NULL; - int ret = -1; - ENTITY_ID eid = -1; - - char* data = NULL; - Py_ssize_t size; - - if (argCount == 4) - ret = PyArg_ParseTuple(args, "s#|O|i|O", &data, &size, &pycallback, &eid, &pyDBInterfaceName); - else if (argCount == 3) - ret = PyArg_ParseTuple(args, "s#|O|i", &data, &size, &pycallback, &eid); - else if (argCount == 2) - ret = PyArg_ParseTuple(args, "s#|O", &data, &size, &pycallback); - else if (argCount == 1) - ret = PyArg_ParseTuple(args, "s#", &data, &size); - - if (ret == -1) - { - PyErr_Format(PyExc_TypeError, "KBEngine::executeRawDatabaseCommand: args error!"); - PyErr_PrintEx(0); - S_Return; - } - - std::string dbInterfaceName = "default"; - if (pyDBInterfaceName) - { - dbInterfaceName = PyUnicode_AsUTF8AndSize(pyDBInterfaceName, NULL); - - if (!g_kbeSrvConfig.dbInterface(dbInterfaceName)) - { - PyErr_Format(PyExc_TypeError, "KBEngine::executeRawDatabaseCommand: args4, incorrect dbInterfaceName(%s)!", - dbInterfaceName.c_str()); - - PyErr_PrintEx(0); - S_Return; - } - } - - Dbmgr::getSingleton().executeRawDatabaseCommand(data, (uint32)size, pycallback, eid, dbInterfaceName); - S_Return; -} - -//------------------------------------------------------------------------------------- -void Dbmgr::executeRawDatabaseCommand(const char* datas, uint32 size, PyObject* pycallback, ENTITY_ID eid, const std::string& dbInterfaceName) -{ - if (datas == NULL) - { - ERROR_MSG("KBEngine::executeRawDatabaseCommand: execute error!\n"); - return; - } - - int dbInterfaceIndex = g_kbeSrvConfig.dbInterfaceName2dbInterfaceIndex(dbInterfaceName); - if (dbInterfaceIndex < 0) - { - ERROR_MSG(fmt::format("KBEngine::executeRawDatabaseCommand: not found dbInterface({})!\n", - dbInterfaceName)); - - return; - } - - //INFO_MSG(fmt::format("KBEngine::executeRawDatabaseCommand{}:{}.\n", (eid > 0 ? fmt::format("(entityID={})", eid) : ""), datas)); - - MemoryStream* pMemoryStream = MemoryStream::createPoolObject(OBJECTPOOL_POINT); - (*pMemoryStream) << eid; - (*pMemoryStream) << (uint16)dbInterfaceIndex; - (*pMemoryStream) << componentID_ << componentType_; - - CALLBACK_ID callbackID = 0; - - if (pycallback && PyCallable_Check(pycallback)) - callbackID = callbackMgr().save(pycallback); - - (*pMemoryStream) << callbackID; - (*pMemoryStream) << size; - (*pMemoryStream).append(datas, size); - executeRawDatabaseCommand(NULL, *pMemoryStream); - MemoryStream::reclaimPoolObject(pMemoryStream); -} - -//------------------------------------------------------------------------------------- -void Dbmgr::onExecuteRawDatabaseCommandCB(KBEngine::MemoryStream& s) -{ - std::string err; - CALLBACK_ID callbackID = 0; - uint32 nrows = 0; - uint32 nfields = 0; - uint64 affectedRows = 0; - uint64 lastInsertID = 0; - - PyObject* pResultSet = NULL; - PyObject* pAffectedRows = NULL; - PyObject* pLastInsertID = NULL; - PyObject* pErrorMsg = NULL; - - s >> callbackID; - s >> err; - - if (err.size() <= 0) - { - s >> nfields; - - pErrorMsg = Py_None; - Py_INCREF(pErrorMsg); - - if (nfields > 0) - { - pAffectedRows = Py_None; - Py_INCREF(pAffectedRows); - - pLastInsertID = Py_None; - Py_INCREF(pLastInsertID); - - s >> nrows; - - pResultSet = PyList_New(nrows); - for (uint32 i = 0; i < nrows; ++i) - { - PyObject* pRow = PyList_New(nfields); - for (uint32 j = 0; j < nfields; ++j) - { - std::string cell; - s.readBlob(cell); - - PyObject* pCell = NULL; - - if (cell == "KBE_QUERY_DB_NULL") - { - Py_INCREF(Py_None); - pCell = Py_None; - } - else - { - pCell = PyBytes_FromStringAndSize(cell.data(), cell.length()); - } - - PyList_SET_ITEM(pRow, j, pCell); - } - - PyList_SET_ITEM(pResultSet, i, pRow); - } - } - else - { - pResultSet = Py_None; - Py_INCREF(pResultSet); - - pErrorMsg = Py_None; - Py_INCREF(pErrorMsg); - - s >> affectedRows; - - pAffectedRows = PyLong_FromUnsignedLongLong(affectedRows); - - s >> lastInsertID; - pLastInsertID = PyLong_FromUnsignedLongLong(lastInsertID); - } - } - else - { - pResultSet = Py_None; - Py_INCREF(pResultSet); - - pErrorMsg = PyUnicode_FromString(err.c_str()); - - pAffectedRows = Py_None; - Py_INCREF(pAffectedRows); - - pLastInsertID = Py_None; - Py_INCREF(pLastInsertID); - } - - s.done(); - - //DEBUG_MSG(fmt::format("Cellapp::onExecuteRawDatabaseCommandCB: nrows={}, nfields={}, err={}.\n", - // nrows, nfields, err.c_str())); - - if (callbackID > 0) - { - SCOPED_PROFILE(SCRIPTCALL_PROFILE); - - PyObjectPtr pyfunc = pyCallbackMgr_.take(callbackID); - if (pyfunc != NULL) - { - PyObject* pyResult = PyObject_CallFunction(pyfunc.get(), - const_cast("OOOO"), - pResultSet, pAffectedRows, pLastInsertID, pErrorMsg); - - if (pyResult != NULL) - Py_DECREF(pyResult); - else - SCRIPT_ERROR_CHECK(); - } - else - { - ERROR_MSG(fmt::format("Cellapp::onExecuteRawDatabaseCommandCB: not found callback:{}.\n", - callbackID)); - } - } - - Py_XDECREF(pResultSet); - Py_XDECREF(pAffectedRows); - Py_XDECREF(pLastInsertID); - Py_XDECREF(pErrorMsg); -} - -//------------------------------------------------------------------------------------- -void Dbmgr::writeEntity(Network::Channel* pChannel, - KBEngine::MemoryStream& s) -{ - ENTITY_ID eid; - DBID entityDBID; - COMPONENT_ID componentID; - uint16 dbInterfaceIndex; - - s >> componentID >> eid >> entityDBID >> dbInterfaceIndex; - - Buffered_DBTasks* pBuffered_DBTasks = findBufferedDBTask(g_kbeSrvConfig.dbInterfaceIndex2dbInterfaceName(dbInterfaceIndex)); - if (!pBuffered_DBTasks) - { - ERROR_MSG(fmt::format("Dbmgr::writeEntity: not found dbInterfaceIndex({})!\n", dbInterfaceIndex)); - s.done(); - return; - } - - pBuffered_DBTasks->addTask(new DBTaskWriteEntity(pChannel->addr(), componentID, eid, entityDBID, s)); - s.done(); - - ++numWrittenEntity_; -} - -//------------------------------------------------------------------------------------- -void Dbmgr::removeEntity(Network::Channel* pChannel, KBEngine::MemoryStream& s) -{ - ENTITY_ID eid; - DBID entityDBID; - COMPONENT_ID componentID; - uint16 dbInterfaceIndex; - - s >> dbInterfaceIndex >> componentID >> eid >> entityDBID; - KBE_ASSERT(entityDBID > 0); - - Buffered_DBTasks* pBuffered_DBTasks = findBufferedDBTask(g_kbeSrvConfig.dbInterfaceIndex2dbInterfaceName(dbInterfaceIndex)); - if (!pBuffered_DBTasks) - { - ERROR_MSG(fmt::format("Dbmgr::removeEntity: not found dbInterfaceIndex({})!\n", dbInterfaceIndex)); - s.done(); - return; - } - - pBuffered_DBTasks->addTask(new DBTaskRemoveEntity(pChannel->addr(), - componentID, eid, entityDBID, s)); - - s.done(); - - ++numRemovedEntity_; -} - -//------------------------------------------------------------------------------------- -void Dbmgr::entityAutoLoad(Network::Channel* pChannel, KBEngine::MemoryStream& s) -{ - COMPONENT_ID componentID; - ENTITY_SCRIPT_UID entityType; - ENTITY_ID start; - ENTITY_ID end; - uint16 dbInterfaceIndex = 0; - - s >> dbInterfaceIndex >> componentID >> entityType >> start >> end; - - DBUtil::pThreadPool(g_kbeSrvConfig.dbInterfaceIndex2dbInterfaceName(dbInterfaceIndex))-> - addTask(new DBTaskEntityAutoLoad(pChannel->addr(), componentID, entityType, start, end)); -} - -//------------------------------------------------------------------------------------- -void Dbmgr::deleteEntityByDBID(Network::Channel* pChannel, KBEngine::MemoryStream& s) -{ - COMPONENT_ID componentID; - ENTITY_SCRIPT_UID sid; - CALLBACK_ID callbackID = 0; - DBID entityDBID; - uint16 dbInterfaceIndex = 0; - - s >> dbInterfaceIndex >> componentID >> entityDBID >> callbackID >> sid; - KBE_ASSERT(entityDBID > 0); - - DBUtil::pThreadPool(g_kbeSrvConfig.dbInterfaceIndex2dbInterfaceName(dbInterfaceIndex))-> - addTask(new DBTaskDeleteEntityByDBID(pChannel->addr(), componentID, entityDBID, callbackID, sid)); -} - -//------------------------------------------------------------------------------------- -void Dbmgr::lookUpEntityByDBID(Network::Channel* pChannel, KBEngine::MemoryStream& s) -{ - COMPONENT_ID componentID; - ENTITY_SCRIPT_UID sid; - CALLBACK_ID callbackID = 0; - DBID entityDBID; - uint16 dbInterfaceIndex = 0; - - s >> dbInterfaceIndex >> componentID >> entityDBID >> callbackID >> sid; - KBE_ASSERT(entityDBID > 0); - - DBUtil::pThreadPool(g_kbeSrvConfig.dbInterfaceIndex2dbInterfaceName(dbInterfaceIndex))-> - addTask(new DBTaskLookUpEntityByDBID(pChannel->addr(), componentID, entityDBID, callbackID, sid)); -} - -//------------------------------------------------------------------------------------- -void Dbmgr::queryEntity(Network::Channel* pChannel, uint16 dbInterfaceIndex, COMPONENT_ID componentID, int8 queryMode, DBID dbid, - std::string& entityType, CALLBACK_ID callbackID, ENTITY_ID entityID) -{ - bufferedDBTasksMaps_[g_kbeSrvConfig.dbInterfaceIndex2dbInterfaceName(dbInterfaceIndex)]. - addTask(new DBTaskQueryEntity(pChannel->addr(), queryMode, entityType, dbid, componentID, callbackID, entityID)); - - numQueryEntity_++; -} - -//------------------------------------------------------------------------------------- -void Dbmgr::syncEntityStreamTemplate(Network::Channel* pChannel, KBEngine::MemoryStream& s) -{ - int rpos = s.rpos(); - EntityTables::ENTITY_TABLES_MAP::iterator iter = EntityTables::sEntityTables.begin(); - for (; iter != EntityTables::sEntityTables.end(); ++iter) - { - KBEAccountTable* pTable = - static_cast(iter->second.findKBETable(KBE_TABLE_PERFIX "_accountinfos")); - - KBE_ASSERT(pTable); - - s.rpos(rpos); - pTable->accountDefMemoryStream(s); - } - - s.done(); -} - -//------------------------------------------------------------------------------------- -void Dbmgr::charge(Network::Channel* pChannel, KBEngine::MemoryStream& s) -{ - findBestInterfacesHandler()->charge(pChannel, s); -} - -//------------------------------------------------------------------------------------- -void Dbmgr::onChargeCB(Network::Channel* pChannel, KBEngine::MemoryStream& s) -{ - findBestInterfacesHandler()->onChargeCB(s); -} - -//------------------------------------------------------------------------------------- -void Dbmgr::eraseClientReq(Network::Channel* pChannel, std::string& logkey) -{ - std::vector::iterator iter = pInterfacesHandlers_.begin(); - for(; iter != pInterfacesHandlers_.end(); ++iter) - (*iter)->eraseClientReq(pChannel, logkey); -} - -//------------------------------------------------------------------------------------- -void Dbmgr::accountActivate(Network::Channel* pChannel, std::string& scode) -{ - INFO_MSG(fmt::format("Dbmgr::accountActivate: code={}.\n", scode)); - findBestInterfacesHandler()->accountActivate(pChannel, scode); -} - -//------------------------------------------------------------------------------------- -void Dbmgr::accountReqResetPassword(Network::Channel* pChannel, std::string& accountName) -{ - INFO_MSG(fmt::format("Dbmgr::accountReqResetPassword: accountName={}.\n", accountName)); - findBestInterfacesHandler()->accountReqResetPassword(pChannel, accountName); -} - -//------------------------------------------------------------------------------------- -void Dbmgr::accountResetPassword(Network::Channel* pChannel, std::string& accountName, std::string& newpassword, std::string& code) -{ - INFO_MSG(fmt::format("Dbmgr::accountResetPassword: accountName={}.\n", accountName)); - findBestInterfacesHandler()->accountResetPassword(pChannel, accountName, newpassword, code); -} - -//------------------------------------------------------------------------------------- -void Dbmgr::accountReqBindMail(Network::Channel* pChannel, ENTITY_ID entityID, std::string& accountName, - std::string& password, std::string& email) -{ - INFO_MSG(fmt::format("Dbmgr::accountReqBindMail: accountName={}, email={}.\n", accountName, email)); - findBestInterfacesHandler()->accountReqBindMail(pChannel, entityID, accountName, password, email); -} - -//------------------------------------------------------------------------------------- -void Dbmgr::accountBindMail(Network::Channel* pChannel, std::string& username, std::string& scode) -{ - INFO_MSG(fmt::format("Dbmgr::accountBindMail: username={}, scode={}.\n", username, scode)); - findBestInterfacesHandler()->accountBindMail(pChannel, username, scode); -} - -//------------------------------------------------------------------------------------- -void Dbmgr::accountNewPassword(Network::Channel* pChannel, ENTITY_ID entityID, std::string& accountName, - std::string& password, std::string& newpassword) -{ - INFO_MSG(fmt::format("Dbmgr::accountNewPassword: accountName={}.\n", accountName)); - findBestInterfacesHandler()->accountNewPassword(pChannel, entityID, accountName, password, newpassword); -} - -//------------------------------------------------------------------------------------- -std::string Dbmgr::selectAccountDBInterfaceName(const std::string& name) -{ - std::string dbInterfaceName = "default"; - - // ɽű - SCOPED_PROFILE(SCRIPTCALL_PROFILE); - PyObject* pyResult = PyObject_CallMethod(getEntryScript().get(), - const_cast("onSelectAccountDBInterface"), - const_cast("s"), - name.c_str()); - - if (pyResult != NULL) - { - dbInterfaceName = PyUnicode_AsUTF8AndSize(pyResult, NULL); - Py_DECREF(pyResult); - } - else - { - SCRIPT_ERROR_CHECK(); - } - - if (dbInterfaceName == "" || g_kbeSrvConfig.dbInterface(dbInterfaceName) == NULL) - { - ERROR_MSG(fmt::format("Dbmgr::selectAccountDBInterfaceName: not found dbInterface({}), accountName={}.\n", dbInterfaceName, name)); - return "default"; - } - - return dbInterfaceName; -} - -//------------------------------------------------------------------------------------- -void Dbmgr::onChannelDeregister(Network::Channel * pChannel) -{ - // app - if (pChannel->isInternal()) - { - Components::ComponentInfos* cinfo = Components::getSingleton().findComponent(pChannel); - if (cinfo) - { - if (cinfo->componentType == BASEAPP_TYPE) - { - loseBaseappts_[cinfo->cid] = timestamp() + uint64(60 * stampsPerSecond()); - WARNING_MSG(fmt::format("Dbmgr::onChannelDeregister(): If the process cannot be resumed, the entitylog(baseapp={}) will be cleaned up after 60 seconds!\n", cinfo->cid)); - } - } - } - - ServerApp::onChannelDeregister(pChannel); -} - -//------------------------------------------------------------------------------------- -} +// Copyright 2008-2018 Yolo Technologies, Inc. All Rights Reserved. https://www.comblockengine.com + + +#include "dbmgr.h" +#include "dbmgr_interface.h" +#include "dbtasks.h" +#include "profile.h" +#include "interfaces_handler.h" +#include "sync_app_datas_handler.h" +#include "update_dblog_handler.h" +#include "db_mysql/kbe_table_mysql.h" +#include "network/common.h" +#include "network/tcp_packet.h" +#include "network/udp_packet.h" +#include "network/message_handler.h" +#include "thread/threadpool.h" +#include "server/components.h" +#include "server/telnet_server.h" +#include "db_interface/db_interface.h" +#include "db_mysql/db_interface_mysql.h" +#include "entitydef/scriptdef_module.h" +#include "entitydef/py_entitydef.h" + +#include "baseapp/baseapp_interface.h" +#include "cellapp/cellapp_interface.h" +#include "baseappmgr/baseappmgr_interface.h" +#include "cellappmgr/cellappmgr_interface.h" +#include "loginapp/loginapp_interface.h" + +namespace KBEngine{ + +ServerConfig g_serverConfig; +KBE_SINGLETON_INIT(Dbmgr); + +//------------------------------------------------------------------------------------- +Dbmgr::Dbmgr(Network::EventDispatcher& dispatcher, + Network::NetworkInterface& ninterface, + COMPONENT_TYPE componentType, + COMPONENT_ID componentID): + PythonApp(dispatcher, ninterface, componentType, componentID), + loopCheckTimerHandle_(), + mainProcessTimer_(), + idServer_(1, 1024), + pGlobalData_(NULL), + pBaseAppData_(NULL), + pCellAppData_(NULL), + bufferedDBTasksMaps_(), + numWrittenEntity_(0), + numRemovedEntity_(0), + numQueryEntity_(0), + numExecuteRawDatabaseCommand_(0), + numCreatedAccount_(0), + pInterfacesHandlers_(), + pSyncAppDatasHandler_(NULL), + pUpdateDBServerLogHandler_(NULL), + pTelnetServer_(NULL), + loseBaseappts_() +{ + KBEngine::Network::MessageHandlers::pMainMessageHandlers = &DbmgrInterface::messageHandlers; +} + +//------------------------------------------------------------------------------------- +Dbmgr::~Dbmgr() +{ + loopCheckTimerHandle_.cancel(); + mainProcessTimer_.cancel(); + KBEngine::sleep(300); + + for (std::vector::iterator iter = pInterfacesHandlers_.begin(); iter != pInterfacesHandlers_.end(); ++iter) + { + SAFE_RELEASE((*iter)); + } +} + +//------------------------------------------------------------------------------------- +ShutdownHandler::CAN_SHUTDOWN_STATE Dbmgr::canShutdown() +{ + if (getEntryScript().get() && PyObject_HasAttrString(getEntryScript().get(), "onReadyForShutDown") > 0) + { + // нű + PyObject* pyResult = PyObject_CallMethod(getEntryScript().get(), + const_cast("onReadyForShutDown"), + const_cast("")); + + if (pyResult != NULL) + { + bool isReady = (pyResult == Py_True); + Py_DECREF(pyResult); + + if (!isReady) + return ShutdownHandler::CAN_SHUTDOWN_STATE_USER_FALSE; + } + else + { + SCRIPT_ERROR_CHECK(); + return ShutdownHandler::CAN_SHUTDOWN_STATE_USER_FALSE; + } + } + + KBEUnordered_map::iterator bditer = bufferedDBTasksMaps_.begin(); + for (; bditer != bufferedDBTasksMaps_.end(); ++bditer) + { + if (bditer->second.size() > 0) + { + thread::ThreadPool* pThreadPool = DBUtil::pThreadPool(bditer->first); + KBE_ASSERT(pThreadPool); + + INFO_MSG(fmt::format("Dbmgr::canShutdown(): Wait for the task to complete, dbInterface={}, tasks{}=[{}], threads={}/{}, threadpoolDestroyed={}!\n", + bditer->first, bditer->second.size(), bditer->second.getTasksinfos(), (pThreadPool->currentThreadCount() - pThreadPool->currentFreeThreadCount()), + pThreadPool->currentThreadCount(), pThreadPool->isDestroyed())); + + return ShutdownHandler::CAN_SHUTDOWN_STATE_FALSE; + } + } + + Components::COMPONENTS& cellapp_components = Components::getSingleton().getComponents(CELLAPP_TYPE); + if (cellapp_components.size() > 0) + { + std::string s; + for (size_t i = 0; i 0) + { + std::string s; + for (size_t i = 0; i("onDBMgrShutDown"), false); +} + +//------------------------------------------------------------------------------------- +void Dbmgr::onShutdownEnd() +{ + PythonApp::onShutdownEnd(); +} + +//------------------------------------------------------------------------------------- +bool Dbmgr::initializeWatcher() +{ + WATCH_OBJECT("numWrittenEntity", numWrittenEntity_); + WATCH_OBJECT("numRemovedEntity", numRemovedEntity_); + WATCH_OBJECT("numQueryEntity", numQueryEntity_); + WATCH_OBJECT("numExecuteRawDatabaseCommand", numExecuteRawDatabaseCommand_); + WATCH_OBJECT("numCreatedAccount", numCreatedAccount_); + + KBEUnordered_map::iterator bditer = bufferedDBTasksMaps_.begin(); + for (; bditer != bufferedDBTasksMaps_.end(); ++bditer) + { + WATCH_OBJECT(fmt::format("DBThreadPool/{}/dbid_tasksSize", bditer->first).c_str(), &bditer->second, &Buffered_DBTasks::dbid_tasksSize); + WATCH_OBJECT(fmt::format("DBThreadPool/{}/entityid_tasksSize", bditer->first).c_str(), &bditer->second, &Buffered_DBTasks::entityid_tasksSize); + WATCH_OBJECT(fmt::format("DBThreadPool/{}/printBuffered_dbid", bditer->first).c_str(), &bditer->second, &Buffered_DBTasks::printBuffered_dbid); + WATCH_OBJECT(fmt::format("DBThreadPool/{}/printBuffered_entityID", bditer->first).c_str(), &bditer->second, &Buffered_DBTasks::printBuffered_entityID); + } + + return ServerApp::initializeWatcher() && DBUtil::initializeWatcher(); +} + +//------------------------------------------------------------------------------------- +bool Dbmgr::run() +{ + return PythonApp::run(); +} + +//------------------------------------------------------------------------------------- +void Dbmgr::handleTimeout(TimerHandle handle, void * arg) +{ + PythonApp::handleTimeout(handle, arg); + + switch (reinterpret_cast(arg)) + { + case TIMEOUT_TICK: + this->handleMainTick(); + break; + case TIMEOUT_CHECK_STATUS: + this->handleCheckStatusTick(); + break; + default: + break; + } +} + +//------------------------------------------------------------------------------------- +void Dbmgr::handleMainTick() +{ + AUTO_SCOPED_PROFILE("mainTick"); + + // time_t t = ::time(NULL); + // static int kbeTime = 0; + // DEBUG_MSG(fmt::format("Dbmgr::handleGameTick[{}]:{}\n", t, ++kbeTime)); + + threadPool_.onMainThreadTick(); + DBUtil::handleMainTick(); + networkInterface().processChannels(&DbmgrInterface::messageHandlers); +} + +//------------------------------------------------------------------------------------- +void Dbmgr::handleCheckStatusTick() +{ + // 鶪ʧ̣һʱ֮Ȼ޷֣Ҫݿentitylog + if (loseBaseappts_.size() > 0) + { + std::map::iterator iter = loseBaseappts_.begin(); + for (; iter != loseBaseappts_.end();) + { + if (timestamp() > iter->second) + { + Components::ComponentInfos* cinfo = Components::getSingleton().findComponent(iter->first); + if (!cinfo) + { + ENGINE_COMPONENT_INFO& dbcfg = g_kbeSrvConfig.getDBMgr(); + std::vector::iterator dbinfo_iter = dbcfg.dbInterfaceInfos.begin(); + for (; dbinfo_iter != dbcfg.dbInterfaceInfos.end(); ++dbinfo_iter) + { + std::string dbInterfaceName = dbinfo_iter->name; + + DBUtil::pThreadPool(dbInterfaceName)-> + addTask(new DBTaskEraseBaseappEntityLog(iter->first)); + } + } + + loseBaseappts_.erase(iter++); + } + else + { + ++iter; + } + } + } +} + +//------------------------------------------------------------------------------------- +bool Dbmgr::initializeBegin() +{ + idServer_.set_range_step(g_kbeSrvConfig.getDBMgr().ids_increasing_range); + return true; +} + +//------------------------------------------------------------------------------------- +bool Dbmgr::inInitialize() +{ + // ʼչģ + // assets/scripts/ + if (!PythonApp::inInitialize()) + return false; + + std::vector scriptBaseTypes; + if(!EntityDef::initialize(scriptBaseTypes, componentType_)){ + return false; + } + + return true; +} + +//------------------------------------------------------------------------------------- +bool Dbmgr::initializeEnd() +{ + PythonApp::initializeEnd(); + + // һtimer ÿһЩ״̬ + loopCheckTimerHandle_ = this->dispatcher().addTimer(1000000, this, + reinterpret_cast(TIMEOUT_CHECK_STATUS)); + + mainProcessTimer_ = this->dispatcher().addTimer(1000000 / 50, this, + reinterpret_cast(TIMEOUT_TICK)); + + // globalData, baseAppData, cellAppData֧ + pGlobalData_ = new GlobalDataServer(GlobalDataServer::GLOBAL_DATA); + pBaseAppData_ = new GlobalDataServer(GlobalDataServer::BASEAPP_DATA); + pCellAppData_ = new GlobalDataServer(GlobalDataServer::CELLAPP_DATA); + pGlobalData_->addConcernComponentType(CELLAPP_TYPE); + pGlobalData_->addConcernComponentType(BASEAPP_TYPE); + pBaseAppData_->addConcernComponentType(BASEAPP_TYPE); + pCellAppData_->addConcernComponentType(CELLAPP_TYPE); + + INFO_MSG(fmt::format("Dbmgr::initializeEnd: digest({})\n", + EntityDef::md5().getDigestStr())); + + SCOPED_PROFILE(SCRIPTCALL_PROFILE); + + // нű + PyObject* pyResult = PyObject_CallMethod(getEntryScript().get(), + const_cast("onDBMgrReady"), + const_cast("")); + + if(pyResult != NULL) + Py_DECREF(pyResult); + else + SCRIPT_ERROR_CHECK(); + + pTelnetServer_ = new TelnetServer(&this->dispatcher(), &this->networkInterface()); + pTelnetServer_->pScript(&this->getScript()); + + bool ret = pTelnetServer_->start(g_kbeSrvConfig.getDBMgr().telnet_passwd, + g_kbeSrvConfig.getDBMgr().telnet_deflayer, + g_kbeSrvConfig.getDBMgr().telnet_port); + + Components::getSingleton().extraData4(pTelnetServer_->port()); + + return ret && initInterfacesHandler() && initDB(); +} + +//------------------------------------------------------------------------------------- +bool Dbmgr::installPyModules() +{ + return PythonApp::installPyModules() && script::entitydef::installModule("EntityDef"); +} + +//------------------------------------------------------------------------------------- +bool Dbmgr::uninstallPyModules() +{ + return script::entitydef::uninstallModule() && PythonApp::uninstallPyModules(); +} + +//------------------------------------------------------------------------------------- +void Dbmgr::onInstallPyModules() +{ + PyObject * module = getScript().getModule(); + + for (int i = 0; i < SERVER_ERR_MAX; i++) + { + if(PyModule_AddIntConstant(module, SERVER_ERR_STR[i], i)) + { + ERROR_MSG( fmt::format("Dbmgr::onInstallPyModules: Unable to set KBEngine.{}.\n", SERVER_ERR_STR[i])); + } + } + + APPEND_SCRIPT_MODULE_METHOD(module, executeRawDatabaseCommand, __py_executeRawDatabaseCommand, METH_VARARGS, 0); +} + +//------------------------------------------------------------------------------------- +bool Dbmgr::initInterfacesHandler() +{ + std::vector< Network::Address > addresses = g_kbeSrvConfig.interfacesAddrs(); + std::string type = addresses.size() == 0 ? "dbmgr" : "interfaces"; + + if (type == "dbmgr") + { + InterfacesHandler* pInterfacesHandler = InterfacesHandlerFactory::create(type); + + INFO_MSG(fmt::format("Dbmgr::initInterfacesHandler: interfaces addr({}), accountType:({}), chargeType:({}).\n", + Network::Address::NONE.c_str(), + type, + type)); + + if (!pInterfacesHandler->initialize()) + return false; + + pInterfacesHandlers_.push_back(pInterfacesHandler); + } + else + { + std::vector< Network::Address >::iterator iter = addresses.begin(); + for (; iter != addresses.end(); ++iter) + { + InterfacesHandler* pInterfacesHandler = InterfacesHandlerFactory::create(type); + + const Network::Address& addr = (*iter); + + INFO_MSG(fmt::format("Dbmgr::initInterfacesHandler: interfaces addr({}), accountType:({}), chargeType:({}).\n", + addr.c_str(), + type, + type)); + + ((InterfacesHandler_Interfaces*)pInterfacesHandler)->setAddr(addr); + + if (!pInterfacesHandler->initialize()) + return false; + + pInterfacesHandlers_.push_back(pInterfacesHandler); + } + } + + return pInterfacesHandlers_.size() > 0; +} + +//------------------------------------------------------------------------------------- +bool Dbmgr::initDB() +{ + ScriptDefModule* pModule = EntityDef::findScriptModule(DBUtil::accountScriptName()); + if(pModule == NULL) + { + ERROR_MSG(fmt::format("Dbmgr::initDB(): not found account script[{}]!\n", + DBUtil::accountScriptName())); + + return false; + } + + ENGINE_COMPONENT_INFO& dbcfg = g_kbeSrvConfig.getDBMgr(); + if (dbcfg.dbInterfaceInfos.size() == 0) + { + ERROR_MSG(fmt::format("DBUtil::initialize: not found dbInterface! (kbengine[_defs].xml->dbmgr->databaseInterfaces)\n")); + return false; + } + + if (!DBUtil::initialize()) + { + ERROR_MSG("Dbmgr::initDB(): can't initialize dbInterface!\n"); + return false; + } + + bool hasDefaultInterface = false; + + std::vector::iterator dbinfo_iter = dbcfg.dbInterfaceInfos.begin(); + for (; dbinfo_iter != dbcfg.dbInterfaceInfos.end(); ++dbinfo_iter) + { + Buffered_DBTasks buffered_DBTasks; + bufferedDBTasksMaps_.insert(std::make_pair((*dbinfo_iter).name, buffered_DBTasks)); + BUFFERED_DBTASKS_MAP::iterator buffered_DBTasks_iter = bufferedDBTasksMaps_.find((*dbinfo_iter).name); + buffered_DBTasks_iter->second.dbInterfaceName((*dbinfo_iter).name); + } + + for (dbinfo_iter = dbcfg.dbInterfaceInfos.begin(); dbinfo_iter != dbcfg.dbInterfaceInfos.end(); ++dbinfo_iter) + { + DBInterface* pDBInterface = DBUtil::createInterface((*dbinfo_iter).name); + if(pDBInterface == NULL) + { + ERROR_MSG("Dbmgr::initDB(): can't create dbInterface!\n"); + return false; + } + + bool ret = DBUtil::initInterface(pDBInterface); + pDBInterface->detach(); + SAFE_RELEASE(pDBInterface); + + if(!ret) + return false; + + if (std::string("default") == (*dbinfo_iter).name) + hasDefaultInterface = true; + } + + if (!hasDefaultInterface) + { + ERROR_MSG("Dbmgr::initDB(): \"default\" dbInterface was not found! (kbengine[_defs].xml->dbmgr->databaseInterfaces)\n"); + return false; + } + + if(pUpdateDBServerLogHandler_ == NULL) + pUpdateDBServerLogHandler_ = new UpdateDBServerLogHandler(); + + return true; +} + +//------------------------------------------------------------------------------------- +void Dbmgr::finalise() +{ + SAFE_RELEASE(pUpdateDBServerLogHandler_); + + SAFE_RELEASE(pGlobalData_); + SAFE_RELEASE(pBaseAppData_); + SAFE_RELEASE(pCellAppData_); + + if (pTelnetServer_) + { + pTelnetServer_->stop(); + SAFE_RELEASE(pTelnetServer_); + } + + DBUtil::finalise(); + PythonApp::finalise(); +} + +//------------------------------------------------------------------------------------- +InterfacesHandler* Dbmgr::findBestInterfacesHandler() +{ + if (pInterfacesHandlers_.size() == 0) + return NULL; + + static size_t i = 0; + + return pInterfacesHandlers_[i++ % pInterfacesHandlers_.size()]; +} + +//------------------------------------------------------------------------------------- +void Dbmgr::onReqAllocEntityID(Network::Channel* pChannel, COMPONENT_ORDER componentType, COMPONENT_ID componentID) +{ + KBEngine::COMPONENT_TYPE ct = static_cast(componentType); + + // ȡһid IDClient + std::pair idRange = idServer_.allocRange(); + Network::Bundle* pBundle = Network::Bundle::createPoolObject(OBJECTPOOL_POINT); + + if(ct == BASEAPP_TYPE) + (*pBundle).newMessage(BaseappInterface::onReqAllocEntityID); + else + (*pBundle).newMessage(CellappInterface::onReqAllocEntityID); + + (*pBundle) << idRange.first; + (*pBundle) << idRange.second; + pChannel->send(pBundle); +} + +//------------------------------------------------------------------------------------- +void Dbmgr::onRegisterNewApp(Network::Channel* pChannel, int32 uid, std::string& username, + COMPONENT_TYPE componentType, COMPONENT_ID componentID, COMPONENT_ORDER globalorderID, COMPONENT_ORDER grouporderID, + uint32 intaddr, uint16 intport, uint32 extaddr, uint16 extport, std::string& extaddrEx) +{ + if(pChannel->isExternal()) + return; + + ServerApp::onRegisterNewApp(pChannel, uid, username, componentType, componentID, globalorderID, grouporderID, + intaddr, intport, extaddr, extport, extaddrEx); + + KBEngine::COMPONENT_TYPE tcomponentType = (KBEngine::COMPONENT_TYPE)componentType; + + COMPONENT_ORDER startGroupOrder = 1; + COMPONENT_ORDER startGlobalOrder = Components::getSingleton().getGlobalOrderLog()[getUserUID()]; + + if(grouporderID > 0) + startGroupOrder = grouporderID; + + if(globalorderID > 0) + startGlobalOrder = globalorderID; + + if(pSyncAppDatasHandler_ == NULL) + pSyncAppDatasHandler_ = new SyncAppDatasHandler(this->networkInterface()); + + // һ: + // ӵdbmgrҪȴappʼϢ + // 磺ʼentityIDԼapp˳ϢǷһbaseapp + if(tcomponentType == BASEAPP_TYPE || + tcomponentType == CELLAPP_TYPE || + tcomponentType == LOGINAPP_TYPE) + { + switch(tcomponentType) + { + case BASEAPP_TYPE: + { + if(grouporderID <= 0) + startGroupOrder = Components::getSingleton().getBaseappGroupOrderLog()[getUserUID()]; + } + break; + case CELLAPP_TYPE: + { + if(grouporderID <= 0) + startGroupOrder = Components::getSingleton().getCellappGroupOrderLog()[getUserUID()]; + } + break; + case LOGINAPP_TYPE: + if(grouporderID <= 0) + startGroupOrder = Components::getSingleton().getLoginappGroupOrderLog()[getUserUID()]; + + break; + default: + break; + } + } + + pSyncAppDatasHandler_->pushApp(componentID, startGroupOrder, startGlobalOrder); + + // baseappcellappԼעᵽbaseappcellapp + if(tcomponentType == BASEAPP_TYPE || + tcomponentType == CELLAPP_TYPE) + { + KBEngine::COMPONENT_TYPE broadcastCpTypes[2] = {BASEAPP_TYPE, CELLAPP_TYPE}; + for(int idx = 0; idx < 2; ++idx) + { + Components::COMPONENTS& cts = Components::getSingleton().getComponents(broadcastCpTypes[idx]); + Components::COMPONENTS::iterator fiter = cts.begin(); + for(; fiter != cts.end(); ++fiter) + { + if((*fiter).cid == componentID) + continue; + + Network::Bundle* pBundle = Network::Bundle::createPoolObject(OBJECTPOOL_POINT); + ENTITTAPP_COMMON_NETWORK_MESSAGE(broadcastCpTypes[idx], (*pBundle), onGetEntityAppFromDbmgr); + + if(tcomponentType == BASEAPP_TYPE) + { + BaseappInterface::onGetEntityAppFromDbmgrArgs11::staticAddToBundle((*pBundle), + uid, username, componentType, componentID, startGlobalOrder, startGroupOrder, + intaddr, intport, extaddr, extport, g_kbeSrvConfig.getConfig().externalAddress); + } + else + { + CellappInterface::onGetEntityAppFromDbmgrArgs11::staticAddToBundle((*pBundle), + uid, username, componentType, componentID, startGlobalOrder, startGroupOrder, + intaddr, intport, extaddr, extport, g_kbeSrvConfig.getConfig().externalAddress); + } + + KBE_ASSERT((*fiter).pChannel != NULL); + (*fiter).pChannel->send(pBundle); + } + } + } +} + +//------------------------------------------------------------------------------------- +void Dbmgr::onGlobalDataClientLogon(Network::Channel* pChannel, COMPONENT_TYPE componentType) +{ + if(BASEAPP_TYPE == componentType) + { + pBaseAppData_->onGlobalDataClientLogon(pChannel, componentType); + pGlobalData_->onGlobalDataClientLogon(pChannel, componentType); + } + else if(CELLAPP_TYPE == componentType) + { + pGlobalData_->onGlobalDataClientLogon(pChannel, componentType); + pCellAppData_->onGlobalDataClientLogon(pChannel, componentType); + } + else + { + ERROR_MSG(fmt::format("Dbmgr::onGlobalDataClientLogon: nonsupport {}!\n", + COMPONENT_NAME_EX(componentType))); + } +} + +//------------------------------------------------------------------------------------- +void Dbmgr::onBroadcastGlobalDataChanged(Network::Channel* pChannel, KBEngine::MemoryStream& s) +{ + uint8 dataType; + std::string key, value; + bool isDelete; + COMPONENT_TYPE componentType; + + s >> dataType; + s >> isDelete; + + s.readBlob(key); + + if(!isDelete) + { + s.readBlob(value); + } + + s >> componentType; + + switch(dataType) + { + case GlobalDataServer::GLOBAL_DATA: + if(isDelete) + pGlobalData_->del(pChannel, componentType, key); + else + pGlobalData_->write(pChannel, componentType, key, value); + break; + case GlobalDataServer::BASEAPP_DATA: + if(isDelete) + pBaseAppData_->del(pChannel, componentType, key); + else + pBaseAppData_->write(pChannel, componentType, key, value); + break; + case GlobalDataServer::CELLAPP_DATA: + if(isDelete) + pCellAppData_->del(pChannel, componentType, key); + else + pCellAppData_->write(pChannel, componentType, key, value); + break; + default: + KBE_ASSERT(false && "dataType error!\n"); + break; + }; +} + +//------------------------------------------------------------------------------------- +void Dbmgr::reqCreateAccount(Network::Channel* pChannel, KBEngine::MemoryStream& s) +{ + std::string registerName, password, datas; + uint8 uatype = 0; + + s >> registerName >> password >> uatype; + s.readBlob(datas); + + if(registerName.size() == 0) + { + ERROR_MSG("Dbmgr::reqCreateAccount: registerName is empty.\n"); + return; + } + + findBestInterfacesHandler()->createAccount(pChannel, registerName, password, datas, ACCOUNT_TYPE(uatype)); + numCreatedAccount_++; +} + +//------------------------------------------------------------------------------------- +void Dbmgr::onCreateAccountCBFromInterfaces(Network::Channel* pChannel, KBEngine::MemoryStream& s) +{ + findBestInterfacesHandler()->onCreateAccountCB(s); +} + +//------------------------------------------------------------------------------------- +void Dbmgr::onAccountLogin(Network::Channel* pChannel, KBEngine::MemoryStream& s) +{ + std::string loginName, password, datas; + s >> loginName >> password; + s.readBlob(datas); + + if(loginName.size() == 0) + { + ERROR_MSG("Dbmgr::onAccountLogin: loginName is empty.\n"); + return; + } + + findBestInterfacesHandler()->loginAccount(pChannel, loginName, password, datas); +} + +//------------------------------------------------------------------------------------- +void Dbmgr::onLoginAccountCBBFromInterfaces(Network::Channel* pChannel, KBEngine::MemoryStream& s) +{ + findBestInterfacesHandler()->onLoginAccountCB(s); +} + +//------------------------------------------------------------------------------------- +void Dbmgr::queryAccount(Network::Channel* pChannel, + std::string& accountName, + std::string& password, + bool needCheckPassword, + COMPONENT_ID componentID, + ENTITY_ID entityID, + DBID entityDBID, + uint32 ip, + uint16 port) +{ + if(accountName.size() == 0) + { + ERROR_MSG("Dbmgr::queryAccount: accountName is empty.\n"); + return; + } + + Buffered_DBTasks* pBuffered_DBTasks = + findBufferedDBTask(Dbmgr::getSingleton().selectAccountDBInterfaceName(accountName)); + + if (!pBuffered_DBTasks) + { + ERROR_MSG(fmt::format("Dbmgr::queryAccount: not found dbInterface({})!\n", + Dbmgr::getSingleton().selectAccountDBInterfaceName(accountName))); + return; + } + + pBuffered_DBTasks->addTask(new DBTaskQueryAccount(pChannel->addr(), accountName, password, needCheckPassword, + componentID, entityID, entityDBID, ip, port)); + + numQueryEntity_++; +} + +//------------------------------------------------------------------------------------- +void Dbmgr::onAccountOnline(Network::Channel* pChannel, + std::string& accountName, + COMPONENT_ID componentID, + ENTITY_ID entityID) +{ + // bufferedDBTasks_.addTask(new DBTaskAccountOnline(pChannel->addr(), + // accountName, componentID, entityID)); +} + +//------------------------------------------------------------------------------------- +void Dbmgr::onEntityOffline(Network::Channel* pChannel, DBID dbid, ENTITY_SCRIPT_UID sid, uint16 dbInterfaceIndex) +{ + Buffered_DBTasks* pBuffered_DBTasks = findBufferedDBTask(g_kbeSrvConfig.dbInterfaceIndex2dbInterfaceName(dbInterfaceIndex)); + if (!pBuffered_DBTasks) + { + ERROR_MSG(fmt::format("Dbmgr::onEntityOffline: not found dbInterfaceIndex({})!\n", dbInterfaceIndex)); + return; + } + + pBuffered_DBTasks->addTask(new DBTaskEntityOffline(pChannel->addr(), dbid, sid)); +} + +//------------------------------------------------------------------------------------- +void Dbmgr::executeRawDatabaseCommand(Network::Channel* pChannel, + KBEngine::MemoryStream& s) +{ + ENTITY_ID entityID = -1; + s >> entityID; + + uint16 dbInterfaceIndex = 0; + s >> dbInterfaceIndex; + + std::string dbInterfaceName = g_kbeSrvConfig.dbInterfaceIndex2dbInterfaceName(dbInterfaceIndex); + if (dbInterfaceName.size() == 0) + { + ERROR_MSG(fmt::format("Dbmgr::executeRawDatabaseCommand: not found dbInterface({})!\n", dbInterfaceName)); + s.done(); + return; + } + + if (entityID == -1) + { + thread::ThreadPool* pThreadPool = DBUtil::pThreadPool(dbInterfaceName); + if (!pThreadPool) + { + ERROR_MSG(fmt::format("Dbmgr::executeRawDatabaseCommand: not found pThreadPool(dbInterface={})!\n", dbInterfaceName)); + s.done(); + return; + } + + pThreadPool->addTask(new DBTaskExecuteRawDatabaseCommand(pChannel ? pChannel->addr() : Network::Address::NONE, s)); + } + else + { + Buffered_DBTasks* pBuffered_DBTasks = findBufferedDBTask(dbInterfaceName); + if (!pBuffered_DBTasks) + { + ERROR_MSG(fmt::format("Dbmgr::executeRawDatabaseCommand: not found pBuffered_DBTasks(dbInterface={})!\n", dbInterfaceName)); + s.done(); + return; + } + + pBuffered_DBTasks->addTask(new DBTaskExecuteRawDatabaseCommandByEntity(pChannel ? pChannel->addr() : Network::Address::NONE, s, entityID)); + } + + s.done(); + + ++numExecuteRawDatabaseCommand_; +} + +//------------------------------------------------------------------------------------- +PyObject* Dbmgr::__py_executeRawDatabaseCommand(PyObject* self, PyObject* args) +{ + int argCount = (int)PyTuple_Size(args); + PyObject* pycallback = NULL; + PyObject* pyDBInterfaceName = NULL; + int ret = -1; + ENTITY_ID eid = -1; + + char* data = NULL; + Py_ssize_t size; + + if (argCount == 4) + ret = PyArg_ParseTuple(args, "s#|O|i|O", &data, &size, &pycallback, &eid, &pyDBInterfaceName); + else if (argCount == 3) + ret = PyArg_ParseTuple(args, "s#|O|i", &data, &size, &pycallback, &eid); + else if (argCount == 2) + ret = PyArg_ParseTuple(args, "s#|O", &data, &size, &pycallback); + else if (argCount == 1) + ret = PyArg_ParseTuple(args, "s#", &data, &size); + + if (ret == -1) + { + PyErr_Format(PyExc_TypeError, "KBEngine::executeRawDatabaseCommand: args error!"); + PyErr_PrintEx(0); + S_Return; + } + + std::string dbInterfaceName = "default"; + if (pyDBInterfaceName) + { + dbInterfaceName = PyUnicode_AsUTF8AndSize(pyDBInterfaceName, NULL); + + if (!g_kbeSrvConfig.dbInterface(dbInterfaceName)) + { + PyErr_Format(PyExc_TypeError, "KBEngine::executeRawDatabaseCommand: args4, incorrect dbInterfaceName(%s)!", + dbInterfaceName.c_str()); + + PyErr_PrintEx(0); + S_Return; + } + } + + Dbmgr::getSingleton().executeRawDatabaseCommand(data, (uint32)size, pycallback, eid, dbInterfaceName); + S_Return; +} + +//------------------------------------------------------------------------------------- +void Dbmgr::executeRawDatabaseCommand(const char* datas, uint32 size, PyObject* pycallback, ENTITY_ID eid, const std::string& dbInterfaceName) +{ + if (datas == NULL) + { + ERROR_MSG("KBEngine::executeRawDatabaseCommand: execute error!\n"); + return; + } + + int dbInterfaceIndex = g_kbeSrvConfig.dbInterfaceName2dbInterfaceIndex(dbInterfaceName); + if (dbInterfaceIndex < 0) + { + ERROR_MSG(fmt::format("KBEngine::executeRawDatabaseCommand: not found dbInterface({})!\n", + dbInterfaceName)); + + return; + } + + //INFO_MSG(fmt::format("KBEngine::executeRawDatabaseCommand{}:{}.\n", (eid > 0 ? fmt::format("(entityID={})", eid) : ""), datas)); + + MemoryStream* pMemoryStream = MemoryStream::createPoolObject(OBJECTPOOL_POINT); + (*pMemoryStream) << eid; + (*pMemoryStream) << (uint16)dbInterfaceIndex; + (*pMemoryStream) << componentID_ << componentType_; + + CALLBACK_ID callbackID = 0; + + if (pycallback && PyCallable_Check(pycallback)) + callbackID = callbackMgr().save(pycallback); + + (*pMemoryStream) << callbackID; + (*pMemoryStream) << size; + (*pMemoryStream).append(datas, size); + executeRawDatabaseCommand(NULL, *pMemoryStream); + MemoryStream::reclaimPoolObject(pMemoryStream); +} + +//------------------------------------------------------------------------------------- +void Dbmgr::onExecuteRawDatabaseCommandCB(KBEngine::MemoryStream& s) +{ + std::string err; + CALLBACK_ID callbackID = 0; + uint32 nrows = 0; + uint32 nfields = 0; + uint64 affectedRows = 0; + uint64 lastInsertID = 0; + + PyObject* pResultSet = NULL; + PyObject* pAffectedRows = NULL; + PyObject* pLastInsertID = NULL; + PyObject* pErrorMsg = NULL; + + s >> callbackID; + s >> err; + + if (err.size() <= 0) + { + s >> nfields; + + pErrorMsg = Py_None; + Py_INCREF(pErrorMsg); + + if (nfields > 0) + { + pAffectedRows = Py_None; + Py_INCREF(pAffectedRows); + + pLastInsertID = Py_None; + Py_INCREF(pLastInsertID); + + s >> nrows; + + pResultSet = PyList_New(nrows); + for (uint32 i = 0; i < nrows; ++i) + { + PyObject* pRow = PyList_New(nfields); + for (uint32 j = 0; j < nfields; ++j) + { + std::string cell; + s.readBlob(cell); + + PyObject* pCell = NULL; + + if (cell == "KBE_QUERY_DB_NULL") + { + Py_INCREF(Py_None); + pCell = Py_None; + } + else + { + pCell = PyBytes_FromStringAndSize(cell.data(), cell.length()); + } + + PyList_SET_ITEM(pRow, j, pCell); + } + + PyList_SET_ITEM(pResultSet, i, pRow); + } + } + else + { + pResultSet = Py_None; + Py_INCREF(pResultSet); + + pErrorMsg = Py_None; + Py_INCREF(pErrorMsg); + + s >> affectedRows; + + pAffectedRows = PyLong_FromUnsignedLongLong(affectedRows); + + s >> lastInsertID; + pLastInsertID = PyLong_FromUnsignedLongLong(lastInsertID); + } + } + else + { + pResultSet = Py_None; + Py_INCREF(pResultSet); + + pErrorMsg = PyUnicode_FromString(err.c_str()); + + pAffectedRows = Py_None; + Py_INCREF(pAffectedRows); + + pLastInsertID = Py_None; + Py_INCREF(pLastInsertID); + } + + s.done(); + + //DEBUG_MSG(fmt::format("Cellapp::onExecuteRawDatabaseCommandCB: nrows={}, nfields={}, err={}.\n", + // nrows, nfields, err.c_str())); + + if (callbackID > 0) + { + SCOPED_PROFILE(SCRIPTCALL_PROFILE); + + PyObjectPtr pyfunc = pyCallbackMgr_.take(callbackID); + if (pyfunc != NULL) + { + PyObject* pyResult = PyObject_CallFunction(pyfunc.get(), + const_cast("OOOO"), + pResultSet, pAffectedRows, pLastInsertID, pErrorMsg); + + if (pyResult != NULL) + Py_DECREF(pyResult); + else + SCRIPT_ERROR_CHECK(); + } + else + { + ERROR_MSG(fmt::format("Cellapp::onExecuteRawDatabaseCommandCB: not found callback:{}.\n", + callbackID)); + } + } + + Py_XDECREF(pResultSet); + Py_XDECREF(pAffectedRows); + Py_XDECREF(pLastInsertID); + Py_XDECREF(pErrorMsg); +} + +//------------------------------------------------------------------------------------- +void Dbmgr::writeEntity(Network::Channel* pChannel, + KBEngine::MemoryStream& s) +{ + ENTITY_ID eid; + DBID entityDBID; + COMPONENT_ID componentID; + uint16 dbInterfaceIndex; + + s >> componentID >> eid >> entityDBID >> dbInterfaceIndex; + + Buffered_DBTasks* pBuffered_DBTasks = findBufferedDBTask(g_kbeSrvConfig.dbInterfaceIndex2dbInterfaceName(dbInterfaceIndex)); + if (!pBuffered_DBTasks) + { + ERROR_MSG(fmt::format("Dbmgr::writeEntity: not found dbInterfaceIndex({})!\n", dbInterfaceIndex)); + s.done(); + return; + } + + pBuffered_DBTasks->addTask(new DBTaskWriteEntity(pChannel->addr(), componentID, eid, entityDBID, s)); + s.done(); + + ++numWrittenEntity_; +} + +//------------------------------------------------------------------------------------- +void Dbmgr::removeEntity(Network::Channel* pChannel, KBEngine::MemoryStream& s) +{ + ENTITY_ID eid; + DBID entityDBID; + COMPONENT_ID componentID; + uint16 dbInterfaceIndex; + + s >> dbInterfaceIndex >> componentID >> eid >> entityDBID; + KBE_ASSERT(entityDBID > 0); + + Buffered_DBTasks* pBuffered_DBTasks = findBufferedDBTask(g_kbeSrvConfig.dbInterfaceIndex2dbInterfaceName(dbInterfaceIndex)); + if (!pBuffered_DBTasks) + { + ERROR_MSG(fmt::format("Dbmgr::removeEntity: not found dbInterfaceIndex({})!\n", dbInterfaceIndex)); + s.done(); + return; + } + + pBuffered_DBTasks->addTask(new DBTaskRemoveEntity(pChannel->addr(), + componentID, eid, entityDBID, s)); + + s.done(); + + ++numRemovedEntity_; +} + +//------------------------------------------------------------------------------------- +void Dbmgr::entityAutoLoad(Network::Channel* pChannel, KBEngine::MemoryStream& s) +{ + COMPONENT_ID componentID; + ENTITY_SCRIPT_UID entityType; + ENTITY_ID start; + ENTITY_ID end; + uint16 dbInterfaceIndex = 0; + + s >> dbInterfaceIndex >> componentID >> entityType >> start >> end; + + DBUtil::pThreadPool(g_kbeSrvConfig.dbInterfaceIndex2dbInterfaceName(dbInterfaceIndex))-> + addTask(new DBTaskEntityAutoLoad(pChannel->addr(), componentID, entityType, start, end)); +} + +//------------------------------------------------------------------------------------- +void Dbmgr::deleteEntityByDBID(Network::Channel* pChannel, KBEngine::MemoryStream& s) +{ + COMPONENT_ID componentID; + ENTITY_SCRIPT_UID sid; + CALLBACK_ID callbackID = 0; + DBID entityDBID; + uint16 dbInterfaceIndex = 0; + + s >> dbInterfaceIndex >> componentID >> entityDBID >> callbackID >> sid; + KBE_ASSERT(entityDBID > 0); + + DBUtil::pThreadPool(g_kbeSrvConfig.dbInterfaceIndex2dbInterfaceName(dbInterfaceIndex))-> + addTask(new DBTaskDeleteEntityByDBID(pChannel->addr(), componentID, entityDBID, callbackID, sid)); +} + +//------------------------------------------------------------------------------------- +void Dbmgr::lookUpEntityByDBID(Network::Channel* pChannel, KBEngine::MemoryStream& s) +{ + COMPONENT_ID componentID; + ENTITY_SCRIPT_UID sid; + CALLBACK_ID callbackID = 0; + DBID entityDBID; + uint16 dbInterfaceIndex = 0; + + s >> dbInterfaceIndex >> componentID >> entityDBID >> callbackID >> sid; + KBE_ASSERT(entityDBID > 0); + + DBUtil::pThreadPool(g_kbeSrvConfig.dbInterfaceIndex2dbInterfaceName(dbInterfaceIndex))-> + addTask(new DBTaskLookUpEntityByDBID(pChannel->addr(), componentID, entityDBID, callbackID, sid)); +} + +//------------------------------------------------------------------------------------- +void Dbmgr::queryEntity(Network::Channel* pChannel, uint16 dbInterfaceIndex, COMPONENT_ID componentID, int8 queryMode, DBID dbid, + std::string& entityType, CALLBACK_ID callbackID, ENTITY_ID entityID) +{ + bufferedDBTasksMaps_[g_kbeSrvConfig.dbInterfaceIndex2dbInterfaceName(dbInterfaceIndex)]. + addTask(new DBTaskQueryEntity(pChannel->addr(), queryMode, entityType, dbid, componentID, callbackID, entityID)); + + numQueryEntity_++; +} + +//------------------------------------------------------------------------------------- +void Dbmgr::syncEntityStreamTemplate(Network::Channel* pChannel, KBEngine::MemoryStream& s) +{ + int rpos = s.rpos(); + EntityTables::ENTITY_TABLES_MAP::iterator iter = EntityTables::sEntityTables.begin(); + for (; iter != EntityTables::sEntityTables.end(); ++iter) + { + KBEAccountTable* pTable = + static_cast(iter->second.findKBETable(KBE_TABLE_PERFIX "_accountinfos")); + + KBE_ASSERT(pTable); + + s.rpos(rpos); + pTable->accountDefMemoryStream(s); + } + + s.done(); +} + +//------------------------------------------------------------------------------------- +void Dbmgr::charge(Network::Channel* pChannel, KBEngine::MemoryStream& s) +{ + findBestInterfacesHandler()->charge(pChannel, s); +} + +//------------------------------------------------------------------------------------- +void Dbmgr::onChargeCB(Network::Channel* pChannel, KBEngine::MemoryStream& s) +{ + findBestInterfacesHandler()->onChargeCB(s); +} + +//------------------------------------------------------------------------------------- +void Dbmgr::eraseClientReq(Network::Channel* pChannel, std::string& logkey) +{ + std::vector::iterator iter = pInterfacesHandlers_.begin(); + for(; iter != pInterfacesHandlers_.end(); ++iter) + (*iter)->eraseClientReq(pChannel, logkey); +} + +//------------------------------------------------------------------------------------- +void Dbmgr::accountActivate(Network::Channel* pChannel, std::string& scode) +{ + INFO_MSG(fmt::format("Dbmgr::accountActivate: code={}.\n", scode)); + findBestInterfacesHandler()->accountActivate(pChannel, scode); +} + +//------------------------------------------------------------------------------------- +void Dbmgr::accountReqResetPassword(Network::Channel* pChannel, std::string& accountName) +{ + INFO_MSG(fmt::format("Dbmgr::accountReqResetPassword: accountName={}.\n", accountName)); + findBestInterfacesHandler()->accountReqResetPassword(pChannel, accountName); +} + +//------------------------------------------------------------------------------------- +void Dbmgr::accountResetPassword(Network::Channel* pChannel, std::string& accountName, std::string& newpassword, std::string& code) +{ + INFO_MSG(fmt::format("Dbmgr::accountResetPassword: accountName={}.\n", accountName)); + findBestInterfacesHandler()->accountResetPassword(pChannel, accountName, newpassword, code); +} + +//------------------------------------------------------------------------------------- +void Dbmgr::accountReqBindMail(Network::Channel* pChannel, ENTITY_ID entityID, std::string& accountName, + std::string& password, std::string& email) +{ + INFO_MSG(fmt::format("Dbmgr::accountReqBindMail: accountName={}, email={}.\n", accountName, email)); + findBestInterfacesHandler()->accountReqBindMail(pChannel, entityID, accountName, password, email); +} + +//------------------------------------------------------------------------------------- +void Dbmgr::accountBindMail(Network::Channel* pChannel, std::string& username, std::string& scode) +{ + INFO_MSG(fmt::format("Dbmgr::accountBindMail: username={}, scode={}.\n", username, scode)); + findBestInterfacesHandler()->accountBindMail(pChannel, username, scode); +} + +//------------------------------------------------------------------------------------- +void Dbmgr::accountNewPassword(Network::Channel* pChannel, ENTITY_ID entityID, std::string& accountName, + std::string& password, std::string& newpassword) +{ + INFO_MSG(fmt::format("Dbmgr::accountNewPassword: accountName={}.\n", accountName)); + findBestInterfacesHandler()->accountNewPassword(pChannel, entityID, accountName, password, newpassword); +} + +//------------------------------------------------------------------------------------- +std::string Dbmgr::selectAccountDBInterfaceName(const std::string& name) +{ + std::string dbInterfaceName = "default"; + + // ɽű + SCOPED_PROFILE(SCRIPTCALL_PROFILE); + PyObject* pyResult = PyObject_CallMethod(getEntryScript().get(), + const_cast("onSelectAccountDBInterface"), + const_cast("s"), + name.c_str()); + + if (pyResult != NULL) + { + dbInterfaceName = PyUnicode_AsUTF8AndSize(pyResult, NULL); + Py_DECREF(pyResult); + } + else + { + SCRIPT_ERROR_CHECK(); + } + + if (dbInterfaceName == "" || g_kbeSrvConfig.dbInterface(dbInterfaceName) == NULL) + { + ERROR_MSG(fmt::format("Dbmgr::selectAccountDBInterfaceName: not found dbInterface({}), accountName={}.\n", dbInterfaceName, name)); + return "default"; + } + + return dbInterfaceName; +} + +//------------------------------------------------------------------------------------- +void Dbmgr::onChannelDeregister(Network::Channel * pChannel) +{ + // app + if (pChannel->isInternal()) + { + Components::ComponentInfos* cinfo = Components::getSingleton().findComponent(pChannel); + if (cinfo) + { + if (cinfo->componentType == BASEAPP_TYPE) + { + loseBaseappts_[cinfo->cid] = timestamp() + uint64(60 * stampsPerSecond()); + WARNING_MSG(fmt::format("Dbmgr::onChannelDeregister(): If the process cannot be resumed, the entitylog(baseapp={}) will be cleaned up after 60 seconds!\n", cinfo->cid)); + } + } + } + + ServerApp::onChannelDeregister(pChannel); +} + +//------------------------------------------------------------------------------------- +} diff --git a/kbe/src/server/dbmgr/dbmgr.h b/kbe/src/server/dbmgr/dbmgr.h index 1194656df2..4128c2d671 100644 --- a/kbe/src/server/dbmgr/dbmgr.h +++ b/kbe/src/server/dbmgr/dbmgr.h @@ -1,276 +1,279 @@ -// Copyright 2008-2018 Yolo Technologies, Inc. All Rights Reserved. https://www.comblockengine.com - -#ifndef KBE_DBMGR_H -#define KBE_DBMGR_H - -#include "db_interface/db_threadpool.h" -#include "buffered_dbtasks.h" -#include "server/kbemain.h" -#include "pyscript/script.h" -#include "pyscript/pyobject_pointer.h" -#include "entitydef/entitydef.h" -#include "server/python_app.h" -#include "server/idallocate.h" -#include "server/serverconfig.h" -#include "server/globaldata_client.h" -#include "server/globaldata_server.h" -#include "server/callbackmgr.h" -#include "common/timer.h" -#include "network/endpoint.h" -#include "resmgr/resmgr.h" -#include "thread/threadpool.h" - - -namespace KBEngine{ - -class DBInterface; -class TelnetServer; -class InterfacesHandler; -class SyncAppDatasHandler; -class UpdateDBServerLogHandler; - -class Dbmgr : public PythonApp, - public Singleton -{ -public: - enum TimeOutType - { - TIMEOUT_TICK = TIMEOUT_PYTHONAPP_MAX + 1, - TIMEOUT_CHECK_STATUS - }; - - Dbmgr(Network::EventDispatcher& dispatcher, - Network::NetworkInterface& ninterface, - COMPONENT_TYPE componentType, - COMPONENT_ID componentID); - - ~Dbmgr(); - - bool run(); - - void handleTimeout(TimerHandle handle, void * arg); - void handleMainTick(); - void handleCheckStatusTick(); - - /* ʼؽӿ */ - bool initializeBegin(); - bool inInitialize(); - bool initializeEnd(); - void finalise(); - void onInstallPyModules(); - - bool initInterfacesHandler(); - - bool initDB(); - - virtual ShutdownHandler::CAN_SHUTDOWN_STATE canShutdown(); - - virtual void onShutdownBegin(); - virtual void onShutdownEnd(); - - /** ȡIDָ */ - IDServer& idServer(void){ return idServer_; } - - /** ӿ - һENTITY_ID - */ - void onReqAllocEntityID(Network::Channel* pChannel, COMPONENT_ORDER componentType, COMPONENT_ID componentID); - - /* ӿ - עһ¼baseappcellappdbmgr - ͨһµappˣ ҪijЩעԼ - */ - virtual void onRegisterNewApp(Network::Channel* pChannel, - int32 uid, - std::string& username, - COMPONENT_TYPE componentType, COMPONENT_ID componentID, COMPONENT_ORDER globalorderID, COMPONENT_ORDER grouporderID, - uint32 intaddr, uint16 intport, uint32 extaddr, uint16 extport, std::string& extaddrEx); - - - /** ӿ - dbmgr㲥globalݵĸı - */ - void onGlobalDataClientLogon(Network::Channel* pChannel, COMPONENT_TYPE componentType); - void onBroadcastGlobalDataChanged(Network::Channel* pChannel, KBEngine::MemoryStream& s); - - /** ӿ - 󴴽˺ - */ - void reqCreateAccount(Network::Channel* pChannel, KBEngine::MemoryStream& s); - void onCreateAccountCBFromInterfaces(Network::Channel* pChannel, KBEngine::MemoryStream& s); - - /** ӿ - ͻ - */ - void eraseClientReq(Network::Channel* pChannel, std::string& logkey); - - /** ӿ - һû¼ ҪϷ - */ - void onAccountLogin(Network::Channel* pChannel, KBEngine::MemoryStream& s); - void onLoginAccountCBBFromInterfaces(Network::Channel* pChannel, KBEngine::MemoryStream& s); - - /** ӿ - baseappѯaccountϢ - */ - void queryAccount(Network::Channel* pChannel, std::string& accountName, std::string& password, bool needCheckPassword, - COMPONENT_ID componentID, ENTITY_ID entityID, DBID entityDBID, uint32 ip, uint16 port); - - /** ӿ - ʵԶع - */ - void entityAutoLoad(Network::Channel* pChannel, KBEngine::MemoryStream& s); - - /** ӿ - ˺Ŵbaseapp - */ - void onAccountOnline(Network::Channel* pChannel, std::string& accountName, - COMPONENT_ID componentID, ENTITY_ID entityID); - - /** ӿ - entity-baseapp - */ - void onEntityOffline(Network::Channel* pChannel, DBID dbid, ENTITY_SCRIPT_UID sid, uint16 dbInterfaceIndex); - - /** ӿ - ִݿѯ - */ - void executeRawDatabaseCommand(Network::Channel* pChannel, KBEngine::MemoryStream& s); - - /** ӿ - ijentity浵 - */ - void writeEntity(Network::Channel* pChannel, KBEngine::MemoryStream& s); - - /** ӿ - ɾijentityĴ浵 - */ - void removeEntity(Network::Channel* pChannel, KBEngine::MemoryStream& s); - - /** ӿ - ͨdbidݿɾһʵĻص - */ - void deleteEntityByDBID(Network::Channel* pChannel, KBEngine::MemoryStream& s); - - /** ӿ - ͨdbidѯһʵǷݿ - */ - void lookUpEntityByDBID(Network::Channel* pChannel, KBEngine::MemoryStream& s); - - /** ӿ - dbȡentity - */ - void queryEntity(Network::Channel* pChannel, uint16 dbInterfaceIndex, COMPONENT_ID componentID, int8 queryMode, DBID dbid, - std::string& entityType, CALLBACK_ID callbackID, ENTITY_ID entityID); - - /** ӿ - ͬentityģ - */ - void syncEntityStreamTemplate(Network::Channel* pChannel, KBEngine::MemoryStream& s); - - virtual bool initializeWatcher(); - - /** ӿ - ֵ - */ - void charge(Network::Channel* pChannel, KBEngine::MemoryStream& s); - - /** ӿ - ֵص - */ - void onChargeCB(Network::Channel* pChannel, KBEngine::MemoryStream& s); - - - /** ӿ - ص - */ - void accountActivate(Network::Channel* pChannel, std::string& scode); - - /** ӿ - ˺ - */ - void accountReqResetPassword(Network::Channel* pChannel, std::string& accountName); - void accountResetPassword(Network::Channel* pChannel, std::string& accountName, - std::string& newpassword, std::string& code); - - /** ӿ - ˺Ű - */ - void accountReqBindMail(Network::Channel* pChannel, ENTITY_ID entityID, std::string& accountName, - std::string& password, std::string& email); - void accountBindMail(Network::Channel* pChannel, std::string& username, std::string& scode); - - /** ӿ - ˺޸ - */ - void accountNewPassword(Network::Channel* pChannel, ENTITY_ID entityID, std::string& accountName, - std::string& password, std::string& newpassword); - - SyncAppDatasHandler* pSyncAppDatasHandler() const { return pSyncAppDatasHandler_; } - void pSyncAppDatasHandler(SyncAppDatasHandler* p){ pSyncAppDatasHandler_ = p; } - - std::string selectAccountDBInterfaceName(const std::string& name); - - Buffered_DBTasks* findBufferedDBTask(const std::string& dbInterfaceName) - { - BUFFERED_DBTASKS_MAP::iterator dbin_iter = bufferedDBTasksMaps_.find(dbInterfaceName); - if (dbin_iter == bufferedDBTasksMaps_.end()) - return NULL; - - return &dbin_iter->second; - } - - virtual void onChannelDeregister(Network::Channel * pChannel); - - InterfacesHandler* findBestInterfacesHandler(); - - /** - dbmgrִһݿ - */ - static PyObject* __py_executeRawDatabaseCommand(PyObject* self, PyObject* args); - void executeRawDatabaseCommand(const char* datas, uint32 size, PyObject* pycallback, ENTITY_ID eid, const std::string& dbInterfaceName); - void onExecuteRawDatabaseCommandCB(KBEngine::MemoryStream& s); - - PY_CALLBACKMGR& callbackMgr() { return pyCallbackMgr_; } - -protected: - TimerHandle loopCheckTimerHandle_; - TimerHandle mainProcessTimer_; - - // entityID - IDServer idServer_; - - // globalData - GlobalDataServer* pGlobalData_; - - // baseAppData - GlobalDataServer* pBaseAppData_; - - // cellAppData - GlobalDataServer* pCellAppData_; - - typedef KBEUnordered_map BUFFERED_DBTASKS_MAP; - BUFFERED_DBTASKS_MAP bufferedDBTasksMaps_; - - // Statistics - uint32 numWrittenEntity_; - uint32 numRemovedEntity_; - uint32 numQueryEntity_; - uint32 numExecuteRawDatabaseCommand_; - uint32 numCreatedAccount_; - - std::vector pInterfacesHandlers_; - - SyncAppDatasHandler* pSyncAppDatasHandler_; - UpdateDBServerLogHandler* pUpdateDBServerLogHandler_; - - TelnetServer* pTelnetServer_; - - std::map loseBaseappts_; - - PY_CALLBACKMGR pyCallbackMgr_; -}; - -} - -#endif // KBE_DBMGR_H +// Copyright 2008-2018 Yolo Technologies, Inc. All Rights Reserved. https://www.comblockengine.com + +#ifndef KBE_DBMGR_H +#define KBE_DBMGR_H + +#include "db_interface/db_threadpool.h" +#include "buffered_dbtasks.h" +#include "server/kbemain.h" +#include "pyscript/script.h" +#include "pyscript/pyobject_pointer.h" +#include "entitydef/entitydef.h" +#include "server/python_app.h" +#include "server/idallocate.h" +#include "server/serverconfig.h" +#include "server/globaldata_client.h" +#include "server/globaldata_server.h" +#include "server/callbackmgr.h" +#include "common/timer.h" +#include "network/endpoint.h" +#include "resmgr/resmgr.h" +#include "thread/threadpool.h" + + +namespace KBEngine{ + +class DBInterface; +class TelnetServer; +class InterfacesHandler; +class SyncAppDatasHandler; +class UpdateDBServerLogHandler; + +class Dbmgr : public PythonApp, + public Singleton +{ +public: + enum TimeOutType + { + TIMEOUT_TICK = TIMEOUT_PYTHONAPP_MAX + 1, + TIMEOUT_CHECK_STATUS + }; + + Dbmgr(Network::EventDispatcher& dispatcher, + Network::NetworkInterface& ninterface, + COMPONENT_TYPE componentType, + COMPONENT_ID componentID); + + ~Dbmgr(); + + bool run(); + + void handleTimeout(TimerHandle handle, void * arg); + void handleMainTick(); + void handleCheckStatusTick(); + + /* ʼؽӿ */ + bool initializeBegin(); + bool inInitialize(); + bool initializeEnd(); + void finalise(); + + bool installPyModules(); + bool uninstallPyModules(); + void onInstallPyModules(); + + bool initInterfacesHandler(); + + bool initDB(); + + virtual ShutdownHandler::CAN_SHUTDOWN_STATE canShutdown(); + + virtual void onShutdownBegin(); + virtual void onShutdownEnd(); + + /** ȡIDָ */ + IDServer& idServer(void){ return idServer_; } + + /** ӿ + һENTITY_ID + */ + void onReqAllocEntityID(Network::Channel* pChannel, COMPONENT_ORDER componentType, COMPONENT_ID componentID); + + /* ӿ + עһ¼baseappcellappdbmgr + ͨһµappˣ ҪijЩעԼ + */ + virtual void onRegisterNewApp(Network::Channel* pChannel, + int32 uid, + std::string& username, + COMPONENT_TYPE componentType, COMPONENT_ID componentID, COMPONENT_ORDER globalorderID, COMPONENT_ORDER grouporderID, + uint32 intaddr, uint16 intport, uint32 extaddr, uint16 extport, std::string& extaddrEx); + + + /** ӿ + dbmgr㲥globalݵĸı + */ + void onGlobalDataClientLogon(Network::Channel* pChannel, COMPONENT_TYPE componentType); + void onBroadcastGlobalDataChanged(Network::Channel* pChannel, KBEngine::MemoryStream& s); + + /** ӿ + 󴴽˺ + */ + void reqCreateAccount(Network::Channel* pChannel, KBEngine::MemoryStream& s); + void onCreateAccountCBFromInterfaces(Network::Channel* pChannel, KBEngine::MemoryStream& s); + + /** ӿ + ͻ + */ + void eraseClientReq(Network::Channel* pChannel, std::string& logkey); + + /** ӿ + һû¼ ҪϷ + */ + void onAccountLogin(Network::Channel* pChannel, KBEngine::MemoryStream& s); + void onLoginAccountCBBFromInterfaces(Network::Channel* pChannel, KBEngine::MemoryStream& s); + + /** ӿ + baseappѯaccountϢ + */ + void queryAccount(Network::Channel* pChannel, std::string& accountName, std::string& password, bool needCheckPassword, + COMPONENT_ID componentID, ENTITY_ID entityID, DBID entityDBID, uint32 ip, uint16 port); + + /** ӿ + ʵԶع + */ + void entityAutoLoad(Network::Channel* pChannel, KBEngine::MemoryStream& s); + + /** ӿ + ˺Ŵbaseapp + */ + void onAccountOnline(Network::Channel* pChannel, std::string& accountName, + COMPONENT_ID componentID, ENTITY_ID entityID); + + /** ӿ + entity-baseapp + */ + void onEntityOffline(Network::Channel* pChannel, DBID dbid, ENTITY_SCRIPT_UID sid, uint16 dbInterfaceIndex); + + /** ӿ + ִݿѯ + */ + void executeRawDatabaseCommand(Network::Channel* pChannel, KBEngine::MemoryStream& s); + + /** ӿ + ijentity浵 + */ + void writeEntity(Network::Channel* pChannel, KBEngine::MemoryStream& s); + + /** ӿ + ɾijentityĴ浵 + */ + void removeEntity(Network::Channel* pChannel, KBEngine::MemoryStream& s); + + /** ӿ + ͨdbidݿɾһʵĻص + */ + void deleteEntityByDBID(Network::Channel* pChannel, KBEngine::MemoryStream& s); + + /** ӿ + ͨdbidѯһʵǷݿ + */ + void lookUpEntityByDBID(Network::Channel* pChannel, KBEngine::MemoryStream& s); + + /** ӿ + dbȡentity + */ + void queryEntity(Network::Channel* pChannel, uint16 dbInterfaceIndex, COMPONENT_ID componentID, int8 queryMode, DBID dbid, + std::string& entityType, CALLBACK_ID callbackID, ENTITY_ID entityID); + + /** ӿ + ͬentityģ + */ + void syncEntityStreamTemplate(Network::Channel* pChannel, KBEngine::MemoryStream& s); + + virtual bool initializeWatcher(); + + /** ӿ + ֵ + */ + void charge(Network::Channel* pChannel, KBEngine::MemoryStream& s); + + /** ӿ + ֵص + */ + void onChargeCB(Network::Channel* pChannel, KBEngine::MemoryStream& s); + + + /** ӿ + ص + */ + void accountActivate(Network::Channel* pChannel, std::string& scode); + + /** ӿ + ˺ + */ + void accountReqResetPassword(Network::Channel* pChannel, std::string& accountName); + void accountResetPassword(Network::Channel* pChannel, std::string& accountName, + std::string& newpassword, std::string& code); + + /** ӿ + ˺Ű + */ + void accountReqBindMail(Network::Channel* pChannel, ENTITY_ID entityID, std::string& accountName, + std::string& password, std::string& email); + void accountBindMail(Network::Channel* pChannel, std::string& username, std::string& scode); + + /** ӿ + ˺޸ + */ + void accountNewPassword(Network::Channel* pChannel, ENTITY_ID entityID, std::string& accountName, + std::string& password, std::string& newpassword); + + SyncAppDatasHandler* pSyncAppDatasHandler() const { return pSyncAppDatasHandler_; } + void pSyncAppDatasHandler(SyncAppDatasHandler* p){ pSyncAppDatasHandler_ = p; } + + std::string selectAccountDBInterfaceName(const std::string& name); + + Buffered_DBTasks* findBufferedDBTask(const std::string& dbInterfaceName) + { + BUFFERED_DBTASKS_MAP::iterator dbin_iter = bufferedDBTasksMaps_.find(dbInterfaceName); + if (dbin_iter == bufferedDBTasksMaps_.end()) + return NULL; + + return &dbin_iter->second; + } + + virtual void onChannelDeregister(Network::Channel * pChannel); + + InterfacesHandler* findBestInterfacesHandler(); + + /** + dbmgrִһݿ + */ + static PyObject* __py_executeRawDatabaseCommand(PyObject* self, PyObject* args); + void executeRawDatabaseCommand(const char* datas, uint32 size, PyObject* pycallback, ENTITY_ID eid, const std::string& dbInterfaceName); + void onExecuteRawDatabaseCommandCB(KBEngine::MemoryStream& s); + + PY_CALLBACKMGR& callbackMgr() { return pyCallbackMgr_; } + +protected: + TimerHandle loopCheckTimerHandle_; + TimerHandle mainProcessTimer_; + + // entityID + IDServer idServer_; + + // globalData + GlobalDataServer* pGlobalData_; + + // baseAppData + GlobalDataServer* pBaseAppData_; + + // cellAppData + GlobalDataServer* pCellAppData_; + + typedef KBEUnordered_map BUFFERED_DBTASKS_MAP; + BUFFERED_DBTASKS_MAP bufferedDBTasksMaps_; + + // Statistics + uint32 numWrittenEntity_; + uint32 numRemovedEntity_; + uint32 numQueryEntity_; + uint32 numExecuteRawDatabaseCommand_; + uint32 numCreatedAccount_; + + std::vector pInterfacesHandlers_; + + SyncAppDatasHandler* pSyncAppDatasHandler_; + UpdateDBServerLogHandler* pUpdateDBServerLogHandler_; + + TelnetServer* pTelnetServer_; + + std::map loseBaseappts_; + + PY_CALLBACKMGR pyCallbackMgr_; +}; + +} + +#endif // KBE_DBMGR_H diff --git a/kbe/src/server/tools/kbcmd/client_sdk_ue4.cpp b/kbe/src/server/tools/kbcmd/client_sdk_ue4.cpp index 4fa2cad4ec..6398bbe028 100644 --- a/kbe/src/server/tools/kbcmd/client_sdk_ue4.cpp +++ b/kbe/src/server/tools/kbcmd/client_sdk_ue4.cpp @@ -15,6 +15,8 @@ namespace KBEngine { static std::string headerBody = "/*\n\tGenerated by KBEngine!\n\tPlease do not modify this file!\n#REPLACE#\ttools = kbcmd\n*/\n\n"; static std::string moduleSuffix = "Base"; static std::string ConflictTypePrefix = "KBE_"; +static std::string namespaceNameBegin = "namespace KBEngine\n{\n\n"; +static std::string namespaceNameEnd = "\n}"; //------------------------------------------------------------------------------------- ClientSDKUE4::ClientSDKUE4(): @@ -272,9 +274,10 @@ bool ClientSDKUE4::writeServerErrorDescrsModuleBegin() strutil::kbe_replace(fileBody(), "#REPLACE#", ""); fileBody() += "#pragma once\n\n"; - fileBody() += "#include \"KBECommon.h\"\n"; + fileBody() += "#include \"KBECommon.h\"\n\n"; - fileBody() += "\n\n// defined in */res/server/server_errors.xml\n\n"; + fileBody() += namespaceNameBegin; + fileBody() += "// defined in */res/server/server_errors.xml\n\n"; fileBody() += fmt::format("class KBENGINEPLUGINS_API {}\n{{\npublic:\n", "ServerErrorDescrs"); fileBody() += "\tServerErrorDescrs():\n\tserverErrs_()\n\t{\n"; @@ -300,7 +303,8 @@ bool ClientSDKUE4::writeServerErrorDescrsModuleEnd() fileBody() += "\tFKServerErr ServerErr(uint16 id)\n\t{\n\t\treturn serverErrs_.FindRef(id);\n\t}\n\n"; fileBody() += "\tprotected:\n\t\tTMap serverErrs_;"; - fileBody() += "\n};\n\n"; + fileBody() += "\n};\n"; + fileBody() += namespaceNameEnd; return true; } @@ -318,6 +322,7 @@ bool ClientSDKUE4::writeEngineMessagesModuleBegin() fileBody() += "#include \"MemoryStream.h\"\n\n"; fileBody() += "// engine-c++ messages\n\n"; + fileBody() += namespaceNameBegin; fileBody() += fmt::format("class KBENGINEPLUGINS_API {}\n{{\npublic:", "Message"); fileBody() += "\n\tMessage()\n\t{\n"; @@ -366,6 +371,8 @@ bool ClientSDKUE4::writeEngineMessagesModuleBegin() fileBody() += "#include \"MemoryStream.h\"\n"; fileBody() += "#include \"KBEngine.h\"\n\n"; + fileBody() += namespaceNameBegin; + fileBody() += "TMap Messages::loginappMessages;\n"; fileBody() += "TMap Messages::baseappMessages;\n"; fileBody() += "TMap Messages::clientMessages;\n"; @@ -467,6 +474,8 @@ bool ClientSDKUE4::writeEngineMessagesModuleMessage(Network::ExposedMessageInfo& //------------------------------------------------------------------------------------- bool ClientSDKUE4::writeEngineMessagesModuleEnd() { + changeContextToSource(); + fileBody() += namespaceNameEnd; changeContextToHeader(); fileBody() += fmt::format("class KBENGINEPLUGINS_API {}\n{{\npublic:\n", "Messages"); @@ -500,8 +509,8 @@ bool ClientSDKUE4::writeEngineMessagesModuleEnd() fileBody() += "\n\t\treturn true;"; fileBody() += "\n\t}"; - fileBody() += "\n};"; - + fileBody() += "\n}\n;"; + fileBody() += namespaceNameEnd; return true; } @@ -517,6 +526,7 @@ bool ClientSDKUE4::writeEntityDefsModuleBegin() fileBody() += "#pragma once\n\n"; fileBody() += "#include \"KBECommon.h\"\n\n"; + fileBody() += namespaceNameBegin; fileBody() += "class Entity;\n"; fileBody() += "class DATATYPE_BASE;\n"; fileBody() += "class ScriptModule;\n\n"; @@ -540,6 +550,7 @@ bool ClientSDKUE4::writeEntityDefsModuleBegin() fileBody() += "\tstatic Entity* createEntity(int utype);\n"; changeContextToSource(); + fileBody() += "#include \"EntityDef.h\"\n"; fileBody() += "#include \"DataTypes.h\"\n"; fileBody() += "#include \"CustomDataTypes.h\"\n"; @@ -566,6 +577,7 @@ bool ClientSDKUE4::writeEntityDefsModuleBegin() } fileBody() += "\n"; + fileBody() += namespaceNameBegin; fileBody() += "TMap EntityDef::datatype2id;\n"; fileBody() += "TMap EntityDef::datatypes;\n"; fileBody() += "TMap EntityDef::id2datatypes;\n"; @@ -660,8 +672,11 @@ bool ClientSDKUE4::writeEntityDefsModuleBegin() //------------------------------------------------------------------------------------- bool ClientSDKUE4::writeEntityDefsModuleEnd() { + changeContextToSource(); + fileBody() += namespaceNameEnd; changeContextToHeader(); fileBody() += "};\n"; + fileBody() += namespaceNameEnd; return true; } @@ -794,12 +809,17 @@ bool ClientSDKUE4::writeEntityCallBegin(ScriptDefModule* pScriptDefModule) } fileBody() += std::string("\n// defined in */scripts/entity_defs/") + pScriptDefModule->getName() + ".def\n\n"; + fileBody() += namespaceNameBegin; return true; } //------------------------------------------------------------------------------------- bool ClientSDKUE4::writeEntityCallEnd(ScriptDefModule* pScriptDefModule) { + changeContextToSource(); + fileBody() += namespaceNameEnd; + changeContextToHeader(); + fileBody() += namespaceNameEnd; return true; } @@ -919,7 +939,8 @@ bool ClientSDKUE4::writeBaseEntityCallBegin(ScriptDefModule* pScriptDefModule) fileBody() += fmt::format("#include \"{}.h\"\n", (std::string("EntityCall") + pScriptDefModule->getName() + moduleSuffix)); fileBody() += fmt::format("#include \"Bundle.h\"\n\n"); - fileBody() += fmt::format("\n{}::{}(int32 eid, const FString& ename) : EntityCall(eid, ename)\n{{\n", newModuleName, newModuleName); + fileBody() += namespaceNameBegin; + fileBody() += fmt::format("{}::{}(int32 eid, const FString& ename) : EntityCall(eid, ename)\n{{\n", newModuleName, newModuleName); fileBody() += initstr; } else @@ -931,9 +952,11 @@ bool ClientSDKUE4::writeBaseEntityCallBegin(ScriptDefModule* pScriptDefModule) fileBody() += fmt::format("\n\tvirtual ~{}();\n", newModuleName); changeContextToSource(); + fileBody() += fmt::format("#include \"{}.h\"\n", (std::string("EntityCall") + pScriptDefModule->getName() + moduleSuffix)); fileBody() += fmt::format("#include \"Bundle.h\"\n\n"); - fileBody() += fmt::format("\n{}::{}(uint16 ecpID, int32 eid) : \nEntityCall(eid, \"{}\"),\nentityComponentPropertyID(0)\n{{\n", newModuleName, newModuleName, pScriptDefModule->getName()); + fileBody() += namespaceNameBegin; + fileBody() += fmt::format("{}::{}(uint16 ecpID, int32 eid) : \nEntityCall(eid, \"{}\"),\nentityComponentPropertyID(0)\n{{\n", newModuleName, newModuleName, pScriptDefModule->getName()); fileBody() += fmt::format("\tentityComponentPropertyID = ecpID;\n"); } @@ -1071,7 +1094,8 @@ bool ClientSDKUE4::writeCustomDataTypesBegin() fileBody() += "#include \"KBETypes.h\"\n"; fileBody() += "#include \"MemoryStream.h\"\n"; fileBody() += "#include \"Bundle.h\"\n"; - fileBody() += "#include \"DataTypes.h\"\n\n"; + fileBody() += "#include \"DataTypes.h\"\n"; + fileBody() += namespaceNameBegin; changeContextToSource(); fileBody() += "#include \"CustomDataTypes.h\"\n"; @@ -1079,15 +1103,18 @@ bool ClientSDKUE4::writeCustomDataTypesBegin() fileBody() += "#include \"KBDebug.h\"\n"; fileBody() += "#include \"DataTypes.h\"\n"; fileBody() += "#include \"Runtime/Core/Public/Misc/Variant.h\"\n\n"; - + fileBody() += namespaceNameBegin; + return true; } //------------------------------------------------------------------------------------- bool ClientSDKUE4::writeCustomDataTypesEnd() { + changeContextToSource(); + fileBody() += namespaceNameEnd; changeContextToHeader(); - fileBody() += "\n"; + fileBody() += namespaceNameEnd; return true; } @@ -1762,15 +1789,16 @@ bool ClientSDKUE4::writeTypesBegin() strutil::kbe_replace(fileBody(), "#REPLACE#", ""); fileBody() += "#pragma once\n\n"; - fileBody() += "#include \"KBECommon.h\"\n"; - - fileBody() += "\n\n// defined in */scripts/entity_defs/types.xml\n\n"; + fileBody() += "#include \"KBECommon.h\"\n\n"; + fileBody() += namespaceNameBegin; + fileBody() += "\n// defined in */scripts/entity_defs/types.xml\n\n"; return true; } //------------------------------------------------------------------------------------- bool ClientSDKUE4::writeTypesEnd() { + fileBody() += namespaceNameEnd; return true; } @@ -2096,6 +2124,7 @@ bool ClientSDKUE4::writeEntityModuleBegin(ScriptDefModule* pEntityScriptDefModul fileBody() += fmt::format("#include \"EntityCall{}.h\"\n\n", newModuleName); + fileBody() += namespaceNameBegin; fileBody() += "class Method;\n"; fileBody() += "class Property;\n"; fileBody() += "class MemoryStream;\n"; @@ -2131,6 +2160,7 @@ bool ClientSDKUE4::writeEntityModuleBegin(ScriptDefModule* pEntityScriptDefModul fileBody() += fmt::format("\tScriptModule* getScriptModule();\n\n"); changeContextToSource(); + fileBody() += fmt::format("#include \"{}.h\"\n", newModuleName); fileBody() += fmt::format("#include \"KBVar.h\"\n"); fileBody() += fmt::format("#include \"Entity.h\"\n"); @@ -2140,7 +2170,8 @@ bool ClientSDKUE4::writeEntityModuleBegin(ScriptDefModule* pEntityScriptDefModul fileBody() += fmt::format("#include \"Method.h\"\n"); fileBody() += fmt::format("#include \"DataTypes.h\"\n"); fileBody() += fmt::format("#include \"CustomDataTypes.h\"\n"); - fileBody() += fmt::format("#include \"MemoryStream.h\"\n"); + fileBody() += fmt::format("#include \"MemoryStream.h\"\n\n"); + fileBody() += namespaceNameBegin; } else { @@ -2161,7 +2192,7 @@ bool ClientSDKUE4::writeEntityModuleBegin(ScriptDefModule* pEntityScriptDefModul fileBody() += fmt::format("#include \"DataTypes.h\"\n"); fileBody() += fmt::format("#include \"CustomDataTypes.h\"\n"); fileBody() += fmt::format("#include \"MemoryStream.h\"\n"); - fileBody() += fmt::format("#include \"EntityComponent.h\"\n"); + fileBody() += fmt::format("#include \"EntityComponent.h\"\n\n"); std::map includesHistroy; ScriptDefModule::PROPERTYDESCRIPTION_MAP clientPropertys = pEntityScriptDefModule->getClientPropertyDescriptions(); @@ -2178,9 +2209,11 @@ bool ClientSDKUE4::writeEntityModuleBegin(ScriptDefModule* pEntityScriptDefModul if (includesHistroy.find(pEntityComponentType->pScriptDefModule()->getName()) != includesHistroy.end()) continue; - fileBody() += fmt::format("#include \"Scripts/Components/{}.h\"\n", pEntityComponentType->pScriptDefModule()->getName()); + fileBody() += fmt::format("#include \"Scripts/Components/{}.h\"\n\n", pEntityComponentType->pScriptDefModule()->getName()); includesHistroy[pEntityComponentType->pScriptDefModule()->getName()] = pPropertyDescription; } + + fileBody() += namespaceNameBegin; } changeContextToHeader(); @@ -2299,11 +2332,14 @@ bool ClientSDKUE4::writeEntityModuleEnd(ScriptDefModule* pEntityScriptDefModule) fileBody() += fmt::format("\t{}->onDetached(this);\n", pPropertyDescription->getName()); } - fileBody() += fmt::format("}}\n\n"); + fileBody() += fmt::format("}}\n"); } + fileBody() += namespaceNameEnd; + changeContextToHeader(); - fileBody() += "\n};\n\n"; + fileBody() += "\n};\n"; + fileBody() += namespaceNameEnd; return true; } diff --git a/kbe/src/server/tools/kbcmd/main.cpp b/kbe/src/server/tools/kbcmd/main.cpp index 52af869765..0967b6da00 100644 --- a/kbe/src/server/tools/kbcmd/main.cpp +++ b/kbe/src/server/tools/kbcmd/main.cpp @@ -1,455 +1,482 @@ -// Copyright 2008-2018 Yolo Technologies, Inc. All Rights Reserved. https://www.comblockengine.com - -#include "server/kbemain.h" -#include "kbcmd.h" -#include "client_sdk.h" -#include "server_assets.h" -#include "entitydef/entitydef.h" -#include "pyscript/py_compression.h" -#include "pyscript/py_platform.h" - -#undef DEFINE_IN_INTERFACE -#include "machine/machine_interface.h" -#define DEFINE_IN_INTERFACE -#include "machine/machine_interface.h" - -#undef DEFINE_IN_INTERFACE -#include "client_lib/client_interface.h" -#define DEFINE_IN_INTERFACE -#include "client_lib/client_interface.h" - -#undef DEFINE_IN_INTERFACE -#include "baseappmgr/baseappmgr_interface.h" -#define DEFINE_IN_INTERFACE -#include "baseappmgr/baseappmgr_interface.h" - -#undef DEFINE_IN_INTERFACE -#include "cellappmgr/cellappmgr_interface.h" -#define DEFINE_IN_INTERFACE -#include "cellappmgr/cellappmgr_interface.h" - -#undef DEFINE_IN_INTERFACE -#include "cellapp/cellapp_interface.h" -#define DEFINE_IN_INTERFACE -#include "cellapp/cellapp_interface.h" - -#undef DEFINE_IN_INTERFACE -#include "baseapp/baseapp_interface.h" -#define DEFINE_IN_INTERFACE -#include "baseapp/baseapp_interface.h" - -#undef DEFINE_IN_INTERFACE -#include "loginapp/loginapp_interface.h" -#define DEFINE_IN_INTERFACE -#include "loginapp/loginapp_interface.h" - -#undef DEFINE_IN_INTERFACE -#include "dbmgr/dbmgr_interface.h" -#define DEFINE_IN_INTERFACE -#include "dbmgr/dbmgr_interface.h" - -#undef DEFINE_IN_INTERFACE -#include "tools/logger/logger_interface.h" -#define DEFINE_IN_INTERFACE -#include "tools/logger/logger_interface.h" - -#undef DEFINE_IN_INTERFACE -#include "tools/bots/bots_interface.h" -#define DEFINE_IN_INTERFACE -#include "tools/bots/bots_interface.h" - -#undef DEFINE_IN_INTERFACE -#include "tools/interfaces/interfaces_interface.h" -#define DEFINE_IN_INTERFACE -#include "tools/interfaces/interfaces_interface.h" - -using namespace KBEngine; - -#define PARSE_COMMAND_ARG_BEGIN() \ - for (int argIdx = 1; argIdx < argc; ++argIdx) \ - { \ - std::string cmd = argv[argIdx]; \ - std::string findcmd; \ - std::string::size_type fi1; \ - -#define PARSE_COMMAND_ARG_DO_FUNC(NAME, EXEC) \ - cmd = argv[argIdx]; \ - findcmd = NAME; \ - fi1 = cmd.find(findcmd); \ - if (fi1 != std::string::npos) \ - { \ - cmd.erase(fi1, findcmd.size()); \ - try \ - { \ - if (EXEC == -1) \ - return -1; \ - } \ - catch (...) \ - { \ - ERROR_MSG("parseCommandArgs: "#NAME"? invalid, no set! type is uint64\n"); \ - } \ - \ - continue; \ - } \ - -#define PARSE_COMMAND_ARG_DO_FUNC_RETURN(NAME, EXEC) \ - cmd = argv[argIdx]; \ - findcmd = NAME; \ - fi1 = cmd.find(findcmd); \ - if (fi1 != std::string::npos) \ - { \ - cmd.erase(fi1, findcmd.size()); \ - try \ - { \ - return EXEC; \ - } \ - catch (...) \ - { \ - ERROR_MSG("parseCommandArgs: "#NAME"? invalid, no set! type is uint64\n"); \ - } \ - \ - continue; \ - } \ - -#define PARSE_COMMAND_ARG_GET_VALUE(NAME, VAL) \ - cmd = argv[argIdx]; \ - findcmd = NAME; \ - fi1 = cmd.find(findcmd); \ - if (fi1 != std::string::npos) \ - { \ - cmd.erase(fi1, findcmd.size()); \ - if (cmd.size() > 0) \ - { \ - try \ - { \ - VAL = cmd; \ - } \ - catch (...) \ - { \ - ERROR_MSG("parseCommandArgs: "#NAME"? invalid, no set! type is uint64\n"); \ - } \ - } \ - \ - continue; \ - } \ - -#define PARSE_COMMAND_ARG_END() } - -int process_make_client_sdk(int argc, char* argv[], const std::string clientType) -{ - Resmgr::getSingleton().initialize(); - setEvns(); - loadConfig(); - - DebugHelper::initialize(g_componentType); - - INFO_MSG("-----------------------------------------------------------------------------------------\n\n\n"); - - Resmgr::getSingleton().print(); - - Network::EventDispatcher dispatcher; - DebugHelper::getSingleton().pDispatcher(&dispatcher); - - Network::g_SOMAXCONN = g_kbeSrvConfig.tcp_SOMAXCONN(g_componentType); - - Network::NetworkInterface networkInterface(&dispatcher); - - DebugHelper::getSingleton().pNetworkInterface(&networkInterface); - - KBCMD app(dispatcher, networkInterface, g_componentType, g_componentID); - - START_MSG(COMPONENT_NAME_EX(g_componentType), g_componentID); - - if (!app.initialize()) - { - ERROR_MSG("app::initialize(): initialization failed!\n"); - - app.finalise(); - - // ־δͬɣ ͬɲŽ - DebugHelper::getSingleton().finalise(); - return -1; - } - - std::vector scriptBaseTypes; - if (!EntityDef::initialize(scriptBaseTypes, g_componentType)) - { - ERROR_MSG("app::initialize(): EntityDef initialization failed!\n"); - - app.finalise(); - - // ־δͬɣ ͬɲŽ - DebugHelper::getSingleton().finalise(); - return -1; - } - - std::string path = ""; - std::string compressionfile = ""; - int compressionType = 0; - - PARSE_COMMAND_ARG_BEGIN(); - PARSE_COMMAND_ARG_GET_VALUE("--outpath=", path); - PARSE_COMMAND_ARG_END(); - - PARSE_COMMAND_ARG_BEGIN(); - PARSE_COMMAND_ARG_GET_VALUE("--zip=", compressionfile); - PARSE_COMMAND_ARG_END(); - - if (compressionfile.size() == 0) - { - PARSE_COMMAND_ARG_BEGIN(); - PARSE_COMMAND_ARG_GET_VALUE("--tar=", compressionfile); - PARSE_COMMAND_ARG_END(); - - compressionType = 2; - } - else - { - compressionType = 1; - } - - // ⵽zipļôzipļõpath - if (compressionfile.size() > 0) - { - std::vector tmpvec; - KBEngine::strutil::kbe_splits(compressionfile, ".", tmpvec); - path = tmpvec[0]; - compressionfile = tmpvec[1]; - } - - if (script::PyPlatform::pathExists(path) && !script::PyPlatform::rmdir(path)) { - ERROR_MSG(fmt::format("app::initialize(): delete directorys({}) error!\n", path)); - } - - ClientSDK* pClientSDK = ClientSDK::createClientSDK(clientType); - - int ret = 0; - - if (pClientSDK) - { - try - { - if (!pClientSDK->create(path)) - { - ret = -1; - } - } - catch (std::exception &err) - { - ERROR_MSG(fmt::format("app::initialize(): create clientsdk error({})!\n", err.what())); - } - } - else - { - ERROR_MSG(fmt::format("app::initialize(): create clientsdk error! nonsupport type={}\n", clientType)); - ret = -1; - } - - // ʼ - if (compressionfile.size() > 0) - { - if (compressionType == 1) - { - if (!script::PyCompression::zipCompressDirectory(path, (path + "." + compressionfile))) - { - ERROR_MSG("app::initialize(): compress zip error!\n"); - } - } - else if (compressionType == 2) - { - if (!script::PyCompression::tarCompressDirectory(path, (path + "." + compressionfile))) - { - ERROR_MSG("app::initialize(): compress tar error!\n"); - } - } - else - { - - } - - if (!script::PyPlatform::rmdir(path)) { - ERROR_MSG(fmt::format("app::initialize(): delete directorys({}) error!\n", path)); - } - } - - app.finalise(); - INFO_MSG(fmt::format("{}({}) has shut down. ClientSDK={}\n", COMPONENT_NAME_EX(g_componentType), g_componentID, pClientSDK->good())); - - // ־δͬɣ ͬɲŽ - DebugHelper::getSingleton().finalise(); - - if(pClientSDK) - delete pClientSDK; - - return ret; -} - -int process_newassets(int argc, char* argv[], const std::string assetsType) -{ - // assetsTypeKBE_RES_PATHĿ - std::string res_path = getenv("KBE_RES_PATH") == NULL ? "" : getenv("KBE_RES_PATH"); - std::string root_path = getenv("KBE_ROOT") == NULL ? "" : getenv("KBE_ROOT"); - - if (root_path[root_path.size() - 1] != '\\' && root_path[root_path.size() - 1] != '/') - root_path += "/"; - - std::string assets_sys_path_root = root_path + "kbe/res/sdk_templates/server/"; - -#if KBE_PLATFORM != PLATFORM_WIN32 - char splitflag = ':'; -#else - char splitflag = ';'; -#endif - - if (assetsType == "python") - assets_sys_path_root += "python_assets/"; - else - assets_sys_path_root += "python_assets/"; - - res_path += fmt::format("{}{}{}{}res/{}{}scripts/", splitflag, assets_sys_path_root, splitflag, assets_sys_path_root, splitflag, assets_sys_path_root); - setenv("KBE_RES_PATH", res_path.c_str(), 1); - - Resmgr::getSingleton().initialize(); - setEvns(); - loadConfig(); - - DebugHelper::initialize(g_componentType); - - INFO_MSG("-----------------------------------------------------------------------------------------\n\n\n"); - - Resmgr::getSingleton().print(); - - Network::EventDispatcher dispatcher; - DebugHelper::getSingleton().pDispatcher(&dispatcher); - - Network::g_SOMAXCONN = g_kbeSrvConfig.tcp_SOMAXCONN(g_componentType); - - Network::NetworkInterface networkInterface(&dispatcher); - - DebugHelper::getSingleton().pNetworkInterface(&networkInterface); - - KBCMD app(dispatcher, networkInterface, g_componentType, g_componentID); - - START_MSG(COMPONENT_NAME_EX(g_componentType), g_componentID); - - if (!app.initialize()) - { - ERROR_MSG("app::initialize(): initialization failed!\n"); - - app.finalise(); - - // ־δͬɣ ͬɲŽ - DebugHelper::getSingleton().finalise(); - return -1; - } - - std::vector scriptBaseTypes; - if (!EntityDef::initialize(scriptBaseTypes, g_componentType)) - { - ERROR_MSG("app::initialize(): EntityDef initialization failed!\n"); - - app.finalise(); - - // ־δͬɣ ͬɲŽ - DebugHelper::getSingleton().finalise(); - return -1; - } - - std::string path = ""; - - PARSE_COMMAND_ARG_BEGIN(); - PARSE_COMMAND_ARG_GET_VALUE("--outpath=", path); - PARSE_COMMAND_ARG_END(); - - ServerAssets* pServerAssets = ServerAssets::createServerAssets(assetsType); - - int ret = 0; - - if (pServerAssets) - { - try - { - if (!pServerAssets->create(path)) - { - ret = -1; - } - } - catch (std::exception &err) - { - ERROR_MSG(fmt::format("app::initialize(): create serverassets error({})!\n", err.what())); - } - } - else - { - ERROR_MSG(fmt::format("app::initialize(): create serverassets error! nonsupport type={}\n", assetsType)); - ret = -1; - } - - app.finalise(); - INFO_MSG(fmt::format("{}({}) has shut down. ServerAssets={}\n", COMPONENT_NAME_EX(g_componentType), g_componentID, pServerAssets->good())); - - // ־δͬɣ ͬɲŽ - DebugHelper::getSingleton().finalise(); - - if (pServerAssets) - delete pServerAssets; - - return ret; -} - -int process_getuid(int argc, char* argv[]) -{ - if (getUserUID() == 0) - { - autoFixUserDigestUID(); - } - - setenv("UID", fmt::format("{}", getUserUID()).c_str(), 1); - printf("%s", fmt::format("{}", getUserUID()).c_str()); - return getUserUID(); -} - -int process_help(int argc, char* argv[]) -{ - printf("Usage:\n"); - printf("--clientsdk:\n"); - printf("\tAutomatically generate client code based on entity_defs file. Environment variables based on KBE.\n"); - printf("\tkbcmd.exe --clientsdk=unity --outpath=c:/unity_kbesdk\n"); - printf("\tkbcmd.exe --clientsdk=ue4 --zip=c:/unity_kbesdk.zip\n"); - printf("\tkbcmd.exe --clientsdk=ue4 --tar=c:/unity_kbesdk.tgz\n"); - printf("\tkbcmd.exe --clientsdk=ue4 --outpath=c:/unity_kbesdk --KBE_ROOT=\"*\" --KBE_RES_PATH=\"*\" --KBE_BIN_PATH=\"*\"\n"); - - printf("\n--getuid\n"); - printf("\tReturns the ID of the server group.\n"); - - printf("\n--newassets\n"); - printf("\tCreate a new server game asset library, contains the necessary files.\n"); - printf("\tkbcmd.exe --newassets=python --outpath=c:/xserver_assets\n"); - - printf("\n--help:\n"); - printf("\tDisplay help information.\n"); - return 0; -} - -int main(int argc, char* argv[]) -{ - g_componentType = TOOL_TYPE; - g_componentID = 0; - - if (argc == 1) - { - return process_help(argc, argv); - } - - parseMainCommandArgs(argc, argv); - - PARSE_COMMAND_ARG_BEGIN(); - PARSE_COMMAND_ARG_DO_FUNC("--clientsdk=", process_make_client_sdk(argc, argv, cmd)); - PARSE_COMMAND_ARG_DO_FUNC_RETURN("--getuid", process_getuid(argc, argv)); - PARSE_COMMAND_ARG_DO_FUNC("--newassets=", process_newassets(argc, argv, cmd)); - PARSE_COMMAND_ARG_DO_FUNC("--help", process_help(argc, argv)); - PARSE_COMMAND_ARG_END(); - - return 0; -} - +// Copyright 2008-2018 Yolo Technologies, Inc. All Rights Reserved. https://www.comblockengine.com + +#include "server/kbemain.h" +#include "kbcmd.h" +#include "client_sdk.h" +#include "server_assets.h" +#include "entitydef/entitydef.h" +#include "entitydef/py_entitydef.h" +#include "pyscript/py_compression.h" +#include "pyscript/py_platform.h" + +#undef DEFINE_IN_INTERFACE +#include "machine/machine_interface.h" +#define DEFINE_IN_INTERFACE +#include "machine/machine_interface.h" + +#undef DEFINE_IN_INTERFACE +#include "client_lib/client_interface.h" +#define DEFINE_IN_INTERFACE +#include "client_lib/client_interface.h" + +#undef DEFINE_IN_INTERFACE +#include "baseappmgr/baseappmgr_interface.h" +#define DEFINE_IN_INTERFACE +#include "baseappmgr/baseappmgr_interface.h" + +#undef DEFINE_IN_INTERFACE +#include "cellappmgr/cellappmgr_interface.h" +#define DEFINE_IN_INTERFACE +#include "cellappmgr/cellappmgr_interface.h" + +#undef DEFINE_IN_INTERFACE +#include "cellapp/cellapp_interface.h" +#define DEFINE_IN_INTERFACE +#include "cellapp/cellapp_interface.h" + +#undef DEFINE_IN_INTERFACE +#include "baseapp/baseapp_interface.h" +#define DEFINE_IN_INTERFACE +#include "baseapp/baseapp_interface.h" + +#undef DEFINE_IN_INTERFACE +#include "loginapp/loginapp_interface.h" +#define DEFINE_IN_INTERFACE +#include "loginapp/loginapp_interface.h" + +#undef DEFINE_IN_INTERFACE +#include "dbmgr/dbmgr_interface.h" +#define DEFINE_IN_INTERFACE +#include "dbmgr/dbmgr_interface.h" + +#undef DEFINE_IN_INTERFACE +#include "tools/logger/logger_interface.h" +#define DEFINE_IN_INTERFACE +#include "tools/logger/logger_interface.h" + +#undef DEFINE_IN_INTERFACE +#include "tools/bots/bots_interface.h" +#define DEFINE_IN_INTERFACE +#include "tools/bots/bots_interface.h" + +#undef DEFINE_IN_INTERFACE +#include "tools/interfaces/interfaces_interface.h" +#define DEFINE_IN_INTERFACE +#include "tools/interfaces/interfaces_interface.h" + +using namespace KBEngine; + +#define PARSE_COMMAND_ARG_BEGIN() \ + for (int argIdx = 1; argIdx < argc; ++argIdx) \ + { \ + std::string cmd = argv[argIdx]; \ + std::string findcmd; \ + std::string::size_type fi1; \ + +#define PARSE_COMMAND_ARG_DO_FUNC(NAME, EXEC) \ + cmd = argv[argIdx]; \ + findcmd = NAME; \ + fi1 = cmd.find(findcmd); \ + if (fi1 != std::string::npos) \ + { \ + cmd.erase(fi1, findcmd.size()); \ + try \ + { \ + if (EXEC == -1) \ + return -1; \ + } \ + catch (...) \ + { \ + ERROR_MSG("parseCommandArgs: "#NAME"? invalid, no set! type is uint64\n"); \ + } \ + \ + continue; \ + } \ + +#define PARSE_COMMAND_ARG_DO_FUNC_RETURN(NAME, EXEC) \ + cmd = argv[argIdx]; \ + findcmd = NAME; \ + fi1 = cmd.find(findcmd); \ + if (fi1 != std::string::npos) \ + { \ + cmd.erase(fi1, findcmd.size()); \ + try \ + { \ + return EXEC; \ + } \ + catch (...) \ + { \ + ERROR_MSG("parseCommandArgs: "#NAME"? invalid, no set! type is uint64\n"); \ + } \ + \ + continue; \ + } \ + +#define PARSE_COMMAND_ARG_GET_VALUE(NAME, VAL) \ + cmd = argv[argIdx]; \ + findcmd = NAME; \ + fi1 = cmd.find(findcmd); \ + if (fi1 != std::string::npos) \ + { \ + cmd.erase(fi1, findcmd.size()); \ + if (cmd.size() > 0) \ + { \ + try \ + { \ + VAL = cmd; \ + } \ + catch (...) \ + { \ + ERROR_MSG("parseCommandArgs: "#NAME"? invalid, no set! type is uint64\n"); \ + } \ + } \ + \ + continue; \ + } \ + +#define PARSE_COMMAND_ARG_END() } + +int process_make_client_sdk(int argc, char* argv[], const std::string clientType) +{ + Resmgr::getSingleton().initialize(); + setEvns(); + loadConfig(); + + DebugHelper::initialize(g_componentType); + + INFO_MSG("-----------------------------------------------------------------------------------------\n\n\n"); + + Resmgr::getSingleton().print(); + + Network::EventDispatcher dispatcher; + DebugHelper::getSingleton().pDispatcher(&dispatcher); + + Network::g_SOMAXCONN = g_kbeSrvConfig.tcp_SOMAXCONN(g_componentType); + + Network::NetworkInterface networkInterface(&dispatcher); + + DebugHelper::getSingleton().pNetworkInterface(&networkInterface); + + KBCMD app(dispatcher, networkInterface, g_componentType, g_componentID); + + START_MSG(COMPONENT_NAME_EX(g_componentType), g_componentID); + + if (!app.initialize()) + { + ERROR_MSG("app::initialize(): initialization failed!\n"); + + app.finalise(); + + // ־δͬɣ ͬɲŽ + DebugHelper::getSingleton().finalise(); + return -1; + } + + if (!script::entitydef::installModule("EntityDef")) + { + ERROR_MSG("app::initialize(): EntityDef initialization failed!\n"); + + app.finalise(); + + // ־δͬɣ ͬɲŽ + DebugHelper::getSingleton().finalise(); + return -1; + } + + std::vector scriptBaseTypes; + if (!EntityDef::initialize(scriptBaseTypes, g_componentType)) + { + ERROR_MSG("app::initialize(): EntityDef initialization failed!\n"); + + script::entitydef::uninstallModule(); + app.finalise(); + + // ־δͬɣ ͬɲŽ + DebugHelper::getSingleton().finalise(); + return -1; + } + + std::string path = ""; + std::string compressionfile = ""; + int compressionType = 0; + + PARSE_COMMAND_ARG_BEGIN(); + PARSE_COMMAND_ARG_GET_VALUE("--outpath=", path); + PARSE_COMMAND_ARG_END(); + + PARSE_COMMAND_ARG_BEGIN(); + PARSE_COMMAND_ARG_GET_VALUE("--zip=", compressionfile); + PARSE_COMMAND_ARG_END(); + + if (compressionfile.size() == 0) + { + PARSE_COMMAND_ARG_BEGIN(); + PARSE_COMMAND_ARG_GET_VALUE("--tar=", compressionfile); + PARSE_COMMAND_ARG_END(); + + compressionType = 2; + } + else + { + compressionType = 1; + } + + // ⵽zipļôzipļõpath + if (compressionfile.size() > 0) + { + std::vector tmpvec; + KBEngine::strutil::kbe_splits(compressionfile, ".", tmpvec); + path = tmpvec[0]; + compressionfile = tmpvec[1]; + } + + if (script::PyPlatform::pathExists(path) && !script::PyPlatform::rmdir(path)) { + ERROR_MSG(fmt::format("app::initialize(): delete directorys({}) error!\n", path)); + } + + ClientSDK* pClientSDK = ClientSDK::createClientSDK(clientType); + + int ret = 0; + + if (pClientSDK) + { + try + { + if (!pClientSDK->create(path)) + { + ret = -1; + } + } + catch (std::exception &err) + { + ERROR_MSG(fmt::format("app::initialize(): create clientsdk error({})!\n", err.what())); + } + } + else + { + ERROR_MSG(fmt::format("app::initialize(): create clientsdk error! nonsupport type={}\n", clientType)); + ret = -1; + } + + // ʼ + if (compressionfile.size() > 0) + { + if (compressionType == 1) + { + if (!script::PyCompression::zipCompressDirectory(path, (path + "." + compressionfile))) + { + ERROR_MSG("app::initialize(): compress zip error!\n"); + } + } + else if (compressionType == 2) + { + if (!script::PyCompression::tarCompressDirectory(path, (path + "." + compressionfile))) + { + ERROR_MSG("app::initialize(): compress tar error!\n"); + } + } + else + { + + } + + if (!script::PyPlatform::rmdir(path)) { + ERROR_MSG(fmt::format("app::initialize(): delete directorys({}) error!\n", path)); + } + } + + script::entitydef::uninstallModule(); + app.finalise(); + INFO_MSG(fmt::format("{}({}) has shut down. ClientSDK={}\n", COMPONENT_NAME_EX(g_componentType), g_componentID, pClientSDK->good())); + + // ־δͬɣ ͬɲŽ + DebugHelper::getSingleton().finalise(); + + if(pClientSDK) + delete pClientSDK; + + return ret; +} + +int process_newassets(int argc, char* argv[], const std::string assetsType) +{ + // assetsTypeKBE_RES_PATHĿ + std::string res_path = getenv("KBE_RES_PATH") == NULL ? "" : getenv("KBE_RES_PATH"); + std::string root_path = getenv("KBE_ROOT") == NULL ? "" : getenv("KBE_ROOT"); + + if (root_path[root_path.size() - 1] != '\\' && root_path[root_path.size() - 1] != '/') + root_path += "/"; + + std::string assets_sys_path_root = root_path + "kbe/res/sdk_templates/server/"; + +#if KBE_PLATFORM != PLATFORM_WIN32 + char splitflag = ':'; +#else + char splitflag = ';'; +#endif + + if (assetsType == "python") + assets_sys_path_root += "python_assets/"; + else + assets_sys_path_root += "python_assets/"; + + res_path += fmt::format("{}{}{}{}res/{}{}scripts/", splitflag, assets_sys_path_root, splitflag, assets_sys_path_root, splitflag, assets_sys_path_root); + setenv("KBE_RES_PATH", res_path.c_str(), 1); + + Resmgr::getSingleton().initialize(); + setEvns(); + loadConfig(); + + DebugHelper::initialize(g_componentType); + + INFO_MSG("-----------------------------------------------------------------------------------------\n\n\n"); + + Resmgr::getSingleton().print(); + + Network::EventDispatcher dispatcher; + DebugHelper::getSingleton().pDispatcher(&dispatcher); + + Network::g_SOMAXCONN = g_kbeSrvConfig.tcp_SOMAXCONN(g_componentType); + + Network::NetworkInterface networkInterface(&dispatcher); + + DebugHelper::getSingleton().pNetworkInterface(&networkInterface); + + KBCMD app(dispatcher, networkInterface, g_componentType, g_componentID); + + START_MSG(COMPONENT_NAME_EX(g_componentType), g_componentID); + + if (!app.initialize()) + { + ERROR_MSG("app::initialize(): initialization failed!\n"); + + app.finalise(); + + // ־δͬɣ ͬɲŽ + DebugHelper::getSingleton().finalise(); + return -1; + } + + if (!script::entitydef::installModule("EntityDef")) + { + ERROR_MSG("app::initialize(): EntityDef initialization failed!\n"); + + app.finalise(); + + // ־δͬɣ ͬɲŽ + DebugHelper::getSingleton().finalise(); + return -1; + } + + std::vector scriptBaseTypes; + if (!EntityDef::initialize(scriptBaseTypes, g_componentType)) + { + ERROR_MSG("app::initialize(): EntityDef initialization failed!\n"); + + script::entitydef::uninstallModule(); + app.finalise(); + + // ־δͬɣ ͬɲŽ + DebugHelper::getSingleton().finalise(); + return -1; + } + + std::string path = ""; + + PARSE_COMMAND_ARG_BEGIN(); + PARSE_COMMAND_ARG_GET_VALUE("--outpath=", path); + PARSE_COMMAND_ARG_END(); + + ServerAssets* pServerAssets = ServerAssets::createServerAssets(assetsType); + + int ret = 0; + + if (pServerAssets) + { + try + { + if (!pServerAssets->create(path)) + { + ret = -1; + } + } + catch (std::exception &err) + { + ERROR_MSG(fmt::format("app::initialize(): create serverassets error({})!\n", err.what())); + } + } + else + { + ERROR_MSG(fmt::format("app::initialize(): create serverassets error! nonsupport type={}\n", assetsType)); + ret = -1; + } + + script::entitydef::uninstallModule(); + app.finalise(); + INFO_MSG(fmt::format("{}({}) has shut down. ServerAssets={}\n", COMPONENT_NAME_EX(g_componentType), g_componentID, pServerAssets->good())); + + // ־δͬɣ ͬɲŽ + DebugHelper::getSingleton().finalise(); + + if (pServerAssets) + delete pServerAssets; + + return ret; +} + +int process_getuid(int argc, char* argv[]) +{ + if (getUserUID() == 0) + { + autoFixUserDigestUID(); + } + + setenv("UID", fmt::format("{}", getUserUID()).c_str(), 1); + printf("%s", fmt::format("{}", getUserUID()).c_str()); + return getUserUID(); +} + +int process_help(int argc, char* argv[]) +{ + printf("Usage:\n"); + printf("--clientsdk:\n"); + printf("\tAutomatically generate client code based on entity_defs file. Environment variables based on KBE.\n"); + printf("\tkbcmd.exe --clientsdk=unity --outpath=c:/unity_kbesdk\n"); + printf("\tkbcmd.exe --clientsdk=ue4 --zip=c:/unity_kbesdk.zip\n"); + printf("\tkbcmd.exe --clientsdk=ue4 --tar=c:/unity_kbesdk.tgz\n"); + printf("\tkbcmd.exe --clientsdk=ue4 --outpath=c:/unity_kbesdk --KBE_ROOT=\"*\" --KBE_RES_PATH=\"*\" --KBE_BIN_PATH=\"*\"\n"); + + printf("\n--getuid\n"); + printf("\tReturns the ID of the server group.\n"); + + printf("\n--newassets\n"); + printf("\tCreate a new server game asset library, contains the necessary files.\n"); + printf("\tkbcmd.exe --newassets=python --outpath=c:/xserver_assets\n"); + + printf("\n--help:\n"); + printf("\tDisplay help information.\n"); + return 0; +} + +int main(int argc, char* argv[]) +{ + g_componentType = TOOL_TYPE; + g_componentID = 0; + + if (argc == 1) + { + return process_help(argc, argv); + } + + parseMainCommandArgs(argc, argv); + + PARSE_COMMAND_ARG_BEGIN(); + PARSE_COMMAND_ARG_DO_FUNC("--clientsdk=", process_make_client_sdk(argc, argv, cmd)); + PARSE_COMMAND_ARG_DO_FUNC_RETURN("--getuid", process_getuid(argc, argv)); + PARSE_COMMAND_ARG_DO_FUNC("--newassets=", process_newassets(argc, argv, cmd)); + PARSE_COMMAND_ARG_DO_FUNC("--help", process_help(argc, argv)); + PARSE_COMMAND_ARG_END(); + + return 0; +} + diff --git a/kbe/tools/server/pycluster/cluster_controller.py b/kbe/tools/server/pycluster/cluster_controller.py index f1e8649d58..2018872dd3 100644 --- a/kbe/tools/server/pycluster/cluster_controller.py +++ b/kbe/tools/server/pycluster/cluster_controller.py @@ -260,7 +260,7 @@ def sendStop(self, showDebug): print("\t\t%s : %i\t%s" % (ctype, len(clist), clist)) # 最好是尽量多的尝试次数，否则可能包未及时恢复造成后续查询错乱 - self.stopServer( COMPONENT_NAME2TYPE[ctype], 0, "", 3 ) + self.stopServer( COMPONENT_NAME2TYPE[ctype], 0, MACHINES_ADDRESS, 3 ) #print ("ClusterStopHandler::do: stop uid=%s, type=%s, send=%s" % (self.uid, ctype, \ # len(self.recvDatas) > 0 and self.recvDatas[0] == b'\x01'))