Skip to content
Permalink
Browse files

Fixed issues with usernames containing unicode characters.

Summary:
Fixed bug in find last of when iterating over a string with unicode characters.
Switched all API calls that took a file path to read the contents of the f ile from memory to avoid the issue where an API did not support reading filepaths with unicode characters.
Fixed issue when initilizing the text editor when running the launcher that would result in a null access.
Fixed ImageProcessor/GeometryProcessor to read the wide version of the command line arguements to properly recieve commands containing unicode characters.
Fixes T2536.

Test Plan: Create a username with unicode characters on windows. Open the launcher. Create a new project, make sure the editor exe being launched is the built one from this change. The path will display ???? for unicode characters not in the font face, but will create the project properly in the correct location.

Reviewers: #zero_engine_developers, joshua.davis

Reviewed By: #zero_engine_developers, joshua.davis

Subscribers: joshua.davis, dane.curbow

Maniphest Tasks: T2536

Differential Revision: https://dev.zeroengine.io/D733
  • Loading branch information...
danecurbow committed Aug 3, 2018
1 parent 03ade06 commit 124a96e67f166c1e8c6c1de91076d11ab4e402c0
@@ -607,8 +607,11 @@ void TextEditor::InsertAutoCompleteText(const char* text, int length, int remove
void TextEditor::UseTextEditorConfig()
{
TextEditorConfig* config = GetConfig();
ConnectThisTo(config, Events::PropertyModified, OnConfigChanged);
this->UpdateConfig(config);
if (config)
{
ConnectThisTo(config, Events::PropertyModified, OnConfigChanged);
this->UpdateConfig(config);
}
}

void TextEditor::OnConfigChanged(PropertyEvent* event)
@@ -645,9 +648,14 @@ void TextEditor::OnColorSchemeChanged(ObjectEvent* event)

TextEditorConfig* TextEditor::GetConfig()
{
auto config = Z::gEditor->mConfig->has(TextEditorConfig);
ErrorIf(config == nullptr, "The config should always have a TextEditorConfig component");
return config;
// Check if the editor is present for when this is called from the launcher
if (Z::gEditor)
{
auto config = Z::gEditor->mConfig->has(TextEditorConfig);
ErrorIf(config == nullptr, "The config should always have a TextEditorConfig component");
return config;
}
return nullptr;
}

void TextEditor::UpdateConfig(TextEditorConfig* textConfig)
@@ -79,12 +79,18 @@ int GeometryImporter::ProcessModelFiles()
else
return GeometryProcessorCodes::Failed;

// set the flags for post process we want to run
// Set the flags for post process we want to run
uint flags = SetupAssimpPostProcess();

// load in the file
mScene = mAssetImporter.ReadFile(mInputFile.c_str(), flags);
ZPrint("Processing model: %s\n", FilePath::GetFileNameWithoutExtension(mInputFile).Data());
// Load the file into Assimp. We must use memory because their
// file functions do not call into our File wrappers. Also
// when file paths contain unicode characters Assimp fails to
// read the file.
DataBlock block = ReadFileIntoDataBlock(mInputFile.c_str());
mScene = mAssetImporter.ReadFileFromMemory(block.Data, block.Size, flags);
zDeallocate(block.Data);
ZPrint("Processing model: %s\n", FilePath::GetFileNameWithoutExtension(mInputFile).Data());


// An error has occurred, no scene imported
if (!mScene)
@@ -28,8 +28,10 @@ int main(int argc, char** argv)
Zero::Console::Add(&stdListener);
ZPrint("Running Geometry Processor\n");

int numArguments = 0;
wchar_t** commandLineArgs = CommandLineToArgvW(GetCommandLineW(), &numArguments);
Array<String> commandLine;
CommandLineToStringArray(commandLine, (cstr*)argv, argc);
CommandLineToStringArray(commandLine, commandLineArgs, numArguments);

Environment environment;
environment.ParseCommandArgs(commandLine);
@@ -28,8 +28,10 @@ int main(int argc, char** argv)
Zero::Console::Add(&stdListener);
ZPrint("Running Image Processor\n");

int numArguments = 0;
wchar_t** commandLineArgs = CommandLineToArgvW(GetCommandLineW(), &numArguments);
Array<String> commandLine;
CommandLineToStringArray(commandLine, (cstr*)argv, argc);
CommandLineToStringArray(commandLine, commandLineArgs, numArguments);

Environment environment;
environment.ParseCommandArgs(commandLine);
@@ -287,6 +287,7 @@ FontRasterizer::~FontRasterizer()
FT_Done_Face(mData->FontFace);
FT_Done_FreeType(mData->Library);
SafeDelete(mData);
SafeDelete(mFontSource.Data);
}

RenderFont* FontRasterizer::RasterNewFont(int fontHeight)
@@ -379,14 +380,14 @@ void FontRasterizer::LoadFontFace(int fontHeight)
//Always face index zero for now.
uint faceIndex = 0;

//Load the font from the font file
int errorCode = FT_New_Face(mData->Library, mFontObject->LoadPath.c_str(), faceIndex, &mData->FontFace);

//Load the font from the font file into memory
mFontSource = ReadFileIntoDataBlock(mFontObject->LoadPath.c_str());
// Create the font face from the file data now stored in memory
int errorCode = FT_New_Memory_Face(mData->Library, mFontSource.Data, mFontSource.Size, faceIndex, &mData->FontFace);

ErrorIf(errorCode == FT_Err_Unknown_File_Format, nullptr, "File is not a valid font file.");
ErrorIf(errorCode != 0, nullptr, "Bad file or path.");

//mGlyphSlot = mFontFace->glyph;

errorCode = FT_Set_Pixel_Sizes(mData->FontFace, 0, fontHeight);
ErrorIf(errorCode != 0, nullptr, "Pixel size failed for some reason.");

@@ -153,6 +153,7 @@ class FontRasterizer


Font* mFontObject;
DataBlock mFontSource;
RenderFont* mRenderFont;
Array<RenderGlyph> mGlyphInfo;
Image mFontImage;
@@ -706,35 +706,34 @@ TEST(UnicodeCompare)
TEST(UnicodeFindFirstOf)
{
WindowsDebugTimer timer("UnicodeFindFirstOf");
// 0000000000111111111122222222223333333333444
// 0123458901234567890123456789012345678901234
String text = Zero::Narrow(L"ąčę ėįšų ū ςερτυ θιοπα σδφγηξκλ΄ζχψωβνμ αω");

String text = Zero::Narrow(L"ąॐ𐋆čቹę ėįᚫš𐋉ų 𐌞ū ςॐ𐋍ερቹτυ θιοॐπα σδ𐌲φγηॐ𐋎ξκλ΄ζχॐ𐌑ψωβᚫνμ αक़𐌕ω𐌚");
StringRange textRange(text);

String first = Zero::Narrow(L"ėįšų");
String first = Zero::Narrow(L"ėįᚫš𐋉ų");
StringRange firstRange = text.FindFirstOf(first);
StringIterator startPos = textRange.Begin() + 4;
StringIterator endPos = textRange.Begin() + 8;
StringIterator startPos = textRange.Begin() + 7;
StringIterator endPos = textRange.Begin() + 13;
CHECK(firstRange.Begin() == startPos);
CHECK(firstRange.End() == endPos);

String second = Zero::Narrow(L"ςερτυ");
String second = Zero::Narrow(L"ςॐ𐋍ερቹτυ");
StringRange secondRange = text.FindFirstOf(second);
startPos = textRange.Begin() + 11;
endPos = textRange.Begin() + 16;
startPos = textRange.Begin() + 17;
endPos = textRange.Begin() + 25;
CHECK(secondRange.Begin() == startPos);
CHECK(secondRange.End() == endPos);

String third = Zero::Narrow(L"σδφγηξκλ");
String third = Zero::Narrow(L"σδ𐌲φγηॐ𐋎ξκλ");
StringRange thirdRange = text.FindFirstOf(third);
startPos = textRange.Begin() + 23;
endPos = textRange.Begin() + 31;
startPos = textRange.Begin() + 33;
endPos = textRange.Begin() + 44;
CHECK(thirdRange.Begin() == startPos);
CHECK(thirdRange.End() == endPos);

String fourth = Zero::Narrow(L"αω");
String fourth = Zero::Narrow(L"αक़𐌕ω𐌚");
StringRange fourthRange = text.FindFirstOf(fourth);
startPos = textRange.Begin() + 40;
startPos = textRange.Begin() + 56;
endPos = textRange.End();
CHECK(fourthRange.Begin() == startPos);
CHECK(fourthRange.End() == endPos);
@@ -743,35 +742,34 @@ TEST(UnicodeFindFirstOf)
TEST(UnicodeFindLastOf)
{
WindowsDebugTimer timer("UnicodeFindLastOf");
// 0000000000111111111122222222223333333333444
// 0123456789012345678901234567890123456789012
String text = Zero::Narrow(L"ąčę ėįšų ū ςερτυ θιοπα σδφγηξκλ΄ζχψωβνμ αω");

String text = Zero::Narrow(L"ąॐ𐋆čቹę ėįᚫš𐋉ų 𐌞ū ςॐ𐋍ερቹτυ θιοॐπα σδ𐌲φγηॐ𐋎ξκλ΄ζχॐ𐌑ψωβᚫνμ αक़𐌕ω𐌚");
StringRange textRange(text);

String first = Zero::Narrow(L"ėįšų");
String first = Zero::Narrow(L"ėįᚫš𐋉ų");
StringRange firstRange = text.FindLastOf(first);
StringIterator startPos = textRange.Begin() + 4;
StringIterator endPos = textRange.Begin() + 8;
StringIterator startPos = textRange.Begin() + 7;
StringIterator endPos = textRange.Begin() + 13;
CHECK(firstRange.Begin() == startPos);
CHECK(firstRange.End() == endPos);

String second = Zero::Narrow(L"ςερτυ");
String second = Zero::Narrow(L"ςॐ𐋍ερቹτυ");
StringRange secondRange = text.FindLastOf(second);
startPos = textRange.Begin() + 11;
endPos = textRange.Begin() + 16;
startPos = textRange.Begin() + 17;
endPos = textRange.Begin() + 25;
CHECK(secondRange.Begin() == startPos);
CHECK(secondRange.End() == endPos);

String third = Zero::Narrow(L"σδφγηξκλ");
String third = Zero::Narrow(L"σδ𐌲φγηॐ𐋎ξκλ");
StringRange thirdRange = text.FindLastOf(third);
startPos = textRange.Begin() + 23;
endPos = textRange.Begin() + 31;
startPos = textRange.Begin() + 33;
endPos = textRange.Begin() + 44;
CHECK(thirdRange.Begin() == startPos);
CHECK(thirdRange.End() == endPos);

String fourth = Zero::Narrow(L"αω");
String fourth = Zero::Narrow(L"αक़𐌕ω𐌚");
StringRange fourthRange = text.FindLastOf(fourth);
startPos = textRange.Begin() + 40;
startPos = textRange.Begin() + 56;
endPos = textRange.End();
CHECK(fourthRange.Begin() == startPos);
CHECK(fourthRange.End() == endPos);
@@ -264,19 +264,26 @@ StringRange StringRange::FindLastByBytes(cstr buffer, uint valueSizeInBytes) con
if (!valueSizeInBytes || valueSizeInBytes > rangeSize)
return StringRange();

size_t i = 0;
size_t last = rangeSize - valueSizeInBytes;

for (size_t i = 0; i <= last; i += UTF8::EncodedCodepointLength(mBegin[i]))
while(i <= last)
{
size_t j;
size_t runePointIndex = last - i;
for (j = 0; j < valueSizeInBytes; ++j)
{
if (mBegin[last - i + j] != buffer[j])
if (mBegin[runePointIndex + j] != buffer[j])
break;
}

if (j == valueSizeInBytes)
return StringRange(mOriginalString, mBegin + (last - i), mBegin + (last - i + j));
return StringRange(mOriginalString, mBegin + runePointIndex, mBegin + (runePointIndex + j));

do
{
--runePointIndex;
++i;
} while (IsContinuationByte(&mBegin[runePointIndex]) && runePointIndex > 0);
}

return StringRange();

0 comments on commit 124a96e

Please sign in to comment.
You can’t perform that action at this time.