issue #8375 Lowercase search does not find non-ASCII uppercase pages …

…and vice versa (part 2)

src/caseconvert.py

@@ -10,10 +10,10 @@
 
 def writeMapping(file,mapping):
     for k,v in sorted(mapping.items()):
-        file.write(u"    case %s /* %s */: return u8\"%s\" /* %s */;\n" %
-                (hex(ord(k[0])),k,v,",".join("0x{:02x}".format(ord(c)) for c in v)))
+        file.write(u"    case %s /* %s */: BSEQ(%s) /* %s */;\n" %
+               (hex(ord(k[0])), k, ",".join(f"0x{b:02x}" for b in v.encode('utf-8')), v))
 
-# create mappings of characters whose upper and lower case differs
+# create mappings of characters whose upper and lower case differ
 for codeValue in range(0,0x1FFFF):
         s = chr(codeValue)
         sl = s.lower()
@@ -32,6 +32,9 @@ def writeMapping(file,mapping):
 #include <cstdint>
 #include <string>
 
+#define BSEQ(...) { static unsigned char s[] = { __VA_ARGS__, 0x00 }; \
+                    return reinterpret_cast<const char *>(s); }
+
 inline const char *convertUnicodeToUpper(uint32_t code)
 {
   switch(code)

src/defgen.cpp

@@ -531,7 +531,7 @@ void generateDEF()
   }
 
   QCString fileName=outputDirectory+"/doxygen.def";
-  std::ofstream t(fileName.str(),std::ostream::out);
+  std::ofstream t(fileName.str(),std::ostream::out | std::ostream::binary);
   if (!t.is_open())
   {
     err("Cannot open file %s for writing!\n",fileName.data());

src/dotfilepatcher.cpp

@@ -466,7 +466,7 @@ bool DotFilePatcher::run() const
     // keep original SVG file so we can refer to it, we do need to replace
     // dummy link by real ones
     fi.open(tmpName,std::ifstream::in);
-    t.open(orgName,std::ofstream::out);
+    t.open(orgName,std::ofstream::out | std::ofstream::binary);
     if (!fi.is_open())
     {
       err("problem opening file %s for reading!\n",tmpName.c_str());

src/doxygen.cpp

@@ -10656,7 +10656,7 @@ static void writeTagFile()
   QCString generateTagFile = Config_getString(GENERATE_TAGFILE);
   if (generateTagFile.isEmpty()) return;
 
-  std::ofstream tagFile(generateTagFile.str(),std::ofstream::out);
+  std::ofstream tagFile(generateTagFile.str(),std::ofstream::out | std::ofstream::binary);
   if (!tagFile.is_open())
   {
     err("cannot open tag file %s for writing\n",

src/formula.cpp

@@ -349,7 +349,7 @@ void FormulaManager::generateImages(const char *path,Format format,HighDPI hd) c
         // read back %s_tmp.eps and replace
         // bounding box values with x1,y1,x2,y2 and remove the HiResBoundingBox
         std::ifstream epsIn(formBase.str()+"_tmp.eps",std::ifstream::in);
-        std::ofstream epsOut(formBase.str()+"_tmp_corr.eps",std::ofstream::out);
+        std::ofstream epsOut(formBase.str()+"_tmp_corr.eps",std::ofstream::out | std::ofstream::binary);
         if (epsIn.is_open() && epsOut.is_open())
         {
           std::string line;
@@ -427,7 +427,7 @@ void FormulaManager::generateImages(const char *path,Format format,HighDPI hd) c
   // generated images represent (we use this next time to avoid regeneration
   // of the images, and to avoid forcing the user to delete all images in order
   // to let a browser refresh the images).
-  t.open("formula.repository",std::ofstream::out);
+  t.open("formula.repository",std::ofstream::out | std::ofstream::binary);
   if (t.is_open())
   {
     for (int i=0; i<(int)p->formulas.size(); i++)

src/htmlgen.cpp

@@ -937,7 +937,7 @@ void HtmlGenerator::init()
   }
 
   {
-    std::ofstream t(dname+"/dynsections.js",std::ofstream::out);
+    std::ofstream t(dname+"/dynsections.js",std::ofstream::out | std::ofstream::binary);
     if (t.is_open())
     {
       t << mgr.getAsString("dynsections.js");
@@ -1011,7 +1011,7 @@ void HtmlGenerator::writeSearchData(const char *dname)
   }
 
   QCString searchDirName = dname;
-  std::ofstream t(searchDirName.str()+"/search.css",std::ofstream::out);
+  std::ofstream t(searchDirName.str()+"/search.css",std::ofstream::out | std::ofstream::binary);
   if (t.is_open())
   {
     QCString searchCss;
@@ -2563,7 +2563,7 @@ void HtmlGenerator::writeSearchPage()
 
   // OPENSEARCH_PROVIDER {
   QCString configFileName = htmlOutput+"/search_config.php";
-  std::ofstream t(configFileName.str(),std::ofstream::out);
+  std::ofstream t(configFileName.str(),std::ofstream::out | std::ofstream::binary);
   if (t.is_open())
   {
     t << "<?php\n\n";
@@ -2593,7 +2593,7 @@ void HtmlGenerator::writeSearchPage()
   // OPENSEARCH_PROVIDER }
 
   QCString fileName = htmlOutput+"/search.php";
-  t.open(fileName.str(),std::ofstream::out);
+  t.open(fileName.str(),std::ofstream::out | std::ofstream::binary);
   if (t.is_open())
   {
     t << substituteHtmlKeywords(g_header,"Search","");
@@ -2631,7 +2631,7 @@ void HtmlGenerator::writeSearchPage()
   t.close();
 
   QCString scriptName = htmlOutput+"/search/search.js";
-  t.open(scriptName.str(),std::ofstream::out);
+  t.open(scriptName.str(),std::ofstream::out | std::ofstream::binary);
   if (t.is_open())
   {
     t << ResourceMgr::instance().getAsString("extsearch.js");
@@ -2647,7 +2647,7 @@ void HtmlGenerator::writeExternalSearchPage()
   bool generateTreeView = Config_getBool(GENERATE_TREEVIEW);
   QCString dname = Config_getString(HTML_OUTPUT);
   QCString fileName = dname+"/search"+Doxygen::htmlFileExtension;
-  std::ofstream t(fileName.str(),std::ofstream::out);
+  std::ofstream t(fileName.str(),std::ofstream::out | std::ofstream::binary);
   if (t.is_open())
   {
     t << substituteHtmlKeywords(g_header,"Search","");
@@ -2698,7 +2698,7 @@ void HtmlGenerator::writeExternalSearchPage()
   t.close();
 
   QCString scriptName = dname+"/search/search.js";
-  t.open(scriptName.str(),std::ofstream::out);
+  t.open(scriptName.str(),std::ofstream::out | std::ofstream::binary);
   if (t.is_open())
   {
     t << "var searchResultsText=["

src/layout.cpp

@@ -1552,7 +1552,7 @@ void LayoutDocManager::parse(const char *fileName)
 
 void writeDefaultLayoutFile(const char *fileName)
 {
-  std::ofstream t(fileName,std::ofstream::out);
+  std::ofstream t(fileName,std::ofstream::out | std::ofstream::binary);
   bool ok = openOutputFile(fileName,t);
   if (!ok)
   {

src/rtfgen.cpp

@@ -2602,7 +2602,7 @@ bool RTFGenerator::preProcessFileInplace(const char *path,const char *name)
   QCString combinedName = (QCString)path+"/combined.rtf";
   QCString mainRTFName  = (QCString)path+"/"+name;
 
-  std::ofstream outt(combinedName.str(),std::ofstream::out);
+  std::ofstream outt(combinedName.str(),std::ofstream::out | std::ofstream::binary);
   if (!outt.is_open())
   {
     err("Failed to open %s for writing!\n",combinedName.data());

src/searchindex.cpp

@@ -16,6 +16,7 @@
 
 #include <ctype.h>
 #include <assert.h>
+#include <sstream>
 
 #include "searchindex.h"
 #include "config.h"
@@ -550,8 +551,25 @@ QCString searchName(const Definition *d)
 
 QCString searchId(const Definition *d)
 {
-  QCString s = searchName(d);
-  return convertUTF8ToLower(s.str());
+  std::string s = searchName(d).str();
+  std::ostringstream t(std::ios_base::ate);
+  for (size_t i=0;i<s.length();i++)
+  {
+    if (isId(s[i]))
+    {
+      t << s[i];
+    }
+    else // escape non-identifier characters
+    {
+      static const char *hex = "0123456789ABCDEF";
+      unsigned char uc = static_cast<unsigned char>(s[i]);
+      t << '_';
+      t << hex[uc>>4];
+      t << hex[uc&0xF];
+    }
+  }
+
+  return convertUTF8ToLower(t.str());
 }
 
 
@@ -1103,7 +1121,8 @@ void writeJavaScriptSearchIndex()
   }
 
   {
-    std::ofstream t(searchDirName.str()+"/searchdata.js",std::ofstream::out);
+    std::ofstream t(searchDirName.str()+"/searchdata.js",
+                    std::ofstream::out | std::ofstream::binary);
     if (t.is_open())
     {
       t << "var indexSectionsWithContent =\n";
@@ -1119,7 +1138,7 @@ void writeJavaScriptSearchIndex()
           for (const auto &kv : sii.symbolMap)
           {
             if ( kv.first == "\"" ) t << "\\";
-            t << kv.first.c_str();
+            t << kv.first;
           }
           t << "\"";
           j++;