Skip to content
Permalink
Browse files

Speedup RideCache Load Time

Implement regex/hash based string substitution object to perform multiple
substitutions in 2 passes. Speeds up athlete data load by 2x.
Use QStringRef to avoid copy
Fixes #3234
  • Loading branch information
ericchristoffersen authored and amtriathlon committed Nov 30, 2019
1 parent b3a4b4a commit c13b24251e4ed0b9e8f17fd2d3322ed5137abbfc
Showing with 125 additions and 25 deletions.
  1. +4 −12 src/Core/RideDB.l
  2. +114 −1 src/Core/Utils.cpp
  3. +3 −0 src/FileIO/JsonRideFile.h
  4. +4 −12 src/FileIO/JsonRideFile.l
@@ -44,20 +44,12 @@ static QString unprotect(char *string)

// this is a lexer string so it will be enclosed
// in quotes. Lets strip those first
QString s = string2.mid(1,string2.length()-2);
QStringRef r = string2.midRef(1,string2.length()-2);

// does it end with a space (to avoid token conflict) ?
if (s.endsWith(" ")) s = s.mid(0, s.length()-1);

// now un-escape the control characters
s.replace("\\t", "\t"); // tab
s.replace("\\n", "\n"); // newline
s.replace("\\r", "\r"); // carriage-return
s.replace("\\b", "\b"); // backspace
s.replace("\\f", "\f"); // formfeed
s.replace("\\/", "/"); // solidus
s.replace("\\\"", "\""); // quote
s.replace("\\\\", "\\"); // backslash
if (r.endsWith(" ")) r = r.mid(0, r.length()-1);

QString s = Utils::RidefileUnEscape(r);

return s;
}
@@ -22,9 +22,122 @@
#include <QStringList>
#include <QDebug>
#include <QDir>
#include <QRegularExpression>

namespace Utils
{

// Class for performing multiple string substitutions in a single
// pass over string. This implementation is only valid when substitutions
// will not inject additional substitution opportunities.
class StringSubstitutionizer
{
QVector<QString> v;
QMap<QString, QString> qm;
QRegularExpression qr;

QString GetSubstitute(QString s) const
{
if (!qm.contains(s)) return QString();
return qm.value(s);
}

void BuildFindAnyRegex()
{
QString qRegexString;

for (int i = 0; i < v.size(); i++)
{
if (i > 0) qRegexString.append("|");

qRegexString.append(v[i]);
}

qr = QRegularExpression(qRegexString);
}

protected:

void PushSubstitution(QString regexstring, QString matchstring, QString substitute)
{
if (!qm.contains(matchstring))
{
v.push_back(regexstring);
qm[matchstring] = substitute;
}
}

void FinalizeInit()
{
BuildFindAnyRegex();
}

QRegularExpression GetFindAnyRegex() const
{
return qr;
}

public:

QString BuildSubstitutedString(QStringRef s) const
{
QRegularExpression qr = GetFindAnyRegex();

QRegularExpressionMatchIterator i = qr.globalMatch(s);

if (!i.hasNext())
return s.toString();

QString newstring;

unsigned iCopyIdx = 0;
do
{
QRegularExpressionMatch match = i.next();
int copysize = match.capturedStart() - iCopyIdx;

if (copysize > 0)
newstring.append(s.mid(iCopyIdx, copysize));

newstring.append(GetSubstitute(match.captured()));

iCopyIdx = (match.capturedStart() + match.captured().size());
} while (i.hasNext());

int copysize = s.size() - iCopyIdx;
if (copysize > 0)
newstring.append(s.mid(iCopyIdx, copysize));

return newstring;
}
};

struct RidefileUnEscaper : public StringSubstitutionizer
{
RidefileUnEscaper()
{
// regex match replacement
PushSubstitution("\\\\t", "\\t", "\t"); // tab
PushSubstitution("\\\\n", "\\n", "\n"); // newline
PushSubstitution("\\\\r", "\\r", "\r"); // carriage-return
PushSubstitution("\\\\b", "\\b", "\b"); // backspace
PushSubstitution("\\\\f", "\\f", "\f"); // formfeed
PushSubstitution("\\\\/", "\\/", "/"); // solidus
PushSubstitution("\\\\\"", "\\\"", "\""); // quote
PushSubstitution("\\\\\\\\", "\\\\", "\\"); // backslash

FinalizeInit();
}
};

QString RidefileUnEscape(const QStringRef s)
{
// Static const object constructs it's search regex at load time.
static const RidefileUnEscaper s_RidefileUnescaper;

return s_RidefileUnescaper.BuildSubstitutedString(s);
}

// when writing xml...
QString xmlprotect(const QString &string)
{
@@ -43,7 +156,7 @@ QString xmlprotect(const QString &string)
return s;
}

// BEWARE: this function is tide closely to RideFile parsing
// BEWARE: this function is tied closely to RideFile parsing
// DO NOT CHANGE IT UNLESS YOU KNOW WHAT YOU ARE DOING
QString unprotect(const QString &buffer)
{
@@ -29,6 +29,9 @@
#include <QDebug>
#define DATETIME_FORMAT "yyyy/MM/dd hh:mm:ss' UTC'"

namespace Utils {
QString RidefileUnEscape(QStringRef);
};

struct JsonFileReader : public RideFileReader {
virtual RideFile *openRideFile(QFile &file, QStringList &errors, QList<RideFile*>* = 0) const;
@@ -44,20 +44,12 @@ static QString unprotect(char *string)

// this is a lexer string so it will be enclosed
// in quotes. Lets strip those first
QString s = string2.mid(1,string2.length()-2);
QStringRef r = string2.midRef(1,string2.length()-2);

// does it end with a space (to avoid token conflict) ?
if (s.endsWith(" ")) s = s.mid(0, s.length()-1);

// now un-escape the control characters
s.replace("\\t", "\t"); // tab
s.replace("\\n", "\n"); // newline
s.replace("\\r", "\r"); // carriage-return
s.replace("\\b", "\b"); // backspace
s.replace("\\f", "\f"); // formfeed
s.replace("\\/", "/"); // solidus
s.replace("\\\"", "\""); // quote
s.replace("\\\\", "\\"); // backslash
if (r.endsWith(" ")) r = r.mid(0, r.length()-1);

QString s = Utils::RidefileUnEscape(r);

return s;
}

0 comments on commit c13b242

Please sign in to comment.
You can’t perform that action at this time.