Skip to content

Commit

Permalink
call perspective api and render results
Browse files Browse the repository at this point in the history
  • Loading branch information
Kirk Shoop committed Sep 3, 2017
1 parent 0dfdabf commit a9ec935
Show file tree
Hide file tree
Showing 4 changed files with 164 additions and 22 deletions.
2 changes: 1 addition & 1 deletion imgui/imgui_impl_sdl_gl3.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
// SDL,GL3W
#include <SDL.h>
#include <SDL_syswm.h>
#include <GL/gl3w.h> // This example is using gl3w to access OpenGL functions (because it is small). You may use glew/glad/glLoadGen/etc. whatever already works for you.
#include <GL/glew.h> // This example is using gl3w to access OpenGL functions (because it is small). You may use glew/glad/glLoadGen/etc. whatever already works for you.

// Data
static double g_Time = 0.0f;
Expand Down
92 changes: 84 additions & 8 deletions main.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -741,7 +741,10 @@ int main(int argc, const char *argv[])
start_with(noop));
}

//
// send tweet text to sentiment api service.
//

auto sentimentupdates = ts |
onlytweets() |
buffer_with_time(milliseconds(500), tweetthread) |
Expand Down Expand Up @@ -780,8 +783,8 @@ int main(int argc, const char *argv[])
});
});
}) |
merge(poolthread) |
nooponerror() |
merge() |
nooponerror("sentiment") |
start_with(noop) |
as_dynamic();

Expand All @@ -793,6 +796,57 @@ int main(int argc, const char *argv[])
switch_on_next()
);

//
// send tweet text to perspective api service.
//

auto perspectiveupdates = ts |
onlytweets() |
rxo::map([=](const Tweet& tw) -> observable<Reducer> {
auto text = tweettext(tw.data->tweet);
auto ts = timestamp_ms(tw);
return perspectiverequest(poolthread, factory, settings["PerspectiveUrl"].get<string>(), settings["PerspectiveKey"].get<string>(), text) |
rxo::map([=](const string& body){
auto response = json::parse(body);

return Reducer([=](Model& m){
auto& scores = response["attributeScores"];

Perspective result{
scores["TOXICITY"]["summaryScore"]["value"].get<float>(),
scores["SPAM"]["summaryScore"]["value"].get<float>(),
scores["INFLAMMATORY"]["summaryScore"]["value"].get<float>()
};

m.data->perspective[tw.data->tweet["id_str"]] = result;

auto toxic = result.toxicity > 0.7f;

for (auto& word: tw.data->words) {
toxic && ++m.data->toxicwords[word];
}

updategroups(m, ts, length, [&](TweetGroup& tg){
toxic && ++tg.toxic;
});

return std::move(m);
});
});
}) |
merge() |
nooponerror("perspective") |
start_with(noop) |
as_dynamic();

reducers.push_back(
usePerspectiveApi |
rxo::map([=](bool use){
return use ? perspectiveupdates : just(noop) | as_dynamic();
}) |
switch_on_next()
);

// group tweets, that arrive, by the timestamp_ms value
reducers.push_back(
ts |
Expand Down Expand Up @@ -1274,6 +1328,15 @@ int main(int argc, const char *argv[])

// by group

if (ImGui::CollapsingHeader("Toxic Tweets Per Minute", ImGuiTreeNodeFlags_Framed))
{
auto& tpm = vm.data->toxictpm;
ImVec2 plotextent(ImGui::GetContentRegionAvailWidth(),100);
ImGui::PushStyleColor(ImGuiCol_PlotLines, negativecolor);
ImGui::PlotLines("", &tpm[0], tpm.size(), 0, nullptr, 0.0f, vm.data->maxtpm, plotextent);
ImGui::PopStyleColor(1);
}

if (ImGui::CollapsingHeader("Negative Tweets Per Minute", ImGuiTreeNodeFlags_Framed))
{
auto& tpm = vm.data->negativetpm;
Expand Down Expand Up @@ -1330,6 +1393,7 @@ int main(int argc, const char *argv[])
ImGui::RadioButton("Selected", &model::scope, scope_selected); ImGui::SameLine();
ImGui::RadioButton("All -", &model::scope, scope_all_negative); ImGui::SameLine();
ImGui::RadioButton("All +", &model::scope, scope_all_positive); ImGui::SameLine();
ImGui::RadioButton("All \u2620", &model::scope, scope_all_toxic); ImGui::SameLine();
ImGui::RadioButton("All", &model::scope, scope_all);

ImGui::Text("%s -> %s", vm.data->scope_begin.c_str(), vm.data->scope_end.c_str());
Expand Down Expand Up @@ -1374,16 +1438,20 @@ int main(int argc, const char *argv[])

for (auto& w : top) {

ImGui::Text("%4.d,", model::scope == scope_selected ? w.count : m.allwords[w.word]); ImGui::SameLine();
auto total = m.allwords[w.word];
ImGui::Text("%4.d,", total); ImGui::SameLine();
auto positive = m.positivewords[w.word];
ImGui::TextColored(positivecolor, " +%4.d,", positive); ImGui::SameLine();
auto negative = m.negativewords[w.word];
ImGui::TextColored(negativecolor, " -%4.d", negative); ImGui::SameLine();
auto toxic = m.toxicwords[w.word];
ImGui::TextColored(negativecolor, " \u2620%4.d", toxic); ImGui::SameLine();
if (negative > positive) {
ImGui::TextColored(negativecolor, " -%6.2fx", negative / std::max(float(positive), 1.0f)); ImGui::SameLine();
} else {
ImGui::TextColored(positivecolor, " +%6.2fx", positive / std::max(float(negative), 1.0f)); ImGui::SameLine();
}
ImGui::TextColored(negativecolor, " \u2620%6.2fx", total / std::max(float(toxic), 1.0f)); ImGui::SameLine();
ImGui::Text(" - %s", w.word.c_str());

ImVec2 plotextent(ImGui::GetContentRegionAvailWidth(),100);
Expand Down Expand Up @@ -1426,7 +1494,7 @@ int main(int argc, const char *argv[])
static vector<ImRect> taken;
taken.clear();

// start a reproducable series each frame.
// start a reproducible series each frame.
mt19937 source;

auto maxCount = 0;
Expand All @@ -1437,7 +1505,12 @@ int main(int argc, const char *argv[])

maxCount = max(maxCount, cursor->count);

auto color = cursor->word[0] == '@' ? mentioncolor : cursor->word[0] == '#' ? hashtagcolor : textcolor;
auto fallbackcolor = textcolor;
fallbackcolor = m.allwords[cursor->word] < m.negativewords[cursor->word] ? negativecolor : fallbackcolor;
fallbackcolor = m.negativewords[cursor->word] < m.positivewords[cursor->word] ? positivecolor : fallbackcolor;
fallbackcolor = m.positivewords[cursor->word] < m.toxicwords[cursor->word] ? negativecolor : fallbackcolor;

auto color = cursor->word[0] == '@' ? mentioncolor : cursor->word[0] == '#' ? hashtagcolor : fallbackcolor;
auto place = Clamp(static_cast<float>(cursor->count)/maxCount, 0.0f, 0.9999f);
auto size = Clamp(font->FontSize*scale*place, font->FontSize*scale*0.25f, font->FontSize*scale);
auto extent = font->CalcTextSizeA(size, fltmax, 0.0f, &cursor->word[0], &cursor->word[0] + cursor->word.size(), nullptr);
Expand Down Expand Up @@ -1485,7 +1558,7 @@ int main(int argc, const char *argv[])
ImGui::ColorEdit3("positivecolor", reinterpret_cast<float*>(&positivecolor), ImGuiColorEditFlags_Float);
ImGui::ColorEdit3("neutralcolor", reinterpret_cast<float*>(&neutralcolor), ImGuiColorEditFlags_Float);
ImGui::ColorEdit3("negativecolor", reinterpret_cast<float*>(&negativecolor), ImGuiColorEditFlags_Float);

if (ImGui::Button("Close"))
ImGui::CloseCurrentPopup();
}
Expand All @@ -1508,14 +1581,17 @@ int main(int argc, const char *argv[])
auto name = tweet["user"]["name"].get<string>();
auto screenName = tweet["user"]["screen_name"].get<string>();
auto sentiment = m.sentiment[tweet["id_str"]];
auto perspective = m.perspective[tweet["id_str"]];
auto color = sentiment == "positive" ? positivecolor : sentiment == "negative" ? negativecolor : neutralcolor;
auto text = tweettext(tweet);
auto passSentiment = model::scope == scope_all_negative ? sentiment == "negative" : model::scope == scope_all_positive ? sentiment == "positive" : true;
if (passSentiment && (filter.PassFilter(name.c_str()) || filter.PassFilter(screenName.c_str()) || filter.PassFilter(text.c_str()))) {
auto passPerspective = model::scope == scope_all_toxic ? perspective.toxicity > 0.7f : true;
if (passSentiment && passPerspective && (filter.PassFilter(name.c_str()) || filter.PassFilter(screenName.c_str()) || filter.PassFilter(text.c_str()))) {
--remaining;
ImGui::Separator();
ImGui::Text("%s (@%s) - ", name.c_str() , screenName.c_str() ); ImGui::SameLine();
ImGui::TextColored(color, "%s", sentiment.c_str());
ImGui::TextColored(color, "%s", sentiment.c_str()); ImGui::SameLine();
ImGui::Text(" \u2620%6.2f, %6.2f, %6.2f", perspective.toxicity, perspective.spam, perspective.inflammatory);
ImGui::TextWrapped("%s", text.c_str());
}
}
Expand Down
62 changes: 53 additions & 9 deletions model.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,8 +95,15 @@ struct TweetGroup
WordCountMap words;
int positive = 0;
int negative = 0;
int toxic = 0;
};

struct Perspective
{
float toxicity;
float spam;
float inflammatory;
};

struct Model
{
Expand All @@ -113,7 +120,9 @@ struct Model
WordCountMap allwords;
WordCountMap positivewords;
WordCountMap negativewords;
WordCountMap toxicwords;
unordered_map<string, string> sentiment;
unordered_map<string, Perspective> perspective;
};
shared_ptr<shared> data = make_shared<shared>();
};
Expand All @@ -122,10 +131,13 @@ using Reducer = function<Model(Model&)>;

auto noop = Reducer([](Model& m){return std::move(m);});

inline function<observable<Reducer>(observable<Reducer>)> nooponerror() {
return [](observable<Reducer> s){
inline function<observable<Reducer>(observable<Reducer>)> nooponerror(string from = string{}) {
return [=](observable<Reducer> s){
return s |
on_error_resume_next([](std::exception_ptr ep){
on_error_resume_next([=](std::exception_ptr ep){
if (!from.empty()) {
cerr << from << " - ";
}
cerr << rxu::what(ep) << endl;
return observable<>::empty<Reducer>();
}) |
Expand All @@ -151,7 +163,8 @@ int idx = 0;
const int scope_all = 1;
const int scope_all_negative = 2;
const int scope_all_positive = 3;
const int scope_selected = 4;
const int scope_all_toxic = 4;
const int scope_selected = 5;
static int scope = scope_selected;

struct ViewModel
Expand Down Expand Up @@ -204,19 +217,31 @@ struct ViewModel
ranges::action::sort([](const WordCount& l, const WordCount& r){
return l.count > r.count;
});
} else {
data->scope_words = &data->allwords;
data->allwords = model.allwords |
} else if (scope == scope_all_toxic) {
data->scope_words = &data->toxicwords;
data->toxicwords = model.toxicwords |
ranges::view::transform([&](const pair<string, int>& word){
return WordCount{word.first, word.second, {}};
});

data->allwords |=
data->toxicwords |=
ranges::action::sort([](const WordCount& l, const WordCount& r){
return l.count > r.count;
});
} else {
data->scope_words = &data->allwords;
}

data->allwords = model.allwords |
ranges::view::transform([&](const pair<string, int>& word){
return WordCount{word.first, word.second, {}};
});

data->allwords |=
ranges::action::sort([](const WordCount& l, const WordCount& r){
return l.count > r.count;
});

data->scope_tweets = &model.tweets;
data->scope_begin = model.groups.empty() ? string{} : utctextfrom(duration_cast<seconds>(model.groups.front().begin));
data->scope_end = model.groups.empty() ? string{} : utctextfrom(duration_cast<seconds>(model.groups.back().end));
Expand Down Expand Up @@ -274,6 +299,23 @@ struct ViewModel
return group.second;
});
}

{
vector<pair<milliseconds, float>> groups = model.groupedtweets |
ranges::view::transform([&](const pair<TimeRange, shared_ptr<TweetGroup>>& group){
return make_pair(group.first.begin, static_cast<float>(group.second->toxic));
});

groups |=
ranges::action::sort([](const pair<milliseconds, float>& l, const pair<milliseconds, float>& r){
return l.first < r.first;
});

data->toxictpm = groups |
ranges::view::transform([&](const pair<milliseconds, float>& group){
return group.second;
});
}
}

Model m;
Expand All @@ -284,10 +326,12 @@ struct ViewModel
vector<WordCount> allwords;
vector<WordCount> negativewords;
vector<WordCount> positivewords;

vector<WordCount> toxicwords;

vector<float> groupedtpm;
vector<float> positivetpm;
vector<float> negativetpm;
vector<float> toxictpm;
float maxtpm = 0.0f;

string scope_begin = {};
Expand Down
30 changes: 26 additions & 4 deletions tweets.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -245,8 +245,7 @@ auto twitterrequest = [](observe_on_one_worker tweetthread, ::rxcurl::rxcurl fac
finally([](){cerr << "end twitter stream request" << endl;}) |
merge(tweetthread);
}) |
twitter_stream_reconnection(tweetthread) |
subscribe_on(tweetthread);
twitter_stream_reconnection(tweetthread);
};

auto sentimentrequest = [](observe_on_one_worker worker, ::rxcurl::rxcurl factory, string url, string key, vector<string> text) -> observable<string> {
Expand Down Expand Up @@ -276,9 +275,32 @@ auto sentimentrequest = [](observe_on_one_worker worker, ::rxcurl::rxcurl factor
tap([=](exception_ptr){
cout << body << endl;
});
}) |
subscribe_on(worker);
});
};

auto perspectiverequest = [](observe_on_one_worker worker, ::rxcurl::rxcurl factory, string url, string key, string text) -> observable<string> {

std::map<string, string> headers;
headers["Content-Type"] = "application/json";

url += "?key=" + key;

auto body = json::parse(R"({"comment": {"text": ""}, "languages": ["en"], "requestedAttributes": {"TOXICITY":{}, "INFLAMMATORY":{}, "SPAM":{}}, "doNotStore": true })");

body["comment"]["text"] = text;

return observable<>::defer([=]() -> observable<string> {

return factory.create(http_request{url, "POST", headers, body.dump()}) |
rxo::map([](http_response r){
return r.body.complete;
}) |
merge(worker) |
tap([=](exception_ptr){
cout << body << endl;
});
});
};


}

0 comments on commit a9ec935

Please sign in to comment.