Skip to content

Commit

Permalink
Migrate CLI print logic to their own classes
Browse files Browse the repository at this point in the history
  • Loading branch information
@Bionus
Bionus committed Aug 22, 2020
1 parent e57c37f commit 4dccf2c
Show file tree
Hide file tree
Showing 10 changed files with 355 additions and 178 deletions.
143 changes: 11 additions & 132 deletions src/cli/src/main.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
#include <QUrl>
#include <stdexcept>
#include "downloader/downloader.h"
#include "downloader/printers/json-printer.h"
#include "downloader/printers/simple-printer.h"
#include "functions.h"
#include "logger.h"
#include "models/filtering/blacklist.h"
Expand All @@ -19,41 +21,6 @@
#include <vendor/qcommandlineparser.h>
#endif

void loadMoreDetails(const QList<QSharedPointer<Image>> &images);
QJsonObject serializeImg(const Image *image, const QMap<QString, Token>& tokens);

void returnJsonArray(const QJsonArray &array)
{
QJsonDocument jsonDoc;
jsonDoc.setArray(array);

QByteArray jsonResult = jsonDoc.toJson(QJsonDocument::Indented);
QTextStream(stdout) << qPrintable(jsonResult);

qApp->quit();
}

void serializeTags(const QList<Tag> &tags)
{
QJsonArray jsonArray;
for (const Tag &tag : tags) {
QJsonObject jsonObj;
tag.write(jsonObj);
jsonArray.append(jsonObj);
}
returnJsonArray(jsonArray);
}

void serializeImages(Profile *profile, const QList<QSharedPointer<Image>> &images)
{
QJsonArray jsonArray;
for (const auto &image : images) {
auto tokens = image->tokens(profile);
auto jsObject = serializeImg(image.data(), tokens);
jsonArray.append(jsObject);
}
returnJsonArray(jsonArray);
}

int main(int argc, char *argv[])
{
Expand Down Expand Up @@ -121,7 +88,7 @@ int main(int argc, char *argv[])
const QCommandLineOption returnCountOption(QStringList() << "rc" << "return-count", "Return total image count.");
const QCommandLineOption returnTagsOption(QStringList() << "rt" << "return-tags", "Return tags for a search.");
const QCommandLineOption returnPureTagsOption(QStringList() << "rp" << "return-pure-tags", "Return tags.");
const QCommandLineOption returnImagesOption(QStringList() << "ri" << "return-images", "Return images url.");
const QCommandLineOption returnImagesOption(QStringList() << "ri" << "return-images", "Return images.");
const QCommandLineOption downloadOption(QStringList() << "download", "Download found images.");
parser.addOption(returnCountOption);
parser.addOption(returnTagsOption);
Expand Down Expand Up @@ -175,6 +142,10 @@ int main(int argc, char *argv[])
}
}

Printer *printer = parser.isSet(jsonOption)
? (Printer*) new JsonPrinter(profile)
: (Printer*) new SimplePrinter(parser.value(tagsFormatOption));

if (parser.isSet(getDetailsOption)) {
if (sites.length() != 1) {
throw std::runtime_error("number of provided sites must be 1");
Expand All @@ -193,14 +164,7 @@ int main(int argc, char *argv[])
QObject::connect(&image, &Image::finishedLoadingTags, &loop, &QEventLoop::quit);
loop.exec();

auto tokens = image.tokens(profile);
auto jsObject = serializeImg(&image, tokens);

QJsonDocument jsonDoc;
jsonDoc.setObject(jsObject);
auto jsonResult = jsonDoc.toJson(QJsonDocument::Indented);
QTextStream(stdout) << qPrintable(jsonResult);

printer->print(image);
exit(0);
}

Expand All @@ -210,7 +174,7 @@ int main(int argc, char *argv[])
}

QString blacklistOverride = parser.value(tagsBlacklistOption);
Downloader *downloader = new Downloader(profile,
Downloader *downloader = new Downloader(profile, printer,
parser.value(tagsOption).split(" ", QString::SkipEmptyParts),
parser.value(postFilteringOption).split(" ", QString::SkipEmptyParts),
sites,
Expand All @@ -225,23 +189,10 @@ int main(int argc, char *argv[])
blacklistOverride.isEmpty() ? profile->getBlacklist() : Blacklist(blacklistOverride.split(' ')),
parser.isSet(noDuplicatesOption),
parser.value(tagsMinOption).toInt(),
parser.value(tagsFormatOption));
parser.isSet(loadDetailsOption));

downloader->setQuit(true);

// JSON output
if (parser.isSet(jsonOption)) {
downloader->setQuit(false);

QObject::connect(downloader, &Downloader::finishedTags, serializeTags);
QObject::connect(downloader, &Downloader::finishedImages, [&](const QList<QSharedPointer<Image>> &images) {
if (parser.isSet(loadDetailsOption)) {
loadMoreDetails(images);
}
serializeImages(profile, images);
});
}

// Load the correct data
if (parser.isSet(returnCountOption)) {
downloader->getPageCount();
Expand All @@ -251,7 +202,7 @@ int main(int argc, char *argv[])
downloader->getTags();
} else if (parser.isSet(returnImagesOption)) {
downloader->getUrls();
} else if (parser.isSet(downloadOption) || parser.isSet(jsonOption)) {
} else if (parser.isSet(downloadOption)) {
downloader->getImages();
} else {
parser.showHelp();
Expand All @@ -261,75 +212,3 @@ int main(int argc, char *argv[])

return app.exec();
}

void loadMoreDetails(const QList<QSharedPointer<Image>> &images)
{
int work = images.length();
QEventLoop loop;
int requestsLimit = 5; // simultan requests
int runningRequests = 0;
for (auto& image : images) {
while (runningRequests >= requestsLimit) {
QCoreApplication::processEvents(QEventLoop::AllEvents, 100);
}
runningRequests++;
image->loadDetails();
QObject::connect(image.data(), &Image::finishedLoadingTags, [&](){
work--;
runningRequests--;
if (!work) {
loop.quit();
}
});
}
loop.exec();
}

QJsonObject serializeImg(const Image *image, const QMap<QString, Token> &tokens)
{
static QStringList ignoreKeys = {"all", "allo", "allos", "all_namespaces", };
QJsonObject jsObject;

for (auto& key : tokens.keys()) {
typedef QVariant::Type Type;
if (ignoreKeys.contains(key)) {
continue;
}
if (key.contains("search_")) {
continue;
}

const QVariant& qvalue = tokens.value(key).value();
auto type = qvalue.type();

if (type == QVariant::Type::StringList) {
QStringList l = qvalue.toStringList();
if (l.isEmpty()) {
continue;
}
jsObject.insert(key, QJsonArray::fromStringList(l));
} else if (type == QVariant::Type::String) {
QString s = qvalue.toString();
if (s.isEmpty()) {
continue;
}
jsObject.insert(key, s);
} else if (type == Type::Url || type == Type::ULongLong || type == Type::LongLong) {
jsObject.insert(key, qvalue.toString());
} else if (type == Type::Int) {
jsObject.insert(key, qvalue.value<int>());
} else if (type == Type::Bool) {
jsObject.insert(key, qvalue.value<bool>());
} else if (type == Type::DateTime) {
jsObject.insert(key, static_cast<int>(qvalue.value<QDateTime>().toTime_t()));
} else {
qDebug() << qvalue;
log(QStringLiteral("using generic QVariant::toString for key: %1").arg(key), Logger::Warning);
jsObject.insert(key, qvalue.toString());
}
}
jsObject.insert("isVideo", image->isVideo());
jsObject.insert("isGallery", image->isGallery());
jsObject.insert("isAnimated", image->isAnimated());
return jsObject;
}
14 changes: 12 additions & 2 deletions src/gui/src/main/main.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@
#include <QTextStream>
#include "analytics.h"
#include "downloader/downloader.h"
#include "downloader/printers/json-printer.h"
#include "downloader/printers/simple-printer.h"
#include "functions.h"
#include "logger.h"
#include "main-window.h"
Expand Down Expand Up @@ -126,6 +128,8 @@ int main(int argc, char *argv[])
const QCommandLineOption verboseOption(QStringList() << "d" << "debug", "Show debug messages.");
const QCommandLineOption tagsMinOption(QStringList() << "tm" << "tags-min", "Minimum count for tags to be returned.", "count", "0");
const QCommandLineOption tagsFormatOption(QStringList() << "tf" << "tags-format", "Format for returning tags.", "format", "%tag\t%count\t%type");
const QCommandLineOption jsonOption(QStringList() << "j" << "json", "output results as json.");
const QCommandLineOption loadDetailsOption(QStringList() << "load-details", "request (more) details on found items.");
parser.addOption(tagsOption);
parser.addOption(sourceOption);
parser.addOption(pageOption);
Expand All @@ -142,6 +146,8 @@ int main(int argc, char *argv[])
parser.addOption(tagsFormatOption);
parser.addOption(noDuplicatesOption);
parser.addOption(verboseOption);
parser.addOption(jsonOption);
parser.addOption(loadDetailsOption);
const QCommandLineOption returnCountOption(QStringList() << "rc" << "return-count", "Return total image count.");
const QCommandLineOption returnTagsOption(QStringList() << "rt" << "return-tags", "Return tags for a search.");
const QCommandLineOption returnPureTagsOption(QStringList() << "rp" << "return-pure-tags", "Return tags.");
Expand Down Expand Up @@ -192,8 +198,12 @@ int main(int argc, char *argv[])
exit(1);
}

Printer *printer = parser.isSet(jsonOption)
? (Printer*) new JsonPrinter(profile)
: (Printer*) new SimplePrinter(parser.value(tagsFormatOption));

QString blacklistOverride = parser.value(tagsBlacklistOption);
Downloader *downloader = new Downloader(profile,
Downloader *downloader = new Downloader(profile, printer,
parser.value(tagsOption).split(" ", QString::SkipEmptyParts),
parser.value(postFilteringOption).split(" ", QString::SkipEmptyParts),
profile->getFilteredSites(parser.value(sourceOption).split(" ", QString::SkipEmptyParts)),
Expand All @@ -208,7 +218,7 @@ int main(int argc, char *argv[])
blacklistOverride.isEmpty() ? profile->getBlacklist() : Blacklist(blacklistOverride.split(' ')),
parser.isSet(noDuplicatesOption),
parser.value(tagsMinOption).toInt(),
parser.value(tagsFormatOption));
parser.isSet(loadDetailsOption));

if (parser.isSet(returnCountOption)) {
downloader->getPageCount();
Expand Down
74 changes: 41 additions & 33 deletions src/lib/src/downloader/downloader.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,12 @@
#include "downloader/downloader.h"
#include <QCoreApplication>
#include <QEventLoop>
#include <QObject>
#include <qmath.h>
#include <iostream>
#include <utility>
#include "downloader/image-downloader.h"
#include "downloader/printers/printer.h"
#include "functions.h"
#include "logger.h"
#include "models/api/api.h"
Expand All @@ -12,8 +16,32 @@
#include "tags/tag-api.h"


Downloader::Downloader(Profile *profile, QStringList tags, QStringList postFiltering, QList<Site*> sources, int page, int max, int perPage, QString location, QString filename, QString user, QString password, bool blacklist, Blacklist blacklistedTags, bool noDuplicates, int tagsMin, QString tagsFormat, Downloader *previous)
: m_profile(profile), m_lastPage(nullptr), m_tags(std::move(tags)), m_postFiltering(std::move(postFiltering)), m_sites(std::move(sources)), m_page(page), m_max(max), m_perPage(perPage), m_waiting(0), m_ignored(0), m_duplicates(0), m_tagsMin(tagsMin), m_location(std::move(location)), m_filename(std::move(filename)), m_user(std::move(user)), m_password(std::move(password)), m_blacklist(blacklist), m_noDuplicates(noDuplicates), m_tagsFormat(std::move(tagsFormat)), m_blacklistedTags(std::move(blacklistedTags)), m_cancelled(false), m_quit(false), m_previous(previous)
void loadMoreDetails(const QList<QSharedPointer<Image>> &images)
{
int work = images.length();
QEventLoop loop;
int requestsLimit = 5; // simultan requests
int runningRequests = 0;
for (auto& image : images) {
while (runningRequests >= requestsLimit) {
QCoreApplication::processEvents(QEventLoop::AllEvents, 100);
}
runningRequests++;
image->loadDetails();
QObject::connect(image.data(), &Image::finishedLoadingTags, [&](){
work--;
runningRequests--;
if (!work) {
loop.quit();
}
});
}
loop.exec();
}


Downloader::Downloader(Profile *profile, Printer *printer, QStringList tags, QStringList postFiltering, QList<Site*> sources, int page, int max, int perPage, QString location, QString filename, QString user, QString password, bool blacklist, Blacklist blacklistedTags, bool noDuplicates, int tagsMin, bool loadMoreDetails, Downloader *previous)
: m_profile(profile), m_printer(printer), m_lastPage(nullptr), m_tags(std::move(tags)), m_postFiltering(std::move(postFiltering)), m_sites(std::move(sources)), m_page(page), m_max(max), m_perPage(perPage), m_waiting(0), m_ignored(0), m_duplicates(0), m_tagsMin(tagsMin), m_loadMoreDetails(loadMoreDetails), m_location(std::move(location)), m_filename(std::move(filename)), m_user(std::move(user)), m_password(std::move(password)), m_blacklist(blacklist), m_noDuplicates(noDuplicates), m_blacklistedTags(std::move(blacklistedTags)), m_cancelled(false), m_quit(false), m_previous(previous)
{}

Downloader::~Downloader()
Expand Down Expand Up @@ -360,11 +388,7 @@ void Downloader::finishedLoadingImages(Page *page)
}
}

if (m_quit) {
downloadImages(images);
} else {
emit finishedImages(images);
}
downloadImages(images);
}

void Downloader::downloadImages(const QList<QSharedPointer<Image>> &images)
Expand Down Expand Up @@ -443,7 +467,7 @@ void Downloader::finishedLoadingUrls(Page *page)
}

QSet<QString> md5s;
QVector<QSharedPointer<Image>> images;
QList<QSharedPointer<Image>> images;
for (Page *p : qAsConst(m_pages)) {
for (const QSharedPointer<Image> &img : p->images()) {
// Blacklisted tags
Expand Down Expand Up @@ -473,51 +497,35 @@ void Downloader::finishedLoadingUrls(Page *page)
}
}

QStringList urls;
int i = 0;
for (const QSharedPointer<Image> &img : images) {
if (m_max <= 0 || i++ < m_max) {
urls.append(img->url().toString());
}
if (m_loadMoreDetails) {
loadMoreDetails(images);
}

if (m_quit) {
returnStringList(urls);
returnImageList(images);
} else {
emit finishedUrls(urls);
emit finishedImages(images);
}
}

void Downloader::returnInt(int ret)
{
std::cout << ret << std::endl;
m_printer->print(ret);
emit quit();
}
void Downloader::returnString(const QString &ret)
{
std::cout << ret.toStdString() << std::endl;
m_printer->print(ret);
emit quit();
}
void Downloader::returnTagList(const QList<Tag> &tags)
{
for (const Tag &tag : tags) {
QString ret = m_tagsFormat;
ret.replace("\\t", "\t");
ret.replace("\\n", "\n");
ret.replace("\\r", "\r");
ret.replace("%tag", tag.text());
ret.replace("%count", QString::number(tag.count()));
ret.replace("%type", tag.type().name());
ret.replace("%stype", QString::number(tag.type().number()));
std::cout << ret.toStdString() << std::endl;
}
m_printer->print(tags);
emit quit();
}
void Downloader::returnStringList(const QStringList &ret)
void Downloader::returnImageList(const QList<QSharedPointer<Image>> &ret)
{
for (const QString &str : ret) {
std::cout << str.toStdString() << std::endl;
}
m_printer->print(ret);
emit quit();
}

Expand Down
Loading

0 comments on commit 4dccf2c

Please sign in to comment.