Permalink
Browse files

added log

  • Loading branch information...
1 parent fae31cc commit ab0cf0c35c5952a5c159c20e6ba6a86c046ca3ef @liseen committed Aug 11, 2010
Showing with 49 additions and 11 deletions.
  1. +11 −1 main.cpp
  2. +33 −9 qfetcher.cpp
  3. +4 −0 qfetcher.h
  4. +1 −1 qfetcher.pro
View
@@ -7,18 +7,25 @@
#include <QNetworkRequest>
#include <QNetworkReply>
#include <QStringList>
-#include <QApplication>
+#include <QCoreApplication>
#include <qcontent_config.h>
+
#include "qfetcher.h"
+static qcontent::QFetcher *global_qfetcher = NULL;
+
static void signal_handler(int sig)
{
switch(sig) {
case SIGTERM:
case SIGHUP:
case SIGINT:
// TODO
+ if (global_qfetcher) {
+ global_qfetcher->stop();
+ }
+
break;
default:
break;
@@ -137,6 +144,9 @@ int main(int argc, char **argv)
qcontent::UrlQueue urlqueue(url_queue_host, url_queue_port);
qcontent::HubQueue crawled_queue(crawled_queue_host.c_str(), crawled_queue_port, crawled_queue_name);
qcontent::QFetcher fetcher(&urlqueue, &crawled_queue, multiple);
+
+ global_qfetcher = &fetcher;
+
fetcher.start();
app.exec();
View
@@ -6,6 +6,7 @@
#include <QUrl>
#include <QDateTime>
#include <QTime>
+#include <QCoreApplication>
#include <cassert>
#include <string>
#include <msgpack.hpp>
@@ -16,7 +17,7 @@ namespace qcontent {
void QFetcher::cycleFetch()
{
- DLOG(INFO) << "cycle fetch";
+ VLOG(1) << __FUNCTION__;
if (m_global_stop) {
return;
}
@@ -35,7 +36,7 @@ void QFetcher::cycleFetch()
void QFetcher::cyclePush()
{
- DLOG(INFO) << __FUNCTION__;
+ VLOG(1) << __FUNCTION__;
QList<std::string> tmp;
int size = m_record_buffer.size();
for (int i = 0; i < size; i++) {
@@ -56,7 +57,7 @@ void QFetcher::cyclePush()
void QFetcher::crawlUrl(const QUrl &qurl, QCrawlerRecord &record)
{
- DLOG(INFO) << "crawl: " << record.url;
+ VLOG(1) << "crawl: " << record.url;
QNetworkRequest request(qurl);
QNetworkReply *reply = m_manager.get(request);
@@ -66,26 +67,49 @@ void QFetcher::crawlUrl(const QUrl &qurl, QCrawlerRecord &record)
void QFetcher::start()
{
+ m_global_stop = false;
+
if (fetch_timer == NULL) {
fetch_timer = new QTimer(this);
+ connect(fetch_timer, SIGNAL(timeout()), this, SLOT(cycleFetch()));
}
if (push_timer == NULL) {
push_timer = new QTimer(this);
+ connect(push_timer, SIGNAL(timeout()), this, SLOT(cyclePush()));
}
- connect(fetch_timer, SIGNAL(timeout()), this, SLOT(cycleFetch()));
- connect(push_timer, SIGNAL(timeout()), this, SLOT(cyclePush()));
fetch_timer->start(1000);
push_timer->start(1000);
}
+void QFetcher::cycleStop()
+{
+ VLOG(1) << __FUNCTION__;
+ if (m_global_stop && m_current_downloads.size() == 0 &&
+ m_record_buffer.size() == 0) {
+ VLOG(1) << "quit application";
+ QCoreApplication::quit();
+ }
+}
+
+void QFetcher::stop()
+{
+ VLOG(1) << __FUNCTION__;
+ m_global_stop = true;
+ if (stop_timer == NULL) {
+ stop_timer = new QTimer(this);
+ connect(stop_timer, SIGNAL(timeout()), this, SLOT(cycleStop()));
+ stop_timer->start(1000);
+ }
+}
+
int QFetcher::fetchCrawlerRecord()
{
QCrawlerRecord record;
std::string record_str;
int ret = m_input_queue->pop_url(record_str);
- DLOG(INFO) << "pop_url ret: " << ret;
+ VLOG(1) << "pop_url ret: " << ret;
if (ret == QCONTENTHUB_OK) {
msgpack::zone zone;
msgpack::object obj;
@@ -94,7 +118,7 @@ int QFetcher::fetchCrawlerRecord()
obj.convert(&record);
} catch (std::exception& e) {
// TODO
- fprintf(stderr, "%s\n", e.what());
+ LOG(ERROR) << "uppack crawler record error " << e.what();
return QCONTENTHUB_ERROR;
}
QUrl qurl(QString::fromUtf8(record.url.c_str()));
@@ -114,7 +138,7 @@ int QFetcher::pushCrawlerRecord(const QCrawlerRecord &record)
int QFetcher::pushCrawlerRecord(const std::string &rec_str)
{
- DLOG(INFO) << __FUNCTION__;
+ VLOG(1) << __FUNCTION__;
int push_ret = m_out_queue->push_nowait(rec_str);
if (push_ret == QCONTENTHUB_OK) {
m_stop_crawl = false;
@@ -169,7 +193,7 @@ void QFetcher::downloadFinished(QNetworkReply *reply)
}
}
- DLOG(INFO) << "download: " << rec.url << " okay";
+ VLOG(1) << "download: " << rec.url << " okay";
m_current_downloads.remove(reply);
reply->deleteLater();
}
View
@@ -25,12 +25,14 @@ class QFetcher: public QObject
QFetcher(UrlQueue *input_queue, HubQueue *out_queue, int concurrent = 10) : m_input_queue(input_queue), m_out_queue(out_queue), m_concurrent(concurrent), m_stop_crawl(false) {
fetch_timer = NULL;
push_timer = NULL;
+ stop_timer = NULL;
m_global_stop = false;
connect(&m_manager, SIGNAL(finished(QNetworkReply*)),
SLOT(downloadFinished(QNetworkReply*)));
}
void start();
+ void stop();
void crawlUrl(const QUrl &qurl, QCrawlerRecord &record);
int fetchCrawlerRecord();
int pushCrawlerRecord(const QCrawlerRecord &record);
@@ -39,6 +41,7 @@ class QFetcher: public QObject
public slots:
void cycleFetch();
void cyclePush();
+ void cycleStop();
void downloadFinished(QNetworkReply *reply);
private:
@@ -49,6 +52,7 @@ public slots:
QList<std::string> m_record_buffer;
QTimer *fetch_timer;
QTimer *push_timer;
+ QTimer *stop_timer;
int m_concurrent;
volatile bool m_stop_crawl;
View
@@ -9,6 +9,6 @@ HEADERS += ../qcontentcommon/qcontent_config.h
INCLUDEPATH = /opt/qcrawler-thirdparty/include ../qcontentcommon ../libqcontenthub
LIBS = -L /opt/qcrawler-thirdparty/lib -Wl,-rpath,../libqcontenthub -L../libqcontenthub -lqcontenthub -lglog
-CONFIG += debug
+#CONFIG += debug
QT += network

0 comments on commit ab0cf0c

Please sign in to comment.