Skip to content
Browse files

tweets are now saving into tweet database

  • Loading branch information...
1 parent 7c8ad63 commit d5a87d2f05245da0b5a398c3728844fe6d389785 @matthewfl committed Jul 26, 2009
Showing with 30 additions and 206 deletions.
  1. +0 −149 #TwitterChatBot.cpp#
  2. +0 −43 #TwitterChatBot.h#
  3. +2 −0 .gitignore
  4. +1 −0 Makefile
  5. +1 −1 README
  6. +24 −11 TwitterChatBot.cpp
  7. +1 −1 config.h
  8. +1 −1 twitterChat.cpp
View
149 #TwitterChatBot.cpp#
@@ -1,149 +0,0 @@
-#include "TwitterChatBot.h"
-#include <pthread.h>
-#include <curl/curl.h>
-#include <matthewfl/json.hpp>
-
-#include <string>
-#include <stdlib.h>
-#include <iostream>
-#include <sstream>
-using namespace std;
-
-vector<vector<string> > SqlQuery(string, sqlite3*);
-
-void TwitterChatBot::startFeed () {
- /// if feed is running do nothing
- if(!feedRunning) {
- feedRunning = true;
- pthread_create(&Pfeed, NULL, StreamThreadStart, (void*)this);
- }
-}
-
-void TwitterChatBot::stopFeed() {
- if(feedRunning) {
- pthread_cancel(Pfeed);
- curl_easy_pause(curl, CURLPAUSE_ALL);
- curl_easy_cleanup(curl);
- curl = NULL;
- feedRunning = false;
- }
-}
-
-void * TwitterChatBot::StreamThreadStart(void * self) {
- TwitterChatBot * s = reinterpret_cast<TwitterChatBot*>(self);
- // open the socket
- cout<<"here\n";
- CURL * curl = curl_easy_init();
- s->curl = curl;
- CURLcode result;
- // login in info
- curl_easy_setopt(curl, CURLOPT_HTTPAUTH, CURLAUTH_BASIC);
- curl_easy_setopt(curl, CURLOPT_USERPWD, s->loginInfo.c_str());
-
- // other the url to use
- curl_easy_setopt(curl, CURLOPT_URL, s->feedUrl.c_str());
-
- // other info
- curl_easy_setopt(curl, CURLOPT_USERAGENT, "libcurl-agent/1.0 matthewfl.com-chatBot/1.0");
- curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1);
- curl_easy_setopt(curl, CURLOPT_HEADER, 0);
- curl_easy_setopt(curl, CURLOPT_WRITEDATA, s);
- curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, StreamCallBack);
-
- cout << "preform\n";
- result = curl_easy_perform(curl);
-
- // thread done running
- s->feedRunning = false;
-
- // close the socket
- curl_easy_cleanup(curl);
- s->curl = NULL;
-
- return NULL; // what am I to return here?
-
-}
-
-size_t TwitterChatBot::StreamCallBack (char * str, size_t size, size_t nmemb, TwitterChatBot * self) {
- string s;
- s.append(str, size * nmemb);
- self->tempHolder += s;
- if(self->tempHolder.find("\n") != string::npos) {
- self->proccessStream(self->tempHolder);
- self->tempHolder="";
- }
- return size * nmemb;
-}
-
-void TwitterChatBot::proccessStream (string & data) {
- //cout << data << endl << endl;
- matthewfl::json j;
- stringstream message;
- j.prase(data);
- if(!j["delete"].empty()) {j["text"].cast<matthewfl::json::String>() // make this safe to insert
- << "\", " << j["user_id"].cast<matthewfl::json::Number>()
- << ", \"" << j["user_name"].cast<matthewfl::json::String>()
- << "\", " << j["id"].cast<matthewfl::json::Number>()
- << ", " << j["in_reply_to_user_id"].empty() ? "0" : j["in_reply_to_user_id"].cast<matthewfl::json::Number>()
- << ");";
- tweetQueue.push(message.str());
- if(!j["in_reply_to_user_id"].empty()) {
- cout << "reply to: ";
- cout << j["in_reply_to_user_id"].cast<matthewfl::json::Number>() << endl << endl;
- }
- }
-}
-
-void * TwitterChatBot::AddTweetData (void * s) {
- TwitterChatBot * self = reinterpret_cast<TwitterChatBot*>(s);
- while(1) {
- while(!self->tweetQueue.empty()) {
- SqlQuery(self->tweetQueue.front(), self->Tweet);
- self->tweetQueue.pop();
- }
- sleep(1);
- }
- return NULL;
-}
-
-unsigned int TwitterChatBot::queueSize() {
- return tweetQueue.size();
-}
-
-void TwitterChatBot::start() {
- if(!SystemRunning) {
- pthread_create(&PtweetData, NULL, AddTweetData, (void*)this);
- }
-}
-
-void TwitterChatBot::stop() {
- if(SystemRunning) {
- stopFeed();
- pthread_cancel(PtweetData);
- }
-}
-
-
-// database code
-
-
-static int SqlCallBack(void * passed, int argc, char **argv, char **azColName) {
- vector<vector<string> > * p = reinterpret_cast<vector<vector<string> >* >(passed);
- vector<string> data;
- for(int i=0; i<argc;i++) {
- data.push_back(string(argv[i]));
- }
- p->push_back(data);
- return 0;
-}
-
-vector<vector<string> > SqlQuery (string q, sqlite3 * db) {
- char *error=0;
- vector<vector<string> > passed;
- if(SQLITE_OK != sqlite3_exec(db, q.c_str(), SqlCallBack, (void*)(&passed), &error)) {
- cerr<<"sql error: "<<error<<endl;
- sqlite3_free(error);
- }
- return passed;
-}
-
View
43 #TwitterChatBot.h#
@@ -1,43 +0,0 @@
-#ifndef _Twitter_ChatBot_h
-#define _Twitter_ChatBot_h
-
-#include <pthread.h>
-#include <curl/curl.h>
-#include <string>
-#include <queue>
-#include <vector>
-#include <sqlite3.h>
-
-class TwitterChatBot
-{
-private:
- sqlite3 * Tweet;
- sqlite3 * Know;
- std::string loginInfo;
- std::queue<std::string> tweetQueue;
- pthread_t Pfeed, PtweetData;
- CURL * curl;
- bool feedRunning;
- std::string tempHolder;
- static void * StreamThreadStart (void*);
- static size_t StreamCallBack (char*, size_t, size_t, TwitterChatBot*);
- void proccessStream (std::string &);
- static void * AddTweetData (void*);
- bool SystemRunning;
-public:
- std::string feedUrl;
-
- TwitterChatBot (std::string user, std::string pass, sqlite3 * tweet, sqlite3 * know): Tweet(tweet), Know(know), loginInfo(user+":"+pass), feedRunning(false) { start(); };
- TwitterChatBot (std::string userpwd, sqlite3 * tweet, sqlite3 * know): Tweet(tweet), Know(know), loginInfo(userpwd), feedRunning(false) { start(); }
- ~TwitterChatBot () { stop(); }
- void startFeed();
- void stopFeed();
- void start();
- void stop();
- unsigned int queueSize();
-};
-
-
-
-
-#endif // _Twitter_ChatBot_h
View
2 .gitignore
@@ -1,4 +1,6 @@
*.o
*.database
+*.log
+\#*
# binary
/twitterChat
View
1 Makefile
@@ -2,6 +2,7 @@ CXXFLAGS = -O2 -Wall -fmessage-length=0
OBJS = twitterChat.o TwitterChatBot.o
+# hoard is not needed
LIBS = -lhoard -lpthread -lcurl -lsqlite3
# download libs from:
# hoard.org # not needed but helps
View
2 README
@@ -7,7 +7,7 @@ To build:
also the system can use hoard for better memory (http://hoard.org/) if not using you need to remove from the
libs in the makefile
- you might want to edit the setting in the make file
+ you might want to edit the setting in the config.h & make file
run make
then do:
sqlite3 tweets.database < tweet.sql
View
35 TwitterChatBot.cpp
@@ -1,6 +1,8 @@
#include "TwitterChatBot.h"
#include <pthread.h>
#include <curl/curl.h>
+#include <boost/algorithm/string/replace.hpp> // used for minor clean up on tweet when saving
+
#include <matthewfl/json.hpp>
#include <string>
@@ -20,6 +22,7 @@ void TwitterChatBot::startFeed () {
}
void TwitterChatBot::stopFeed() {
+ /// if feed not running do nothing
if(feedRunning) {
pthread_cancel(Pfeed);
curl_easy_pause(curl, CURLPAUSE_ALL);
@@ -76,7 +79,7 @@ size_t TwitterChatBot::StreamCallBack (char * str, size_t size, size_t nmemb, Tw
}
void TwitterChatBot::proccessStream (string & data) {
- //cout << data << endl << endl;
+ cout << data << endl << endl;
matthewfl::json j;
stringstream message;
j.prase(data);
@@ -88,22 +91,30 @@ void TwitterChatBot::proccessStream (string & data) {
<< d["id"].cast<matthewfl::json::Number>();
// TODO: make a delete spool for deleting tweets out of the data base
- tweetQueue.push(message.str());
+ // TODO: add back in when saving of tweets is working
+ // tweetQueue.push(message.str());
return;
}
if(!j["text"].empty()) {
- cout << j["text"].cast<matthewfl::json::String>() << endl;
+
+ matthewfl::json user = j["user"];
message << "INSERT INTO tweet (text, byID, byName, tweetID, toID) VALUES (\""
- << j["text"].cast<matthewfl::json::String>() // make this safe to insert
- << "\", " << j["user_id"].cast<matthewfl::json::Number>()
- << ", \"" << j["user_name"].cast<matthewfl::json::String>()
- << "\", " << j["id"].cast<matthewfl::json::Number>()
- << ", " << j["in_reply_to_user_id"].cast<matthewfl::json::Number>()
- << ");";
+ << boost::replace_all_copy(j["text"].cast<matthewfl::json::String>(), "\"", "\\\"") // make this safe to insert
+ << "\", " << user["id"].cast<matthewfl::json::Number>()
+ << ", \"" << user["screen_name"].cast<matthewfl::json::String>()
+ << "\", " << j["id"].cast<matthewfl::json::Number>() << ", ";
+ if(j["in_reply_to_user_id"].empty())
+ message << "NULL";
+ else
+ message << j["in_reply_to_user_id"].cast<matthewfl::json::Number>();
+ message << ");";
tweetQueue.push(message.str());
+ cout << "@" << user["screen_name"].cast<matthewfl::json::String>() << " : "
+ << j["text"].cast<matthewfl::json::String>() <<endl;
if(!j["in_reply_to_user_id"].empty()) {
- cout << "reply to: ";
- cout << j["in_reply_to_user_id"].cast<matthewfl::json::Number>() << endl << endl;
+ cout << "\t\t\treply to: ";
+ cout << j["in_reply_to_user_id"].cast<matthewfl::json::Number>()
+ << " " << j["in_reply_to_screen_name"].cast<matthewfl::json::String>() << endl;
}
}
}
@@ -115,12 +126,14 @@ void * TwitterChatBot::AddTweetData (void * s) {
SqlQuery(self->tweetQueue.front(), self->Tweet);
self->tweetQueue.pop();
}
+ cout << "database sleep\n";
sleep(1);
}
return NULL;
}
unsigned int TwitterChatBot::queueSize() {
+ /// sum all of the queues that are used
return tweetQueue.size();
}
View
2 config.h
@@ -17,7 +17,7 @@
// the max size of all the queues
#define Twitter_Queue_maxsize 500
-#define Twitter_Queue_checkTime 5
+#define Twitter_Queue_checkTime_sec 5
#endif // _tiwtter_config_h
View
2 twitterChat.cpp
@@ -42,7 +42,7 @@ int main (int argc, char *argv[]) {
}else{
twit.startFeed();
}
- sleep(Twitter_Queue_checkTime);
+ sleep(Twitter_Queue_checkTime_sec);
}
twit.stop();
sqlite3_close(dbTweet);

0 comments on commit d5a87d2

Please sign in to comment.
Something went wrong with that request. Please try again.