From 0acbbc5f3cef472fdd0a0fb56bbfc00979009669 Mon Sep 17 00:00:00 2001 From: Will Hunt Date: Thu, 30 Jun 2016 13:37:33 +0100 Subject: [PATCH] Database backend added & various fixes * Added Sqlite database for caching * Fixed process_tweet never resolving * Fixed messages not being emptied from queue * Profile names automatically update * Moved content upload function to util.js --- config.sample.yaml | 1 - package.json | 1 + src/AccountServices.js | 10 +- src/HashtagHandler.js | 2 - src/MatrixTwitter.js | 252 ++++++++++++++++++++++++++------------ src/TwitterDB.js | 200 ++++++++++++++++++++++++++++++ src/TwitterRoomHandler.js | 9 +- src/util.js | 51 ++++++++ twitter-as.js | 69 +++-------- 9 files changed, 460 insertions(+), 135 deletions(-) create mode 100644 src/TwitterDB.js create mode 100644 src/util.js diff --git a/config.sample.yaml b/config.sample.yaml index 2668305..d6a7f5c 100644 --- a/config.sample.yaml +++ b/config.sample.yaml @@ -5,4 +5,3 @@ bridge: app_auth: #Get these from twitter by creating an application through the API site. consumer_key: BLAHBLAHBLAH consumer_secret: BLAHBLAHBLAH - diff --git a/package.json b/package.json index 74de66c..afdb7b7 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ "matrix-js-sdk": "git+ssh://git@github.com/matrix-org/matrix-js-sdk.git#develop", "npmlog": "^3.0.0", "oauth": "^0.9.14", + "sqlite3": "^3.1.4", "twitter": "^1.3.0" } } diff --git a/src/AccountServices.js b/src/AccountServices.js index 8e2a07d..ff79bc5 100644 --- a/src/AccountServices.js +++ b/src/AccountServices.js @@ -63,7 +63,7 @@ AccountServices.prototype.processMessage = function (event, request, context){ } var intent = this._bridge.getIntent(); this._oauth_getAccessToken(pin,remoteSender).then(() => { - intent.sendMessage(event.room_id,{"body":"All good. You should now be able to use your twitter account on matrix.","msgtype":"m.text"}); + intent.sendMessage(event.room_id,{"body":"All good. You should now be able to use your Twitter account on Matrix.","msgtype":"m.text"}); }).catch(err => { intent.sendMessage(event.room_id,{"body":"We couldn't verify this PIN :(. Maybe you typed it wrong or you might need to request it again.","msgtype":"m.text"}); log.error("Handler.AccountServices","OAuth Access Token Failed:%s", err); @@ -77,7 +77,7 @@ AccountServices.prototype._oauth_getAccessToken = function (pin,remoteUser) { if(data && data.oauth_token != "" && data.oauth_secret != ""){ this._oauth.getOAuthAccessToken(data.oauth_token, data.oauth_secret, pin, (error,access_token,access_token_secret) =>{ if(error){ - reject(error); + reject(error.statusCode + ": " + error.data); } data.access_token = access_token; data.access_token_secret = access_token_secret; @@ -86,13 +86,15 @@ AccountServices.prototype._oauth_getAccessToken = function (pin,remoteUser) { resolve(); }); } - reject("User has no associated token request data"); + else { + reject("User has no associated token request data"); + } }); } AccountServices.prototype._oauth_getUrl = function(event,remoteUser){ return new Promise((resolve,reject) => { - this._oauth.getOAuthRequestToken({"x_auth_access_type":"write"},(error, oAuthToken, oAuthTokenSecret, results) => { + this._oauth.getOAuthRequestToken({"x_auth_access_type":"dm"},(error, oAuthToken, oAuthTokenSecret, results) => { if(error){ reject(["Couldn't get token for user.\n%s",error]); } diff --git a/src/HashtagHandler.js b/src/HashtagHandler.js index 7f2ff57..235408b 100644 --- a/src/HashtagHandler.js +++ b/src/HashtagHandler.js @@ -8,8 +8,6 @@ var HashtagHandler = function (bridge, twitter) { HashtagHandler.prototype.processInvite = function (event, request, context) { - var intent = this._bridge.getIntent(event.state_key); - intent.leave(event.room_id); return;//No invites } diff --git a/src/MatrixTwitter.js b/src/MatrixTwitter.js index d87d5bc..515b707 100644 --- a/src/MatrixTwitter.js +++ b/src/MatrixTwitter.js @@ -4,10 +4,12 @@ var fs = require('fs'); var log = require('npmlog'); var Buffer = require('buffer').Buffer; var HTMLDecoder = new require('html-entities').AllHtmlEntities; +var MatrixRoom = require("matrix-appservice-bridge").MatrixRoom; var ProcessedTweetList = require("./ProcessedTweetList.js"); +var util = require("./util.js"); -const TWITTER_PROFILE_INTERVAL_MS = 60000; +const TWITTER_PROFILE_INTERVAL_MS = 300000; const TWITTER_CLIENT_INTERVAL_MS = 60000; const TWITTER_MSG_QUEUE_INTERVAL_MS = 1500; @@ -15,11 +17,11 @@ const TWITTER_MSG_QUEUE_INTERVAL_MS = 1500; We pass in the full configuration to the constructor since eventually other settings will be placed into config (for flags and other forms of auth). */ -var MatrixTwitter = function (bridge, config) { +var MatrixTwitter = function (bridge, config, storage) { this.app_auth = config.app_auth; this.app_twitter = null; - this.tuser_cache = {}; this.tclients = {}; + this.storage = storage; this.timeline_list = []; this.timeline_queue = []; @@ -102,7 +104,7 @@ MatrixTwitter.prototype.get_bearer_token = function () { fs.readFile('bearer.tok',{encoding:'utf-8'}, (err, content) => { if(err){ log.warn('Twitter',"Token file not found or unreadable. Requesting new token."); - console.log(err); + log.error("Twitter",err); return this._get_bearer_http(); } //Test the token @@ -123,7 +125,7 @@ MatrixTwitter.prototype.get_bearer_token = function () { resolve(content); } else { - console.log(error); + log.error("Twitter",error); reject("Unexpected response to application/rate_limit_status during bearer token validation. Bailing."); } }); @@ -132,7 +134,39 @@ MatrixTwitter.prototype.get_bearer_token = function () { } MatrixTwitter.prototype._update_user_timeline_profile = function(profile){ - log.info("Twitter","[STUB] Update user profile for %s",profile.screen_name); + //It's a free request, store the data. + log.info("Twitter","Updating profile for %s", profile.screen_name); + var ts = new Date().getTime(); + return this.storage.get_profile_by_id(profile.id).then((old)=>{ + if(old != null){ + var sname = (old.profile.screen_name != profile.screen_name); + var name = (old.profile.name != profile.name); + var avatar = (old.profile.profile_background_image_url_https != profile.profile_background_image_url_https); + } + else { + sname = true; + name = true; + avatar = true; + } + + var muser = "@twitter_" + profile.id_str + ":" + this._bridge.opts.domain; + var intent = this._bridge.getIntent(muser); + if(name || sname){ + intent.setDisplayName(profile.name + " (@" + profile.screen_name + ")"); + log.info("Twitter","Setting new display name for %s", profile.screen_name); + } + + if(avatar){ + log.info("Twitter","Setting new avatar for %s", profile.screen_name); + //util.uploadContentFromUrl(this._bridge,profile.profile_background_image_url_https,muser).then((uri) =>{ + // intent.setAvatarUrl(uri); + //}); + } + + + + }); + this.storage.set_twitter_profile(profile.id,profile.screen_name,profile,ts); } MatrixTwitter.prototype._get_twitter_client = function(sender){ @@ -171,7 +205,6 @@ MatrixTwitter.prototype._get_twitter_client = function(sender){ client.get("account/verify_credentials",(error,profile) =>{ if(error){ delete this.tclients[id];//Invalidate it - console.log(error); log.error("Twitter","We couldn't reauthenticate with the supplied access token for " + id + ". Look into this."); reject("Twitter account could not be reauthenticated."); //TODO: Possibly find a way to get another key. @@ -242,8 +275,7 @@ MatrixTwitter.prototype.send_tweet_to_timeline = function(remote,sender,body,ext client.post("statuses/update",status,(error,tweet) => { if(error){ - log.error("Twitter","Failed to send tweet."); - console.log(error); + log.error("Twitter","Failed to send tweet. %s",error); return; } var id = sender.getId(); @@ -251,70 +283,49 @@ MatrixTwitter.prototype.send_tweet_to_timeline = function(remote,sender,body,ext log.info("Twitter","Tweet sent from %s!",id); }); }).catch(err =>{ - log.error("Twiter","Failed to send tweet. %s",err); + log.error("Twitter","Failed to send tweet. %s",err); }); } -MatrixTwitter.prototype.get_user_by_id = function(id) { - var ts = new Date().getTime(); - return new Promise((resolve, reject) => { - for (var i in this.tuser_cache) { - var cached = this.tuser_cache[i]; - if (ts - cached.cache_time > TWITTER_PROFILE_INTERVAL_MS) { - continue; - } - if (cached.user != null && cached.user.id_str == id) { - resolve(cached.user); - } +MatrixTwitter.prototype._get_user_from_twitter = function(data) { + return new Promise((resolve,reject) => { + this.app_twitter.get('users/show', data, (error, user, response) => { + if (error) { + log.error('Twitter',"get_user_by_id: GET /users/show returned: ", error); + reject(error); } - this.app_twitter.get('users/show', { - user_id: id - }, (error, user, response) => { - if (error) { - log.error('Twitter',"get_user_by_id: GET /users/show returned: ", error); - reject(error); - } else { - this.tuser_cache[user.screen_name] = { - cache_time: ts, - user: user - }; - resolve(user); - } - }); + ts = new Date().getTime(); + this._update_user_timeline_profile(user); + resolve(user); }); + }); } -MatrixTwitter.prototype.get_user = function(name) { - log.info('Twitter',"Looking up @" + name); - return new Promise((resolve, reject) => { - var ts = new Date().getTime(); - if (this.tuser_cache[name] != undefined) { - log.info('Twitter',"Checking cache for @" + name); - var cached = this.tuser_cache[name]; - if (ts - cached.cache_time < TWITTER_PROFILE_INTERVAL_MS) { - resolve(cached.user); - } - } - - log.info('Twitter',"Checking twitter for @" + name); - this.app_twitter.get('users/show', { - screen_name: name - }, (error, user, response) => { - if (error) { - this.tuser_cache[name] = { - cache_time: ts, - user: null - }; - reject(error); - } - this.tuser_cache[name] = { - cache_time: ts, - user: user - }; - resolve(user); - }); - }); +MatrixTwitter.prototype.get_user_by_id = function(id) { + log.info("Twitter","Looking up T" + id); + var ts = new Date().getTime(); + return this.storage.get_profile_by_id(id).then((profile)=>{ + if(profile != null){ + if(ts - profile.timestamp < TWITTER_PROFILE_INTERVAL_MS){ + return profile.profile; + } + } + return this._get_user_from_twitter({user_id:id}); + }); +} +MatrixTwitter.prototype.get_user = function(name) { + log.info("Twitter","Looking up @" + name); + var ts = new Date().getTime(); + return this.storage.get_profile_by_name(name).then((profile)=>{ + if(profile != null){ + if(ts - profile.timestamp < TWITTER_PROFILE_INTERVAL_MS){ + resolve(profile.profile); + return; + } + } + return this._get_user_from_twitter({screen_name:name}); + }); } /* @@ -369,7 +380,7 @@ MatrixTwitter.prototype.tweet_to_matrix_content = function(tweet, type) { MatrixTwitter.prototype._process_head_of_msg_queue = function(){ if(this.msg_queue.length > 0){ var msg = this.msg_queue.pop(); - log.info("Twitter","Pulling off queue:",msg.content.body); + //log.info("Twitter","Pulling off queue:",msg.content.body); var intent = this._bridge.getIntent(msg.userId); intent.sendEvent(msg.roomId, msg.type, msg.content).then( (id) => { //TODO: Cache this for..reasons. @@ -394,6 +405,24 @@ MatrixTwitter.prototype._push_to_msg_queue = function(muser,roomid,tweet,type){ this.msg_queue.push(newmsg); } +//Check user display name +//TODO: Check avatar. Probably with a stored key +MatrixTwitter.prototype._get_matrix_twitter_user = function(twitter_user){ + + // var cli = intent.getClient(); + // var current_user = cli.getUserIdLocalpart() + ":" + this._bridge.opts.domain; + // console.log(current_user); + // cli.getProfileInfo(current_user,'displayname').then(info =>{ + // console.log(info); + // var disp = tweet.user.name + " (@" + tweet.user.screen_name + ")"; + // if(info.displayname != disp){ + // intent.setDisplayName(disp); + // } + // }).catch(err => { + // log.error("Twitter",err); + // }); +} + /* Process a given tweet (including resolving any parent tweets), and submit it to the given room. This function is recursive, limited to the depth @@ -404,17 +433,16 @@ MatrixTwitter.prototype._push_to_msg_queue = function(muser,roomid,tweet,type){ function calls itself. */ MatrixTwitter.prototype.process_tweet = function(roomid, tweet, depth) { - //console.log(tweet); depth--; - var muser = "@twitter_" + tweet.user.id_str + ":" + bridge.opts.domain; + var muser = "@twitter_" + tweet.user.id_str + ":" + this._bridge.opts.domain; var intent = this._bridge.getIntent(muser); var type = "m.text"; if (tweet.in_reply_to_status_id_str != null) { type = "m.notice"; // A nicer way to show previous tweets } - log.info("Twitter","Processing tweet:",tweet.text); + //log.info("Twitter","Processing tweet:",tweet.text); return new Promise( (resolve) => { if (tweet.in_reply_to_status_id_str != null && depth > 0) { @@ -440,13 +468,18 @@ MatrixTwitter.prototype.process_tweet = function(roomid, tweet, depth) { resolve(); } }).then(() => { - log.info("Twitter","Putting on queue:",tweet.text); + + this._update_user_timeline_profile(tweet.user); + if(this.processed_tweets.contains(tweet.id_str)){ log.info("Twitter","Repeated tweet detected, not processing"); return; } + this.processed_tweets.push(tweet.id_str); this._push_to_msg_queue(muser,roomid,tweet,type); + return; + }); } @@ -472,7 +505,7 @@ MatrixTwitter.prototype._process_timeline = function(self) { this.app_twitter.get('statuses/user_timeline', req, (error, feed, response) => { if(error){ - log.error("Twitter","_process_timeline: GET /statuses/user_timeline returned: %s", error); + log.error("Twitter","_process_timeline: GET /statuses/user_timeline returned: %s", error.msg); return; } if (feed.length > 0) { @@ -485,8 +518,8 @@ MatrixTwitter.prototype._process_timeline = function(self) { feed.reverse().forEach((item) => { promises.push(this.process_tweet(tline.local.roomId, item, 3)); }); - Promise.all(promises).then(() =>{ + console.log("Done!"); this._bridge.getRoomStore().setRemoteRoom(tline.remote); this.msg_queue_intervalID = setInterval(() => {this._process_head_of_msg_queue();}, TWITTER_MSG_QUEUE_INTERVAL_MS); }); @@ -529,12 +562,14 @@ MatrixTwitter.prototype._process_hashtag_feed = function(){ return; } + var feed = this.hashtag_queue.shift(); var req = { q: "%23"+feed.hashtag, result_type: 'recent' }; + var since = feed.remote.get("twitter_since"); if (since != undefined) { req.since_id = since; @@ -578,15 +613,82 @@ MatrixTwitter.prototype.send_matrix_event_as_tweet = function(event,user,room){ this._downloadImage(url).then((buffer) =>{ return this.twitter.upload_media(user,buffer); }).then ((mediaId) => { - console.log(mediaId); this.send_tweet_to_timeline(room,user,"",{media:[mediaId]}); }).catch(err => { - log.error("Twitter","Failed to send image to timeline."); - console.error(error); + log.error("Twitter","Failed to send image to timeline. %s", err); }); } } +MatrixTwitter.prototype.attach_user_stream = function(ruser){ + return this._get_twitter_client(ruser).then((c) => { + var stream = c.stream('user', {with: "user"}); + stream.on('data', (data) => { + if(data.hasOwnProperty("direct_message")){ + this._process_incoming_dm(data.direct_message); + } + }); + + stream.on('error', function(error) { + log.error("Twitter","Stream gave an error %s",error); + }); + }); +} + +MatrixTwitter.prototype._process_incoming_dm = function(msg){ + //Find the room + var users = [msg.sender_id_str,msg.recipient_id_str].sort(); + this._bridge.getRoomStore().getMatrixRoom({twitter_type:"dm",twitter_users:users}).then((room) => { + //TODO: Check to see if the message exists. + if(!room){ + var intent = this._bridge.getIntentFromLocalpart("twitter_" + msg.sender_id_str); + log.info("Twitter.DM","Creating a new room for DMs from %s(%s) => %s(%s)",msg.sender_id_str,msg.sender_screen_name,msg.recipient_id_str,msg.recipient_screen_name); + intent.createRoom( + false, + opts = { + visibility: "private", + room_alias_name: "twitterdm_#"+msg.sender_id_str+"_"+msg.recipient_id_str, + invite:[ + "@twitter_" + msg.sender_id_str + ":" + this._bridge.opts.domain, + "@twitter_" + msg.recipient_id_str + ":" + this._bridge.opts.domain + ] + //name: "[Twitter] DM "+, + //topic: "Twitter feed for #"+name, + // initial_state: [ + // { + // "type": "m.room.join_rules", + // "content": { + // "join_rule": "public" + // }, + // "state_key": "" + // } + // ] + } + ).then(roomid => { + var mroom = new MatrixRoom(roomid); + mroom.set("twitter_type","dm"); + mroom.set("twitter_users",users); + this._bridge.getRoomStore().setMatrixRoom(mroom); + this._put_dm_in_room(roomid,msg); + }).catch(err => { + log.error("Twitter.DM","Failed to create DM room: %s",err); + }); + } + else{ + this._put_dm_in_room(room.roomId,msg); + } + }); +} + +//Send from sender to recipient + MatrixTwitter.prototype._put_dm_in_room = function(room,msg){ + var intent = this._bridge.getIntentFromLocalpart("twitter_" + msg.sender_id_str); + log.info("Twitter.DM","Attempting to send DM from %s(%s) => %s(%s)",msg.sender_id_str,msg.sender_screen_name,msg.recipient_id_str,msg.recipient_screen_name); + intent.sendEvent(room.roomId, "m.text", msg.text).then( (id) => { + //TODO: Cache this for....reasons + }); +} + module.exports = { MatrixTwitter: MatrixTwitter } diff --git a/src/TwitterDB.js b/src/TwitterDB.js new file mode 100644 index 0000000..9cbd2a0 --- /dev/null +++ b/src/TwitterDB.js @@ -0,0 +1,200 @@ +var SQLite3 = require('sqlite3').verbose(); +var log = require('npmlog'); + +var TwitterDB = function(filepath){ + this.db = new SQLite3.Database(filepath,(err) => { + if(err){ + log.error("TwitDB","Error opening database, %s"); + } + }); +} + +TwitterDB.prototype.init = function() { + this._create_profile_cache(); + this._create_twitter_table(); +} + +TwitterDB.prototype.get_profile_by_id = function(id){ + log.info("TwitDB","Retrieving profile: %s",id); + return new Promise((resolve,reject) =>{ + this.db.get( + ` + SELECT profile, timestamp + FROM user_cache + WHERE user_cache.id = $id; + ` + ,{ + $id: id + } + ,(err,row) =>{ + if(err != null){ + log.error("TwitDB","Error retrieving profile: %s",err.Error); + reject(err); + } + if(row !== undefined){ + row.profile = JSON.parse(row.profile); + resolve(row); + } + else { + resolve(null); + } + }); + }); +} + +TwitterDB.prototype.get_profile_by_name = function(name){ + log.info("TwitDB","Retrieving profile: %s",name); + return new Promise((resolve,reject) =>{ + this.db.get( + ` + SELECT profile, timestamp + FROM user_cache + WHERE user_cache.screenname = $name; + ` + ,{ + $id: id + } + ,(err,row) =>{ + if(err != null){ + log.error("TwitDB","Error retrieving profile: %s",err.Error); + reject(err); + return; + } + if(row !== undefined){ + row.profile = JSON.parse(row.profile); + resolve(row); + } + else { + resolve(null); + } + }); + }); +} + +TwitterDB.prototype.set_twitter_profile = function(id,name,data,timestamp){ + this.db.run( + ` + REPLACE INTO user_cache VALUES ($id,$name,$data,$timestamp); + ` + ,{ + $id: id, + $name: name, + $data: JSON.stringify(data), + $timestamp: timestamp + }, + function (err) { + if(err){ + log.error("TwitDB","Error storing profile: %s",err); + return; + } + log.info("TwitDB","Stored profile for %s",name); + }); +} + +//Caches every user profile we grab from Twitter so as to not go over our limits. +TwitterDB.prototype._create_profile_cache = function(){ + this.db.run( + ` + CREATE TABLE IF NOT EXISTS user_cache ( + id INTEGER UNIQUE NOT NULL, + screenname TEXT NOT NULL, + profile TEXT NOT NULL, + timestamp INTEGER NOT NULL, + PRIMARY KEY(id) + ) + `, + function (err) { + if(err){ + log.error("TwitDB","Error creating 'user_cache': %s",err); + return; + } + } + ); +} + +//Keeps track of links between matrix users and their accounts +TwitterDB.prototype._create_twitter_table = function(){ + this.db.run( + ` + CREATE TABLE IF NOT EXISTS twitter_account ( + user_id INTEGER UNIQUE NOT NULL, + oauth_token TEXT, + oauth_secret TEXT, + access_token TEXT, + access_token_secret TEXT, + twitter_id INTEGER, + PRIMARY KEY(user_id) + ) + `, + function (err) { + if(err){ + log.error("TwitDB","Error creating 'twitter_account': %s",err); + return; + } + } + ); +} + +TwitterDB.prototype.get_client_data = function(user_id){ + log.info("TwitDB","Retrieving client data: %s",user_id); + return new Promise((resolve,reject) =>{ + this.db.get( + ` + SELECT * + FROM twitter_account + WHERE twitter_account.user_id = $user_id; + ` + ,{ + $user_id: user_id + } + ,(err,row) =>{ + if(err != null){ + log.error("TwitDB","Error retrieving client data: %s",err.Error); + reject(err); + } + if(row !== undefined){ + resolve(row); + } + else { + resolve(null); + } + }); + }); +} + +TwitterDB.prototype.set_client_data = function(user_id,twitter_id,data){ + this.db.run( + ` + REPLACE INTO twitter_account VALUES ( + $user_id, + $oauth_token, + $oauth_secret, + $access_token, + $access_token_secret, + $twitter_id + ); + ` + ,{ + $user_id: user_id, + $twitter_id: twitter_id, + $oauth_token: data.oauth_token, + $oauth_secret: data.oauth_secret, + $access_token: data.access_token, + $access_token_secret: data.access_token_secret + }, + function (err) { + if(err){ + log.error("TwitDB","Error storing client data: %s",err); + return; + } + log.info("TwitDB","Stored client data for %s",user_id); + }); +} + +TwitterDB.prototype.close = function() { + this.db.close(); +} + +module.exports = { + TwitterDB: TwitterDB +} diff --git a/src/TwitterRoomHandler.js b/src/TwitterRoomHandler.js index 95bdc7e..864a959 100644 --- a/src/TwitterRoomHandler.js +++ b/src/TwitterRoomHandler.js @@ -70,15 +70,18 @@ TwitterRoomHandler.prototype.passEvent = function (request, context){ } TwitterRoomHandler.prototype.processAliasQuery = function(alias, aliasLocalpart){ - var type = aliasLocalpart.substr("twitter_".length,1); + var type = aliasLocalpart.substr("twitter_".length,2); var part = aliasLocalpart.substr("twitter_.".length); - if(type == '@'){ //User timeline + if(type[0] == '@'){ //User timeline return this.handlers.timeline.processAliasQuery(part); } - else if(type == '#') { //Hashtag + else if(type[0] == '#') { //Hashtag return this.handlers.hashtag.processAliasQuery(part); } + /*else if(type == 'DM') { //Hashtag + return this.handlers.directmessage.processAliasQuery(part.substr(1)); + }*/ else { //Unknown return null; diff --git a/src/util.js b/src/util.js new file mode 100644 index 0000000..1923ce5 --- /dev/null +++ b/src/util.js @@ -0,0 +1,51 @@ +var https = require('https'); +var Buffer = require("buffer").Buffer; +var log = require('npmlog'); + +/* + This function will take a URL, upload it to Matrix and return the corresponding + MXC url in a Promise. The content will be uploaded on the users behalf using + the ID, or the AS bot if set to null. +*/ +function uploadContentFromUrl(bridge, url, id = null, name = null) { + var contenttype; + return new Promise((resolve, reject) => { + https.get((url), (res) => { + contenttype = res.headers["content-type"]; + if (name == null) { + name = url.split("/"); + name = name[name.length - 1]; + } + var size = parseInt(res.headers["content-length"]); + var buffer = Buffer.alloc(size); + var bsize = 0; + res.on('data', (d) => { + d.copy(buffer, bsize); + bsize += d.length; + }); + res.on('error', () => { + reject("Failed to download."); + }); + res.on('end', () => { + resolve(buffer); + }); + }) + }).then((buffer) => { + return bridge.getIntent(id).getClient().uploadContent({ + stream: buffer, + name: name, + type: contenttype + }); + }).then((response) => { + var content_uri = JSON.parse(response).content_uri; + return content_uri; + log.info("UploadContent","Media uploaded to %s", content_uri); + }).catch(function(reason) { + log.error("UploadContent","Failed to get image from url:\n%s", reason) + }) + +} + +module.exports = { + uploadContentFromUrl: uploadContentFromUrl +} diff --git a/twitter-as.js b/twitter-as.js index dc87484..ed0880a 100644 --- a/twitter-as.js +++ b/twitter-as.js @@ -1,5 +1,3 @@ -var https = require('https'); -var Buffer = require("buffer").Buffer; var log = require('npmlog'); var Cli = require("matrix-appservice-bridge").Cli; @@ -12,7 +10,10 @@ var TwitterRoomHandler = require("./src/TwitterRoomHandler.js").TwitterRoomHandl var AccountServices = require("./src/AccountServices.js").AccountServices; var TimelineHandler = require("./src/TimelineHandler.js").TimelineHandler; var HashtagHandler = require("./src/HashtagHandler.js").HashtagHandler; +//var DirectMessageHandler = require("./src/DirectMessageHandler.js").DirectMessageHandler; +var TwitterDB = require("./src/TwitterDB.js").TwitterDB; +var util = require('./src/util.js'); var twitter; var troomstore; @@ -34,6 +35,7 @@ new Cli({ reg.addRegexPattern("users", "@twitter_.*", true); reg.addRegexPattern("aliases", "#twitter_@.*", true); reg.addRegexPattern("aliases", "#twitter_#.*", true); + reg.addRegexPattern("aliases", "#twitter_DM.*", true); callback(reg); }, run: function(port, config) { @@ -57,13 +59,17 @@ new Cli({ }); log.info("AppServ","Matrix-side listening on port %s", port); //Setup twitter - - twitter = new MatrixTwitter(bridge, config); + + var tstorage = new TwitterDB('twitter.db'); + tstorage.init(); + + twitter = new MatrixTwitter(bridge, config, tstorage); troomstore = new TwitterRoomHandler(bridge, config, { services: new AccountServices(bridge, config.app_auth), timeline: new TimelineHandler(bridge, twitter), - hashtag: new HashtagHandler(bridge, twitter) + hashtag: new HashtagHandler(bridge, twitter), + //directmessage: new DirectMessageHandler(bridge,twitter) } ); @@ -73,6 +79,13 @@ new Cli({ return bridge.loadDatabases(); }).then(() => { roomstore = bridge.getRoomStore(); + + // bridge.getUserStore().getRemoteUser("twitter_M@Half-Shot:localhost").then(value => { + // twitter.attach_user_stream(value); + // }).catch(reason =>{ + // throw reason; + // }); + return roomstore.getRemoteRooms({}); }).then((rooms) => { rooms.forEach((rroom, i, a) => { @@ -101,7 +114,7 @@ function userQuery(queriedUser) { return twitter.get_user_by_id(queriedUser.localpart.substr("twitter_".length)).then( (twitter_user) => { /* Even users with a default avatar will still have an avatar url set. This *should* always work. */ - return uploadContentFromUrl(bridge, twitter_user.profile_image_url_https, queriedUser.getId()).then((uri) => { + return util.uploadContentFromUrl(bridge, twitter_user.profile_image_url_https, queriedUser.getId()).then((uri) => { return { name: twitter_user.name + " (@" + twitter_user.screen_name + ")", url: uri, @@ -112,47 +125,3 @@ function userQuery(queriedUser) { log.error("UserQuery","Couldn't find the user.\nReason: %s",error); }); } - -/* - This function will take a URL, upload it to Matrix and return the corresponding - MXC url in a Promise. The content will be uploaded on the users behalf using - the ID, or the AS bot if set to null. -*/ -function uploadContentFromUrl(bridge, url, id = null, name = null) { - var contenttype; - return new Promise((resolve, reject) => { - https.get((url), (res) => { - contenttype = res.headers["content-type"]; - if (name == null) { - name = url.split("/"); - name = name[name.length - 1]; - } - var size = parseInt(res.headers["content-length"]); - var buffer = Buffer.alloc(size); - var bsize = 0; - res.on('data', (d) => { - d.copy(buffer, bsize); - bsize += d.length; - }); - res.on('error', () => { - reject("Failed to download."); - }); - res.on('end', () => { - resolve(buffer); - }); - }) - }).then((buffer) => { - return bridge.getIntent(id).getClient().uploadContent({ - stream: buffer, - name: name, - type: contenttype - }); - }).then((response) => { - var content_uri = JSON.parse(response).content_uri; - return content_uri; - log.info("UploadContent","Media uploaded to %s", content_uri); - }).catch(function(reason) { - log.error("UploadContent","Failed to get image from url:\n%s", reason) - }) - -}