From 04bb7117a99e576b486055ba885f9a81badfe996 Mon Sep 17 00:00:00 2001 From: Alex Osman Date: Sat, 10 Feb 2018 19:52:38 -0500 Subject: [PATCH 1/2] filtering out posts and updating comments and votes correctly --- src/blockchainAPI/steemAPI.ts | 5 ++- src/bot.ts | 56 +++++++++++++++++------ src/classes/post.ts | 1 + src/database/database.ts | 5 ++- src/database/sqlDatabase.ts | 83 +++++++++++++++++++---------------- src/functions.ts | 12 +++++ src/main.ts | 12 +++-- src/settings.ts | 18 ++++++++ tsconfig.json | 4 ++ 9 files changed, 137 insertions(+), 59 deletions(-) create mode 100644 src/functions.ts create mode 100644 src/settings.ts diff --git a/src/blockchainAPI/steemAPI.ts b/src/blockchainAPI/steemAPI.ts index 3dce14c..9ffb3d5 100644 --- a/src/blockchainAPI/steemAPI.ts +++ b/src/blockchainAPI/steemAPI.ts @@ -11,13 +11,13 @@ export class SteemAPI { return new Promise((resolve, reject) => { steem.api.getDiscussionsByCreated({ "tag": tag, - "limit": 10 + "limit": 100 }, (err, result) => { if (err) { reject(err) } else { resolve(result - .filter(x => x.category === tag) + // .filter(x => x.category === tag) .map(post => ({ author: post.author, permlink: post.permlink, @@ -25,6 +25,7 @@ export class SteemAPI { votes: post.active_votes.length, did_comment: false, did_vote: false, + is_approved: false, }) as Post) ) } diff --git a/src/bot.ts b/src/bot.ts index 1fc21da..27322d4 100644 --- a/src/bot.ts +++ b/src/bot.ts @@ -3,14 +3,18 @@ import { Database } from './database/database' import { BlockchainAPI } from './blockchainAPI/blockchainAPI' import { User } from './classes/user'; import { weekFilter } from './filters'; +import { Post } from './classes/post'; +import { cleanScrape } from './functions'; const steem = require('steem') export class Bot { - week: number; // 5 + week: number; communityName: string; // nowplaying username: string; // nowplaying-music password: string; + users: User[]; + posts: Post[]; private _broadcaster: Broadcaster; private _blockchainAPI: BlockchainAPI; @@ -37,28 +41,54 @@ export class Bot { async setDatabase(database: Database): Promise { this._database = database await this._database.setup() + this.users = await this._database.getUsers() } async scrape(): Promise { try { - console.log(this._blockchainAPI) - const posts = await this._blockchainAPI.getPosts(this.communityName) - const write = await this._database.writePosts(posts, async post => { - try { - const comment = await this._broadcaster.makeComment(post) - } catch(e) { } - try { - const vote = await this._broadcaster.makeVote(post) - } catch(e) { } - return { } - }) - console.log(write) + const x = await this._blockchainAPI.getPosts(this.communityName) + const posts = await cleanScrape(x, this.users) + console.log(await this._database.writePosts(posts)) } catch(e) { console.log(e) console.log('got err') } } + async comment(): Promise { + try { + const allPosts = await this._database.getPosts() + const toCommentPosts = allPosts.filter(post => !post.did_comment) + + // Comment on each one with 20 second breaks + toCommentPosts.forEach((post, index) => { + setTimeout(() => { + this._broadcaster.makeComment(post) + this._database.writeComment(post) + }, index * 22 * 1000) + }) + } catch(e) { + console.log('something went wrong', e) + } + } + + async vote(): Promise { + try { + const allPosts = await this._database.getPosts() + const toVotePosts = allPosts.filter(post => !post.did_vote) + + // Comment on each one with 20 second breaks + toVotePosts.forEach((post, index) => { + setTimeout(() => { + this._broadcaster.makeVote(post) + this._database.writeVote(post) + }, index * 22 * 1000) + }) + } catch(e) { + console.log('something went wrong', e) + } + } + async payout(totalPayout: number): Promise { const allUsers = await this._database.getUsers() const weekUsers = allUsers.filter(weekFilter(this.week)) diff --git a/src/classes/post.ts b/src/classes/post.ts index 33f2296..e5c1447 100644 --- a/src/classes/post.ts +++ b/src/classes/post.ts @@ -13,6 +13,7 @@ export class Post { // SQL did_comment: boolean; did_vote: boolean; + is_approved: boolean; constructor() { this.jsonMetadata = {} as JsonMetadata diff --git a/src/database/database.ts b/src/database/database.ts index f167926..f46ae4e 100644 --- a/src/database/database.ts +++ b/src/database/database.ts @@ -5,5 +5,8 @@ export interface Database { setup: () => Promise getUsers: () => Promise getPosts: () => Promise - writePosts: (posts: Post[], onInsert: (post: Post) => Promise) => Promise<{ created: number, updated: number, total: number }> + + writePosts: (posts: Post[]) => Promise<{ created: number, updated: number, total: number }> + writeComment: (post: Post) => Promise + writeVote: (post: Post) => Promise } \ No newline at end of file diff --git a/src/database/sqlDatabase.ts b/src/database/sqlDatabase.ts index 27e0514..e4a2684 100644 --- a/src/database/sqlDatabase.ts +++ b/src/database/sqlDatabase.ts @@ -22,9 +22,12 @@ export class sqlDatabase { author: data.author, votes: data.votes, created: data.created, - permlink: data.permlink + permlink: data.permlink, + did_comment: data.did_comment, + did_vote: data.did_vote, + is_approved: data.is_approved }) as Post) - .filter((post) => post.author != 'loubega' && post.author != 'nowplaying-music') + .filter((post) => post.author != 'loubega') .reduce((users: User[], post) => { const user = users.find(user => user.username === post.author) if (user) { @@ -51,45 +54,47 @@ export class sqlDatabase { }) as Post) } - async writePosts(posts: Post[], onInsert: (post: Post) => Promise): Promise { - let count = 0; - try { - const insertResponses: { post: Post, result: any }[] = await Promise.all( - posts.map(async post => { - let result: any = { - err: true - } - try { - result = await this._con.query('INSERT INTO posts SET ?', [post]) - setTimeout(() => onInsert(post), 20*1000*count) - count++; - } catch (e) { - // console.log('error inserting prolly cause im there already') - } - return { - post, - result - } - }) - ) - const toUpdate = insertResponses.filter(res => !res.result.changedRows) - const updateResponses = await Promise.all(toUpdate.map(async postObj => ({ - post: postObj.post, - result: await this._con.query('UPDATE posts SET votes=? WHERE author=? AND permlink=?', [postObj.post.votes, postObj.post.author, postObj.post.permlink]) - }))) - return { - created: insertResponses.length - toUpdate.length, - updated: updateResponses.filter(res => res.result.changedRows).length, - total: insertResponses.length - } - } catch (e) { - if (e.code === 'ER_DUP_ENTRY') { + async writePosts(posts: Post[]): Promise { + const insertResponses: { post: Post, result: any }[] = await Promise.all( + posts.map(async post => { + let result: any = { + err: true + } + try { + result = await this._con.query('INSERT INTO posts SET ?', [post]) + } catch (e) { + // console.log('error inserting prolly cause im there already') + } return { - already: "there" + post, + result } - } else { - throw e - } + }) + ) + + const toUpdate = insertResponses.filter(res => !res.result.changedRows) + + const updateResponses = await Promise.all(toUpdate.map(async postObj => ({ + post: postObj.post, + result: await this._con.query('UPDATE posts SET votes=? WHERE author=? AND permlink=?', [postObj.post.votes, postObj.post.author, postObj.post.permlink]) + }))) + + return { + created: insertResponses.length - toUpdate.length, + updated: updateResponses.filter(res => res.result.changedRows).length, + total: insertResponses.length } } + + async writeComment(post: Post): Promise { + const result = await this._con.query('UPDATE posts SET did_comment=1 WHERE author=? AND permlink=?', [post.author, post.permlink]) + console.log(result) + return new Promise(resolve => resolve('hello')) + } + + async writeVote(post: Post): Promise { + const result = await this._con.query('UPDATE posts SET did_vote=1 WHERE author=? AND permlink=?', [post.author, post.permlink]) + console.log(result) + return new Promise(resolve => resolve('hello')) + } } \ No newline at end of file diff --git a/src/functions.ts b/src/functions.ts new file mode 100644 index 0000000..073aab7 --- /dev/null +++ b/src/functions.ts @@ -0,0 +1,12 @@ +import { Post } from './classes/post'; +import { User } from './classes/user'; +import { Settings } from './settings' + +export const cleanScrape = (posts: Post[], users: User[]): Post[] => posts + .filter(post => !Settings.blacklist.includes(post.author)) + .map(post => { + if (users.map(u => u.username).includes(post.author)) { + post.is_approved = true + } + return post + }) \ No newline at end of file diff --git a/src/main.ts b/src/main.ts index e5ba64c..f7e4f3e 100644 --- a/src/main.ts +++ b/src/main.ts @@ -7,9 +7,9 @@ import { SteemAPI } from './blockchainAPI/steemAPI'; const mysql = require('promise-mysql'); const local = { - host: 'localhost', - user: 'root', //process.env.DB_USER, - password: 'password', //process.env.DB_PASS, + host: '192.168.0.2', + user: 'nodice', //process.env.DB_USER, + password: '', //process.env.DB_PASS, database: 'nowplaying' }; @@ -23,9 +23,13 @@ const main = async () => { bot.setBroadcaster(new SteemBroadcaster()) bot.setBlockchainAPI(new SteemAPI()) await bot.setDatabase(new sqlDatabase(local)) + // bot.scrape() + bot.comment() + bot.vote() + // bot.scrape() // setInterval(() => bot.scrape(), 1000 * 60) // every minute - const payout = await bot.payout(1.425) + // const payout = await bot.payout(1.425) // console.log(payout) } diff --git a/src/settings.ts b/src/settings.ts new file mode 100644 index 0000000..2bc248c --- /dev/null +++ b/src/settings.ts @@ -0,0 +1,18 @@ +export const Settings = { + week: 6, + blacklist: [ + 'nowplaying-music', + + 'arsaljr', + 'clicknpict', + 'dianasteem7', + 'heroheri', + 'loubega', + 'maulinda', + 'rahmatz', + 'ryanananda', + 'safrizalpotret', + 'tasyazgs588', + ] + +} \ No newline at end of file diff --git a/tsconfig.json b/tsconfig.json index 345de55..2a0c1df 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,5 +1,9 @@ { "compilerOptions": { + "lib": [ + "dom", + "es7" + ], "target": "es6", "module": "commonjs", "moduleResolution": "node", From 8c76f62bb6ea09dd6741794a175bc512bab977c6 Mon Sep 17 00:00:00 2001 From: Alex Osman Date: Sat, 10 Feb 2018 19:54:05 -0500 Subject: [PATCH 2/2] set up the main method --- src/main.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/main.ts b/src/main.ts index f7e4f3e..fc23bbe 100644 --- a/src/main.ts +++ b/src/main.ts @@ -24,11 +24,15 @@ const main = async () => { bot.setBlockchainAPI(new SteemAPI()) await bot.setDatabase(new sqlDatabase(local)) // bot.scrape() - bot.comment() - bot.vote() + + + // every minute scrape, vote, and comment + setInterval(() => { + bot.scrape() + bot.vote() + bot.comment() + }, 1000 * 60) - // bot.scrape() - // setInterval(() => bot.scrape(), 1000 * 60) // every minute // const payout = await bot.payout(1.425) // console.log(payout) }