Merge pull request #46 from alex-osman/improved-filtering

Improved filtering

src/blockchainAPI/steemAPI.ts

@@ -11,20 +11,21 @@ export class SteemAPI {
         return new Promise((resolve, reject) => {
             steem.api.getDiscussionsByCreated({
                 "tag": tag,
-                "limit": 10
+                "limit": 100
             }, (err, result) => {
                 if (err) {
                     reject(err)
                 } else {
                     resolve(result
-                        .filter(x => x.category === tag)
+                        // .filter(x => x.category === tag)
                         .map(post => ({
                             author: post.author,
                             permlink: post.permlink,
                             created: post.created,
                             votes: post.active_votes.length,
                             did_comment: false,
                             did_vote: false,
+                            is_approved: false,
                         }) as Post)
                     )
                 }

src/bot.ts

@@ -3,14 +3,18 @@ import { Database } from './database/database'
 import { BlockchainAPI } from './blockchainAPI/blockchainAPI'
 import { User } from './classes/user';
 import { weekFilter } from './filters';
+import { Post } from './classes/post';
+import { cleanScrape } from './functions';
 
 const steem = require('steem')
 
 export class Bot {
-    week: number;          // 5
+    week: number;
     communityName: string; // nowplaying
     username: string;      // nowplaying-music
     password: string;
+    users: User[];
+    posts: Post[];
 
     private _broadcaster: Broadcaster;
     private _blockchainAPI: BlockchainAPI;
@@ -37,28 +41,54 @@ export class Bot {
     async setDatabase(database: Database): Promise<void> {
         this._database = database
         await this._database.setup()
+        this.users = await this._database.getUsers()
     }
 
     async scrape(): Promise<any> {
         try {
-            console.log(this._blockchainAPI)
-            const posts = await this._blockchainAPI.getPosts(this.communityName)
-            const write = await this._database.writePosts(posts, async post => {
-                try {
-                    const comment = await this._broadcaster.makeComment(post)
-                } catch(e) { }
-                try {
-                    const vote = await this._broadcaster.makeVote(post)
-                } catch(e) { }
-                return { }
-            })
-            console.log(write)
+            const x = await this._blockchainAPI.getPosts(this.communityName)
+            const posts = await cleanScrape(x, this.users)
+            console.log(await this._database.writePosts(posts))
         } catch(e) {
             console.log(e)
             console.log('got err')
         }
     }
 
+    async comment(): Promise<any> {
+        try {
+            const allPosts = await this._database.getPosts()
+            const toCommentPosts = allPosts.filter(post => !post.did_comment)
+
+            // Comment on each one with 20 second breaks
+            toCommentPosts.forEach((post, index) => {
+                setTimeout(() => {
+                    this._broadcaster.makeComment(post)
+                    this._database.writeComment(post)
+                }, index * 22 * 1000)
+            })
+        } catch(e) {
+            console.log('something went wrong', e)
+        }
+    }
+
+    async vote(): Promise<any> {
+        try {
+            const allPosts = await this._database.getPosts()
+            const toVotePosts = allPosts.filter(post => !post.did_vote)
+
+            // Comment on each one with 20 second breaks
+            toVotePosts.forEach((post, index) => {
+                setTimeout(() => {
+                    this._broadcaster.makeVote(post)
+                    this._database.writeVote(post)
+                }, index * 22 * 1000)
+            })
+        } catch(e) {
+            console.log('something went wrong', e)
+        }
+    }
+
     async payout(totalPayout: number): Promise<any> {
         const allUsers = await this._database.getUsers()
         const weekUsers = allUsers.filter(weekFilter(this.week))

src/classes/post.ts

@@ -13,6 +13,7 @@ export class Post {
     // SQL
     did_comment: boolean;
     did_vote: boolean;
+    is_approved: boolean;
 
     constructor() {
         this.jsonMetadata = {} as JsonMetadata

src/database/database.ts

@@ -5,5 +5,8 @@ export interface Database {
     setup: () => Promise<void>
     getUsers: () => Promise<User[]>
     getPosts: () => Promise<Post[]>
-    writePosts: (posts: Post[], onInsert: (post: Post) => Promise<any>) => Promise<{ created: number, updated: number, total: number }>
+
+    writePosts: (posts: Post[]) => Promise<{ created: number, updated: number, total: number }>
+    writeComment: (post: Post) => Promise<any>
+    writeVote: (post: Post) => Promise<any>
 }

src/database/sqlDatabase.ts

@@ -22,9 +22,12 @@ export class sqlDatabase {
                 author: data.author,
                 votes: data.votes,
                 created: data.created,
-                permlink: data.permlink
+                permlink: data.permlink,
+                did_comment: data.did_comment,
+                did_vote: data.did_vote,
+                is_approved: data.is_approved
             }) as Post)
-            .filter((post) => post.author != 'loubega' && post.author != 'nowplaying-music')
+            .filter((post) => post.author != 'loubega')
             .reduce((users: User[], post) => {
                 const user = users.find(user => user.username === post.author)
                 if (user) {
@@ -51,45 +54,47 @@ export class sqlDatabase {
         }) as Post)
     }
 
-    async writePosts(posts: Post[], onInsert: (post: Post) => Promise<any>): Promise<any> {
-        let count = 0;
-        try {
-            const insertResponses: { post: Post, result: any }[] = await Promise.all(
-                posts.map(async post => {
-                    let result: any = {
-                        err: true
-                    }
-                    try {
-                        result = await this._con.query('INSERT INTO posts SET ?', [post])
-                        setTimeout(() => onInsert(post), 20*1000*count)
-                        count++;
-                    } catch (e) {
-                        // console.log('error inserting prolly cause im there already')
-                    }
-                    return {
-                        post,
-                        result
-                    }
-                })
-            )
-            const toUpdate = insertResponses.filter(res => !res.result.changedRows)
-            const updateResponses = await Promise.all(toUpdate.map(async postObj => ({
-                post: postObj.post,
-                result: await this._con.query('UPDATE posts SET votes=? WHERE author=? AND permlink=?', [postObj.post.votes, postObj.post.author, postObj.post.permlink])
-            })))
-            return {
-                created: insertResponses.length - toUpdate.length,
-                updated: updateResponses.filter(res => res.result.changedRows).length,
-                total: insertResponses.length
-            }
-        } catch (e) {
-            if (e.code === 'ER_DUP_ENTRY') {
+    async writePosts(posts: Post[]): Promise<any> {
+        const insertResponses: { post: Post, result: any }[] = await Promise.all(
+            posts.map(async post => {
+                let result: any = {
+                    err: true
+                }
+                try {
+                    result = await this._con.query('INSERT INTO posts SET ?', [post])
+                } catch (e) {
+                    // console.log('error inserting prolly cause im there already')
+                }
                 return {
-                    already: "there"
+                    post,
+                    result
                 }
-            } else {
-                throw e
-            }
+            })
+        )
+
+        const toUpdate = insertResponses.filter(res => !res.result.changedRows)
+
+        const updateResponses = await Promise.all(toUpdate.map(async postObj => ({
+            post: postObj.post,
+            result: await this._con.query('UPDATE posts SET votes=? WHERE author=? AND permlink=?', [postObj.post.votes, postObj.post.author, postObj.post.permlink])
+        })))
+
+        return {
+            created: insertResponses.length - toUpdate.length,
+            updated: updateResponses.filter(res => res.result.changedRows).length,
+            total: insertResponses.length
         }
     }
+
+    async writeComment(post: Post): Promise<any> {
+        const result = await this._con.query('UPDATE posts SET did_comment=1 WHERE author=? AND permlink=?', [post.author, post.permlink])
+        console.log(result)
+        return new Promise(resolve => resolve('hello'))
+    }
+
+    async writeVote(post: Post): Promise<any> {
+        const result = await this._con.query('UPDATE posts SET did_vote=1 WHERE author=? AND permlink=?', [post.author, post.permlink])
+        console.log(result)
+        return new Promise(resolve => resolve('hello'))
+    }
 }

src/functions.ts

@@ -0,0 +1,12 @@
+import { Post } from './classes/post';
+import { User } from './classes/user';
+import { Settings } from './settings'
+
+export const cleanScrape = (posts: Post[], users: User[]): Post[] => posts
+    .filter(post => !Settings.blacklist.includes(post.author))
+    .map(post => {
+        if (users.map(u => u.username).includes(post.author)) {
+            post.is_approved = true
+        }
+        return post
+    })

src/main.ts

@@ -7,9 +7,9 @@ import { SteemAPI } from './blockchainAPI/steemAPI';
 const mysql = require('promise-mysql');
 
 const local = {
-  host: 'localhost',
-  user: 'root', //process.env.DB_USER,
-  password: 'password', //process.env.DB_PASS,
+  host: '192.168.0.2',
+  user: 'nodice', //process.env.DB_USER,
+  password: '', //process.env.DB_PASS,
   database: 'nowplaying'
 };
 
@@ -24,8 +24,16 @@ const main = async () => {
   bot.setBlockchainAPI(new SteemAPI())
   await bot.setDatabase(new sqlDatabase(local))
   // bot.scrape()
-  // setInterval(() => bot.scrape(), 1000 * 60) // every minute
-  const payout = await bot.payout(1.425)
+
+
+  // every minute scrape, vote, and comment
+  setInterval(() => {
+    bot.scrape()
+    bot.vote()
+    bot.comment()
+  }, 1000 * 60)
+
+  // const payout = await bot.payout(1.425)
   // console.log(payout)
 }
 

src/settings.ts

@@ -0,0 +1,18 @@
+export const Settings = {
+    week: 6,
+    blacklist: [
+        'nowplaying-music',
+
+        'arsaljr',
+        'clicknpict',
+        'dianasteem7',
+        'heroheri',
+        'loubega',
+        'maulinda',
+        'rahmatz',
+        'ryanananda',
+        'safrizalpotret',
+        'tasyazgs588',
+    ]
+
+}

tsconfig.json

@@ -1,5 +1,9 @@
 {
   "compilerOptions": {
+    "lib": [
+      "dom",
+      "es7"
+    ],
     "target": "es6",
     "module": "commonjs",
     "moduleResolution": "node",