Merge pull request #157 from GolosChain/143_script_posts_contract

Create script to convert post/comments from GolosChain to CyberWay #143

scripts/posting_script/convert.py

@@ -0,0 +1,15 @@
+#!/usr/bin/env python3
+
+import posts
+import votes
+
+def print_success():
+    print('SUCCESS!')
+
+print('posts converting: ')
+if posts.convert_posts():
+    print_success()
+
+print('votes converting: ')
+if votes.convert_votes():
+    print_success()

scripts/posting_script/dbs.py

@@ -0,0 +1,24 @@
+from pymongo import MongoClient
+import hashlib
+
+from decimal import Decimal
+from bson.decimal128 import Decimal128
+
+
+client = MongoClient('172.17.0.1', 27017)
+
+golos_db = client['Golos']
+cyberway_db = client['_CYBERWAY_TEST_']
+
+def convert_hash(param):
+    hashobj = hashlib.sha256(param.encode('utf-8')).hexdigest()
+    return Decimal128(Decimal(int(hashobj[:16], 16))) 
+
+def printProgressBar (iteration, total, prefix = '', suffix = '', decimals = 1, length = 100, fill = '█'):
+    percent = ("{0:." + str(decimals) + "f}").format(100 * (iteration / float(total)))
+    filledLength = int(length * iteration // total)
+    bar = fill * filledLength + '-' * (length - filledLength)
+    print('\r%s |%s| %s%% %s' % (prefix, bar, percent, suffix), end = '\r')
+    if iteration == total:
+        print()
+

scripts/posting_script/posts.py

@@ -0,0 +1,101 @@
+import dbs
+import json
+from datetime import datetime
+
+def create_tags(metadata_tags):
+    tags = []
+    for tag in metadata_tags or []:
+        tag_obj = {
+            "tag_name": tag
+        }
+        tags.append(tag_obj)
+
+    return tags
+
+def level_post(parent_id):
+    if (parent_id):
+        cb_posts = dbs.cyberway_db["posttable"]
+        parent_post = cb_posts.find_one({"id": parent_id})
+        if not parent_post:
+              return 0
+
+        return parent_post["level"] + 1
+
+    return 0
+
+def convert_posts():
+    golos_posts = dbs.golos_db['comment_object']
+
+    length = golos_posts.count()
+    dbs.printProgressBar(0, length, prefix = 'Progress:', suffix = 'Complete', length = 50)
+
+    i = 0 
+    for doc in golos_posts.find():
+
+        try:
+            if (doc["removed"]):
+                continue
+
+            messagestate = {
+                "absshares": doc["abs_rshares"],
+                "netshares": doc["net_rshares"],
+                "voteshares": doc["vote_rshares"],
+                "sumcuratorsw": 0
+            }
+
+            message = {
+                "id": dbs.convert_hash(doc["permlink"]),
+                "date": doc["last_update"],
+                "parentacc": doc["parent_author"],
+                "parent_id": dbs.convert_hash(doc["parent_permlink"]),
+                "tokenprop": 0,
+                "beneficiaries": "",
+                "rewardweight": doc["reward_weight"],
+                "state": messagestate,
+                "childcount": doc["children"],
+                "closed": "false",
+                "level": level_post(dbs.convert_hash(doc["parent_permlink"])),
+                "_SCOPE_": doc["author"],
+                "_PAYER_": "gls.publish",
+                "_SIZE_": 50
+            }
+            dbs.cyberway_db['posttable'].save(message)
+
+            tags = []
+            if (isinstance(doc["json_metadata"], dict)):
+                if ("tags" in doc["json_metadata"]):
+                    tags = create_tags(doc["json_metadata"]["tags"])
+
+            if(isinstance(doc["json_metadata"], str)):                
+                try:
+                    if (doc["json_metadata"]):
+                        json_str = doc["json_metadata"]
+                        if ((json_str.find("\"") == 0) and (json_str.rfind("\"") == len(json_str)-1)):
+                            json_str = json_str[1: len(json_str)-1]
+                        dict_metadata = json.loads(json_str)                    
+                        if (dict_metadata["tags"]):
+                            tags = create_tags(dict_metadata["tags"])
+                except Exception:
+                    tags= []
+
+            content = {
+                "id": dbs.convert_hash(doc["permlink"]),
+                "headermssg": doc["title"],
+                "bodymssg": doc["body"],
+                "languagemssg": "",
+                "tags": tags,
+                "jsonmetadata": doc["json_metadata"],
+                "_SCOPE_": doc["author"],
+                "_PAYER_": doc["author"],
+                "_SIZE_": 50
+            }
+            dbs.cyberway_db['contenttable'].save(content) 
+
+            i += 1
+            dbs.printProgressBar(i + 1, length, prefix = 'Progress:', suffix = 'Complete', length = 50)
+        except Exception as e:
+            print(doc)
+            print(traceback.format_exc())
+
+    return True
+

scripts/posting_script/votes.py

@@ -0,0 +1,40 @@
+import dbs
+import bson
+from time import sleep
+
+
+def convert_votes():
+    golos_votes = dbs.golos_db['comment_vote_object']
+    cyberway_messages = dbs.cyberway_db['posttable']
+
+    length = golos_votes.count()
+    dbs.printProgressBar(0, length, prefix = 'Progress:', suffix = 'Complete', length = 50)
+
+    id_count = 0
+
+    for doc in golos_votes.find():
+        try:
+            post = cyberway_messages.find_one({"id": dbs.convert_hash(doc["permlink"])})
+            if not post:
+                continue
+
+            vote = {
+                "id" : id_count,
+                "message_id" : post["id"],
+                "voter" : doc["voter"],
+                "weight" : doc["weight"],
+                "time" : doc["last_update"],
+                "count" : 0,
+                "curatorsw": 0,
+                "_SCOPE_" : post["_SCOPE_"],
+                "_PAYER_" : doc["author"],
+                "_SIZE_" : 50
+            }
+            dbs.cyberway_db['votetable'].save(vote)
+
+            id_count += 1
+            dbs.printProgressBar(id_count + 1, length, prefix = 'Progress:', suffix = 'Complete', length = 50)
+        except Exception:
+            print(post)
+    return True
+