-
Notifications
You must be signed in to change notification settings - Fork 5
/
move_posts_comments.py
31 lines (24 loc) · 1.55 KB
/
move_posts_comments.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
# -*- coding: utf-8 -*-
#==============================================================================================================================#
# Script to clean the text in status message to get a well formatted CSV for hive #
# storage( removing commans and new line characters) #
#==============================================================================================================================#
import csv
import pandas as pd
import re
from subprocess import call
#function to copy files to hdfs from local
def movePostsHdfs():
comm = 'hdfs dfs -copyFromLocal -f /home/daasuser/facebook/posts_comments/MTNLoaded_facebook_statuses.csv /user/daasuser/facebook/MTNLoaded_facebook_statuses/'
call(comm,shell = True)
def moveCommentsHdfs():
comm = 'hdfs dfs -copyFromLocal -f /home/daasuser/facebook/posts_comments/MTNLoaded_facebook_comments.csv /user/daasuser/facebook/MTNLoaded_facebook_comments/'
call(comm,shell = True)
status=pd.read_csv('/home/daasuser/facebook/posts_comments/MTNLoaded_facebook_statuses.csv')
regex=re.compile('[,\n]')
for i in range(0,len(status)):
status.loc[i,'status_message']=regex.sub(' ',status.loc[i,'status_message'])
status.loc[i,'status_message']=re.sub('\n+',' ',status.loc[i,'status_message'])
status.to_csv('/home/daasuser/facebook/posts_comments/MTNLoaded_facebook_statuses.csv',index=None)
movePostsHdfs()
moveCommentsHdfs()