-
Notifications
You must be signed in to change notification settings - Fork 1
/
transform.py
55 lines (46 loc) · 1.78 KB
/
transform.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
from gensim import corpora, models
from nltk.corpus import stopwords
import json
import re
import pandas as pd
import ijson
from pandas import *
import ast
def convert(x):
''' Convert a json string to a flat python dictionary
which can be passed into Pandas. '''
ob = json.loads(x)
for k, v in ob.items():
if isinstance(v, list):
ob[k] = ','.join(v)
elif isinstance(v, dict):
for kk, vv in v.items():
ob['%s_%s' % (k, kk)] = vv
del ob[k]
return ob
if __name__ == '__main__':
#
#Load Bussiness Data from File to Pandas Dataframe
businessJsonFileName = "./dataset/yelp_dataset_challenge_academic_dataset/yelp_academic_dataset_business.json"
df = pd.DataFrame([convert(line) for line in file(businessJsonFileName)])
#Filter out the required columns from dataframe
df1 = df[["business_id","name","city",'categories']]
#Filter out Business data for restaurants at Pittsburgh location
df2 = df1[df1.city == "Pittsburgh"]
business_id = df2['business_id'].tolist()
#Save Business id into sets for review data filtering
business_id = set(business_id)
reviewJsonFileName = "./dataset/yelp_dataset_challenge_academic_dataset/yelp_academic_dataset_review.json"
finalReviewJsonFile = "review_json_file_pittsburgh_restaurant.json"
outfile = open (finalReviewJsonFile,"w")
data = []
rowcnt = 0
#Filter out review data for business ids in pittsburgh
with open(reviewJsonFileName, 'r') as f:
for row in f:
rowcnt = rowcnt + 1
if ast.literal_eval(row)["business_id"] in business_id:
data.append(row)
#Write data to the file
for item in data:
outfile.write(item)