-
Notifications
You must be signed in to change notification settings - Fork 4
/
data_loading_and_preprocessing.py
111 lines (93 loc) · 2.78 KB
/
data_loading_and_preprocessing.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
import pandas as pd
import os
import pickle
import io
import demoji
from nltk.tokenize import word_tokenize
from nltk.corpus import stopwords,brown,words
import re
import string
from nltk.stem.porter import PorterStemmer
from nltk.stem.snowball import SnowballStemmer
import nltk
import random
import numpy as np
import tensorflow as tf
import fasttext as ft
from keras.models import *
from keras.layers import *
from keras.callbacks import *
import matplotlib.pyplot as plt
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from sklearn.metrics import roc_curve
from sklearn.metrics import auc
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import RandomForestClassifier as rfc
from sklearn.svm import SVC
import re
from nltk.util import ngrams
import demoji
demoji.download_codes()
nltk.download('punkt')
nltk.download('stopwords')
nltk.download('brown')
nltk.download('words')
users = dict()
labels = ['positive','negative']
path = input("enter the path to the tweet dataset")
for lab in labels:
users[lab]=[]
temp_path=os.path.join(path,lab,'tweet')
for files in os.listdir(temp_path):
print(files,' opened ')
dt = pd.read_json(os.path.join(temp_path,files),lines=True)
users[lab].append(dt['text'])
print(dt['text'])
usess=dict()
for i in uses:
tab = []
c=0
for j in uses[i]:
print(c)
print(j[0])
tab.append(j[0])
c=c+1
usess[i]=tab
usesss=dict()
for i in usess:
usesss[i]=[]
for k in usess[i]:
s=''
h=demoji.findall(k)
for j in range(len(k)):
if k[j] in h :
s=s+h[k[j]]
else:
s=s+k[j]
usesss[i].append(s)
usesssss={'tweet':[],'class':[],'tokens':[],'vectors':[]}
for i in usess:
usesss[i]=[]
for k in usess[i]:
s=''
h=demoji.findall(k)
for j in range(len(k)):
if k[j] in h :
s=s+h[k[j]]
else:
s=s+k[j]
usesssss['tweet'].append(s)
usesssss['class'].append(labels.index(i))
print(len(usesss['negative']))
usesss['positive']=usesss['positive'][0:5385]
print(len(usesss['positive']))
stop_words = set(stopwords.words('english'))
for tweet in usesssss['tweet'] :
temp = ' '.join(re.sub("(@[A-Za-z0-9]+)|([^0-9A-Za-z \t])|(\w+:\/\/\S+)"," ",tweet).split())
temp = (word_tokenize(temp))
temp = [word for word in temp if word not in stop_words and (word.encode('utf-8').isalpha()) ]
temp = [porter.stem(word) for word in temp]
usesssss['tokens'].append(temp)
us_file = open("uses.pickle","ab")
pickle.dump(usesssss,us_file)