-
Notifications
You must be signed in to change notification settings - Fork 3
/
vectorizer.py
67 lines (52 loc) · 1.76 KB
/
vectorizer.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
import pickle
import pandas
import numpy as np
import scipy.io as sio
from sklearn.feature_extraction.text import TfidfVectorizer
udf = pandas.read_csv('/home/itdept/users.tsv',delimiter='\t')
udf = udf.loc[0:6999]
udf = udf.fillna(0)
jobroles = udf['jobroles'].replace(',',' ',regex = True)
jobroles=jobroles.tolist()
print type(jobroles[0])
tfidf = TfidfVectorizer(max_df =0.95,min_df = 2 )
vecs = tfidf.fit_transform(jobroles)
jobroles_vector = np.array(vecs.todense())
discipline = udf['discipline_id']
discipline =discipline.as_matrix()
discipline = discipline.astype(str)
vecs = tfidf.fit_transform(discipline)
discipline_vector = np.array(vecs.todense())
region = udf['region']
region =region.as_matrix()
region = region.astype(str)
vecs = tfidf.fit_transform(region)
region_vector = np.array(vecs.todense())
tfidf= TfidfVectorizer(max_df=0.95, min_df=2)
industry=udf['industry_id'].tolist()
#print industry
#industry=industry.astype(str)
industry = list(map(str,industry ))
industry_vector=tfidf.fit_transform(industry)
#print industry_vector.shape
industry_vector=np.array(industry_vector.todense())
print industry_vector
country=udf['country'].tolist()
#print country
#industry=industry.astype(str)
country = list(map(str,country ))
country_vector=tfidf.fit_transform(country)
#print country_vector.shape
country_vector=np.array(country_vector.todense())
print country_vector
'''
education=udf['edu_fieldofstudies'].astype(str).replace(',',' ',regex = True)
print education
tfidf= TfidfVectorizer(min_df=1)
education = education.tolist()
print type(education[0])
print education
education_vector=tfidf.fit_transform(education)
#education_vector=np.array(education_vector.todense())
#print education_vector
'''