-
Notifications
You must be signed in to change notification settings - Fork 1
/
bigdata.py
96 lines (74 loc) · 2.34 KB
/
bigdata.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
#!/usr/bin/env python
# coding: utf-8
## Import the CSV File into the DataFrame.
import pandas as pd
empdata = pd.read_csv('C:\\Users\\XXXXX\\emp.csv', index_col=False, delimiter = ',')
empdata.head()
## Read file data in Jupyter Notenook
data = pd.read_csv('emp.csv')
df = pd.DataFrame(data)
print(df)
## Connect to the MySQL and create a Database
import os
import pymysql
import pandas as pd
host = os.getenv('MYSQL_HOST')
port = os.getenv('MYSQL_PORT')
user = os.getenv('MYSQL_USER')
password = os.getenv('MYSQL_PASSWORD')
database = os.getenv('MYSQL_DATABASE')
conn = pymysql.connect(
host="localhost",
port=int(3306),
user="root",
passwd="",
db="csv",
charset='utf8mb4')
cursor = conn.cursor()
cursor.execute("select database();")
record = cursor.fetchone()
print("You're connected to database: ", record)
mydb.commit()
cursor.close()
print ("Done")
## Create a table & Import the CSV data into the MySQL table with timestamp
import os
import pymysql
import pandas as pd
import datetime;
## Timestamp
ct = datetime.datetime.now()
print("current time:-", ct)
## Import CSV
data = pd.read_csv('emp.csv')
df = pd.DataFrame(data)
host = os.getenv('MYSQL_HOST')
user = os.getenv('MYSQL_USER')
password = os.getenv('MYSQL_PASSWORD')
database = os.getenv('MYSQL_DATABASE')
conn = pymysql.connect(host="localhost",user="root",passwd="",db="csv",charset='utf8mb4')
cursor = conn.cursor()
cursor.execute("select database();")
record = cursor.fetchone()
print("You're connected to database: ", record)
cursor.execute('DROP TABLE IF EXISTS employee_data;')
cursor.execute("CREATE TABLE employee_data(first_name varchar(255),last_name varchar(255),company_name varchar(255),address varchar(255),city varchar(255),county varchar(255),state varchar(255),zip int,phone1 varchar(255),phone2 varchar(255),email varchar(255),web varchar(255))")
print("Table is created....")
for i,row in data.iterrows():
sql = "INSERT INTO csv.employee_data VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)"
cursor.execute(sql, tuple(row))
print("Record inserted")
conn.commit()
## timestamp
ct = datetime.datetime.now()
print("current time:-", ct)
ts = ct.timestamp()
print("timestamp:-", ts)
## Query the Table
# Execute query
sql = "SELECT * FROM csv.employee_data"
cursor.execute(sql)
# Fetch all the records
result = cursor.fetchall()
for i in result:
print(i)