-
Notifications
You must be signed in to change notification settings - Fork 0
/
main.py
78 lines (71 loc) · 2.9 KB
/
main.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
import re
import time
from os.path import join, dirname
import pandas as pd
import requests
import json
path = dirname(__file__)
data_dir = join(path, 'data')
class Miner:
def __init__(self, path):
self.df = pd.read_csv(path)
self.issue_keys = list(self.df['Issue key'].values)
self.base_url = 'https://api.github.com/search/commits?q=repo:apache/activemq+'
def find_commits(self):
headers = {'content-type': 'application/json',
'accept': 'application/vnd.github.cloak-preview'}
keys = []
ids = []
messages = []
for issue in self.issue_keys:
print(issue)
with open(join(path, 'auth.conf'), 'r') as file:
lines = file.readlines()
username = lines[0].split('\n')[0]
password = lines[1].split('\n')[0]
session = requests.Session()
session.auth = (username, password)
response = session.get(self.base_url + issue, headers=headers)
time.sleep(1)
response_dict = json.loads(response.text)
commits = response_dict.get('items')
for commit in commits:
if not self.remove_modify_commit(commit):
keys.append(issue)
ids.append(commit.get('sha'))
messages.append(commit.get('commit').get('message'))
with open(join(data_dir, 'issue_commit.txt'), 'a') as file:
file.write(str(keys[-1]) + ', ' +
str(ids[-1]) + ', ' +
str(messages[-1]) + '\n')
print('\tfor', issue, ' commit', ids[-1], ' added.')
commit_df = pd.DataFrame({'Issue key': keys,
'SHA': ids,
'Message': messages})
df = commit_df.join(self.df.set_index('Issue key'), on='Issue key')
return df
@staticmethod
def remove_modify_commit(commit):
headers = {'content-type': 'application/json',
'accept': 'application/vnd.github.v3.patch'}
url = commit.get('url')
with open(join(path, 'auth.conf'), 'r') as file:
lines = file.readlines()
username = lines[0].split('\n')[0]
password = lines[1].split('\n')[0]
session = requests.Session()
session.auth = (username, password)
response = session.get(url, headers=headers)
time.sleep(1)
if response.text == '':
return True
text = response.text.split('\n')
p = re.compile(', [0-9]+ deletion(s{0,1})\(-\)')
for line in text:
if bool(p.search(line)):
return True
return False
if __name__ == '__main__':
miner = Miner(path=join(data_dir, 'issues.csv'))
df = miner.find_commits()
df.to_csv(join(data_dir, 'issue_commit.csv'), index=False)