This repository has been archived by the owner on Jan 23, 2024. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 0
/
cluster_projects.py
132 lines (97 loc) · 4.08 KB
/
cluster_projects.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
# Import the necessary libraries
from sklearn.decomposition import PCA
from sklearn.feature_extraction.text import CountVectorizer
from matplotlib import colormaps
import matplotlib.pyplot as plt
import numpy as np
import os, re
from datetime import datetime
from ClusterProperties import ClusterProperties
from Experiment import Experiment
from Project import Project
# Setup parameters for filtering projects
min_dr_scratch = 16 # Min Dr Scratch score to be clustered
max_samples = 50_000 # Max number of samples to cluster
# Setup experiments
eps = np.arange(8.0, 10.1, 0.5)
min_samples = np.arange(10, 21, 5)
# Look for best fitting hyperparameters
experiments = [
Experiment("Data only", True, False, False, np.arange(0.5, 10.1, 0.5), np.arange(5, 61, 5)),
Experiment("Names only", False, False, True, np.arange(0.5, 10.1, 0.5), np.arange(5, 61, 5))
]
# The experiments to run
# NOTE: This overrides the `experiments` array to find the best fitting hyperparameters
experiments = [
Experiment("Data only", True, False, False, eps, min_samples),
Experiment("Normalised only", False, True, False, eps, min_samples),
# Experiment("Names only", False, False, True, eps, min_samples),
Experiment("Data and normalised", True, True, False, eps, min_samples),
Experiment("Data and names", True, False, True, eps, min_samples),
Experiment("All", True, True, True, eps, min_samples)
]
def format_time(diff):
minutes = int(diff.seconds / 60)
milliseconds = int(diff.microseconds / 1000)
return str(minutes) + "m " + "{:.3f}".format((diff.seconds % 60) + (milliseconds / 1000)) + "s"
def read_data():
projects = []
file = open("data\\merged_data.csv", encoding="utf-8")
file.readline()
for line in file:
projects.append(Project(line))
return projects
def vectorise_project_names(projects):
# Remove special characters (`?!()[]`) and collapse (multiple) whitespaces and dashes into one whitespace
names = list(map(lambda p: re.sub("[\?!\(\)\[\]]", "", re.sub("[\s-]+", " ", p.name)), projects))
# Vectorise the names
alg = CountVectorizer()
result = alg.fit_transform(names).toarray()
# Add the name vectors to projects
for i, p in enumerate(projects):
p.name_vector = result[i]
# Show histogram of Dr. Scratch scores
def dr_scratch_scores_histogram(projects):
scores = list(map(lambda p: p.dr_scratch, projects))
fig, ax = plt.subplots()
ax.hist(scores, range(min(scores), 23), align='left')
ax.set_xticks(range(min(scores), 22))
plt.title("Dr. Scratch scores distribution")
plt.xlabel("Dr. Scratch score")
plt.ylabel("Number of projects")
fig.savefig("figures/histogram_dr_scratch_" + str(min(scores)) + ".png") # Uncomment this line to generate the histograms
def main():
main_start = datetime.now()
print("Reading and parsing projects...")
projects = read_data()
dr_scratch_scores_histogram(projects)
print("Filtering projects...")
projects = list(filter(lambda p: p.dr_scratch >= min_dr_scratch, projects))
print(str(len(projects)) + " projects found.")
dr_scratch_scores_histogram(projects)
# Exit if there are too many projects
if len(projects) > max_samples:
print("Error: too many projects. Limit is " + str(max_samples) + " projects. Exiting.")
exit()
# Vectorise project names
vectorise_project_names(projects)
# Setup directory to save the results
root_folder = datetime.now().strftime("%Y%m%d %H.%M.%S")
os.mkdir("results/" + root_folder)
os.mkdir("figures/" + root_folder)
# Run and save the experiments
print("")
for i, experiment in enumerate(experiments):
print("Performing experiment " + str(i + 1) + " of " + str(len(experiments)) + ", \"" + experiment.name + "\"...")
folder = "/".join([root_folder, experiment.name])
os.mkdir("results/" + folder)
os.mkdir("figures/" + folder)
# Setup timer to calculate time needed for each experiment
start = datetime.now()
experiment.run_all(projects, folder)
# Experiment done
print("Experiment " + str(i + 1) + " completed in " + format_time(datetime.now() - start) + ".\r\n")
# Program done
print("Program completed in " + format_time(datetime.now() - main_start) + ".")
if __name__ == "__main__":
main()