-
Notifications
You must be signed in to change notification settings - Fork 0
/
renderer.js
268 lines (231 loc) · 7.58 KB
/
renderer.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
const fs = require('fs');
var tf = require('@tensorflow/tfjs');
var Jimp = require("jimp");
const screenshot = require('screenshot-desktop')
const remote = require('electron').remote;
const path = remote.getGlobal('curr_path')
var {ipcRenderer} = require('electron');
let classes = {
6: "Doomfist",
1: "Genji",
11: "McCree",
16: "Pharah",
2: "Reaper",
0: "Soldier: 76",
19: "Sombra",
22: "Tracer",
4: "Bastion",
8: "Hanzo",
9: "Junkrat",
12: "Mei",
21: "Torbjorn",
23: "Widowmaker",
7: "D.Va",
15: "Orisa",
17: "Reinhardt",
18: "Roadhog",
24: "Winston",
25: "Zarya",
3: "Ana",
13: "Mercy",
10: "Lucio",
14: "Moira",
20: "Symmetra",
26: "Zenyatta",
5: "Brigitte"
}
let model;
let curr = null;
let heroClassPredictions = [];
let heroProbabilityValues =[]
let currentJimpBuffer = null;
// Load model from AWS, and than start taking screenshots.
const loadModel = async () => {
console.log("Loading Model...");
model = await tf.loadModel('https://s3-us-west-2.amazonaws.com/mood1995/neural_net_all_heroes/model.json');
console.log("Model loaded!");
setInterval(startScreen, 500);
}
loadModel();
async function predict(imgElement) {
const logits = tf.tidy(() => {
const b = tf.scalar(255);
const img = tf.fromPixels(imgElement).toFloat().div(b);
// img.print();
const batched = img.reshape([1, 40, 85, 3]);
return model.predict(batched);
});
const classes = await getTopKClasses(logits, 1);
let prob = classes[0].probability;
let pred = classes[0].className;
console.log(pred);
console.log(prob);
if(prob < 0.97) {
console.log("Not a high enough probability");
return;
}
// check last 5 predictions, and only then switch playlists.
if (heroClassPredictions.length == 4) {
// remove oldest prediction to keep array at constant size < 5.
heroClassPredictions.shift();
heroProbabilityValues.shift()
// push newest prediction.
heroClassPredictions.push(pred);
heroProbabilityValues.push(prob);
} else {
console.log("Need more examples!");
heroClassPredictions.push(pred);
heroProbabilityValues.push(prob);
return;
}
console.log(heroClassPredictions);
console.log(heroProbabilityValues);
let newH = checkChamp(heroClassPredictions, heroProbabilityValues);
// not sure about current champ at all, backout!
if (newH == null) {
return
}
if (curr == null || newH != curr) {
// right before we switch, make sure we aren't on the POTG screen.
// we only need to check this once upon a switch!
// if it is the POTG screen, clear our arrays, and return, do NOT
// change the playlist!
console.log("Attempting to switch, first need to check POTG!")
detectPOTGScreen(function(detectedPOTG) {
if (detectedPOTG == true) {
console.log("POTG TRUEEEEEE");
heroClassPredictions = [];
heroProbabilityValues = [];
return;
} else {
webview.removeEventListener('did-start-loading', loadstart)
webview.removeEventListener('did-stop-loading', loadstop)
webview.addEventListener('did-start-loading', loadstart)
webview.addEventListener('did-stop-loading', loadstop)
webview.loadURL('https://mood.gg/owchoosechampspecial?championname=' + newH);
curr = newH;
}
});
}
}
function startScreen() {
if (heroDetectionOn == false) {
return;
}
screenshot().then((fullImage) => {
fs.writeFile(path + "/screenshot.png", fullImage, function(err) {
if(err) {
return console.log(err);
}
// Once every second, check if we are on the POTG screen.
Jimp.read(fullImage, function (err, jimpImage) {
if (err) throw err;
// I save the buffer for POTG detection
currentJimpBuffer = jimpImage.clone();
// work required for hero detection.
jimpImage.clone().crop(1650, 920, 170, 80)
.resize(85, 40)
.quality(100)
.write(path + "/screenshot-cropped.jpg", function(err) {
if (err) throw err;
var image = new Image();
image.onload = function () {
// actually predict hero.
predict(image)
};
image.src = path + '/screenshot-cropped.jpg?' + new Date().getTime();
});
});
});
}).catch((err) => {
console.log("Screenshot failed", err);
});
}
// Simply takes an array of predictions and makes sure we predicted the same thing 5 times.
// Otherwise, just return null.
function checkChamp(arr, arrTwo) {
// First we check that all the predictions beleive they found the SAME hero.
let s = new Set(arr)
if(s.size == 1) {
var it = s.values();
var first = it.next();
// Lastly, we do a quick check to make sure we had a "100%" prediction at least once.
if (arrTwo.filter(item => item == 1).length >= 2) { // >= 2
return first.value;
} else {
return null;
}
} else {
return null;
}
}
async function getTopKClasses(logits, topK) {
const values = await logits.data();
const valuesAndIndices = [];
for (let i = 0; i < values.length; i++) {
valuesAndIndices.push({value: values[i], index: i});
}
valuesAndIndices.sort((a, b) => {
return b.value - a.value;
});
const topkValues = new Float32Array(topK);
const topkIndices = new Int32Array(topK);
for (let i = 0; i < topK; i++) {
topkValues[i] = valuesAndIndices[i].value;
topkIndices[i] = valuesAndIndices[i].index;
}
const topClassesAndProbs = [];
for (let i = 0; i < topkIndices.length; i++) {
topClassesAndProbs.push({
className: classes[topkIndices[i]],
probability: topkValues[i]
})
}
return topClassesAndProbs;
}
function detectPOTGScreen(callback) {
if(currentJimpBuffer == null) {
console.log("currentJimpBuffer was null!");
callback(false)
}
currentJimpBuffer.clone().crop(35, 35, 210, 35)
.quality(100)
.write(path + "/screenshot-potg-cropped.jpg", function(err) {
if (err) throw err;
var image = new Image();
image.onload = function () {
callback(doTemplateMatch(image))
};
image.src = path + '/screenshot-potg-cropped.jpg?' + new Date().getTime();
});
}
// This does a template match against the play of the game screen
// AND the kill cam screen. This was added later.
function doTemplateMatch(imgElement) {
if (!(openCVReady)) {
console.log("OpenCV still not ready!");
return false;
}
if(template == null) {
console.log("Templates still null");
return false;
}
let src = cv.imread(imgElement);
let dst = new cv.Mat();
let mask = new cv.Mat();
cv.matchTemplate(src, template, dst, cv.TM_CCOEFF_NORMED, mask);
// sq_diff normed 1 - minVal
// coeff_normed max_val
let result = cv.minMaxLoc(dst, mask);
src.delete(); dst.delete(); mask.delete();
console.log("template match gave us probability ... ", result.maxVal);
// threshold
if(result.maxVal < 0.80) {
return false;
}
// successfully detected!
else {
console.log("Successful detection of POTG!")
return true;
}
}