-
Notifications
You must be signed in to change notification settings - Fork 0
/
prepare_batches.py
53 lines (36 loc) · 1.41 KB
/
prepare_batches.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
import os
import pickle
import random
import bot_params as params
from replay_memory import DepthMemory
files = os.listdir(params.train_data_folder)
examples_in_file = 64
desired_examples = 64*2
filtered_files = [file for file in files if file.startswith("DepthMemory") and file.endswith(".pickle")]
loaded_files = [None for x in filtered_files]
processed_examples = set()
count = 0
def get_example(pos_tuple):
global loaded_files
file_idx = pos_tuple[0]
ex_idx = pos_tuple[1]
if loaded_files[file_idx] is None:
with open(params.train_data_folder + filtered_files[file_idx], "rb") as f:
loaded_files[file_idx] = pickle.load(f)
return loaded_files[file_idx].inputs[ex_idx], loaded_files[file_idx].outputs[ex_idx]
current_batch = None
def add_example(inp, out):
global current_batch
if current_batch is None:
current_batch = DepthMemory()
if current_batch.inputs is not None and current_batch.inputs.shape[0] >= examples_in_file:
current_batch.save_data("_64")
current_batch = DepthMemory()
current_batch.add_episode(inp, out)
while desired_examples > count:
randex_addr = (random.randint(0, len(filtered_files)-1), random.randint(0, params.num_examples_to_dump)-1)
if randex_addr not in processed_examples:
inp, out = get_example(randex_addr)
add_example(inp, out)
processed_examples.add(randex_addr)
count += 1