-
Notifications
You must be signed in to change notification settings - Fork 9
/
partial_preprocess.py
79 lines (58 loc) · 2.27 KB
/
partial_preprocess.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
from models import *
import os
import numpy as np
from torch.utils.data import Dataset
from torchvision import datasets, transforms
import matplotlib.pyplot as plt
def partial_mnistm(mnistm_path="/home/neo/dataset/mnistm/"):
train_data, train_label = torch.load(
os.path.join(mnistm_path, "mnistm_pytorch_train"))
test_data, test_label = torch.load(
os.path.join(mnistm_path, "mnistm_pytorch_test"))
train_data = train_data.numpy()
train_label = train_label.numpy()
test_data = test_data.numpy()
test_label = test_label.numpy()
""" Training Data """
idx = np.where(train_label % 3 == 0)
train_data = train_data[idx]
train_label = train_label[idx]
train_data = torch.from_numpy(train_data)
train_label = torch.from_numpy(train_label)
""" Testing Data """
idx = np.where(test_label % 3 == 0)
test_data = test_data[idx]
test_label = test_label[idx]
test_data = torch.from_numpy(test_data)
test_label = torch.from_numpy(test_label)
""" Save Dataset """
partial_train = (train_data, train_label)
partial_test = (test_data, test_label)
with open(os.path.join(mnistm_path, "partial_mnistm_pytorch_train"), "wb") as f:
torch.save(partial_train, f)
with open(os.path.join(mnistm_path, "partial_mnistm_pytorch_test"), "wb") as f:
torch.save(partial_test, f)
print("Done")
def partial_usps(usps_path="/home/neo/dataset/usps/"):
train_data, train_label = torch.load(
os.path.join(usps_path, "usps_pytorch_train"))
test_data, test_label = torch.load(
os.path.join(usps_path, "usps_pytorch_test"))
""" Training Data """
idx = np.where(train_label % 3 == 0)
train_data = train_data[idx]
train_label = train_label[idx]
""" Testing Data """
idx = np.where(test_label % 3 == 0)
test_data = test_data[idx]
test_label = test_label[idx]
""" Save Dataset """
partial_train = (train_data, train_label)
partial_test = (test_data, test_label)
with open(os.path.join(usps_path, "partial_usps_pytorch_train"), "wb") as f:
torch.save(partial_train, f)
with open(os.path.join(usps_path, "partial_usps_pytorch_test"), "wb") as f:
torch.save(partial_test, f)
print("Done")
partial_usps()
partial_mnistm()