/
dataset-mnist.R
172 lines (145 loc) · 5.98 KB
/
dataset-mnist.R
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
#' MNIST dataset
#'
#' Prepares the MNIST dataset and optionally downloads it.
#'
#' @param root (string): Root directory of dataset where
#' `MNIST/processed/training.pt` and `MNIST/processed/test.pt` exist.
#' @param train (bool, optional): If True, creates dataset from
#' `training.pt`, otherwise from `test.pt`.
#' @param download (bool, optional): If true, downloads the dataset from the
#' internet and puts it in root directory. If dataset is already downloaded,
#' it is not downloaded again.
#' @param transform (callable, optional): A function/transform that takes in an
#' PIL image and returns a transformed version. E.g,
#' `transforms.RandomCrop`.
#' @param target_transform (callable, optional): A function/transform that takes
#' in the target and transforms it.
#'
#' @export
mnist_dataset <- dataset(
name = "mnist",
resources = list(
c("https://storage.googleapis.com/torch-datasets/mnist/train-images-idx3-ubyte.gz", "f68b3c2dcbeaaa9fbdd348bbdeb94873"),
c("https://storage.googleapis.com/torch-datasets/mnist/train-labels-idx1-ubyte.gz", "d53e105ee54ea40749a09fcbcd1e9432"),
c("https://storage.googleapis.com/torch-datasets/mnist/t10k-images-idx3-ubyte.gz", "9fb629c4189551a2d022fa330f9573f3"),
c("https://storage.googleapis.com/torch-datasets/mnist/t10k-labels-idx1-ubyte.gz", "ec29112dd5afa0611ce80d1b7f02629c")
),
training_file = 'training.rds',
test_file = 'test.rds',
classes = c('0 - zero', '1 - one', '2 - two', '3 - three', '4 - four',
'5 - five', '6 - six', '7 - seven', '8 - eight', '9 - nine'),
initialize = function(root, train = TRUE, transform = NULL, target_transform = NULL,
download = FALSE) {
self$root_path <- root
self$transform <- transform
self$target_transform <- target_transform
self$train <- train
if (download)
self$download()
if (!self$check_exists())
runtime_error("Dataset not found. You can use `download = TRUE` to download it.")
if (self$train)
data_file <- self$training_file
else
data_file <- self$test_file
data <- readRDS(file.path(self$processed_folder, data_file))
self$data <- data[[1]]
self$targets <- data[[2]] + 1L
},
download = function() {
if (self$check_exists())
return(NULL)
fs::dir_create(self$raw_folder)
fs::dir_create(self$processed_folder)
for (r in self$resources) {
filename <- tail(strsplit(r[1], "/")[[1]], 1)
destpath <- file.path(self$raw_folder, filename)
p <- download_and_cache(r[1], prefix = class(self)[1])
fs::file_copy(p, destpath)
if (!tools::md5sum(destpath) == r[2])
runtime_error("MD5 sums are not identical for file: {r[1]}.")
}
rlang::inform("Processing...")
training_set <- list(
read_sn3_pascalvincent(file.path(self$raw_folder, 'train-images-idx3-ubyte.gz')),
read_sn3_pascalvincent(file.path(self$raw_folder, 'train-labels-idx1-ubyte.gz'))
)
test_set <- list(
read_sn3_pascalvincent(file.path(self$raw_folder, 't10k-images-idx3-ubyte.gz')),
read_sn3_pascalvincent(file.path(self$raw_folder, 't10k-labels-idx1-ubyte.gz'))
)
saveRDS(training_set, file.path(self$processed_folder, self$training_file))
saveRDS(test_set, file.path(self$processed_folder, self$test_file))
rlang::inform("Done!")
},
check_exists = function() {
fs::file_exists(file.path(self$processed_folder, self$training_file)) &&
fs::file_exists(file.path(self$processed_folder, self$test_file))
},
.getitem = function(index) {
img <- self$data[index, ,]
target <- self$targets[index]
if (!is.null(self$transform))
img <- self$transform(img)
if (!is.null(self$target_transform))
target <- self$target_transform(target)
list(x = img, y = target)
},
.length = function() {
dim(self$data)[1]
},
active = list(
raw_folder = function() {
file.path(self$root_path, "mnist", "raw")
},
processed_folder = function() {
file.path(self$root_path, "mnist", "processed")
}
)
)
#' Kuzushiji-MNIST
#'
#' Prepares the [Kuzushiji-MNIST](https://github.com/rois-codh/kmnist) dataset
#' and optionally downloads it.
#'
#' @param root (string): Root directory of dataset where
#' `KMNIST/processed/training.pt` and `KMNIST/processed/test.pt` exist.
#' @param train (bool, optional): If TRUE, creates dataset from `training.pt`,
#' otherwise from `test.pt`.
#' @param download (bool, optional): If true, downloads the dataset from the
#' internet and puts it in root directory. If dataset is already downloaded,
#' it is not downloaded again.
#' @param transform (callable, optional): A function/transform that takes in an
#' PIL image and returns a transformed version. E.g, `transforms.RandomCrop`.
#' @param target_transform (callable, optional): A function/transform that takes
#' in the target and transforms it.
#'
#' @export
kmnist_dataset <- dataset(
name = "kminst_dataset",
inherit = mnist_dataset,
resources = list(
c("http://codh.rois.ac.jp/kmnist/dataset/kmnist/train-images-idx3-ubyte.gz", "bdb82020997e1d708af4cf47b453dcf7"),
c("http://codh.rois.ac.jp/kmnist/dataset/kmnist/train-labels-idx1-ubyte.gz", "e144d726b3acfaa3e44228e80efcd344"),
c("http://codh.rois.ac.jp/kmnist/dataset/kmnist/t10k-images-idx3-ubyte.gz", "5c965bf0a639b31b8f53240b1b52f4d7"),
c("http://codh.rois.ac.jp/kmnist/dataset/kmnist/t10k-labels-idx1-ubyte.gz", "7320c461ea6c1c855c0b718fb2a4b134")
),
classes = c('o', 'ki', 'su', 'tsu', 'na', 'ha', 'ma', 'ya', 're', 'wo')
)
read_sn3_pascalvincent <- function(path) {
x <- gzfile(path, open = "rb")
on.exit({close(x)})
magic <- readBin(x, endian = "big", what = integer(), n = 1)
n_dimensions <- magic %% 256
ty <- magic %/% 256
dim <- readBin(x, what = integer(), size = 4, endian = "big",
n = n_dimensions)
a <- readBin(
x,
what = "int", endian = "big", n = prod(dim),
size = 1, signed = FALSE
)
a <- array(a, dim = rev(dim))
a <- aperm(a, perm = rev(seq_along(dim)))
a
}