-
Notifications
You must be signed in to change notification settings - Fork 18
/
test_fsspec_to_message.py
301 lines (253 loc) · 12.7 KB
/
test_fsspec_to_message.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
"""Tests for the fsspec to message features."""
import json
import os
import socket
from copy import deepcopy
import pytest
import pytroll_collectors.fsspec_to_message
from pytroll_collectors.fsspec_to_message import extract_local_files_to_message_for_remote_use
from pytroll_collectors.tests.test_s3stalker import ls_output, fs_json, subject, zip_content, zip_json, zip_json_fo
class TestMessageComposer:
"""Test case for the message composer."""
def setup(self):
"""Set up message composer tests."""
self.ls_output = deepcopy(ls_output)
def test_message_is_created_with_fs(self):
"""Test the message is created with a filesystem."""
message = pytroll_collectors.fsspec_to_message.create_message_with_json_fs(fs_json, self.ls_output[0], subject)
assert 'filesystem' in message.data
assert message.data['filesystem'] == {"cls": "s3fs.core.S3FileSystem",
"protocol": "s3", "args": [], "anon": True}
def test_message_is_created_with_uri(self):
"""Test message has a uri."""
message = pytroll_collectors.fsspec_to_message.create_message_with_json_fs(fs_json, self.ls_output[0], subject)
assert 'uri' in message.data
assert message.data['uri'] == 's3://' + self.ls_output[0]['name']
def test_message_is_created_with_uid(self):
"""Test message has a uid."""
message = pytroll_collectors.fsspec_to_message.create_message_with_json_fs(fs_json, self.ls_output[0], subject)
assert 'uid' in message.data
def test_message_is_created_with_file(self):
"""Test message has a type of file."""
message = pytroll_collectors.fsspec_to_message.create_message_with_json_fs(fs_json, self.ls_output[0], subject)
assert message.type == 'file'
def test_message_is_created_with_dataset(self):
"""Test message is created with dataset."""
message = pytroll_collectors.fsspec_to_message.create_message_with_json_fs(fs_json, self.ls_output, subject)
assert 'dataset' in message.data
assert len(message.data['dataset']) > 0
filenames = ["s3://" + item['name'] for item in self.ls_output]
for item in message.data['dataset']:
assert item['uri'] in filenames
assert message.type == 'dataset'
def test_message_from_zip_is_created_with_uid(self):
"""Test message has a uri."""
zip_output = list(zip_content.values())
message = pytroll_collectors.fsspec_to_message.create_message_with_json_fs(zip_json, zip_output[0], subject)
assert 'uid' in message.data
assert message.data['uid'] == 'zip://' + zip_output[0]['name']
def test_message_from_zip_with_fo_is_created_with_uid(self):
"""Test message has a uri."""
zip_output = list(zip_content.values())
message = pytroll_collectors.fsspec_to_message.create_message_with_json_fs(zip_json_fo, zip_output[0], subject)
assert 'uid' in message.data
assert message.data['uid'] == 'zip://' + zip_output[0]['name']
def test_message_include_file_metadata(self):
"""Test message has metadata."""
self.ls_output[0]['metadata'] = dict(platform_name='S3A')
message = pytroll_collectors.fsspec_to_message.create_message_with_json_fs(fs_json, self.ls_output[0], subject)
assert message.data['platform_name'] == 'S3A'
def test_message_include_metadata_parameter(self):
"""Test message has metadata from parameter."""
metadata = dict(platform_name='S3A')
message = pytroll_collectors.fsspec_to_message.create_message_with_json_fs(fs_json,
self.ls_output[0],
subject,
metadata)
assert message.data['platform_name'] == 'S3A'
def create_tar_file(filenames, path):
"""Create a tarred file."""
from tarfile import TarFile
with TarFile(path, mode="w") as packfile:
for filename in filenames:
packfile.add(filename)
def create_zip_file(filenames, path):
"""Create a zipped file."""
from zipfile import ZipFile
with ZipFile(path, mode="w") as packfile:
for filename in filenames:
packfile.write(filename)
def create_pack_path(tmp_path):
"""Create the pack path."""
pack_path = tmp_path / "pack"
os.mkdir(pack_path)
return pack_path
class TestUnpackMessage:
"""Test unpacking a file to message."""
def create_files_to_pack(self, tmp_path):
"""Create files to pack."""
filenames = []
for filename_number in range(10):
filename = tmp_path / f"file_to_pack{filename_number}.txt"
with open(filename, mode="w") as fd:
fd.write("Very important stuff.\n")
filenames.append(filename)
return filenames
@pytest.mark.parametrize(
("packing", "create_packfile", "filesystem_class"),
[
("tar", create_tar_file, "fsspec.implementations.tar.TarFileSystem"),
("zip", create_zip_file, "fsspec.implementations.zip.ZipFileSystem"),
]
)
def test_pack_file_extract(self, packing, create_packfile, filesystem_class, tmp_path):
"""Test extracting packed files."""
filenames = self.create_files_to_pack(tmp_path)
pack_path = create_pack_path(tmp_path)
path = pack_path / ("packfile." + packing)
create_packfile(filenames, path)
from pytroll_collectors.fsspec_to_message import extract_files_to_message
import fsspec
fs = fsspec.filesystem("file")
packed_file = path
topic = "interesting_topic"
msg = extract_files_to_message(packed_file, fs, topic, packing)
expected_data = {"dataset": [
{"filesystem": {"cls": filesystem_class,
"protocol": packing, "args": [],
"fo": f"{path}",
"target_protocol": "file", "target_options": {}},
"uid": f"{packing}:/{filename}",
"uri": f"{packing}:/{filename}::file://{path}"} for filename in filenames]}
assert msg.data == expected_data
assert msg.subject == topic
@pytest.mark.parametrize(
("packing", "create_packfile", "filesystem_class"),
[
("tar", create_tar_file, "fsspec.implementations.tar.TarFileSystem"),
("zip", create_zip_file, "fsspec.implementations.zip.ZipFileSystem"),
]
)
def test_pack_local_file_extract(self, packing, create_packfile, filesystem_class, tmp_path):
"""Test extracting packed files."""
filenames = self.create_files_to_pack(tmp_path)
pack_path = create_pack_path(tmp_path)
path = pack_path / ("packfile." + packing)
create_packfile(filenames, path)
packed_file = path
topic = "interesting_topic"
msg = extract_local_files_to_message_for_remote_use(packed_file, topic, packing=packing)
host = socket.gethostname()
expected_data = {"dataset": [
{"filesystem": {"cls": filesystem_class,
"protocol": packing, "args": [],
"fo": f"{path}",
"target_protocol": "ssh", "target_options": {"host": host}},
"uid": f"{packing}:/{filename}",
"uri": f"{packing}:/{filename}::ssh://{host}{path}"} for filename in filenames]}
assert msg.data == expected_data
assert msg.subject == topic
@pytest.mark.skipif(os.getenv("GITHUB_ACTIONS", "false") == "true", reason="SSH filesystem shaky on github actions")
@pytest.mark.parametrize(
("packing", "create_packfile", "filesystem_class"),
[
("tar", create_tar_file, "fsspec.implementations.tar.TarFileSystem"),
("zip", create_zip_file, "fsspec.implementations.zip.ZipFileSystem"),
]
)
def test_pack_local_file_extract_filesystem(self, packing, create_packfile, filesystem_class, tmp_path):
"""Test that extracting packed files give a valid filesystem."""
filenames = self.create_files_to_pack(tmp_path)
pack_path = create_pack_path(tmp_path)
path = pack_path / ("packfile." + packing)
create_packfile(filenames, path)
packed_file = path
topic = "interesting_topic"
msg = extract_local_files_to_message_for_remote_use(packed_file, topic, packing=packing)
filesystem_info = json.dumps(msg.data["dataset"][0]["filesystem"])
self.check_filesystem_is_understood_by_fsspec(filesystem_info)
def check_filesystem_is_understood_by_fsspec(self, filesystem_info):
"""Check that the filesystem info is understood by fsspec."""
from fsspec import AbstractFileSystem
return AbstractFileSystem.from_json(filesystem_info)
@pytest.mark.parametrize(
("packing", "create_packfile", "filesystem_class"),
[
("tar", create_tar_file, "fsspec.implementations.tar.TarFileSystem"),
("zip", create_zip_file, "fsspec.implementations.zip.ZipFileSystem"),
]
)
def test_pack_local_file_extract_with_custom_options(self, packing, create_packfile, filesystem_class, tmp_path):
"""Test extracting packed files using custom options."""
filenames = self.create_files_to_pack(tmp_path)
pack_path = create_pack_path(tmp_path)
path = pack_path / ("packfile." + packing)
create_packfile(filenames, path)
packed_file = path
topic = "interesting_topic"
host = socket.gethostname()
username = "me"
protocol = "ssh"
port = 22
target_options = {"host": host,
"username": username,
"protocol": protocol,
"port": port}
msg = extract_local_files_to_message_for_remote_use(packed_file, topic,
target_options=target_options, packing=packing)
expected_data = {"dataset": [
{"filesystem": {"cls": filesystem_class,
"protocol": packing, "args": [],
"fo": f"{path}",
"target_protocol": protocol, "target_options": target_options},
"uid": f"{packing}:/{filename}",
"uri": f"{packing}:/{filename}::{protocol}://{username}@{host}:{port}{path}"}
for filename in filenames]}
assert msg.data == expected_data
assert msg.subject == topic
class TestSingleFile:
"""Test creating messages from a single file."""
def test_local_file_to_remote_message(self, tmp_path):
"""Test creating a message for a single file with custom options."""
filename = tmp_path / "important_file"
with open(filename, mode="w") as fd:
fd.write("Very important stuff.\n")
topic = "interesting_topic"
host = socket.gethostname()
username = "me"
protocol = "ssh"
port = 22
target_options = {"host": host,
"username": username,
"protocol": protocol,
"port": port}
msg = extract_local_files_to_message_for_remote_use(filename, topic,
target_options=target_options)
filesystem_dict = {"cls": "fsspec.implementations.sftp.SFTPFileSystem",
"protocol": "ssh", "args": []}
filesystem_dict.update(target_options)
expected_data = {
"filesystem": filesystem_dict,
"uid": f"{protocol}://{username}@{host}:{port}{filename}",
"uri": f"{protocol}://{username}@{host}:{port}{filename}"}
assert msg.data == expected_data
assert msg.subject == topic
def test_remote_message_cannot_contain_password(self, tmp_path):
"""Test creating a message with a password crashes."""
filename = tmp_path / "important_file"
with open(filename, mode="w") as fd:
fd.write("Very important stuff.\n")
topic = "interesting_topic"
host = socket.gethostname()
username = "me"
password = "the_password"
protocol = "ssh"
port = 22
target_options = {"host": host,
"username": username,
"password": password,
"protocol": protocol,
"port": port}
with pytest.raises(RuntimeError):
_ = extract_local_files_to_message_for_remote_use(filename, topic,
target_options=target_options)