/
test_s3.py
58 lines (47 loc) · 1.8 KB
/
test_s3.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not
# use this file except in compliance with the License. You may obtain a copy of
# the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations under
# the License.
# ==============================================================================
"""Tests for S3 file system"""
import os
import sys
import time
import tempfile
import tensorflow as tf
import tensorflow_io as tfio
import pytest
@pytest.mark.skipif(
sys.platform in ("win32", "darwin"),
reason="TODO Localstack not setup properly on macOS/Windows yet",
)
def test_read_file():
"""Test case for reading S3"""
import boto3
os.environ["AWS_REGION"] = "us-east-1"
os.environ["AWS_ACCESS_KEY_ID"] = "ACCESS_KEY"
os.environ["AWS_SECRET_ACCESS_KEY"] = "SECRET_KEY"
client = boto3.client(
"s3", region_name="us-east-1", endpoint_url="http://localhost:4566"
)
body = b"1234567"
# Setup the S3 bucket and key
key_name = "TEST"
bucket_name = f"s3e{time.time()}e"
client.create_bucket(Bucket=bucket_name)
client.put_object(Bucket=bucket_name, Key=key_name, Body=body)
response = client.get_object(Bucket=bucket_name, Key=key_name)
assert response["Body"].read() == body
os.environ["S3_VERIFY_SSL"] = "0"
os.environ["S3_ENDPOINT"] = "http://localhost:4566"
content = tf.io.read_file(f"s3://{bucket_name}/{key_name}")
assert content == body