From b9fcd5f2e56d8e7eb13a8937f294b1eeb2b705b9 Mon Sep 17 00:00:00 2001 From: "polecito.em@gmail.com" Date: Thu, 29 Sep 2016 15:41:25 -0700 Subject: [PATCH 1/6] Making sure that GcsBufferedReader implements the iterator protocol --- sdks/python/apache_beam/io/gcsio.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/sdks/python/apache_beam/io/gcsio.py b/sdks/python/apache_beam/io/gcsio.py index 5a83004446d8..7406130c244b 100644 --- a/sdks/python/apache_beam/io/gcsio.py +++ b/sdks/python/apache_beam/io/gcsio.py @@ -307,6 +307,14 @@ def __init__(self, def _get_object_metadata(self, get_request): return self.client.objects.Get(get_request) + def __iter__(self): + return self + + def next(self): + """Read one line delimited by '\\n' from the file. + """ + return self.readline() + def read(self, size=-1): """Read data from a GCS file. From 2d93943fad5e8179909e21dac739e3a5ce9eab55 Mon Sep 17 00:00:00 2001 From: "polecito.em@gmail.com" Date: Tue, 4 Oct 2016 15:40:43 -0700 Subject: [PATCH 2/6] Adding raise StopIteration to GcsBufferedReader.next --- sdks/python/apache_beam/io/gcsio.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sdks/python/apache_beam/io/gcsio.py b/sdks/python/apache_beam/io/gcsio.py index 7406130c244b..cf64f439d8d2 100644 --- a/sdks/python/apache_beam/io/gcsio.py +++ b/sdks/python/apache_beam/io/gcsio.py @@ -313,7 +313,10 @@ def __iter__(self): def next(self): """Read one line delimited by '\\n' from the file. """ - return self.readline() + line = self.readline() + if not line: + raise StopIteration() + return line def read(self, size=-1): """Read data from a GCS file. From 30c66d8919cb038fa01f7d46ed66294d51201a8f Mon Sep 17 00:00:00 2001 From: "polecito.em@gmail.com" Date: Tue, 4 Oct 2016 17:03:14 -0700 Subject: [PATCH 3/6] Adding unit tests, and __next__ for Python 3 --- sdks/python/apache_beam/io/gcsio.py | 5 ++++ sdks/python/apache_beam/io/gcsio_test.py | 30 ++++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/sdks/python/apache_beam/io/gcsio.py b/sdks/python/apache_beam/io/gcsio.py index cf64f439d8d2..b4fdf9cb7be7 100644 --- a/sdks/python/apache_beam/io/gcsio.py +++ b/sdks/python/apache_beam/io/gcsio.py @@ -310,6 +310,11 @@ def _get_object_metadata(self, get_request): def __iter__(self): return self + def __next__(self): + """Read one line delimited by '\\n' from the file. + """ + return self.next() + def next(self): """Read one line delimited by '\\n' from the file. """ diff --git a/sdks/python/apache_beam/io/gcsio_test.py b/sdks/python/apache_beam/io/gcsio_test.py index 919e9d20182b..437195b0af85 100644 --- a/sdks/python/apache_beam/io/gcsio_test.py +++ b/sdks/python/apache_beam/io/gcsio_test.py @@ -341,6 +341,36 @@ def test_file_random_seek(self): f.read(end - start + 1), random_file.contents[start:end + 1]) self.assertEqual(f.tell(), end + 1) + def test_file_iterator(self): + file_name = 'gs://gcsio-test/iterating_file' + lines = [] + line_count = 10 + for _ in range(line_count): + line_length = random.randint(100,500) + line = os.urandom(line_length).replace('\n', ' ') + '\n' + lines.append(line) + + contents = ''.join(lines) + file_size = len(contents) + bucket, name = gcsio.parse_gcs_path(file_name) + self.client.objects.add_file(FakeFile(bucket, name, contents, 1)) + + f = self.gcs.open(file_name) + + stop_it_thrown = False + read_lines = 0 + while True: + try: + next_line = next(f) + read_lines += 1 + if read_lines > 10: + break + except StopIteration: + stop_it_thrown = True + break + self.assertEqual(read_lines, line_count) + self.assertTrue(stop_it_thrown) + def test_file_read_line(self): file_name = 'gs://gcsio-test/read_line_file' lines = [] From 475f7e0c475c8939cd685e2761b1bd02486ca884 Mon Sep 17 00:00:00 2001 From: "polecito.em@gmail.com" Date: Tue, 4 Oct 2016 17:22:19 -0700 Subject: [PATCH 4/6] Fixing some lint issues in unit test --- sdks/python/apache_beam/io/gcsio_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sdks/python/apache_beam/io/gcsio_test.py b/sdks/python/apache_beam/io/gcsio_test.py index 437195b0af85..3d8efc004b52 100644 --- a/sdks/python/apache_beam/io/gcsio_test.py +++ b/sdks/python/apache_beam/io/gcsio_test.py @@ -346,12 +346,11 @@ def test_file_iterator(self): lines = [] line_count = 10 for _ in range(line_count): - line_length = random.randint(100,500) + line_length = random.randint(100, 500) line = os.urandom(line_length).replace('\n', ' ') + '\n' lines.append(line) contents = ''.join(lines) - file_size = len(contents) bucket, name = gcsio.parse_gcs_path(file_name) self.client.objects.add_file(FakeFile(bucket, name, contents, 1)) @@ -361,13 +360,14 @@ def test_file_iterator(self): read_lines = 0 while True: try: - next_line = next(f) + next(f) read_lines += 1 if read_lines > 10: break except StopIteration: stop_it_thrown = True break + self.assertEqual(read_lines, line_count) self.assertTrue(stop_it_thrown) From 387f824d15b6d9caaed8d01633ca162aadef9797 Mon Sep 17 00:00:00 2001 From: "polecito.em@gmail.com" Date: Wed, 5 Oct 2016 17:47:52 -0700 Subject: [PATCH 5/6] Improving test readability. --- sdks/python/apache_beam/io/gcsio.py | 2 +- sdks/python/apache_beam/io/gcsio_test.py | 13 ++----------- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/sdks/python/apache_beam/io/gcsio.py b/sdks/python/apache_beam/io/gcsio.py index b4fdf9cb7be7..9fcce5bdc3d4 100644 --- a/sdks/python/apache_beam/io/gcsio.py +++ b/sdks/python/apache_beam/io/gcsio.py @@ -320,7 +320,7 @@ def next(self): """ line = self.readline() if not line: - raise StopIteration() + raise StopIteration return line def read(self, size=-1): diff --git a/sdks/python/apache_beam/io/gcsio_test.py b/sdks/python/apache_beam/io/gcsio_test.py index 3d8efc004b52..2e9945a99757 100644 --- a/sdks/python/apache_beam/io/gcsio_test.py +++ b/sdks/python/apache_beam/io/gcsio_test.py @@ -356,20 +356,11 @@ def test_file_iterator(self): f = self.gcs.open(file_name) - stop_it_thrown = False read_lines = 0 - while True: - try: - next(f) - read_lines += 1 - if read_lines > 10: - break - except StopIteration: - stop_it_thrown = True - break + for line in f: + read_lines += 1 self.assertEqual(read_lines, line_count) - self.assertTrue(stop_it_thrown) def test_file_read_line(self): file_name = 'gs://gcsio-test/read_line_file' From 51b4e775af86891f41efdfb0136317c9724e40b7 Mon Sep 17 00:00:00 2001 From: Pablo Date: Thu, 6 Oct 2016 13:44:34 -0700 Subject: [PATCH 6/6] Trying to unify the TOX_HOME variable --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 8d126681d501..4aa61a03c887 100644 --- a/.travis.yml +++ b/.travis.yml @@ -58,9 +58,9 @@ before_install: - if [ "$TRAVIS_OS_NAME" == "linux" ]; then jdk_switcher use "$CUSTOM_JDK"; fi # Python SDK environment settings. - export TOX_ENV=py27 - - if [ "$TRAVIS_OS_NAME" == "osx" ]; then export TOX_HOME=$HOME/Library/Python/2.7/bin; fi - - if [ "$TRAVIS_OS_NAME" == "linux" ]; then export TOX_HOME=$HOME/.local/bin; fi - + #- if [ "$TRAVIS_OS_NAME" == "osx" ]; then export TOX_HOME=$HOME/Library/Python/2.7/bin; fi + #- if [ "$TRAVIS_OS_NAME" == "linux" ]; then export TOX_HOME=$HOME/.local/bin; fi + - export TOX_HOME=$HOME/.local/bin; install: - if [ ! "$TEST_PYTHON" ]; then travis_retry mvn -B install clean -U -DskipTests=true; fi - if [ "$TEST_PYTHON" ]; then travis_retry pip install tox --user `whoami`; fi