From 7248c793f755a559b0cbf7e2efe8519a27ed0a15 Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Mon, 18 Sep 2017 19:46:36 -0400 Subject: [PATCH 1/2] Explicitly close owned file handles in ParquetWriter.close to avoid flakiness on Windows --- python/pyarrow/_parquet.pyx | 5 +++++ python/pyarrow/tests/test_parquet.py | 3 --- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx index aea6fb608501e..ea665de180415 100644 --- a/python/pyarrow/_parquet.pyx +++ b/python/pyarrow/_parquet.pyx @@ -562,6 +562,7 @@ cdef class ParquetWriter: cdef: unique_ptr[FileWriter] writer shared_ptr[OutputStream] sink + bint own_sink cdef readonly: object use_dictionary @@ -588,8 +589,10 @@ cdef class ParquetWriter: check_status(FileOutputStream.Open(c_where, &filestream)) self.sink = filestream + self.own_sink = True else: get_writer(where, &self.sink) + self.own_sink = False self.use_dictionary = use_dictionary self.compression = compression @@ -664,6 +667,8 @@ cdef class ParquetWriter: def close(self): with nogil: check_status(self.writer.get().Close()) + if self.own_sink: + self.sink.get().Close() def write_table(self, Table table, row_group_size=None): cdef CTable* ctable = table.table diff --git a/python/pyarrow/tests/test_parquet.py b/python/pyarrow/tests/test_parquet.py index 9b5a4bcc43c51..790b47832f226 100644 --- a/python/pyarrow/tests/test_parquet.py +++ b/python/pyarrow/tests/test_parquet.py @@ -1202,9 +1202,6 @@ def test_write_error_deletes_incomplete_file(tmpdir): except pa.ArrowException: pass - # Ensure that object has been destructed; this causes test failures on - # Windows - gc.collect() assert not os.path.exists(filename) From 863827c4ad1f483b83af505d05b0e75cf7c4955a Mon Sep 17 00:00:00 2001 From: Wes McKinney Date: Mon, 18 Sep 2017 19:53:03 -0400 Subject: [PATCH 2/2] Check status --- python/pyarrow/_parquet.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyarrow/_parquet.pyx b/python/pyarrow/_parquet.pyx index ea665de180415..b096fa1b4d337 100644 --- a/python/pyarrow/_parquet.pyx +++ b/python/pyarrow/_parquet.pyx @@ -668,7 +668,7 @@ cdef class ParquetWriter: with nogil: check_status(self.writer.get().Close()) if self.own_sink: - self.sink.get().Close() + check_status(self.sink.get().Close()) def write_table(self, Table table, row_group_size=None): cdef CTable* ctable = table.table