ARROW-8245: [Python][Parquet] Skip hidden directories when reading pa…

…rtitioned parquet files

Closes #6821 from caleboverman/master

Authored-by: caleboverman <coverman@fanthreesixty.com>
Signed-off-by: Wes McKinney <wesm+git@apache.org>

python/pyarrow/parquet.py

@@ -935,7 +935,7 @@ def _parse_hive_partition(value):
 
 def _is_private_directory(x):
     _, tail = os.path.split(x)
-    return tail.startswith('_') and '=' not in tail
+    return (tail.startswith('_') or tail.startswith('.')) and '=' not in tail
 
 
 def _path_split(path, sep):

python/pyarrow/tests/test_parquet.py

@@ -2337,15 +2337,16 @@ def _make_example_multifile_dataset(base_path, nfiles=10, file_nrows=5):
 
 
 @pytest.mark.pandas
-def test_ignore_private_directories(tempdir):
+@pytest.mark.parametrize('dir_prefix', ['_', '.'])
+def test_ignore_private_directories(tempdir, dir_prefix):
     dirpath = tempdir / guid()
     dirpath.mkdir()
 
     paths = _make_example_multifile_dataset(dirpath, nfiles=10,
                                             file_nrows=5)
 
     # private directory
-    (dirpath / '_impala_staging').mkdir()
+    (dirpath / '{}staging'.format(dir_prefix)).mkdir()
 
     dataset = pq.ParquetDataset(dirpath)
     assert set(map(str, paths)) == {x.path for x in dataset.pieces}