Skip to content

Commit

Permalink
Change orphan file log to warning (#31835)
Browse files Browse the repository at this point in the history
  • Loading branch information
Abacn committed Jul 11, 2024
1 parent 8d5c3b5 commit 00bf1c6
Showing 1 changed file with 7 additions and 7 deletions.
14 changes: 7 additions & 7 deletions sdks/python/apache_beam/io/fileio.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,6 @@

import collections
import logging
import os
import random
import uuid
from collections import namedtuple
Expand Down Expand Up @@ -702,9 +701,10 @@ def process(self, element, w=beam.DoFn.WindowParam):

move_from = [f.file_name for f in temp_file_results]
move_to = [f.file_name for f in final_file_results]

_LOGGER.info(
'Moving temporary files %s to dir: %s as %s',
map(os.path.basename, move_from),
'Moving %d temporary files to dir: %s as %s',
len(move_from),
self.path.get(),
move_to)

Expand Down Expand Up @@ -745,13 +745,13 @@ def _check_orphaned_files(self, writer_key):
orphaned_files = [m.path for m in match_result[0].metadata_list]

if len(orphaned_files) > 0:
_LOGGER.info(
_LOGGER.warning(
'Some files may be left orphaned in the temporary folder: %s. '
'This may be a result of insufficient permissions to delete'
'these temp files.',
'This may be a result of retried work items or insufficient'
'permissions to delete these temp files.',
orphaned_files)
except BeamIOError as e:
_LOGGER.info('Exceptions when checking orphaned files: %s', e)
_LOGGER.warning('Exceptions when checking orphaned files: %s', e)


class _WriteShardedRecordsFn(beam.DoFn):
Expand Down

0 comments on commit 00bf1c6

Please sign in to comment.