Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Port util/process_handler.py and util/tarutil.py #6082

Merged
merged 3 commits into from Jul 9, 2018
Merged

Port util/process_handler.py and util/tarutil.py #6082

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
940e3e8
Port process_handler.py
Jul 9, 2018
32d08ca
Port tarutil
Jul 9, 2018
7089612
Stop treating Tarutil differently between Py2 and Py3
Jul 9, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
5 changes: 3 additions & 2 deletions src/python/pants/util/BUILD
View file
Open in desktop
Expand Up @@ -101,8 +101,8 @@ python_library(
python_library(
name = 'process_handler',
sources = ['process_handler.py'],
dependencies=[
'3rdparty/python:six',
dependencies= [
'3rdparty/python:future',
'3rdparty/python:subprocess32',
':meta',
]
Expand Down Expand Up @@ -154,6 +154,7 @@ python_library(
name = 'tarutil',
sources = ['tarutil.py'],
dependencies = [
'3rdparty/python:future',
'3rdparty/python:six',
],
)
Expand Down
8 changes: 4 additions & 4 deletions src/python/pants/util/process_handler.py
View file
Open in desktop
Expand Up @@ -5,13 +5,13 @@
from __future__ import (absolute_import, division, generators, nested_scopes, print_function,
unicode_literals, with_statement)

import io
import multiprocessing
import os
import StringIO
import sys
from abc import abstractmethod

import six
import future

from pants.util.meta import AbstractClass

Expand All @@ -20,7 +20,7 @@
# NB: As recommended here: https://github.com/google/python-subprocess32/blob/master/README.md
# which accounts for non-posix, ie: Windows. Although we don't support Windows yet, this sets the
# pattern up in anticipation.
if os.name == 'posix' and six.PY2:
if os.name == 'posix' and future.utils.PY2:
import subprocess32 as subprocess3
else:
import subprocess as subprocess3
Expand Down Expand Up @@ -106,7 +106,7 @@ def join_and_get_output():


def _tee(infile, outfile, return_function):
accumulator = StringIO.StringIO()
accumulator = io.BytesIO()
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

StringIO.StringIO() allows for both bytes and unicode. The library was removed in Python3.

Instead, we're using io.BytesIO(), which allows for only bytes. This is what we want - io.StringIO() fails the tests whereas this passes.

for line in iter(infile.readline, ""):
accumulator.write(line)
outfile.write(line)
Expand Down
128 changes: 64 additions & 64 deletions src/python/pants/util/tarutil.py
View file
Open in desktop
Expand Up @@ -6,77 +6,77 @@
unicode_literals, with_statement)

import tarfile
from builtins import str

import six
from future.utils import implements_iterator


if six.PY2:
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Now creating the same class for both Py2 and Py3. I'm not sure why the original author only patched the Py2 behavior - Py3 still needs the patch (see new line 64).

class TarFile(tarfile.TarFile):
def next(self):
"""A copy and modification of the next() method in tarfile module.
@implements_iterator
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Py2 expects the function next() whereas Py3 expects __next__(). This decorator handles that discrepancy.

class TarFile(tarfile.TarFile):

The copy is from tarfile.py of CPython @102457:95df96aa2f5a
def __next__(self):
"""A copy and modification of the next() method in tarfile module.

# Copyright (C) 2002 Lars Gustäbel <lars@gustaebel.de>
# All rights reserved.
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
# files (the "Software"), to deal in the Software without
# restriction, including without limitation the rights to use,
# copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following
# conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""
self._check("ra")
if self.firstmember is not None:
m = self.firstmember
self.firstmember = None
return m
The copy is from tarfile.py of CPython @102457:95df96aa2f5a

# Advance the file pointer.
if self.offset != self.fileobj.tell():
self.fileobj.seek(self.offset - 1)
if not self.fileobj.read(1):
raise tarfile.ReadError("unexpected end of data")
# Copyright (C) 2002 Lars Gustäbel <lars@gustaebel.de>
# All rights reserved.
#
# Permission is hereby granted, free of charge, to any person
# obtaining a copy of this software and associated documentation
# files (the "Software"), to deal in the Software without
# restriction, including without limitation the rights to use,
# copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the
# Software is furnished to do so, subject to the following
# conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
"""
self._check("ra")
if self.firstmember is not None:
m = self.firstmember
self.firstmember = None
return m

# Read the next block.
tarinfo = None
while True:
try:
tarinfo = self.tarinfo.fromtarfile(self)
except tarfile.EOFHeaderError as e:
if self.ignore_zeros:
self._dbg(2, "0x%X: %s" % (self.offset, e))
self.offset += tarfile.BLOCKSIZE
continue
except tarfile.InvalidHeaderError as e:
if self.ignore_zeros:
self._dbg(2, "0x%X: %s" % (self.offset, e))
self.offset += tarfile.BLOCKSIZE
continue
# Modify here, to raise exceptions if errorlevel is bigger than 0.
elif self.errorlevel > 0:
raise tarfile.ReadError(str(e))
except tarfile.EmptyHeaderError:
if self.offset == 0:
raise tarfile.ReadError("empty file")
except tarfile.TruncatedHeaderError as e:
if self.offset == 0:
raise tarfile.ReadError(str(e))
except tarfile.SubsequentHeaderError as e:
# Advance the file pointer.
if self.offset != self.fileobj.tell():
self.fileobj.seek(self.offset - 1)
if not self.fileobj.read(1):
raise tarfile.ReadError("unexpected end of data")

# Read the next block.
tarinfo = None
while True:
try:
tarinfo = self.tarinfo.fromtarfile(self)
except tarfile.EOFHeaderError as e:
if self.ignore_zeros:
self._dbg(2, "0x%X: %s" % (self.offset, e))
self.offset += tarfile.BLOCKSIZE
continue
except tarfile.InvalidHeaderError as e:
if self.ignore_zeros:
self._dbg(2, "0x%X: %s" % (self.offset, e))
self.offset += tarfile.BLOCKSIZE
continue
# Modify here, to raise exceptions if errorlevel is bigger than 0.
elif self.errorlevel > 0:
raise tarfile.ReadError(str(e))
except tarfile.EmptyHeaderError:
if self.offset == 0:
raise tarfile.ReadError("empty file")
except tarfile.TruncatedHeaderError as e:
if self.offset == 0:
raise tarfile.ReadError(str(e))
break
except tarfile.SubsequentHeaderError as e:
raise tarfile.ReadError(str(e))
break

if tarinfo is not None:
self.members.append(tarinfo)
else:
self._loaded = True
if tarinfo is not None:
self.members.append(tarinfo)
else:
self._loaded = True

return tarinfo
else:
TarFile = tarfile.TarFile
return tarinfo