ARROW-5757: [Python] Remove Python 2.7 support

Part of the changes were done using [pyupgrade](https://github.com/asottile/pyupgrade). Closes #6410 from pitrou/ARROW-5757-py2-goodbye and squashes the following commits: f0f9f51 <Antoine Pitrou> Address review comments 561ac96 <Antoine Pitrou> ARROW-5757: Remove Python 2.7 support Authored-by: Antoine Pitrou <antoine@python.org> Signed-off-by: Wes McKinney <wesm+git@apache.org>
apache · Feb 13, 2020 · 412145b · 412145b
1 parent 3bc01ec
commit 412145b
Show file tree

Hide file tree

Showing 98 changed files with 426 additions and 1,123 deletions.
diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
@@ -72,7 +72,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python: [2.7, 3.6, 3.8]
+        python: [3.6, 3.8]
     env:
       PYTHON: ${{ matrix.python }}
       PANDAS: latest

diff --git a/ci/docker/conda-python.dockerfile b/ci/docker/conda-python.dockerfile
@@ -24,7 +24,6 @@ ARG python=3.6
 COPY ci/conda_env_python.yml /arrow/ci/
 RUN conda install -q \
         --file arrow/ci/conda_env_python.yml \
-        $([ "$python" == "2.7" ] && echo "futures enum34") \
         $([ "$python" == "3.6" -o "$python" == "3.7" ] && echo "pickle5") \
         python=${python} \
         nomkl && \

diff --git a/cpp/src/arrow/python/common.h b/cpp/src/arrow/python/common.h
@@ -134,8 +134,6 @@ auto SafeCallIntoPython(Function&& func) -> decltype(func()) {
   return maybe_status;
 }
 
-#define PYARROW_IS_PY2 PY_MAJOR_VERSION <= 2
-
 // A RAII primitive that DECREFs the underlying PyObject* when it
 // goes out of scope.
 class ARROW_PYTHON_EXPORT OwnedRef {
@@ -211,21 +209,13 @@ struct PyBytesView {
   }
 
   Status FromUnicode(PyObject* obj) {
-#if PY_MAJOR_VERSION >= 3
     Py_ssize_t size;
     // The utf-8 representation is cached on the unicode object
     const char* data = PyUnicode_AsUTF8AndSize(obj, &size);
     RETURN_IF_PYERROR();
     this->bytes = data;
     this->size = size;
     this->ref.reset();
-#else
-    PyObject* converted = PyUnicode_AsUTF8String(obj);
-    RETURN_IF_PYERROR();
-    this->bytes = PyBytes_AS_STRING(converted);
-    this->size = PyBytes_GET_SIZE(converted);
-    this->ref.reset(converted);
-#endif
     return Status::OK();
   }
 

diff --git a/cpp/src/arrow/python/config.h b/cpp/src/arrow/python/config.h
@@ -23,10 +23,6 @@
 #include "arrow/python/numpy_interop.h"
 #include "arrow/python/visibility.h"
 
-#if PY_MAJOR_VERSION >= 3
-#define PyString_Check PyUnicode_Check
-#endif
-
 namespace arrow {
 namespace py {
 

diff --git a/cpp/src/arrow/python/datetime.h b/cpp/src/arrow/python/datetime.h
@@ -121,36 +121,23 @@ inline TimePoint PyDateTime_to_TimePoint(PyDateTime_DateTime* pydatetime) {
 ARROW_PYTHON_EXPORT
 inline int64_t PyDelta_to_s(PyDateTime_Delta* pytimedelta) {
   int64_t total_seconds = 0;
-#if PY_VERSION_HEX >= 0x03000000
   total_seconds += PyDateTime_DELTA_GET_SECONDS(pytimedelta);
   total_seconds += PyDateTime_DELTA_GET_DAYS(pytimedelta) * 24 * 3600;
-#else
-  total_seconds += pytimedelta->seconds;
-  total_seconds += pytimedelta->days * 24 * 3600;
-#endif
   return total_seconds;
 }
 
 ARROW_PYTHON_EXPORT
 inline int64_t PyDelta_to_ms(PyDateTime_Delta* pytimedelta) {
   int64_t total_ms = PyDelta_to_s(pytimedelta) * 1000;
-#if PY_VERSION_HEX >= 0x03000000
   total_ms += PyDateTime_DELTA_GET_MICROSECONDS(pytimedelta) / 1000;
-#else
-  total_ms += pytimedelta->microseconds / 1000;
-#endif
   return total_ms;
 }
 
 ARROW_PYTHON_EXPORT
 inline int64_t PyDelta_to_us(PyDateTime_Delta* pytimedelta) {
   int64_t total_us = 0;
   total_us += PyDelta_to_s(pytimedelta) * 1000 * 1000;
-#if PY_VERSION_HEX >= 0x03000000
   total_us += PyDateTime_DELTA_GET_MICROSECONDS(pytimedelta);
-#else
-  total_us += pytimedelta->microseconds;
-#endif
   return total_us;
 }
 

diff --git a/cpp/src/arrow/python/deserialize.cc b/cpp/src/arrow/python/deserialize.cc
@@ -123,12 +123,6 @@ Status GetValue(PyObject* context, const Array& arr, int64_t index, int8_t type,
       return Status::OK();
     case PythonType::PY2INT:
     case PythonType::INT: {
-#if PY_MAJOR_VERSION < 3
-      if (type == PythonType::PY2INT) {
-        *result = PyInt_FromSsize_t(checked_cast<const Int64Array&>(arr).Value(index));
-        return Status::OK();
-      }
-#endif
       *result = PyLong_FromSsize_t(checked_cast<const Int64Array&>(arr).Value(index));
       return Status::OK();
     }

diff --git a/cpp/src/arrow/python/helpers.cc b/cpp/src/arrow/python/helpers.cc
@@ -96,33 +96,22 @@ std::string PyBytes_AsStdString(PyObject* obj) {
 
 Status PyUnicode_AsStdString(PyObject* obj, std::string* out) {
   DCHECK(PyUnicode_Check(obj));
-#if PY_MAJOR_VERSION >= 3
   Py_ssize_t size;
   // The utf-8 representation is cached on the unicode object
   const char* data = PyUnicode_AsUTF8AndSize(obj, &size);
   RETURN_IF_PYERROR();
   *out = std::string(data, size);
   return Status::OK();
-#else
-  OwnedRef bytes_ref(PyUnicode_AsUTF8String(obj));
-  RETURN_IF_PYERROR();
-  *out = PyBytes_AsStdString(bytes_ref.obj());
-  return Status::OK();
-#endif
 }
 
 std::string PyObject_StdStringRepr(PyObject* obj) {
-#if PY_MAJOR_VERSION >= 3
   OwnedRef unicode_ref(PyObject_Repr(obj));
   OwnedRef bytes_ref;
 
   if (unicode_ref) {
     bytes_ref.reset(
         PyUnicode_AsEncodedString(unicode_ref.obj(), "utf8", "backslashreplace"));
   }
-#else
-  OwnedRef bytes_ref(PyObject_Repr(obj));
-#endif
   if (!bytes_ref) {
     PyErr_Clear();
     std::stringstream ss;
@@ -135,12 +124,7 @@ std::string PyObject_StdStringRepr(PyObject* obj) {
 Status PyObject_StdStringStr(PyObject* obj, std::string* out) {
   OwnedRef string_ref(PyObject_Str(obj));
   RETURN_IF_PYERROR();
-#if PY_MAJOR_VERSION >= 3
   return PyUnicode_AsStdString(string_ref.obj(), out);
-#else
-  *out = PyBytes_AsStdString(string_ref.obj());
-  return Status::OK();
-#endif
 }
 
 Status ImportModule(const std::string& module_name, OwnedRef* ref) {
@@ -259,18 +243,10 @@ template Status CIntFromPython(PyObject*, uint64_t*, const std::string&);
 
 inline bool MayHaveNaN(PyObject* obj) {
   // Some core types can be very quickly type-checked and do not allow NaN values
-#if PYARROW_IS_PY2
-  const int64_t non_nan_tpflags = Py_TPFLAGS_INT_SUBCLASS | Py_TPFLAGS_LONG_SUBCLASS |
-                                  Py_TPFLAGS_LIST_SUBCLASS | Py_TPFLAGS_TUPLE_SUBCLASS |
-                                  Py_TPFLAGS_STRING_SUBCLASS |
-                                  Py_TPFLAGS_UNICODE_SUBCLASS | Py_TPFLAGS_DICT_SUBCLASS |
-                                  Py_TPFLAGS_BASE_EXC_SUBCLASS | Py_TPFLAGS_TYPE_SUBCLASS;
-#else
   const int64_t non_nan_tpflags = Py_TPFLAGS_LONG_SUBCLASS | Py_TPFLAGS_LIST_SUBCLASS |
                                   Py_TPFLAGS_TUPLE_SUBCLASS | Py_TPFLAGS_BYTES_SUBCLASS |
                                   Py_TPFLAGS_UNICODE_SUBCLASS | Py_TPFLAGS_DICT_SUBCLASS |
                                   Py_TPFLAGS_BASE_EXC_SUBCLASS | Py_TPFLAGS_TYPE_SUBCLASS;
-#endif
   return !PyType_HasFeature(Py_TYPE(obj), non_nan_tpflags);
 }
 
@@ -313,10 +289,6 @@ Status UnboxIntegerAsInt64(PyObject* obj, int64_t* out) {
     if (overflow) {
       return Status::Invalid("PyLong is too large to fit int64");
     }
-#if PY_MAJOR_VERSION < 3
-  } else if (PyInt_Check(obj)) {
-    *out = static_cast<int64_t>(PyInt_AS_LONG(obj));
-#endif
   } else if (PyArray_IsScalar(obj, UByte)) {
     *out = reinterpret_cast<PyUByteScalarObject*>(obj)->obval;
   } else if (PyArray_IsScalar(obj, Short)) {

diff --git a/cpp/src/arrow/python/helpers.h b/cpp/src/arrow/python/helpers.h
@@ -65,13 +65,7 @@ ARROW_PYTHON_EXPORT
 Status ImportFromModule(PyObject* module, const std::string& name, OwnedRef* ref);
 
 // \brief Check whether obj is an integer, independent of Python versions.
-inline bool IsPyInteger(PyObject* obj) {
-#if PYARROW_IS_PY2
-  return PyLong_Check(obj) || PyInt_Check(obj);
-#else
-  return PyLong_Check(obj);
-#endif
-}
+inline bool IsPyInteger(PyObject* obj) { return PyLong_Check(obj); }
 
 // \brief Use pandas missing value semantics to check if a value is null
 ARROW_PYTHON_EXPORT

diff --git a/cpp/src/arrow/python/io.cc b/cpp/src/arrow/python/io.cc
@@ -123,14 +123,6 @@ class PythonFile {
   Status Write(const std::shared_ptr<Buffer>& buffer) {
     RETURN_NOT_OK(CheckClosed());
 
-#if PY_MAJOR_VERSION < 3
-    // On Python 2, a write() method can typically call str() on its argument
-    // to get its bytes payload (this is the case with socket.makefile()).
-    // Unfortunately, on non-bytes buffer-like objects this will give out
-    // the repr() of the object rather than its data.  So fall back on
-    // copying the data to a bytes object.
-    return Write(buffer->data(), buffer->size());
-#else
     PyObject* py_data = wrap_buffer(buffer);
     PY_RETURN_IF_ERROR(StatusCode::IOError);
 
@@ -139,7 +131,6 @@ class PythonFile {
     Py_XDECREF(result);
     PY_RETURN_IF_ERROR(StatusCode::IOError);
     return Status::OK();
-#endif
   }
 
   Result<int64_t> Tell() {

diff --git a/cpp/src/arrow/python/numpy_internal.h b/cpp/src/arrow/python/numpy_internal.h
@@ -161,11 +161,6 @@ inline bool PyFloatScalar_Check(PyObject* obj) {
 }
 
 inline bool PyIntScalar_Check(PyObject* obj) {
-#if PY_MAJOR_VERSION < 3
-  if (PyInt_Check(obj)) {
-    return true;
-  }
-#endif
   return PyLong_Check(obj) || PyArray_IsScalar(obj, Integer);
 }
 

diff --git a/cpp/src/arrow/python/python_test.cc b/cpp/src/arrow/python/python_test.cc
@@ -85,9 +85,6 @@ TEST(OwnedRefNoGIL, TestMoves) {
 std::string FormatPythonException(const std::string& exc_class_name) {
   std::stringstream ss;
   ss << "Python exception: ";
-#if PY_MAJOR_VERSION < 3
-  ss << "exceptions.";
-#endif
   ss << exc_class_name;
   return ss.str();
 }

diff --git a/cpp/src/arrow/python/serialize.cc b/cpp/src/arrow/python/serialize.cc
@@ -103,11 +103,6 @@ class SequenceBuilder {
     return AppendPrimitive(&bools_, data, PythonType::BOOL);
   }
 
-  // Appending a python 2 int64_t to the sequence
-  Status AppendPy2Int64(const int64_t data) {
-    return AppendPrimitive(&py2_ints_, data, PythonType::PY2INT);
-  }
-
   // Appending an int64_t to the sequence
   Status AppendInt64(const int64_t data) {
     return AppendPrimitive(&ints_, data, PythonType::INT);
@@ -252,7 +247,6 @@ class SequenceBuilder {
 
   std::shared_ptr<BooleanBuilder> bools_;
   std::shared_ptr<Int64Builder> ints_;
-  std::shared_ptr<Int64Builder> py2_ints_;
   std::shared_ptr<BinaryBuilder> bytes_;
   std::shared_ptr<StringBuilder> strings_;
   std::shared_ptr<HalfFloatBuilder> half_floats_;
@@ -458,10 +452,6 @@ Status Append(PyObject* context, PyObject* elem, SequenceBuilder* builder,
       RETURN_NOT_OK(
           builder->AppendDict(context, serialized_object, recursion_depth, blobs_out));
     }
-#if PY_MAJOR_VERSION < 3
-  } else if (PyInt_Check(elem)) {
-    RETURN_NOT_OK(builder->AppendPy2Int64(static_cast<int64_t>(PyInt_AS_LONG(elem))));
-#endif
   } else if (PyBytes_Check(elem)) {
     auto data = reinterpret_cast<uint8_t*>(PyBytes_AS_STRING(elem));
     int32_t size = -1;

diff --git a/cpp/src/arrow/python/serialize.h b/cpp/src/arrow/python/serialize.h
@@ -118,7 +118,7 @@ struct PythonType {
   enum type {
     BOOL,
     INT,
-    PY2INT,
+    PY2INT,  // Kept for compatibility
     BYTES,
     STRING,
     HALF_FLOAT,

diff --git a/dev/tasks/conda-recipes/.ci_support/linux_python2.7.yaml b/dev/tasks/conda-recipes/.ci_support/linux_python2.7.yaml
diff --git a/dev/tasks/conda-recipes/.ci_support/osx_python2.7.yaml b/dev/tasks/conda-recipes/.ci_support/osx_python2.7.yaml