Merge pull request #972 from davidmarin/drop-python2.5

Drop Python 2.5

CHANGES.txt

@@ -31,6 +31,8 @@ v0.4.3, 2014-03-?? -- mostly cleanup and bugfixes
  * Fix to avoid interpreting windows paths as URIs (#880)
  * Better error message when ssh keyfile is missing (#858)
  * Update EMR tool ISO8601 parsing to be consistent with EMR runner (#870)
+ * Dropped support for Python 2.5 (#713)
+   * Dropped support for the 1.x EMR AMI series, which uses Python 2.5
 
 v0.4.2, 2013-11-27 -- that's one small step for a JAR
  * jobs:

README.rst

@@ -3,7 +3,7 @@ mrjob: the Python MapReduce library
 
 .. image:: docs/logos/logo_medium.png
 
-mrjob is a Python 2.5+ package that helps you write and run Hadoop Streaming
+mrjob is a Python 2.6+ package that helps you write and run Hadoop Streaming
 jobs.
 
 `Stable version (v0.4.2) documentation <http://packages.python.org/mrjob/>`_

docs/guides/configs-all-runners.rst

@@ -201,7 +201,7 @@ Job execution context
 
     Interpreter to launch your script with. Defaults to the value of
     **python_bin**, which is deprecated. Change this if you're using a
-    language besides Python 2.5-2.7.
+    language besides Python 2.6-2.7.
 
 .. mrjob-opt::
     :config: python_bin

docs/guides/configs-reference.rst

@@ -17,9 +17,9 @@ You can set an option by:
 
     runners:
         local:
-            python_bin: python2.6  # only used in local runner
+            python_bin: python2.7  # only used in local runner
         emr:
-            python_bin: python2.5  # only used in Elastic MapReduce runner
+            python_bin: python2.6  # only used in Elastic MapReduce runner
 
   See :ref:`mrjob.conf` for information on where to put config files.
 

docs/guides/emr-opts.rst

@@ -74,6 +74,11 @@ Job flow creation and configuration
     .. _`the AWS docs on specifying the AMI version`:
         http://docs.amazonwebservices.com/ElasticMapReduce/latest/DeveloperGuide/EnvironmentConfig_AMIVersion.html
 
+    .. warning::
+
+        The 1.x series of AMIs is no longer supported because they use Python
+        2.5.
+
 .. mrjob-opt::
     :config: aws_availability_zone
     :switch: --aws-availability-zone

docs/guides/emr-troubleshooting.rst

@@ -70,19 +70,19 @@ Now you can use the job flow ID to start the troublesome job::
     Traceback (most recent call last):
       File "buggy_job.py", line 36, in <module>
         MRWordFreqCount.run()
-      File "/usr/lib/python2.5/site-packages/mrjob/job.py", line 448, in run
+      File "/usr/lib/python2.7/site-packages/mrjob/job.py", line 448, in run
         mr_job.execute()
-      File "/usr/lib/python2.5/site-packages/mrjob/job.py", line 455, in execute
+      File "/usr/lib/python2.7/site-packages/mrjob/job.py", line 455, in execute
         self.run_mapper(self.options.step_num)
-      File "/usr/lib/python2.5/site-packages/mrjob/job.py", line 548, in run_mapper
+      File "/usr/lib/python2.7/site-packages/mrjob/job.py", line 548, in run_mapper
         for out_key, out_value in mapper(key, value) or ():
       File "buggy_job.py", line 24, in mapper
         raise IndexError
     IndexError
     Traceback (most recent call last):
       File "wrapper.py", line 16, in <module>
         check_call(sys.argv[1:])
-      File "/usr/lib/python2.5/subprocess.py", line 462, in check_call
+      File "/usr/lib/python2.7/subprocess.py", line 462, in check_call
         raise CalledProcessError(retcode, cmd)
     subprocess.CalledProcessError: Command '['python', 'buggy_job.py', '--step-num=0', '--mapper']' returned non-zero exit status 1
     (while reading from s3://scratch-bucket/tmp/buggy_job.username.20110811.185410.536519/input/00000-README.rst)
@@ -108,19 +108,19 @@ you can use the :py:mod:`mrjob.tools.emr.fetch_logs` tool to scan the logs::
     Traceback (most recent call last):
       File "buggy_job.py", line 36, in <module>
         MRWordFreqCount.run()
-      File "/usr/lib/python2.5/site-packages/mrjob/job.py", line 448, in run
+      File "/usr/lib/python2.7/site-packages/mrjob/job.py", line 448, in run
         mr_job.execute()
-      File "/usr/lib/python2.5/site-packages/mrjob/job.py", line 455, in execute
+      File "/usr/lib/python2.7/site-packages/mrjob/job.py", line 455, in execute
         self.run_mapper(self.options.step_num)
-      File "/usr/lib/python2.5/site-packages/mrjob/job.py", line 548, in run_mapper
+      File "/usr/lib/python2.7/site-packages/mrjob/job.py", line 548, in run_mapper
         for out_key, out_value in mapper(key, value) or ():
       File "buggy_job.py", line 24, in mapper
         raise IndexError
     IndexError
     Traceback (most recent call last):
       File "wrapper.py", line 16, in <module>
         check_call(sys.argv[1:])
-      File "/usr/lib/python2.5/subprocess.py", line 462, in check_call
+      File "/usr/lib/python2.7/subprocess.py", line 462, in check_call
         raise CalledProcessError(retcode, cmd)
     subprocess.CalledProcessError: Command '['python', 'buggy_job.py', '--step-num=0', '--mapper']' returned non-zero exit status 1
     (while reading from s3://scratch-bucket/tmp/buggy_job.username.20110811.185410.536519/input/00000-README.rst)
@@ -145,19 +145,19 @@ problem. You can look at the logs yourself by using Amazon's `elastic-mapreduce
     Traceback (most recent call last):
       File "mr_word_freq_count.py", line 36, in <module>
         MRWordFreqCount.run()
-      File "/usr/lib/python2.5/site-packages/mrjob/job.py", line 448, in run
+      File "/usr/lib/python2.7/site-packages/mrjob/job.py", line 448, in run
         mr_job.execute()
-      File "/usr/lib/python2.5/site-packages/mrjob/job.py", line 455, in execute
+      File "/usr/lib/python2.7/site-packages/mrjob/job.py", line 455, in execute
         self.run_mapper(self.options.step_num)
-      File "/usr/lib/python2.5/site-packages/mrjob/job.py", line 548, in run_mapper
+      File "/usr/lib/python2.7/site-packages/mrjob/job.py", line 548, in run_mapper
         for out_key, out_value in mapper(key, value) or ():
       File "mr_word_freq_count.py", line 24, in mapper
         raise IndexError
     IndexError
     Traceback (most recent call last):
       File "wrapper.py", line 16, in <module>
         check_call(sys.argv[1:])
-      File "/usr/lib/python2.5/subprocess.py", line 462, in check_call
+      File "/usr/lib/python2.7/subprocess.py", line 462, in check_call
         raise CalledProcessError(retcode, cmd)
     subprocess.CalledProcessError: Command '['python', 'mr_word_freq_count.py', '--step-num=0', '--mapper']' returned non-zero exit status 1
     java.lang.RuntimeException: PipeMapRed.waitOutputThreads(): subprocess failed with code 1

docs/guides/runners.rst

@@ -72,8 +72,6 @@ Running on your own Hadoop cluster
 ----------------------------------
 
 * Set up a hadoop cluster (see http://hadoop.apache.org/common/docs/current/)
-* If running Python 2.5 on your cluster, install the :py:mod:`simplejson`
-  module on all nodes (recommended but not required for Python 2.6+).
 * Make sure :envvar:`HADOOP_HOME` is set
 * Run your job with ``-r hadoop``::
 
@@ -115,8 +113,6 @@ This pattern can also be used to write integration tests (see :doc:`testing`).
 
 ::
 
-    from __future__ import with_statement # only needed on Python 2.5
-
     mr_job = MRWordCounter(args=['-r', 'emr'])
     with mr_job.make_runner() as runner:
         runner.run()
@@ -236,8 +232,6 @@ example below demonstrates the use of counters in a test case.
 ``test_counters.py``
 ::
 
-    from __future__ import with_statement
-
     try:
         import unittest2 as unittest
     except ImportError:

docs/guides/testing.rst

@@ -38,12 +38,7 @@ See :py:class:`~mrjob.local.LocalMRJobRunner` for reference about its behavior.
 Anatomy of a test case
 ----------------------
 
-mrjob's test cases use the :py:mod:`unittest2` module, which is available
-for Python 2.3 and up. Most tests also require the :keyword:`with` statement.
-
-::
-
-    from __future__ import with_statement
+mrjob's test cases use the :py:mod:`unittest2` module::
 
     try:
         import unittest2 as unittest

docs/index.rst

@@ -1,7 +1,7 @@
 mrjob
 =====
 
-**mrjob lets you write MapReduce jobs in Python 2.5+ and run them on several
+**mrjob lets you write MapReduce jobs in Python 2.6+ and run them on several
 platforms.** You can:
 
 * Write multi-step MapReduce jobs in pure Python

docs/whats-new.rst

@@ -18,6 +18,8 @@ Job flows are now visible to all IAM users by default.
 
 Tests now use pytest and support tox.
 
+Support for Python 2.5 has been dropped.
+
 This release also contains many `bugfixes
 <https://github.com/Yelp/mrjob/blob/master/CHANGES.txt>`_, including
 problems with documentation.

mrjob/aws.py

@@ -15,7 +15,6 @@
 """General information about Amazon Web Services, such as region-to-endpoint
 mappings.
 """
-from __future__ import with_statement
 
 ### EC2 Instances ###
 

mrjob/cmd.py

@@ -11,13 +11,9 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
-from __future__ import with_statement
-
 from sys import argv
 from sys import stderr
 
-
 commands = {}
 
 descriptions = {}

mrjob/conf.py

@@ -16,9 +16,6 @@
 """"mrjob.conf" is the name of both this module, and the global config file
 for :py:mod:`mrjob`.
 """
-
-from __future__ import with_statement
-
 import glob
 from itertools import chain
 import logging
@@ -152,7 +149,7 @@ def conf_object_at_path(conf_path):
         else:
             try:
                 return json.load(f)
-            except ValueError, e:
+            except ValueError as e:
                 msg = ('If your mrjob.conf is in YAML, you need to install'
                        ' yaml; see http://pypi.python.org/pypi/PyYAML/')
                 # Use msg attr if it's set

mrjob/emr.py

@@ -12,7 +12,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from __future__ import with_statement
 
 import logging
 import os
@@ -221,7 +220,7 @@ def describe_all_job_flows(emr_conn, states=None, jobflow_ids=None,
             results = emr_conn.describe_jobflows(
                 states=states, jobflow_ids=jobflow_ids,
                 created_after=created_after, created_before=created_before)
-        except boto.exception.BotoServerError, ex:
+        except boto.exception.BotoServerError as ex:
             if 'ValidationError' in ex.body:
                 log.debug(
                     '  reached earliest allowed created_before time, done!')
@@ -565,6 +564,11 @@ def __init__(self, **kwargs):
         else:
             self._output_dir = self._s3_tmp_uri + 'output/'
 
+        # check AMI version
+        if self._opts['ami_version'].startswith('1.'):
+            log.warning('1.x AMIs will probably not work because they use'
+                        ' Python 2.5. Use a later AMI version or mrjob v0.4.2')
+
         # manage working dir for bootstrap script
         self._bootstrap_dir_mgr = BootstrapWorkingDirManager()
 
@@ -1000,7 +1004,7 @@ def cleanup(self, mode=None):
                 try:
                     os.kill(self._ssh_proc.pid, signal.SIGKILL)
                     self._ssh_proc = None
-                except Exception, e:
+                except Exception as e:
                     log.exception(e)
 
         # stop the job flow if it belongs to us (it may have stopped on its
@@ -1012,7 +1016,7 @@ def cleanup(self, mode=None):
             log.info('Terminating job flow: %s' % self._emr_job_flow_id)
             try:
                 self.make_emr_conn().terminate_jobflow(self._emr_job_flow_id)
-            except Exception, e:
+            except Exception as e:
                 log.exception(e)
 
     def _cleanup_remote_scratch(self):
@@ -1022,7 +1026,7 @@ def _cleanup_remote_scratch(self):
                 log.info('Removing all files in %s' % self._s3_tmp_uri)
                 self.rm(self._s3_tmp_uri)
                 self._s3_tmp_uri = None
-            except Exception, e:
+            except Exception as e:
                 log.exception(e)
 
     def _cleanup_logs(self):
@@ -1036,7 +1040,7 @@ def _cleanup_logs(self):
                 log.info('Removing all files in %s' % self._s3_job_log_uri)
                 self.rm(self._s3_job_log_uri)
                 self._s3_job_log_uri = None
-            except Exception, e:
+            except Exception as e:
                 log.exception(e)
 
     def _cleanup_job(self):
@@ -1080,7 +1084,7 @@ def _cleanup_job_flow(self):
         try:
             log.info("Attempting to terminate job flow")
             emr_conn.terminate_jobflow(self._emr_job_flow_id)
-        except Exception, e:
+        except Exception as e:
             # Something happened with boto and the user should know.
             log.exception(e)
             return
@@ -1631,7 +1635,7 @@ def _ls_ssh_logs(self, relative_path):
             self._enable_slave_ssh_access()
             log.debug('Search %s for logs' % self._ssh_path(relative_path))
             return self.ls(self._ssh_path(relative_path))
-        except IOError, e:
+        except IOError as e:
             raise LogFetchError(e)
 
     def _ls_slave_ssh_logs(self, addr, relative_path):
@@ -1798,7 +1802,7 @@ def _fetch_counters_s3(self, step_nums, skip_s3_wait=False):
                              " mrjob fetch-logs --counters %s" %
                              job_flow.jobflowid)
             return results
-        except LogFetchError, e:
+        except LogFetchError as e:
             log.info("Unable to fetch counters: %s" % e)
             return {}
 
@@ -1851,7 +1855,7 @@ def _find_probable_cause_of_failure_ssh(self, step_nums,
             task_attempt_logs = self.ls_task_attempt_logs_ssh(step_nums)
             step_logs = self.ls_step_logs_ssh(lg_step_nums)
             job_logs = self.ls_job_logs_ssh(step_nums)
-        except IOError, e:
+        except IOError as e:
             raise LogFetchError(e)
         log.info('Scanning SSH logs for probable cause of failure')
         return best_error_from_logs(self, task_attempt_logs, step_logs,

mrjob/examples/mr_postfix_bounce/mr_postfix_bounce.py

@@ -17,8 +17,6 @@
 The emitted email addresses can then be unconfirmed or handled in some other
 way.
 """
-from __future__ import with_statement
-
 __author__ = 'Adam Derewecki <derewecki@gmail.com>'
 
 import datetime
@@ -28,7 +26,6 @@
 
 from mrjob.job import MRJob
 
-
 PROCESS_TYPE_PATTERN = re.compile(
     r'postfix-(?P<queue>[^/]+)/(?P<process>[^[]+)\[\d+\]:')
 MESSAGE_ID_PATTERN = re.compile(

mrjob/fs/composite.py

@@ -52,7 +52,7 @@ def _do_action(self, action, path, *args, **kwargs):
             if fs.can_handle_path(path):
                 try:
                     return getattr(fs, action)(path, *args, **kwargs)
-                except IOError, e:
+                except IOError as e:
                     if first_exception is None:
                         first_exception = e
 

mrjob/fs/local.py

@@ -9,8 +9,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-from __future__ import with_statement
-
 import glob
 import hashlib
 import logging
@@ -21,7 +19,6 @@
 from mrjob.parse import is_uri
 from mrjob.util import read_file
 
-
 log = logging.getLogger(__name__)
 
 

mrjob/fs/s3.py

@@ -243,7 +243,7 @@ def get_s3_key(self, uri, s3_conn=None):
 
         try:
             bucket = s3_conn.get_bucket(bucket_name, validate=VALIDATE_BUCKET)
-        except boto.exception.S3ResponseError, e:
+        except boto.exception.S3ResponseError as e:
             if e.status != 404:
                 raise e
             key = None

mrjob/hadoop.py

@@ -374,7 +374,7 @@ def treat_eio_as_eof(iter):
             while True:
                 try:
                     yield iter.next()  # okay for StopIteration to bubble up
-                except IOError, e:
+                except IOError as e:
                     if e.errno == errno.EIO:
                         return
                     else:
@@ -509,7 +509,7 @@ def _cleanup_local_scratch(self):
 
             try:
                 self.invoke_hadoop(['fs', '-rmr', self._hdfs_tmp_dir])
-            except Exception, e:
+            except Exception as e:
                 log.exception(e)
 
     ### LOG FETCHING/PARSING ###