Skip to content
Browse files

use minput as default env

  • Loading branch information...
1 parent 29e0d5d commit 8d414fe2b44b4b7c0046224749ae1a6a7bdb39d8 @wsxiaoys committed Sep 24, 2012
Showing with 172 additions and 199 deletions.
  1. +169 −196 hadoopy/_main.c
  2. +3 −3 hadoopy/_main.pyx
View
365 hadoopy/_main.c
@@ -1,4 +1,4 @@
-/* Generated by Cython 0.17 on Mon Sep 24 21:43:03 2012 */
+/* Generated by Cython 0.17 on Mon Sep 24 21:52:05 2012 */
#define PY_SSIZE_T_CLEAN
#include "Python.h"
@@ -626,14 +626,14 @@ static PyObject *__pyx_pf_13_hadoopy_main_11HadoopyTask_18read_in_map(struct __p
static PyObject *__pyx_pf_13_hadoopy_main_11HadoopyTask_20read_in_reduce(struct __pyx_obj_13_hadoopy_main_HadoopyTask *__pyx_v_self, PyObject *__pyx_v_type); /* proto */
static PyObject *__pyx_pf_13_hadoopy_main_11HadoopyTask_22is_on_hadoop(CYTHON_UNUSED struct __pyx_obj_13_hadoopy_main_HadoopyTask *__pyx_v_self); /* proto */
static PyObject *__pyx_pf_13_hadoopy_main_11HadoopyTask_24flush_tb_writes(CYTHON_UNUSED struct __pyx_obj_13_hadoopy_main_HadoopyTask *__pyx_v_self); /* proto */
-static char __pyx_k_4[] = "stream_reduce_output";
-static char __pyx_k_6[] = "%s\t%s\n";
-static char __pyx_k_7[] = "read_offset_value_text";
-static char __pyx_k_8[] = "mapred_input_format_class";
-static char __pyx_k_9[] = "hadoopy_flush_tb_writes";
-static char __pyx_k_10[] = "Brandyn A. White <bwhite@cs.umd.edu>";
-static char __pyx_k_11[] = "GPL V3";
-static char __pyx_k_12[] = "hadoopy._runner";
+static char __pyx_k_2[] = "stream_reduce_output";
+static char __pyx_k_3[] = "%s\t%s\n";
+static char __pyx_k_4[] = "read_offset_value_text";
+static char __pyx_k_5[] = "mapred_input_format_class";
+static char __pyx_k_6[] = "hadoopy_flush_tb_writes";
+static char __pyx_k_7[] = "Brandyn A. White <bwhite@cs.umd.edu>";
+static char __pyx_k_8[] = "GPL V3";
+static char __pyx_k_9[] = "hadoopy._runner";
static char __pyx_k__r[] = "r";
static char __pyx_k__os[] = "os";
static char __pyx_k__get[] = "get";
@@ -691,13 +691,13 @@ static char __pyx_k__stream_map_input[] = "stream_map_input";
static char __pyx_k__stream_map_output[] = "stream_map_output";
static char __pyx_k__read_key_value_text[] = "read_key_value_text";
static char __pyx_k__stream_reduce_input[] = "stream_reduce_input";
-static PyObject *__pyx_kp_s_10;
-static PyObject *__pyx_kp_s_11;
-static PyObject *__pyx_n_s_12;
+static PyObject *__pyx_n_s_2;
+static PyObject *__pyx_kp_s_3;
static PyObject *__pyx_n_s_4;
-static PyObject *__pyx_kp_s_6;
-static PyObject *__pyx_n_s_7;
-static PyObject *__pyx_n_s_8;
+static PyObject *__pyx_n_s_5;
+static PyObject *__pyx_n_s_6;
+static PyObject *__pyx_kp_s_7;
+static PyObject *__pyx_kp_s_8;
static PyObject *__pyx_n_s_9;
static PyObject *__pyx_n_s__AttributeError;
static PyObject *__pyx_n_s__RawBytesFile;
@@ -758,9 +758,6 @@ static PyObject *__pyx_n_s__writes;
static PyObject *__pyx_int_0;
static PyObject *__pyx_int_1;
static PyObject *__pyx_k_tuple_1;
-static PyObject *__pyx_k_tuple_2;
-static PyObject *__pyx_k_tuple_3;
-static PyObject *__pyx_k_tuple_5;
/* Python wrapper */
static int __pyx_pw_13_hadoopy_main_14KeyValueStream_1__init__(PyObject *__pyx_v_self, PyObject *__pyx_args, PyObject *__pyx_kwds); /*proto*/
@@ -2699,8 +2696,8 @@ static PyObject *__pyx_pf_13_hadoopy_main_11HadoopyTask_2run(struct __pyx_obj_13
__Pyx_RefNannyDeclarations
PyObject *__pyx_t_1 = NULL;
PyObject *__pyx_t_2 = NULL;
- int __pyx_t_3;
- PyObject *__pyx_t_4 = NULL;
+ PyObject *__pyx_t_3 = NULL;
+ int __pyx_t_4;
PyObject *__pyx_t_5 = NULL;
PyObject *__pyx_t_6 = NULL;
int __pyx_lineno = 0;
@@ -2713,7 +2710,7 @@ static PyObject *__pyx_pf_13_hadoopy_main_11HadoopyTask_2run(struct __pyx_obj_13
* def run(self):
* env = os.environ # <<<<<<<<<<<<<<
* minput = env.get('stream_map_input', 'text')
- * moutput = env.get('stream_map_output', 'text')
+ * moutput = env.get('stream_map_output', minput)
*/
__pyx_t_1 = __Pyx_GetName(__pyx_m, __pyx_n_s__os); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 190; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
__Pyx_GOTREF(__pyx_t_1);
@@ -2727,8 +2724,8 @@ static PyObject *__pyx_pf_13_hadoopy_main_11HadoopyTask_2run(struct __pyx_obj_13
* def run(self):
* env = os.environ
* minput = env.get('stream_map_input', 'text') # <<<<<<<<<<<<<<
- * moutput = env.get('stream_map_output', 'text')
- * rinput = env.get('stream_reduce_input', 'text')
+ * moutput = env.get('stream_map_output', minput)
+ * rinput = env.get('stream_reduce_input', minput)
*/
__pyx_t_2 = PyObject_GetAttr(__pyx_v_env, __pyx_n_s__get); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 191; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
__Pyx_GOTREF(__pyx_t_2);
@@ -2741,59 +2738,86 @@ static PyObject *__pyx_pf_13_hadoopy_main_11HadoopyTask_2run(struct __pyx_obj_13
/* "hadoopy/_main.pyx":192
* env = os.environ
* minput = env.get('stream_map_input', 'text')
- * moutput = env.get('stream_map_output', 'text') # <<<<<<<<<<<<<<
- * rinput = env.get('stream_reduce_input', 'text')
- * routput = env.get('stream_reduce_output', 'text')
+ * moutput = env.get('stream_map_output', minput) # <<<<<<<<<<<<<<
+ * rinput = env.get('stream_reduce_input', minput)
+ * routput = env.get('stream_reduce_output', minput)
*/
__pyx_t_1 = PyObject_GetAttr(__pyx_v_env, __pyx_n_s__get); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 192; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
__Pyx_GOTREF(__pyx_t_1);
- __pyx_t_2 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_k_tuple_2), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 192; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+ __pyx_t_2 = PyTuple_New(2); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 192; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
__Pyx_GOTREF(__pyx_t_2);
+ __Pyx_INCREF(((PyObject *)__pyx_n_s__stream_map_output));
+ PyTuple_SET_ITEM(__pyx_t_2, 0, ((PyObject *)__pyx_n_s__stream_map_output));
+ __Pyx_GIVEREF(((PyObject *)__pyx_n_s__stream_map_output));
+ __Pyx_INCREF(__pyx_v_minput);
+ PyTuple_SET_ITEM(__pyx_t_2, 1, __pyx_v_minput);
+ __Pyx_GIVEREF(__pyx_v_minput);
+ __pyx_t_3 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 192; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+ __Pyx_GOTREF(__pyx_t_3);
__Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
- __pyx_v_moutput = __pyx_t_2;
- __pyx_t_2 = 0;
+ __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0;
+ __pyx_v_moutput = __pyx_t_3;
+ __pyx_t_3 = 0;
/* "hadoopy/_main.pyx":193
* minput = env.get('stream_map_input', 'text')
- * moutput = env.get('stream_map_output', 'text')
- * rinput = env.get('stream_reduce_input', 'text') # <<<<<<<<<<<<<<
- * routput = env.get('stream_reduce_output', 'text')
+ * moutput = env.get('stream_map_output', minput)
+ * rinput = env.get('stream_reduce_input', minput) # <<<<<<<<<<<<<<
+ * routput = env.get('stream_reduce_output', minput)
*
*/
- __pyx_t_2 = PyObject_GetAttr(__pyx_v_env, __pyx_n_s__get); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 193; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+ __pyx_t_3 = PyObject_GetAttr(__pyx_v_env, __pyx_n_s__get); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 193; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+ __Pyx_GOTREF(__pyx_t_3);
+ __pyx_t_2 = PyTuple_New(2); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 193; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
__Pyx_GOTREF(__pyx_t_2);
- __pyx_t_1 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_k_tuple_3), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 193; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+ __Pyx_INCREF(((PyObject *)__pyx_n_s__stream_reduce_input));
+ PyTuple_SET_ITEM(__pyx_t_2, 0, ((PyObject *)__pyx_n_s__stream_reduce_input));
+ __Pyx_GIVEREF(((PyObject *)__pyx_n_s__stream_reduce_input));
+ __Pyx_INCREF(__pyx_v_minput);
+ PyTuple_SET_ITEM(__pyx_t_2, 1, __pyx_v_minput);
+ __Pyx_GIVEREF(__pyx_v_minput);
+ __pyx_t_1 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 193; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
__Pyx_GOTREF(__pyx_t_1);
- __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+ __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
+ __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0;
__pyx_v_rinput = __pyx_t_1;
__pyx_t_1 = 0;
/* "hadoopy/_main.pyx":194
- * moutput = env.get('stream_map_output', 'text')
- * rinput = env.get('stream_reduce_input', 'text')
- * routput = env.get('stream_reduce_output', 'text') # <<<<<<<<<<<<<<
+ * moutput = env.get('stream_map_output', minput)
+ * rinput = env.get('stream_reduce_input', minput)
+ * routput = env.get('stream_reduce_output', minput) # <<<<<<<<<<<<<<
*
* if self.task_type == 'map':
*/
__pyx_t_1 = PyObject_GetAttr(__pyx_v_env, __pyx_n_s__get); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 194; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
__Pyx_GOTREF(__pyx_t_1);
- __pyx_t_2 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_k_tuple_5), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 194; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+ __pyx_t_2 = PyTuple_New(2); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 194; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
__Pyx_GOTREF(__pyx_t_2);
+ __Pyx_INCREF(((PyObject *)__pyx_n_s_2));
+ PyTuple_SET_ITEM(__pyx_t_2, 0, ((PyObject *)__pyx_n_s_2));
+ __Pyx_GIVEREF(((PyObject *)__pyx_n_s_2));
+ __Pyx_INCREF(__pyx_v_minput);
+ PyTuple_SET_ITEM(__pyx_t_2, 1, __pyx_v_minput);
+ __Pyx_GIVEREF(__pyx_v_minput);
+ __pyx_t_3 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 194; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+ __Pyx_GOTREF(__pyx_t_3);
__Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
- __pyx_v_routput = __pyx_t_2;
- __pyx_t_2 = 0;
+ __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0;
+ __pyx_v_routput = __pyx_t_3;
+ __pyx_t_3 = 0;
/* "hadoopy/_main.pyx":196
- * routput = env.get('stream_reduce_output', 'text')
+ * routput = env.get('stream_reduce_output', minput)
*
* if self.task_type == 'map': # <<<<<<<<<<<<<<
* return self.process_inout(self.mapper, self.read_in_map(minput), self.print_out(moutput), 'map')
* elif self.task_type == 'reduce':
*/
- __pyx_t_2 = PyObject_RichCompare(__pyx_v_self->task_type, ((PyObject *)__pyx_n_s__map), Py_EQ); __Pyx_XGOTREF(__pyx_t_2); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 196; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_t_2); if (unlikely(__pyx_t_3 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 196; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
- if (__pyx_t_3) {
+ __pyx_t_3 = PyObject_RichCompare(__pyx_v_self->task_type, ((PyObject *)__pyx_n_s__map), Py_EQ); __Pyx_XGOTREF(__pyx_t_3); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 196; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+ __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_3); if (unlikely(__pyx_t_4 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 196; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+ __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
+ if (__pyx_t_4) {
/* "hadoopy/_main.pyx":197
*
@@ -2803,48 +2827,48 @@ static PyObject *__pyx_pf_13_hadoopy_main_11HadoopyTask_2run(struct __pyx_obj_13
* return self.process_inout(self.reducer, self.read_in_reduce(rinput), self.print_out(routput), 'reduce')
*/
__Pyx_XDECREF(__pyx_r);
- __pyx_t_2 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_n_s__process_inout); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 197; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+ __pyx_t_3 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_n_s__process_inout); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 197; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+ __Pyx_GOTREF(__pyx_t_3);
+ __pyx_t_2 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_n_s__read_in_map); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 197; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
__Pyx_GOTREF(__pyx_t_2);
- __pyx_t_1 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_n_s__read_in_map); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 197; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+ __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 197; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
__Pyx_GOTREF(__pyx_t_1);
- __pyx_t_4 = PyTuple_New(1); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 197; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_4);
__Pyx_INCREF(__pyx_v_minput);
- PyTuple_SET_ITEM(__pyx_t_4, 0, __pyx_v_minput);
+ PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_minput);
__Pyx_GIVEREF(__pyx_v_minput);
- __pyx_t_5 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_t_4), NULL); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 197; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+ __pyx_t_5 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 197; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
__Pyx_GOTREF(__pyx_t_5);
- __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
- __Pyx_DECREF(((PyObject *)__pyx_t_4)); __pyx_t_4 = 0;
- __pyx_t_4 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_n_s__print_out); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 197; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_4);
- __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 197; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+ __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+ __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0;
+ __pyx_t_1 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_n_s__print_out); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 197; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
__Pyx_GOTREF(__pyx_t_1);
+ __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 197; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+ __Pyx_GOTREF(__pyx_t_2);
__Pyx_INCREF(__pyx_v_moutput);
- PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_moutput);
+ PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_moutput);
__Pyx_GIVEREF(__pyx_v_moutput);
- __pyx_t_6 = PyObject_Call(__pyx_t_4, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 197; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+ __pyx_t_6 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 197; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
__Pyx_GOTREF(__pyx_t_6);
- __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
- __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0;
- __pyx_t_1 = PyTuple_New(4); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 197; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_1);
+ __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+ __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0;
+ __pyx_t_2 = PyTuple_New(4); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 197; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+ __Pyx_GOTREF(__pyx_t_2);
__Pyx_INCREF(__pyx_v_self->mapper);
- PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_self->mapper);
+ PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_self->mapper);
__Pyx_GIVEREF(__pyx_v_self->mapper);
- PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_t_5);
+ PyTuple_SET_ITEM(__pyx_t_2, 1, __pyx_t_5);
__Pyx_GIVEREF(__pyx_t_5);
- PyTuple_SET_ITEM(__pyx_t_1, 2, __pyx_t_6);
+ PyTuple_SET_ITEM(__pyx_t_2, 2, __pyx_t_6);
__Pyx_GIVEREF(__pyx_t_6);
__Pyx_INCREF(((PyObject *)__pyx_n_s__map));
- PyTuple_SET_ITEM(__pyx_t_1, 3, ((PyObject *)__pyx_n_s__map));
+ PyTuple_SET_ITEM(__pyx_t_2, 3, ((PyObject *)__pyx_n_s__map));
__Pyx_GIVEREF(((PyObject *)__pyx_n_s__map));
__pyx_t_5 = 0;
__pyx_t_6 = 0;
- __pyx_t_6 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 197; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+ __pyx_t_6 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 197; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
__Pyx_GOTREF(__pyx_t_6);
- __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
- __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0;
+ __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
+ __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0;
__pyx_r = __pyx_t_6;
__pyx_t_6 = 0;
goto __pyx_L0;
@@ -2859,9 +2883,9 @@ static PyObject *__pyx_pf_13_hadoopy_main_11HadoopyTask_2run(struct __pyx_obj_13
* elif self.task_type == 'combine':
*/
__pyx_t_6 = PyObject_RichCompare(__pyx_v_self->task_type, ((PyObject *)__pyx_n_s__reduce), Py_EQ); __Pyx_XGOTREF(__pyx_t_6); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 198; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_t_6); if (unlikely(__pyx_t_3 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 198; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+ __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_6); if (unlikely(__pyx_t_4 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 198; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
__Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
- if (__pyx_t_3) {
+ if (__pyx_t_4) {
/* "hadoopy/_main.pyx":199
* return self.process_inout(self.mapper, self.read_in_map(minput), self.print_out(moutput), 'map')
@@ -2873,48 +2897,48 @@ static PyObject *__pyx_pf_13_hadoopy_main_11HadoopyTask_2run(struct __pyx_obj_13
__Pyx_XDECREF(__pyx_r);
__pyx_t_6 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_n_s__process_inout); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 199; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
__Pyx_GOTREF(__pyx_t_6);
- __pyx_t_1 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_n_s__read_in_reduce); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 199; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_1);
- __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 199; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+ __pyx_t_2 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_n_s__read_in_reduce); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 199; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
__Pyx_GOTREF(__pyx_t_2);
+ __pyx_t_3 = PyTuple_New(1); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 199; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+ __Pyx_GOTREF(__pyx_t_3);
__Pyx_INCREF(__pyx_v_rinput);
- PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_rinput);
+ PyTuple_SET_ITEM(__pyx_t_3, 0, __pyx_v_rinput);
__Pyx_GIVEREF(__pyx_v_rinput);
- __pyx_t_5 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 199; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+ __pyx_t_5 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_3), NULL); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 199; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
__Pyx_GOTREF(__pyx_t_5);
- __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
- __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0;
- __pyx_t_2 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_n_s__print_out); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 199; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+ __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
+ __Pyx_DECREF(((PyObject *)__pyx_t_3)); __pyx_t_3 = 0;
+ __pyx_t_3 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_n_s__print_out); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 199; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+ __Pyx_GOTREF(__pyx_t_3);
+ __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 199; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
__Pyx_GOTREF(__pyx_t_2);
- __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 199; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_1);
__Pyx_INCREF(__pyx_v_routput);
- PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_routput);
+ PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_routput);
__Pyx_GIVEREF(__pyx_v_routput);
- __pyx_t_4 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 199; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_4);
- __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
- __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0;
- __pyx_t_1 = PyTuple_New(4); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 199; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+ __pyx_t_1 = PyObject_Call(__pyx_t_3, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 199; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
__Pyx_GOTREF(__pyx_t_1);
+ __Pyx_DECREF(__pyx_t_3); __pyx_t_3 = 0;
+ __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0;
+ __pyx_t_2 = PyTuple_New(4); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 199; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+ __Pyx_GOTREF(__pyx_t_2);
__Pyx_INCREF(__pyx_v_self->reducer);
- PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_self->reducer);
+ PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_self->reducer);
__Pyx_GIVEREF(__pyx_v_self->reducer);
- PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_t_5);
+ PyTuple_SET_ITEM(__pyx_t_2, 1, __pyx_t_5);
__Pyx_GIVEREF(__pyx_t_5);
- PyTuple_SET_ITEM(__pyx_t_1, 2, __pyx_t_4);
- __Pyx_GIVEREF(__pyx_t_4);
+ PyTuple_SET_ITEM(__pyx_t_2, 2, __pyx_t_1);
+ __Pyx_GIVEREF(__pyx_t_1);
__Pyx_INCREF(((PyObject *)__pyx_n_s__reduce));
- PyTuple_SET_ITEM(__pyx_t_1, 3, ((PyObject *)__pyx_n_s__reduce));
+ PyTuple_SET_ITEM(__pyx_t_2, 3, ((PyObject *)__pyx_n_s__reduce));
__Pyx_GIVEREF(((PyObject *)__pyx_n_s__reduce));
__pyx_t_5 = 0;
- __pyx_t_4 = 0;
- __pyx_t_4 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 199; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_4);
+ __pyx_t_1 = 0;
+ __pyx_t_1 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 199; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+ __Pyx_GOTREF(__pyx_t_1);
__Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
- __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0;
- __pyx_r = __pyx_t_4;
- __pyx_t_4 = 0;
+ __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0;
+ __pyx_r = __pyx_t_1;
+ __pyx_t_1 = 0;
goto __pyx_L0;
goto __pyx_L3;
}
@@ -2926,10 +2950,10 @@ static PyObject *__pyx_pf_13_hadoopy_main_11HadoopyTask_2run(struct __pyx_obj_13
* return self.process_inout(self.combiner, self.read_in_reduce(rinput), self.print_out(routput), 'reduce')
* else:
*/
- __pyx_t_4 = PyObject_RichCompare(__pyx_v_self->task_type, ((PyObject *)__pyx_n_s__combine), Py_EQ); __Pyx_XGOTREF(__pyx_t_4); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 200; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __pyx_t_3 = __Pyx_PyObject_IsTrue(__pyx_t_4); if (unlikely(__pyx_t_3 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 200; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
- if (__pyx_t_3) {
+ __pyx_t_1 = PyObject_RichCompare(__pyx_v_self->task_type, ((PyObject *)__pyx_n_s__combine), Py_EQ); __Pyx_XGOTREF(__pyx_t_1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 200; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+ __pyx_t_4 = __Pyx_PyObject_IsTrue(__pyx_t_1); if (unlikely(__pyx_t_4 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 200; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+ __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+ if (__pyx_t_4) {
/* "hadoopy/_main.pyx":201
* return self.process_inout(self.reducer, self.read_in_reduce(rinput), self.print_out(routput), 'reduce')
@@ -2939,50 +2963,50 @@ static PyObject *__pyx_pf_13_hadoopy_main_11HadoopyTask_2run(struct __pyx_obj_13
* return 1
*/
__Pyx_XDECREF(__pyx_r);
- __pyx_t_4 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_n_s__process_inout); if (unlikely(!__pyx_t_4)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 201; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_4);
- __pyx_t_1 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_n_s__read_in_reduce); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 201; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+ __pyx_t_1 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_n_s__process_inout); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 201; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
__Pyx_GOTREF(__pyx_t_1);
+ __pyx_t_2 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_n_s__read_in_reduce); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 201; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+ __Pyx_GOTREF(__pyx_t_2);
__pyx_t_6 = PyTuple_New(1); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 201; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
__Pyx_GOTREF(__pyx_t_6);
__Pyx_INCREF(__pyx_v_rinput);
PyTuple_SET_ITEM(__pyx_t_6, 0, __pyx_v_rinput);
__Pyx_GIVEREF(__pyx_v_rinput);
- __pyx_t_5 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_t_6), NULL); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 201; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+ __pyx_t_5 = PyObject_Call(__pyx_t_2, ((PyObject *)__pyx_t_6), NULL); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 201; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
__Pyx_GOTREF(__pyx_t_5);
- __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+ __Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
__Pyx_DECREF(((PyObject *)__pyx_t_6)); __pyx_t_6 = 0;
__pyx_t_6 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_n_s__print_out); if (unlikely(!__pyx_t_6)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 201; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
__Pyx_GOTREF(__pyx_t_6);
- __pyx_t_1 = PyTuple_New(1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 201; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_1);
+ __pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 201; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+ __Pyx_GOTREF(__pyx_t_2);
__Pyx_INCREF(__pyx_v_routput);
- PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_routput);
+ PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_routput);
__Pyx_GIVEREF(__pyx_v_routput);
- __pyx_t_2 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 201; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_2);
+ __pyx_t_3 = PyObject_Call(__pyx_t_6, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 201; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+ __Pyx_GOTREF(__pyx_t_3);
__Pyx_DECREF(__pyx_t_6); __pyx_t_6 = 0;
- __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0;
- __pyx_t_1 = PyTuple_New(4); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 201; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_1);
+ __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0;
+ __pyx_t_2 = PyTuple_New(4); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 201; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+ __Pyx_GOTREF(__pyx_t_2);
__Pyx_INCREF(__pyx_v_self->combiner);
- PyTuple_SET_ITEM(__pyx_t_1, 0, __pyx_v_self->combiner);
+ PyTuple_SET_ITEM(__pyx_t_2, 0, __pyx_v_self->combiner);
__Pyx_GIVEREF(__pyx_v_self->combiner);
- PyTuple_SET_ITEM(__pyx_t_1, 1, __pyx_t_5);
+ PyTuple_SET_ITEM(__pyx_t_2, 1, __pyx_t_5);
__Pyx_GIVEREF(__pyx_t_5);
- PyTuple_SET_ITEM(__pyx_t_1, 2, __pyx_t_2);
- __Pyx_GIVEREF(__pyx_t_2);
+ PyTuple_SET_ITEM(__pyx_t_2, 2, __pyx_t_3);
+ __Pyx_GIVEREF(__pyx_t_3);
__Pyx_INCREF(((PyObject *)__pyx_n_s__reduce));
- PyTuple_SET_ITEM(__pyx_t_1, 3, ((PyObject *)__pyx_n_s__reduce));
+ PyTuple_SET_ITEM(__pyx_t_2, 3, ((PyObject *)__pyx_n_s__reduce));
__Pyx_GIVEREF(((PyObject *)__pyx_n_s__reduce));
__pyx_t_5 = 0;
- __pyx_t_2 = 0;
- __pyx_t_2 = PyObject_Call(__pyx_t_4, ((PyObject *)__pyx_t_1), NULL); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 201; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_t_2);
- __Pyx_DECREF(__pyx_t_4); __pyx_t_4 = 0;
- __Pyx_DECREF(((PyObject *)__pyx_t_1)); __pyx_t_1 = 0;
- __pyx_r = __pyx_t_2;
- __pyx_t_2 = 0;
+ __pyx_t_3 = 0;
+ __pyx_t_3 = PyObject_Call(__pyx_t_1, ((PyObject *)__pyx_t_2), NULL); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 201; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+ __Pyx_GOTREF(__pyx_t_3);
+ __Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
+ __Pyx_DECREF(((PyObject *)__pyx_t_2)); __pyx_t_2 = 0;
+ __pyx_r = __pyx_t_3;
+ __pyx_t_3 = 0;
goto __pyx_L0;
goto __pyx_L3;
}
@@ -3007,7 +3031,7 @@ static PyObject *__pyx_pf_13_hadoopy_main_11HadoopyTask_2run(struct __pyx_obj_13
__pyx_L1_error:;
__Pyx_XDECREF(__pyx_t_1);
__Pyx_XDECREF(__pyx_t_2);
- __Pyx_XDECREF(__pyx_t_4);
+ __Pyx_XDECREF(__pyx_t_3);
__Pyx_XDECREF(__pyx_t_5);
__Pyx_XDECREF(__pyx_t_6);
__Pyx_AddTraceback("_hadoopy_main.HadoopyTask.run", __pyx_clineno, __pyx_lineno, __pyx_filename);
@@ -3702,7 +3726,7 @@ static PyObject *__pyx_pf_13_hadoopy_main_11HadoopyTask_6print_out_text(struct _
__Pyx_INCREF(__pyx_v_v);
PyTuple_SET_ITEM(__pyx_t_5, 1, __pyx_v_v);
__Pyx_GIVEREF(__pyx_v_v);
- __pyx_t_7 = PyNumber_Remainder(((PyObject *)__pyx_kp_s_6), ((PyObject *)__pyx_t_5)); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 235; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+ __pyx_t_7 = PyNumber_Remainder(((PyObject *)__pyx_kp_s_3), ((PyObject *)__pyx_t_5)); if (unlikely(!__pyx_t_7)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 235; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
__Pyx_GOTREF(((PyObject *)__pyx_t_7));
__Pyx_DECREF(((PyObject *)__pyx_t_5)); __pyx_t_5 = 0;
__pyx_t_5 = PyTuple_New(2); if (unlikely(!__pyx_t_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 235; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
@@ -4263,7 +4287,7 @@ static PyObject *__pyx_f_13_hadoopy_main_11HadoopyTask_read_offset_value_text(st
if (unlikely(__pyx_skip_dispatch)) ;
/* Check if overriden in Python */
else if (unlikely(Py_TYPE(((PyObject *)__pyx_v_self))->tp_dictoffset != 0)) {
- __pyx_t_1 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_n_s_7); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 274; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+ __pyx_t_1 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_n_s_4); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 274; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
__Pyx_GOTREF(__pyx_t_1);
if (!PyCFunction_Check(__pyx_t_1) || (PyCFunction_GET_FUNCTION(__pyx_t_1) != (PyCFunction)__pyx_pw_13_hadoopy_main_11HadoopyTask_17read_offset_value_text)) {
__Pyx_XDECREF(__pyx_r);
@@ -4597,7 +4621,7 @@ static PyObject *__pyx_pf_13_hadoopy_main_11HadoopyTask_18read_in_map(struct __p
* def read_in_reduce(self, type):
*/
__Pyx_XDECREF(__pyx_r);
- __pyx_t_3 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_n_s_7); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 304; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+ __pyx_t_3 = PyObject_GetAttr(((PyObject *)__pyx_v_self), __pyx_n_s_4); if (unlikely(!__pyx_t_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 304; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
__Pyx_GOTREF(__pyx_t_3);
__pyx_t_2 = PyTuple_New(1); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 304; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
__Pyx_GOTREF(__pyx_t_2);
@@ -4826,7 +4850,7 @@ static PyObject *__pyx_pf_13_hadoopy_main_11HadoopyTask_22is_on_hadoop(CYTHON_UN
__pyx_t_2 = PyObject_GetAttr(__pyx_t_1, __pyx_n_s__environ); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 318; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
__Pyx_GOTREF(__pyx_t_2);
__Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
- __pyx_t_3 = (__Pyx_PySequence_Contains(((PyObject *)__pyx_n_s_8), __pyx_t_2, Py_EQ)); if (unlikely(__pyx_t_3 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 318; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+ __pyx_t_3 = (__Pyx_PySequence_Contains(((PyObject *)__pyx_n_s_5), __pyx_t_2, Py_EQ)); if (unlikely(__pyx_t_3 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 318; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
__Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
__pyx_t_2 = __Pyx_PyBool_FromLong(__pyx_t_3); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 318; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
__Pyx_GOTREF(__pyx_t_2);
@@ -4887,7 +4911,7 @@ static PyObject *__pyx_pf_13_hadoopy_main_11HadoopyTask_24flush_tb_writes(CYTHON
__pyx_t_2 = PyObject_GetAttr(__pyx_t_1, __pyx_n_s__environ); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 321; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
__Pyx_GOTREF(__pyx_t_2);
__Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
- __pyx_t_3 = (__Pyx_PySequence_Contains(((PyObject *)__pyx_n_s_9), __pyx_t_2, Py_EQ)); if (unlikely(__pyx_t_3 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 321; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+ __pyx_t_3 = (__Pyx_PySequence_Contains(((PyObject *)__pyx_n_s_6), __pyx_t_2, Py_EQ)); if (unlikely(__pyx_t_3 < 0)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 321; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
__Pyx_DECREF(__pyx_t_2); __pyx_t_2 = 0;
__pyx_t_2 = __Pyx_PyBool_FromLong(__pyx_t_3); if (unlikely(!__pyx_t_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 321; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
__Pyx_GOTREF(__pyx_t_2);
@@ -5799,13 +5823,13 @@ static struct PyModuleDef __pyx_moduledef = {
#endif
static __Pyx_StringTabEntry __pyx_string_tab[] = {
- {&__pyx_kp_s_10, __pyx_k_10, sizeof(__pyx_k_10), 0, 0, 1, 0},
- {&__pyx_kp_s_11, __pyx_k_11, sizeof(__pyx_k_11), 0, 0, 1, 0},
- {&__pyx_n_s_12, __pyx_k_12, sizeof(__pyx_k_12), 0, 0, 1, 1},
+ {&__pyx_n_s_2, __pyx_k_2, sizeof(__pyx_k_2), 0, 0, 1, 1},
+ {&__pyx_kp_s_3, __pyx_k_3, sizeof(__pyx_k_3), 0, 0, 1, 0},
{&__pyx_n_s_4, __pyx_k_4, sizeof(__pyx_k_4), 0, 0, 1, 1},
- {&__pyx_kp_s_6, __pyx_k_6, sizeof(__pyx_k_6), 0, 0, 1, 0},
- {&__pyx_n_s_7, __pyx_k_7, sizeof(__pyx_k_7), 0, 0, 1, 1},
- {&__pyx_n_s_8, __pyx_k_8, sizeof(__pyx_k_8), 0, 0, 1, 1},
+ {&__pyx_n_s_5, __pyx_k_5, sizeof(__pyx_k_5), 0, 0, 1, 1},
+ {&__pyx_n_s_6, __pyx_k_6, sizeof(__pyx_k_6), 0, 0, 1, 1},
+ {&__pyx_kp_s_7, __pyx_k_7, sizeof(__pyx_k_7), 0, 0, 1, 0},
+ {&__pyx_kp_s_8, __pyx_k_8, sizeof(__pyx_k_8), 0, 0, 1, 0},
{&__pyx_n_s_9, __pyx_k_9, sizeof(__pyx_k_9), 0, 0, 1, 1},
{&__pyx_n_s__AttributeError, __pyx_k__AttributeError, sizeof(__pyx_k__AttributeError), 0, 0, 1, 1},
{&__pyx_n_s__RawBytesFile, __pyx_k__RawBytesFile, sizeof(__pyx_k__RawBytesFile), 0, 0, 1, 1},
@@ -5881,8 +5905,8 @@ static int __Pyx_InitCachedConstants(void) {
* def run(self):
* env = os.environ
* minput = env.get('stream_map_input', 'text') # <<<<<<<<<<<<<<
- * moutput = env.get('stream_map_output', 'text')
- * rinput = env.get('stream_reduce_input', 'text')
+ * moutput = env.get('stream_map_output', minput)
+ * rinput = env.get('stream_reduce_input', minput)
*/
__pyx_k_tuple_1 = PyTuple_New(2); if (unlikely(!__pyx_k_tuple_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 191; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
__Pyx_GOTREF(__pyx_k_tuple_1);
@@ -5893,57 +5917,6 @@ static int __Pyx_InitCachedConstants(void) {
PyTuple_SET_ITEM(__pyx_k_tuple_1, 1, ((PyObject *)__pyx_n_s__text));
__Pyx_GIVEREF(((PyObject *)__pyx_n_s__text));
__Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_1));
-
- /* "hadoopy/_main.pyx":192
- * env = os.environ
- * minput = env.get('stream_map_input', 'text')
- * moutput = env.get('stream_map_output', 'text') # <<<<<<<<<<<<<<
- * rinput = env.get('stream_reduce_input', 'text')
- * routput = env.get('stream_reduce_output', 'text')
- */
- __pyx_k_tuple_2 = PyTuple_New(2); if (unlikely(!__pyx_k_tuple_2)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 192; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_k_tuple_2);
- __Pyx_INCREF(((PyObject *)__pyx_n_s__stream_map_output));
- PyTuple_SET_ITEM(__pyx_k_tuple_2, 0, ((PyObject *)__pyx_n_s__stream_map_output));
- __Pyx_GIVEREF(((PyObject *)__pyx_n_s__stream_map_output));
- __Pyx_INCREF(((PyObject *)__pyx_n_s__text));
- PyTuple_SET_ITEM(__pyx_k_tuple_2, 1, ((PyObject *)__pyx_n_s__text));
- __Pyx_GIVEREF(((PyObject *)__pyx_n_s__text));
- __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_2));
-
- /* "hadoopy/_main.pyx":193
- * minput = env.get('stream_map_input', 'text')
- * moutput = env.get('stream_map_output', 'text')
- * rinput = env.get('stream_reduce_input', 'text') # <<<<<<<<<<<<<<
- * routput = env.get('stream_reduce_output', 'text')
- *
- */
- __pyx_k_tuple_3 = PyTuple_New(2); if (unlikely(!__pyx_k_tuple_3)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 193; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_k_tuple_3);
- __Pyx_INCREF(((PyObject *)__pyx_n_s__stream_reduce_input));
- PyTuple_SET_ITEM(__pyx_k_tuple_3, 0, ((PyObject *)__pyx_n_s__stream_reduce_input));
- __Pyx_GIVEREF(((PyObject *)__pyx_n_s__stream_reduce_input));
- __Pyx_INCREF(((PyObject *)__pyx_n_s__text));
- PyTuple_SET_ITEM(__pyx_k_tuple_3, 1, ((PyObject *)__pyx_n_s__text));
- __Pyx_GIVEREF(((PyObject *)__pyx_n_s__text));
- __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_3));
-
- /* "hadoopy/_main.pyx":194
- * moutput = env.get('stream_map_output', 'text')
- * rinput = env.get('stream_reduce_input', 'text')
- * routput = env.get('stream_reduce_output', 'text') # <<<<<<<<<<<<<<
- *
- * if self.task_type == 'map':
- */
- __pyx_k_tuple_5 = PyTuple_New(2); if (unlikely(!__pyx_k_tuple_5)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 194; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
- __Pyx_GOTREF(__pyx_k_tuple_5);
- __Pyx_INCREF(((PyObject *)__pyx_n_s_4));
- PyTuple_SET_ITEM(__pyx_k_tuple_5, 0, ((PyObject *)__pyx_n_s_4));
- __Pyx_GIVEREF(((PyObject *)__pyx_n_s_4));
- __Pyx_INCREF(((PyObject *)__pyx_n_s__text));
- PyTuple_SET_ITEM(__pyx_k_tuple_5, 1, ((PyObject *)__pyx_n_s__text));
- __Pyx_GIVEREF(((PyObject *)__pyx_n_s__text));
- __Pyx_GIVEREF(((PyObject *)__pyx_k_tuple_5));
__Pyx_RefNannyFinishContext();
return 0;
__pyx_L1_error:;
@@ -6076,7 +6049,7 @@ PyMODINIT_FUNC PyInit__hadoopy_main(void)
* __license__ = 'GPL V3'
*
*/
- if (PyObject_SetAttr(__pyx_m, __pyx_n_s____author__, ((PyObject *)__pyx_kp_s_10)) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 17; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+ if (PyObject_SetAttr(__pyx_m, __pyx_n_s____author__, ((PyObject *)__pyx_kp_s_7)) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 17; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
/* "hadoopy/_main.pyx":18
*
@@ -6085,7 +6058,7 @@ PyMODINIT_FUNC PyInit__hadoopy_main(void)
*
* import sys
*/
- if (PyObject_SetAttr(__pyx_m, __pyx_n_s____license__, ((PyObject *)__pyx_kp_s_11)) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 18; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+ if (PyObject_SetAttr(__pyx_m, __pyx_n_s____license__, ((PyObject *)__pyx_kp_s_8)) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 18; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
/* "hadoopy/_main.pyx":20
* __license__ = 'GPL V3'
@@ -6130,7 +6103,7 @@ PyMODINIT_FUNC PyInit__hadoopy_main(void)
*
*
*/
- __pyx_t_1 = __Pyx_Import(((PyObject *)__pyx_n_s_12), 0, -1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 23; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
+ __pyx_t_1 = __Pyx_Import(((PyObject *)__pyx_n_s_9), 0, -1); if (unlikely(!__pyx_t_1)) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 23; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
__Pyx_GOTREF(__pyx_t_1);
if (PyObject_SetAttr(__pyx_m, __pyx_n_s__hadoopy, __pyx_t_1) < 0) {__pyx_filename = __pyx_f[0]; __pyx_lineno = 23; __pyx_clineno = __LINE__; goto __pyx_L1_error;}
__Pyx_DECREF(__pyx_t_1); __pyx_t_1 = 0;
View
6 hadoopy/_main.pyx
@@ -189,9 +189,9 @@ cdef class HadoopyTask(object):
def run(self):
env = os.environ
minput = env.get('stream_map_input', 'text')
- moutput = env.get('stream_map_output', 'text')
- rinput = env.get('stream_reduce_input', 'text')
- routput = env.get('stream_reduce_output', 'text')
+ moutput = env.get('stream_map_output', minput)
+ rinput = env.get('stream_reduce_input', minput)
+ routput = env.get('stream_reduce_output', minput)
if self.task_type == 'map':
return self.process_inout(self.mapper, self.read_in_map(minput), self.print_out(moutput), 'map')

0 comments on commit 8d414fe

Please sign in to comment.
Something went wrong with that request. Please try again.