Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

fix NPE and wordcount results

  • Loading branch information...
commit 89a1c4e715b21baf2131544c032950b02d7ae93f 1 parent 1aefc32
Jake Luciani authored
0  examples/hadoop_word_count/bin/word_count_counters 100644 → 100755
File mode changed
16 examples/hadoop_word_count/src/WordCount.java
@@ -20,7 +20,6 @@
20 20 import java.nio.ByteBuffer;
21 21 import java.util.*;
22 22
23   -import org.apache.cassandra.cql3.CFDefinition;
24 23 import org.apache.cassandra.thrift.*;
25 24 import org.apache.cassandra.hadoop.ColumnFamilyOutputFormat;
26 25 import org.slf4j.Logger;
@@ -80,15 +79,21 @@ public static void main(String[] args) throws Exception
80 79 protected void setup(org.apache.hadoop.mapreduce.Mapper.Context context)
81 80 throws IOException, InterruptedException
82 81 {
83   - sourceColumn = ByteBufferUtil.bytes(context.getConfiguration().get(CONF_COLUMN_NAME));
84 82 }
85 83
86 84 public void map(ByteBuffer key, SortedMap<ByteBuffer, IColumn> columns, Context context) throws IOException, InterruptedException
87 85 {
88 86 for (IColumn column : columns.values())
89 87 {
90   - String value = ByteBufferUtil.string(column.value());
91   - logger.debug("read " + key + ":" + value + " from " + context.getInputSplit());
  88 + String name = ByteBufferUtil.string(column.name());
  89 + String value = null;
  90 +
  91 + if (name.contains("int"))
  92 + value = String.valueOf(ByteBufferUtil.toInt(column.value()));
  93 + else
  94 + value = ByteBufferUtil.string(column.value());
  95 +
  96 + System.err.println("read " + ByteBufferUtil.string(key) + ":" +name + ":" + value + " from " + context.getInputSplit());
92 97
93 98 StringTokenizer itr = new StringTokenizer(value);
94 99 while (itr.hasMoreTokens())
@@ -185,6 +190,7 @@ public int run(String[] args) throws Exception
185 190 job.setOutputFormatClass(ColumnFamilyOutputFormat.class);
186 191
187 192 ConfigHelper.setOutputColumnFamily(job.getConfiguration(), KEYSPACE, OUTPUT_COLUMN_FAMILY);
  193 + job.getConfiguration().set(CONF_COLUMN_NAME, "sum");
188 194 }
189 195
190 196 job.setInputFormatClass(ColumnFamilyInputFormat.class);
@@ -205,7 +211,7 @@ public int run(String[] args) throws Exception
205 211 if (i == 5)
206 212 {
207 213 // this will cause the predicate to be ignored in favor of scanning everything as a wide row
208   - ConfigHelper.setInputColumnFamily(job.getConfiguration(), KEYSPACE, COLUMN_FAMILY, CFDefinition.Kind.DYNAMIC);
  214 + ConfigHelper.setInputColumnFamily(job.getConfiguration(), KEYSPACE, COLUMN_FAMILY, true);
209 215 }
210 216
211 217 ConfigHelper.setOutputInitialAddress(job.getConfiguration(), "localhost");
10 src/java/org/apache/cassandra/hadoop/ColumnFamilyRecordReader.java
@@ -453,9 +453,15 @@ private void maybeInit()
453 453 {
454 454 rows = client.get_paged_slice(cfName, keyRange, startColumn, consistencyLevel);
455 455
  456 + // nothing found?
  457 + if (rows == null || rows.isEmpty() || rows.get(0).columns.isEmpty())
  458 + {
  459 + rows = null;
  460 + return;
  461 + }
  462 +
456 463 // nothing new? reached the end
457   - if (rows.get(0).columns.isEmpty()
458   - || (rows.get(0).key.equals(lastRow.key) && rows.get(0).columns.get(0).column.equals(startColumn)))
  464 + if (lastRow != null && (rows.get(0).key.equals(lastRow.key) || rows.get(0).columns.get(0).column.equals(startColumn)))
459 465 {
460 466 rows = null;
461 467 return;

0 comments on commit 89a1c4e

Please sign in to comment.
Something went wrong with that request. Please try again.