Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Remove old twittertopiccount example

  • Loading branch information...
commit 02dca79e607f3b2bfe20b7c9c40330d9f457fb56 1 parent 7aaba95
Bruce Robbins authored
View
2  README.md
@@ -63,7 +63,7 @@ Running the Twitter Topic Count Example
git clone https://github.com/s4/s4.git
# Create image
-gradlew allImage
+./gradlew allImage
# set the S4_IMAGE environmental variable
cd build/s4-image/
View
22 s4-examples/twittertopiccount/build.gradle
@@ -1,22 +0,0 @@
-/*
- * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
- * either express or implied. See the License for the specific
- * language governing permissions and limitations under the
- * License. See accompanying LICENSE file.
- */
-dependencies {
- compile( libraries.json )
- compile( libraries.commons_codec )
- compile( libraries.commons_httpclient )
- compile project(':s4-core')
-}
-
View
118 s4-examples/twittertopiccount/src/main/java/io/s4/example/twittertopiccount/DirectToFilePersister.java
@@ -1,118 +0,0 @@
-/*
- * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
- * either express or implied. See the License for the specific
- * language governing permissions and limitations under the
- * License. See accompanying LICENSE file.
- */
-package io.s4.example.twittertopiccount;
-
-import java.io.FileWriter;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.log4j.Logger;
-
-import io.s4.persist.Persister;
-
-public class DirectToFilePersister implements Persister {
- private String outputFilename;
- private int persistCount;
-
- public void setOutputFilename(String outputFilename) {
- this.outputFilename = outputFilename;
- }
-
- @Override
- public int cleanOutGarbage() throws InterruptedException {
- return 0;
- }
-
- @Override
- public Object get(String arg0) throws InterruptedException {
- return null;
- }
-
- @Override
- public Map<String, Object> getBulk(String[] arg0)
- throws InterruptedException {
- return new HashMap<String, Object>();
- }
-
- @Override
- public Map<String, Object> getBulkObjects(String[] arg0)
- throws InterruptedException {
- return new HashMap<String, Object>();
- }
-
- @Override
- public int getCacheEntryCount() {
- return 1;
- }
-
- @Override
- public Object getObject(String arg0) throws InterruptedException {
- return null;
- }
-
- @Override
- public int getPersistCount() {
- return persistCount;
- }
-
- @Override
- public int getQueueSize() {
- return 0;
- }
-
- @Override
- public Set<String> keySet() {
- return new HashSet<String>();
- }
-
- @Override
- public void remove(String arg0) throws InterruptedException {
-
- }
-
- @Override
- public void set(String key, Object value, int persistTime)
- throws InterruptedException {
-
- FileWriter fw = null;
- try {
- fw = new FileWriter(outputFilename);
- fw.write(String.valueOf(value));
- } catch (IOException e) {
- // TODO Auto-generated catch block
- Logger.getLogger("s4").error(e);
- } finally {
- if (fw != null) {
- try {
- fw.close();
- } catch (Exception e) {
- }
- }
- }
- }
-
- @Override
- public void setAsynch(String key, Object value, int persistTime) {
- try {
- set(key, value, persistTime);
- } catch (InterruptedException ie) {
- }
- }
-
-}
View
154 s4-examples/twittertopiccount/src/main/java/io/s4/example/twittertopiccount/Status.java
@@ -1,154 +0,0 @@
-/*
- * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
- * either express or implied. See the License for the specific
- * language governing permissions and limitations under the
- * License. See accompanying LICENSE file.
- */
-package io.s4.example.twittertopiccount;
-
-public class Status {
- private long id;
- private long inReplyToStatusId;
- private String text;
- private boolean truncated;
- private String source;
- private String inReplyToScreenName;
- private boolean favorited;
- private User user;
- private long inReplyToUserId;
- private String createdAt;
-
- public long getId() {
- return id;
- }
-
- public void setId(long id) {
- this.id = id;
- }
-
- public long getInReplyToStatusId() {
- return inReplyToStatusId;
- }
-
- public void setInReplyToStatusId(long inReplyToStatusId) {
- this.inReplyToStatusId = inReplyToStatusId;
- }
-
- public String getText() {
- return text;
- }
-
- public void setText(String text) {
- this.text = text;
- }
-
- public boolean isTruncated() {
- return truncated;
- }
-
- public void setTruncated(boolean truncated) {
- this.truncated = truncated;
- }
-
- public String getSource() {
- return source;
- }
-
- public void setSource(String source) {
- this.source = source;
- }
-
- public String getInReplyToScreenName() {
- return inReplyToScreenName;
- }
-
- public void setInReplyToScreenName(String inReplyToScreenName) {
- this.inReplyToScreenName = inReplyToScreenName;
- }
-
- public boolean isFavorited() {
- return favorited;
- }
-
- public void setFavorited(boolean favorited) {
- this.favorited = favorited;
- }
-
- public User getUser() {
- return user;
- }
-
- public void setUser(User user) {
- this.user = user;
- }
-
- public long getInReplyToUserId() {
- return inReplyToUserId;
- }
-
- public void setInReplyToUserId(long inReplyToUserId) {
- this.inReplyToUserId = inReplyToUserId;
- }
-
- public String getCreatedAt() {
- return createdAt;
- }
-
- public void setCreatedAt(String createdAt) {
- this.createdAt = createdAt;
- }
-
- public String toString() {
- StringBuffer sb = new StringBuffer();
- sb.append("{")
- .append("id:")
- .append(id)
- .append(",")
- .append("inReplyToStatusId:")
- .append(inReplyToStatusId)
- .append(",")
- .append("text:")
- .append(text)
- .append(",")
- .append("truncated:")
- .append(truncated)
- .append(",")
- .append("source:")
- .append(source)
- .append(",")
- .append("inReplyToScreenName:")
- .append(inReplyToScreenName)
- .append(",")
- .append("favorited:")
- .append(favorited)
- .append(",")
- .append("user:")
- .append(user)
- .append(",")
- .append("inReplyToUserId:")
- .append(inReplyToUserId)
- .append(",")
- .append("createdAt:")
- .append(createdAt)
- .append("}");
-
- return sb.toString();
- }
-
- public Object clone() {
- try {
- return super.clone();
- } catch (CloneNotSupportedException cnse) {
- throw new RuntimeException(cnse);
- }
- }
-}
View
178 s4-examples/twittertopiccount/src/main/java/io/s4/example/twittertopiccount/TopNTopicPE.java
@@ -1,178 +0,0 @@
-/*
- * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
- * either express or implied. See the License for the specific
- * language governing permissions and limitations under the
- * License. See accompanying LICENSE file.
- */
-package io.s4.example.twittertopiccount;
-
-import io.s4.persist.Persister;
-import io.s4.processor.AbstractPE;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-import org.apache.log4j.Logger;
-import org.json.JSONArray;
-import org.json.JSONObject;
-
-import com.google.gson.Gson;
-
-public class TopNTopicPE extends AbstractPE {
- private String id;
- private Persister persister;
- private int entryCount = 10;
- private Map<String, Integer> topicMap = new ConcurrentHashMap<String, Integer>();
- private int persistTime;
- private String persistKey = "myapp:topNTopics";
-
- public void setId(String id) {
- this.id = id;
- }
-
- public Persister getPersister() {
- return persister;
- }
-
- public void setPersister(Persister persister) {
- this.persister = persister;
- }
-
- public int getEntryCount() {
- return entryCount;
- }
-
- public void setEntryCount(int entryCount) {
- this.entryCount = entryCount;
- }
-
- public int getPersistTime() {
- return persistTime;
- }
-
- public void setPersistTime(int persistTime) {
- this.persistTime = persistTime;
- }
-
- public String getPersistKey() {
- return persistKey;
- }
-
- public void setPersistKey(String persistKey) {
- this.persistKey = persistKey;
- }
-
- public void processEvent(TopicSeen topicSeen) {
- topicMap.put(topicSeen.getTopic(), topicSeen.getCount());
- }
-
- public ArrayList<TopNEntry> getTopTopics() {
- if (entryCount < 1)
- return null;
-
- ArrayList<TopNEntry> sortedList = new ArrayList<TopNEntry>();
-
- for (String key : topicMap.keySet()) {
- sortedList.add(new TopNEntry(key, topicMap.get(key)));
- }
-
- Collections.sort(sortedList);
-
- // truncate: Yuck!!
- // unfortunately, Kryo cannot deserialize RandomAccessSubList
- // if we use ArrayList.subList(...)
- while (sortedList.size() > entryCount)
- sortedList.remove(sortedList.size() - 1);
-
- return sortedList;
- }
-
- @Override
- public void output() {
- List<TopNEntry> sortedList = new ArrayList<TopNEntry>();
-
- for (String key : topicMap.keySet()) {
- sortedList.add(new TopNEntry(key, topicMap.get(key)));
- }
-
- Collections.sort(sortedList);
-
- try {
- JSONObject message = new JSONObject();
- JSONArray jsonTopN = new JSONArray();
-
- for (int i = 0; i < entryCount; i++) {
- if (i == sortedList.size()) {
- break;
- }
- TopNEntry tne = sortedList.get(i);
- JSONObject jsonEntry = new JSONObject();
- jsonEntry.put("topic", tne.getTopic());
- jsonEntry.put("count", tne.getCount());
- jsonTopN.put(jsonEntry);
- }
- message.put("topN", jsonTopN);
- persister.set(persistKey, message.toString()+"\n", persistTime);
- } catch (Exception e) {
- Logger.getLogger("s4").error(e);
- }
- }
-
- @Override
- public String getId() {
- return this.id;
- }
-
- public static class TopNEntry implements Comparable<TopNEntry> {
- public TopNEntry(String topic, int count) {
- this.topic = topic;
- this.count = count;
- }
-
- public TopNEntry() {}
-
- String topic = null;
- int count = 0;
-
- public String getTopic() {
- return topic;
- }
-
- public void setTopic(String topic) {
- this.topic = topic;
- }
-
- public int getCount() {
- return count;
- }
-
- public void setCount(int count) {
- this.count = count;
- }
-
- public int compareTo(TopNEntry topNEntry) {
- if (topNEntry.getCount() < this.count) {
- return -1;
- } else if (topNEntry.getCount() > this.count) {
- return 1;
- }
- return 0;
- }
-
- public String toString() {
- return "topic:" + topic + " count:" + count;
- }
- }
-}
View
76 s4-examples/twittertopiccount/src/main/java/io/s4/example/twittertopiccount/TopicCountAndReportPE.java
@@ -1,76 +0,0 @@
-/*
- * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
- * either express or implied. See the License for the specific
- * language governing permissions and limitations under the
- * License. See accompanying LICENSE file.
- */
-package io.s4.example.twittertopiccount;
-
-import io.s4.dispatcher.EventDispatcher;
-import io.s4.processor.AbstractPE;
-
-public class TopicCountAndReportPE extends AbstractPE {
- private String id;
- private EventDispatcher dispatcher;
- private String outputStreamName;
- private int threshold;
- private int count;
-
- public void setId(String id) {
- this.id = id;
- }
-
- public EventDispatcher getDispatcher() {
- return dispatcher;
- }
-
- public void setDispatcher(EventDispatcher dispatcher) {
- this.dispatcher = dispatcher;
- }
-
- public String getOutputStreamName() {
- return outputStreamName;
- }
-
- public void setOutputStreamName(String outputStreamName) {
- this.outputStreamName = outputStreamName;
- }
-
- public int getThreshold() {
- return threshold;
- }
-
- public void setThreshold(int threshold) {
- this.threshold = threshold;
- }
-
- public void processEvent(TopicSeen topicSeen) {
- count += topicSeen.getCount();
- }
-
- @Override
- public void output() {
- if (count < threshold) {
- return;
- }
- TopicSeen topicSeen = new TopicSeen((String) this.getKeyValue().get(0),
- count);
- topicSeen.setReportKey("1");
- dispatcher.dispatchEvent(outputStreamName, topicSeen);
- }
-
- @Override
- public String getId() {
- return this.id;
- }
-
-}
View
114 s4-examples/twittertopiccount/src/main/java/io/s4/example/twittertopiccount/TopicExtractorPE.java
@@ -1,114 +0,0 @@
-/*
- * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
- * either express or implied. See the License for the specific
- * language governing permissions and limitations under the
- * License. See accompanying LICENSE file.
- */
-package io.s4.example.twittertopiccount;
-
-import io.s4.dispatcher.Dispatcher;
-import io.s4.dispatcher.EventDispatcher;
-import io.s4.processor.AbstractPE;
-
-public class TopicExtractorPE extends AbstractPE {
- private String id;
- private EventDispatcher dispatcher;
- private String outputStreamName;
-
- public EventDispatcher getDispatcher() {
- return dispatcher;
- }
-
- public void setDispatcher(EventDispatcher dispatcher) {
- this.dispatcher = dispatcher;
- }
-
- public String getOutputStreamName() {
- return outputStreamName;
- }
-
- public void setOutputStreamName(String outputStreamName) {
- this.outputStreamName = outputStreamName;
- }
-
- public void setId(String id) {
- this.id = id;
- }
-
- public void processEvent(Status status) {
- String text = status.getText();
- if (text == null) {
- return;
- }
-
- int textLength = text.length();
- int index = 0;
- int prevIndex = 0;
- while ((index = text.indexOf('#', prevIndex)) != -1) {
- prevIndex = index + 1;
- if (prevIndex == textLength) { // if hash is the last character
- break; // get out
- }
- StringBuffer sb = new StringBuffer();
- for (int i = index + 1; i < textLength; i++) {
- char ch = text.charAt(i);
- if (!Character.isLetterOrDigit(ch)) {
- break;
- }
- sb.append(ch);
- }
-
- if (sb.length() == 0) {
- continue;
- }
-
- TopicSeen topicSeen = new TopicSeen(sb.toString().toLowerCase(), 1);
- dispatcher.dispatchEvent(outputStreamName, topicSeen);
- }
- }
-
- @Override
- public void output() {
- // TODO Auto-generated method stub
-
- }
-
- @Override
- public String getId() {
- return this.id;
- }
-
- static class DummyDispatcher extends Dispatcher {
- public void dispatchEvent(String streamName, Object event) {
- System.out.println(event);
- }
- }
-
- public static void main(String args[]) {
- TopicExtractorPE te = new TopicExtractorPE();
- te.setDispatcher(new DummyDispatcher());
- te.setOutputStreamName("test");
-
- Status status = new Status();
- status.setText("Hey this is a test");
- te.processEvent(status);
-
- status.setText("This is an edge test #");
- te.processEvent(status);
-
- status.setText("#GLOB this is a test");
- te.processEvent(status);
-
- status.setText("Hey there #FLOB, this is a test #GLOB");
- te.processEvent(status);
- }
-}
View
67 s4-examples/twittertopiccount/src/main/java/io/s4/example/twittertopiccount/TopicSeen.java
@@ -1,67 +0,0 @@
-/*
- * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
- * either express or implied. See the License for the specific
- * language governing permissions and limitations under the
- * License. See accompanying LICENSE file.
- */
-package io.s4.example.twittertopiccount;
-
-public class TopicSeen {
- private String topic;
- private int count;
- private String reportKey;
-
- public TopicSeen() {
-
- }
-
- public TopicSeen(String topic, int count) {
- this.topic = topic;
- this.count = count;
- }
-
- public String getTopic() {
- return topic;
- }
-
- public void setTopic(String topic) {
- this.topic = topic;
- }
-
- public int getCount() {
- return count;
- }
-
- public void setCount(int count) {
- this.count = count;
- }
-
- public String getReportKey() {
- return reportKey;
- }
-
- public void setReportKey(String reportKey) {
- this.reportKey = reportKey;
- }
-
- public String toString() {
- return "{topic:" + topic + "}";
- }
-
- public Object clone() {
- try {
- return super.clone();
- } catch (CloneNotSupportedException cnse) {
- throw new RuntimeException(cnse);
- }
- }
-}
View
355 s4-examples/twittertopiccount/src/main/java/io/s4/example/twittertopiccount/TwitterFeedListener.java
@@ -1,355 +0,0 @@
-/*
- * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
- * either express or implied. See the License for the specific
- * language governing permissions and limitations under the
- * License. See accompanying LICENSE file.
- */
-package io.s4.example.twittertopiccount;
-
-import java.io.BufferedReader;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.net.URL;
-import java.net.URLConnection;
-import java.util.HashSet;
-import java.util.Set;
-import java.util.concurrent.LinkedBlockingQueue;
-
-import org.apache.log4j.Logger;
-import org.json.JSONObject;
-import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.httpclient.util.EncodingUtil;
-
-import io.s4.collector.EventWrapper;
-import io.s4.listener.EventHandler;
-import io.s4.listener.EventProducer;
-
-public class TwitterFeedListener implements EventProducer, Runnable {
- private String userid;
- private String password;
- private String urlString;
- private long maxBackoffTime = 30 * 1000; // 5 seconds
- private long messageCount = 0;
- private long blankCount = 0;
- private String streamName;
-
- private LinkedBlockingQueue<String> messageQueue = new LinkedBlockingQueue<String>();
- private Set<io.s4.listener.EventHandler> handlers = new HashSet<io.s4.listener.EventHandler>();
-
- public void setUserid(String userid) {
- this.userid = userid;
- }
-
- public void setPassword(String password) {
- this.password = password;
- }
-
- public void setUrlString(String urlString) {
- this.urlString = urlString;
- }
-
- public void setMaxBackoffTime(long maxBackoffTime) {
- this.maxBackoffTime = maxBackoffTime;
- }
-
- public void setStreamName(String streamName) {
- this.streamName = streamName;
- }
-
- public void init() {
- for (int i = 0; i < 12; i++) {
- Dequeuer dequeuer = new Dequeuer(i);
- Thread t = new Thread(dequeuer);
- t.start();
- }
- (new Thread(this)).start();
- }
-
- public void run() {
- long backoffTime = 1000;
- while (!Thread.interrupted()) {
- try {
- connectAndRead();
- } catch (Exception e) {
- Logger.getLogger("s4").error("Exception reading feed", e);
- try {
- Thread.sleep(backoffTime);
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- }
- backoffTime = backoffTime * 2;
- if (backoffTime > maxBackoffTime) {
- backoffTime = maxBackoffTime;
- }
- }
- }
- }
-
- public void connectAndRead() throws Exception {
- URL url = new URL(urlString);
-
- URLConnection connection = url.openConnection();
- String userPassword = userid + ":" + password;
- String encoded = EncodingUtil.getAsciiString(Base64.encodeBase64(EncodingUtil.getAsciiBytes(userPassword)));
- connection.setRequestProperty("Authorization", "Basic " + encoded);
- connection.connect();
-
- InputStream is = connection.getInputStream();
- InputStreamReader isr = new InputStreamReader(is);
- BufferedReader br = new BufferedReader(isr);
-
- String inputLine = null;
- while ((inputLine = br.readLine()) != null) {
- if (inputLine.trim().length() == 0) {
- blankCount++;
- continue;
- }
- messageCount++;
- messageQueue.add(inputLine);
- }
- }
-
- class Dequeuer implements Runnable {
- private int id;
-
- public Dequeuer(int id) {
- this.id = id;
- }
-
- public void run() {
- while (!Thread.interrupted()) {
- try {
- String message = messageQueue.take();
- JSONObject jsonObject = new JSONObject(message);
-
- // ignore delete records for now
- if (jsonObject.has("delete")) {
- continue;
- }
-
- Status status = getStatus(jsonObject);
-
- EventWrapper ew = new EventWrapper(streamName, status, null);
- for (io.s4.listener.EventHandler handler : handlers) {
- try {
- handler.processEvent(ew);
- } catch (Exception e) {
- Logger.getLogger("s4")
- .error("Exception in raw event handler", e);
- }
- }
- } catch (InterruptedException ie) {
- Thread.currentThread().interrupt();
- } catch (Exception e) {
- Logger.getLogger("s4")
- .error("Exception processing message", e);
- }
- }
- }
-
- public Status getStatus(JSONObject jsonObject) {
- try {
- if (jsonObject == null || jsonObject.equals(JSONObject.NULL)) {
- return null;
- }
-
- Status status = new Status();
-
- status.setUser(getUser((JSONObject) jsonObject.opt("user")));
-
- Object value = jsonObject.opt("id");
- if (value != null && !value.equals(JSONObject.NULL)) {
- status.setId(((Number) value).longValue());
- }
-
- value = jsonObject.opt("in_reply_to_status_id");
- if (value != null && !value.equals(JSONObject.NULL)) {
- status.setInReplyToStatusId(((Number) value).longValue());
- }
-
- value = jsonObject.opt("text");
- if (value != null && !value.equals(JSONObject.NULL)) {
- status.setText((String) value);
- }
-
- value = jsonObject.opt("truncated");
- if (value != null && !value.equals(JSONObject.NULL)) {
- status.setTruncated((Boolean) value);
- }
-
- value = jsonObject.opt("source");
- if (value != null && !value.equals(JSONObject.NULL)) {
- status.setSource((String) value);
- }
-
- value = jsonObject.opt("in_reply_to_screen_name");
- if (value != null && !value.equals(JSONObject.NULL)) {
- status.setInReplyToScreenName((String) value);
- }
-
- value = jsonObject.opt("favorited");
- if (value != null && !value.equals(JSONObject.NULL)) {
- status.setFavorited((Boolean) value);
- }
-
- value = jsonObject.opt("in_reply_to_user_id");
- if (value != null && !value.equals(JSONObject.NULL)) {
- status.setInReplyToUserId(((Number) value).longValue());
- }
-
- value = jsonObject.opt("created_at");
- if (value != null && !value.equals(JSONObject.NULL)) {
- status.setCreatedAt((String) value);
- }
-
- return status;
- } catch (Exception e) {
- Logger.getLogger("s4").error(e);
- }
-
- return null;
- }
-
- public User getUser(JSONObject jsonObject) {
- try {
- if (jsonObject == null || jsonObject.equals(JSONObject.NULL)) {
- return null;
- }
-
- User user = new User();
-
- Object value = jsonObject.opt("id");
- if (value != null && !value.equals(JSONObject.NULL)) {
- user.setId(((Number) value).longValue());
- }
-
- value = jsonObject.opt("screen_name");
- if (value != null && !value.equals(JSONObject.NULL)) {
- user.setScreenName((String) value);
- }
-
- value = jsonObject.opt("name");
- if (value != null && !value.equals(JSONObject.NULL)) {
- user.setName((String) value);
- }
-
- value = jsonObject.opt("url");
- if (value != null && !value.equals(JSONObject.NULL)) {
- user.setUrl((String) value);
- }
-
- value = jsonObject.opt("followers_count");
- if (value != null && !value.equals(JSONObject.NULL)) {
- user.setFollowersCount(((Number) value).intValue());
- }
-
- value = jsonObject.opt("lang");
- if (value != null && !value.equals(JSONObject.NULL)) {
- user.setLang((String) value);
- }
-
- value = jsonObject.opt("verified");
- if (value != null && !value.equals(JSONObject.NULL)) {
- user.setVerified((Boolean) value);
- }
-
- value = jsonObject.opt("profile_image_url");
- if (value != null && !value.equals(JSONObject.NULL)) {
- user.setProfileImageUrl((String) value);
- }
-
- value = jsonObject.opt("friends_count");
- if (value != null && !value.equals(JSONObject.NULL)) {
- user.setFriendsCount(((Number) value).intValue());
- }
-
- value = jsonObject.opt("description");
- if (value != null && !value.equals(JSONObject.NULL)) {
- user.setDescription((String) value);
- }
-
- value = jsonObject.opt("favourites_Count");
- if (value != null && !value.equals(JSONObject.NULL)) {
- user.setFavouritesCount(((Number) value).intValue());
- }
-
- value = jsonObject.opt("geo_enabled");
- if (value != null && !value.equals(JSONObject.NULL)) {
- user.setGeoEnabled((Boolean) value);
- }
-
- value = jsonObject.opt("listed_count");
- if (value != null && !value.equals(JSONObject.NULL)) {
- user.setListedCount(((Number) value).intValue());
- }
-
- value = jsonObject.opt("profile_background_image_url");
- if (value != null && !value.equals(JSONObject.NULL)) {
- user.setProfileBackgroundImageUrl((String) value);
- }
-
- value = jsonObject.opt("protected_user");
- if (value != null && !value.equals(JSONObject.NULL)) {
- user.setProtectedUser((Boolean) value);
- }
-
- value = jsonObject.opt("location");
- if (value != null && !value.equals(JSONObject.NULL)) {
- user.setLocation((String) value);
- }
-
- value = jsonObject.opt("statuses_count");
- if (value != null && !value.equals(JSONObject.NULL)) {
- user.setStatusesCount(((Number) value).longValue());
- }
-
- value = jsonObject.opt("time_zone");
- if (value != null && !value.equals(JSONObject.NULL)) {
- user.setTimeZone((String) value);
- }
-
- value = jsonObject.opt("contributors_enabled");
- if (value != null && !value.equals(JSONObject.NULL)) {
- user.setContributorsEnabled((Boolean) value);
- }
-
- value = jsonObject.opt("utc_offset");
- if (value != null && !value.equals(JSONObject.NULL)) {
- user.setUtcOffset(((Number) value).intValue());
- }
-
- value = jsonObject.opt("created_at");
- if (value != null && !value.equals(JSONObject.NULL)) {
- user.setCreatedAt((String) value);
- }
-
- return user;
- } catch (Exception e) {
- Logger.getLogger("s4").error(e);
- }
-
- return null;
- }
- }
-
- @Override
- public void addHandler(EventHandler handler) {
- handlers.add(handler);
-
- }
-
- @Override
- public boolean removeHandler(EventHandler handler) {
- return handlers.remove(handler);
- }
-
-}
View
280 s4-examples/twittertopiccount/src/main/java/io/s4/example/twittertopiccount/User.java
@@ -1,280 +0,0 @@
-/*
- * Copyright (c) 2010 Yahoo! Inc. All rights reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
- * either express or implied. See the License for the specific
- * language governing permissions and limitations under the
- * License. See accompanying LICENSE file.
- */
-package io.s4.example.twittertopiccount;
-
-public class User {
- private long id;
- private String screenName;
- private String name;
- private String url;
- private int followersCount;
- private String lang;
- private boolean verified;
- private String profileImageUrl;
- private int friendsCount;
- private String description;
- private int favouritesCount;
- private boolean geoEnabled;
- private int listedCount;
- private String profileBackgroundImageUrl;
- private boolean protectedUser;
- private String location;
- private long statusesCount;
- private String timeZone;
- private boolean contributorsEnabled;
- private int utcOffset;
- private String createdAt;
-
- public long getId() {
- return id;
- }
-
- public void setId(long id) {
- this.id = id;
- }
-
- public String getScreenName() {
- return screenName;
- }
-
- public void setScreenName(String screenName) {
- this.screenName = screenName;
- }
-
- public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
- }
-
- public String getUrl() {
- return url;
- }
-
- public void setUrl(String url) {
- this.url = url;
- }
-
- public int getFollowersCount() {
- return followersCount;
- }
-
- public void setFollowersCount(int followersCount) {
- this.followersCount = followersCount;
- }
-
- public String getLang() {
- return lang;
- }
-
- public void setLang(String lang) {
- this.lang = lang;
- }
-
- public boolean isVerified() {
- return verified;
- }
-
- public void setVerified(boolean verified) {
- this.verified = verified;
- }
-
- public String getProfileImageUrl() {
- return profileImageUrl;
- }
-
- public void setProfileImageUrl(String profileImageUrl) {
- this.profileImageUrl = profileImageUrl;
- }
-
- public int getFriendsCount() {
- return friendsCount;
- }
-
- public void setFriendsCount(int friendsCount) {
- this.friendsCount = friendsCount;
- }
-
- public String getDescription() {
- return description;
- }
-
- public void setDescription(String description) {
- this.description = description;
- }
-
- public int getFavouritesCount() {
- return favouritesCount;
- }
-
- public void setFavouritesCount(int favouritesCount) {
- this.favouritesCount = favouritesCount;
- }
-
- public boolean isGeoEnabled() {
- return geoEnabled;
- }
-
- public void setGeoEnabled(boolean geoEnabled) {
- this.geoEnabled = geoEnabled;
- }
-
- public int getListedCount() {
- return listedCount;
- }
-
- public void setListedCount(int listedCount) {
- this.listedCount = listedCount;
- }
-
- public String getProfileBackgroundImageUrl() {
- return profileBackgroundImageUrl;
- }
-
- public void setProfileBackgroundImageUrl(String profileBackgroundImageUrl) {
- this.profileBackgroundImageUrl = profileBackgroundImageUrl;
- }
-
- public boolean isProtectedUser() {
- return protectedUser;
- }
-
- public void setProtectedUser(boolean protectedUser) {
- this.protectedUser = protectedUser;
- }
-
- public String getLocation() {
- return location;
- }
-
- public void setLocation(String location) {
- this.location = location;
- }
-
- public long getStatusesCount() {
- return statusesCount;
- }
-
- public void setStatusesCount(long statusesCount) {
- this.statusesCount = statusesCount;
- }
-
- public String getTimeZone() {
- return timeZone;
- }
-
- public void setTimeZone(String timeZone) {
- this.timeZone = timeZone;
- }
-
- public boolean isContributorsEnabled() {
- return contributorsEnabled;
- }
-
- public void setContributorsEnabled(boolean contributorsEnabled) {
- this.contributorsEnabled = contributorsEnabled;
- }
-
- public int getUtcOffset() {
- return utcOffset;
- }
-
- public void setUtcOffset(int utcOffset) {
- this.utcOffset = utcOffset;
- }
-
- public String getCreatedAt() {
- return createdAt;
- }
-
- public void setCreatedAt(String createdAt) {
- this.createdAt = createdAt;
- }
-
- public String toString() {
- StringBuffer sb = new StringBuffer();
- sb.append("{")
- .append("id:")
- .append(id)
- .append(",")
- .append("screenName:")
- .append(screenName)
- .append(",")
- .append("name:")
- .append(name)
- .append(",")
- .append("url:")
- .append(url)
- .append(",")
- .append("followersCount:")
- .append(followersCount)
- .append(",")
- .append("lang:")
- .append(lang)
- .append(",")
- .append("verified:")
- .append(verified)
- .append(",")
- .append("profileImageUrl:")
- .append(profileImageUrl)
- .append(",")
- .append("friendsCount:")
- .append(friendsCount)
- .append(",")
- .append("description:")
- .append(description)
- .append(",")
- .append("favouritesCount:")
- .append(favouritesCount)
- .append(",")
- .append("geoEnabled:")
- .append(geoEnabled)
- .append(",")
- .append("listedCount:")
- .append(listedCount)
- .append(",")
- .append("profileBackgroundImageUrl:")
- .append(profileBackgroundImageUrl)
- .append(",")
- .append("protectedUser:")
- .append(protectedUser)
- .append(",")
- .append("location:")
- .append(location)
- .append(",")
- .append("statusesCount:")
- .append(statusesCount)
- .append(",")
- .append("timeZone:")
- .append(timeZone)
- .append(",")
- .append("contributorsEnabled:")
- .append(contributorsEnabled)
- .append(",")
- .append("utcOffset:")
- .append(utcOffset)
- .append(",")
- .append("createdAt:")
- .append(createdAt)
- .append("}");
-
- return sb.toString();
-
- }
-
-}
View
17 s4-examples/twittertopiccount/src/main/resources/adapter-conf.xml
@@ -1,17 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-
-<beans xmlns="http://www.springframework.org/schema/beans"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://www.springframework.org/schema/beans
- http://www.springframework.org/schema/beans/spring-beans-2.0.xsd">
-
-
- <bean id="twitterFeedListener" class="io.s4.example.twittertopiccount.TwitterFeedListener"
- init-method="init">
- <property name="userid" value=""/>
- <property name="password" value=""/>
- <property name="urlString" value="http://stream.twitter.com/1/statuses/sample.json"/>
- <property name="streamName" value="RawStatus"/>
- </bean>
-
-</beans>
View
121 s4-examples/twittertopiccount/src/main/resources/s4-example-twittertopiccount-conf.xml
@@ -1,121 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<beans xmlns="http://www.springframework.org/schema/beans" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans-2.0.xsd">
-
- <!-- <bean id="printEventPE" class="io.s4.processor.PrintEventPE">
- <property name="id" value="printEventPE"/>
- <property name="keys">
- <list>
- <value>TopicSeen topic</value>
- </list>
- </property>
- </bean> -->
-
- <bean id="topicExtractorPE" class="io.s4.example.twittertopiccount.TopicExtractorPE">
- <property name="id" value="topicSeenPE"/>
- <property name="keys">
- <list>
- <value>RawStatus *</value>
- </list>
- </property>
- <property name="dispatcher" ref="twitDispatcher"/>
- <property name="outputStreamName" value="TopicSeen"/>
- </bean>
-
- <bean id="topicCountAndReportPE" class="io.s4.example.twittertopiccount.TopicCountAndReportPE">
- <property name="id" value="topicCountAndReportPE"/>
- <property name="keys">
- <list>
- <value>TopicSeen topic</value>
- </list>
- </property>
- <property name="threshold" value="4"/>
- <property name="outputFrequencyByTimeBoundary" value="5"/>
- <property name="dispatcher" ref="forkdispatcher"/>
- <property name="outputStreamName" value="AggregatedTopicSeen"/>
- <property name="ttl" value="36000"/>
- </bean>
-
- <bean id="dtfPersister" class="io.s4.example.twittertopiccount.DirectToFilePersister">
- <property name="outputFilename" value="/tmp/top_n_hashtags"/>
- </bean>
-
- <bean id="top10TopicPE" class="io.s4.example.twittertopiccount.TopNTopicPE">
- <property name="id" value="top10TopicPE"/>
- <property name="keys">
- <list>
- <value>AggregatedTopicSeen reportKey</value>
- </list>
- </property>
- <property name="entryCount" value="10"/>
- <property name="outputFrequencyByTimeBoundary" value="10"/>
- <property name="persister" ref="dtfPersister"/>
- <property name="persistTime" value="864000"/>
- <property name="persistKey" value="myapp:top10Topics"/>
- <property name="ttl" value="36000"/>
- </bean>
-
- <bean id="topicSeenPartitioner" class="io.s4.dispatcher.partitioner.DefaultPartitioner">
- <property name="streamNames">
- <list>
- <value>TopicSeen</value>
- </list>
- </property>
- <property name="hashKey">
- <list>
- <value>topic</value>
- </list>
- </property>
- <property name="hasher" ref="hasher"/>
- <property name="debug" value="false"/>
- </bean>
-
- <bean id="aggregatedTopicSeenPartitioner" class="io.s4.dispatcher.partitioner.DefaultPartitioner">
- <property name="streamNames">
- <list>
- <value>AggregatedTopicSeen</value>
- </list>
- </property>
- <property name="hashKey">
- <list>
- <value>reportKey</value>
- </list>
- </property>
- <property name="hasher" ref="hasher"/>
- <property name="debug" value="false"/>
- </bean>
-
- <bean id="twitDispatcher" class="io.s4.dispatcher.Dispatcher" init-method="init">
- <property name="partitioners">
- <list>
- <ref bean="topicSeenPartitioner"/>
- <ref bean="aggregatedTopicSeenPartitioner"/>
- </list>
- </property>
- <property name="eventEmitter" ref="commLayerEmitter"/>
- <property name="loggerName" value="s4"/>
- </bean>
-
- <!-- dispatcher to S4 and client adapter -->
- <bean id="forkdispatcher" class="io.s4.dispatcher.MultiDispatcher">
- <property name="dispatchers">
- <list>
-
- <!-- send everything through the S4 dispatcher -->
- <ref bean="twitDispatcher"/>
-
-
- <!-- send some streams to client adapters -->
- <bean id="selectiveDispatchToAdapter" class="io.s4.dispatcher.StreamSelectingDispatcher">
- <property name="dispatcher" ref="dispatcherToClientAdapters"/>
- <property name="streams">
- <list>
- <value>AggregatedTopicSeen</value>
- </list>
- </property>
- </bean>
-
- </list>
- </property>
- </bean>
-
-</beans>
View
2  settings.gradle
@@ -19,7 +19,6 @@ include 's4-driver'
include 's4-example-driver'
-include 's4-example-twittertopiccount'
include 's4-example-twittertopiccount-scala'
include 's4-example-speech01'
include 's4-example-speech01-scala'
@@ -31,7 +30,6 @@ include 's4-tools-loadgenerator'
/* TODO: Write code to map proj name to dir. */
project(':s4-driver').projectDir = new File(settingsDir, 's4-driver/java')
project(':s4-example-driver').projectDir = new File(settingsDir, 's4-driver/examples')
-project(':s4-example-twittertopiccount').projectDir = new File(settingsDir, 's4-examples/twittertopiccount')
project(':s4-example-twittertopiccount-scala').projectDir = new File(settingsDir, 's4-examples/twittertopiccount-scala')
project(':s4-example-speech01').projectDir = new File(settingsDir, 's4-examples/speech01')
project(':s4-example-speech01-scala').projectDir = new File(settingsDir, 's4-examples/speech01-scala')
Please sign in to comment.
Something went wrong with that request. Please try again.