From 73da730a6e5878ffeeff708929501d69c3619072 Mon Sep 17 00:00:00 2001 From: Rakuraku Jyo Date: Mon, 28 Nov 2011 23:26:20 +0900 Subject: [PATCH] changes for personal use --- Makefile | 16 +- core/src/main/java/h2weibo/InitServlet.java | 10 +- core/src/main/java/h2weibo/S3BackupTask.java | 91 ------------ default.properties | 5 + h2weibo.iml | 1 - .../h2weibo/controllers/CommandServlet.java | 139 ++++++++++-------- .../java/h2weibo/filters/AdminFilter.java | 3 +- web/src/main/webapp/WEB-INF/web.xml | 20 ++- .../java/h2weibo/worker/WorkerServlet.java | 2 - 9 files changed, 109 insertions(+), 178 deletions(-) delete mode 100644 core/src/main/java/h2weibo/S3BackupTask.java create mode 100644 default.properties diff --git a/Makefile b/Makefile index 56f78c8..d49ef7d 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,7 @@ MVN_INSTALL_FLAGS = install:install-file -Dfile=./lib/cron4j-2.2.3.jar -DgroupId=cron4j -DartifactId=cron4j -Dversion=2.2.3 -Dpackaging=jar WEB_TARGET_DIR = ./web/target/h2weibo -WORKER_TARGET_DIR = ./worker/target/h2weibo-worker -WEB_NAME = h2weibo -WORKER_NAME = h2weibo-w1 +WEB_NAME = h2w-single compile: clean @mvn -Dmaven.test.skip=true package @@ -21,27 +19,15 @@ run: update: compile @vmc update ${WEB_NAME} --path $(WEB_TARGET_DIR) -update_worker: compile - @vmc update ${WORKER_NAME} --path $(WORKER_TARGET_DIR) - logs: @vmc logs ${WEB_NAME} -logs_worker: - @vmc logs ${WORKER_NAME} - stats: @echo stats for ${WEB_NAME} @vmc stats ${WEB_NAME} - @echo stats for ${WORKER_NAME} - @vmc stats ${WORKER_NAME} restart: @vmc stop ${WEB_NAME} @vmc start ${WEB_NAME} -restart_worker: - @vmc stop ${WORKER_NAME} - @vmc start ${WORKER_NAME} - .PHONY: compile diff --git a/core/src/main/java/h2weibo/InitServlet.java b/core/src/main/java/h2weibo/InitServlet.java index 1fd8396..407a6ff 100644 --- a/core/src/main/java/h2weibo/InitServlet.java +++ b/core/src/main/java/h2weibo/InitServlet.java @@ -77,15 +77,17 @@ public JedisPool getPool(ServletContext context) { public void init(ServletConfig config) throws ServletException { super.init(config); + System.setProperty("h2weibo.admin.user", config.getInitParameter("admin")); + // Key for Weibo App - System.setProperty("weibo4j.oauth.consumerKey", "2440858351"); - System.setProperty("weibo4j.oauth.consumerSecret", "1faf4ed7b4af302907e25429a0b88dfc"); + System.setProperty("weibo4j.oauth.consumerKey", config.getInitParameter("w_k")); + System.setProperty("weibo4j.oauth.consumerSecret", config.getInitParameter("w_s")); System.setProperty("weibo4j.debug", "false"); // Key for Twitter App - System.setProperty("twitter4j.oauth.consumerKey", "Scwn2HbdT7v3yOEjkAQrfQ"); - System.setProperty("twitter4j.oauth.consumerSecret", "QIz4dbgb5ABzNMjfP1Sb0YdwKTY2oKQwhLoehk0ug"); + System.setProperty("twitter4j.oauth.consumerKey", config.getInitParameter("t_k")); + System.setProperty("twitter4j.oauth.consumerSecret", config.getInitParameter("t_s")); log.info("System initialized."); diff --git a/core/src/main/java/h2weibo/S3BackupTask.java b/core/src/main/java/h2weibo/S3BackupTask.java deleted file mode 100644 index c86e588..0000000 --- a/core/src/main/java/h2weibo/S3BackupTask.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * (The MIT License) - * - * Copyright (c) 2011 Rakuraku Jyo - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the 'Software'), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies - * of the Software, and to permit persons to whom the Software is furnished to - * do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - */ - -package h2weibo; - -import org.apache.commons.lang.time.DateFormatUtils; -import org.apache.log4j.Logger; -import org.jets3t.service.S3Service; -import org.jets3t.service.impl.rest.httpclient.RestS3Service; -import org.jets3t.service.model.S3Bucket; -import org.jets3t.service.model.S3Object; -import org.jets3t.service.security.AWSCredentials; -import weibo4j.Weibo; - -import java.io.BufferedReader; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.Date; - -/** - * @author Rakuraku Jyo - */ -public class S3BackupTask extends DBTask { - private static final Logger log = Logger.getLogger(S3BackupTask.class.getName()); - private String awsAccessKey; - private String awsSecretAccessKey; - - public S3BackupTask() { - this.awsAccessKey = System.getProperty("h2weibo.awsAccessKey"); - this.awsSecretAccessKey = System.getProperty("h2weibo.awsSecretAccessKey"); - } - - public S3BackupTask(String awsAccessKey, String awsSecretAccessKey) { - this.awsAccessKey = awsAccessKey; - this.awsSecretAccessKey = awsSecretAccessKey; - } - - public void run() { - String dump = getHelper().dump(); - - try { - AWSCredentials awsCredentials = new AWSCredentials(awsAccessKey, awsSecretAccessKey); - S3Service s3Service = new RestS3Service(awsCredentials); - S3Bucket backup = s3Service.getBucket("h2weibo.backup"); - - String dateString = DateFormatUtils.format(new Date(), "yyyy-MM-dd_HH"); - S3Object object = new S3Object(dateString + ".json", dump); - object = s3Service.putObject(backup, object); - System.out.println("S3Object after upload: " + object); - } catch (Exception e) { - log.error("Failed to upload to S3."); - log.error(e); - } - } - - public void restore(String from) { - try { - AWSCredentials awsCredentials = new AWSCredentials(awsAccessKey, awsSecretAccessKey); - S3Service s3Service = new RestS3Service(awsCredentials); - - S3Object object = s3Service.getObject("h2weibo.backup", from + ".json"); - InputStream inputStream = object.getDataInputStream(); - BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream)); - - String line; - String data = ""; - while ((line = reader.readLine()) != null) { - data += line; - } - reader.close(); - - getHelper().restore(data); - } catch (Exception e) { - log.error("Failed to restore from S3."); - log.error(e); - } - } -} diff --git a/default.properties b/default.properties new file mode 100644 index 0000000..c8a4d58 --- /dev/null +++ b/default.properties @@ -0,0 +1,5 @@ +w_k=341680771 +w_s=60b0a26cf3e400082630d12e279adcd7 +t_k=Scwn2HbdT7v3yOEjkAQrfQ +t_s=QIz4dbgb5ABzNMjfP1Sb0YdwKTY2oKQwhLoehk0ug +admin.user=xu_lele \ No newline at end of file diff --git a/h2weibo.iml b/h2weibo.iml index 62bf13c..5bd48dd 100644 --- a/h2weibo.iml +++ b/h2weibo.iml @@ -3,7 +3,6 @@ - diff --git a/web/src/main/java/h2weibo/controllers/CommandServlet.java b/web/src/main/java/h2weibo/controllers/CommandServlet.java index 5fcca2f..8d5d184 100644 --- a/web/src/main/java/h2weibo/controllers/CommandServlet.java +++ b/web/src/main/java/h2weibo/controllers/CommandServlet.java @@ -49,60 +49,45 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) t final DBHelper helper = (DBHelper) request.getAttribute(Keys.REQUEST_DB_HELPER); - if (router.is(":cmd", "dump")) { - S3BackupTask task = new S3BackupTask(); - task.setHelper(helper); - task.run(); + if (router.is(":cmd", "users")) { + Set ids = helper.getAuthorizedIds(); + Map mappings = helper.getMappings(); + + writer.println(String.format("Syncing user list: (%d users)", ids.size())); + for (String id : ids) { + writer.println(String.format(" %s => %s", id, mappings.get(id))); + } + } else if (router.is(":cmd", "mapping")) { + Thread t = new Thread(new Runnable() { + @Override + public void run() { + helper.createUserMap(); + } + }); + t.start(); response.sendRedirect("/"); - } else if (router.is(":cmd", "restore")) { - S3BackupTask task = new S3BackupTask(); + } else if (router.is(":cmd", "info")) { + String info = helper.getJedisInfo(); + writer.println(info); + } else if (router.is(":cmd", "del")) { if (router.has(":id")) { - task.restore(router.get(":id")); - response.sendRedirect("/"); - } else { - writer.println("Parameter error"); + String user = router.get(":id"); + helper.deleteUser(helper.findOneByUser(user)); + response.sendRedirect("/u/" + user); } - } else { - if (router.is(":cmd", "users")) { - Set ids = helper.getAuthorizedIds(); - Map mappings = helper.getMappings(); - - writer.println(String.format("Syncing user list: (%d users)", ids.size())); - for (String id : ids) { - writer.println(String.format(" %s => %s", id, mappings.get(id))); - } - } else if (router.is(":cmd", "mapping")) { - Thread t = new Thread(new Runnable() { - @Override - public void run() { - helper.createUserMap(); - } - }); - t.start(); - response.sendRedirect("/"); - } else if (router.is(":cmd", "info")) { - String info = helper.getJedisInfo(); - writer.println(info); - } else if (router.is(":cmd", "del")) { - if (router.has(":id")) { - String user = router.get(":id"); - helper.deleteUser(helper.findOneByUser(user)); - response.sendRedirect("/u/" + user); - } - } else if (router.is(":cmd", "u")) { - if (router.has(":id")) { - T2WUser u = helper.findOneByUser(router.get(":id")); - writer.println(String.format("Latest tweet ID is %d", u.getLatestId())); - writer.println(String.format("Twitter ID is %s", router.get(":id"))); - writer.println(String.format("Weibo ID is %s", helper.getWeiboId(u.getUserId()))); - writer.println(String.format("Twitter Token %s", u.getTwitterToken())); - writer.println(String.format("Twitter Secret %s", u.getTwitterTokenSecret())); - writer.println(String.format("Weibo Token %s", u.getToken())); - writer.println(String.format("Weibo Secret %s", u.getTokenSecret())); - } - } else { - response.sendRedirect("/"); + } else if (router.is(":cmd", "u")) { + if (router.has(":id")) { + T2WUser u = helper.findOneByUser(router.get(":id")); + writer.println(String.format("Latest tweet ID is %d", u.getLatestId())); + writer.println(String.format("Twitter ID is %s", router.get(":id"))); + writer.println(String.format("Weibo ID is %s", helper.getWeiboId(u.getUserId()))); + writer.println(String.format("Twitter Token %s", u.getTwitterToken())); + writer.println(String.format("Twitter Secret %s", u.getTwitterTokenSecret())); + writer.println(String.format("Weibo Token %s", u.getToken())); + writer.println(String.format("Weibo Secret %s", u.getTokenSecret())); } + } else { + response.sendRedirect("/"); } writer.close(); } @@ -113,26 +98,60 @@ public void init(ServletConfig config) throws ServletException { log.info("Web started."); - JedisPool jedisPool = getPool(getServletContext()); - DBHelper helper = DBHelperFactory.createHelper(jedisPool); + JedisPool pool = getPool(getServletContext()); + DBHelper helper = DBHelperFactory.createHelper(pool); // clear the queue helper.clearQueue(); Scheduler scheduler = new Scheduler(); QueueTask task = new QueueTask(); - task.setHelper(DBHelperFactory.createHelper(jedisPool)); + task.setHelper(DBHelperFactory.createHelper(pool)); scheduler.schedule("*/2 * * * *", task); - System.setProperty("h2weibo.awsAccessKey", config.getInitParameter("accessKey")); - System.setProperty("h2weibo.awsSecretAccessKey", config.getInitParameter("secretAccessKey")); - - S3BackupTask task2 = new S3BackupTask(); - task2.setHelper(DBHelperFactory.createHelper(jedisPool)); - scheduler.schedule("0 * * * *", task2); - scheduler.start(); log.info("Cron scheduler started."); + + + log.info("Worker started."); + // 1 Threads to handle the sync job + new Thread(new SyncWorkerRunnable(DBHelperFactory.createHelper(pool))).start(); } + + private static class SyncWorkerRunnable implements Runnable { + private DBHelper helper; + + public SyncWorkerRunnable(DBHelper helper) { + this.helper = helper; + } + + @Override + public void run() { + long size = helper.getQueueSize(); + log.info("Start running, queue size = " + size + ", user count = " + helper.getUserCount()); + + int multiper = 1; + while (true) { + T2WUser user = helper.pop(); + if (user != null) { + multiper = 1; + String userId = user.getUserId(); + Twitter2Weibo weibo = new Twitter2Weibo(helper); + weibo.syncTwitter(userId); + log.debug("Syncing for " + userId); + } else { + try { + log.info("No task found, sleeping"); + Thread.sleep(4000 * multiper); + multiper *= 2; + } catch (InterruptedException e) { + log.error("Error when try to sleep thread.", e); + } + } + } + + } + } + } diff --git a/web/src/main/java/h2weibo/filters/AdminFilter.java b/web/src/main/java/h2weibo/filters/AdminFilter.java index 8a5f0c2..21b59b8 100644 --- a/web/src/main/java/h2weibo/filters/AdminFilter.java +++ b/web/src/main/java/h2weibo/filters/AdminFilter.java @@ -37,7 +37,8 @@ public void doFilter(ServletRequest req, ServletResponse res, FilterChain chain) HttpSession session = ((HttpServletRequest) req).getSession(); String user = (String) session.getAttribute(Keys.SESSION_LOGIN_USER); - if (!"xu_lele".equals(user)) { + String admin = System.getProperty("h2weibo.admin.user", "xu_lele"); + if (!admin.equals(user)) { log.info("Not logged in. Redirect to twitter login."); session.setAttribute(Keys.SESSION_REQUEST_URL, ((HttpServletRequest) req).getRequestURL().toString()); ((HttpServletResponse) res).sendRedirect("/auth/twitter"); diff --git a/web/src/main/webapp/WEB-INF/web.xml b/web/src/main/webapp/WEB-INF/web.xml index 76a4be2..e360844 100644 --- a/web/src/main/webapp/WEB-INF/web.xml +++ b/web/src/main/webapp/WEB-INF/web.xml @@ -8,12 +8,24 @@ h2weibo.controllers.CommandServlet 1 - accessKey - ${s3.access} + t_k + ${t_k} - secretAccessKey - ${s3.secret} + t_s + ${t_s} + + + w_k + ${w_k} + + + w_s + ${w_s} + + + admin + ${admin.user} diff --git a/worker/src/main/java/h2weibo/worker/WorkerServlet.java b/worker/src/main/java/h2weibo/worker/WorkerServlet.java index 4924e8a..f97ed6a 100644 --- a/worker/src/main/java/h2weibo/worker/WorkerServlet.java +++ b/worker/src/main/java/h2weibo/worker/WorkerServlet.java @@ -43,8 +43,6 @@ public void init(final ServletConfig config) throws ServletException { // 3 Threads to handle the sync job new Thread(new SyncWorkerRunnable(DBHelperFactory.createHelper(pool))).start(); - new Thread(new SyncWorkerRunnable(DBHelperFactory.createHelper(pool))).start(); - new Thread(new SyncWorkerRunnable(DBHelperFactory.createHelper(pool))).start(); } private static class SyncWorkerRunnable implements Runnable {