changes for personal use

rjyo · Nov 28, 2011 · 73da730 · 73da730
1 parent 0e024aa
commit 73da730
Show file tree

Hide file tree

Showing 9 changed files with 109 additions and 178 deletions.
diff --git a/Makefile b/Makefile
@@ -1,9 +1,7 @@
 MVN_INSTALL_FLAGS = install:install-file -Dfile=./lib/cron4j-2.2.3.jar -DgroupId=cron4j -DartifactId=cron4j -Dversion=2.2.3 -Dpackaging=jar
 
 WEB_TARGET_DIR = ./web/target/h2weibo
-WORKER_TARGET_DIR = ./worker/target/h2weibo-worker
-WEB_NAME = h2weibo
-WORKER_NAME = h2weibo-w1
+WEB_NAME = h2w-single
 
 compile: clean
 	@mvn -Dmaven.test.skip=true package
@@ -21,27 +19,15 @@ run:
 update: compile
 	@vmc update ${WEB_NAME} --path $(WEB_TARGET_DIR)
 
-update_worker: compile
-	@vmc update ${WORKER_NAME} --path $(WORKER_TARGET_DIR)
-
 logs:
 	@vmc logs ${WEB_NAME}
 
-logs_worker:
-	@vmc logs ${WORKER_NAME}
-
 stats:
 	@echo stats for ${WEB_NAME}
 	@vmc stats ${WEB_NAME}
-	@echo stats for ${WORKER_NAME}
-	@vmc stats ${WORKER_NAME}
 
 restart:
 	@vmc stop ${WEB_NAME}
 	@vmc start ${WEB_NAME}
 
-restart_worker:
-	@vmc stop ${WORKER_NAME}
-	@vmc start ${WORKER_NAME}
-
 .PHONY: compile
diff --git a/core/src/main/java/h2weibo/InitServlet.java b/core/src/main/java/h2weibo/InitServlet.java
@@ -77,15 +77,17 @@ public JedisPool getPool(ServletContext context) {
     public void init(ServletConfig config) throws ServletException {
         super.init(config);
 
+        System.setProperty("h2weibo.admin.user", config.getInitParameter("admin"));
+
         // Key for Weibo App
-        System.setProperty("weibo4j.oauth.consumerKey", "2440858351");
-        System.setProperty("weibo4j.oauth.consumerSecret", "1faf4ed7b4af302907e25429a0b88dfc");
+        System.setProperty("weibo4j.oauth.consumerKey", config.getInitParameter("w_k"));
+        System.setProperty("weibo4j.oauth.consumerSecret", config.getInitParameter("w_s"));
 
         System.setProperty("weibo4j.debug", "false");
 
         // Key for Twitter App
-        System.setProperty("twitter4j.oauth.consumerKey", "Scwn2HbdT7v3yOEjkAQrfQ");
-        System.setProperty("twitter4j.oauth.consumerSecret", "QIz4dbgb5ABzNMjfP1Sb0YdwKTY2oKQwhLoehk0ug");
+        System.setProperty("twitter4j.oauth.consumerKey", config.getInitParameter("t_k"));
+        System.setProperty("twitter4j.oauth.consumerSecret", config.getInitParameter("t_s"));
 
         log.info("System initialized.");
 

diff --git a/core/src/main/java/h2weibo/S3BackupTask.java b/core/src/main/java/h2weibo/S3BackupTask.java
diff --git a/default.properties b/default.properties
@@ -0,0 +1,5 @@
+w_k=341680771
+w_s=60b0a26cf3e400082630d12e279adcd7
+t_k=Scwn2HbdT7v3yOEjkAQrfQ
+t_s=QIz4dbgb5ABzNMjfP1Sb0YdwKTY2oKQwhLoehk0ug
+admin.user=xu_lele
diff --git a/h2weibo.iml b/h2weibo.iml
@@ -3,7 +3,6 @@
   <component name="NewModuleRootManager" inherit-compiler-output="false">
     <output url="file://$MODULE_DIR$/target/classes" />
     <output-test url="file://$MODULE_DIR$/target/test-classes" />
-    <exclude-output />
     <content url="file://$MODULE_DIR$">
       <excludeFolder url="file://$MODULE_DIR$/.idea" />
       <excludeFolder url="file://$MODULE_DIR$/lib" />

diff --git a/web/src/main/java/h2weibo/controllers/CommandServlet.java b/web/src/main/java/h2weibo/controllers/CommandServlet.java
@@ -49,60 +49,45 @@ protected void doGet(HttpServletRequest request, HttpServletResponse response) t
 
         final DBHelper helper = (DBHelper) request.getAttribute(Keys.REQUEST_DB_HELPER);
 
-        if (router.is(":cmd", "dump")) {
-            S3BackupTask task = new S3BackupTask();
-            task.setHelper(helper);
-            task.run();
+        if (router.is(":cmd", "users")) {
+            Set<String> ids = helper.getAuthorizedIds();
+            Map<String, String> mappings = helper.getMappings();
+
+            writer.println(String.format("Syncing user list: (%d users)", ids.size()));
+            for (String id : ids) {
+                writer.println(String.format("  %s => %s", id, mappings.get(id)));
+            }
+        } else if (router.is(":cmd", "mapping")) {
+            Thread t = new Thread(new Runnable() {
+                @Override
+                public void run() {
+                    helper.createUserMap();
+                }
+            });
+            t.start();
             response.sendRedirect("/");
-        } else if (router.is(":cmd", "restore")) {
-            S3BackupTask task = new S3BackupTask();
+        } else if (router.is(":cmd", "info")) {
+            String info = helper.getJedisInfo();
+            writer.println(info);
+        } else if (router.is(":cmd", "del")) {
             if (router.has(":id")) {
-                task.restore(router.get(":id"));
-                response.sendRedirect("/");
-            } else {
-                writer.println("Parameter error");
+                String user = router.get(":id");
+                helper.deleteUser(helper.findOneByUser(user));
+                response.sendRedirect("/u/" + user);
             }
-        } else {
-            if (router.is(":cmd", "users")) {
-                Set<String> ids = helper.getAuthorizedIds();
-                Map<String, String> mappings = helper.getMappings();
-
-                writer.println(String.format("Syncing user list: (%d users)", ids.size()));
-                for (String id : ids) {
-                    writer.println(String.format("  %s => %s", id, mappings.get(id)));
-                }
-            } else if (router.is(":cmd", "mapping")) {
-                Thread t = new Thread(new Runnable() {
-                    @Override
-                    public void run() {
-                        helper.createUserMap();
-                    }
-                });
-                t.start();
-                response.sendRedirect("/");
-            } else if (router.is(":cmd", "info")) {
-                String info = helper.getJedisInfo();
-                writer.println(info);
-            } else if (router.is(":cmd", "del")) {
-                if (router.has(":id")) {
-                    String user = router.get(":id");
-                    helper.deleteUser(helper.findOneByUser(user));
-                    response.sendRedirect("/u/" + user);
-                }
-            } else if (router.is(":cmd", "u")) {
-                if (router.has(":id")) {
-                    T2WUser u = helper.findOneByUser(router.get(":id"));
-                    writer.println(String.format("Latest tweet ID is %d", u.getLatestId()));
-                    writer.println(String.format("Twitter ID is %s", router.get(":id")));
-                    writer.println(String.format("Weibo ID is %s", helper.getWeiboId(u.getUserId())));
-                    writer.println(String.format("Twitter Token %s", u.getTwitterToken()));
-                    writer.println(String.format("Twitter Secret %s", u.getTwitterTokenSecret()));
-                    writer.println(String.format("Weibo Token %s", u.getToken()));
-                    writer.println(String.format("Weibo Secret %s", u.getTokenSecret()));
-                }
-            } else {
-                response.sendRedirect("/");
+        } else if (router.is(":cmd", "u")) {
+            if (router.has(":id")) {
+                T2WUser u = helper.findOneByUser(router.get(":id"));
+                writer.println(String.format("Latest tweet ID is %d", u.getLatestId()));
+                writer.println(String.format("Twitter ID is %s", router.get(":id")));
+                writer.println(String.format("Weibo ID is %s", helper.getWeiboId(u.getUserId())));
+                writer.println(String.format("Twitter Token %s", u.getTwitterToken()));
+                writer.println(String.format("Twitter Secret %s", u.getTwitterTokenSecret()));
+                writer.println(String.format("Weibo Token %s", u.getToken()));
+                writer.println(String.format("Weibo Secret %s", u.getTokenSecret()));
             }
+        } else {
+            response.sendRedirect("/");
         }
         writer.close();
     }
@@ -113,26 +98,60 @@ public void init(ServletConfig config) throws ServletException {
 
         log.info("Web started.");
 
-        JedisPool jedisPool = getPool(getServletContext());
-        DBHelper helper = DBHelperFactory.createHelper(jedisPool);
+        JedisPool pool = getPool(getServletContext());
+        DBHelper helper = DBHelperFactory.createHelper(pool);
         // clear the queue
         helper.clearQueue();
 
         Scheduler scheduler = new Scheduler();
 
         QueueTask task = new QueueTask();
-        task.setHelper(DBHelperFactory.createHelper(jedisPool));
+        task.setHelper(DBHelperFactory.createHelper(pool));
         scheduler.schedule("*/2 * * * *", task);
 
-        System.setProperty("h2weibo.awsAccessKey", config.getInitParameter("accessKey"));
-        System.setProperty("h2weibo.awsSecretAccessKey", config.getInitParameter("secretAccessKey"));
-
-        S3BackupTask task2 = new S3BackupTask();
-        task2.setHelper(DBHelperFactory.createHelper(jedisPool));
-        scheduler.schedule("0 * * * *", task2);
-
         scheduler.start();
 
         log.info("Cron scheduler started.");
+
+
+        log.info("Worker started.");
+        // 1 Threads to handle the sync job
+        new Thread(new SyncWorkerRunnable(DBHelperFactory.createHelper(pool))).start();
     }
+
+    private static class SyncWorkerRunnable implements Runnable {
+        private DBHelper helper;
+
+        public SyncWorkerRunnable(DBHelper helper) {
+            this.helper = helper;
+        }
+
+        @Override
+        public void run() {
+            long size = helper.getQueueSize();
+            log.info("Start running, queue size = " + size + ", user count = " + helper.getUserCount());
+
+            int multiper = 1;
+            while (true) {
+                T2WUser user = helper.pop();
+                if (user != null) {
+                    multiper = 1;
+                    String userId = user.getUserId();
+                    Twitter2Weibo weibo = new Twitter2Weibo(helper);
+                    weibo.syncTwitter(userId);
+                    log.debug("Syncing for " + userId);
+                } else {
+                    try {
+                        log.info("No task found, sleeping");
+                        Thread.sleep(4000 * multiper);
+                        multiper *= 2;
+                    } catch (InterruptedException e) {
+                        log.error("Error when try to sleep thread.", e);
+                    }
+                }
+            }
+
+        }
+    }
+
 }
diff --git a/web/src/main/java/h2weibo/filters/AdminFilter.java b/web/src/main/java/h2weibo/filters/AdminFilter.java
@@ -37,7 +37,8 @@ public void doFilter(ServletRequest req, ServletResponse res, FilterChain chain)
         HttpSession session = ((HttpServletRequest) req).getSession();
         String user = (String) session.getAttribute(Keys.SESSION_LOGIN_USER);
 
-        if (!"xu_lele".equals(user)) {
+        String admin = System.getProperty("h2weibo.admin.user", "xu_lele");
+        if (!admin.equals(user)) {
             log.info("Not logged in. Redirect to twitter login.");
             session.setAttribute(Keys.SESSION_REQUEST_URL, ((HttpServletRequest) req).getRequestURL().toString());
             ((HttpServletResponse) res).sendRedirect("/auth/twitter");

diff --git a/web/src/main/webapp/WEB-INF/web.xml b/web/src/main/webapp/WEB-INF/web.xml
@@ -8,12 +8,24 @@
         <servlet-class>h2weibo.controllers.CommandServlet</servlet-class>
         <load-on-startup>1</load-on-startup>
         <init-param>
-            <param-name>accessKey</param-name>
-            <param-value>${s3.access}</param-value>
+            <param-name>t_k</param-name>
+            <param-value>${t_k}</param-value>
         </init-param>
         <init-param>
-            <param-name>secretAccessKey</param-name>
-            <param-value>${s3.secret}</param-value>
+            <param-name>t_s</param-name>
+            <param-value>${t_s}</param-value>
+        </init-param>
+        <init-param>
+            <param-name>w_k</param-name>
+            <param-value>${w_k}</param-value>
+        </init-param>
+        <init-param>
+            <param-name>w_s</param-name>
+            <param-value>${w_s}</param-value>
+        </init-param>
+        <init-param>
+            <param-name>admin</param-name>
+            <param-value>${admin.user}</param-value>
         </init-param>
     </servlet>
 

diff --git a/worker/src/main/java/h2weibo/worker/WorkerServlet.java b/worker/src/main/java/h2weibo/worker/WorkerServlet.java
@@ -43,8 +43,6 @@ public void init(final ServletConfig config) throws ServletException {
 
         // 3 Threads to handle the sync job
         new Thread(new SyncWorkerRunnable(DBHelperFactory.createHelper(pool))).start();
-        new Thread(new SyncWorkerRunnable(DBHelperFactory.createHelper(pool))).start();
-        new Thread(new SyncWorkerRunnable(DBHelperFactory.createHelper(pool))).start();
     }
 
     private static class SyncWorkerRunnable implements Runnable {