Develop

README.md

@@ -3,15 +3,15 @@ LTS 轻量级分布式任务调度框架(Light Task Schedule)
 
 ## 框架概况：
  LTS是一个轻量级分布式任务调度框架，参考hadoop的部分思想。有三种角色, JobClient, JobTracker, TaskTracker。各个节点都是无状态的，可以部署多个，来实现负载均衡，实现更大的负载量, 并且框架具有很好的容错能力。
- 采用多种注册中心（Zookeeper，redis等）进行节点信息暴露，master选举。Mongo存储任务队列和任务执行日志, netty做底层通信。
+ 采用多种注册中心（Zookeeper，redis等）进行节点信息暴露，master选举。(Mongo or Mysql)存储任务队列和任务执行日志, netty做底层通信。
 * JobClient : 主要负责提交任务, 和 接收任务执行反馈结果。
 * JobTracker : 负责接收并分配任务，任务调度。
 * TaskTracker: 负责执行任务，执行完反馈给JobTracker。
 
 框架支持实时任务，也支持定时任务，同时也支持CronExpression, 有问题，请加QQ群：109500214 一起完善，探讨
 
 ##架构图
-![Aaron Swartz](https://raw.githubusercontent.com/qq254963746/light-task-schedule/master/data/%E6%9E%B6%E6%9E%84%E5%9B%BE.png)
+![Aaron Swartz](https://raw.githubusercontent.com/qq254963746/light-task-schedule/master/doc/%E6%9E%B6%E6%9E%84%E5%9B%BE.png)
 ##节点组:
 * 1. 一个节点组等同于一个集群，同一个节点组中的各个节点是对等的，外界无论连接节点组中的任务一个节点都是可以的。
 * 2. 每个节点组中都有一个master节点(master宕机，会自动选举出新的master节点)，框架会提供接口API来监听master节点的变化，用户可以自己使用master节点做自己想做的事情。
@@ -23,7 +23,7 @@ LTS 轻量级分布式任务调度框架(Light Task Schedule)
 * 1. JobClient 提交一个 任务 给 JobTracker, 这里我提供了两种客户端API, 一种是如果JobTracker 不存在或者提交失败，直接返回提交失败。另一种客户端是重试客户端, 如果提交失败，先存储到本地leveldb(可以使用NFS来达到同个节点组共享leveldb文件的目的,多线程访问，做了文件锁处理)，返回给客户端提交成功的信息，待JobTracker可用的时候，再将任务提交。
 * 2. JobTracker 收到JobClient提交来的任务，先生成一个唯一的JobID。然后将任务储存在Mongo集群中。JobTracker 发现有（任务执行的）可用的TaskTracker节点(组) 之后，将优先级最大，最先提交的任务分发给TaskTracker。这里JobTracker会优先分配给比较空闲的TaskTracker节点，达到负载均衡。
 * 3. TaskTracker 收到JobTracker分发来的任务之后，执行。执行完毕之后，再反馈任务执行结果给JobTracker（成功or 失败[失败有失败错误信息]），如果发现JobTacker不可用，那么存储本地leveldb，等待TaskTracker可用的时候再反馈。反馈结果的同时，询问JobTacker有没有新的任务要执行。
-* 4. JobTacker收到TaskTracker节点的任务结果信息，生成并插入(mongo)任务执行日志。根据任务信息决定要不要反馈给客户端。不需要反馈的直接删除, 需要反馈的（同样JobClient不可用存储文件，等待可用重发）。
+* 4. JobTacker收到TaskTracker节点的任务结果信息，生成并插入(mongo or mysql)任务执行日志。根据任务信息决定要不要反馈给客户端。不需要反馈的直接删除, 需要反馈的（同样JobClient不可用存储文件，等待可用重发）。
 * 5. JobClient 收到任务执行结果，进行自己想要的逻辑处理。
 
 ## 特性

job-admin/pom.xml → lts-admin/pom.xml

@@ -3,13 +3,13 @@
          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
-        <artifactId>job-parent</artifactId>
+        <artifactId>lts-parent</artifactId>
         <groupId>com.lts</groupId>
-        <version>1.4.2-SNAPSHOT</version>
+        <version>1.5-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <packaging>war</packaging>
-    <artifactId>job-admin</artifactId>
+    <artifactId>lts-admin</artifactId>
 
     <properties>
         <spring.version>3.2.4.RELEASE</spring.version>
@@ -35,7 +35,7 @@
     <dependencies>
         <dependency>
             <groupId>com.lts</groupId>
-            <artifactId>job-core</artifactId>
+            <artifactId>lts-core</artifactId>
             <version>${project.version}</version>
         </dependency>
         <dependency>

job-admin/src/main/java/com/lts/job/web/support/AppConfigurer.java → lts-admin/src/main/java/com/lts/job/web/support/AppConfigurer.java

@@ -1,5 +1,7 @@
 package com.lts.job.web.support;
 
+import java.util.*;
+
 /**
  * @author Robert HG (254963746@qq.com) on 5/11/15.
  */

lts-admin/src/main/webapp/WEB-INF/web.xml

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<web-app xmlns="http://java.sun.com/xml/ns/javaee"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://java.sun.com/xml/ns/javaee http://java.sun.com/xml/ns/javaee/web-app_2_5.xsd "
+         metadata-complete="false"
+         version="2.5">
+
+    <display-name>LTS Admin</display-name>
+
+</web-app>

job-core/pom.xml → lts-core/pom.xml

@@ -3,13 +3,13 @@
          xmlns="http://maven.apache.org/POM/4.0.0"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
-        <artifactId>job-parent</artifactId>
+        <artifactId>lts-parent</artifactId>
         <groupId>com.lts</groupId>
-        <version>1.4.2-SNAPSHOT</version>
+        <version>1.5-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <packaging>jar</packaging>
-    <artifactId>job-core</artifactId>
+    <artifactId>lts-core</artifactId>
 
     <dependencies>
         <dependency>

job-core/src/main/java/com/lts/job/core/remoting/HeartBeatMonitor.java → lts-core/src/main/java/com/lts/job/core/remoting/HeartBeatMonitor.java

@@ -8,9 +8,13 @@
 import com.lts.job.core.protocol.JobProtos;
 import com.lts.job.core.protocol.command.HeartBeatRequest;
 import com.lts.job.core.util.CollectionUtils;
+import com.lts.job.core.util.Holder;
+import com.lts.job.remoting.InvokeCallback;
+import com.lts.job.remoting.netty.ResponseFuture;
 import com.lts.job.remoting.protocol.RemotingCommand;
 
 import java.util.List;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;

job-core/src/main/java/com/lts/job/store/mongo/MongoFactoryBean.java → lts-core/src/main/java/com/lts/job/store/mongo/MongoFactoryBean.java

@@ -3,10 +3,7 @@
 import com.lts.job.core.logger.Logger;
 import com.lts.job.core.logger.LoggerFactory;
 import com.lts.job.core.util.StringUtils;
-import com.mongodb.MongoClient;
-import com.mongodb.MongoClientOptions;
-import com.mongodb.MongoCredential;
-import com.mongodb.ServerAddress;
+import com.mongodb.*;
 
 import java.util.ArrayList;
 import java.util.List;

job-core/src/test/java/com/lts/job/core/failstore/rocksdb/RocksdbFailStoreTest.java → lts-core/src/test/java/com/lts/job/core/failstore/rocksdb/RocksdbFailStoreTest.java

@@ -7,13 +7,16 @@
 import com.lts.job.core.domain.KVPair;
 import com.lts.job.core.failstore.FailStore;
 import com.lts.job.core.failstore.FailStoreException;
+import com.lts.job.core.failstore.berkeleydb.BerkeleydbFailStore;
 import com.lts.job.core.util.CollectionUtils;
 import com.lts.job.core.util.JSONUtils;
 import org.junit.Before;
 import org.junit.Test;
 
 import java.util.List;
 
+import static org.junit.Assert.*;
+
 /**
  * Robert HG (254963746@qq.com) on 5/27/15.
  */

job-core/src/test/java/com/lts/job/core/file/FileLockTest.java → lts-core/src/test/java/com/lts/job/core/file/FileLockTest.java

@@ -2,6 +2,8 @@
 
 import java.io.IOException;
 
+import static org.junit.Assert.*;
+
 /**
  * Robert HG (254963746@qq.com) on 5/27/15.
  */

job-example/pom.xml → lts-example/pom.xml

@@ -3,53 +3,53 @@
          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
-        <artifactId>job-parent</artifactId>
+        <artifactId>lts-parent</artifactId>
         <groupId>com.lts</groupId>
-        <version>1.4.2-SNAPSHOT</version>
+        <version>1.5-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
-    <artifactId>job-example</artifactId>
+    <artifactId>lts-example</artifactId>
 
     <dependencies>
         <dependency>
             <groupId>com.lts</groupId>
-            <artifactId>job-tracker</artifactId>
+            <artifactId>lts-job-tracker</artifactId>
             <version>${project.version}</version>
         </dependency>
         <dependency>
             <groupId>com.lts</groupId>
-            <artifactId>job-task-tracker</artifactId>
+            <artifactId>lts-task-tracker</artifactId>
             <version>${project.version}</version>
         </dependency>
         <dependency>
             <groupId>com.lts</groupId>
-            <artifactId>job-client</artifactId>
+            <artifactId>lts-job-client</artifactId>
             <version>${project.version}</version>
         </dependency>
 
         <dependency>
             <groupId>com.lts</groupId>
-            <artifactId>job-logger-mongo</artifactId>
+            <artifactId>lts-logger-mongo</artifactId>
             <version>${project.version}</version>
         </dependency>
         <dependency>
             <groupId>com.lts</groupId>
-            <artifactId>job-logger-console</artifactId>
+            <artifactId>lts-logger-console</artifactId>
             <version>${project.version}</version>
         </dependency>
         <dependency>
             <groupId>com.lts</groupId>
-            <artifactId>job-logger-mysql</artifactId>
+            <artifactId>lts-logger-mysql</artifactId>
             <version>${project.version}</version>
         </dependency>
         <dependency>
             <groupId>com.lts</groupId>
-            <artifactId>job-queue-mongo</artifactId>
+            <artifactId>lts-queue-mongo</artifactId>
             <version>${project.version}</version>
         </dependency>
         <dependency>
             <groupId>com.lts</groupId>
-            <artifactId>job-queue-mysql</artifactId>
+            <artifactId>lts-queue-mysql</artifactId>
             <version>${project.version}</version>
         </dependency>
         <dependency>
@@ -80,10 +80,6 @@
             <groupId>org.fusesource.leveldbjni</groupId>
             <artifactId>leveldbjni-all</artifactId>
         </dependency>
-        <dependency>
-            <groupId>com.alibaba</groupId>
-            <artifactId>druid</artifactId>
-        </dependency>
         <dependency>
             <groupId>junit</groupId>
             <artifactId>junit</artifactId>

job-example/src/main/java/com/lts/job/example/api/JobTrackerTest.java → lts-example/src/main/java/com/lts/job/example/api/JobTrackerTest.java

@@ -12,9 +12,9 @@ public class JobTrackerTest {
     public static void main(String[] args) {
 
         // 1. 使用mongo做任务队列
-        testMongoQueue();
+//        testMongoQueue();
 //         2. 使用mysql做任务队列
-//        testMysqlQueue();
+        testMysqlQueue();
     }
 
     /**

job-extensions/job-ext-spring/pom.xml → lts-extensions/lts-ext-spring/pom.xml

@@ -5,7 +5,7 @@
     <parent>
         <artifactId>job-extensions</artifactId>
         <groupId>com.lts</groupId>
-        <version>1.4.2-SNAPSHOT</version>
+        <version>1.5-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
@@ -28,19 +28,19 @@
         </dependency>
         <dependency>
             <groupId>com.lts</groupId>
-            <artifactId>job-client</artifactId>
+            <artifactId>lts-job-client</artifactId>
             <version>${project.version}</version>
             <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>com.lts</groupId>
-            <artifactId>job-task-tracker</artifactId>
+            <artifactId>lts-task-tracker</artifactId>
             <version>${project.version}</version>
             <scope>provided</scope>
         </dependency>
         <dependency>
             <groupId>com.lts</groupId>
-            <artifactId>job-tracker</artifactId>
+            <artifactId>lts-job-tracker</artifactId>
             <version>${project.version}</version>
             <scope>provided</scope>
         </dependency>

job-extensions/pom.xml → lts-extensions/pom.xml

@@ -3,9 +3,9 @@
          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
-        <artifactId>job-parent</artifactId>
+        <artifactId>lts-parent</artifactId>
         <groupId>com.lts</groupId>
-        <version>1.4.2-SNAPSHOT</version>
+        <version>1.5-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <packaging>pom</packaging>

job-client/pom.xml → lts-job-client/pom.xml

@@ -3,18 +3,18 @@
          xmlns="http://maven.apache.org/POM/4.0.0"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
-        <artifactId>job-parent</artifactId>
+        <artifactId>lts-parent</artifactId>
         <groupId>com.lts</groupId>
-        <version>1.4.2-SNAPSHOT</version>
+        <version>1.5-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
-    <artifactId>job-client</artifactId>
+    <artifactId>lts-job-client</artifactId>
 
     <dependencies>
         <dependency>
             <groupId>com.lts</groupId>
-            <artifactId>job-core</artifactId>
+            <artifactId>lts-core</artifactId>
             <version>${project.version}</version>
         </dependency>
     </dependencies>

job-client/src/main/java/com/lts/job/client/RetryJobClient.java → lts-job-client/src/main/java/com/lts/job/client/RetryJobClient.java

@@ -8,6 +8,7 @@
 import com.lts.job.core.domain.Job;
 import com.lts.job.core.support.RetryScheduler;
 import com.lts.job.core.util.JSONUtils;
+import sun.security.util.Resources_es;
 
 import java.util.Arrays;
 import java.util.List;

job-tracker/pom.xml → lts-job-tracker/pom.xml

@@ -3,28 +3,28 @@
          xmlns="http://maven.apache.org/POM/4.0.0"
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
     <parent>
-        <artifactId>job-parent</artifactId>
+        <artifactId>lts-parent</artifactId>
         <groupId>com.lts</groupId>
-        <version>1.4.2-SNAPSHOT</version>
+        <version>1.5-SNAPSHOT</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
-    <artifactId>job-tracker</artifactId>
+    <artifactId>lts-job-tracker</artifactId>
 
     <dependencies>
         <dependency>
             <groupId>com.lts</groupId>
-            <artifactId>job-core</artifactId>
+            <artifactId>lts-core</artifactId>
             <version>${project.version}</version>
         </dependency>
         <dependency>
             <groupId>com.lts</groupId>
-            <artifactId>job-queue-api</artifactId>
+            <artifactId>lts-queue-api</artifactId>
             <version>${project.version}</version>
         </dependency>
         <dependency>
             <groupId>com.lts</groupId>
-            <artifactId>job-logger-api</artifactId>
+            <artifactId>lts-logger-api</artifactId>
             <version>${project.version}</version>
         </dependency>
     </dependencies>

job-tracker/src/main/java/com/lts/job/tracker/domain/JobTrackerApplication.java → lts-job-tracker/src/main/java/com/lts/job/tracker/domain/JobTrackerApplication.java

@@ -9,10 +9,10 @@
 import com.lts.job.queue.JobFeedbackQueue;
 import com.lts.job.tracker.channel.ChannelManager;
 import com.lts.job.tracker.id.IdGenerator;
-import com.lts.job.tracker.support.OldDataHandler;
-import com.lts.job.tracker.support.checker.ExecutingDeadJobChecker;
 import com.lts.job.tracker.support.cluster.JobClientManager;
+import com.lts.job.tracker.support.OldDataHandler;
 import com.lts.job.tracker.support.cluster.TaskTrackerManager;
+import com.lts.job.tracker.support.checker.ExecutingDeadJobChecker;
 
 /**
  * JobTracker Application