Async job schedule #1030

zhoney · 2020-06-11T09:33:29Z

No description provided.

javeme · 2020-06-15T07:44:01Z

hugegraph-api/src/main/java/com/baidu/hugegraph/api/job/TaskAPI.java

@@ -161,11 +161,11 @@ public void delete(@Context GraphManager manager,

        TaskScheduler scheduler = graph(manager, graph).taskScheduler();
        HugeTask<?> task = scheduler.task(IdGenerator.of(id));
-        if (!task.completed() && scheduler.cancel(task)) {
+        if (!task.completed() &&
+            (scheduler.cancel(task) || task.cancelling())) {


move task.cancelling() before scheduler.cancel(task)

scheduler.cancel(task) will change task status to CANCELLING

hugegraph-api/src/main/java/com/baidu/hugegraph/api/job/TaskAPI.java

hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java

javeme · 2020-06-15T07:49:32Z

hugegraph-core/src/main/java/com/baidu/hugegraph/task/HugeTask.java

+            ServerInfoManager manager = scheduler.serverManager();
+            HugeServerInfo serverInfo = manager.serverInfo();
+            serverInfo.load(serverInfo.load() - this.load);
+            manager.save(serverInfo);


ensure don't throw any exception out of done(), add a method ServerInfoManager.decreaseLoad(int) and just log in that method if error

javeme · 2020-06-15T07:54:49Z

hugegraph-core/src/main/java/com/baidu/hugegraph/task/TaskManager.java

    }

    public void addScheduler(HugeGraphParams graph) {
        E.checkArgumentNotNull(graph, "The graph can't be null");
        ExecutorService task = this.taskExecutor;
        ExecutorService db = this.dbExecutor;
-        this.schedulers.put(graph, new StandardTaskScheduler(graph, task, db));
+        TaskScheduler taskScheduler = new StandardTaskScheduler(graph, task,


rename to scheduler to not wrap line

hugegraph-core/src/main/java/com/baidu/hugegraph/cluster/ServerInfoManager.java

javeme · 2020-06-15T08:07:21Z

hugegraph-core/src/main/java/com/baidu/hugegraph/cluster/HugeServerInfo.java

+            this.graph.schemaTransaction().addVertexLabel(label);
+
+            // Create index
+            this.createIndexLabel(label, P.ROLE);


seems no need to query by role

query by role in initServerInfo of com/baidu/hugegraph/cluster/ServerInfoManager.java

javeme · 2020-06-15T08:10:38Z

hugegraph-core/src/main/java/com/baidu/hugegraph/cluster/HugeServerInfo.java

+
+public class HugeServerInfo {
+
+    public static final int MAX_LOAD = 1000;


report MAX_LOAD by itself and add maxLoad field

javeme · 2020-06-15T08:13:17Z

hugegraph-core/src/main/java/com/baidu/hugegraph/task/TaskManager.java

+                             1, TASK_SCHEDULER);
+        // Start after 10s waiting for HugeGraphServer startup
+        this.taskScheduler.scheduleWithFixedDelay(this::periodicJob,
+                                                  10L, 3, TimeUnit.SECONDS);


define 3 as const var SCHEDULE_PERIOD

javeme · 2020-06-15T08:14:32Z

hugegraph-core/src/main/java/com/baidu/hugegraph/cluster/HugeServerInfo.java

+public class HugeServerInfo {
+
+    public static final int MAX_LOAD = 1000;
+    public static final long EXPIRED_INTERVAL = 5000L;


SCHEDULE_PERIOD * 3

javeme · 2020-06-22T17:37:48Z

hugegraph-api/src/main/java/com/baidu/hugegraph/config/ServerOptions.java

+                    "server.role",
+                    "The role of nodes in the cluster, available type are " +
+                    "[master, worker]",
+                    disallowEmpty(),


use allowValues

javeme · 2020-06-22T17:38:18Z

hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedSchemaTransaction.java

@@ -47,16 +47,15 @@
    private EventListener storeEventListener;
    private EventListener cacheEventListener;

-    private final Map<HugeType, Boolean> cachedTypes;
+    private static final Map<HugeGraphParams, CachedTypes> CACHED_TYPES =


Map<String, CachedTypes>

hugegraph-core/src/main/java/com/baidu/hugegraph/task/TaskManager.java

javeme · 2020-06-22T17:42:37Z

hugegraph-core/src/main/java/com/baidu/hugegraph/task/TaskManager.java

+                // Update server heartbeat
+                server.heartbeat();
+
+                // Master schedule tasks not found suitable server when created


improve "not found suitable server when created"

javeme · 2020-06-23T07:03:24Z

hugegraph-core/src/main/java/com/baidu/hugegraph/StandardHugeGraph.java

@@ -267,6 +267,8 @@ public void truncateBackend() {

        this.storeProvider.truncate();
        this.storeProvider.initSystemInfo(this);
+        this.serverStarted(this.serverManager.serverId().asString(),


why not change the parameter type

javeme · 2020-06-23T07:03:47Z

hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedSchemaTransaction.java

@@ -47,16 +47,15 @@
    private EventListener storeEventListener;
    private EventListener cacheEventListener;

-    private final Map<HugeType, Boolean> cachedTypes;
+    private static final Map<String, CachedTypes> CACHED_TYPES =
+            new ConcurrentHashMap<>();


prefer align with CACHED_TYPES

javeme · 2020-06-23T07:04:12Z

hugegraph-core/src/main/java/com/baidu/hugegraph/backend/cache/CachedSchemaTransaction.java

            }
            return results;
        }
    }
+
+    private static class CachedTypes
+            extends ConcurrentHashMap<HugeType, Boolean> {}


align with class

javeme · 2020-06-23T07:07:02Z

hugegraph-core/src/main/java/com/baidu/hugegraph/task/HugeTask.java

-                                    e, e.getMessage());
-        } finally {
-            ((StandardTaskScheduler) task.scheduler()).remove(id);
+            // ignore


don't ignore exception here

javeme · 2020-06-23T07:09:13Z

hugegraph-core/src/main/java/com/baidu/hugegraph/task/HugeTask.java

-                if (task.status.code() > TaskStatus.QUEUED.code()) {
-                    task.get();
-                }
+                task = this.scheduler().task(this.id());


should sleep between tow times of task-read

javeme · 2020-06-23T07:09:37Z

hugegraph-core/src/main/java/com/baidu/hugegraph/task/StandardTaskScheduler.java

@@ -322,12 +325,13 @@ private void unlistenChanges() {
        HugeServerInfo server;
        do {
            Iterator<HugeServerInfo> servers = this.serverManager()
-                                                   .serverInfos(100L, page);
+                                                   .serverInfos(10L, page);


add const var

javeme · 2020-06-23T07:10:23Z

hugegraph-core/src/main/java/com/baidu/hugegraph/task/StandardTaskScheduler.java

@@ -388,9 +392,10 @@ protected ServerInfoManager serverManager() {
        return this.graph.serverManager();
    }

-    protected void remove(Id id) {
+    public void remove(Id id) {


keep protected

javeme · 2020-06-23T07:13:33Z

hugegraph-test/src/main/java/com/baidu/hugegraph/core/VertexLabelCoreTest.java

+        task = scheduler.task(id);
+        Assert.assertEquals(TaskStatus.FAILED, task.status());
+        Assert.assertContains("Not allowed to remove vertex label 'book' " +
+                              "because the edge label 'write' still link with" +


wrap line after "link "

javeme · 2020-06-23T07:14:58Z

hugegraph-test/src/main/java/com/baidu/hugegraph/core/TaskCoreTest.java

+        @Override
+        public void done() {
+            StandardTaskScheduler scheduler = Whitebox.getInternalState(
+                                              this.task(), "scheduler");


why not call this.task().scheduler()

HugeTask doesn't have scheduler()
Already change to this.graph().taskScheduler()

javeme · 2020-06-23T07:16:22Z

hugegraph-core/src/main/java/com/baidu/hugegraph/task/StandardTaskScheduler.java

        E.checkArgumentNotNull(task, "Task can't be null");
        if (!this.serverManager().master()) {
            return;
        }
        if (!task.completed()) {
            // The task scheduled to workers, waiting for worker cancel
+            task = this.task(task.id());


why read again

javeme · 2020-06-23T07:21:28Z

hugegraph-core/src/main/java/com/baidu/hugegraph/task/StandardTaskScheduler.java

    }

-    private <V> Future<?> submitTask(HugeTask<V> task) {
+    public  <V> Future<?> submitTask(HugeTask<V> task) {


keep private

Linary · 2020-06-22T08:24:44Z

hugegraph-api/src/main/java/com/baidu/hugegraph/config/ServerOptions.java

+    public static final ConfigOption<String> SERVER_ROLE =
+            new ConfigOption<>(
+                    "server.role",
+                    "The role of nodes in the cluster, available type are " +


Linary · 2020-06-22T08:29:04Z

hugegraph-core/src/main/java/com/baidu/hugegraph/cluster/HugeServerInfo.java

+    protected Object[] asArray() {
+        E.checkState(this.id != null, "Server id can't be null");
+
+        List<Object> list = new ArrayList<>(8);


initialCapacity should be 10, actually DEFAULT_CAPACITY is 10

javeme · 2020-06-23T07:24:34Z

hugegraph-core/src/main/java/com/baidu/hugegraph/task/TaskManager.java


+    public static final int SCHEDULE_PERIOD = 3;


add comment: Unit second

javeme · 2020-06-23T07:31:09Z

hugegraph-test/src/main/java/com/baidu/hugegraph/core/MultiGraphsTest.java

        for (String graphName : graphNames) {
            config.setProperty(CoreOptions.STORE.name(), graphName);
+            if (backend.equals("rocksdb")) {
+                String dataPath = data + "/" + graphName;
+                config.setProperty(RocksDBOptions.DATA_PATH.name(), dataPath);


prefer set DATA_PATH before openGraphs() in each test

javeme · 2020-06-23T07:34:19Z

hugegraph-core/src/main/java/com/baidu/hugegraph/task/StandardTaskScheduler.java

                                     .enableLabelIndex(true)
                                     .build();
            this.params().schemaTransaction().addVertexLabel(label);

            // Create index
            this.createIndexLabel(label, P.STATUS);
+            this.createIndexLabel(label, P.SERVER);


unneeded index

javeme · 2020-06-23T07:37:26Z

hugegraph-core/src/main/java/com/baidu/hugegraph/task/StandardTaskScheduler.java

+
+    private synchronized <V> HugeServerInfo pickWorker(HugeTask<V> task) {
+        HugeServerInfo master = null;
+        HugeServerInfo minServer = null;


minServer -- the meaning is incomprehensible

hugegraph-core/src/main/java/com/baidu/hugegraph/task/StandardTaskScheduler.java

javeme · 2020-06-23T07:41:19Z

hugegraph-core/src/main/java/com/baidu/hugegraph/task/StandardTaskScheduler.java

@@ -162,6 +170,7 @@ private void unlistenChanges() {
    public <V> void restoreTasks() {
        boolean supportsPaging = this.graph().backendStoreFeatures()
                                             .supportsQueryByPage();
+        Id server = this.serverManager().serverId();


rename to selfServer

javeme · 2020-06-23T07:42:36Z

hugegraph-core/src/main/java/com/baidu/hugegraph/config/CoreOptions.java

@@ -114,7 +114,7 @@ public static synchronized CoreOptions instance() {
                    "Timeout in seconds for waiting for the task to complete," +
                    "such as when truncating or clearing the backend.",
                    rangeInt(0L, Long.MAX_VALUE),
-                    10L
+                    30L


too much time that client may time out, 20s is ok

javeme · 2020-06-23T07:46:08Z

hugegraph-core/src/main/java/com/baidu/hugegraph/cluster/ServerInfoManager.java

+        HugeServerInfo serverInfo = new HugeServerInfo(server, nodeRole);
+        serverInfo.maxLoad(this.calcMaxLoad());
+        this.serverId = serverInfo.id();
+        this.serverRole = nodeRole;


move serverId and serverRole assignment to the begin of this method after refactor initServerInfo params

javeme · 2020-06-23T07:46:38Z

hugegraph-core/src/main/java/com/baidu/hugegraph/cluster/ServerInfoManager.java

+        if (nodeRole.master()) {
+            String page = PAGE_NONE;
+            do {
+                Iterator<HugeServerInfo> servers = this.serverInfos(10L, page);


add const var

javeme · 2020-06-23T12:15:02Z

hugegraph-test/src/main/java/com/baidu/hugegraph/core/MultiGraphsTest.java

        for (String graphName : graphNames) {
            config.setProperty(CoreOptions.STORE.name(), graphName);
+            if (backend.equals("rocksdb")) {
+                String dataPath = data + "/" + graphName;
+                config.setProperty(RocksDBOptions.DATA_PATH.name(), dataPath);


javeme · 2020-06-23T12:24:17Z

hugegraph-core/src/main/java/com/baidu/hugegraph/task/HugeTask.java

-            }
-            throw new HugeException("Async task failed with error: %s",
-                                    cause, cause.getMessage());
+            this.scheduler().waitUntilTaskCompleted(this.id(), 20L, 10);


seems can delete syncWait() method

javeme · 2020-06-23T12:27:36Z

hugegraph-core/src/main/java/com/baidu/hugegraph/task/StandardTaskScheduler.java

+                }
+                HugeServerInfo server = this.pickWorker(task);
+                if (server == null) {
+                    LOG.debug("The master can not find suitable server to " +


javeme · 2020-06-23T12:56:41Z

hugegraph-test/src/main/java/com/baidu/hugegraph/core/TaskCoreTest.java

@@ -325,6 +316,7 @@ public void testGremlinJobWithFailure() throws TimeoutException {
               .job(new GremlinJob());
        HugeTask<Object> task = builder.schedule();
        scheduler.waitUntilTaskCompleted(task.id(), 10);
+        task = scheduler.task(task.id());


task = scheduler.waitUntilTaskCompleted()

javeme · 2020-06-23T13:08:17Z

test errors:

[ERROR] Tests run: 288, Failures: 0, Errors: 2, Skipped: 1, Time elapsed: 289.868 s <<< FAILURE! - in com.baidu.hugegraph.unit.UnitTestSuite
[ERROR] testResetCachedAllIfReachedCapacity(com.baidu.hugegraph.unit.cache.CachedSchemaTransactionTest)  Time elapsed: 0.137 s  <<< ERROR!
java.lang.RuntimeException: Unable to get internal state on field 'cachedTypes' of com.baidu.hugegraph.backend.cache.CachedSchemaTransaction@2d7ee91c
	at com.baidu.hugegraph.unit.cache.CachedSchemaTransactionTest.testResetCachedAllIfReachedCapacity(CachedSchemaTransactionTest.java:186)
Caused by: java.lang.RuntimeException: Not declared field 'cachedTypes' in class 'Object'
	at com.baidu.hugegraph.unit.cache.CachedSchemaTransactionTest.testResetCachedAllIfReachedCapacity(CachedSchemaTransactionTest.java:186)
[ERROR] com.baidu.hugegraph.unit.core.SecurityManagerTest  Time elapsed: 0.402 s  <<< ERROR!
com.baidu.hugegraph.HugeException: The worker can't schedule task
	at com.baidu.hugegraph.unit.core.SecurityManagerTest.runGremlinJob(SecurityManagerTest.java:311)
	at com.baidu.hugegraph.unit.core.SecurityManagerTest.init(SecurityManagerTest.java:62)

houzhizhen · 2020-06-24T01:29:47Z

hugegraph-core/src/main/java/com/baidu/hugegraph/cluster/HugeServerInfo.java

-        if (this.updateTime.getTime() + 5000L < now ||
-            this.load() + task.load() > MAX_LOAD) {
+    public <V> boolean suitableFor(HugeTask<V> task) {
+        if (this.alive() || this.load() + task.load() > MAX_LOAD) {


seem contrary to common sense, it should accept new task if it is alive

houzhizhen · 2020-06-24T01:40:21Z

hugegraph-core/src/main/java/com/baidu/hugegraph/cluster/ServerInfoManager.java

+            if (servers.hasNext()) {
+                existed = servers.next();
+                E.checkArgument(!existed.alive(),
+                                "Already existed master '%s' in current " +


Exception information may not correct.

zhoney · 2020-06-24T08:55:40Z

[ERROR] testList(com.baidu.hugegraph.api.EdgeLabelApiTest)  Time elapsed: 0.286 s  <<< FAILURE!
java.lang.AssertionError: Response with status 400 and content {"exception":"class java.lang.IllegalArgumentException","message":"Undefined source vertex label 'person' in edge label 'created'","cause":""} expected:<201> but was:<400>
	at com.baidu.hugegraph.api.EdgeLabelApiTest.testList(EdgeLabelApiTest.java:115)
[ERROR] testBatchUpdate(com.baidu.hugegraph.api.EdgeApiTest)  Time elapsed: 0.21 s  <<< ERROR!
com.baidu.hugegraph.HugeException: Failed to get vertex id: {"exception":"class java.lang.IllegalArgumentException","message":"Undefined vertex label: 'person'","cause":""}
	at com.baidu.hugegraph.api.EdgeApiTest.testBatchUpdate(EdgeApiTest.java:65)

zhoney · 2020-06-26T05:57:12Z

hbase

[ERROR] testGremlinJobAndCancel(com.baidu.hugegraph.core.TaskCoreTest)  Time elapsed: 3.219 s  <<< FAILURE!
java.lang.AssertionError: expected:<CANCELLED> but was:<CANCELLING>
	at com.baidu.hugegraph.core.TaskCoreTest.testGremlinJobAndCancel(TaskCoreTest.java:457)
[ERROR] testGremlinJobAndRestore(com.baidu.hugegraph.core.TaskCoreTest)  Time elapsed: 0.066 s  <<< ERROR!
com.baidu.hugegraph.HugeException: Failed to update/query TaskStore: java.util.concurrent.ExecutionException: java.lang.IllegalStateException: Can't delete incomplete task '1760' in status CANCELLING
	at com.baidu.hugegraph.core.TaskCoreTest.setup(TaskCoreTest.java:64)
Caused by: java.util.concurrent.ExecutionException: java.lang.IllegalStateException: Can't delete incomplete task '1760' in status CANCELLING
	at com.baidu.hugegraph.core.TaskCoreTest.setup(TaskCoreTest.java:64)
Caused by: java.lang.IllegalStateException: Can't delete incomplete task '1760' in status CANCELLING

javeme · 2020-06-27T10:04:04Z

[ERROR] Tests run: 289, Failures: 0, Errors: 2, Skipped: 1, Time elapsed: 277.599 s <<< FAILURE! - in com.baidu.hugegraph.unit.UnitTestSuite
[ERROR] com.baidu.hugegraph.unit.core.SecurityManagerTest  Time elapsed: 0.403 s  <<< ERROR!
com.baidu.hugegraph.HugeException: Failed to update/query server info: java.util.concurrent.ExecutionException: java.lang.IllegalStateException: Invalid PageState 'null'
	at com.baidu.hugegraph.unit.core.SecurityManagerTest.loadGraph(SecurityManagerTest.java:330)
	at com.baidu.hugegraph.unit.core.SecurityManagerTest.init(SecurityManagerTest.java:63)
Caused by: java.util.concurrent.ExecutionException: java.lang.IllegalStateException: Invalid PageState 'null'
	at com.baidu.hugegraph.unit.core.SecurityManagerTest.loadGraph(SecurityManagerTest.java:330)
	at com.baidu.hugegraph.unit.core.SecurityManagerTest.init(SecurityManagerTest.java:63)
Caused by: java.lang.IllegalStateException: Invalid PageState 'null'
[ERROR] com.baidu.hugegraph.unit.core.SecurityManagerTest  Time elapsed: 0.404 s  <<< ERROR!
java.lang.NullPointerException
	at com.baidu.hugegraph.unit.core.SecurityManagerTest.clear(SecurityManagerTest.java:71)

zhoney · 2020-06-28T08:44:49Z

[ERROR] testCreateMultiGraphs(com.baidu.hugegraph.core.MultiGraphsTest)  Time elapsed: 96.731 s  <<< FAILURE!
java.lang.AssertionError: java.lang.IllegalStateException: Ensure tx closed in all threads when closing graph 'g12345678901234567890123456789012345678901234567'
	at com.baidu.hugegraph.core.MultiGraphsTest.destoryGraphs(MultiGraphsTest.java:344)
	at com.baidu.hugegraph.core.MultiGraphsTest.testCreateMultiGraphs(MultiGraphsTest.java:62)
[ERROR] testCopySchemaWithMultiGraphsWithConflict(com.baidu.hugegraph.core.MultiGraphsTest)  Time elapsed: 96.539 s  <<< FAILURE!
java.lang.AssertionError: java.lang.IllegalStateException: Ensure tx closed in all threads when closing graph 'schema_g2'
	at com.baidu.hugegraph.core.MultiGraphsTest.destoryGraphs(MultiGraphsTest.java:344)
	at com.baidu.hugegraph.core.MultiGraphsTest.testCopySchemaWithMultiGraphsWithConflict(MultiGraphsTest.java:209)

codecov · 2020-06-29T04:18:04Z

Codecov Report

Merging #1030 into master will increase coverage by 9.39%.
The diff coverage is 72.78%.

@@             Coverage Diff              @@
##             master    #1030      +/-   ##
============================================
+ Coverage     59.92%   69.32%   +9.39%     
- Complexity     4571     5538     +967     
============================================
  Files           328      331       +3     
  Lines         26400    26852     +452     
  Branches       3773     3838      +65     
============================================
+ Hits          15820    18614    +2794     
+ Misses         8857     6422    -2435     
- Partials       1723     1816      +93

Impacted Files	Coverage Δ	Complexity Δ
...a/com/baidu/hugegraph/auth/HugeGraphAuthProxy.java	`4.22% <0.00%> (-0.05%)`	`0.00 <0.00> (ø)`
...e/src/main/java/com/baidu/hugegraph/HugeGraph.java	`65.30% <ø> (ø)`	`9.00 <0.00> (ø)`
...va/com/baidu/hugegraph/type/define/SerialEnum.java	`64.28% <0.00%> (ø)`	`4.00 <0.00> (ø)`
...main/java/com/baidu/hugegraph/api/job/TaskAPI.java	`51.02% <33.33%> (-4.30%)`	`0.00 <0.00> (ø)`
...om/baidu/hugegraph/task/StandardTaskScheduler.java	`80.31% <68.14%> (-2.50%)`	`85.00 <26.00> (+24.00)`	⬇️
.../java/com/baidu/hugegraph/task/HugeServerInfo.java	`68.70% <68.70%> (ø)`	`18.00 <18.00> (?)`
...java/com/baidu/hugegraph/type/define/NodeRole.java	`68.75% <68.75%> (ø)`	`6.00 <6.00> (?)`
...in/java/com/baidu/hugegraph/core/GraphManager.java	`55.22% <71.42%> (+0.60%)`	`0.00 <0.00> (ø)`
...va/com/baidu/hugegraph/task/ServerInfoManager.java	`78.37% <78.37%> (ø)`	`32.00 <32.00> (?)`
...c/main/java/com/baidu/hugegraph/task/HugeTask.java	`72.32% <78.57%> (+4.55%)`	`79.00 <6.00> (+15.00)`
... and 156 more

Continue to review full report at Codecov.

Legend - Click here to learn more
Δ = absolute <relative> (impact), ø = not affected, ? = missing data
Powered by Codecov. Last update ac02492...c21e704. Read the comment docs.

Change-Id: I06478c6dcf64699aba336aa5c032337bd297783b

Change-Id: I39306140f3d4682b7de88bfa2b4277c622ca8560

Change-Id: I804eab3fe05807b8ca3482207268340bb1b09a71

Change-Id: I55ab99d497c13eb7803e76da3dcc7d9f8850c57b

Change-Id: Ie5dd63e0e977eda9dd0a795b34f0898252b45d2b

Change-Id: If48dac267863c6757de16eb4d6ea12f26cdc6087

Change-Id: Iccad390f86a6b5d5605f127fc49e50713cc264df

Change-Id: I016eebc92314e23b7cfd622dbca78df2f4ab63c2

Change-Id: Idf77f479c766aaa0a29b74fbd5ab92828ffebfba

Change-Id: Ia3b1c163f5f90c07a748b89c4be652ccc2ae6fd9

Change-Id: I2c2dcd5abe710f5684ddda13363b17414c050fb2

Change-Id: Id6960cfed0e4cce401b3e94546b86222a8c6583f

Change-Id: I273b4a3edf6607a1319e0d6b185c31a1c7f3c83b

Change-Id: Ib244ba6782d76edbc1547358f1b3c1cbc44168c0

Change-Id: I0e4cae424a788fcb0b61e7ee8708e4b6e05b27bb

Change-Id: I7e94e3111a735f5d25de8711321a92fb3ef45e20

2. wait task completed in delete tests of api test Change-Id: I5fc7f828c704b49b6caada37f0648ed9e54258ae

Change-Id: I663d8d22be72dba90d2a8413ef60463de1ad8363

Change-Id: I812a482100509d224bc3b26265121e2f007a837c

Change-Id: I13323f44edb5553e5cd1fc35280e1283c9aa16ba

2. change cassandra ttl type from int to long Change-Id: I6780c7829cfeae5dedb60bfc50a9dbed5657f29d

Change-Id: I5a3b595ea399f38d29d544dbc76c87629490bfe5

Change-Id: Ica887e3c11d74389a7582d5a003d4436a7eb5ed9

Change-Id: I1e40fd6a10d6d2f069912ffa529ef5096c3f6bd3

Change-Id: Ib11947591ef312a8f38c0831013d06e404823b58

javeme · 2020-06-30T09:19:55Z

hugegraph-core/src/main/java/com/baidu/hugegraph/task/StandardTaskScheduler.java

        } while (page != null);
    }

    protected <V> void executeTasksForWorker(Id server) {
-        String page = PageInfo.PAGE_NONE;
+        boolean supportsPaging = this.graph().backendStoreFeatures()
+                                     .supportsQueryByPage();


define a field or method supportsPaging

javeme · 2020-06-30T09:23:57Z

hugegraph-api/src/main/java/com/baidu/hugegraph/config/ServerOptions.java

+                    "server.role",
+                    "The role of nodes in the cluster, available types are " +
+                    "[master, worker]",
+                    allowValues("master", "worker"),


add "computer"

javeme · 2020-06-30T09:36:00Z

hugegraph-core/src/main/java/com/baidu/hugegraph/task/HugeTask.java

@@ -591,19 +636,22 @@ private void checkPropertySize(int propertyLength, String propertyName) {
    }

    public void syncWait() {
+        long timeout = ((HugeConfig) this.scheduler.graph().configuration())


not supported by auth graph

javeme · 2020-06-30T09:37:20Z

hugegraph-core/src/main/java/com/baidu/hugegraph/task/HugeTask.java

-            }
-            throw new HugeException("Async task failed with error: %s",
-                                    cause, cause.getMessage());
+            this.scheduler().waitUntilTaskCompleted(this.id(), timeout, 10L);


can read timeout in scheduler.waitUntilTaskCompleted()

javeme · 2020-06-30T09:48:27Z

hugegraph-core/src/main/java/com/baidu/hugegraph/task/TaskManager.java

+        };
+
+        try {
+            this.schedulerExecutor.submit(closeTx);


ensure completed, call this.schedulerExecutor.submit(closeTx).get()

Change-Id: Iff72fc89737c8b05ff22ffcc86cc9c00bc555144

javeme · 2020-06-30T12:06:47Z

hugegraph-api/src/main/java/com/baidu/hugegraph/config/ServerOptions.java

@@ -62,7 +62,7 @@ public static synchronized ServerOptions instance() {
                    "server.role",
                    "The role of nodes in the cluster, available types are " +
                    "[master, worker]",
-                    allowValues("master", "worker"),
+                    allowValues("master", "worker", "computer"),


also update line 64

Change-Id: Iaad40388421a650761f01699eedcc4f5ad002f28

zhoney · 2020-06-30T12:13:12Z

异步任务等待10ms时，CI测试时间每个延长3分钟左右

javeme · 2020-07-08T09:41:17Z

improved by https://github.com/hugegraph/hugegraph/pull/1079:

javeme requested changes Jun 15, 2020

View reviewed changes

zhoney force-pushed the async-job-schedule branch from f277c9f to e96baf2 Compare June 22, 2020 12:04

javeme reviewed Jun 22, 2020

View reviewed changes

zhoney force-pushed the async-job-schedule branch 3 times, most recently from 29f2258 to e68cc2a Compare June 23, 2020 03:57

javeme requested changes Jun 23, 2020

View reviewed changes

Linary requested changes Jun 23, 2020

View reviewed changes

javeme requested changes Jun 23, 2020

View reviewed changes

javeme reviewed Jun 23, 2020

View reviewed changes

houzhizhen reviewed Jun 24, 2020

View reviewed changes

houzhizhen previously approved these changes Jun 24, 2020

View reviewed changes

zhoney dismissed houzhizhen’s stale review via 7e6632e June 24, 2020 08:04

zhoney added 9 commits June 29, 2020 15:22

first version schedule async job on worker

8ddd1e0

Change-Id: I06478c6dcf64699aba336aa5c032337bd297783b

close taskScheduler when shutdown

bd4377e

Change-Id: I39306140f3d4682b7de88bfa2b4277c622ca8560

implement Server info schema and r/w

9e7fbd6

Change-Id: I804eab3fe05807b8ca3482207268340bb1b09a71

support cancel and restore tasks in cluster

0937abb

Change-Id: I55ab99d497c13eb7803e76da3dcc7d9f8850c57b

master submit or cancel tasks immediately

e63045d

Change-Id: Ie5dd63e0e977eda9dd0a795b34f0898252b45d2b

master schedule task periodic and decrease server load when task done

e5fc6f1

Change-Id: If48dac267863c6757de16eb4d6ea12f26cdc6087

only schedule and cancel tasks in periodic job for master

a0267b4

Change-Id: Iccad390f86a6b5d5605f127fc49e50713cc264df

check server existed

8a0bab3

Change-Id: I016eebc92314e23b7cfd622dbca78df2f4ab63c2

add default server config in rest-server.properties

50e3e3e

Change-Id: Idf77f479c766aaa0a29b74fbd5ab92828ffebfba

zhoney added 16 commits June 29, 2020 15:22

fix CachedSchemaTransaciton cachedTypes error

c88dd47

Change-Id: Ia3b1c163f5f90c07a748b89c4be652ccc2ae6fd9

improve

7e104d6

Change-Id: I2c2dcd5abe710f5684ddda13363b17414c050fb2

improve

3c718d5

Change-Id: Id6960cfed0e4cce401b3e94546b86222a8c6583f

improve

f794a05

Change-Id: I273b4a3edf6607a1319e0d6b185c31a1c7f3c83b

improve

fe576b9

Change-Id: Ib244ba6782d76edbc1547358f1b3c1cbc44168c0

api test set "check_exist" to false in schema create

3597b63

Change-Id: I0e4cae424a788fcb0b61e7ee8708e4b6e05b27bb

wait task completed in api test teardown

d08d9d3

Change-Id: I7e94e3111a735f5d25de8711321a92fb3ef45e20

1. add dbexeutor for ServerInfoManager

e820879

2. wait task completed in delete tests of api test Change-Id: I5fc7f828c704b49b6caada37f0648ed9e54258ae

improve

85b5dc3

Change-Id: I663d8d22be72dba90d2a8413ef60463de1ad8363

shutdown serverinfodbexecutor when shutdown

2e12b3c

Change-Id: I812a482100509d224bc3b26265121e2f007a837c

improve

5dd9c9a

Change-Id: I13323f44edb5553e5cd1fc35280e1283c9aa16ba

1. make server info query compatible with memory(no paging)

0a81f86

2. change cassandra ttl type from int to long Change-Id: I6780c7829cfeae5dedb60bfc50a9dbed5657f29d

improve

44ba74b

Change-Id: I5a3b595ea399f38d29d544dbc76c87629490bfe5

improve

e7f2e02

Change-Id: Ica887e3c11d74389a7582d5a003d4436a7eb5ed9

improve

2d6cdcb

Change-Id: I1e40fd6a10d6d2f069912ffa529ef5096c3f6bd3

improve

d456f91

Change-Id: Ib11947591ef312a8f38c0831013d06e404823b58

javeme reviewed Jun 30, 2020

View reviewed changes

zhoney force-pushed the async-job-schedule branch from abf01c7 to c2dab85 Compare June 30, 2020 10:16

houzhizhen previously approved these changes Jun 30, 2020

View reviewed changes

wait close scheduler tx done

4d073d3

Change-Id: Iff72fc89737c8b05ff22ffcc86cc9c00bc555144

zhoney dismissed houzhizhen’s stale review via 4d073d3 June 30, 2020 11:31

zhoney force-pushed the async-job-schedule branch from c2dab85 to 4d073d3 Compare June 30, 2020 11:31

javeme reviewed Jun 30, 2020

View reviewed changes

improve

c21e704

Change-Id: Iaad40388421a650761f01699eedcc4f5ad002f28

javeme approved these changes Jul 1, 2020

View reviewed changes

Linary approved these changes Jul 2, 2020

View reviewed changes

zhoney merged commit 530707d into master Jul 2, 2020

zhoney deleted the async-job-schedule branch July 2, 2020 02:51


		public class HugeServerInfo {

		public static final int MAX_LOAD = 1000;

Async job schedule #1030

Async job schedule #1030

Conversation

zhoney commented Jun 11, 2020

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

javeme commented Jun 23, 2020

Choose a reason for hiding this comment

Choose a reason for hiding this comment

zhoney commented Jun 24, 2020

zhoney commented Jun 26, 2020

javeme commented Jun 27, 2020

zhoney commented Jun 28, 2020

codecov bot commented Jun 29, 2020 • edited

Codecov Report

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

Choose a reason for hiding this comment

zhoney commented Jun 30, 2020

javeme commented Jul 8, 2020

codecov bot commented Jun 29, 2020 •

edited