Skip to content
Permalink
Browse files
Improve log info for files under dir (#223)
  • Loading branch information
Linary committed Jul 29, 2021
1 parent 960e358 commit 5e30894d2e1b8a404309f34f22724c64958449be
Showing 6 changed files with 26 additions and 18 deletions.
@@ -18,7 +18,7 @@ jobs:
env:
TRAVIS_DIR: assembly/travis
STATIC_DIR: assembly/static
COMMIT_ID: c0dff5b233e853716ca6f3f28a5cda05e6f3d639
COMMIT_ID: e65641517361d7da7ef1801489c15e3af7863a39
DB_USER: root
DB_PASS: root
DB_DATABASE: load_test
@@ -62,7 +62,7 @@
<dependency>
<groupId>com.baidu.hugegraph</groupId>
<artifactId>hugegraph-common</artifactId>
<version>1.8.8</version>
<version>1.8.9</version>
<exclusions>
<exclusion>
<groupId>com.google.guava</groupId>
@@ -73,7 +73,7 @@
<dependency>
<groupId>com.baidu.hugegraph</groupId>
<artifactId>hugegraph-client</artifactId>
<version>1.9.4</version>
<version>1.9.6</version>
</dependency>
<dependency>
<groupId>commons-io</groupId>
@@ -228,7 +228,7 @@ private void loadStructs(List<InputStruct> structs) {
* Let load task worked in pipeline mode
*/
private void loadStruct(InputStruct struct, InputReader reader) {
LOG.info("Start parsing '{}'", struct);
LOG.info("Start loading '{}'", struct);
LoadMetrics metrics = this.context.summary().metrics(struct);
metrics.startInFlight();

@@ -276,7 +276,7 @@ private void loadStruct(InputStruct struct, InputReader reader) {
}

metrics.stopInFlight();
LOG.info("Finish parsing '{}'", struct);
LOG.info("Finish loading '{}'", struct);
}

/**
@@ -23,7 +23,6 @@
import java.util.Map;
import java.util.concurrent.atomic.LongAdder;

import com.baidu.hugegraph.loader.HugeGraphLoader;
import com.baidu.hugegraph.loader.mapping.EdgeMapping;
import com.baidu.hugegraph.loader.mapping.ElementMapping;
import com.baidu.hugegraph.loader.mapping.InputStruct;
@@ -51,10 +50,10 @@ public LoadMetrics(InputStruct struct) {
this.vertexMetrics = new HashMap<>();
this.edgeMetrics = new HashMap<>();
for (VertexMapping mapping : struct.vertices()) {
vertexMetrics.put(mapping.label(), new Metrics());
this.vertexMetrics.put(mapping.label(), new Metrics());
}
for (EdgeMapping mapping : struct.edges()) {
edgeMetrics.put(mapping.label(), new Metrics());
this.edgeMetrics.put(mapping.label(), new Metrics());
}
}

@@ -95,17 +94,11 @@ public void stopInFlight() {
}

public void plusFlighting(int num) {
if (this.inFlight && this.flightingNums.longValue() == 0L) {
HugeGraphLoader.LOG.info("Start loading '{}'", struct);
}
this.flightingNums.add(num);
}

public void minusFlighting(int num) {
this.flightingNums.add(-num);
if (!this.inFlight && this.flightingNums.longValue() == 0L) {
HugeGraphLoader.LOG.info("Finish loading '{}'", this.struct);
}
}

public long parseSuccess(ElementMapping mapping) {
@@ -161,7 +161,7 @@ private boolean openNextReadable() {
continue;
}

LOG.debug("Ready to open '{}'", this.readable);
LOG.info("In loading '{}'", this.readable);
this.fetcher.openReader(this.readable);
if (status == LoadStatus.LOADED_HALF) {
long offset = this.oldProgress.loadingOffset();
@@ -10,12 +10,27 @@
<PatternLayout charset="${log-charset}" pattern="%-d{yyyy-MM-dd HH:mm:ss} [%t] [%-5p] %c{1.} %x - %m%n" />
</Console>

<RollingFile name="file" fileName="logs/hugegraph-loader.log"
<RollingRandomAccessFile name="file" fileName="logs/hugegraph-loader.log"
filePattern="logs/hugegraph-loader-%d{yyyy-MM-dd}-%i.log">
<ThresholdFilter level="INFO" onMatch="ACCEPT" onMismatch="DENY"/>
<PatternLayout charset="${log-charset}" pattern="%-d{yyyy-MM-dd HH:mm:ss} [%t] [%-5p] %c{1.} %x - %m%n" />
<SizeBasedTriggeringPolicy size="10MB"/>
</RollingFile>
<!-- Trigger after exceeding 1day or 100MB -->
<Policies>
<SizeBasedTriggeringPolicy size="100MB"/>
<TimeBasedTriggeringPolicy interval="1" modulate="true" />
</Policies>
<!-- Keep max 5 files per day & auto delete after over 1GB or 100 files -->
<DefaultRolloverStrategy max="5">
<Delete basePath="logs" maxDepth="2">
<IfFileName glob="*/*.log"/>
<!-- Limit log amount & size -->
<IfAny>
<IfAccumulatedFileSize exceeds="1GB" />
<IfAccumulatedFileCount exceeds="100" />
</IfAny>
</Delete>
</DefaultRolloverStrategy>
</RollingRandomAccessFile>
</appenders>
<loggers>
<root level="INFO">

0 comments on commit 5e30894

Please sign in to comment.