Skip to content

Feat metrics #9484

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
easonyipj wants to merge 2 commits into from
Closed

Feat metrics #9484

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
481fc32
[feat]
easonyipj Jul 14, 2022
b982f0b
[feat]add golang stack metric
easonyipj Jul 16, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion apm-protocol/apm-network/src/main/proto
Open in desktop
Submodule proto updated from e89af1 to eaacba
44 changes: 44 additions & 0 deletions ...ava/org/apache/skywalking/oap/server/analyzer/provider/golang/GolangSourceDispatcher.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
package org.apache.skywalking.oap.server.analyzer.provider.golang;

import org.apache.skywalking.apm.network.language.agent.v3.GolangMetric;
import org.apache.skywalking.oap.server.core.CoreModule;
import org.apache.skywalking.oap.server.core.analysis.IDManager;
import org.apache.skywalking.oap.server.core.analysis.TimeBucket;
import org.apache.skywalking.oap.server.core.source.ServiceInstanceGolangStack;
import org.apache.skywalking.oap.server.core.source.SourceReceiver;
import org.apache.skywalking.oap.server.library.module.ModuleManager;

public class GolangSourceDispatcher {
private final SourceReceiver sourceReceiver;

public GolangSourceDispatcher(ModuleManager moduleManager) {
this.sourceReceiver = moduleManager.find(CoreModule.NAME).provider().getService(SourceReceiver.class);
}

public void sendMetric(String service, String serviceInstance, GolangMetric golangMetric) {
long minuteTimeBucket = TimeBucket.getMinuteTimeBucket(golangMetric.getTime());

final String serviceId = IDManager.ServiceID.buildId(service, true);
final String serviceInstanceId = IDManager.ServiceInstanceID.buildId(serviceId, serviceInstance);

this.sendToStackProcess(service, serviceId, serviceInstance, serviceInstanceId, minuteTimeBucket, golangMetric.getStackInUse());
}

private void sendToStackProcess(String service,
String serviceId,
String serviceInstance,
String serviceInstanceId,
long timeBucket,
long heapUsed) {
ServiceInstanceGolangStack serviceInstanceGolangStack = new ServiceInstanceGolangStack();
serviceInstanceGolangStack.setId(serviceInstanceId);
serviceInstanceGolangStack.setName(serviceInstance);
serviceInstanceGolangStack.setServiceId(serviceId);
serviceInstanceGolangStack.setServiceName(service);
serviceInstanceGolangStack.setTimeBucket(timeBucket);
serviceInstanceGolangStack.setUsed(heapUsed);
this.sourceReceiver.receive(serviceInstanceGolangStack);
}


}
1 change: 1 addition & 0 deletions oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALLexer.g4
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ SRC_SERVICE_INSTANCE_CLR_GC: 'ServiceInstanceCLRGC';
SRC_SERVICE_INSTANCE_CLR_THREAD: 'ServiceInstanceCLRThread';
SRC_ENVOY_INSTANCE_METRIC: 'EnvoyInstanceMetric';
SRC_EVENT: 'Event';
SRC_SERVICE_INSTANCE_GOLANG_STACK: 'ServiceInstanceGolangStack';

// Browser keywords
SRC_BROWSER_APP_PERF: 'BrowserAppPerf';
Expand Down
2 changes: 1 addition & 1 deletion oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALParser.g4
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ source
SRC_ENVOY_INSTANCE_METRIC |
SRC_BROWSER_APP_PERF | SRC_BROWSER_APP_PAGE_PERF | SRC_BROWSER_APP_SINGLE_VERSION_PERF |
SRC_BROWSER_APP_TRAFFIC | SRC_BROWSER_APP_PAGE_TRAFFIC | SRC_BROWSER_APP_SINGLE_VERSION_TRAFFIC |
SRC_EVENT
SRC_EVENT | SRC_SERVICE_INSTANCE_GOLANG_STACK
;

disableSource
Expand Down
8 changes: 8 additions & 0 deletions ...r-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,14 @@ public class DefaultScopeDefine {
public static final int EBPF_PROFILING_DATA = 48;
public static final int SERVICE_LABEL = 49;
public static final int TAG_AUTOCOMPLETE = 50;
public static final int SERVICE_INSTANCE_GOLANG_HEAP = 51;
public static final int SERVICE_INSTANCE_GOLANG_STACK = 52;
public static final int SERVICE_INSTANCE_GOLANG_GC_NUM = 53;
public static final int SERVICE_INSTANCE_GOLANG_GC_TIME = 54;
public static final int SERVICE_INSTANCE_GOLANG_GOROUTINE_NUM = 55;
public static final int SERVICE_INSTANCE_GOLANG_THREAD_NUM = 56;
public static final int SERVICE_INSTANCE_GOLANG_CPU = 57;
public static final int SERVICE_INSTANCE_GOLANG_MEM = 58;

/**
* Catalog of scope, the metrics processor could use this to group all generated metrics by oal rt.
Expand Down
4 changes: 4 additions & 0 deletions ...rc/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceGolangGcNum.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
package org.apache.skywalking.oap.server.core.source;

public class ServiceInstanceGolangGcNum {
}
4 changes: 4 additions & 0 deletions ...n/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceGolangGcPauseTime.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
package org.apache.skywalking.oap.server.core.source;

public class ServiceInstanceGolangGcPauseTime {
}
4 changes: 4 additions & 0 deletions ...src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceGolangHeap.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
package org.apache.skywalking.oap.server.core.source;

public class ServiceInstanceGolangHeap {
}
39 changes: 39 additions & 0 deletions ...rc/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceGolangStack.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
package org.apache.skywalking.oap.server.core.source;

import lombok.Getter;
import lombok.Setter;

import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.*;

@ScopeDeclaration(id = SERVICE_INSTANCE_GOLANG_STACK, name = "ServiceInstanceGolangStack", catalog = SERVICE_INSTANCE_CATALOG_NAME)
@ScopeDefaultColumn.VirtualColumnDefinition(fieldName = "entityId", columnName = "entity_id", isID = true, type = String.class)
public class ServiceInstanceGolangStack extends Source {
@Override
public int scope() {
return DefaultScopeDefine.SERVICE_INSTANCE_GOLANG_STACK;
}

@Override
public String getEntityId() {
return String.valueOf(id);
}

@Getter
@Setter
private String id;
@Getter
@Setter
@ScopeDefaultColumn.DefinedByField(columnName = "name", requireDynamicActive = true)
private String name;
@Getter
@Setter
@ScopeDefaultColumn.DefinedByField(columnName = "service_name", requireDynamicActive = true)
private String serviceName;
@Getter
@Setter
@ScopeDefaultColumn.DefinedByField(columnName = "service_id")
private String serviceId;
@Getter
@Setter
private long used;
}
1 change: 1 addition & 0 deletions oap-server/server-receiver-plugin/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
<module>skywalking-mesh-receiver-plugin</module>
<module>skywalking-management-receiver-plugin</module>
<module>skywalking-jvm-receiver-plugin</module>
<module>skywalking-golang-receiver-plugin</module>
<module>envoy-metrics-receiver-plugin</module>
<module>skywalking-sharing-server-plugin</module>
<module>skywalking-clr-receiver-plugin</module>
Expand Down
28 changes: 28 additions & 0 deletions oap-server/server-receiver-plugin/skywalking-golang-receiver-plugin/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>server-receiver-plugin</artifactId>
<groupId>org.apache.skywalking</groupId>
<version>9.1.0-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

<artifactId>skywalking-golang-receiver-plugin</artifactId>
<packaging>jar</packaging>

<dependencies>
<dependency>
<groupId>org.apache.skywalking</groupId>
<artifactId>agent-analyzer</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.skywalking</groupId>
<artifactId>skywalking-sharing-server-plugin</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>

</project>
13 changes: 13 additions & 0 deletions ...n/src/main/java/org/apache/skywalking/oap/server/receiver/golang/module/GolangModule.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
package org.apache.skywalking.oap.server.receiver.golang.module;
import org.apache.skywalking.oap.server.library.module.ModuleDefine;

public class GolangModule extends ModuleDefine {
public GolangModule() {
super("receiver-golang");
}

@Override
public Class[] services() {
return new Class[0];
}
}
61 changes: 61 additions & 0 deletions .../java/org/apache/skywalking/oap/server/receiver/golang/provider/GolangModuleProvider.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
package org.apache.skywalking.oap.server.receiver.golang.provider;

import org.apache.skywalking.oap.server.analyzer.provider.golang.GolangSourceDispatcher;
import org.apache.skywalking.oap.server.core.CoreModule;
import org.apache.skywalking.oap.server.core.config.NamingControl;
import org.apache.skywalking.oap.server.core.oal.rt.OALEngineLoaderService;
import org.apache.skywalking.oap.server.core.server.GRPCHandlerRegister;
import org.apache.skywalking.oap.server.library.module.*;
import org.apache.skywalking.oap.server.receiver.golang.module.GolangModule;
import org.apache.skywalking.oap.server.receiver.golang.provider.handler.GolangMetricHandler;
import org.apache.skywalking.oap.server.receiver.sharing.server.SharingServerModule;

public class GolangModuleProvider extends ModuleProvider {

@Override
public String name() {
return "default";
}

@Override
public Class<? extends ModuleDefine> module() {
return GolangModule.class;
}

@Override
public ModuleConfig createConfigBeanIfAbsent() {
return null;
}

@Override
public void prepare() {
}

@Override
public void start() throws ModuleStartException {
// load official analysis
getManager().find(CoreModule.NAME)
.provider()
.getService(OALEngineLoaderService.class)
.load(GolangOALDefine.INSTANCE);

GRPCHandlerRegister grpcHandlerRegister = getManager().find(SharingServerModule.NAME)
.provider()
.getService(GRPCHandlerRegister.class);
GolangMetricHandler golangMetricHandler = new GolangMetricHandler(getManager());
grpcHandlerRegister.addHandler(golangMetricHandler);
}

@Override
public void notifyAfterCompleted() {

}

@Override
public String[] requiredModules() {
return new String[] {
CoreModule.NAME,
SharingServerModule.NAME
};
}
}
16 changes: 16 additions & 0 deletions .../main/java/org/apache/skywalking/oap/server/receiver/golang/provider/GolangOALDefine.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
package org.apache.skywalking.oap.server.receiver.golang.provider;

import org.apache.skywalking.oap.server.core.oal.rt.OALDefine;

public class GolangOALDefine extends OALDefine {

public static final GolangOALDefine INSTANCE = new GolangOALDefine();

private GolangOALDefine() {
super(
"oal/golang-agent.oal",
"org.apache.skywalking.oap.server.core.source"
);
}

}
42 changes: 42 additions & 0 deletions ...rg/apache/skywalking/oap/server/receiver/golang/provider/handler/GolangMetricHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
package org.apache.skywalking.oap.server.receiver.golang.provider.handler;

import io.grpc.stub.StreamObserver;
import lombok.extern.slf4j.Slf4j;
import org.apache.skywalking.apm.network.common.v3.Commands;
import org.apache.skywalking.apm.network.language.agent.v3.GolangMetricCollection;
import org.apache.skywalking.apm.network.language.agent.v3.GolangMetricReportServiceGrpc;
import org.apache.skywalking.oap.server.analyzer.provider.golang.GolangSourceDispatcher;
import org.apache.skywalking.oap.server.core.CoreModule;
import org.apache.skywalking.oap.server.core.config.NamingControl;
import org.apache.skywalking.oap.server.library.module.ModuleManager;
import org.apache.skywalking.oap.server.library.server.grpc.GRPCHandler;


@Slf4j
public class GolangMetricHandler extends GolangMetricReportServiceGrpc.GolangMetricReportServiceImplBase implements GRPCHandler {

private final GolangSourceDispatcher golangSourceDispatcher;
private final NamingControl namingControl;


public GolangMetricHandler(ModuleManager moduleManager) {
this.golangSourceDispatcher = new GolangSourceDispatcher(moduleManager);
this.namingControl = moduleManager.find(CoreModule.NAME)
.provider()
.getService(NamingControl.class);
}
@Override
public void collect(GolangMetricCollection request, StreamObserver<Commands> responseObserver) {
log.info(request.toString());
final GolangMetricCollection.Builder builder = request.toBuilder();
builder.setService(namingControl.formatServiceName(builder.getService()));
builder.setServiceInstance(namingControl.formatInstanceName(builder.getServiceInstance()));

builder.getMetricsList().forEach(golangMetric -> {
golangSourceDispatcher.sendMetric(builder.getService(), builder.getServiceInstance(), golangMetric);
});

responseObserver.onNext(Commands.newBuilder().build());
responseObserver.onCompleted();
}
}
19 changes: 19 additions & 0 deletions .../resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleDefine
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#

org.apache.skywalking.oap.server.receiver.golang.module.GolangModule
19 changes: 19 additions & 0 deletions ...esources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
#

org.apache.skywalking.oap.server.receiver.golang.provider.GolangModuleProvider
5 changes: 5 additions & 0 deletions oap-server/server-starter/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,11 @@
<artifactId>skywalking-jvm-receiver-plugin</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.skywalking</groupId>
<artifactId>skywalking-golang-receiver-plugin</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.skywalking</groupId>
<artifactId>skywalking-trace-receiver-plugin</artifactId>
Expand Down
10 changes: 7 additions & 3 deletions oap-server/server-starter/src/main/resources/application.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ core:
# Turn it on then automatically grouping endpoint by the given OpenAPI definitions.
enableEndpointNameGroupingByOpenapi: ${SW_CORE_ENABLE_ENDPOINT_NAME_GROUPING_BY_OPAENAPI:true}
storage:
selector: ${SW_STORAGE:h2}
selector: ${SW_STORAGE:mysql}
elasticsearch:
namespace: ${SW_NAMESPACE:""}
clusterNodes: ${SW_STORAGE_ES_CLUSTER_NODES:localhost:9200}
Expand Down Expand Up @@ -177,7 +177,7 @@ storage:
properties:
jdbcUrl: ${SW_JDBC_URL:"jdbc:mysql://localhost:3306/swtest?rewriteBatchedStatements=true"}
dataSource.user: ${SW_DATA_SOURCE_USER:root}
dataSource.password: ${SW_DATA_SOURCE_PASSWORD:root@1234}
dataSource.password: ${SW_DATA_SOURCE_PASSWORD:1874ypj!}
dataSource.cachePrepStmts: ${SW_DATA_SOURCE_CACHE_PREP_STMTS:true}
dataSource.prepStmtCacheSize: ${SW_DATA_SOURCE_PREP_STMT_CACHE_SQL_SIZE:250}
dataSource.prepStmtCacheSqlLimit: ${SW_DATA_SOURCE_PREP_STMT_CACHE_SQL_LIMIT:2048}
Expand Down Expand Up @@ -402,7 +402,7 @@ query:
# abort a query if the total number of data fields queried exceeds the defined threshold.
maxQueryComplexity: ${SW_QUERY_MAX_QUERY_COMPLEXITY:1000}
# Allow user add, disable and update UI template
enableUpdateUITemplate: ${SW_ENABLE_UPDATE_UI_TEMPLATE:false}
enableUpdateUITemplate: ${SW_ENABLE_UPDATE_UI_TEMPLATE:true}
# "On demand log" allows users to fetch Pod containers' log in real time,
# because this might expose secrets in the logs (if any), users need
# to enable this manually, and add permissions to OAP cluster role.
Expand Down Expand Up @@ -502,3 +502,7 @@ receiver-event:
receiver-ebpf:
selector: ${SW_RECEIVER_EBPF:default}
default:

receiver-golang:
selector: ${SW_RECEIVER_GOLANG:default}
default:
Loading