Connect OpenTelemetry to Skywalking

[Hemant0601]

This is my docker-compose.yaml

`version: "3.8"
services:
postgres:
container_name: postgres
image: postgres:15.7
ports:
- 5432:5432
volumes:
- ~/apps/postgres:/var/lib/postgresql/data
- ./postgres_config/pg_hba.conf:/etc/postgresql/pg_hba.conf
environment:
- POSTGRES_PASSWORD=XkFKRNke3nr3
- POSTGRES_USER=skywalking
- POSTGRES_DB=skywalking
restart: always # Add restart policy to ensure PostgreSQL container restarts if it stops
networks:
- sw
healthcheck:
test: ["CMD-SHELL", "pg_isready -U skywalking -d skywalking"]
interval: 10s
retries: 5

oap:
image: apache/skywalking-oap-server:10.0.0
container_name: oap
restart: always
depends_on:
postgres:
condition: service_healthy
links:
- postgres
networks:
- sw
ports:
- 11800:11800
- 12800:12800
#healthcheck:
# test: [ "CMD-SHELL", "/skywalking/bin/swctl ch" ]
# interval: 30s
# timeout: 10s
# retries: 3
# start_period: 10s
environment:
SW_OTEL_RECEIVER: default
SW_OTEL_RECEIVER_ENABLED_HANDLERS: otlp-metrics,otlp-traces,otlp-logs
SW_OTEL_RECEIVER_ENABLED_OTEL_METRICS_RULES: vm
SW_HEALTH_CHECKER: default
SW_STORAGE: postgresql
SW_JDBC_URL: "jdbc:postgresql://postgres:5432/skywalking" # Adjusted to point to PostgreSQL service
SW_DATA_SOURCE_USER: skywalking # Changed user to skywalking
SW_DATA_SOURCE_PASSWORD: XkFKRNke3nr3 # Changed password to XkFKRNke3nr3
SW_STORAGE_MYSQL_QUERY_MAX_SIZE: 5000 # Adjusted to PostgreSQL-specific property name
SW_STORAGE_MAX_SIZE_OF_ARRAY_COLUMN: 20
SW_STORAGE_NUM_OF_SEARCHABLE_VALUES_PER_TAG: 2
SW_STORAGE_MAX_SIZE_OF_BATCH_SQL: 2000
SW_STORAGE_ASYNC_BATCH_PERSISTENT_POOL_SIZE: 4
SW_DATA_SOURCE_CACHE_PREP_STMTS: true # Included dataSource.cachePrepStmts parameter
SW_DATA_SOURCE_PREP_STMT_CACHE_SQL_SIZE: 250 # Included dataSource.prepStmtCacheSize parameter
SW_DATA_SOURCE_PREP_STMT_CACHE_SQL_LIMIT: 2048 # Included dataSource.prepStmtCacheSqlLimit parameter
SW_DATA_SOURCE_USE_SERVER_PREP_STMTS: true # Included dataSource.useServerPrepStmts parameter
JAVA_OPTS: "-Xms2048m -Xmx2048m"

node-exporter:
image: prom/node-exporter:latest
container_name: node-exporter
restart: always
volumes:
- /proc:/host/proc:ro # Mount the proc directory from the host
- /sys:/host/sys:ro # Mount the sys directory from the host
- /:/rootfs:ro # Mount the root filesystem from the host
command:
- '--path.procfs=/host/proc' # Path to the procfs directory
- '--path.rootfs=/rootfs' # Path to the root filesystem
- '--path.sysfs=/host/sys' # Path to the sysfs directory
- '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)' # Exclude certain filesystem mount points from collection
ports:
- "9100:9100"
expose:
- 9100 # Expose ports from the container
networks:
- sw

otel-collector:
container_name: otel
# Specify the image to start the container from
image: otel/opentelemetry-collector:0.92.0
# Set the otel-collector configfile
command: ["--config=/etc/otel-collector-config.yaml"]
# Mapping the configfile to host directory
restart: always
volumes:
- ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
ports:
- "13133:13133" # health_check extension
- "12800" # OpenCensus receiver
- "11800"
networks:
- sw

ui:
image: apache/skywalking-ui:10.0.0
container_name: ui
restart: always
#depends_on:
# oap:
# condition: service_healthy
networks:
- sw
links:
- oap
ports:
- "8080:8080"
environment:
SW_OAP_ADDRESS: http://oap:12800

networks:
sw: `

And this is my otel-collector-config.yaml

`extensions:
health_check:

A receiver is how data gets into the OpenTelemetry Collector

receivers:

Set Prometheus Receiver to collects metrics from targets

It’s supports the full set of Prometheus configuration

prometheus:
config:
scrape_configs:
- job_name: 'vm-monitoring'
scrape_interval: 10s
static_configs:
# Replace the IP to your VMs‘s IP which has installed Node Exporter
- targets: ['10.101.39.149:9100']

processors:
batch:

An exporter is how data gets sent to different systems/back-ends

exporters:
otlp:
endpoint: 'oap:11800' # The OAP Server address
tls:
insecure: true
#logging: debug

service:
extensions: [health_check]
pipelines:
metrics:
receivers: [prometheus]
processors: [batch]
exporters: [otlp]
`

The flow should work like this

node-exporter -> otel -> oap -> postgres
oap -> UI

But otel -> oap is broken and oap -> UI is broken.

Help me fix it one by one

below error is of otel logs .
I am unable to make connection to the OAP server
2024-05-17T03:24:12.928Z info exporterhelper/retry_sender.go:129 Exporting failed. Will retry the request after interval. {"kind": "exporter", "data_type": "metrics", "name": "otlp", "error": "rpc error: code = Unavailable desc = connection error: desc = \"transport: Error while dialing: dial tcp 192.168.16.4:11800: connect: connection refused\"", "interval": "22.8217393s"}

And in the UI also I am getting error

2024-05-17 03:23:09,958 com.linecorp.armeria.client.endpoint.healthcheck.HealthCheckedEndpointGroup 183 [armeria-common-worker-epoll-2-2] WARN [] - The first health check failed for all endpoints. numCandidates: 1 candidates: [Endpoint{10.101.39.149:12800, weight=1000}] java.util.concurrent.CompletionException: com.linecorp.armeria.common.ClosedSessionException at java.util.concurrent.CompletableFuture.reportJoin(Unknown Source) ~[?:?] at java.util.concurrent.CompletableFuture.join(Unknown Source) ~[?:?] at com.linecorp.armeria.common.util.EventLoopCheckingFuture.join(EventLoopCheckingFuture.java:87) ~[skywalking-webapp.jar:?] at com.linecorp.armeria.client.endpoint.healthcheck.HealthCheckContextGroup.lambda$initialize$2(HealthCheckContextGroup.java:87) ~[skywalking-webapp.jar:?] at java.util.concurrent.CompletableFuture.uniHandle(Unknown Source) [?:?] at java.util.concurrent.CompletableFuture$UniHandle.tryFire(Unknown Source) [?:?] at java.util.concurrent.CompletableFuture.postComplete(Unknown Source) [?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(Unknown Source) [?:?] at com.linecorp.armeria.client.endpoint.healthcheck.DefaultHealthCheckerContext.updateHealth(DefaultHealthCheckerContext.java:184) [skywalking-webapp.jar:?] at com.linecorp.armeria.client.endpoint.healthcheck.HttpHealthChecker$HealthCheckResponseSubscriber.updateHealth(HttpHealthChecker.java:321) [skywalking-webapp.jar:?] at com.linecorp.armeria.client.endpoint.healthcheck.HttpHealthChecker$HealthCheckResponseSubscriber.onError(HttpHealthChecker.java:248) [skywalking-webapp.jar:?] at com.linecorp.armeria.common.stream.CancellableStreamMessage$CloseEvent.notifySubscriber(CancellableStreamMessage.java:269) [skywalking-webapp.jar:?] at com.linecorp.armeria.common.stream.DefaultStreamMessage.notifySubscriberOfCloseEvent0(DefaultStreamMessage.java:303) [skywalking-webapp.jar:?] at com.linecorp.armeria.common.stream.DefaultStreamMessage.notifySubscriberOfCloseEvent(DefaultStreamMessage.java:295) [skywalking-webapp.jar:?] at com.linecorp.armeria.common.stream.DefaultStreamMessage.handleCloseEvent(DefaultStreamMessage.java:437) [skywalking-webapp.jar:?] at com.linecorp.armeria.common.stream.DefaultStreamMessage.notifySubscriber0(DefaultStreamMessage.java:380) [skywalking-webapp.jar:?] at com.linecorp.armeria.common.stream.DefaultStreamMessage.notifySubscriber(DefaultStreamMessage.java:336) [skywalking-webapp.jar:?] at com.linecorp.armeria.common.stream.DefaultStreamMessage.addObjectOrEvent(DefaultStreamMessage.java:317) [skywalking-webapp.jar:?] at com.linecorp.armeria.common.stream.DefaultStreamMessage.tryClose(DefaultStreamMessage.java:466) [skywalking-webapp.jar:?] at com.linecorp.armeria.common.stream.DefaultStreamMessage.close(DefaultStreamMessage.java:455) [skywalking-webapp.jar:?] at com.linecorp.armeria.client.HttpResponseWrapper.closeAction(HttpResponseWrapper.java:231) [skywalking-webapp.jar:?] at com.linecorp.armeria.client.HttpResponseWrapper.cancelTimeoutOrLog(HttpResponseWrapper.java:262) [skywalking-webapp.jar:?] at com.linecorp.armeria.client.HttpResponseWrapper.close(HttpResponseWrapper.java:210) [skywalking-webapp.jar:?] at com.linecorp.armeria.client.HttpResponseWrapper.close(HttpResponseWrapper.java:201) [skywalking-webapp.jar:?] at com.linecorp.armeria.client.AbstractHttpResponseDecoder.failUnfinishedResponses(AbstractHttpResponseDecoder.java:131) [skywalking-webapp.jar:?] at com.linecorp.armeria.client.HttpSessionHandler.channelInactive(HttpSessionHandler.java:536) [skywalking-webapp.jar:?] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) [skywalking-webapp.jar:?] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) [skywalking-webapp.jar:?] at io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) [skywalking-webapp.jar:?] at com.linecorp.armeria.client.Http1ResponseDecoder.channelInactive(Http1ResponseDecoder.java:157) [skywalking-webapp.jar:?] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:305) [skywalking-webapp.jar:?] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) [skywalking-webapp.jar:?] at io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) [skywalking-webapp.jar:?] at io.netty.channel.CombinedChannelDuplexHandler$DelegatingChannelHandlerContext.fireChannelInactive(CombinedChannelDuplexHandler.java:418) [skywalking-webapp.jar:?] at io.netty.handler.codec.ByteToMessageDecoder.channelInputClosed(ByteToMessageDecoder.java:412) [skywalking-webapp.jar:?] at io.netty.handler.codec.ByteToMessageDecoder.channelInactive(ByteToMessageDecoder.java:377) [skywalking-webapp.jar:?] at io.netty.handler.codec.http.HttpClientCodec$Decoder.channelInactive(HttpClientCodec.java:410) [skywalking-webapp.jar:?] at io.netty.channel.CombinedChannelDuplexHandler.channelInactive(CombinedChannelDuplexHandler.java:221) [skywalking-webapp.jar:?] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) [skywalking-webapp.jar:?] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) [skywalking-webapp.jar:?] at io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) [skywalking-webapp.jar:?] at io.netty.handler.logging.LoggingHandler.channelInactive(LoggingHandler.java:206) [skywalking-webapp.jar:?] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:303) [skywalking-webapp.jar:?] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) [skywalking-webapp.jar:?] at io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:274) [skywalking-webapp.jar:?] at io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1405) [skywalking-webapp.jar:?] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:301) [skywalking-webapp.jar:?] at io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:281) [skywalking-webapp.jar:?] at io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:901) [skywalking-webapp.jar:?] at io.netty.channel.AbstractChannel$AbstractUnsafe$7.run(AbstractChannel.java:813) [skywalking-webapp.jar:?] at io.netty.util.concurrent.AbstractEventExecutor.runTask(AbstractEventExecutor.java:173) [skywalking-webapp.jar:?] at io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:166) [skywalking-webapp.jar:?] at io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:470) [skywalking-webapp.jar:?] at io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:413) [skywalking-webapp.jar:?] at io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) [skywalking-webapp.jar:?] at io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) [skywalking-webapp.jar:?] at io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) [skywalking-webapp.jar:?] at java.lang.Thread.run(Unknown Source) [?:?] Caused by: com.linecorp.armeria.common.ClosedSessionException at com.linecorp.armeria.common.ClosedSessionException.get(ClosedSessionException.java:35) ~[skywalking-webapp.jar:?] at com.linecorp.armeria.internal.client.ClosedStreamExceptionUtil.newClosedSessionException(ClosedStreamExceptionUtil.java:51) ~[skywalking-webapp.jar:?] at com.linecorp.armeria.internal.client.ClosedStreamExceptionUtil.newClosedSessionException(ClosedStreamExceptionUtil.java:45) ~[skywalking-webapp.jar:?] at com.linecorp.armeria.client.HttpSessionHandler.channelInactive(HttpSessionHandler.java:535) ~[skywalking-webapp.jar:?] ... 32 more 2024-05-17 03:23:10,134 com.linecorp.armeria.common.util.SystemInfo 237 [main] INFO [] - hostname: 29b21ef293d1 (from /proc/sys/kernel/hostname) 2024-05-17 03:23:10,303 com.linecorp.armeria.server.Server 808 [armeria-boss-http-*:8080] INFO [] - Serving HTTP at /0:0:0:0:0:0:0:0%0:8080 - http://127.0.0.1:8080/ 2024-05-17 03

Although when I see OAP logs everything seems fine, it shows gRPC server at 11800 and http at 12800 are properly strated.

I have gone through almost all github tickets referring to this issue but not able to find a solution to this. Documentation mention that its stratightforward but it doesnt work that way at all. I am very keen to use skywalking thats why want to resolve it and use this tool only.

[wu-sheng]

Could you format your contents? It's hard to read.

[Hemant0601]

Hi. I was able to solve this issue. There was some network issue which caused this.
P.S : It should add one line in documentation that the metrics will auto pop up and nothing required in UI.
Marking this closed. Also, let me know if anyone faces this issue, I can sent my configurations.

[wu-sheng]

P.S : It should add one line in documentation that the metrics will auto pop up and nothing required in UI.

I don't think that is the case. At least, MAL should be added.

[Hemant0601]

I have tried below configuration, can you mention where MAL is used, as I havent mentioned it specifically anywhere.

#docker-compose.yaml

` services:
postgres:
container_name: postgres
image: postgres:15.7
ports:
- 5432:5432
volumes:
- ~/apps/postgres:/var/lib/postgresql/data
- ./postgres_config/pg_hba.conf:/etc/postgresql/pg_hba.conf
environment:
- POSTGRES_PASSWORD=skywalking
- POSTGRES_USER=skywalking
- POSTGRES_DB=skywalking
restart: always # Add restart policy to ensure PostgreSQL container restarts if it stops
networks:
- sw
healthcheck:
test: ["CMD-SHELL", "pg_isready -U skywalking -d skywalking"]
interval: 10s
retries: 5

oap:
image: apache/skywalking-oap-server:10.0.0
container_name: oap
restart: always
depends_on:
postgres:
condition: service_healthy
links:
- postgres
networks:
- sw
ports:
- 11800:11800
- 12800:12800
environment:
SW_OTEL_RECEIVER: default
SW_OTEL_RECEIVER_ENABLED_HANDLERS: otlp-metrics,otlp-traces,otlp-logs
SW_OTEL_RECEIVER_ENABLED_OTEL_METRICS_RULES: vm
SW_CORE_RECORD_DATA_TTL: 3
SW_CORE_METRICS_DATA_TTL: 7
SW_HEALTH_CHECKER: default
SW_STORAGE: postgresql
SW_JDBC_URL: "jdbc:postgresql://postgres:5432/skywalking" # Adjusted to point to PostgreSQL service
SW_DATA_SOURCE_USER: skywalking # Changed user to skywalking
SW_DATA_SOURCE_PASSWORD: skywalking # Changed password to skywalking
SW_STORAGE_MYSQL_QUERY_MAX_SIZE: 5000 # Adjusted to PostgreSQL-specific property name
SW_STORAGE_MAX_SIZE_OF_ARRAY_COLUMN: 20
SW_STORAGE_NUM_OF_SEARCHABLE_VALUES_PER_TAG: 2
SW_STORAGE_MAX_SIZE_OF_BATCH_SQL: 2000
SW_STORAGE_ASYNC_BATCH_PERSISTENT_POOL_SIZE: 4
SW_DATA_SOURCE_CACHE_PREP_STMTS: true # Included dataSource.cachePrepStmts parameter
SW_DATA_SOURCE_PREP_STMT_CACHE_SQL_SIZE: 250 # Included dataSource.prepStmtCacheSize parameter
SW_DATA_SOURCE_PREP_STMT_CACHE_SQL_LIMIT: 2048 # Included dataSource.prepStmtCacheSqlLimit parameter
SW_DATA_SOURCE_USE_SERVER_PREP_STMTS: true # Included dataSource.useServerPrepStmts parameter
JAVA_OPTS: "-Xms2048m -Xmx2048m"

node-exporter:
image: prom/node-exporter:latest
container_name: node-exporter
restart: always
volumes:
- /proc:/host/proc:ro # Mount the proc directory from the host
- /sys:/host/sys:ro # Mount the sys directory from the host
- /:/rootfs:ro # Mount the root filesystem from the host
command:
- '--path.procfs=/host/proc' # Path to the procfs directory
- '--path.rootfs=/rootfs' # Path to the root filesystem
- '--path.sysfs=/host/sys' # Path to the sysfs directory
- '--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)' # Exclude certain filesystem mount points from collection
ports:
- "9100:9100"
expose:
- 9100 # Expose ports from the container
networks:
- sw
fluent-bit:
image: fluent/fluent-bit:latest
container_name: fluent-bit
restart: always
networks:
- sw
volumes:
- ./fluent-bit.conf:/fluent-bit/etc/fluent-bit.conf
- ./parsers.conf:/fluent-bit/etc/parsers.conf
- /var/log/fluent-bit:/var/log/fluent-bit
ports:
- "5140:5140"
otel-collector:
container_name: otel
# Specify the image to start the container from
image: otel/opentelemetry-collector:0.92.0
# Set the otel-collector configfile
command: ["--config=/etc/otel-collector-config.yaml"]
# Mapping the configfile to host directory
restart: always
volumes:
- ./otel-collector-config.yaml:/etc/otel-collector-config.yaml
ports:
- "13133:13133" # health_check extension
- "4318:4318" # http receiver
- "4317:4317" # gRpc receiver
- "11800"
networks:
- sw

ui:
image: apache/skywalking-ui:10.0.0
container_name: ui
restart: always
networks:
- sw
links:
- oap
ports:
- "8080:8080"
environment:
SW_OAP_ADDRESS: oap:12800
networks:
sw: `

otel-configuration.yaml

extensions:
health_check:
receivers:
prometheus:
config:
scrape_configs:
- job_name: 'vm-monitoring'
scrape_interval: 5s
static_configs:
# Replace the IP to your VMs‘s IP which has installed Node Exporter
- targets: ['10.101.39.149:9100']
otlp:
protocols:
grpc:
endpoint: 0.0.0.0:4317
http:
endpoint: 0.0.0.0:4318
processors:
batch:
resource:
attributes:
- key: service.name
value: "skywalking-server"
action: insert
exporters:
otlp:
endpoint: 'oap:11800'
tls:
insecure: true
logging:
loglevel: debug
service:
extensions: [health_check]
pipelines:
metrics:
receivers: [prometheus]
processors: [batch]
exporters: [otlp]
logs:
receivers: [otlp]
processors: [batch,resource]
exporters: [otlp]
traces:
receivers: [otlp]
processors: [batch,resource]
exporters: [logging,otlp]

[wu-sheng]

If you are using official dashboard metrics, then you are correct.
But if it is custom metrics, MAL scripts are required.