[DBNode] CPUCore() throw error: SIGILL: illegal instruction #3105

shaneing · 2021-01-20T08:01:28Z

When I execute command as follows:

curl -X POST http://localhost:7201/api/v1/json/write -d '{
  "tags": 
    {
      "__name__": "third_avenue",
      "city": "new_york",
      "checkout": "1"
    },
    "timestamp": '\"$(date "+%s")\"',
    "value": 3347.26
}'

The m3dbnode will throw error:

SIGILL: illegal instruction
PC=0xd24610 m=13 sigcode=2

goroutine 16338 [running]:
github.com/m3db/m3/src/x/sync.getCore(0xc00dda54d0, 0x3, 0x3, 0x1, 0x2308140, 0xc004090300, 0x22aaac0, 0xc004090300, 0x22aaaa0, 0xc004090300, ...)
        /go/src/github.com/m3db/m3/src/x/sync/cpu_linux_amd64.s:9 fp=0xc00f52c218 sp=0xc00f52c210 pc=0xd24610
github.com/m3db/m3/src/x/sync.CPUCore(...)
        /go/src/github.com/m3db/m3/src/x/sync/index_cpu.go:72
github.com/m3db/m3/src/dbnode/storage.(*dbShardInsertQueue).Insert(0xc000771ad0, 0xc00f4f0210, 0x0, 0x0, 0xed799d2d5, 0x32a1140, 0x40aa26851eb851ec, 0x2, 0x0, 0x0, ...)
        /go/src/github.com/m3db/m3/src/dbnode/storage/shard_insert_queue.go:291 +0x26b fp=0xc00f52c288 sp=0xc00f52c218 pc=0x11a7afb
github.com/m3db/m3/src/dbnode/storage.(*dbShard).insertSeriesAsyncBatched(0xc004090300, 0x2308ec0, 0xc03dcf7ce0, 0x7f751c5a2df0, 0xc06956c0c0, 0x0, 0x0, 0xed799d2d5, 0x32a1140, 0x40aa26851eb851ec, ...)
        /go/src/github.com/m3db/m3/src/dbnode/storage/shard.go:1389 +0x209 fp=0xc00f52c580 sp=0xc00f52c288 pc=0x1197c79
github.com/m3db/m3/src/dbnode/storage.(*dbShard).writeAndIndex(0xc004090300, 0x231eae0, 0xc082aca620, 0x2308ec0, 0xc03dcf7ce0, 0x7f751c5a2df0, 0xc06956c0c0, 0x0, 0xed799d2d5, 0x32a1140, ...)
        /go/src/github.com/m3db/m3/src/dbnode/storage/shard.go:1023 +0x24e fp=0xc00f52cb40 sp=0xc00f52c580 pc=0x11951de
github.com/m3db/m3/src/dbnode/storage.(*dbShard).WriteTagged(0xc004090300, 0x231eae0, 0xc082aca620, 0x2308ec0, 0xc03dcf7ce0, 0x7f751c5a2df0, 0xc06956c0c0, 0x0, 0xed799d2d5, 0x32a1140, ...)
        /go/src/github.com/m3db/m3/src/dbnode/storage/shard.go:910 +0x15e fp=0xc00f52cde0 sp=0xc00f52cb40 pc=0x1194cee
github.com/m3db/m3/src/dbnode/storage.(*dbNamespace).WriteTagged(0xc0a52b0e00, 0x231eae0, 0xc082aca620, 0x2308ec0, 0xc03dcf7ce0, 0x7f751c5a2df0, 0xc06956c0c0, 0x0, 0xed799d2d5, 0x32a1140, ...)
        /go/src/github.com/m3db/m3/src/dbnode/storage/namespace.go:729 +0x35c fp=0xc00f52d0f0 sp=0xc00f52cde0 pc=0x1173f1c
github.com/m3db/m3/src/dbnode/storage.(*db).writeBatch(0xc09efece00, 0x231eae0, 0xc082aca620, 0x2308ec0, 0xc03dcf7c80, 0x7f751c5a2db8, 0xc0902c2660, 0x22aa820, 0xc0697902a0, 0xed799d201, ...)
        /go/src/github.com/m3db/m3/src/dbnode/storage/database.go:783 +0x34f fp=0xc00f52d808 sp=0xc00f52d0f0 pc=0x114136f
github.com/m3db/m3/src/dbnode/storage.(*db).WriteTaggedBatch(0xc09efece00, 0x231eae0, 0xc082aca620, 0x2308ec0, 0xc03dcf7c80, 0x7f751c5a2db8, 0xc0902c2660, 0x22aa820, 0xc0697902a0, 0x0, ...)
        /go/src/github.com/m3db/m3/src/dbnode/storage/database.go:741 +0x99 fp=0xc00f52d878 sp=0xc00f52d808 pc=0x1140fe9
github.com/m3db/m3/src/dbnode/storage/cluster.(*clusterDB).WriteTaggedBatch(0xc0dbfbc380, 0x231eae0, 0xc082aca620, 0x2308ec0, 0xc03dcf7c80, 0x7f751c5a2db8, 0xc0902c2660, 0x22aa820, 0xc0697902a0, 0x0, ...)
        <autogenerated>:1 +0xa9 fp=0xc00f52d8e0 sp=0xc00f52d878 pc=0x182afe9
github.com/m3db/m3/src/dbnode/network/server/tchannelthrift/node.(*service).WriteTaggedBatchRaw(0xc0038ed200, 0x7f751e7185b0, 0xc00f456e00, 0xc00f51e360, 0x0, 0x0)
        /go/src/github.com/m3db/m3/src/dbnode/network/server/tchannelthrift/node/service.go:1699 +0xa82 fp=0xc00f52dad8 sp=0xc00f52d8e0 pc=0x1813432
github.com/m3db/m3/src/dbnode/generated/thrift/rpc.(*tchanNodeServer).handleWriteTaggedBatchRaw(0xc09efe8dc0, 0x7f751e7185b0, 0xc00f456e00, 0x2338b60, 0xc0956ae900, 0x1cdb7a0, 0xc004f93ba8, 0x47c7c0, 0x327eda0, 0xc0088b7800)
        /go/src/github.com/m3db/m3/src/dbnode/generated/thrift/rpc/tchan-rpc.go:2015 +0xc4 fp=0xc00f52db30 sp=0xc00f52dad8 pc=0xeecee4
github.com/m3db/m3/src/dbnode/generated/thrift/rpc.(*tchanNodeServer).Handle(0xc09efe8dc0, 0x7f751e7185b0, 0xc00f456e00, 0xc00f520066, 0x13, 0x2338b60, 0xc0956ae900, 0xc00f51e2d0, 0xc00f4f2200, 0x6dcbd7, ...)
        /go/src/github.com/m3db/m3/src/dbnode/generated/thrift/rpc/tchan-rpc.go:1162 +0x16c1 fp=0xc00f52dbb8 sp=0xc00f52db30 pc=0xee7bc1
github.com/uber/tchannel-go/thrift.(*Server).handle(0xc09efa59a0, 0x22f49a0, 0xc00f456d70, 0x22deaa0, 0xc09efe8dc0, 0x1f8be50, 0xc00f520066, 0x13, 0xc00dbc3e00, 0x0, ...)
        /go/pkg/mod/github.com/uber/tchannel-go@v1.14.0/thrift/server.go:163 +0x3ad fp=0xc00f52dd40 sp=0xc00f52dbb8 pc=0xe2762d
github.com/uber/tchannel-go/thrift.(*Server).Handle(0xc09efa59a0, 0x22f49a0, 0xc00f456d70, 0xc00dbc3e00)
        /go/pkg/mod/github.com/uber/tchannel-go@v1.14.0/thrift/server.go:237 +0x182 fp=0xc00f52de08 sp=0xc00f52dd40 pc=0xe28162
github.com/uber/tchannel-go.(*handlerMap).Handle(0xc09efea4e0, 0x22f49a0, 0xc00f456d70, 0xc00dbc3e00)
        /go/pkg/mod/github.com/uber/tchannel-go@v1.14.0/handlers.go:118 +0xf9 fp=0xc00f52df10 sp=0xc00f52de08 pc=0xdec869
github.com/uber/tchannel-go.channelHandler.Handle(0xc09efec000, 0x22f49a0, 0xc00f456d70, 0xc00dbc3e00)
        /go/pkg/mod/github.com/uber/tchannel-go@v1.14.0/handlers.go:126 +0x8c fp=0xc00f52df58 sp=0xc00f52df10 pc=0xdece2c
github.com/uber/tchannel-go.(*Connection).dispatchInbound(0xc0a3b52280, 0xd00000003, 0xc00dbc3e00, 0xc0a4058ea0)
        /go/pkg/mod/github.com/uber/tchannel-go@v1.14.0/inbound.go:194 +0x362 fp=0xc00f52dfc0 sp=0xc00f52df58 pc=0xdefd02
runtime.goexit()
        /usr/local/go/src/runtime/asm_amd64.s:1357 +0x1 fp=0xc00f52dfc8 sp=0xc00f52dfc0 pc=0x45f0a1
created by github.com/uber/tchannel-go.(*Connection).handleCallReq
        /go/pkg/mod/github.com/uber/tchannel-go@v1.14.0/inbound.go:125 +0xc97

The text was updated successfully, but these errors were encountered:

gibbscullen · 2021-01-20T18:26:44Z

@shaneing -- looking into this. Also, what version of M3 are you using?

shaneing · 2021-01-21T02:06:09Z

@gibbscullen I am using image quay.io/m3db/m3dbnode:v1.0.0.

ChrisChinchilla · 2021-01-21T09:17:10Z

@shaneing What OS are you using?

shaneing · 2021-01-21T09:37:20Z

@ChrisChinchilla

NAME="CentOS Linux"
VERSION="7 (Core)"
ID="centos"
ID_LIKE="rhel fedora"
VERSION_ID="7"
PRETTY_NAME="CentOS Linux 7 (Core)"
ANSI_COLOR="0;31"
CPE_NAME="cpe:/o:centos:centos:7"
HOME_URL="https://www.centos.org/"
BUG_REPORT_URL="https://bugs.centos.org/"

CENTOS_MANTISBT_PROJECT="CentOS-7"
CENTOS_MANTISBT_PROJECT_VERSION="7"
REDHAT_SUPPORT_PRODUCT="centos"
REDHAT_SUPPORT_PRODUCT_VERSION="7"

4.15.6-1.el7.elrepo.x86_64 #1 SMP Sun Feb 25 20:57:32 EST 2018 x86_64 x86_64 x86_64 GNU/Linux

ChrisChinchilla · 2021-01-21T11:32:52Z

@shaneing OK, I tested on debian, Windows, and macOS, but not Centos. I will download a VM and try. And unless I am wrong, you're using the Docker image not the binaries?

shaneing · 2021-01-22T09:02:49Z

@ChrisChinchilla Yes, I am using the docker image.

ChrisChinchilla · 2021-01-22T13:08:50Z

@shaneing Hmm, I was not able to replicate this. I tried running the whole tutorial though on a CentOS 7 VM, and it all worked.

One thing to check, as we've found a few people hitting possibly related errors around the quickstart.

Make sure that you have followed the quickstart steps in order, including the prerequisites (for example, even JQ), and if you ever restart the quickstart, make sure to delete the Docker persistent directory (m3db_data in this case).

In the meantime I am going to try and make it clearer about running through the steps in order, and clarify a few small things.

shaneing · 2021-01-25T06:43:02Z

@ChrisChinchilla Sorry, I have found the problem. The reason is that my vm has not supported the instrument rdtscp.

ChrisChinchilla · 2021-01-25T10:51:10Z

OK!

JamesJoe-C · 2021-03-22T06:42:29Z

Thinks, This also helped me.

angelala00 · 2021-08-13T11:39:46Z

@ChrisChinchilla Sorry, I have found the problem. The reason is that my vm has not supported the instrument rdtscp.

I got the same error, how did you solve this?

gibbscullen self-assigned this Jan 20, 2021

ChrisChinchilla self-assigned this Jan 21, 2021

ChrisChinchilla closed this as completed Jan 25, 2021

angelala00 mentioned this issue Aug 16, 2021

CPUCore() throw error: SIGILL: illegal instruction #3659

Closed

jandubois mentioned this issue Jan 25, 2022

m3db docker image crashes on rancher desktop, runs fine on other docker environments rancher-sandbox/rancher-desktop#1324

Open

moio mentioned this issue May 2, 2022

qemu: expose RDTSCP to guests Homebrew/homebrew-core#100645

Closed

6 tasks

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

[DBNode] CPUCore() throw error: SIGILL: illegal instruction #3105

[DBNode] CPUCore() throw error: SIGILL: illegal instruction #3105

shaneing commented Jan 20, 2021

gibbscullen commented Jan 20, 2021

shaneing commented Jan 21, 2021 •

edited

ChrisChinchilla commented Jan 21, 2021

shaneing commented Jan 21, 2021

ChrisChinchilla commented Jan 21, 2021

shaneing commented Jan 22, 2021

ChrisChinchilla commented Jan 22, 2021

shaneing commented Jan 25, 2021

ChrisChinchilla commented Jan 25, 2021

JamesJoe-C commented Mar 22, 2021

angelala00 commented Aug 13, 2021

[DBNode] CPUCore() throw error: SIGILL: illegal instruction #3105

[DBNode] CPUCore() throw error: SIGILL: illegal instruction #3105

Comments

shaneing commented Jan 20, 2021

gibbscullen commented Jan 20, 2021

shaneing commented Jan 21, 2021 • edited

ChrisChinchilla commented Jan 21, 2021

shaneing commented Jan 21, 2021

ChrisChinchilla commented Jan 21, 2021

shaneing commented Jan 22, 2021

ChrisChinchilla commented Jan 22, 2021

shaneing commented Jan 25, 2021

ChrisChinchilla commented Jan 25, 2021

JamesJoe-C commented Mar 22, 2021

angelala00 commented Aug 13, 2021

shaneing commented Jan 21, 2021 •

edited