SEGFAULT with COPY INTO BEST EFFORT and skipping input columns

[monetdb-team]

Date: 2017-10-20 23:52:07 +0200
From: @drstmane
To: SQL devs <>
Version: 11.27.5 (Jul2017-SP1)
CC: @njnes

Last updated: 2017-12-14 14:45:59 +0100

Comment 25774

Date: 2017-10-20 23:52:07 +0200
From: @drstmane

Created attachment 574
test script

COPY INTO BEST EFFORT makes mserver5 segfault in case input columns are skipped, as, e.g., in the following examples.
Please find a sample GDB trace below and an exhaustive test script attached.

start transaction;
create table t ( c1 int , c2 int );
copy 1 records into t (c1) from stdin (c1,c2) best effort;
1|2
rollback;

start transaction;
create table t ( c1 int , c2 int );
copy 1 records into t (c2) from stdin (c1,c2) best effort;
1|2
rollback;

start transaction;
create table t ( c1 int , c2 int );
copy 1 records into t (c1,c2) from stdin (x,c1,c2) best effort;
0|1|2
rollback;

start transaction;
create table t ( c1 int , c2 int );
copy 1 records into t (c1,c2) from stdin (c1,x,c2) best effort;
1|0|2
rollback;

start transaction;
create table t ( c1 int , c2 int );
copy 1 records into t (c1,c2) from stdin (c1,c2,x) best effort;
1|2|0
rollback;

========
Thread 6 "mserver5" received signal SIGSEGV, Segmentation fault.
[Switching to Thread 0x7fffd49d5700 (LWP 26217)]
0x00007ffff7b0ee61 in SQLload_file (cntxt=0x7ffff0cc8338, as=0x7fffd49d4170, b=0x7fffc0002970, out=0x7fffcc002b70, csep=0x7fffc00e8360 "|", rsep=0x7fffc00e8390 "\n", quote=0 '\000', skip=0, maxrow=1, best=1)
at /export/scratch1/home/manegold//Monet/HG/Dec2016/source/MonetDB/monetdb5/modules/mal/tablet.c:1836
1836 width = as->format[attr].c->twidth;
Missing separate debuginfos, use: dnf debuginfo-install R-core-3.4.1-1.fc26.x86_64 SuperLU-5.2.0-2.fc26.x86_64 armadillo-7.900.1-1.fc26.x86_64 arpack-3.5.0-1.fc26.x86_64 atlas-3.10.2-16.fc26.x86_64 blas-3.6.1-4.fc26.x86_64 boost-atomic-1.63.0-6.fc26.x86_64 boost-chrono-1.63.0-6.fc26.x86_64 boost-date-time-1.63.0-6.fc26.x86_64 boost-filesystem-1.63.0-6.fc26.x86_64 boost-iostreams-1.63.0-6.fc26.x86_64 boost-program-options-1.63.0-6.fc26.x86_64 boost-regex-1.63.0-6.fc26.x86_64 boost-system-1.63.0-6.fc26.x86_64 boost-thread-1.63.0-6.fc26.x86_64 bzip2-libs-1.0.6-22.fc26.x86_64 cfitsio-3.370-7.fc26.x86_64 cyrus-sasl-lib-2.1.26-32.fc26.x86_64 expat-2.2.4-1.fc26.x86_64 fontconfig-2.12.1-4.fc26.x86_64 freetype-2.7.1-9.fc26.x86_64 freexl-1.0.3-1.fc26.x86_64 gdal-libs-2.1.3-1.fc26.x86_64 geos-3.6.1-2.fc26.x86_64 giflib-4.1.6-16.fc26.x86_64 hdf5-1.8.18-5.fc26.x86_64 jasper-libs-2.0.12-1.fc26.x86_64 jbigkit-libs-2.1-6.fc26.x86_64 json-c-0.12.1-2.fc26.x86_64 keyutils-libs-1.5.10-1.fc26.x86_64 krb5-libs-1.15.1-28.fc26.x86_64 lapack-3.6.1-4.fc26.x86_64 laszip-2.2.0-7.fc26.x86_64 lcms2-2.8-3.fc26.x86_64 libICE-1.0.9-9.fc26.x86_64 libSM-1.2.2-5.fc26.x86_64 libX11-1.6.5-2.fc26.x86_64 libXau-1.0.8-7.fc26.x86_64 libatomic_ops-7.4.4-2.fc26.x86_64 libcom_err-1.43.4-2.fc26.x86_64 libcrypt-nss-2.25-10.fc26.x86_64 libcurl-7.53.1-10.fc26.x86_64 libdap-3.18.3-2.fc26.x86_64 libgcc-7.1.1-3.fc26.x86_64 libgeotiff-1.4.0-9.fc26.x86_64 libgfortran-7.1.1-3.fc26.x86_64 libgomp-7.1.1-3.fc26.x86_64 libgta-1.0.7-4.fc26.x86_64 libicu-57.1-6.fc26.x86_64 libidn2-2.0.4-1.fc26.x86_64 libjpeg-turbo-1.5.1-0.fc26.x86_64 libkml-1.3.0-4.fc26.x86_64 liblas-1.8.0-16.fc26.x86_64 libnghttp2-1.21.1-1.fc26.x86_64 libpng-1.6.28-2.fc26.x86_64 libpsl-0.18.0-1.fc26.x86_64 libquadmath-7.1.1-3.fc26.x86_64 libselinux-2.6-7.fc26.x86_64 libspatialite-4.3.0a-4.fc26.x86_64 libssh2-1.8.0-2.fc26.x86_64 libstdc++-7.1.1-3.fc26.x86_64 libtiff-4.0.8-1.fc26.x86_64 libtool-ltdl-2.4.6-17.fc26.x86_64 libunistring-0.9.7-1.fc26.x86_64 libuuid-2.30.1-1.fc26.x86_64 libwebp-0.6.0-2.fc26.x86_64 libxcb-1.12-3.fc26.x86_64 libxml2-2.9.4-2.fc26.x86_64 mariadb-libs-10.1.26-2.fc26.x86_64 minizip-1.2.11-2.fc26.x86_64 ncurses-libs-6.0-8.20170212.fc26.x86_64 netcdf-4.4.1.1-4.fc26.x86_64 nspr-4.16.0-1.fc26.x86_64 nss-3.32.0-1.1.fc26.x86_64 nss-softokn-freebl-3.32.0-1.2.fc26.x86_64 nss-util-3.32.0-1.0.fc26.x86_64 ogdi-3.2.0-1.fc26.x86_64 openblas-Rblas-0.2.20-2.fc26.x86_64 openblas-openmp-0.2.20-2.fc26.x86_64 openjpeg2-2.2.0-3.fc26.x86_64 openldap-2.4.45-1.fc26.x86_64 openssl-libs-1.1.0f-7.fc26.x86_64 pcre-8.41-1.fc26.x86_64 poppler-0.52.0-4.fc26.x86_64 postgresql-libs-9.6.4-1.fc26.x86_64 proj-4.9.3-2.fc26.x86_64 readline-7.0-5.fc26.x86_64 sqlite-libs-3.20.1-1.fc26.x86_64 tre-0.8.0-19.20140228gitc2f5d13.fc26.x86_64 unixODBC-2.3.4-5.fc26.x86_64 uriparser-0.8.4-4.fc26.x86_64 xerces-c-3.1.4-2.fc26.x86_64 xz-libs-5.2.3-2.fc26.x86_64 zlib-1.2.11-2.fc26.x86_64
(gdb) bt
0 0x00007ffff7b0ee61 in SQLload_file (cntxt=0x7ffff0cc8338, as=0x7fffd49d4170, b=0x7fffc0002970, out=0x7fffcc002b70, csep=0x7fffc00e8360 "|", rsep=0x7fffc00e8390 "\n", quote=0 '\000', skip=0, maxrow=1, best=1)
at /export/scratch1/home/manegold//Monet/HG/Dec2016/source/MonetDB/monetdb5/modules/mal/tablet.c:1836
1 0x00007fffef36bb8a in mvc_import_table (cntxt=0x7ffff0cc8338, bats=0x7fffd49d4250, m=0x7fffc000adf0, bs=0x7fffc0002970, t=0x7fffc0070a40, sep=0x7fffc00e8360 "|", rsep=0x7fffc00e8390 "\n", ssep=0x0, ns=0x7fffc006b020 "null", sz=1, offset=0,
locked=0, best=1) at /export/scratch1/home/manegold//Monet/HG/Dec2016/source/MonetDB/sql/backends/monet5/sql_result.c:982
2 0x00007fffef314340 in mvc_import_table_wrap (cntxt=0x7ffff0cc8338, mb=0x7fffc05de750, stk=0x7fffc01f6a60, pci=0x7fffc00879a0) at /export/scratch1/home/manegold//Monet/HG/Dec2016/source/MonetDB/sql/backends/monet5/sql.c:2477
3 0x00007ffff79e98f3 in runMALsequence (cntxt=0x7ffff0cc8338, mb=0x7fffc05de750, startpc=1, stoppc=0, stk=0x7fffc01f6a60, env=0x0, pcicaller=0x0) at /export/scratch1/home/manegold//Monet/HG/Dec2016/source/MonetDB/monetdb5/mal/mal_interpreter.c:645
4 0x00007ffff79e8496 in runMAL (cntxt=0x7ffff0cc8338, mb=0x7fffc05de750, mbcaller=0x0, env=0x0) at /export/scratch1/home/manegold//Monet/HG/Dec2016/source/MonetDB/monetdb5/mal/mal_interpreter.c:354
5 0x00007fffef327be8 in SQLrun (c=0x7ffff0cc8338, be=0x7fffc006ae20, m=0x7fffc000adf0) at /export/scratch1/home/manegold//Monet/HG/Dec2016/source/MonetDB/sql/backends/monet5/sql_execute.c:352
6 0x00007fffef32945b in SQLengineIntern (c=0x7ffff0cc8338, be=0x7fffc006ae20) at /export/scratch1/home/manegold//Monet/HG/Dec2016/source/MonetDB/sql/backends/monet5/sql_execute.c:744
7 0x00007fffef3262cf in SQLengine (c=0x7ffff0cc8338) at /export/scratch1/home/manegold//Monet/HG/Dec2016/source/MonetDB/sql/backends/monet5/sql_scenario.c:1243
8 0x00007ffff7a11f00 in runPhase (c=0x7ffff0cc8338, phase=4) at /export/scratch1/home/manegold//Monet/HG/Dec2016/source/MonetDB/monetdb5/mal/mal_scenario.c:506
9 0x00007ffff7a120d9 in runScenarioBody (c=0x7ffff0cc8338) at /export/scratch1/home/manegold//Monet/HG/Dec2016/source/MonetDB/monetdb5/mal/mal_scenario.c:548
10 0x00007ffff7a121e7 in runScenario (c=0x7ffff0cc8338) at /export/scratch1/home/manegold//Monet/HG/Dec2016/source/MonetDB/monetdb5/mal/mal_scenario.c:568
11 0x00007ffff7a13d9c in MSserveClient (dummy=0x7ffff0cc8338) at /export/scratch1/home/manegold//Monet/HG/Dec2016/source/MonetDB/monetdb5/mal/mal_session.c:473
12 0x00007ffff7a1388c in MSscheduleClient (command=0x7fffc00008d0 "\020\372\005\300\377\177", challenge=0x7fffd49d4d6b "FdsTcHQWd", fin=0x7fffc0002970, fout=0x7fffcc002b70, protocol=PROTOCOL_9, blocksize=8190, compute_column_widths=0)
at /export/scratch1/home/manegold//Monet/HG/Dec2016/source/MonetDB/monetdb5/mal/mal_session.c:364
13 0x00007ffff7ab3711 in doChallenge (data=0x7fffcc0008d0) at /export/scratch1/home/manegold//Monet/HG/Dec2016/source/MonetDB/monetdb5/modules/mal/mal_mapi.c:273
14 0x00007ffff768c23a in thread_starter (arg=0x7fffcc004c60) at /export/scratch1/home/manegold//Monet/HG/Dec2016/source/MonetDB/gdk/gdk_system.c:485
15 0x00007ffff4caf36d in start_thread () from /lib64/libpthread.so.0
16 0x00007ffff49e7bbf in clone () from /lib64/libc.so.6
(gdb) li
1831 if (best && BATcount(as->format[firstcol].c)) {
1832 BUN limit;
1833 int width;
1834
1835 for (attr = 0; attr < as->nr_attrs; attr++) {
1836 width = as->format[attr].c->twidth;
1837 switch (width){
1838 case 1:
1839 trimerrors(bte);
1840 break;
(gdb) p attr
$1 = 1
(gdb) p as
$2 = (Tablet *) 0x7fffd49d4170
(gdb) p *as
$3 = {offset = 0, nr = 1, nr_attrs = 2, format = 0x7fffc00b2440, error = 0x0, tryall = 0, filename = 0x0, complaints = 0x0}
(gdb) p as->nr_attrs
$4 = 2
(gdb) p as->format
$5 = (Column *) 0x7fffc00b2440
(gdb) p as->format[attr]
$6 = {name = 0x7fffc0070d70 "%cn", sep = 0x7fffc00e8390 "\n", rsep = 0x7fffc00e8390 "\n", seplen = 1, type = 0x7fffc006b050 "oid", adt = 6, c = 0x0, ci = {b = 0x0, tvid = 0}, p = 0, tabs = 0, nullstr = 0x7fffc006b020 "null", null_length = 4,
width = 0, maxwidth = 0, fieldstart = 0, fieldwidth = 0, scale = 0, precision = 0, tostr = 0x7fffef36a9f9 <_ASCIIadt_toStr>, frstr = 0x7fffef36a617 <_ASCIIadt_frStr>, extra = 0x7fffc0070ce0, data = 0x7fffc01d6a30, skip = 1, len = 8, nillen = 8,
ws = 1 '\001', quote = 0 '\000', nildata = 0x7ffff7738218 <lng_nil>, size = 8}
(gdb) p as->format[attr].c
$7 = (BAT *) 0x0
(gdb) p as->format[attr].c->twidth
There is no member named twidth.

Attached file: best-effort-crash.sql (application/sql, 3651 bytes)
Description: test script

Comment 25869

Date: 2017-11-12 15:22:02 +0100
From: MonetDB Mercurial Repository <>

Changeset 03187aaf584a made by Niels Nes niels@cwi.nl in the MonetDB repo, refers to this bug.

For complete details, see https//devmonetdborg/hg/MonetDB?cmd=changeset;node=03187aaf584a

Changeset description:

fixed bug #6442, skip columns also in the best effort case

Comment 25870

Date: 2017-11-12 15:22:46 +0100
From: @njnes

skip not needed columns (solves crash on correctly mis
sing bat)

Comment 25871

Date: 2017-11-12 15:31:17 +0100
From: @drstmane

Thanks!