Skip to content

Commit

Permalink
Fix btree byte swapping for overflow data
Browse files Browse the repository at this point in the history
When operating on a btree database file of the opposite endianness,
libdb2 was swapping the wrong bytes if a record had a short key but
overflow data.  Fix this bug by not incrementing p when swapping a
P_BIGKEY overflow pointer, and by always skipping the full key size
before swapping a P_BIGDATA overflow pointer (instead of assuming that
a P_BIGKEY pointer always precedes a P_BIGDATA pointer).

(cherry picked from commit b6238a6)

ticket: 8485
version_fixed: 1.14.4
  • Loading branch information
tlyu committed Sep 2, 2016
1 parent e4f49ab commit 927825c
Showing 1 changed file with 9 additions and 6 deletions.
15 changes: 9 additions & 6 deletions src/plugins/kdb/db2/libdb2/btree/bt_conv.c
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ static char sccsid[] = "@(#)bt_conv.c 8.5 (Berkeley) 8/17/94";
#include <sys/param.h>

#include <stdio.h>
#include <string.h>

#include "db-int.h"
#include "btree.h"
Expand All @@ -67,6 +68,7 @@ __bt_pgin(t, pg, pp)
indx_t i, top;
u_char flags;
char *p;
u_int32_t ksize;

if (!F_ISSET(((BTREE *)t), B_NEEDSWAP))
return;
Expand Down Expand Up @@ -104,6 +106,7 @@ __bt_pgin(t, pg, pp)
M_16_SWAP(h->linp[i]);
p = (char *)GETBLEAF(h, i);
P_32_SWAP(p);
memcpy(&ksize, p, sizeof(ksize));
p += sizeof(u_int32_t);
P_32_SWAP(p);
p += sizeof(u_int32_t);
Expand All @@ -112,11 +115,10 @@ __bt_pgin(t, pg, pp)
p += sizeof(u_char);
if (flags & P_BIGKEY) {
P_32_SWAP(p);
p += sizeof(db_pgno_t);
P_32_SWAP(p);
P_32_SWAP(p + sizeof(db_pgno_t));
}
if (flags & P_BIGDATA) {
p += sizeof(u_int32_t);
p += ksize;
P_32_SWAP(p);
p += sizeof(db_pgno_t);
P_32_SWAP(p);
Expand All @@ -135,6 +137,7 @@ __bt_pgout(t, pg, pp)
indx_t i, top;
u_char flags;
char *p;
u_int32_t ksize;

if (!F_ISSET(((BTREE *)t), B_NEEDSWAP))
return;
Expand Down Expand Up @@ -162,6 +165,7 @@ __bt_pgout(t, pg, pp)
}
else if ((h->flags & P_TYPE) == P_BLEAF)
for (i = 0; i < top; i++) {
ksize = GETBLEAF(h, i)->ksize;
p = (char *)GETBLEAF(h, i);
P_32_SWAP(p);
p += sizeof(u_int32_t);
Expand All @@ -172,11 +176,10 @@ __bt_pgout(t, pg, pp)
p += sizeof(u_char);
if (flags & P_BIGKEY) {
P_32_SWAP(p);
p += sizeof(db_pgno_t);
P_32_SWAP(p);
P_32_SWAP(p + sizeof(db_pgno_t));
}
if (flags & P_BIGDATA) {
p += sizeof(u_int32_t);
p += ksize;
P_32_SWAP(p);
p += sizeof(db_pgno_t);
P_32_SWAP(p);
Expand Down

0 comments on commit 927825c

Please sign in to comment.