Skip to content

Commit 6a4fe99

Browse files
montywimontywi
committed
Improvements for myisamchk
These changes was done as part of fixing MDEV-36858 MariaDB MyISAM secondary indexes silently break for tables > 10B rows Changes done in myisamchk: - Tables that are checked are opened in readonly mode if --force is not used. - *.MYD files will be opened in readonly mode for repair if --quick is used. - Added information about check progress if --verbose is used. - Output information about repaired/checked rows every 10000 rows instead of every 1000 rows. Note that this also affects aria_chk - Store open file mode in share->index_mode and share->data_mode instead of in share->mode. - Added new option --keys-active= as a simpler version of keys-used. - Changed output for "myisamchk -dvv" to get nicer output for tables with 10 billion rows.
1 parent 675dffb commit 6a4fe99

File tree

12 files changed

+135
-46
lines changed

12 files changed

+135
-46
lines changed

‎include/my_base.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,13 @@
5050
#define HA_OPEN_FOR_CREATE 4096U
5151
#define HA_OPEN_FOR_DROP (1U << 13) /* Open part of drop */
5252
#define HA_OPEN_GLOBAL_TMP_TABLE (1U << 14) /* TMP table used by repliction */
53+
/*
54+
This is to signal that the table will not be cached by the caller
55+
and the table should be open in read-only mode if the tool requests
56+
that
57+
*/
58+
#define HA_OPEN_FORCE_MODE (1U << 15) /* Force open mode */
59+
#define HA_OPEN_DATA_READONLY (1U << 16) /* Use readonly for data */
5360

5461
/*
5562
Allow opening even if table is incompatible as this is for ALTER TABLE which

‎include/my_global.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1027,7 +1027,7 @@ typedef ulong myf; /* Type of MyFlags in my_funcs */
10271027
#define YESNO(X) ((X) ? "yes" : "no")
10281028

10291029
#define MY_HOW_OFTEN_TO_ALARM 2 /* How often we want info on screen */
1030-
#define MY_HOW_OFTEN_TO_WRITE 10000 /* How often we want info on screen */
1030+
#define MY_HOW_OFTEN_TO_WRITE 100000 /* How often we want info on screen */
10311031

10321032
#include <my_byteorder.h>
10331033

‎include/myisamchk.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,7 @@ typedef struct st_handler_check_param
116116
int (*fix_record)(struct st_myisam_info *info, uchar *record, int keynum);
117117

118118
mysql_mutex_t print_msg_mutex;
119-
my_bool need_print_msg_lock;
119+
my_bool need_print_msg_lock, status_reporting;
120120
myf malloc_flags;
121121
} HA_CHECK;
122122

‎mysql-test/main/long_unique.result‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,8 @@ Ignored NO
4242
MyISAM file: DATADIR/test/t1
4343
Record format: Packed
4444
Character set: latin1_swedish_ci (8)
45-
Data records: 10 Deleted blocks: 0
46-
Recordlength: 12
45+
Data records: 10 Deleted blocks: 0
46+
Recordlength: 12
4747

4848
table description:
4949
Key Start Len Index Type
@@ -130,8 +130,8 @@ insert into t1 values(1),(2),(3),(4),(5),(8),(7);
130130
MyISAM file: DATADIR/test/t1
131131
Record format: Packed
132132
Character set: latin1_swedish_ci (8)
133-
Data records: 7 Deleted blocks: 0
134-
Recordlength: 12
133+
Data records: 7 Deleted blocks: 0
134+
Recordlength: 12
135135

136136
table description:
137137
Key Start Len Index Type
@@ -365,8 +365,8 @@ t1 0 e 1 e A NULL NULL NULL YES HASH NO
365365
MyISAM file: DATADIR/test/t1
366366
Record format: Packed
367367
Character set: latin1_swedish_ci (8)
368-
Data records: 8 Deleted blocks: 0
369-
Recordlength: 3040
368+
Data records: 8 Deleted blocks: 0
369+
Recordlength: 3040
370370

371371
table description:
372372
Key Start Len Index Type
@@ -722,8 +722,8 @@ t1 0 b 4 h A NULL NULL NULL YES HASH NO
722722
MyISAM file: DATADIR/test/t1
723723
Record format: Packed
724724
Character set: latin1_swedish_ci (8)
725-
Data records: 9 Deleted blocks: 0
726-
Recordlength: 5059
725+
Data records: 9 Deleted blocks: 0
726+
Recordlength: 5059
727727

728728
table description:
729729
Key Start Len Index Type

‎mysql-test/main/myisam.result‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2402,8 +2402,8 @@ KEY (c2)
24022402
MyISAM file: MYSQLD_DATADIR/test/t1
24032403
Record format: Packed
24042404
Character set: utf8mb3_general_ci (33)
2405-
Data records: 0 Deleted blocks: 0
2406-
Recordlength: 94
2405+
Data records: 0 Deleted blocks: 0
2406+
Recordlength: 94
24072407

24082408
table description:
24092409
Key Start Len Index Type
@@ -2690,8 +2690,8 @@ KEY (d)
26902690
MyISAM file: MYSQLD_DATADIR/test/t1
26912691
Record format: Fixed length
26922692
Character set: latin1_swedish_ci (8)
2693-
Data records: 0 Deleted blocks: 0
2694-
Recordlength: 13
2693+
Data records: 0 Deleted blocks: 0
2694+
Recordlength: 13
26952695

26962696
table description:
26972697
Key Start Len Index Type

‎mysql-test/suite/vcol/r/vcol_keys_myisam.result‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -244,8 +244,8 @@ a b c
244244
MyISAM file: datadir/test/t1
245245
Record format: Fixed length
246246
Character set: latin1_swedish_ci (8)
247-
Data records: 1001 Deleted blocks: 0
248-
Recordlength: 9
247+
Data records: 1001 Deleted blocks: 0
248+
Recordlength: 9
249249

250250
table description:
251251
Key Start Len Index Type

‎storage/myisam/mi_check.c‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -469,6 +469,11 @@ int chk_key(HA_CHECK *param, register MI_INFO *info)
469469
if (chk_index(param,info,keyinfo,share->state.key_root[key],info->buff,
470470
&keys, param->key_crc+key,1))
471471
DBUG_RETURN(-1);
472+
if ((param->testflag & T_WRITE_LOOP) && param->verbose)
473+
{
474+
puts(" \r");
475+
fflush(stdout);
476+
}
472477
if(!(keyinfo->flag & (HA_FULLTEXT | HA_SPATIAL)))
473478
{
474479
if (keys != info->state->records)
@@ -565,7 +570,8 @@ int chk_key(HA_CHECK *param, register MI_INFO *info)
565570
puts("");
566571
}
567572
if (param->key_file_blocks != info->state->key_file_length &&
568-
param->keys_in_use != ~(ulonglong) 0)
573+
mi_is_all_keys_active(share->state.key_map, share->base.keys) &&
574+
!full_text_keys)
569575
mi_check_print_warning(param, "Some data are unreferenced in keyfile");
570576
if (found_keys != full_text_keys)
571577
param->record_checksum=old_record_checksum-init_checksum; /* Remove delete links */
@@ -870,6 +876,15 @@ static int chk_index(HA_CHECK *param, MI_INFO *info, MI_KEYDEF *keyinfo,
870876
goto err;
871877
}
872878
param->record_checksum+=(ha_checksum) record;
879+
if ((param->testflag & T_WRITE_LOOP) && param->verbose &&
880+
(*keys % WRITE_COUNT) == 0)
881+
{
882+
char llbuff[22];
883+
ulonglong records= info->state->records;
884+
printf("%15s (%3.4f%%)\r", llstr(*keys, llbuff),
885+
((double) *keys / (records > *keys ? records : *keys)) *100);
886+
fflush(stdout);
887+
}
873888
}
874889
if (keypos != endpos)
875890
{
@@ -3878,6 +3893,7 @@ static int sort_key_write(MI_SORT_PARAM *sort_param, const void *a)
38783893
{
38793894
mi_check_print_error(param,
38803895
"Internal error: Keys are not in order from sort");
3896+
DBUG_ASSERT(0);
38813897
return(1);
38823898
}
38833899
#endif

‎storage/myisam/mi_close.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ int mi_close(register MI_INFO *info)
8282
(the server might want to reopen it, and mi_lock_database() only
8383
writes the state for non-temp ones)
8484
*/
85-
if (share->mode != O_RDONLY &&
85+
if (share->index_mode != O_RDONLY &&
8686
(mi_is_crashed(info) || (share->temporary && !share->deleting)))
8787
mi_state_info_write(share->kfile, &share->state, 1);
8888
/* Decrement open count must be last I/O on this file. */

‎storage/myisam/mi_dynrec.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ my_bool mi_dynmap_file(MI_INFO *info, my_off_t size)
8383
*/
8484
info->s->file_map= (uchar*)
8585
my_mmap(0, (size_t) size,
86-
info->s->mode==O_RDONLY ? PROT_READ :
86+
info->s->index_mode == O_RDONLY ? PROT_READ :
8787
PROT_READ | PROT_WRITE,
8888
MAP_SHARED | MAP_NORESERVE,
8989
info->dfile, 0L);

‎storage/myisam/mi_open.c‎

Lines changed: 28 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,8 @@ MI_INFO *test_if_reopen(char *filename)
8282

8383
MI_INFO *mi_open(const char *name, int mode, uint open_flags)
8484
{
85-
int lock_error,kfile,open_mode,save_errno,have_rtree=0, realpath_err;
85+
int lock_error,kfile,save_errno,have_rtree=0, realpath_err;
86+
int open_mode, try_open_mode;
8687
uint i,j,len,errpos,head_length,base_pos,offset,info_length,keys,
8788
key_parts,unique_key_parts,base_key_parts,fulltext_keys,uniques;
8889
uint internal_table= open_flags & HA_OPEN_INTERNAL_TABLE;
@@ -138,18 +139,31 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags)
138139
});
139140

140141
DEBUG_SYNC_C("mi_open_kfile");
142+
143+
/*
144+
We first try to open the file on read-write mode to ensure that
145+
the table is usable for future read and write queries in
146+
MariaDB. Only if the read-write mode fails we try to readonly.
147+
*/
148+
try_open_mode= (open_flags & HA_OPEN_FORCE_MODE) ? mode : O_RDWR;
149+
141150
if ((kfile= mysql_file_open(mi_key_file_kfile, name_buff,
142-
(open_mode= O_RDWR) | O_SHARE | O_NOFOLLOW | O_CLOEXEC,
151+
(open_mode= try_open_mode) | O_SHARE |
152+
O_NOFOLLOW | O_CLOEXEC,
143153
MYF(MY_NOSYMLINKS))) < 0)
144154
{
145-
if ((errno != EROFS && errno != EACCES) ||
155+
if ((errno != EROFS && errno != EACCES) || open_mode == O_RDONLY ||
146156
mode != O_RDONLY ||
147157
(kfile= mysql_file_open(mi_key_file_kfile, name_buff,
148-
(open_mode= O_RDONLY) | O_SHARE| O_NOFOLLOW | O_CLOEXEC,
158+
(open_mode= O_RDONLY) | O_SHARE |
159+
O_NOFOLLOW | O_CLOEXEC,
149160
MYF(MY_NOSYMLINKS))) < 0)
150161
goto err;
151162
}
152-
share->mode=open_mode;
163+
share->index_mode= share->data_mode= open_mode;
164+
if (open_flags & HA_OPEN_DATA_READONLY)
165+
share->data_mode= O_RDONLY;
166+
153167
errpos=1;
154168
if (mysql_file_read(kfile, (uchar*)&share->state.header, head_length,
155169
MYF(MY_NABP)))
@@ -200,7 +214,9 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags)
200214
my_errno= HA_WRONG_CREATE_OPTION;
201215
goto err;
202216
}
203-
share->mode|= O_NOFOLLOW; /* all symlinks are resolved by realpath() */
217+
/* all symlinks are resolved by realpath() */
218+
share->index_mode|= O_NOFOLLOW;
219+
share->data_mode|= O_NOFOLLOW;
204220
}
205221

206222
info_length=mi_uint2korr(share->state.header.header_length);
@@ -588,7 +604,7 @@ MI_INFO *mi_open(const char *name, int mode, uint open_flags)
588604
else
589605
{
590606
share= old_info->s;
591-
if (mode == O_RDWR && share->mode == O_RDONLY)
607+
if (mode == O_RDWR && share->index_mode == O_RDONLY)
592608
{
593609
my_errno=EACCES; /* Can't open in write mode */
594610
goto err;
@@ -1279,10 +1295,11 @@ active seek-positions.
12791295

12801296
int mi_open_datafile(MI_INFO *info, MYISAM_SHARE *share)
12811297
{
1282-
myf flags= MY_WME | (share->mode & O_NOFOLLOW ? MY_NOSYMLINKS: 0);
1298+
myf flags= MY_WME | (share->data_mode & O_NOFOLLOW ? MY_NOSYMLINKS: 0);
12831299
DEBUG_SYNC_C("mi_open_datafile");
12841300
info->dfile= mysql_file_open(mi_key_file_dfile, share->data_file_name,
1285-
share->mode | O_SHARE | O_CLOEXEC, MYF(flags));
1301+
share->data_mode | O_SHARE | O_CLOEXEC,
1302+
MYF(flags));
12861303
return info->dfile >= 0 ? 0 : 1;
12871304
}
12881305

@@ -1291,7 +1308,8 @@ int mi_open_keyfile(MYISAM_SHARE *share)
12911308
{
12921309
if ((share->kfile= mysql_file_open(mi_key_file_kfile,
12931310
share->unique_file_name,
1294-
share->mode | O_SHARE | O_NOFOLLOW | O_CLOEXEC,
1311+
share->index_mode | O_SHARE | O_NOFOLLOW
1312+
| O_CLOEXEC,
12951313
MYF(MY_NOSYMLINKS | MY_WME))) < 0)
12961314
return 1;
12971315
return 0;

0 commit comments

Comments
 (0)