/
DataScan.cc
1709 lines (1498 loc) · 52.1 KB
/
DataScan.cc
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
// vim: ts=8 sw=2 smarttab
/*
* Ceph - scalable distributed file system
*
* Copyright (C) 2015 Red Hat
*
* This is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License version 2.1, as published by the Free Software
* Foundation. See file COPYING.
*
*/
#include "common/errno.h"
#include "common/ceph_argparse.h"
#include <fstream>
#include "include/util.h"
#include "mds/CInode.h"
#include "cls/cephfs/cls_cephfs_client.h"
#include "DataScan.h"
#include "include/compat.h"
#define dout_subsys ceph_subsys_mds
#undef dout_prefix
#define dout_prefix *_dout << "datascan." << __func__ << ": "
void DataScan::usage()
{
std::cout << "Usage: \n"
<< " cephfs-data-scan init [--force-init]\n"
<< " cephfs-data-scan scan_extents [--force-pool] <data pool name>\n"
<< " cephfs-data-scan scan_inodes [--force-pool] [--force-corrupt] <data pool name>\n"
<< "\n"
<< " --force-corrupt: overrite apparently corrupt structures\n"
<< " --force-init: write root inodes even if they exist\n"
<< " --force-pool: use data pool even if it is not in FSMap\n"
<< "\n"
<< " cephfs-data-scan scan_frags [--force-corrupt]\n"
<< std::endl;
generic_client_usage();
}
bool DataScan::parse_kwarg(
const std::vector<const char*> &args,
std::vector<const char *>::const_iterator &i,
int *r)
{
if (i + 1 == args.end()) {
return false;
}
const std::string arg(*i);
const std::string val(*(i + 1));
if (arg == std::string("--output-dir")) {
if (driver != NULL) {
derr << "Unexpected --output-dir: output already selected!" << dendl;
*r = -EINVAL;
return false;
}
dout(4) << "Using local file output to '" << val << "'" << dendl;
driver = new LocalFileDriver(val, data_io);
return true;
} else if (arg == std::string("--worker_n")) {
std::string err;
n = strict_strtoll(val.c_str(), 10, &err);
if (!err.empty()) {
std::cerr << "Invalid worker number '" << val << "'" << std::endl;
*r = -EINVAL;
return false;
}
return true;
} else if (arg == std::string("--worker_m")) {
std::string err;
m = strict_strtoll(val.c_str(), 10, &err);
if (!err.empty()) {
std::cerr << "Invalid worker count '" << val << "'" << std::endl;
*r = -EINVAL;
return false;
}
return true;
} else if (arg == std::string("--filter-tag")) {
filter_tag = val;
dout(10) << "Applying tag filter: '" << filter_tag << "'" << dendl;
return true;
} else if (arg == std::string("--filesystem")) {
std::shared_ptr<const Filesystem> fs;
*r = fsmap->parse_filesystem(val, &fs);
if (*r != 0) {
std::cerr << "Invalid filesystem '" << val << "'" << std::endl;
return false;
}
fscid = fs->fscid;
return true;
} else {
return false;
}
}
bool DataScan::parse_arg(
const std::vector<const char*> &args,
std::vector<const char *>::const_iterator &i)
{
const std::string arg(*i);
if (arg == "--force-pool") {
force_pool = true;
return true;
} else if (arg == "--force-corrupt") {
force_corrupt = true;
return true;
} else if (arg == "--force-init") {
force_init = true;
return true;
} else {
return false;
}
}
int DataScan::main(const std::vector<const char*> &args)
{
// Parse args
// ==========
if (args.size() < 1) {
usage();
return -EINVAL;
}
// Common RADOS init: open metadata pool
// =====================================
librados::Rados rados;
int r = rados.init_with_context(g_ceph_context);
if (r < 0) {
derr << "RADOS unavailable" << dendl;
return r;
}
std::string const &command = args[0];
std::string data_pool_name;
std::string metadata_pool_name;
// Consume any known --key val or --flag arguments
for (std::vector<const char *>::const_iterator i = args.begin() + 1;
i != args.end(); ++i) {
if (parse_kwarg(args, i, &r)) {
// Skip the kwarg value field
++i;
continue;
} else if (r) {
return r;
}
if (parse_arg(args, i)) {
continue;
}
// Trailing positional argument
if (i + 1 == args.end() &&
(command == "scan_inodes" || command == "scan_extents")) {
data_pool_name = *i;
continue;
}
// Fall through: unhandled
std::cerr << "Unknown argument '" << *i << "'" << std::endl;
return -EINVAL;
}
// If caller didn't specify a namespace, try to pick
// one if only one exists
if (fscid == FS_CLUSTER_ID_NONE) {
if (fsmap->get_filesystems().size() == 1) {
fscid = fsmap->get_filesystems().begin()->first;
} else {
std::cerr << "Specify a filesystem with --filesystem" << std::endl;
return -EINVAL;
}
}
auto fs = fsmap->get_filesystem(fscid);
assert(fs != nullptr);
// Default to output to metadata pool
if (driver == NULL) {
driver = new MetadataDriver();
driver->set_force_corrupt(force_corrupt);
driver->set_force_init(force_init);
dout(4) << "Using metadata pool output" << dendl;
}
dout(4) << "connecting to RADOS..." << dendl;
r = rados.connect();
if (r < 0) {
std::cerr << "couldn't connect to cluster: " << cpp_strerror(r)
<< std::endl;
return r;
}
r = driver->init(rados, fsmap, fscid);
if (r < 0) {
return r;
}
// Initialize data_io for those commands that need it
if (command == "scan_inodes"
|| command == "scan_extents") {
if (data_pool_name.empty()) {
std::cerr << "Data pool not specified" << std::endl;
usage();
return -EINVAL;
}
data_pool_id = rados.pool_lookup(data_pool_name.c_str());
if (data_pool_id < 0) {
std::cerr << "Data pool '" << data_pool_name << "' not found!" << std::endl;
return -ENOENT;
} else {
dout(4) << "data pool '" << data_pool_name
<< "' has ID " << data_pool_id << dendl;
}
if (!fs->mds_map.is_data_pool(data_pool_id)) {
std::cerr << "Warning: pool '" << data_pool_name << "' is not a "
"CephFS data pool!" << std::endl;
if (!force_pool) {
std::cerr << "Use --force-pool to continue" << std::endl;
return -EINVAL;
}
}
dout(4) << "opening data pool '" << data_pool_name << "'" << dendl;
r = rados.ioctx_create(data_pool_name.c_str(), data_io);
if (r != 0) {
return r;
}
}
// Initialize metadata_io from MDSMap for scan_frags
if (command == "scan_frags") {
const auto fs = fsmap->get_filesystem(fscid);
if (fs == nullptr) {
std::cerr << "Filesystem id " << fscid << " does not exist" << std::endl;
return -ENOENT;
}
int64_t const metadata_pool_id = fs->mds_map.get_metadata_pool();
dout(4) << "resolving metadata pool " << metadata_pool_id << dendl;
std::string metadata_pool_name;
int r = rados.pool_reverse_lookup(metadata_pool_id, &metadata_pool_name);
if (r < 0) {
std::cerr << "Pool " << metadata_pool_id
<< " identified in MDS map not found in RADOS!" << std::endl;
return r;
}
r = rados.ioctx_create(metadata_pool_name.c_str(), metadata_io);
if (r != 0) {
return r;
}
}
// Finally, dispatch command
if (command == "scan_inodes") {
return scan_inodes();
} else if (command == "scan_extents") {
return scan_extents();
} else if (command == "scan_frags") {
return scan_frags();
} else if (command == "init") {
return driver->init_roots(fs->mds_map.get_first_data_pool());
} else {
std::cerr << "Unknown command '" << command << "'" << std::endl;
return -EINVAL;
}
}
int MetadataDriver::inject_unlinked_inode(
inodeno_t inono, int mode, int64_t data_pool_id)
{
const object_t oid = InodeStore::get_object_name(inono, frag_t(), ".inode");
// Skip if exists
bool already_exists = false;
int r = root_exists(inono, &already_exists);
if (r) {
return r;
}
if (already_exists && !force_init) {
std::cerr << "Inode 0x" << std::hex << inono << std::dec << " already"
" exists, skipping create. Use --force-init to overwrite"
" the existing object." << std::endl;
return 0;
}
// Compose
InodeStore inode;
inode.inode.ino = inono;
inode.inode.version = 1;
inode.inode.xattr_version = 1;
inode.inode.mode = 0500 | mode;
// Fake dirstat.nfiles to 1, so that the directory doesn't appear to be empty
// (we won't actually give the *correct* dirstat here though)
inode.inode.dirstat.nfiles = 1;
inode.inode.ctime =
inode.inode.mtime = ceph_clock_now(g_ceph_context);
inode.inode.nlink = 1;
inode.inode.truncate_size = -1ull;
inode.inode.truncate_seq = 1;
inode.inode.uid = g_conf->mds_root_ino_uid;
inode.inode.gid = g_conf->mds_root_ino_gid;
// Force layout to default: should we let users override this so that
// they don't have to mount the filesystem to correct it?
inode.inode.layout = file_layout_t::get_default();
inode.inode.layout.pool_id = data_pool_id;
// Assume that we will get our stats wrong, and that we may
// be ignoring dirfrags that exist
inode.damage_flags |= (DAMAGE_STATS | DAMAGE_RSTATS | DAMAGE_FRAGTREE);
// Serialize
bufferlist inode_bl;
::encode(std::string(CEPH_FS_ONDISK_MAGIC), inode_bl);
inode.encode(inode_bl, CEPH_FEATURES_SUPPORTED_DEFAULT);
// Write
r = metadata_io.write_full(oid.name, inode_bl);
if (r != 0) {
derr << "Error writing '" << oid.name << "': " << cpp_strerror(r) << dendl;
return r;
}
return r;
}
int MetadataDriver::root_exists(inodeno_t ino, bool *result)
{
object_t oid = InodeStore::get_object_name(ino, frag_t(), ".inode");
uint64_t size;
time_t mtime;
int r = metadata_io.stat(oid.name, &size, &mtime);
if (r == -ENOENT) {
*result = false;
return 0;
} else if (r < 0) {
return r;
}
*result = true;
return 0;
}
int MetadataDriver::init_roots(int64_t data_pool_id)
{
int r = 0;
r = inject_unlinked_inode(MDS_INO_ROOT, S_IFDIR|0755, data_pool_id);
if (r != 0) {
return r;
}
r = inject_unlinked_inode(MDS_INO_MDSDIR(0), S_IFDIR, data_pool_id);
if (r != 0) {
return r;
}
bool created = false;
r = find_or_create_dirfrag(MDS_INO_MDSDIR(0), frag_t(), &created);
if (r != 0) {
return r;
}
return 0;
}
int MetadataDriver::check_roots(bool *result)
{
int r;
r = root_exists(MDS_INO_ROOT, result);
if (r != 0) {
return r;
}
if (!*result) {
return 0;
}
r = root_exists(MDS_INO_MDSDIR(0), result);
if (r != 0) {
return r;
}
if (!*result) {
return 0;
}
return 0;
}
/**
* Stages:
*
* SERIAL init
* 0. Create root inodes if don't exist
* PARALLEL scan_extents
* 1. Size and mtime recovery: scan ALL objects, and update 0th
* objects with max size and max mtime seen.
* PARALLEL scan_inodes
* 2. Inode recovery: scan ONLY 0th objects, and inject metadata
* into dirfrag OMAPs, creating blank dirfrags as needed. No stats
* or rstats at this stage. Inodes without backtraces go into
* lost+found
* TODO: SERIAL "recover stats"
* 3. Dirfrag statistics: depth first traverse into metadata tree,
* rebuilding dir sizes.
* TODO PARALLEL "clean up"
* 4. Cleanup; go over all 0th objects (and dirfrags if we tagged
* anything onto them) and remove any of the xattrs that we
* used for accumulating.
*/
int parse_oid(const std::string &oid, uint64_t *inode_no, uint64_t *obj_id)
{
if (oid.find(".") == std::string::npos || oid.find(".") == oid.size() - 1) {
return -EINVAL;
}
std::string err;
std::string inode_str = oid.substr(0, oid.find("."));
*inode_no = strict_strtoll(inode_str.c_str(), 16, &err);
if (!err.empty()) {
return -EINVAL;
}
std::string pos_string = oid.substr(oid.find(".") + 1);
*obj_id = strict_strtoll(pos_string.c_str(), 16, &err);
if (!err.empty()) {
return -EINVAL;
}
return 0;
}
int DataScan::scan_extents()
{
return forall_objects(data_io, false, [this](
std::string const &oid,
uint64_t obj_name_ino,
uint64_t obj_name_offset) -> int
{
// Read size
uint64_t size;
time_t mtime;
int r = data_io.stat(oid, &size, &mtime);
if (r != 0) {
dout(4) << "Cannot stat '" << oid << "': skipping" << dendl;
return r;
}
// I need to keep track of
// * The highest object ID seen
// * The size of the highest object ID seen
// * The largest object seen
//
// Given those things, I can later infer the object chunking
// size, the offset of the last object (chunk size * highest ID seen)
// and the actual size (offset of last object + size of highest ID seen)
//
// This logic doesn't take account of striping.
r = ClsCephFSClient::accumulate_inode_metadata(
data_io,
obj_name_ino,
obj_name_offset,
size,
mtime);
if (r < 0) {
derr << "Failed to accumulate metadata data from '"
<< oid << "': " << cpp_strerror(r) << dendl;
return r;
}
return r;
});
}
int DataScan::probe_filter(librados::IoCtx &ioctx)
{
bufferlist filter_bl;
ClsCephFSClient::build_tag_filter("test", &filter_bl);
librados::ObjectCursor range_i;
librados::ObjectCursor range_end;
std::vector<librados::ObjectItem> tmp_result;
librados::ObjectCursor tmp_next;
int r = ioctx.object_list(ioctx.object_list_begin(), ioctx.object_list_end(),
1, filter_bl, &tmp_result, &tmp_next);
return r >= 0;
}
int DataScan::forall_objects(
librados::IoCtx &ioctx,
bool untagged_only,
std::function<int(std::string, uint64_t, uint64_t)> handler
)
{
librados::ObjectCursor range_i;
librados::ObjectCursor range_end;
ioctx.object_list_slice(
ioctx.object_list_begin(),
ioctx.object_list_end(),
n,
m,
&range_i,
&range_end);
bufferlist filter_bl;
bool legacy_filtering = false;
if (untagged_only) {
// probe to deal with older OSDs that don't support
// the cephfs pgls filtering mode
legacy_filtering = !probe_filter(ioctx);
if (!legacy_filtering) {
ClsCephFSClient::build_tag_filter(filter_tag, &filter_bl);
}
}
int r = 0;
while(range_i < range_end) {
std::vector<librados::ObjectItem> result;
int r = ioctx.object_list(range_i, range_end, 1,
filter_bl, &result, &range_i);
if (r < 0) {
derr << "Unexpected error listing objects: " << cpp_strerror(r) << dendl;
return r;
}
for (const auto &i : result) {
const std::string &oid = i.oid;
uint64_t obj_name_ino = 0;
uint64_t obj_name_offset = 0;
r = parse_oid(oid, &obj_name_ino, &obj_name_offset);
if (r != 0) {
dout(4) << "Bad object name '" << oid << "', skipping" << dendl;
continue;
}
if (untagged_only && legacy_filtering) {
dout(20) << "Applying filter to " << oid << dendl;
// We are only interested in 0th objects during this phase: we touched
// the other objects during scan_extents
if (obj_name_offset != 0) {
dout(20) << "Non-zeroth object" << dendl;
continue;
}
bufferlist scrub_tag_bl;
int r = ioctx.getxattr(oid, "scrub_tag", scrub_tag_bl);
if (r >= 0) {
std::string read_tag;
bufferlist::iterator q = scrub_tag_bl.begin();
try {
::decode(read_tag, q);
if (read_tag == filter_tag) {
dout(20) << "skipping " << oid << " because it has the filter_tag"
<< dendl;
continue;
}
} catch (const buffer::error &err) {
}
dout(20) << "read non-matching tag '" << read_tag << "'" << dendl;
} else {
dout(20) << "no tag read (" << r << ")" << dendl;
}
} else if (untagged_only) {
assert(obj_name_offset == 0);
dout(20) << "OSD matched oid " << oid << dendl;
}
int this_oid_r = handler(oid, obj_name_ino, obj_name_offset);
if (r == 0 && this_oid_r < 0) {
r = this_oid_r;
}
}
}
return r;
}
int DataScan::scan_inodes()
{
bool roots_present;
int r = driver->check_roots(&roots_present);
if (r != 0) {
derr << "Unexpected error checking roots: '"
<< cpp_strerror(r) << "'" << dendl;
return r;
}
if (!roots_present) {
std::cerr << "Some or all system inodes are absent. Run 'init' from "
"one node before running 'scan_inodes'" << std::endl;
return -EIO;
}
return forall_objects(data_io, true, [this](
std::string const &oid,
uint64_t obj_name_ino,
uint64_t obj_name_offset) -> int
{
int r = 0;
AccumulateResult accum_res;
inode_backtrace_t backtrace;
file_layout_t loaded_layout = file_layout_t::get_default();
r = ClsCephFSClient::fetch_inode_accumulate_result(
data_io, oid, &backtrace, &loaded_layout, &accum_res);
if (r == -EINVAL) {
dout(4) << "Accumulated metadata missing from '"
<< oid << ", did you run scan_extents?" << dendl;
return r;
} else if (r < 0) {
dout(4) << "Unexpected error loading accumulated metadata from '"
<< oid << "': " << cpp_strerror(r) << dendl;
// FIXME: this creates situation where if a client has a corrupt
// backtrace/layout, we will fail to inject it. We should (optionally)
// proceed if the backtrace/layout is corrupt but we have valid
// accumulated metadata.
return r;
}
const time_t file_mtime = accum_res.max_mtime;
uint64_t file_size = 0;
bool have_backtrace = !(backtrace.ancestors.empty());
// This is the layout we will use for injection, populated either
// from loaded_layout or from best guesses
file_layout_t guessed_layout;
guessed_layout.pool_id = data_pool_id;
// Calculate file_size, guess the layout
if (accum_res.ceiling_obj_index > 0) {
uint32_t chunk_size = file_layout_t::get_default().object_size;
// When there are multiple objects, the largest object probably
// indicates the chunk size. But not necessarily, because files
// can be sparse. Only make this assumption if size seen
// is a power of two, as chunk sizes typically are.
if ((accum_res.max_obj_size & (accum_res.max_obj_size - 1)) == 0) {
chunk_size = accum_res.max_obj_size;
}
if (loaded_layout.pool_id == -1) {
// If no stashed layout was found, guess it
guessed_layout.object_size = chunk_size;
guessed_layout.stripe_unit = chunk_size;
guessed_layout.stripe_count = 1;
} else if (!loaded_layout.is_valid() ||
loaded_layout.object_size < accum_res.max_obj_size) {
// If the max size seen exceeds what the stashed layout claims, then
// disbelieve it. Guess instead. Same for invalid layouts on disk.
dout(4) << "bogus xattr layout on 0x" << std::hex << obj_name_ino
<< std::dec << ", ignoring in favour of best guess" << dendl;
guessed_layout.object_size = chunk_size;
guessed_layout.stripe_unit = chunk_size;
guessed_layout.stripe_count = 1;
} else {
// We have a stashed layout that we can't disprove, so apply it
guessed_layout = loaded_layout;
dout(20) << "loaded layout from xattr:"
<< " os: " << guessed_layout.object_size
<< " sc: " << guessed_layout.stripe_count
<< " su: " << guessed_layout.stripe_unit
<< dendl;
// User might have transplanted files from a pool with a different
// ID, so whatever the loaded_layout says, we'll force the injected
// layout to point to the pool we really read from
guessed_layout.pool_id = data_pool_id;
}
if (guessed_layout.stripe_count == 1) {
// Unstriped file: simple chunking
file_size = guessed_layout.object_size * accum_res.ceiling_obj_index
+ accum_res.ceiling_obj_size;
} else {
// Striped file: need to examine the last stripe_count objects
// in the file to determine the size.
// How many complete (i.e. not last stripe) objects?
uint64_t complete_objs = 0;
if (accum_res.ceiling_obj_index > guessed_layout.stripe_count - 1) {
complete_objs = (accum_res.ceiling_obj_index / guessed_layout.stripe_count) * guessed_layout.stripe_count;
} else {
complete_objs = 0;
}
// How many potentially-short objects (i.e. last stripe set) objects?
uint64_t partial_objs = accum_res.ceiling_obj_index + 1 - complete_objs;
dout(10) << "calculating striped size from complete objs: "
<< complete_objs << ", partial objs: " << partial_objs
<< dendl;
// Maximum amount of data that may be in the incomplete objects
uint64_t incomplete_size = 0;
// For each short object, calculate the max file size within it
// and accumulate the maximum
for (uint64_t i = complete_objs; i < complete_objs + partial_objs; ++i) {
char buf[60];
snprintf(buf, sizeof(buf), "%llx.%08llx",
(long long unsigned)obj_name_ino, (long long unsigned)i);
uint64_t osize(0);
time_t omtime(0);
r = data_io.stat(std::string(buf), &osize, &omtime);
if (r == 0) {
if (osize > 0) {
// Upper bound within this object
uint64_t upper_size = (osize - 1) / guessed_layout.stripe_unit
* (guessed_layout.stripe_unit * guessed_layout.stripe_count)
+ (i % guessed_layout.stripe_count)
* guessed_layout.stripe_unit + (osize - 1)
% guessed_layout.stripe_unit + 1;
incomplete_size = MAX(incomplete_size, upper_size);
}
} else if (r == -ENOENT) {
// Absent object, treat as size 0 and ignore.
} else {
// Unexpected error, carry r to outer scope for handling.
break;
}
}
if (r != 0 && r != -ENOENT) {
derr << "Unexpected error checking size of ino 0x" << std::hex
<< obj_name_ino << std::dec << ": " << cpp_strerror(r) << dendl;
return r;
}
file_size = complete_objs * guessed_layout.object_size
+ incomplete_size;
}
} else {
file_size = accum_res.ceiling_obj_size;
if (loaded_layout.pool_id < 0
|| loaded_layout.object_size < accum_res.max_obj_size) {
// No layout loaded, or inconsistent layout, use default
guessed_layout = file_layout_t::get_default();
guessed_layout.pool_id = data_pool_id;
} else {
guessed_layout = loaded_layout;
}
}
// Santity checking backtrace ino against object name
if (have_backtrace && backtrace.ino != obj_name_ino) {
dout(4) << "Backtrace ino 0x" << std::hex << backtrace.ino
<< " doesn't match object name ino 0x" << obj_name_ino
<< std::dec << dendl;
have_backtrace = false;
}
InodeStore dentry;
build_file_dentry(obj_name_ino, file_size, file_mtime, guessed_layout, &dentry);
// Inject inode to the metadata pool
if (have_backtrace) {
inode_backpointer_t root_bp = *(backtrace.ancestors.rbegin());
if (MDS_INO_IS_MDSDIR(root_bp.dirino)) {
/* Special case for strays: even if we have a good backtrace,
* don't put it in the stray dir, because while that would technically
* give it linkage it would still be invisible to the user */
r = driver->inject_lost_and_found(obj_name_ino, dentry);
if (r < 0) {
dout(4) << "Error injecting 0x" << std::hex << backtrace.ino
<< std::dec << " into lost+found: " << cpp_strerror(r) << dendl;
if (r == -EINVAL) {
dout(4) << "Use --force-corrupt to overwrite structures that "
"appear to be corrupt" << dendl;
}
}
} else {
/* Happy case: we will inject a named dentry for this inode */
r = driver->inject_with_backtrace(backtrace, dentry);
if (r < 0) {
dout(4) << "Error injecting 0x" << std::hex << backtrace.ino
<< std::dec << " with backtrace: " << cpp_strerror(r) << dendl;
if (r == -EINVAL) {
dout(4) << "Use --force-corrupt to overwrite structures that "
"appear to be corrupt" << dendl;
}
}
}
} else {
/* Backtrace-less case: we will inject a lost+found dentry */
r = driver->inject_lost_and_found(
obj_name_ino, dentry);
if (r < 0) {
dout(4) << "Error injecting 0x" << std::hex << obj_name_ino
<< std::dec << " into lost+found: " << cpp_strerror(r) << dendl;
if (r == -EINVAL) {
dout(4) << "Use --force-corrupt to overwrite structures that "
"appear to be corrupt" << dendl;
}
}
}
return r;
});
}
bool DataScan::valid_ino(inodeno_t ino) const
{
return (ino >= inodeno_t((1ull << 40)))
|| (MDS_INO_IS_STRAY(ino))
|| (MDS_INO_IS_MDSDIR(ino))
|| ino == MDS_INO_ROOT
|| ino == MDS_INO_CEPH;
}
int DataScan::scan_frags()
{
bool roots_present;
int r = driver->check_roots(&roots_present);
if (r != 0) {
derr << "Unexpected error checking roots: '"
<< cpp_strerror(r) << "'" << dendl;
return r;
}
if (!roots_present) {
std::cerr << "Some or all system inodes are absent. Run 'init' from "
"one node before running 'scan_inodes'" << std::endl;
return -EIO;
}
return forall_objects(metadata_io, true, [this](
std::string const &oid,
uint64_t obj_name_ino,
uint64_t obj_name_offset) -> int
{
int r = 0;
r = parse_oid(oid, &obj_name_ino, &obj_name_offset);
if (r != 0) {
dout(4) << "Bad object name '" << oid << "', skipping" << dendl;
return r;
}
if (obj_name_ino < (1ULL << 40)) {
// FIXME: we're skipping stray dirs here: if they're
// orphaned then we should be resetting them some other
// way
dout(10) << "Skipping system ino " << obj_name_ino << dendl;
return 0;
}
AccumulateResult accum_res;
inode_backtrace_t backtrace;
// Default to inherit layout (i.e. no explicit layout on dir) which is
// expressed as a zeroed layout struct (see inode_t::has_layout)
file_layout_t loaded_layout;
int parent_r = 0;
bufferlist parent_bl;
int layout_r = 0;
bufferlist layout_bl;
bufferlist op_bl;
librados::ObjectReadOperation op;
op.getxattr("parent", &parent_bl, &parent_r);
op.getxattr("layout", &layout_bl, &layout_r);
r = metadata_io.operate(oid, &op, &op_bl);
if (r != 0 && r != -ENODATA) {
derr << "Unexpected error reading backtrace: " << cpp_strerror(parent_r) << dendl;
return r;
}
if (parent_r != -ENODATA) {
try {
bufferlist::iterator q = parent_bl.begin();
backtrace.decode(q);
} catch (buffer::error &e) {
dout(4) << "Corrupt backtrace on '" << oid << "': " << e << dendl;
if (!force_corrupt) {
return -EINVAL;
} else {
// Treat backtrace as absent: we'll inject into lost+found
backtrace = inode_backtrace_t();
}
}
}
if (layout_r != -ENODATA) {
try {
bufferlist::iterator q = layout_bl.begin();
::decode(loaded_layout, q);
} catch (buffer::error &e) {
dout(4) << "Corrupt layout on '" << oid << "': " << e << dendl;
if (!force_corrupt) {
return -EINVAL;
}
}
}
bool have_backtrace = !(backtrace.ancestors.empty());
// Santity checking backtrace ino against object name
if (have_backtrace && backtrace.ino != obj_name_ino) {
dout(4) << "Backtrace ino 0x" << std::hex << backtrace.ino
<< " doesn't match object name ino 0x" << obj_name_ino
<< std::dec << dendl;
have_backtrace = false;
}
uint64_t fnode_version = 0;
fnode_t fnode;
r = read_fnode(obj_name_ino, frag_t(), &fnode, &fnode_version);
if (r == -EINVAL) {
derr << "Corrupt fnode on " << oid << dendl;
if (force_corrupt) {
fnode.fragstat.mtime = 0;
fnode.fragstat.nfiles = 1;
fnode.fragstat.nsubdirs = 0;
fnode.accounted_fragstat = fnode.fragstat;
} else {
return r;
}
}
InodeStore dentry;
build_dir_dentry(obj_name_ino, fnode.accounted_fragstat,
loaded_layout, &dentry);
// Inject inode to the metadata pool
if (have_backtrace) {
inode_backpointer_t root_bp = *(backtrace.ancestors.rbegin());
if (MDS_INO_IS_MDSDIR(root_bp.dirino)) {
/* Special case for strays: even if we have a good backtrace,
* don't put it in the stray dir, because while that would technically
* give it linkage it would still be invisible to the user */
r = driver->inject_lost_and_found(obj_name_ino, dentry);
if (r < 0) {
dout(4) << "Error injecting 0x" << std::hex << backtrace.ino
<< std::dec << " into lost+found: " << cpp_strerror(r) << dendl;
if (r == -EINVAL) {
dout(4) << "Use --force-corrupt to overwrite structures that "
"appear to be corrupt" << dendl;
}
}
} else {
/* Happy case: we will inject a named dentry for this inode */
r = driver->inject_with_backtrace(backtrace, dentry);
if (r < 0) {
dout(4) << "Error injecting 0x" << std::hex << backtrace.ino
<< std::dec << " with backtrace: " << cpp_strerror(r) << dendl;
if (r == -EINVAL) {
dout(4) << "Use --force-corrupt to overwrite structures that "
"appear to be corrupt" << dendl;
}
}
}
} else {
/* Backtrace-less case: we will inject a lost+found dentry */
r = driver->inject_lost_and_found(
obj_name_ino, dentry);
if (r < 0) {
dout(4) << "Error injecting 0x" << std::hex << obj_name_ino
<< std::dec << " into lost+found: " << cpp_strerror(r) << dendl;
if (r == -EINVAL) {
dout(4) << "Use --force-corrupt to overwrite structures that "
"appear to be corrupt" << dendl;
}
}
}
return r;
});
}
int MetadataTool::read_fnode(
inodeno_t ino, frag_t frag, fnode_t *fnode,
uint64_t *last_version)
{
assert(fnode != NULL);
object_t frag_oid = InodeStore::get_object_name(ino, frag, "");
bufferlist fnode_bl;
int r = metadata_io.omap_get_header(frag_oid.name, &fnode_bl);
*last_version = metadata_io.get_last_version();
if (r < 0) {
return r;
}
bufferlist::iterator old_fnode_iter = fnode_bl.begin();
try {
(*fnode).decode(old_fnode_iter);