-
Notifications
You must be signed in to change notification settings - Fork 0
/
content_db_to_berkeley_db_btree.pl
90 lines (62 loc) · 1.88 KB
/
content_db_to_berkeley_db_btree.pl
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
#!/usr/bin/perl
# Author: Trizen
# Date: 03 April 2023
# https://github.com/trizen
# Convert a GDBM database to a Berkeley database, and use JSON for encoding in order to make it portable.
use 5.036;
use DB_File;
use GDBM_File;
no warnings 'once';
use JSON::XS qw(encode_json);
use Storable qw(thaw);
use constant {
# Compress the values of the content database with Zstandard.
# When enabled, the content database will be ~3x smaller.
USE_ZSTD => 1,
};
scalar(@ARGV) == 2 or die "usage: $0 [input.dbm] [output.dbm]";
my $input_file = $ARGV[0];
my $output_file = $ARGV[1];
if (not -f $input_file) {
die "Input file <<$input_file>> does not exist!\n";
}
if (-e $output_file) {
die "Output file <<$output_file>> already exists!\n";
}
tie(my %input, 'GDBM_File', $input_file, &GDBM_READER, 0555)
or die "Can't access database <<$input_file>>: $!";
tie(my %output, 'DB_File', $output_file, O_CREAT | O_RDWR, 0666, $DB_BTREE)
or die "Can't create database <<$output_file>>: $!";
if (USE_ZSTD) {
require IO::Compress::Zstd;
require IO::Uncompress::UnZstd;
}
sub zstd_encode ($data) {
IO::Compress::Zstd::zstd(\$data, \my $zstd_data)
or die "zstd failed: $IO::Compress::Zstd::ZstdError\n";
return $zstd_data;
}
sub zstd_decode ($zstd_data) {
IO::Uncompress::UnZstd::unzstd(\$zstd_data, \my $decoded_data)
or die "unzstd failed: $IO::Uncompress::UnZstd::UnZstdError\n";
return $decoded_data;
}
sub decode_content_entry ($entry) {
my $data = $entry;
if (USE_ZSTD) {
$data = zstd_decode($data);
}
return thaw($data);
}
sub encode_content_entry ($entry) {
my $data = encode_json($entry);
if (USE_ZSTD) {
$data = zstd_encode($data);
}
return $data;
}
while (my ($key, $value) = each %input) {
$output{$key} = encode_content_entry(decode_content_entry($value));
}
untie(%input);
untie(%output);