/
rzip.h
306 lines (259 loc) · 7.26 KB
/
rzip.h
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
/*
Copyright (C) Andrew Tridgell 1998,
Con Kolivas 2006-2010
This program is free software; you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation; either version 2 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program; if not, write to the Free Software
Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*/
#define LRZIP_MAJOR_VERSION 0
#define LRZIP_MINOR_VERSION 5
#define LRZIP_MINOR_SUBVERSION 43
#define NUM_STREAMS 2
#include "config.h"
#include <sys/types.h>
#include <unistd.h>
#include <stdio.h>
#include <stddef.h>
#include <stdarg.h>
#include <stdlib.h>
#include <string.h>
#include <signal.h>
#include <bzlib.h>
#include <zlib.h>
#include <pthread.h>
#include <semaphore.h>
#include <sys/resource.h>
#include <netinet/in.h>
#include <sys/time.h>
#include <sys/mman.h>
#include <sys/syscall.h>
#include <lzo/lzoconf.h>
#include <lzo/lzo1x.h>
/* LZMA C Wrapper */
#include "lzma/C/LzmaLib.h"
#ifdef HAVE_STRING_H
#include <string.h>
#endif
#ifdef HAVE_MALLOC_H
#include <malloc.h>
#endif
#include <fcntl.h>
#include <sys/stat.h>
#ifdef HAVE_CTYPE_H
#include <ctype.h>
#endif
#include <errno.h>
#include <sys/mman.h>
/* needed for CRC routines */
#include "lzma/C/7zCrc.h"
#ifndef uchar
#define uchar unsigned char
#endif
#ifndef int32
#if (SIZEOF_INT == 4)
#define int32 int
#elif (SIZEOF_LONG == 4)
#define int32 long
#elif (SIZEOF_SHORT == 4)
#define int32 short
#endif
#endif
#ifndef int16
#if (SIZEOF_INT == 2)
#define int16 int
#elif (SIZEOF_SHORT == 2)
#define int16 short
#endif
#endif
#ifndef uint32
#define uint32 unsigned int32
#endif
#ifndef uint16
#define uint16 unsigned int16
#endif
#ifndef MIN
#define MIN(a, b) ((a) < (b)? (a): (b))
#endif
#ifndef MAX
#define MAX(a, b) ((a) > (b)? (a): (b))
#endif
#if !HAVE_STRERROR
extern char *sys_errlist[];
#define strerror(i) sys_errlist[i]
#endif
#ifndef HAVE_ERRNO_DECL
extern int errno;
#endif
#define likely(x) __builtin_expect(!!(x), 1)
#define unlikely(x) __builtin_expect(!!(x), 0)
typedef long long int i64;
typedef uint16_t u16;
typedef uint32_t u32;
#ifndef MAP_ANONYMOUS
#define MAP_ANONYMOUS MAP_ANON
#endif
#if defined(NOTHREAD) || !defined(_SC_NPROCESSORS_ONLN)
#define PROCESSORS (1)
#else
#define PROCESSORS (sysconf(_SC_NPROCESSORS_ONLN))
#endif
#ifdef _SC_PAGE_SIZE
#define PAGE_SIZE (sysconf(_SC_PAGE_SIZE))
#else
#define PAGE_SIZE (4096)
#endif
#ifdef __APPLE__
#include <sys/sysctl.h>
#define fmemopen fake_fmemopen
#define open_memstream fake_open_memstream
#define memstream_update_buffer fake_open_memstream_update_buffer
#define mremap fake_mremap
static inline i64 get_ram(void)
{
int mib[2];
size_t len;
i64 *p, ramsize;
mib[0] = CTL_HW;
mib[1] = HW_MEMSIZE;
sysctl(mib, 2, NULL, &len, NULL, 0);
p = malloc(len);
sysctl(mib, 2, p, &len, NULL, 0);
ramsize = *p;
return ramsize;
}
#else /* __APPLE__ */
#define memstream_update_buffer(A, B, C) (0)
static inline i64 get_ram(void)
{
return (i64)sysconf(_SC_PHYS_PAGES) * (i64)sysconf(_SC_PAGE_SIZE);
}
#endif
#define FLAG_SHOW_PROGRESS 2
#define FLAG_KEEP_FILES 4
#define FLAG_TEST_ONLY 8
#define FLAG_FORCE_REPLACE 16
#define FLAG_DECOMPRESS 32
#define FLAG_NO_COMPRESS 64
#define FLAG_LZO_COMPRESS 128
#define FLAG_BZIP2_COMPRESS 256
#define FLAG_ZLIB_COMPRESS 512
#define FLAG_ZPAQ_COMPRESS 1024
#define FLAG_VERBOSITY 2048
#define FLAG_VERBOSITY_MAX 4096
#define FLAG_STDIN 8192
#define FLAG_STDOUT 16384
#define FLAG_INFO 32768
#define FLAG_MAXRAM 65536
#define FLAG_UNLIMITED 131072
#define FLAG_VERBOSE (FLAG_VERBOSITY | FLAG_VERBOSITY_MAX)
#define FLAG_NOT_LZMA (FLAG_NO_COMPRESS | FLAG_LZO_COMPRESS | FLAG_BZIP2_COMPRESS | FLAG_ZLIB_COMPRESS | FLAG_ZPAQ_COMPRESS)
#define LZMA_COMPRESS (!(control.flags & FLAG_NOT_LZMA))
#define SHOW_PROGRESS (control.flags & FLAG_SHOW_PROGRESS)
#define KEEP_FILES (control.flags & FLAG_KEEP_FILES)
#define TEST_ONLY (control.flags & FLAG_TEST_ONLY)
#define FORCE_REPLACE (control.flags & FLAG_FORCE_REPLACE)
#define DECOMPRESS (control.flags & FLAG_DECOMPRESS)
#define NO_COMPRESS (control.flags & FLAG_NO_COMPRESS)
#define LZO_COMPRESS (control.flags & FLAG_LZO_COMPRESS)
#define BZIP2_COMPRESS (control.flags & FLAG_BZIP2_COMPRESS)
#define ZLIB_COMPRESS (control.flags & FLAG_ZLIB_COMPRESS)
#define ZPAQ_COMPRESS (control.flags & FLAG_ZPAQ_COMPRESS)
#define VERBOSE (control.flags & FLAG_VERBOSE)
#define VERBOSITY (control.flags & FLAG_VERBOSITY)
#define MAX_VERBOSE (control.flags & FLAG_VERBOSITY_MAX)
#define STDIN (control.flags & FLAG_STDIN)
#define STDOUT (control.flags & FLAG_STDOUT)
#define INFO (control.flags & FLAG_INFO)
#define MAXRAM (control.flags & FLAG_MAXRAM)
#define UNLIMITED (control.flags & FLAG_UNLIMITED)
#define BITS32 (sizeof(long) == 4)
#define CTYPE_NONE 3
#define CTYPE_BZIP2 4
#define CTYPE_LZO 5
#define CTYPE_LZMA 6
#define CTYPE_GZIP 7
#define CTYPE_ZPAQ 8
struct rzip_control {
char *infile;
char *outname;
char *outfile;
char *outdir;
FILE *msgout; //stream for output messages
const char *suffix;
int compression_level;
unsigned char lzma_properties[5]; // lzma properties, encoded
double threshold;
i64 window;
unsigned long flags;
i64 ramsize;
i64 max_chunk;
i64 max_mmap;
int threads;
int nice_val; // added for consistency
int major_version;
int minor_version;
i64 st_size;
long page_size;
} control;
struct stream {
i64 last_head;
uchar *buf;
i64 buflen;
i64 bufp;
int eos;
};
struct stream_info {
struct stream *s;
int num_streams;
int fd;
i64 bufsize;
i64 cur_pos;
i64 initial_pos;
i64 total_read;
long thread_no;
long next_thread;
};
void fatal(const char *format, ...);
void sighandler();
i64 runzip_fd(int fd_in, int fd_out, int fd_hist, i64 expected_size);
void rzip_fd(int fd_in, int fd_out);
void *open_stream_out(int f, int n, i64 limit);
void *open_stream_in(int f, int n);
int write_stream(void *ss, int stream, uchar *p, i64 len);
i64 read_stream(void *ss, int stream, uchar *p, i64 len);
int close_stream_out(void *ss);
int close_stream_in(void *ss);
void flush_buffer(struct stream_info *sinfo, int stream);
void read_config(struct rzip_control *s);
ssize_t write_1g(int fd, void *buf, i64 len);
ssize_t read_1g(int fd, void *buf, i64 len);
void zpipe_compress(FILE *in, FILE *out, FILE *msgout, long long int buf_len, int progress, long thread);
void zpipe_decompress(FILE *in, FILE *out, FILE *msgout, long long int buf_len, int progress, long thread);
const i64 two_gig;
#define print_err(format, args...) do {\
fprintf(stderr, format, ##args); \
} while (0)
#define print_output(format, args...) do {\
fprintf(control.msgout, format, ##args); \
fflush(control.msgout); \
} while (0)
#define print_progress(format, args...) do {\
if (SHOW_PROGRESS) \
print_output(format, ##args); \
} while (0)
#define print_verbose(format, args...) do {\
if (VERBOSE) \
print_output(format, ##args); \
} while (0)
#define print_maxverbose(format, args...) do {\
if (MAX_VERBOSE) \
print_output(format, ##args); \
} while (0)