-
-
Notifications
You must be signed in to change notification settings - Fork 91
/
tag_duplicates.c
106 lines (90 loc) · 2.53 KB
/
tag_duplicates.c
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
#include <stdlib.h>
#include <string.h>
#include <stdio.h>
#include "tag_duplicates.h"
#include "const.h"
#include "config.h"
#include "thpool.h"
#include "file.h"
#include "chunk.h"
#include "hmap.h"
#include "status.h"
#include "error.h"
#if MULTITHREADING == 1
/** For each chunk following `parent`, add a cleanout_chunk() worker.
*/
static void tag_subchunks(threadpool thpool, const t_chunk *parent)
{
t_chunk chunk;
t_chunk *heap_chunk;
int chunk_id = 1;
memcpy(&chunk, parent, sizeof(t_chunk));
while (get_next_chunk(&chunk, g_file))
{
heap_chunk = malloc(sizeof(t_chunk));
if (heap_chunk == NULL)
die("could not malloc() heap_chunk");
memcpy(heap_chunk, &chunk, sizeof(t_chunk));
/* thpool already prints error unless DISABLE_PRINT is defined */
if (thpool_add_work(thpool, (void*)cleanout_chunk, heap_chunk) != 0)
exit(1);
chunk_id ++;
}
}
/** For each chunk, load it into hmap and remove all duplicates
* in following chunks through `tag_subchunks()`.
*/
void tag_duplicates(void)
{
threadpool thpool;
t_chunk main_chunk = {
.ptr = NULL,
.endptr = NULL
};
/* thpool already prints error unless DISABLE_PRINT is defined */
if ((thpool = thpool_init(g_conf.threads)) == NULL)
exit(1);
while (get_next_chunk(&main_chunk, g_file))
{
populate_hmap(&main_chunk);
tag_subchunks(thpool, &main_chunk);
thpool_wait(thpool);
}
thpool_destroy(thpool);
}
#else /* MULTITHREADING not defined */
/** Cleanout each chunk following the parent
*/
static void tag_subchunks(const t_chunk *parent)
{
t_chunk chunk;
t_chunk *heap_chunk;
memcpy(&chunk, parent, sizeof(t_chunk));
while (get_next_chunk(&chunk, g_file))
{
heap_chunk = malloc(sizeof(t_chunk));
if (heap_chunk == NULL)
die("could not malloc() heap_chunk");
memcpy(heap_chunk, &chunk, sizeof(t_chunk));
cleanout_chunk(heap_chunk);
update_status(CTASK_DONE);
}
}
/** For each chunk, load it into hmap and remove all duplicates
* in following chunks through `tag_subchunks()`.
*/
void tag_duplicates(void)
{
t_chunk main_chunk = {
.ptr = NULL,
.endptr = NULL
};
while (get_next_chunk(&main_chunk, g_file))
{
populate_hmap(&main_chunk);
update_status(CTASK_DONE);
tag_subchunks(&main_chunk);
update_status(CHUNK_DONE);
}
}
#endif