/
slurper.conf
80 lines (65 loc) · 2.59 KB
/
slurper.conf
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
#############################################################################
#
# A configuration file which can be used with the
# "--config-file" option.
#
#############################################################################
# A name used for the PID file, as well as the log filename, to support
# multiple Slurpers instances working from the same installation directory.
#
DATASOURCE_NAME = test
# The source directory. This must be a fully_qualified URI.
#
SRC_DIR = file:/tmp/slurper/in
# The work directory. This must be a fully_qualified URI, and must be on the
# same file system as SRC_DIR.
#
WORK_DIR = file:/tmp/slurper/work
# The completed directory. This must be a fully_qualified URI, and must be on the
# same file system as SRC_DIR.
#
COMPLETE_DIR = file:/tmp/slurper/complete
# Whether the file should be removed after the copy has completed.
# Either this or COMPLETE_DIR must be specified.
#
REMOVE_AFTER_COPY = false
# The error directory. This must be a fully_qualified URI, and must be on the
# same file system as SRC_DIR.
#
ERROR_DIR = file:/tmp/slurper/error
# The destination staging directory. This must be a fully_qualified URI, and must be on the
# same file system as DEST_DIR.
#
DEST_STAGING_DIR = hdfs:/tmp/slurper/stage
# The destination directory. This must be a fully_qualified URI.
#
DEST_DIR = hdfs:/tmp/slurper/dest
# The compression codec which should be used to compress the output.
#
# COMPRESSION_CODEC = com.hadoop.compression.lzo.LzopCodec
# If the destination file is LZOP, this option will create an index file.
#
# CREATE_LZO_INDEX = true
# Reads the destination file after the copy has completed and verifies
# its integrity.
#
# VERIFY = true
# A script which can be called to dynamically determine the destination path.
# The standard input will contain a single line with the fully qualified URI
# of the source file, and the script must put the destination fully qualified URI
# on standard out.
#
# This and the DEST_DIR configuration setting are mutually exclusive; only one can be set.
#
# SCRIPT = /tmp/sample-python.py
# A script is called prior to the file being copied.
# The standard input will contain a single line with the fully qualified URI
# of the source file in the source work directory (see WORK_DIR above).
# The script must put the modified (or unchanged) fully qualified URI of the
# file in the WORK_DIR. This script can perform pre-processing manipulations
# such as decompressing the file, or injecting a date/time into the filename.
#
# WORK_SCRIPT = /tmp/sample-stage-python.py
# The number of threads to be used for slurping
#
THREADS = 1