Skip to content

Commit

Permalink
refactor: CLI arg docstring formatting
Browse files Browse the repository at this point in the history
  • Loading branch information
@Sieboldianus
Sieboldianus committed Mar 9, 2021
1 parent d12b4c1 commit aedd00c
Showing 1 changed file with 48 additions and 12 deletions.
60 changes: 48 additions & 12 deletions lbsntransform/config/config.py
View file
Expand Up @@ -93,9 +93,9 @@ def parse_args(self):
default=0,
help='Input source type (id). '
' '
'Defaults to `0`: LBSN '
'* Defaults to `0`: LBSN '
' '
'Other possible values:'
'Other possible values: '
'* `1` - Instagram '
'* `2` - Flickr '
'* `21` - Flickr YFCC100M '
Expand All @@ -117,8 +117,8 @@ def parse_args(self):
'parameter `--input_path_url`. '
'* To specify file types, e.g. whether to '
'process data from `json` '
'or `csv`, or from URLs, see '
'`--file_type` ')
'or `csv`, or from URLs, '
' see `--file_type` ')
local_input_args.add_argument("--file_type",
default='json',
help='Specify filetype '
Expand Down Expand Up @@ -146,6 +146,7 @@ def parse_args(self):
default=False,
help='Input is stacked json. '
' '
' '
'* The typical form of json is '
'`[{json1},{json2}]` '
'* If `--is_stacked_json` is set, '
Expand All @@ -156,6 +157,7 @@ def parse_args(self):
default=False,
help='Json is line separated '
' '
' '
'* The typical form is '
'`[{json1},{json2}]` '
'* If `--is_line_separated_json` is set, '
Expand All @@ -172,6 +174,7 @@ def parse_args(self):
help='Password for hllworker '
'database (Default = empty)'
' '
' '
'* If reading data into `hlldb`, all '
'HLL Worker parameters must be supplied by'
'default. '
Expand All @@ -191,6 +194,7 @@ def parse_args(self):
hllworker_args.add_argument("--dbserveraddress_hllworker",
help='IP for hllworker db, '
' '
' '
'* e.g. `111.11.11.11` '
'* Optionally add '
'port the to use, e.g. `111.11.11.11:5432`. '
Expand All @@ -199,6 +203,7 @@ def parse_args(self):
hllworker_args.add_argument("--dbname_hllworker",
help='DB name for hllworker db '
' '
' '
'* e.g. `hllworkerdb` ',
type=str)
# DB Output
Expand All @@ -216,6 +221,7 @@ def parse_args(self):
dboutput_args.add_argument('-a', "--dbserveraddress_output",
help='IP for output db, '
' '
' '
'* e.g. `111.11.11.11` '
'* Optionally add '
'port to use, e.g. `111.11.11.11:5432`. '
Expand All @@ -224,12 +230,14 @@ def parse_args(self):
dboutput_args.add_argument('-n', "--dbname_output",
help='DB name for output db '
' '
' '
'* e.g. `rawdb` or `hlldb` ',
type=str)
dboutput_args.add_argument("--dbformat_output",
default="lbsn",
help='Format of the out-db. '
' '
' '
'* Either `hll` or `lbsn`. '
'* This setting '
'affects how data is stored, either '
Expand All @@ -250,6 +258,7 @@ def parse_args(self):
dbinput_args.add_argument("--dbserveraddress_input",
help='IP for input-db, '
' '
' '
'* e.g. `111.11.11.11` '
'* Optionally add port to use, e.g. '
'`111.11.11.11:5432`. '
Expand All @@ -258,25 +267,28 @@ def parse_args(self):
dbinput_args.add_argument("--dbname_input",
help='DB name for input-db, '
' '
' '
'* e.g.: `rawdb` ',
type=str)
dbinput_args.add_argument("--dbformat_input",
default="json",
help='Format of the input-db. '
' '
' '
'* Either `lbsn` or `json` '
'* If lbsn is used, the native lbsn raw '
'input mapping (`0`) will be used '
'* If `json` is used, a custom mapping for '
' json must be provided, for mapping database '
' json\'s to the lbsn structure. See '
' [input mappings](/input-mappings/) ',
'json must be provided, for mapping database '
'json\'s to the lbsn structure. See '
'[input mappings](/input-mappings/) ',
type=str)
# Additional args
settings_args = parser.add_argument_group('Additional settings')
settings_args.add_argument('-t', "--transferlimit",
help='Abort after x records. '
' '
' '
'* This can be used to limit the number of '
'records that will be processed. '
'* e.g. `--transferlimit 10000` will process '
Expand All @@ -290,6 +302,7 @@ def parse_args(self):
default=50000,
help='Transfer batch limit x. '
' '
' '
'* Defines after how many '
'parsed records the results will be '
'transferred to the DB. '
Expand All @@ -305,6 +318,7 @@ def parse_args(self):
default=10000,
help='Fetch x records /batch. '
' '
' '
'* If retrieving data from a db (`lbsn`), '
'limit the number of records to fetch at once. '
'* Defaults to 10000 ',
Expand All @@ -314,6 +328,7 @@ def parse_args(self):
action='store_true',
help='Disable reactions. '
' '
' '
'* If set, processing of lbsn reactions will be skipped, '
'* only original posts are transferred. '
'* This is usefull to reduce '
Expand All @@ -326,6 +341,7 @@ def parse_args(self):
default=False,
help='Disable reactions-refs. '
' '
' '
'Enable this option in args '
'to prevent empty posts being stored '
'due to Foreign-Key-Exists Requirement. '
Expand All @@ -339,11 +355,13 @@ def parse_args(self):
action='store_true',
help='Ignore none-geotagged. '
' '
' '
'If set, posts that are not geotagged '
'are ignored during processing.')
settings_args.add_argument("--startwith_db_rownumber",
help='Start with db row x. '
' '
' '
'* Provide a number (row-id) to start '
'processing from live db. '
'* If input db type '
Expand All @@ -356,6 +374,7 @@ def parse_args(self):
settings_args.add_argument("--endwith_db_rownumber",
help='End with db row x. '
' '
' '
'Provide a number (row-id) to end '
'processing from live db', type=int)
settings_args.add_argument("--debug_mode",
Expand All @@ -364,6 +383,7 @@ def parse_args(self):
settings_args.add_argument("--geocode_locations",
help='Path to loc-geocodes. '
' '
' '
'* Provide path to a CSV file with '
'location geocodes '
'* CSV Header must be: '
Expand All @@ -375,17 +395,20 @@ def parse_args(self):
settings_args.add_argument("--ignore_input_source_list",
help='Path to input ignore. '
' '
' '
'Provide a path to a list of input_source '
'types that will be ignored (e.g. to '
'ignore certain bots etc.)',
type=str)
settings_args.add_argument("--mappings_path",
help='Path mappings folder. '
' '
' '
'Provide a path to a custom folder '
'that contains '
'one or more input mapping modules (`*.py`). '
' '
' '
'* Have a look at the two sample mappings '
'in [the resources folder](https://gitlab.vgiscience.de/lbsn/lbsntransform/-/tree/master/resources/mappings). '
'* See how to define custom input mappings '
Expand All @@ -394,6 +417,7 @@ def parse_args(self):
settings_args.add_argument("--input_lbsn_type",
help='Input sub-type '
' '
' '
'* e.g. `post`, `profile`, '
'`friendslist`, `followerslist` etc. '
'* This can be used to select an appropiate '
Expand All @@ -403,6 +427,7 @@ def parse_args(self):
action='store_true',
help='Map full relations. '
' '
' '
'Set to true to map full relations, '
'e.g. many-to-many relationships, '
'such as `user_follows`, '
Expand All @@ -413,41 +438,47 @@ def parse_args(self):
action='store_true',
help='Store to local CSV. '
' '
' '
'If set, will store all '
'submit values to local CSV instead. '
'Currently, this type of output is not available.')
settings_args.add_argument("--csv_allow_linebreaks",
action='store_true',
help=repr('Disable linebreak-rem. '
' '
' '
'If set, will not '
'remove intext-linebreaks (`\r` or `\n`) '
'in output CSVs'))
settings_args.add_argument("--csv_delimiter",
default=",",
help=repr('CSV delimiter. '
' '
' '
'* Provide the CSV delimiter to be used. '
'* Default is comma (`,`). '
'* Note: to pass tab, '
'use variable substitution (`$"\t"`) '),
'use variable substitution (`$"\t"`) ')[1:-1],
type=str)
settings_args.add_argument("--recursive_load",
action='store_true', default=False,
help='Recursive local sub dirs. '
' '
' '
'If set, process input directories '
'recursively (default depth: `2`)')
settings_args.add_argument("--skip_until_file",
help='Skip until file x. '
' '
' '
'If local input, skip all files '
'until file with name `x` appears '
'(default: start immediately)',
type=str)
settings_args.add_argument("--skip_until_record",
help='Skip until record x. '
' '
' '
'If local input, skip all records '
'until record` x` '
'(default: start with first)',
Expand All @@ -456,21 +487,24 @@ def parse_args(self):
action='store_true', default=False,
help='Zip records parallel. '
' '
'Use this flag to zip records of '
'multiple input files, e.g. '
'`List1[A,B,C]`, `List2[1,2,3]` will be '
' '
'* Use this flag to zip records of '
'multiple input files '
'* e.g. `List1[A,B,C]`, `List2[1,2,3]` will be '
'combined (zipped) on read to '
'`List[A1,B2,C3]`')
'`List[A1,B2,C3]` ')
settings_args.add_argument("--min_geoaccuracy",
help='Min geoaccuracy to use '
' '
' '
'Set to `latlng`, `place`, '
'or `city` to limit processing of records based '
'on mininum geoaccuracy (default: no limit)',
type=str)
settings_args.add_argument("--include_lbsn_objects",
help='lbsn objects to process '
' '
' '
'If processing from lbsn db (`rawdb`), '
'provide a comma separated list '
'of [lbsn objects][1] to include. '
Expand All @@ -496,6 +530,7 @@ def parse_args(self):
settings_args.add_argument("--include_lbsn_bases",
help='lbsn bases to update '
' '
' '
'If the target output type is `hll`, '
'provide a comma separated list '
'of lbsn bases to include/update/store to. '
Expand Down Expand Up @@ -533,6 +568,7 @@ def parse_args(self):
settings_args.add_argument("--override_lbsn_query_schema",
help='Override schema and table name '
' '
' '
'This can be used to redirect lbsn queries on '
'the given object from input db to a specific schema/table '
'such as a materialized view. '
Expand Down

0 comments on commit aedd00c

Please sign in to comment.