diff --git a/.gitignore b/.gitignore index 94b831e..a8ce387 100644 --- a/.gitignore +++ b/.gitignore @@ -8,14 +8,14 @@ build/ dist/ docs/_build/ -tests/output-archmap_users.txt -tests/output-archmap_pretty_users.txt +tests/output-archmap.txt +tests/output-archmap_pretty.txt tests/output-archmap.csv tests/output-archmap.geojson tests/output-archmap.kml -tests/interactive_output-archmap_users.txt -tests/interactive_output-archmap_pretty_users.txt +tests/interactive_output-archmap.txt +tests/interactive_output-archmap_pretty.txt tests/interactive_output-archmap.csv tests/interactive_output-archmap.geojson tests/interactive_output-archmap.kml diff --git a/README.rst b/README.rst index db1bb3d..00566b1 100644 --- a/README.rst +++ b/README.rst @@ -37,7 +37,7 @@ Synopsis -------- By default, running ``archmap`` will output four files to /tmp; -``archmap_users.txt``, ``archmap.geojson``, ``archmap.kml`` and ``archmap.csv``. +``archmap.txt``, ``archmap.geojson``, ``archmap.kml`` and ``archmap.csv``. This can be overridden by either using the config file or by the command line switches. The config file should be placed in ``/etc/archmap.conf``, this can be overridden by using ``--config `` @@ -51,7 +51,7 @@ Running ``archmap --help`` will display this help message: .. code-block:: none usage: - archmap [-h] [-v] [-q] [--config FILE] [--url URL] [--file FILE] [--pretty] [--users FILE] [--geojson FILE] [--kml FILE] [--csv FILE] + archmap [-h] [-v] [-q] [--config FILE] [--url URL] [--file FILE] [--pretty] [--text FILE] [--geojson FILE] [--kml FILE] [--csv FILE] optional arguments: -h, --help show this help message and exit @@ -61,7 +61,7 @@ Running ``archmap --help`` will display this help message: --url URL Use an alternative URL to parse the wiki list from --file FILE Use a file to parse the wiki list from --pretty Prettify the text user list. Only works if user output is enabled - --users FILE Output the user list to FILE, use 'no' to disable output or '-' to print to stdout + --text FILE Output the raw-text to FILE, use 'no' to disable output or '-' to print to stdout --geojson FILE Output the GeoJSON to FILE, use 'no' to disable output or '-' to print to stdout --kml FILE Output the KML to FILE, use 'no' to disable output or '-' to print to stdout --csv FILE Output the CSV to FILE, use 'no' to disable output or '-' to print to stdout @@ -72,6 +72,6 @@ License Everything in the `ArchMap repo `_ is `unlicensed `_. -All of the files that this script can generate (``archmap_users.txt``, ``archmap.geojson``, ``archmap.kml``, and ``archmap.csv``) +All of the files that this script can generate (``archmap.txt``, ``archmap.geojson``, ``archmap.kml``, and ``archmap.csv``) will contain text from the `ArchWiki `_ which puts them under the `GNU Free Documentation License 1.3 or later `_. diff --git a/archmap.conf b/archmap.conf index 06e26b3..b4dbbcb 100644 --- a/archmap.conf +++ b/archmap.conf @@ -7,7 +7,7 @@ file = # The location to save the list of users and the GIS files, # use 'no' or leave blank to disable output, # use '-' to print to stdout. -users = /tmp/archmap_users.txt +text = /tmp/archmap.txt geojson = /tmp/archmap.geojson kml = /tmp/archmap.kml csv = /tmp/archmap.csv diff --git a/archmap.py b/archmap.py index 431e4b6..8557bb9 100755 --- a/archmap.py +++ b/archmap.py @@ -38,14 +38,14 @@ default_url = 'https://wiki.archlinux.org/index.php/ArchMap/List' default_file = '' -# If set to True, the columns in the raw user list text will be aligned +# If set to True, the columns in the raw-text list will be aligned default_pretty = False -# Set the output locations for users, GeoJSON, KML and CSV. +# Set the output locations for the raw-text, GeoJSON, KML and CSV files. # Setting any of the following to 'no' or leaving it blank will disable the output, # use '-' to print the generated text to stdout. # These settings are overridden by the config file, if it exists. -default_users = '/tmp/archmap_users.txt' +default_text = '/tmp/archmap.txt' default_geojson = '/tmp/archmap.geojson' default_kml = '/tmp/archmap.kml' default_csv = '/tmp/archmap.csv' @@ -71,7 +71,7 @@ def get_users(url='https://wiki.archlinux.org/index.php/ArchMap/List', local='') local (str): Path to a local copy of the ArchWiki ArchMap source Returns: - str or None: The extracted raw text list of users or None if not avaliable + str or None: The extracted raw-text list of users or None if not avaliable """ if local == '': # Open and decode the page from the URL containing the list of users. @@ -97,10 +97,11 @@ def get_users(url='https://wiki.archlinux.org/index.php/ArchMap/List', local='') def parse_users(users): - """This function parses the wiki text from ``users`` into it's components. + """This function parses the raw-text list (``users``) that has been extracted from the wiki page + and splits it into a list of namedtuples containing the latitude, longitude, name and comment. Args: - users (str): Raw user data from the ArchWiki + users (str): raw-text list from the ArchWiki Returns: :obj:`list` of :obj:`collections.namedtuple` \ @@ -130,7 +131,7 @@ def parse_users(users): # 8. Comment re_whole = re.compile(str(re_coord + r'\s*,\s*' + re_coord + r'[^a-zA-Z]*' + re_name + r'\s*#*\s*' + re_comment)) - log.info('Parsing ArchWiki text') + log.info('Parsing ArchWiki list') for line_number, line in enumerate(users, start=1): # Retun None unless the line fully matches the RE re_whole_result = re_whole.fullmatch(line) @@ -149,9 +150,9 @@ def parse_users(users): return parsed -def make_users(parsed_users, output_file='', pretty=False): - """This function reads the user data supplied by ``parsed_users``, it then generates - a list according to the formatting specifications on the wiki and writes it to ``output_file``. +def make_text(parsed_users, output_file='', pretty=False): + """This function reads the user data supplied by ``parsed_users``, it then generates a raw-text list + according to the formatting specifications on the wiki and writes it to ``output_file``. Args: parsed_users (:obj:`list` of :obj:`collections.namedtuple` \ @@ -163,7 +164,7 @@ def make_users(parsed_users, output_file='', pretty=False): Returns: str: The text written to the output file """ - users = '' + text_str = '' longest_latitude = 1 longest_longitude = 1 @@ -171,7 +172,7 @@ def make_users(parsed_users, output_file='', pretty=False): longest_comment = 1 if pretty: - log.debug('Finding longest strings for prettying the raw user list') + log.debug('Finding longest strings for prettifying the raw-text') # Go through all of the elements in each list and track the length of the longest string for user in parsed_users: if longest_latitude < len(str(user.latitude)): @@ -183,31 +184,31 @@ def make_users(parsed_users, output_file='', pretty=False): if longest_comment < len(str(user.comment)): longest_comment = len(str(user.comment)) - log.debug('Making raw users') + log.debug('Making raw-text') for user in parsed_users: # This follows the formatting defined here: # https://wiki.archlinux.org/index.php/ArchMap/List#Adding_yourself_to_the_list # # If pretty printing is enabled, the 'longest_' lengths are used to align the elements in the string # Change the '<', '^' or '>' to change the justification (< = left, > = right, ^ = center) - users += '{:<{}},{:<{}} "{:^{}}" # {:>{}}\n'.format(user.latitude, longest_latitude, - user.longitude, longest_longitude, - user.name, longest_name, - user.comment, longest_comment) + text_str += '{:<{}},{:<{}} "{:^{}}" # {:>{}}\n'.format(user.latitude, longest_latitude, + user.longitude, longest_longitude, + user.name, longest_name, + user.comment, longest_comment) # If the last user didnt have a comment, go back to that line # and strip the trailing whitespace then replace the newline (prevents editor errors) - users = users.strip('\n').strip() + '\n' + text_str = text_str.strip('\n').strip() + '\n' if output_file == '-': - print(users) + print(text_str) elif output_file != '': - log.info('Writing raw user list to ' + output_file) + log.info('Writing raw-text to ' + output_file) with open(output_file, 'w') as output: - output.write(users) + output.write(text_str) - return users + return text_str def make_geojson(parsed_users, output_file=''): @@ -336,9 +337,9 @@ def main(): parser.add_argument('--file', metavar='FILE', help='Use a file to parse the wiki list from') parser.add_argument('--pretty', action='store_true', - help='Prettify the text user list. Only works if user output is enabled') - parser.add_argument('--users', metavar='FILE', - help="Output the user list to FILE, use 'no' to disable output or '-' to print to stdout") + help='Prettify the raw-text. Only works if user output is enabled') + parser.add_argument('--text', metavar='FILE', + help="Output the raw-text to FILE, use 'no' to disable output or '-' to print to stdout") parser.add_argument('--geojson', metavar='FILE', help="Output the GeoJSON to FILE, use 'no' to disable output or '-' to print to stdout") parser.add_argument('--kml', metavar='FILE', @@ -360,7 +361,7 @@ def main(): pretty = config.getboolean('extras', 'pretty', fallback=default_pretty) input_url = config.get('files', 'url', fallback=default_url) input_file = config.get('files', 'file', fallback=default_file) - output_file_users = config.get('files', 'users', fallback=default_users) + output_file_text = config.get('files', 'text', fallback=default_text) output_file_geojson = config.get('files', 'geojson', fallback=default_geojson) output_file_kml = config.get('files', 'kml', fallback=default_kml) output_file_csv = config.get('files', 'csv', fallback=default_csv) @@ -388,8 +389,8 @@ def main(): if args.file is not None: input_file = args.file - if args.users is not None: - output_file_users = args.users + if args.text is not None: + output_file_text = args.text if args.geojson is not None: output_file_geojson = args.geojson @@ -402,15 +403,15 @@ def main(): # Do what's needed. dont_run = ['', 'no'] - if output_file_users in dont_run and \ + if output_file_text in dont_run and \ output_file_geojson in dont_run and \ output_file_kml in dont_run and \ output_file_csv in dont_run: log.warning('There is nothing to do') else: pipe_claims = [] - if output_file_users == '-': - pipe_claims.append('Users') + if output_file_text == '-': + pipe_claims.append('Text') if output_file_geojson == '-': pipe_claims.append('GeoJSON') if output_file_kml == '-': @@ -426,8 +427,8 @@ def main(): return None parsed_users = parse_users(users) - if output_file_users not in dont_run: - make_users(parsed_users, output_file_users, pretty=pretty) + if output_file_text not in dont_run: + make_text(parsed_users, output_file_text, pretty=pretty) if output_file_geojson not in dont_run: make_geojson(parsed_users, output_file_geojson) if output_file_kml not in dont_run: diff --git a/docs/usage/advanced_usage.rst b/docs/usage/advanced_usage.rst index 3676123..d3c2a36 100644 --- a/docs/usage/advanced_usage.rst +++ b/docs/usage/advanced_usage.rst @@ -11,7 +11,7 @@ Getting and parsing user data Output generators ----------------- -.. autofunction:: archmap.make_users +.. autofunction:: archmap.make_text .. autofunction:: archmap.make_geojson .. autofunction:: archmap.make_kml .. autofunction:: archmap.make_csv diff --git a/docs/usage/basic_usage.rst b/docs/usage/basic_usage.rst index 888e118..31c6e4d 100644 --- a/docs/usage/basic_usage.rst +++ b/docs/usage/basic_usage.rst @@ -16,7 +16,7 @@ The **--help** flag will output a help message with all of the available options Basic use --------- -By default, running **archmap** will output three files to /tmp, **archmap_users.txt**, **archmap.geojson** and **archmap.kml**, +By default, running **archmap** will output three files to /tmp, **archmap.txt**, **archmap.geojson** and **archmap.kml**, this can be overridden by either using the config file or by the following command line switches. Using the **--verbose** flag will print information on what the script is doing: @@ -29,7 +29,7 @@ You can specify the output location for the user list text, GeoJSON, KML and CSV .. code-block:: bash - archmap --users /tmp/archmap_users.txt --geojson /tmp/archmap.geojson --kml /tmp/archmap.kml --csv /tmp/archmap.csv + archmap --text /tmp/archmap.txt --geojson /tmp/archmap.geojson --kml /tmp/archmap.kml --csv /tmp/archmap.csv If you would like to parse an alternate copy of the wiki list, simply pass either the --url or --file flags:: diff --git a/tests/sample-archmap_users.txt b/tests/sample-archmap.txt similarity index 100% rename from tests/sample-archmap_users.txt rename to tests/sample-archmap.txt diff --git a/tests/sample-archmap_pretty_users.txt b/tests/sample-archmap_pretty.txt similarity index 100% rename from tests/sample-archmap_pretty_users.txt rename to tests/sample-archmap_pretty.txt diff --git a/tests/raw_users.txt b/tests/sample-raw.txt similarity index 100% rename from tests/raw_users.txt rename to tests/sample-raw.txt diff --git a/tests/tests.py b/tests/tests.py index dac675a..8c4cf84 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -22,9 +22,9 @@ class WikiParserTestCase(unittest.TestCase): # including the tags needed for parsing and some test data wiki_html = 'tests/ArchMap_List-stripped.html' - # 'raw_users.txt' contains the extracted list from 'ArchMap_List-stripped.html' + # 'sample-raw.txt' contains the extracted list from 'ArchMap_List-stripped.html' # The trailing newline needs to be stripped to match the output from 'get_users' - with open('tests/raw_users.txt', 'r') as raw_users: + with open('tests/sample-raw.txt', 'r') as raw_users: raw_users = raw_users.read().rstrip('\n') def setUp(self): @@ -70,16 +70,16 @@ class ListParserTestCase(unittest.TestCase): """These tests test that the list parser is working correctly """ - # 'raw_users.txt' contains an unformatted 'raw' sample list - with open('tests/raw_users.txt', 'r') as raw_users_file: + # 'sample-raw.txt' contains an unformatted 'raw' sample list + with open('tests/sample-raw.txt', 'r') as raw_users_file: raw_users = raw_users_file.read() - # 'sample_users.txt' contains a formatted sample list equivilent to the raw version above - with open('tests/sample-archmap_users.txt', 'r') as sample_users_file: - sample_users = sample_users_file.read() + # 'sample_archmap.txt' contains a formatted sample list equivilent to the raw version above + with open('tests/sample-archmap.txt', 'r') as sample_text_file: + sample_text = sample_text_file.read() - # 'sample-parsed_users.pickle' is a pickled list that was generated with a known good list - # ('parse_users()' was run on 'sample-archmap_users.txt' and the output was pickled) + # 'sample_parsed_users.pickle' is a pickled list that was generated with a known good list + # ('parse_users()' was run on 'sample-archmap.txt' and the output was pickled) with open('tests/sample-parsed_users.pickle', 'rb') as pickled_input: sample_parsed_users = pickle.load(pickled_input) @@ -92,26 +92,26 @@ def test_list_parser_raw(self): self.assertEqual(self.sample_parsed_users, parsed_raw_users) def test_list_parser_cleaned(self): - parsed_cleaned_users = archmap.parse_users(self.sample_users) + parsed_cleaned_users = archmap.parse_users(self.sample_text) self.assertEqual(self.sample_parsed_users, parsed_cleaned_users) class OutputTestCase(unittest.TestCase): - """These tests compare the output of ``make_users()``, ``make_geojson()``, ``make_kml()`` and ``make csv()`` + """These tests compare the output of ``make_text()``, ``make_geojson()``, ``make_kml()`` and ``make csv()`` with pre-generated versions that have been checked manually, these *sample* files were generated by running ``archmap.py`` on the stripped-down/handmade ``ArchMap_List-stripped.html'``. """ - # 'sample-parsed_users.pickle' is a pickled list that was generated with a known good list - # ('parse_users()' was run on 'sample-archmap_users.txt' and the output was pickled) + # 'sample_parsed_users.pickle' is a pickled list that was generated with a known good list + # ('parse_users()' was run on 'sample-archmap.txt' and the output was pickled) with open('tests/sample-parsed_users.pickle', 'rb') as pickled_input: parsed_users = pickle.load(pickled_input) def setUp(self): - self.sample_users = 'tests/sample-archmap_users.txt' - self.output_users = 'tests/output-archmap_users.txt' - self.sample_pretty_users = 'tests/sample-archmap_pretty_users.txt' - self.output_pretty_users = 'tests/output-archmap_pretty_users.txt' + self.sample_text = 'tests/sample-archmap.txt' + self.output_text = 'tests/output-archmap.txt' + self.sample_pretty_text = 'tests/sample-archmap_pretty.txt' + self.output_pretty_text = 'tests/output-archmap_pretty.txt' self.sample_geojson = 'tests/sample-archmap.geojson' self.output_geojson = 'tests/output-archmap.geojson' self.sample_kml = 'tests/sample-archmap.kml' @@ -124,33 +124,33 @@ def setUp(self): def tearDown(self): try: - os.remove(self.output_users) - os.remove(self.output_pretty_users) + os.remove(self.output_text) + os.remove(self.output_pretty_text) os.remove(self.output_geojson) os.remove(self.output_kml) os.remove(self.output_csv) except FileNotFoundError: pass - def test_users(self): - archmap.make_users(self.parsed_users, self.output_users) + def test_text(self): + archmap.make_text(self.parsed_users, self.output_text) - with open(self.sample_users, 'r') as file: - sample_users = file.read() - with open(self.output_users, 'r') as file: - output_users = file.read() + with open(self.sample_text, 'r') as file: + sample_text = file.read() + with open(self.output_text, 'r') as file: + output_text = file.read() - self.assertEqual(sample_users, output_users) + self.assertEqual(sample_text, output_text) - def test_pretty_users(self): - archmap.make_users(self.parsed_users, self.output_pretty_users, pretty=True) + def test_pretty_text(self): + archmap.make_text(self.parsed_users, self.output_pretty_text, pretty=True) - with open(self.sample_pretty_users, 'r') as file: - sample_pretty_users = file.read() - with open(self.output_pretty_users, 'r') as file: - output_pretty_users = file.read() + with open(self.sample_pretty_text, 'r') as file: + sample_pretty_text = file.read() + with open(self.output_pretty_text, 'r') as file: + output_pretty_text = file.read() - self.assertEqual(sample_pretty_users, output_pretty_users) + self.assertEqual(sample_pretty_text, output_pretty_text) def test_geojson(self): archmap.make_geojson(self.parsed_users, self.output_geojson) @@ -184,19 +184,19 @@ def test_csv(self): class ReturnedTestCase(unittest.TestCase): - """These tests compare the return values of ``make_users()``, ``make_geojson()``, ``make_kml()`` and ``make csv()`` + """These tests compare the return values of ``make_text()``, ``make_geojson()``, ``make_kml()`` and ``make csv()`` with pre-generated versions that have been checked manually, these *sample* files were generated by running ``archmap.py`` on the stripped-down/handmade ``ArchMap_List-stripped.html'``. """ - # 'sample-parsed_users.pickle' is a pickled list that was generated with a known good list - # ('parse_users()' was run on 'sample-archmap_users.txt' and the output was pickled) + # 'sample_parsed_users.pickle' is a pickled list that was generated with a known good list + # ('parse_users()' was run on 'sample-archmap.txt' and the output was pickled) with open('tests/sample-parsed_users.pickle', 'rb') as pickled_input: parsed_users = pickle.load(pickled_input) def setUp(self): - self.sample_users = 'tests/sample-archmap_users.txt' - self.sample_pretty_users = 'tests/sample-archmap_pretty_users.txt' + self.sample_text = 'tests/sample-archmap.txt' + self.sample_pretty_text = 'tests/sample-archmap_pretty.txt' self.sample_geojson = 'tests/sample-archmap.geojson' self.sample_kml = 'tests/sample-archmap.kml' self.sample_csv = 'tests/sample-archmap.csv' @@ -204,19 +204,19 @@ def setUp(self): # Set 'maxDiff' to 'None' to be able to see long diffs when something goes wrong. self.maxDiff = None - def test_users(self): - returned_users = archmap.make_users(self.parsed_users) + def test_text(self): + returned_text = archmap.make_text(self.parsed_users) - with open(self.sample_users, 'r') as file: - sample_users = file.read() - self.assertEqual(sample_users, returned_users) + with open(self.sample_text, 'r') as file: + sample_text = file.read() + self.assertEqual(sample_text, returned_text) - def test_pretty_users(self): - returned_pretty_users = archmap.make_users(self.parsed_users, pretty=True) + def test_pretty_text(self): + returned_pretty_text = archmap.make_text(self.parsed_users, pretty=True) - with open(self.sample_pretty_users, 'r') as file: - sample_pretty_users = file.read() - self.assertEqual(sample_pretty_users, returned_pretty_users) + with open(self.sample_pretty_text, 'r') as file: + sample_pretty_text = file.read() + self.assertEqual(sample_pretty_text, returned_pretty_text) def test_geojson(self): returned_geojson = archmap.make_geojson(self.parsed_users) @@ -245,10 +245,10 @@ class InteractiveTestCase(unittest.TestCase): """ def setUp(self): - self.sample_users = 'tests/sample-archmap_users.txt' - self.output_users = 'tests/interactive_output-archmap_users.txt' - self.sample_pretty_users = 'tests/sample-archmap_pretty_users.txt' - self.output_pretty_users = 'tests/interactive_output-archmap_pretty_users.txt' + self.sample_text = 'tests/sample-archmap.txt' + self.output_text = 'tests/interactive_output-archmap.txt' + self.sample_pretty_text = 'tests/sample-archmap_pretty.txt' + self.output_pretty_text = 'tests/interactive_output-archmap_pretty.txt' self.sample_geojson = 'tests/sample-archmap.geojson' self.output_geojson = 'tests/interactive_output-archmap.geojson' self.sample_kml = 'tests/sample-archmap.kml' @@ -262,7 +262,7 @@ def setUp(self): sys.argv = ['test', '--quiet', '--file', 'tests/ArchMap_List-stripped.html', - '--users', self.output_users, + '--text', self.output_text, '--geojson', self.output_geojson, '--kml', self.output_kml, '--csv', self.output_csv] @@ -274,7 +274,7 @@ def setUp(self): '--quiet', '--file', 'tests/ArchMap_List-stripped.html', '--pretty', - '--users', self.output_pretty_users, + '--text', self.output_pretty_text, '--geojson', 'no', '--kml', 'no', '--csv', 'no'] @@ -285,29 +285,29 @@ def setUp(self): def tearDown(self): try: - os.remove(self.output_users) - os.remove(self.output_pretty_users) + os.remove(self.output_text) + os.remove(self.output_pretty_text) os.remove(self.output_geojson) os.remove(self.output_kml) os.remove(self.output_csv) except FileNotFoundError: pass - def test_users(self): - with open(self.sample_users, 'r') as file: - sample_users = file.read() - with open(self.output_users, 'r') as file: - output_users = file.read() + def test_text(self): + with open(self.sample_text, 'r') as file: + sample_text = file.read() + with open(self.output_text, 'r') as file: + output_text = file.read() - self.assertEqual(sample_users, output_users) + self.assertEqual(sample_text, output_text) - def test_pretty_users(self): - with open(self.sample_pretty_users, 'r') as file: - sample_pretty_users = file.read() - with open(self.output_pretty_users, 'r') as file: - output_pretty_users = file.read() + def test_pretty_text(self): + with open(self.sample_pretty_text, 'r') as file: + sample_pretty_text = file.read() + with open(self.output_pretty_text, 'r') as file: + output_pretty_text = file.read() - self.assertEqual(sample_pretty_users, output_pretty_users) + self.assertEqual(sample_pretty_text, output_pretty_text) def test_geojson(self): with open(self.sample_geojson, 'r') as file: @@ -338,7 +338,7 @@ def test_piped_output(self): sys.argv = ['test', '--quiet', '--file', 'tests/ArchMap_List-stripped.html', - '--users', '-', + '--text', '-', '--geojson', '-', '--kml', '-', '--csv', '-'] @@ -348,8 +348,8 @@ def test_piped_output(self): archmap.main() piped_output_str = piped_output.getvalue() - with open(self.sample_users, 'r') as file: - sample_users = file.read() + '\n' + with open(self.sample_text, 'r') as file: + sample_text = file.read() + '\n' with open(self.sample_geojson, 'r') as file: sample_geojson = file.read() + '\n' with open(self.sample_kml, 'r') as file: @@ -357,7 +357,7 @@ def test_piped_output(self): with open(self.sample_csv, 'r') as file: sample_csv = file.read() + '\n' - combined_string = sample_users + sample_geojson + sample_kml + sample_csv + combined_string = sample_text + sample_geojson + sample_kml + sample_csv self.assertEqual(combined_string, piped_output_str)