Skip to content

Improve logging #273

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rustprooflabs merged 4 commits into from
Nov 29, 2022
Merged

Improve logging #273

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e1f140f
Improve logging output with append mode
rustprooflabs Nov 26, 2022
9a3494e
Replace subprocess.run with subprocess.Popen to provide more logging …
rustprooflabs Nov 26, 2022
85b153a
Continue moving from subprocess.run to subprocess.Popen
rustprooflabs Nov 29, 2022
fd20908
Cleanup
rustprooflabs Nov 29, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 39 additions & 9 deletions docker/helpers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import subprocess
import os
import sys
from time import sleep

import db

Expand All @@ -23,6 +24,40 @@ def get_today():
return today


def run_command_via_subprocess(cmd, cwd):
"""Wraps around subprocess.Popen to run commands outside of Python. Prints
output as it goes, returns the status code from the command.

Parameters
-----------------------
cmd : list
Parts of the command to run.
cwd : str or None
Set the working directory, or to None.

Returns
-----------------------
status : int
Return code from command
"""
logger = logging.getLogger('pgosm-flex')
with subprocess.Popen(cmd, cwd=cwd, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT
) as process:
while True:
output = process.stdout.readline()
if process.poll() is not None and output == b'':
break

if output:
logger.info(output.strip().decode('utf-8'))
else:
# Only sleep when there wasn't output
sleep(1)
status = process.poll()
return status


def verify_checksum(md5_file, path):
"""If verfication fails calls `sys.exit()`

Expand All @@ -35,16 +70,11 @@ def verify_checksum(md5_file, path):
logger = logging.getLogger('pgosm-flex')
logger.debug(f'Validating {md5_file} in {path}')

output = subprocess.run(['md5sum', '-c', md5_file],
text=True,
check=False,
cwd=path,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
returncode = run_command_via_subprocess(cmd=['md5sum', '-c', md5_file],
cwd=path)

if output.returncode != 0:
err_msg = 'Failed to validate md5sum. Return code: '
err_msg += f'{output.returncode} {output.stdout}'
if returncode != 0:
err_msg = f'Failed to validate md5sum. Return code: {output.returncode}'
logger.error(err_msg)
sys.exit(err_msg)

Expand Down
69 changes: 21 additions & 48 deletions docker/pgosm_flex.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@
import os
from pathlib import Path
import sys
import subprocess

import click

Expand Down Expand Up @@ -107,7 +106,7 @@ def run_pgosm_flex(ram, region, subregion, append, data_only, debug,
success = run_replication_update(skip_nested=skip_nested,
flex_path=paths['flex_path'])
else:
logger.info('Running normal osm2pgsql mode')
logger.info('Running osm2pgsql without replication')
success = run_osm2pgsql_standard(input_file=input_file,
out_path=paths['out_path'],
flex_path=paths['flex_path'],
Expand Down Expand Up @@ -206,18 +205,11 @@ def run_replication_update(skip_nested, flex_path):
-d $PGOSM_CONN
"""
update_cmd = update_cmd.replace('-d $PGOSM_CONN', f'-d {conn_string}')
returncode = helpers.run_command_via_subprocess(cmd=update_cmd.split(),
cwd=flex_path)

output = subprocess.run(update_cmd.split(),
text=True,
check=False,
cwd=flex_path,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)

logger.info(f'osm2pgsql-replication output:\n{output.stdout}')

if output.returncode != 0:
err_msg = f'Failure. Return code: {output.returncode}'
if returncode != 0:
err_msg = f'Failure. Return code: {returncode}'
logger.warning(err_msg)
return False

Expand Down Expand Up @@ -299,7 +291,6 @@ def get_paths():
return paths



def get_export_filename(input_file):
"""Returns the .sql filename to use for pg_dump.

Expand Down Expand Up @@ -344,7 +335,6 @@ def get_export_full_path(out_path, export_filename):
-----------------
export_path : str
"""

if os.path.isabs(export_filename):
export_path = export_filename
else:
Expand All @@ -364,21 +354,15 @@ def run_osm2pgsql(osm2pgsql_command, flex_path):
logger = logging.getLogger('pgosm-flex')
logger.info('Running osm2pgsql')

output = subprocess.run(osm2pgsql_command.split(),
text=True,
cwd=flex_path,
check=False,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)

logger.info(f'osm2pgsql output: \n {output.stdout}\nEND PgOSM Flex output')
returncode = helpers.run_command_via_subprocess(cmd=osm2pgsql_command.split(),
cwd=flex_path)

if output.returncode != 0:
err_msg = f'Failed to run osm2pgsql. Return code: {output.returncode}'
if returncode != 0:
err_msg = f'Failed to run osm2pgsql. Return code: {returncode}'
logger.error(err_msg)
sys.exit(f'{err_msg} - Check the log output for details.')
sys.exit(f'{err_msg} - Check the log output for details')

logger.info('osm2pgsql completed.')
logger.info('osm2pgsql completed')


def check_layerset_places(flex_path):
Expand All @@ -393,7 +377,6 @@ def check_layerset_places(flex_path):
skip_nested : boolean
"""
logger = logging.getLogger('pgosm-flex')

layerset = os.environ.get('PGOSM_LAYERSET')
layerset_path = os.environ.get('PGOSM_LAYERSET_PATH')

Expand Down Expand Up @@ -442,7 +425,6 @@ def run_post_processing(flex_path, skip_nested):

if not post_processing_sql:
return False

return True


Expand All @@ -461,7 +443,6 @@ def dump_database(input_file, out_path, skip_dump, data_only, schema_name):
logging.getLogger('pgosm-flex').info('Skipping pg_dump')
else:
export_filename = get_export_filename(input_file)

export_path = get_export_full_path(out_path, export_filename)

db.run_pg_dump(export_path=export_path,
Expand All @@ -481,20 +462,16 @@ def check_replication_exists():
logger.debug(f'Command to check DB for replication status:\n{check_cmd}')
conn_string = db.connection_string()
check_cmd = check_cmd.replace('-d $PGOSM_CONN', f'-d {conn_string}')
output = subprocess.run(check_cmd.split(),
text=True,
check=False,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)

logger.debug(f'osm2pgsql-replication output:\n{output.stdout}')
returncode = helpers.run_command_via_subprocess(cmd=check_cmd.split(),
cwd=None)

if output.returncode != 0:
err_msg = f'Failure. Return code: {output.returncode}'
logger.warning(err_msg)
if returncode != 0:
logger.info('Replication not previously set up, fresh import.')
logger.debug(f'Return code: {returncode}')
return False

logger.debug('osm2pgsql-replication status checked.')
logger.debug('Replication set up, candidate for update.')
return True


Expand All @@ -513,16 +490,12 @@ def run_osm2pgsql_replication_init(pbf_path, pbf_filename):
logger.debug(f'Initializing DB for replication with command:\n{init_cmd}')
conn_string = db.connection_string()
init_cmd = init_cmd.replace('-d $PGOSM_CONN', f'-d {conn_string}')
output = subprocess.run(init_cmd.split(),
text=True,
check=False,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)

logger.info(f'osm2pgsql-replication output:\n{output.stdout}')
returncode = helpers.run_command_via_subprocess(cmd=init_cmd.split(),
cwd=None)

if output.returncode != 0:
err_msg = f'Failed to run osm2pgsql-replication. Return code: {output.returncode}'
if returncode != 0:
err_msg = f'Failed to run osm2pgsql-replication. Return code: {returncode}'
logger.error(err_msg)
sys.exit(f'{err_msg} - Check the log output for details.')

Expand Down