Skip to content

Commit

Permalink
takeover changes from #40
Browse files Browse the repository at this point in the history
  • Loading branch information
@treee111
treee111 committed Oct 19, 2021
1 parent a7ed8f8 commit 4d77ea1
Show file tree
Hide file tree
Showing 2 changed files with 135 additions and 32 deletions.
49 changes: 44 additions & 5 deletions common_python/constants.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -189,8 +189,47 @@
noregion_geofabrik = ['russia','asia']

# Tags to keep
filtered_tags = ['access', 'admin_level', 'aerialway', 'aeroway', 'barrier',
'boundary', 'bridge', 'highway', 'natural', 'oneway', 'place',
'railway', 'tracktype', 'tunnel', 'waterway']

FILTERED_TAGS_WIN = 'access= admin_level= aerialway= aeroway= barrier= boundary= bridge= highway= natural= oneway= place= railway= tracktype= tunnel= waterway='
#filtered_tags = ['access', 'admin_level', 'aerialway', 'aeroway', 'barrier',
# 'boundary', 'bridge', 'highway', 'natural', 'oneway', 'place',
# 'railway', 'tracktype', 'tunnel', 'waterway']
filtered_tags = 'access \
area=yes \
bicycle \
bridge \
foot=ft_yes, foot_designated \
highway=abandoned, bus_guideway, disused, bridleway, byway, construction, cycleway, footway, living_street, motorway, motorway_link, path, pedestrian, primary, primary_link, residential, road, secondary, secondary_link, service, steps, tertiary, tertiary_link, track, trunk, trunk_link, unclassified \
natural=coastline, nosea, sea, beach, land, scrub, water, wetland =wood \
leisure=park, nature_reserve \
railway=abandoned, bus_guideway, disused, funicular, light_rail, miniature, narrow_gauge, preserved, rail, subway, tram \
surface \
tracktype \
tunnel \
waterway=canal, drain, river, riverbank \
wood=deciduous'

filtered_tags_names = 'admin_level=2 \
area=yes \
mountain_pass \
natural \
place=city, hamlet, island, isolated_dwelling, islet, locality, suburb, town, village, country'

FILTERED_TAGS_WIN = 'access= \
area=yes \
bicycle= \
bridge= \
foot=ft_yes =foot_designated \
highway=abandoned =bus_guideway =disused =bridleway =byway =construction =cycleway =footway =living_street =motorway =motorway_link =path =pedestrian =primary =primary_link =residential =road =secondary =secondary_link =service =steps =tertiary =tertiary_link =track =trunk =trunk_link =unclassified \
natural=coastline =nosea =sea =beach =land =scrub =water =wetland =wood \
leisure=park =nature_reserve \
railway=abandoned =bus_guideway =disused =funicular =light_rail =miniature =narrow_gauge =preserved =rail =subway =tram \
surface= \
tracktype= \
tunnel= \
waterway=canal =drain =river =riverbank \
wood=deciduous'

FILTERED_TAGS_WIN_NAMES = 'admin_level=2 \
area=yes \
mountain_pass= \
natural= \
place=city =hamlet =island =isolated_dwelling =islet =locality =suburb =town =village =country'
118 changes: 91 additions & 27 deletions common_python/osm_maps_functions.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,14 +104,14 @@ def filter_tags_from_country_osm_pbf_files(self):
if platform.system() == "Windows":
for key, val in self.border_countries.items():
# print(key, val)
out_file = os.path.join(fd_fct.OUTPUT_DIR,
f'filtered-{key}.osm.pbf')
out_file_o5m = os.path.join(fd_fct.OUTPUT_DIR,
f'outFile-{key}.o5m')
out_file_o5m_filtered = os.path.join(fd_fct.OUTPUT_DIR,
f'outFileFiltered-{key}.o5m')
out_file_o5m_filtered_names = os.path.join(fd_fct.OUTPUT_DIR,
f'outFileFiltered-{key}-Names.o5m')

if not os.path.isfile(out_file) or self.force_processing is True:
if not os.path.isfile(out_file_o5m_filtered) or self.force_processing is True:
print(f'\n+ Converting map of {key} to o5m format')
cmd = [os.path.join(fd_fct.TOOLING_WIN_DIR, 'osmconvert')]
cmd.extend(['-v', '--hash-memory=2500', '--complete-ways',
Expand All @@ -130,46 +130,68 @@ def filter_tags_from_country_osm_pbf_files(self):
cmd = [os.path.join(fd_fct.TOOLING_WIN_DIR, 'osmfilter')]
cmd.append(out_file_o5m)
cmd.append('--keep="' + constants.FILTERED_TAGS_WIN + '"')
cmd.append('--keep-tags=all type= layer= "' +
constants.FILTERED_TAGS_WIN + '"')
cmd.append('-o=' + out_file_o5m_filtered)
# print(cmd)
result = subprocess.run(cmd, check=True)
if result.returncode != 0:
print(f'Error in OSMFilter with country: {key}')
sys.exit()

print(
f'\n# Converting map of {key} back to osm.pbf format')
cmd = [os.path.join(fd_fct.TOOLING_WIN_DIR, 'osmconvert'), '-v',
'--hash-memory=2500', out_file_o5m_filtered]
cmd.append('-o='+out_file)
cmd = [os.path.join(fd_fct.TOOLING_WIN_DIR, 'osmfilter')]
cmd.append(out_file_o5m)
cmd.append(
'--keep="' + constants.FILTERED_TAGS_WIN_NAMES + '"')
cmd.append('--keep-tags=all type= name= layer= "' +
constants.FILTERED_TAGS_WIN_NAMES + '"')
cmd.append('-o=' + out_file_o5m_filtered_names)
# print(cmd)
result = subprocess.run(cmd, check=True)
if result.returncode != 0:
print(f'Error in OSMConvert with country: {key}')
print(f'Error in OSMFilter with country: {key}')
sys.exit()

os.remove(out_file_o5m)
os.remove(out_file_o5m_filtered)

self.border_countries[key]['filtered_file'] = out_file
self.border_countries[key]['filtered_file'] = out_file_o5m_filtered
self.border_countries[key]['filtered_file_names'] = out_file_o5m_filtered_names

# Non-Windows
else:
for key, val in self.border_countries.items():
## print(key, val)
out_file = os.path.join(fd_fct.OUTPUT_DIR,
f'filtered-{key}.osm.pbf')
# print(outFile)
if not os.path.isfile(out_file):
out_file_o5m_filtered = os.path.join(fd_fct.OUTPUT_DIR,
f'filtered-{key}.o5m')
out_file_o5m_filtered_names = os.path.join(fd_fct.OUTPUT_DIR,
f'outFileFiltered-{key}-Names.o5m')
# print(out_file_o5m_filtered)
if not os.path.isfile(out_file_o5m_filtered):
print(f'+ Create filtered country file for {key}')

cmd = ['osmium', 'tags-filter']
cmd = ['osmium', 'tags-filter', '--remove-tags']
cmd.append(val['map_file'])
cmd.extend(constants.filtered_tags)
cmd.extend(['-o', out_file])
cmd.extend('type layer ' + constants.filtered_tags)
cmd.extend(['-o', out_file_o5m_filtered])
# print(cmd)
result = subprocess.run(cmd, check=True)
if result.returncode != 0:
print(f'Error in Osmium with country: {key}')
sys.exit()

cmd = ['osmium', 'tags-filter', '--remove-tags']
cmd.append(val['map_file'])
cmd.extend('type name layer' +
constants.filtered_tags_names)
cmd.extend(['-o', out_file_o5m_filtered_names])
# print(cmd)
subprocess.run(cmd, check=True)
self.border_countries[key]['filtered_file'] = out_file
result = subprocess.run(cmd, check=True)
if result.returncode != 0:
print(f'Error in Osmium with country: {key}')
sys.exit()

self.border_countries[key]['filtered_file'] = out_file_o5m_filtered
self.border_countries[key]['filtered_file_names'] = out_file_o5m_filtered_names

# logging
print('# Filter tags from country osm.pbf files: OK')
Expand Down Expand Up @@ -265,11 +287,15 @@ def split_filtered_country_files_to_tiles(self):
f'+ Splitting tile {tile_count} of {len(self.tiles)} for Coordinates: {tile["x"]},{tile["y"]} from map of {country}')
out_file = os.path.join(fd_fct.OUTPUT_DIR,
f'{tile["x"]}', f'{tile["y"]}', f'split-{country}.osm.pbf')
if not os.path.isfile(out_file) or self.force_processing is True:
out_file_names = os.path.join(fd_fct.OUTPUT_DIR,
f'{tile["x"]}', f'{tile["y"]}', f'split-{country}-names.osm.pbf')
outMerged = os.path.join(fd_fct.OUTPUT_DIR,
f'{tile["x"]}', f'{tile["y"]}', f'merged.osm.pbf')
if not os.path.isfile(outMerged) or self.force_processing is True:
# Windows
if platform.system() == "Windows":
#cmd = ['.\\osmosis\\bin\\osmosis.bat', '--rbf',border_countries[c]['filtered_file'],'workers='+workers, '--buffer', 'bufferCapacity=12000', '--bounding-box', 'completeWays=yes', 'completeRelations=yes']
#cmd.extend(['left='+f'{tile["left"]}', 'bottom='+f'{tile["bottom"]}', 'right='+f'{tile["right"]}', 'top='+f'{tile["top"]}', '--buffer', 'bufferCapacity=12000', '--wb'])
# cmd = ['.\\osmosis\\bin\\osmosis.bat', '--rbf',border_countries[c]['filtered_file'],'workers='+workers, '--buffer', 'bufferCapacity=12000', '--bounding-box', 'completeWays=yes', 'completeRelations=yes']
# cmd.extend(['left='+f'{tile["left"]}', 'bottom='+f'{tile["bottom"]}', 'right='+f'{tile["right"]}', 'top='+f'{tile["top"]}', '--buffer', 'bufferCapacity=12000', '--wb'])
# cmd.append('file='+outFile)
# cmd.append('omitmetadata=true')
cmd = [os.path.join(
Expand All @@ -281,6 +307,21 @@ def split_filtered_country_files_to_tiles(self):
cmd.append(val['filtered_file'])
cmd.append('-o='+out_file)

# print(cmd)
result = subprocess.run(cmd, check=True)
if result.returncode != 0:
print(f'Error in Osmosis with country: {country}')
sys.exit()

cmd = [os.path.join(
fd_fct.TOOLING_WIN_DIR, 'osmconvert'), '-v', '--hash-memory=2500']
cmd.append('-b='+f'{tile["left"]}' + ',' + f'{tile["bottom"]}' +
',' + f'{tile["right"]}' + ',' + f'{tile["top"]}')
cmd.extend(
['--complete-ways', '--complete-multipolygons', '--complete-boundaries'])
cmd.append(val['filtered_file_names'])
cmd.append('-o='+out_file_names)

# print(cmd)
result = subprocess.run(cmd, check=True)
if result.returncode != 0:
Expand All @@ -299,7 +340,25 @@ def split_filtered_country_files_to_tiles(self):
cmd.extend(['--overwrite'])

# print(cmd)
subprocess.run(cmd, check=True)
result = subprocess.run(cmd, check=True)
if result.returncode != 0:
print(f'Error in Osmium with country: {country}')
sys.exit()

cmd = ['osmium', 'extract']
cmd.extend(
['-b', f'{tile["left"]},{tile["bottom"]},{tile["right"]},{tile["top"]}'])
cmd.append(val['filtered_file_names'])
cmd.extend(['-s', 'smart'])
cmd.extend(['-o', out_file_names])
cmd.extend(['--overwrite'])

# print(cmd)
result = subprocess.run(cmd, check=True)
if result.returncode != 0:
print(f'Error in Osmium with country: {country}')
sys.exit()

print(val['filtered_file'])

tile_count += 1
Expand Down Expand Up @@ -355,9 +414,14 @@ def merge_splitted_tiles_with_land_and_sea(self, calc_border_countries):
if calc_border_countries or country in self.border_countries:
cmd.append(os.path.join(fd_fct.OUTPUT_DIR,
f'{tile["x"]}', f'{tile["y"]}', f'split-{country}.osm.pbf'))
cmd.append(os.path.join(fd_fct.OUTPUT_DIR,
f'{tile["x"]}', f'{tile["y"]}', f'split-{country}-names.osm.pbf'))

cmd.append(os.path.join(fd_fct.OUTPUT_DIR,
f'{tile["x"]}', f'{tile["y"]}', 'land1.osm'))
land_files = glob.glob(os.path.join(fd_fct.OUTPUT_DIR,
f'{tile["x"]}', f'{tile["y"]}', 'land*.osm'))
for land in land_files:
cmd.append(os.path.join(fd_fct.OUTPUT_DIR,
f'{tile["x"]}', f'{tile["y"]}', f'{land}'))
cmd.append(os.path.join(fd_fct.OUTPUT_DIR,
f'{tile["x"]}', f'{tile["y"]}', 'sea.osm'))
cmd.extend(['-o', out_file])
Expand Down Expand Up @@ -454,7 +518,7 @@ def zip_map_files(self):
path_7za = os.path.join(fd_fct.TOOLING_WIN_DIR, '7za')
cmd = [path_7za, 'a', '-tzip', '-m0=lzma', '-mx9',
'-mfb=273', '-md=1536m', self.country_name + '.zip']
#cmd = ['7za', 'a', '-tzip', '-m0=lzma', countryName[1] + '.zip']
# cmd = ['7za', 'a', '-tzip', '-m0=lzma', countryName[1] + '.zip']
# Non-Windows
else:
cmd = ['zip', '-r', self.country_name + '.zip']
Expand Down

0 comments on commit 4d77ea1

Please sign in to comment.