Skip to content

Commit

Permalink
Merge pull request #124 from nismod/finish_arc_fixed
Browse files Browse the repository at this point in the history 
Finish arc fixed
  • Loading branch information
@edwardoughton
edwardoughton committed Nov 13, 2019
2 parents 0406624 + 1c8a704 commit 497b04b
Showing 1 changed file with 102 additions and 73 deletions.
175 changes: 102 additions & 73 deletions scripts/arc_fixed.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@
import configparser
import csv
import os

from collections import defaultdict
from itertools import tee

CONFIG = configparser.ConfigParser()
Expand All @@ -19,6 +21,9 @@
DATA = os.path.join(BASE_PATH,'..','arc_fixed_bb')
RESULTS = os.path.join(BASE_PATH, '..','arc_fixed_bb', 'results')

COST_LOWER_BOUND = 500


def read_data(path):

with open(path, 'r') as csvfile:
Expand Down Expand Up @@ -57,7 +62,8 @@ def processing_dwellings(data):
'oa11cd': dwelling_item['oa11cd'],
'lad11cd': dwelling_item['lad11cd'],
'lad11nm': dwelling_item['lad11nm'],
'dwellings_oa__final': dwelling_item['dwellings_oa__final'],
'dwellings_oa__final': int(dwelling_item['dwellings_oa__final']),
'dwellings_oa__initial': int(dwelling_item['dwellings_oa__initial']),
})

return dwelling_data
Expand All @@ -77,6 +83,7 @@ def process_geotypes(dwelling_data, urban_rural_lut, lad_areas):
'lad11cd': dwelling_item['lad11cd'],
'lad11nm': dwelling_item['lad11nm'],
'dwellings_oa__final': dwelling_item['dwellings_oa__final'],
'dwellings_oa__initial': dwelling_item['dwellings_oa__initial'],
'geotype': int(geotype_item['geotype']),
'geotype_name': geotype_item['geotype_name']
})
Expand All @@ -88,62 +95,55 @@ def process_area_data(dwelling_data, area_data):

final_data = []

for area in area_data:
for item in dwelling_data:
if area['oa11cd'] == item['oa11cd']:
final_data.append({
'scenario': item['scenario'],
'oa11cd': item['oa11cd'],
'lad11cd': item['lad11cd'],
'lad11nm': item['lad11nm'],
'dwellings_oa__final': item['dwellings_oa__final'],
'geotype': int(item['geotype']),
'geotype_name': item['geotype_name'],
'dwelling_density': (
int(item['dwellings_oa__final']) /
(float(area['st_areasha']) / 100)),
'area_km2': float(area['st_areasha']) / 100,
})
area_lu = {area['oa11cd']: float(area['st_areasha']) / 1e6 for area in area_data}

for item in dwelling_data:
area = area_lu[item['oa11cd']]
final_data.append({
'scenario': item['scenario'],
'oa11cd': item['oa11cd'],
'lad11cd': item['lad11cd'],
'lad11nm': item['lad11nm'],
'dwellings_oa__final': item['dwellings_oa__final'],
'dwellings_oa__initial': item['dwellings_oa__initial'],
'geotype': item['geotype'],
'geotype_name': item['geotype_name'],
'dwelling_density': item['dwellings_oa__final'] / area,
'area_km2': area,
})

return final_data


def lad_dwelling_density(dwelling_data, urban_rural_lut):

unique_scenarios = set()

for item in dwelling_data:
unique_scenarios.add(item['scenario'])

"""Calculate initial/baseline LAD dwelling density
"""
interim = []

unique_lads = set()

for oa in dwelling_data:
unique_lads.add(oa['lad11cd'])

for scenario in list(unique_scenarios):
for lad in list(unique_lads):
area_of_lad = 0
dwellings_in_lad = 0
for oa in dwelling_data:
if scenario == oa['scenario']:
area_of_lad += float(oa['area_km2'])
dwellings_in_lad += float(oa['dwellings_oa__final'])
interim.append({
'scenario': scenario,
'lad11cd': lad,
'area_of_lad': area_of_lad,
'dwellings_in_lad': dwellings_in_lad,
})
for lad in list(unique_lads):
area_of_lad = 0
dwellings_in_lad = 0
for oa in dwelling_data:
if oa['scenario'] == 'baseline' and lad == oa['lad11cd']:
area_of_lad += float(oa['area_km2'])
dwellings_in_lad += float(oa['dwellings_oa__initial'])
interim.append({
'lad11cd': lad,
'area_of_lad': area_of_lad,
'dwellings_in_lad': dwellings_in_lad,
})

output = []

for lad in interim:
for item in urban_rural_lut:
if lad['lad11cd'] == item['lad11cd']:
output.append({
'scenario': lad['scenario'],
'lad11cd': lad['lad11cd'],
'area_of_lad': lad['area_of_lad'],
'dwellings_in_lad': lad['dwellings_in_lad'],
Expand Down Expand Up @@ -179,7 +179,6 @@ def add_costs_to_lad_lut(lad_dwelling_density_lut, cost_data):
for cost_item in cost_data:
if int(item['geotype']) == int(cost_item['geotype']):
output.append({
'scenario': item['scenario'],
'lad11cd': item['lad11cd'],
'area_of_lad': item['area_of_lad'],
'dwellings_in_lad': item['dwellings_in_lad'],
Expand All @@ -192,21 +191,42 @@ def add_costs_to_lad_lut(lad_dwelling_density_lut, cost_data):

return output

def cost_for_mean_density(lad_density_cost):
"""Calculate mean density of LADs by geotype
"""
geotype_strategy_densities = defaultdict(list)
for item in lad_density_cost:
geotype_strategy_densities[item['geotype'], item['strategy']] \
.append(item['dwelling_density'])

density_cost = []
for (geotype, strategy), densities in geotype_strategy_densities.items():
for item in lad_density_cost:
if item['geotype'] == geotype and item['strategy'] == strategy:
density_cost.append({
'geotype': geotype,
'geotype_name': item['geotype_name'],
'strategy': strategy,
'cost': item['cost'],
'dwelling_density': sum(densities) / len(densities)
})
break
return density_cost

def add_cost_to_oas(oa_dwelling_data, lad_dwelling_density_lut):

def add_cost_to_oas(oa_dwelling_data, cost_density_lut):

unique_strategies = set()

for item in lad_dwelling_density_lut:
for item in cost_density_lut:
unique_strategies.add(item['strategy'])


output = []

for oa in oa_dwelling_data:
for strategy in unique_strategies:
dwelling_density = oa['dwelling_density']
cost = lookup_cost(dwelling_density, strategy, lad_dwelling_density_lut)
cost, debug = lookup_cost(dwelling_density, strategy, cost_density_lut)
output.append({
'scenario': oa['scenario'],
'strategy': strategy,
Expand All @@ -216,12 +236,11 @@ def add_cost_to_oas(oa_dwelling_data, lad_dwelling_density_lut):
'dwellings_oa__final': oa['dwellings_oa__final'],
'geotype': int(oa['geotype']),
'geotype_name': oa['geotype_name'],
'dwelling_density': (
int(oa['dwellings_oa__final']) /
(float(oa['area_km2']) / 100)),
'area_km2': float(oa['area_km2']) / 100,
'dwelling_density': oa['dwelling_density'],
'area_km2': float(oa['area_km2']),
'cost_per_dwelling': cost,
'total_cost': cost * int(oa['dwellings_oa__final']),
'debug_cost_density': debug,
})

return output
Expand Down Expand Up @@ -249,27 +268,43 @@ def lookup_cost(dwelling_density, strategy, lad_dwelling_density_lut):
(float(item['dwelling_density']), float(item['cost']))
)

density_costs = sorted(density_costs, key=lambda d: d[0])

lowest_density, lowest_cost = density_costs[0]
next_density, next_cost = density_costs[1]
if dwelling_density < lowest_density:
return interpolate(0, 0, lowest_density, lowest_cost, dwelling_density) #Tom can you check this please?
return interpolate(
lowest_density, lowest_cost, next_density, next_cost, dwelling_density
), "<{}".format(int(lowest_density))

for a, b in pairwise(density_costs):
lower_density, lower_capacity = a
upper_density, upper_capacity = b
lower_density, lower_cost = a
upper_density, upper_cost = b
if lower_density <= dwelling_density and dwelling_density < upper_density:
return interpolate(lower_density, lower_capacity, upper_density, upper_capacity, dwelling_density)
return interpolate(
lower_density, lower_cost, upper_density, upper_cost, dwelling_density
), "{}-{} ({}-{}) {}".format(
int(lower_density), int(upper_density),
int(lower_cost), int(upper_cost), int(dwelling_density))

highest_density, highest_capacity = density_costs[-1]
next_highest_density, next_highest_cost = density_costs[-2]
highest_density, highest_cost = density_costs[-1]

return highest_capacity
cost = interpolate(
next_highest_density, next_highest_cost,
highest_density, highest_cost, dwelling_density)

return max(cost, COST_LOWER_BOUND), ">{}".format(highest_density)


def interpolate(x0, y0, x1, y1, x):
"""Linear interpolation between two values.
"""
Linear interpolation between two values.
"""
y = (y0 * (x1 - x) + y1 * (x - x0)) / (x1 - x0)
try:
y = (y0 * (x1 - x) + y1 * (x - x0)) / (x1 - x0)
except ZeroDivisionError as e:
print(x1,x0)
raise e
return y


Expand All @@ -281,9 +316,7 @@ def csv_writer(data, directory, filename):
if not os.path.exists(directory):
os.makedirs(directory)

fieldnames = []
for name, value in data[0].items():
fieldnames.append(name)
fieldnames = list(data[0].keys())

with open(os.path.join(directory, filename), 'w') as csv_file:
writer = csv.DictWriter(csv_file, fieldnames, lineterminator = '\n')
Expand Down Expand Up @@ -324,17 +357,12 @@ def csv_writer(data, directory, filename):

print('load geotype lut')
path = os.path.join(DATA, 'RUC11_LAD11_ENv2.csv')
urban_rural_lut = read_data(path)

print('process geotype lut')
urban_rural_lut = lad_geotypes(urban_rural_lut)
urban_rural_lut = lad_geotypes(read_data(path))
csv_writer(urban_rural_lut, RESULTS, 'urban_rural_lut.csv')

print('load dwelling data')
path = os.path.join(DATA, 'processed','oa_dwellings.csv')
data = read_data(path)

print('processing dwelling data')
oa_dwelling_data = processing_dwellings(data)[:200]
oa_dwelling_data = processing_dwellings(read_data(path))

print('process geotypes')
oa_dwelling_data = process_geotypes(oa_dwelling_data, urban_rural_lut, LAD_AREAS)
Expand All @@ -351,15 +379,16 @@ def csv_writer(data, directory, filename):

print('load nic costs')
path = os.path.join(DATA, 'nic_costs.csv')
cost_data = read_data(path)

print('process cost data')
cost_data = process_cost_data(cost_data)
cost_data = process_cost_data(read_data(path))

print('add costs to lad lut')
lad_dwelling_density_lut = add_costs_to_lad_lut(lad_dwelling_density_lut, cost_data)
csv_writer(lad_dwelling_density_lut, RESULTS, 'lad_dwelling_density_lut.csv')

cost_density_lut = cost_for_mean_density(lad_dwelling_density_lut)
csv_writer(cost_density_lut, RESULTS, 'cost_density_lut.csv')

print('add costs to oas lut')
output = add_cost_to_oas(oa_dwelling_data, lad_dwelling_density_lut)
output = add_cost_to_oas(oa_dwelling_data, cost_density_lut)

csv_writer(output, RESULTS, 'results.csv')

0 comments on commit 497b04b

Please sign in to comment.