Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Redo #160 #170

Closed
wants to merge 9 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 27 additions & 7 deletions bw2io/extractors/simapro_csv.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,16 @@
from ..compatibility import SIMAPRO_BIOSPHERE
from ..strategies.simapro import normalize_simapro_formulae
from bw2data.logs import get_io_logger, close_log
from asteval import Interpreter
from bw2parameters import ParameterSet
from numbers import Number
from stats_arrays import *
from stats_arrays import (
LognormalUncertainty,
NormalUncertainty,
TriangularUncertainty,
UndefinedUncertainty,
UniformUncertainty,
)
import csv
import math
import os
Expand Down Expand Up @@ -51,7 +58,13 @@ def to_number(obj):
return float(obj.replace("%", "").strip()) / 100.0
try:
# Eval for simple expressions like "1/2"
return float(eval(obj.replace(",", ".").strip()))
number = Interpreter().eval(
obj.replace(",", ".").replace("^", "**").strip()
)
if isinstance(number, (int, float)):
return float(number)
else:
return obj
except NameError:
# Formula with a variable which isn't in scope - raises NameError
return obj
Expand Down Expand Up @@ -93,9 +106,16 @@ class SimaProCSVExtractor(object):
@classmethod
def extract(cls, filepath, delimiter=";", name=None, encoding="cp1252"):
assert os.path.exists(filepath), "Can't find file %s" % filepath
log, logfile = get_io_logger("SimaPro-extractor")

log.info(INTRODUCTION % (filepath, repr(delimiter), name,))
log, _ = get_io_logger("SimaPro-extractor")

log.info(
INTRODUCTION
% (
filepath,
repr(delimiter),
name,
)
)
with open(filepath, "r", encoding=encoding) as csv_file:
reader = csv.reader(csv_file, delimiter=delimiter)
lines = [
Expand Down Expand Up @@ -207,7 +227,7 @@ def get_project_name(cls, data):
return line[0][9:-1].strip()

@classmethod
def invalid_uncertainty_data(cls, amount, kind, field1, field2, field3):
def invalid_uncertainty_data(cls, amount, kind, field1):
if kind == "Lognormal" and (not amount or field1 == "0"):
return True

Expand All @@ -220,7 +240,7 @@ def create_distribution(cls, amount, kind, field1, field2, field3):
"loc": amount,
"amount": amount,
}
elif cls.invalid_uncertainty_data(amount, kind, field1, field2, field3):
elif cls.invalid_uncertainty_data(amount, kind, field1):
# TODO: Log invalid data?
return {
"uncertainty type": UndefinedUncertainty.id,
Expand Down
25 changes: 22 additions & 3 deletions bw2io/strategies/simapro.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import re
import numpy as np
from stats_arrays import LognormalUncertainty
import bw2parameters
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This means we have to add bw2parameters to the setup requirements. What do you say @cmutel?



# Pattern for SimaPro munging of ecoinvent names
Expand Down Expand Up @@ -48,9 +49,27 @@ def sp_allocate_products(db):
for product in products:
product = copy.deepcopy(product)
if product["allocation"]:
product["amount"] = (
product["amount"] * 1 / (product["allocation"] / 100)
)
allocation = product["allocation"]
if type(product["allocation"]) is str and "parameters" in ds:
interp = bw2parameters.DefaultParameterSet(
ds["parameters"]
).get_interpreter()
interp.add_symbols(
bw2parameters.DefaultParameterSet(
ds["parameters"]
).evaluate_and_set_amount_field()
)
allocation = interp(
normalize_simapro_formulae(
product["allocation"].lower(),
settings={"Decimal separator": ","},
)
)

if allocation != 0:
product["amount"] = product["amount"] * 1 / (allocation / 100)
else:
product["amount"] = 0 # Infinity as zero? :-/
else:
product["amount"] = 0
copied = copy.deepcopy(ds)
Expand Down
1 change: 1 addition & 0 deletions requirements-test.txt
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
asteval
bw2calc>=1.3
bw2data>=3.3
bw_migrations
Expand Down
11 changes: 9 additions & 2 deletions setup.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
from setuptools import setup

REQUIREMENTS = [
"asteval>=0.9.30",
"bw2calc>=1.7.4",
"bw2data>=3.5.1",
"bw2parameters",
"bw_migrations",
"lxml",
"mrio_common_metadata",
"numpy==1.23",
"numpy>=1.23",
"openpyxl",
"psutil",
"pyprind",
Expand All @@ -30,7 +32,12 @@
"bw2io.strategies",
],
package_data={
"bw2io": ["data/*.*", "data/examples/*.*", "data/lci/*.*", "data/lcia/*.*",]
"bw2io": [
"data/*.*",
"data/examples/*.*",
"data/lci/*.*",
"data/lcia/*.*",
]
},
author="Chris Mutel",
author_email="cmutel@gmail.com",
Expand Down
6 changes: 6 additions & 0 deletions tests/simapro.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
get_biosphere_2_3_category_migration_data,
get_biosphere_2_3_name_migration_data,
)
from bw2io.extractors.simapro_csv import to_number

# from bw2data.utils import recursive_str_to_unicode as _
# from stats_arrays import UndefinedUncertainty, NoUncertainty
Expand Down Expand Up @@ -114,6 +115,11 @@ def test_set_lognormal_loc_value_on_import():
assert np.abs(loc - np.log(amount)) < 1e-10


@bw2test
def test_to_number():
assert to_number("(38-15)*4185*30/0.9*10^-6") == 3.2085


class SimaProCSVImporterTest(BW2DataTest):
# def extra_setup(self):
# # SimaPro importer always wants biosphere database
Expand Down