Merge branch 'master' into pr/3601

.github/workflows/main.yml

@@ -28,4 +28,6 @@ jobs:
 
       - name: Run tests
         shell: bash -l {0}
-        run: cea test --workflow slow
+        run: |
+          cea test --type unittest
+          cea test --type integration

.github/workflows/pull.request.yml

@@ -66,4 +66,6 @@ jobs:
 
       - name: Run tests
         shell: bash -l {0}
-        run: cea test --workflow quick
+        run: |
+          cea test --type unittest
+          cea test --type integration

.github/workflows/setup_build.yml

@@ -61,13 +61,13 @@ jobs:
           repository: architecture-building-systems/CityEnergyAnalyst-GUI
 
       - name: Package CEA GUI
-        shell: bash
+        shell: bash -el {0}
         env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GUI_GH_TOKEN }}
         run: |
           cd $GITHUB_WORKSPACE/gui
           yarn
-          yarn version --new-version $CEA_VERSION --no-git-tag-version
+          yarn version $CEA_VERSION
           yarn electron:release
           mv "out/CityEnergyAnalyst-GUI Setup ${CEA_VERSION}.exe" $GITHUB_WORKSPACE/setup/gui_setup.exe
 
@@ -92,53 +92,59 @@ jobs:
           compression-level: 0 
 
   build_mac_dmg:
-    runs-on: ${{ matrix.os }}
-    strategy:
-      matrix:
-        os: ["macos-12", "macos-14"]
+    runs-on: "macos-12"
     steps:
       - uses: actions/checkout@v4
         with:
           path: cea
 
-      - uses: actions/checkout@v4
-        with:
-          path: gui
-          repository: architecture-building-systems/CityEnergyAnalyst-GUI
-
-      - uses: mamba-org/setup-micromamba@v1
+      - uses: actions/setup-python@v5
         with:
-          micromamba-binary-path: gui/dependencies/micromamba
-          environment-name: cea
-          create-args: >-
-            python=3.8
+          python-version: '3.8'
 
-      - name: Install CEA to environment
-        shell: bash -el {0}
-        run: pip install ./cea
+      - name: Install CEA to python environment
+        run: pip install ${{ github.workspace }}/cea
 
       - name: Get CEA version
-        shell: bash -el {0}
+        shell: bash
         run: echo "CEA_VERSION=$(python -c "import cea; print(cea.__version__)")" >> "$GITHUB_ENV"
 
+      - uses: actions/checkout@v4
+        with:
+          path: gui
+          repository: architecture-building-systems/CityEnergyAnalyst-GUI
+
+      - name: Fetch micromamba
+        run: |
+          cd gui
+          mkdir -p ./dependencies/arm64
+          curl -Ls https://micro.mamba.pm/api/micromamba/osx-arm64/latest | tar -xvj -C ./dependencies/arm64 --strip-components=1 bin/micromamba
+          mkdir -p ./dependencies/x64
+          curl -Ls https://micro.mamba.pm/api/micromamba/osx-64/latest | tar -xvj -C ./dependencies/x64 --strip-components=1 bin/micromamba 
+
       - name: Save Apple API Key secret to file
         env:
           APPLE_API_KEY_BASE64: ${{ secrets.APPLE_API_KEY_BASE64 }}
         run: |
           echo -n $APPLE_API_KEY_BASE64 | base64 -d -o ${{ github.workspace }}/gui/api_key.p8
+  
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+
+      - name: Enable Corepack
+        run: corepack enable
 
       - name: Package CEA GUI
-        shell: bash
         env:
           APPLE_API_ISSUER: ${{ secrets.APPLE_API_ISSUER }}
           APPLE_API_KEY: ${{ github.workspace }}/gui/api_key.p8
           APPLE_API_KEY_ID: ${{ secrets.APPLE_API_KEY_ID }}
           CSC_LINK: ${{ secrets.BUILD_CERTIFICATE_BASE64 }}
           CSC_KEY_PASSWORD: ${{ secrets.P12_PASSWORD }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GUI_GH_TOKEN }}
         run: |
           cd gui
           yarn
-          yarn version --new-version $CEA_VERSION --no-git-tag-version
-          yarn electron:release
-
+          yarn version $CEA_VERSION
+          yarn electron:release --arm64 --x64

CHANGELOG.md

@@ -1,3 +1,29 @@
+- 2024-05-27 - 3.36.0 - #3586 Add nullable info for column schema
+- 2024-05-21 - 3.36.0 - #3584 Release 3.36.0
+- 2024-05-16 - 3.35.6 - #3583 Use MKL for Windows to solve memory errors
+- 2024-05-15 - 3.35.6 - #3574 Add trees as shading for radiation script
+- 2024-05-14 - 3.35.6 - #3576 Consider adjacent walls by default
+- 2024-05-14 - 3.35.6 - #3579 Hotfix two decentralised optimisation workflow test
+- 2024-05-10 - 3.35.6 - #3575 Hotfix-decentralised-optimisaiton-test
+- 2024-05-07 - 3.35.6 - #3550 Optimisation - Calculate the baseline system
+- 2024-05-07 - 3.35.6 - #3572 Fixing database to fit the long unit test
+- 2024-05-07 - 3.35.6 - #3552 3524 memory error in optimisation script
+- 2024-05-07 - 3.35.6 - #3570 removing _new.xlsx in the databases
+- 2024-04-30 - 3.35.6 - #3567 Fixing oudated excel writer
+- 2024-04-29 - 3.35.6 - #3566 Push Docker build to GitHub Container Registry
+- 2024-04-29 - 3.35.6 - #3521 Add German Building Database
+- 2024-04-26 - 3.35.6 - #3563 Return empty data even if surroundings file is empty
+- 2024-04-25 - 3.35.6 - #3562 Remove `MULTIUSER_PAGE_INSTALLMODE`
+- 2024-04-25 - 3.35.6 - #3561 Update deprecated pandas append function
+- 2024-04-24 - 3.35.6 - #3559 Remove multiuser option from setup
+- 2024-04-18 - 3.35.6 - #3553 Enable auto update for windows
+- 2024-04-18 - 3.35.6 - #3543 Fixing interaction between energy potentials and supply system operation
+- 2024-04-18 - 3.35.6 - #3549 Handling zero interest rates
+- 2024-04-12 - 3.35.6 - #3548 Update mac build workflow
+- 2024-04-02 - 3.35.5 - #3539 Add build tools to environment
+- 2024-03-21 - 3.35.5 - #3536 Fix missing html template for plots
+- 2024-03-21 - 3.35.5 - #3535 Update workflow and docs based on new GUI build steps
+- 2024-03-21 - 3.35.5 - #3533 Release 3.35.5
 - 2024-03-20 - 3.35.4 - #3531 Correctly set installation directory for CurrentUser mode for Windows setup
 - 2024-03-20 - 3.35.4 - #3530 Add missing schema keys
 - 2024-03-16 - 3.35.4 - #3525 Fix Windows setup bugs regarding developer version

CREDITS.md

@@ -1,7 +1,7 @@
 How to Cite
 ===========
 
-The CEA Team. (2024). City Energy Analyst (v3.35.5). Zenodo. https://doi.org/10.5281/zenodo.10847779
+The CEA Team. (2024). City Energy Analyst (v3.36.1). Zenodo. https://doi.org/10.5281/zenodo.11382063
 
 The CEA team
 ============
@@ -11,6 +11,65 @@ and split into the categories of developers, product owner, project sponsor and
 
 
 <!-- credits -->
+- Version 3.36.1 - May 2024
+
+    Developers:
+    * Reynold Mok
+    * Martín Mosteiro Romero
+    * Mathias Niffeler
+    * Zhongming Shi
+
+    Product lead:
+    * Zhongming Shi
+
+    Product sponsor:
+    * [Arno Schlueter](https://systems.arch.ethz.ch/arno-schlueter)
+    * [Toni Piëch Foundation](https://www.tonipiechfoundation.org/)
+
+    Others:
+    * https://cityenergyanalyst.com/people
+
+
+- Version 3.36.0 - May 2024
+
+    Developers:
+    * [Amr Elesawy](https://cityenergyanalyst.com/people)
+    * [Jimeno A. Fonseca](https://cityenergyanalyst.com/people)
+    * [Gabriel Happle](https://cityenergyanalyst.com/people)
+    * [Shanshan Hsieh](https://cityenergyanalyst.com/people)
+    * [Reynold Mok](https://cityenergyanalyst.com/people)
+    * [Martín Mosteiro Romero](https://cityenergyanalyst.com/people)
+    * [Mathias Niffeler](https://cityenergyanalyst.com/people)
+    * [Anastasiya Popova](https://cityenergyanalyst.com/people)
+    * [Zhongming Shi](https://cityenergyanalyst.com/people)
+    * [Luis Santos](https://cityenergyanalyst.com/people)
+    * [Bhargava Krishna Sreepathi](https://cityenergyanalyst.com/people)
+    * [Daren Thomas](https://cityenergyanalyst.com/people)
+
+    Product lead:
+    * [Zhongming Shi](https://cityenergyanalyst.com/people)
+
+    Product sponsor:
+    * [Arno Schlueter](https://systems.arch.ethz.ch/arno-schlueter)
+    * [Toni Piëch Foundation](https://www.tonipiechfoundation.org/)
+
+    Collaborators:
+    * Jose Bello
+    * Kian Wee Chen
+    * Jack Hawthorne
+    * Fazel Khayatian
+    * Victor Marty
+    * Rowan Molony
+    * Paul Neitzel
+    * Thuy-An Nguyen
+    * Bo Lie Ong
+    * Emanuel Riegelbauer
+    * Lennart Rogenhofer
+    * Toivo Säwén
+    * Sebastian Troiztsch
+    * Tim Vollrath
+
+
 - Version 3.35.5 - March 2024
 
     Developers:

README.rst

@@ -4,10 +4,8 @@
     :alt: GitHub license
 .. |repo_size| image:: https://img.shields.io/github/repo-size/architecture-building-systems/CityEnergyAnalyst
     :alt: Repo Size
-.. |lines_of_code| image:: https://img.shields.io/tokei/lines/github/architecture-building-systems/CityEnergyAnalyst
-    :alt: Lines of code
-.. |zenodo| image:: https://zenodo.org/badge/DOI/10.5281/zenodo.10847779.svg
-   :target: https://doi.org/10.5281/zenodo.10847779
+.. |zenodo| image:: https://zenodo.org/badge/DOI/10.5281/zenodo.11382063.svg
+   :target: https://doi.org/10.5281/zenodo.11382063
 
 .. image:: cea_logo.png
     :scale: 25 %
@@ -33,4 +31,4 @@ We invite all CEA users to get acquainted with the CEA Dashboard and CEA Console
 Cite us:
 --------
 
-The CEA Team. (2024). City Energy Analyst (v3.35.5). Zenodo. https://doi.org/10.5281/zenodo.10847779
+The CEA Team. (2024). City Energy Analyst (v3.36.1). Zenodo. https://doi.org/10.5281/zenodo.11382063

bin/create_trace_graphviz.py

@@ -12,7 +12,7 @@
 
 import yaml
 import cea.config
-from cea.tests.trace_inputlocator import create_graphviz_output
+from cea.utilities.trace_inputlocator.trace_inputlocator import create_graphviz_output
 
 def main(config):
     with open(config.trace_inputlocator.yaml_output_file, 'r') as f:

cea/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "3.35.6"
+__version__ = "3.36.1"
 
 
 class ConfigError(Exception):

cea/datamanagement/databases_verification.py

@@ -47,7 +47,7 @@ def assert_input_geometry_acceptable_values_floor_height(zone_df: pd.DataFrame):
     # Rule 2. Where floor height is less than 1m on average above ground.
     rule2 = (zone_df['height_ag'] < zone_df['floors_ag']).any()
     if rule2:
-        raise Exception('one of more buildings have less report less than 1m height per floor. This is not possible'
+        raise Exception('one of more buildings report less than 1m height per floor. This is not possible'
                         'to simulate in CEA at the moment. Please verify your Zone or Surroundings shapefile file')
 
     # Rule 3. floors below ground cannot be negative

cea/datamanagement/surroundings_helper.py

@@ -10,6 +10,7 @@
 import pandas as pd
 from geopandas import GeoDataFrame as gdf
 from geopandas.tools import sjoin as spatial_join
+from shapely import MultiPolygon
 
 import cea.config
 import cea.inputlocator
@@ -35,7 +36,11 @@ def calc_surrounding_area(zone_gdf, buffer_m):
     :param float buffer_m: Buffer to add to zone building geometries
     :return: Surrounding area GeoDataFrame
     """
-    surrounding_area = gdf(geometry=[zone_gdf.geometry.buffer(buffer_m).unary_union], crs=zone_gdf.crs)
+    merged_zone = zone_gdf.geometry.unary_union
+    if isinstance(merged_zone, MultiPolygon):
+        merged_zone = merged_zone.convex_hull
+
+    surrounding_area = gdf(geometry=[merged_zone.buffer(buffer_m)], crs=zone_gdf.crs)
     return surrounding_area
 
 
@@ -135,7 +140,6 @@ def erase_no_surrounding_areas(all_surroundings, zone, area_with_buffer):
     :return: GeoDataFrame with surrounding buildings
     """
     buffer_polygon = area_with_buffer.to_crs(zone.crs).geometry.values[0]
-    zone_area = gdf(geometry=[zone.geometry.unary_union], crs=zone.crs)
 
     within_buffer = all_surroundings.geometry.intersects(buffer_polygon)
     surroundings = all_surroundings[within_buffer]
@@ -144,7 +148,7 @@ def erase_no_surrounding_areas(all_surroundings, zone, area_with_buffer):
                                    if not any([s_building_footprint.intersects(z_building_footprint)
                                                for z_building_footprint in zone.geometry])]
     footprints_gdf = gdf(geometry=footprints_without_overlaps, crs=surroundings.crs)
-    relevant_surroundings = spatial_join(surroundings, footprints_gdf, op='within')
+    relevant_surroundings = spatial_join(surroundings, footprints_gdf, predicate='within')
 
     return relevant_surroundings.copy()
 
@@ -175,8 +179,7 @@ def geometry_extractor_osm(locator, config):
     # get footprints of all the surroundings
     print("Getting building footprints")
     area_with_buffer_polygon = area_with_buffer.to_crs(get_geographic_coordinate_system()).geometry.values[0]
-    all_surroundings = osmnx.geometries.geometries_from_polygon(polygon=area_with_buffer_polygon,
-                                                                tags={"building": True})
+    all_surroundings = osmnx.features_from_polygon(polygon=area_with_buffer_polygon, tags={"building": True})
     all_surroundings = all_surroundings.to_crs(get_projected_coordinate_system(float(lat), float(lon)))
 
     # erase overlapping area

cea/datamanagement/trees_helper.py

@@ -0,0 +1,58 @@
+import os
+import warnings
+
+import fiona
+import geopandas as gpd
+from osgeo import gdal, osr
+
+import cea.config
+import cea.inputlocator
+from cea.schemas import schemas
+
+
+def check_terrain_bounds(tree_geometries, terrain_raster):
+    terrian_projection = terrain_raster.GetProjection()
+    proj4_str = osr.SpatialReference(wkt=terrian_projection).ExportToProj4()
+
+    # minx, miny, maxx, maxy
+    geometry_bounds = tree_geometries.to_crs(proj4_str).total_bounds
+
+    (upper_left_x, x_size, x_rotation, upper_left_y, y_rotation, y_size) = terrain_raster.GetGeoTransform()
+    minx = upper_left_x
+    maxy = upper_left_y
+    maxx = minx + x_size * terrain_raster.RasterXSize
+    miny = maxy + y_size * terrain_raster.RasterYSize
+
+    if minx > geometry_bounds[0] or miny > geometry_bounds[1] or maxx < geometry_bounds[2] or maxy < \
+            geometry_bounds[3]:
+        warnings.warn("Terrain and trees geometries do not overlap. Please ")
+
+
+def verify_tree_properties(tree_df):
+    required_columns = schemas(plugins=[])["get_tree_geometry"]["schema"]["columns"].keys()
+    diff = required_columns - tree_df.columns
+
+    if len(diff) > 0:
+        raise ValueError(f"{diff} columns are missing for tree properties.")
+
+
+def main(config):
+    locator = cea.inputlocator.InputLocator(config.scenario)
+
+    trees_df = gpd.read_file(config.trees_helper.trees)
+    terrain_raster = gdal.Open(locator.get_terrain())
+
+    # Set trees to zone crs
+    with fiona.open(locator.get_zone_geometry(), 'r') as f:
+        zone_crs = f.crs
+    trees_df.to_crs(zone_crs, inplace=True)
+
+    verify_tree_properties(trees_df)
+    check_terrain_bounds(trees_df.geometry, terrain_raster)
+
+    os.makedirs(locator.get_tree_geometry_folder(), exist_ok=True)
+    trees_df.to_file(locator.get_tree_geometry())
+
+
+if __name__ == '__main__':
+    main(cea.config.Configuration())