Cim test coverage

.gitignore

@@ -22,6 +22,5 @@ data/
 maro_venv/
 pyvenv.cfg
 htmlcov/
-.coverage
-
-.coveragerc
+.coverage 
+.coveragerc 

maro/data_lib/cim/cim_data_container.py

@@ -151,7 +151,7 @@ def full_return_buffers(self) -> PortBufferTickWrapper:
             .. code-block:: python
 
                 # Get full return buffer tick of port 0.
-                buffer_tick = data_cnr.full_return_buffers[0]
+                buffer_tick = data_cntr.full_return_buffers[0]
         """
         return self._full_return_buffer_wrapper
 
@@ -241,7 +241,7 @@ def reset(self):
         self._is_need_reset_seed = True
 
     def _reset_seed(self):
-        """Reset internal seed for generate reproduceable data"""
+        """Reset internal seed for generate reproduce-able data"""
         random.reset_seed(BUFFER_TICK_RAND_KEY)
 
     @abstractmethod
@@ -288,7 +288,7 @@ def get_orders(self, tick: int, total_empty_container: int) -> List[Order]:
 
             self._is_need_reset_seed = False
 
-        if tick >= self._data_collection.max_tick:
+        if tick >= self._data_collection.max_tick:  # pragma: no cover
             warnings.warn(f"{tick} out of max tick {self._data_collection.max_tick}")
             return []
 
@@ -317,7 +317,7 @@ def _gen_orders(self, tick: int, total_empty_container: int) -> List[Order]:
         orders_to_gen = int(order_proportion[tick])
 
         # if under unfixed mode, we will consider current empty container as factor
-        if order_mode == OrderGenerateMode.UNFIXED:
+        if order_mode == OrderGenerateMode.UNFIXED:  # pragma: no cover. TODO: remove this mark later
             delta = total_containers - total_empty_container
 
             if orders_to_gen <= delta:
@@ -428,11 +428,8 @@ def get_orders(self, tick: int, total_empty_container: int) -> List[Order]:
 
             self._is_need_reset_seed = False
 
-        if tick >= self._data_collection.max_tick:
+        if tick >= self._data_collection.max_tick:  # pragma: no cover
             warnings.warn(f"{tick} out of max tick {self._data_collection.max_tick}")
             return []
 
-        if tick not in self._orders:
-            return []
-
-        return self._orders[tick]
+        return self._orders[tick] if tick in self._orders else []

maro/data_lib/cim/cim_data_container_helpers.py

@@ -9,7 +9,7 @@
 from maro.simulator.utils import random, seed
 
 from .cim_data_container import CimBaseDataContainer, CimRealDataContainer, CimSyntheticDataContainer
-from .cim_data_generator import CimDataGenerator
+from .cim_data_generator import gen_cim_data
 from .cim_data_loader import load_from_folder, load_real_data_from_folder
 from .utils import DATA_CONTAINER_INIT_SEED_LIMIT, ROUTE_INIT_RAND_KEY
 
@@ -40,11 +40,13 @@ def _init_data_container(self, topology_seed: int = None):
                 config_path=config_path, max_tick=self._max_tick, start_tick=self._start_tick,
                 topology_seed=topology_seed
             )
+        elif os.path.exists(os.path.join(self._config_path, "order_proportion.csv")):
+            self._data_cntr = data_from_dumps(dumps_folder=self._config_path)
         else:
             # Real Data Mode: read data from input data files, no need for any config.yml.
             self._data_cntr = data_from_files(data_folder=self._config_path)
 
-    def reset(self, keep_seed):
+    def reset(self, keep_seed: bool):
         """Reset data container internal state"""
         if not keep_seed:
             self._init_data_container(random[ROUTE_INIT_RAND_KEY].randint(0, DATA_CONTAINER_INIT_SEED_LIMIT - 1))
@@ -88,7 +90,7 @@ def data_from_generator(config_path: str, max_tick: int, start_tick: int = 0,
     Returns:
         CimSyntheticDataContainer: Data container used to provide cim data related interfaces.
     """
-    data_collection = CimDataGenerator.gen_data(
+    data_collection = gen_cim_data(
         config_path, start_tick=start_tick, max_tick=max_tick, topology_seed=topology_seed)
 
     return CimSyntheticDataContainer(data_collection)

maro/data_lib/cim/cim_data_dump.py

@@ -8,7 +8,7 @@
 import numpy as np
 from yaml import safe_dump
 
-from .cim_data_generator import CimDataGenerator
+from .cim_data_generator import gen_cim_data
 from .entities import CimSyntheticDataCollection, SyntheticPortSetting
 
 
@@ -248,7 +248,7 @@ def dump_from_config(config_file: str, output_folder: str, max_tick: int):
     assert output_folder is not None and os.path.exists(output_folder), f"Got output folder path: {output_folder}"
     assert max_tick is not None and max_tick > 0, f"Got max tick: {max_tick}"
 
-    data_collection = CimDataGenerator.gen_data(config_file, max_tick=max_tick, start_tick=0, topology_seed=None)
+    data_collection = gen_cim_data(config_file, max_tick=max_tick, start_tick=0, topology_seed=None)
 
     dump_util = CimDataDumpUtil(data_collection)
 

maro/data_lib/cim/cim_data_generator.py

@@ -7,10 +7,9 @@
 from yaml import safe_load
 
 from maro.simulator.utils import random, seed
-from maro.utils.exception.data_lib_exception import CimGeneratorInvalidParkingDuration
 
 from .entities import CimSyntheticDataCollection, OrderGenerateMode, Stop
-from .parsers import GlobalOrderProportion, PortsParser, RoutesParser, VesselsParser
+from .parsers import parse_global_order_proportion, parse_ports, parse_routes, parse_vessels
 from .utils import ROUTE_INIT_RAND_KEY, apply_noise
 
 CIM_GENERATOR_VERSION = 0x000001
@@ -62,9 +61,7 @@ def _extend_route(
 
             # apply noise to parking duration
             parking_duration = ceil(apply_noise(duration, duration_noise, random[ROUTE_INIT_RAND_KEY]))
-
-            if parking_duration <= 0:
-                raise CimGeneratorInvalidParkingDuration()
+            assert parking_duration > 0
 
             # a new stop
             stop = Stop(
@@ -108,88 +105,81 @@ def _extend_route(
     return vessel_stops, vessel_period_without_noise
 
 
-class CimDataGenerator:
-    """Utility to generate cim data from configuration file."""
-
-    def __init__(self) -> None:
-        pass
-
-    @staticmethod
-    def gen_data(
-        config_file: str, max_tick: int,
-        start_tick: int = 0,
-        topology_seed: int = None
-    ) -> CimSyntheticDataCollection:
-        """Generate data with specified configurations.
-
-        Args:
-            config_file(str): File of configuration (yaml).
-            max_tick(int): Max tick to generate.
-            start_tick(int): Start tick to generate.
-            topology_seed(int): Random seed of the business engine. \
-                'None' means using the seed in the configuration file.
-
-        Returns:
-            CimSyntheticDataCollection: Data collection contains all cim data.
-        """
-
-        # read config
-        with open(config_file, "r") as fp:
-            conf: dict = safe_load(fp)
-
-        if topology_seed is None:
-            topology_seed = conf["seed"]
-
-        # set seed to generate data
-        seed(topology_seed)
-
-        # misc configurations
-        total_containers = conf["total_containers"]
-        past_stop_number, future_stop_number = conf["stop_number"]
-        container_volumes = conf["container_volumes"]
-
-        # parse configurations
-        vessel_mapping, vessels_setting = VesselsParser.parse(conf["vessels"])
-        port_mapping, ports_setting = PortsParser.parse(conf["ports"], total_containers)
-        route_mapping, routes = RoutesParser.parse(conf["routes"])
-        global_order_proportion = GlobalOrderProportion.parse(
-            conf["container_usage_proportion"],
-            total_containers, start_tick=start_tick, max_tick=max_tick)
-
-        # extend routes with specified tick range
-        vessel_stops, vessel_period_without_noise = _extend_route(
-            future_stop_number, max_tick, vessels_setting, port_mapping, routes, route_mapping)
-
-        return CimSyntheticDataCollection(
-            # Port
-            port_settings=ports_setting,
-            port_mapping=port_mapping,
-            # Vessel
-            vessel_settings=vessels_setting,
-            vessel_mapping=vessel_mapping,
-            # Stop
-            vessel_stops=vessel_stops,
-            # Route
-            routes=routes,
-            route_mapping=route_mapping,
-            # Vessel Period
-            vessel_period_without_noise=vessel_period_without_noise,
-            # Volume/Container
-            container_volume=container_volumes[0],
-            # Cost Factors
-            load_cost_factor=conf["load_cost_factor"],
-            dsch_cost_factor=conf["dsch_cost_factor"],
-            # Visible Voyage Window
-            past_stop_number=past_stop_number,
-            future_stop_number=future_stop_number,
-            # Time Length of the Data Collection
-            max_tick=max_tick,
-            # Random Seed for Data Generation
-            seed=topology_seed,
-            # For Order Generation
-            total_containers=total_containers,
-            order_mode=OrderGenerateMode(conf["order_generate_mode"]),
-            order_proportion=global_order_proportion,
-            # Data Generator Version
-            version=str(CIM_GENERATOR_VERSION)
-        )
+def gen_cim_data(
+    config_file: str, max_tick: int,
+    start_tick: int = 0,
+    topology_seed: int = None
+) -> CimSyntheticDataCollection:
+    """Generate data with specified configurations.
+
+    Args:
+        config_file(str): File of configuration (yaml).
+        max_tick(int): Max tick to generate.
+        start_tick(int): Start tick to generate.
+        topology_seed(int): Random seed of the business engine. \
+            'None' means using the seed in the configuration file.
+
+    Returns:
+        CimSyntheticDataCollection: Data collection contains all cim data.
+    """
+
+    # read config
+    with open(config_file, "r") as fp:
+        conf: dict = safe_load(fp)
+
+    if topology_seed is None:
+        topology_seed = conf["seed"]
+
+    # set seed to generate data
+    seed(topology_seed)
+
+    # misc configurations
+    total_containers = conf["total_containers"]
+    past_stop_number, future_stop_number = conf["stop_number"]
+    container_volumes = conf["container_volumes"]
+
+    # parse configurations
+    vessel_mapping, vessels_setting = parse_vessels(conf["vessels"])
+    port_mapping, ports_setting = parse_ports(conf["ports"], total_containers)
+    route_mapping, routes = parse_routes(conf["routes"])
+    global_order_proportion = parse_global_order_proportion(
+        conf["container_usage_proportion"],
+        total_containers, start_tick=start_tick, max_tick=max_tick)
+
+    # extend routes with specified tick range
+    vessel_stops, vessel_period_without_noise = _extend_route(
+        future_stop_number, max_tick, vessels_setting, port_mapping, routes, route_mapping)
+
+    return CimSyntheticDataCollection(
+        # Port
+        port_settings=ports_setting,
+        port_mapping=port_mapping,
+        # Vessel
+        vessel_settings=vessels_setting,
+        vessel_mapping=vessel_mapping,
+        # Stop
+        vessel_stops=vessel_stops,
+        # Route
+        routes=routes,
+        route_mapping=route_mapping,
+        # Vessel Period
+        vessel_period_without_noise=vessel_period_without_noise,
+        # Volume/Container
+        container_volume=container_volumes[0],
+        # Cost Factors
+        load_cost_factor=conf["load_cost_factor"],
+        dsch_cost_factor=conf["dsch_cost_factor"],
+        # Visible Voyage Window
+        past_stop_number=past_stop_number,
+        future_stop_number=future_stop_number,
+        # Time Length of the Data Collection
+        max_tick=max_tick,
+        # Random Seed for Data Generation
+        seed=topology_seed,
+        # For Order Generation
+        total_containers=total_containers,
+        order_mode=OrderGenerateMode(conf["order_generate_mode"]),
+        order_proportion=global_order_proportion,
+        # Data Generator Version
+        version=str(CIM_GENERATOR_VERSION)
+    )

maro/data_lib/cim/cim_data_loader.py

@@ -22,7 +22,7 @@
 def _load_misc(data_folder: str) -> dict:
     """Load misc items from yaml"""
     misc_file_path = os.path.join(data_folder, "misc.yml")
-    for _ in range(3):
+    for _ in range(3):  # pragma: no cover
         if not os.path.exists(misc_file_path):
             time.sleep(10)
     with open(misc_file_path, "rt") as fp:
@@ -31,7 +31,7 @@ def _load_misc(data_folder: str) -> dict:
 
 def _read_csv_lines(file_path: str):
     """Helper to read and yield line from csv file"""
-    for _ in range(3):
+    for _ in range(3):  # pragma: no cover
         if not os.path.exists(file_path):
             time.sleep(10)
     with open(file_path, "rt") as fp: