Add performance metrics in report.py

.github/workflows/test.yml

@@ -8,11 +8,11 @@ on:
         type: string
 
 env:
-  up_tamer_commit: "db6709d7c465d52db6d1522206b199e30aed23a1"
-  up_pyperplan_commit: "d6d54f8a8b67e7653599e007a8373245fb2fffd3"
+  up_tamer_commit: "4d179b5c2722758127bf684d732a6a44431a1732"
+  up_pyperplan_commit: "6cec570b88d62ca5c4b8029a60c704414fe7f6a9"
   up_fast_downward_commit: "090977461c6a55812bc16d5e2df55e556dbbdfb5"
   up_enhsp_commit: "af93cd799974626ac17b8aaf1a83975f75c073ab"
-  up_fmap_commit: "98a837f42b9d8d1aaf4c2afe39c9a78cbd5f4043"
+  up_fmap_commit: "f29e66c8483abcb8f17ff1c46a0745ee9b1e95fa"
 
 jobs:
   basic-check:

setup.py

@@ -23,23 +23,23 @@
         "dev": ["tarski[arithmetic]", "pytest", "pytest-cov", "mypy"],
         "grpc": ["grpcio", "grpcio-tools", "grpc-stubs"],
         "tarski": ["tarski[arithmetic]"],
-        "pyperplan": ["up-pyperplan==1.0.0.4.dev1"],
-        "tamer": ["up-tamer==1.0.0.4.dev1"],
+        "pyperplan": ["up-pyperplan==1.0.0.6.dev1"],
+        "tamer": ["up-tamer==1.0.0.6.dev1"],
         "enhsp": ["up-enhsp==0.0.21"],
         "fast-downward": ["up-fast-downward==0.4.0"],
-        "lpg": ["up-lpg==0.0.7"],
-        "fmap": ["up-fmap==0.0.10"],
-        "aries": ["up-aries>=0.2.0"],
+        "lpg": ["up-lpg==0.0.10"],
+        "fmap": ["up-fmap==0.0.12"],
+        "aries": ["up-aries>=0.3.1"],
         "symk": ["up-symk>=1.0.1"],
         "engines": [
             "tarski[arithmetic]",
-            "up-pyperplan==1.0.0.4.dev1",
-            "up-tamer==1.0.0.4.dev1",
+            "up-pyperplan==1.0.0.6.dev1",
+            "up-tamer==1.0.0.6.dev1",
             "up-enhsp==0.0.21",
             "up-fast-downward==0.4.0",
-            "up-lpg==0.0.7",
-            "up-fmap==0.0.10",
-            "up-aries>=0.2.0",
+            "up-lpg==0.0.10",
+            "up-fmap==0.0.12",
+            "up-aries>=0.3.1",
             "up-symk>=1.0.1",
         ],
         "plot": [

unified_planning/engines/oversubscription_planner.py

@@ -171,7 +171,6 @@ def _solve(
                     status,
                     res.plan,
                     self.name,
-                    metrics=res.metrics,
                     log_messages=res.log_messages,
                 )
             elif res.status == PlanGenerationResultStatus.TIMEOUT:

unified_planning/engines/pddl_planner.py

@@ -187,6 +187,7 @@ def _solve(
                 cmd = self._get_anytime_cmd(
                     domain_filename, problem_filename, plan_filename
                 )
+            process_start = time.time()
             if output_stream is None:
                 # If we do not have an output stream to write to, we simply call
                 # a subprocess and retrieve the final output and error with communicate
@@ -229,6 +230,7 @@ def _solve(
                         )
                 timeout_occurred, (proc_out, proc_err), retval = exec_res
 
+            process_end = time.time()
             logs.append(up.engines.results.LogMessage(LogLevel.INFO, "".join(proc_out)))
             logs.append(
                 up.engines.results.LogMessage(LogLevel.ERROR, "".join(proc_err))
@@ -237,18 +239,25 @@ def _solve(
                 plan = self._plan_from_file(
                     problem, plan_filename, self._writer.get_item_named
                 )
+            metrics = {}
+            metrics["engine_internal_time"] = str(process_end - process_start)
             if timeout_occurred and retval != 0:
                 return PlanGenerationResult(
                     PlanGenerationResultStatus.TIMEOUT,
                     plan=plan,
-                    log_messages=logs,
                     engine_name=self.name,
+                    log_messages=logs,
+                    metrics=metrics,
                 )
         status: PlanGenerationResultStatus = self._result_status(
             problem, plan, retval, logs
         )
         res = PlanGenerationResult(
-            status, plan, log_messages=logs, engine_name=self.name
+            status,
+            plan,
+            engine_name=self.name,
+            log_messages=logs,
+            metrics=metrics,
         )
         problem_kind = problem.kind
         if problem_kind.has_continuous_time() or problem_kind.has_discrete_time():

unified_planning/engines/results.py

@@ -274,6 +274,7 @@ class ValidationResult(Result):
     )
     reason: Optional[FailedValidationReason] = field(default=None)
     inapplicable_action: Optional[up.plans.ActionInstance] = field(default=None)
+    metrics: Optional[Dict[str, str]] = field(default=None)
 
     def __post_init__(self):
         assert (
@@ -313,6 +314,7 @@ class CompilerResult(Result):
     ]
     engine_name: str
     log_messages: Optional[List[LogMessage]] = field(default=None)
+    metrics: Optional[Dict[str, str]] = field(default=None)
 
     def _post_init(self):
         # Check that compiled problem and map_back_action_instance are consistent with each other

unified_planning/grpc/proto_reader.py

@@ -848,10 +848,10 @@ def _convert_plan_generation_result(
             raise UPException(f"Unknown Planner Status: {result.status}")
 
         log_messages = None
-        metrics = None
+        engine_metrics = None
 
         if bool(result.metrics):
-            metrics = dict(result.metrics)
+            engine_metrics = dict(result.metrics)
 
         if len(result.log_messages) > 0:
             log_messages = [self.convert(log) for log in result.log_messages]
@@ -862,7 +862,7 @@ def _convert_plan_generation_result(
             if result.HasField("plan")
             else None,
             engine_name=result.engine.name,
-            metrics=metrics,
+            metrics=engine_metrics,
             log_messages=log_messages,
         )
 
@@ -912,12 +912,16 @@ def _convert_compiler_result(
                 original_action_instance.action,
                 original_action_instance.actual_parameters,
             )
+        engine_metrics = None
+        if bool(result.metrics):
+            engine_metrics = dict(result.metrics)
         return unified_planning.engines.CompilerResult(
             problem=problem,
             map_back_action_instance=partial(
                 unified_planning.engines.compilers.utils.lift_action_instance, map=map
             ),
             engine_name=result.engine.name,
+            metrics=engine_metrics,
             log_messages=[self.convert(log) for log in result.log_messages],
         )
 
@@ -939,8 +943,12 @@ def _convert_validation_result(
             r_status = unified_planning.engines.ValidationResultStatus.UNKNOWN
         else:
             raise UPException(f"Unexpected ValidationResult status: {result.status}")
+        engine_metrics = None
+        if bool(result.metrics):
+            engine_metrics = dict(result.metrics)
         return unified_planning.engines.ValidationResult(
             status=r_status,
             engine_name=result.engine.name,
+            metrics=engine_metrics,
             log_messages=[self.convert(log) for log in result.log_messages],
         )

unified_planning/grpc/proto_writer.py

@@ -980,6 +980,7 @@ def _convert_compiler_result(
         return proto.CompilerResult(
             problem=self.convert(result.problem),
             map_back_plan=map,
+            metrics=result.metrics,
             log_messages=[self.convert(log) for log in log_messages],
             engine=proto.Engine(name=result.engine_name),
         )
@@ -990,6 +991,7 @@ def _convert_validation_result(
     ) -> proto.ValidationResult:
         return proto.ValidationResult(
             status=self.convert(result.status),
+            metrics=result.metrics,
             log_messages=[self.convert(log) for log in result.log_messages],
             engine=proto.Engine(name=result.engine_name),
         )

unified_planning/grpc/unified_planning.proto

@@ -844,6 +844,9 @@ message ValidationResult {
     }
     ValidationResultStatus status = 1;
 
+    // A set of engine specific values that can be reported
+    map<string, string> metrics = 4;
+
     // Optional. Information given by the engine to the user.
     repeated LogMessage log_messages = 2;
 
@@ -861,6 +864,9 @@ message CompilerResult {
     // compiled problem to the original ActionInstance.
     map<string, ActionInstance> map_back_plan = 2;
 
+    // A set of engine specific values that can be reported
+    map<string, string> metrics = 5;
+
     // Optional. Information given by the engine to the user.
     repeated LogMessage log_messages = 3;
 

unified_planning/io/ma_pddl_writer.py

@@ -760,28 +760,26 @@ def get_ma_problem_agent(self, agent_name) -> str:
     def write_ma_domain(self, directory_name):
         """Dumps to file the `MA-PDDL` domains."""
         ag_domains = self._write_domain()
-        outdir_ma_pddl = "ma_pddl_" + directory_name
-        osy.makedirs(outdir_ma_pddl, exist_ok=True)
+        osy.makedirs(directory_name, exist_ok=True)
         for ag, domain in ag_domains.items():
             if not self.unfactored:
-                name_domain = ag
+                name_domain = ag + "_"
             else:
-                name_domain = _get_pddl_name(self.problem)
-            path_ma_pddl = osy.path.join(outdir_ma_pddl, name_domain + "_domain.pddl")
+                name_domain = ""
+            path_ma_pddl = osy.path.join(directory_name, name_domain + "domain.pddl")
             with open(path_ma_pddl, "w") as f:
                 f.write(domain)
 
     def write_ma_problem(self, directory_name):
         """Dumps to file the `MA-PDDL` problems."""
         ag_problems = self._write_problem()
-        outdir_ma_pddl = "ma_pddl_" + directory_name
-        osy.makedirs(outdir_ma_pddl, exist_ok=True)
+        osy.makedirs(directory_name, exist_ok=True)
         for ag, problem in ag_problems.items():
             if not self.unfactored:
-                name_problem = ag
+                name_problem = ag + "_"
             else:
-                name_problem = _get_pddl_name(self.problem)
-            path_ma_pddl = osy.path.join(outdir_ma_pddl, name_problem + "_problem.pddl")
+                name_problem = ""
+            path_ma_pddl = osy.path.join(directory_name, name_problem + "problem.pddl")
             with open(path_ma_pddl, "w") as f:
                 f.write(problem)
 

up_test_cases/builtin/classical/depots/__init__.py

@@ -0,0 +1,9 @@
+import os
+from functools import partial
+
+from utils import _get_pddl_test_cases
+
+FILE_DIR = os.path.dirname(os.path.abspath(__file__))
+PDDL_FILES_DIR = os.path.join(FILE_DIR, "pddl_files")
+
+get_test_cases = partial(_get_pddl_test_cases, PDDL_FILES_DIR, filter=("1"))

up_test_cases/builtin/classical/depots/pddl_files/depots_domain.pddl

@@ -0,0 +1,39 @@
+(define (domain depot)
+(:predicates
+	 (at ?x ?y) (on ?x ?y) (in ?x ?y) (lifting ?x ?y) (available ?x) (clear ?x)(place ?x) (locatable ?x) (depot ?x) (distributor ?x) (truck ?x) (hoist ?x) (surface ?x) (pallet ?x) (crate ?x) )
+(:action drive
+ :parameters ( ?x ?y ?z)
+ :precondition
+	(and (truck ?x) (place ?y) (place ?z)  (at ?x ?y))
+ :effect
+	(and (at ?x ?z) (not (at ?x ?y))))
+
+(:action lift
+ :parameters ( ?x ?y ?z ?p)
+ :precondition
+	(and (hoist ?x) (crate ?y) (surface ?z) (place ?p)  (at ?x ?p) (available ?x) (at ?y ?p) (on ?y ?z) (clear ?y))
+ :effect
+	(and (lifting ?x ?y) (clear ?z) (not (at ?y ?p)) (not (clear ?y)) (not (available ?x)) (not (on ?y ?z))))
+
+(:action drop
+ :parameters ( ?x ?y ?z ?p)
+ :precondition
+	(and (hoist ?x) (crate ?y) (surface ?z) (place ?p)  (at ?x ?p) (at ?z ?p) (clear ?z) (lifting ?x ?y))
+ :effect
+	(and (available ?x) (at ?y ?p) (clear ?y) (on ?y ?z) (not (lifting ?x ?y)) (not (clear ?z))))
+
+(:action load
+ :parameters ( ?x ?y ?z ?p)
+ :precondition
+	(and (hoist ?x) (crate ?y) (truck ?z) (place ?p)  (at ?x ?p) (at ?z ?p) (lifting ?x ?y))
+ :effect
+	(and (in ?y ?z) (available ?x) (not (lifting ?x ?y))))
+
+(:action unload
+ :parameters ( ?x ?y ?z ?p)
+ :precondition
+	(and (hoist ?x) (crate ?y) (truck ?z) (place ?p)  (at ?x ?p) (at ?z ?p) (available ?x) (in ?y ?z))
+ :effect
+	(and (lifting ?x ?y) (not (in ?y ?z)) (not (available ?x))))
+
+)

up_test_cases/builtin/classical/depots/pddl_files/depots_p01.pddl

@@ -0,0 +1,47 @@
+(define (problem depotprob1818) (:domain Depot)
+(:objects
+	depot0 distributor0 distributor1 truck0 truck1 pallet0 pallet1 pallet2 crate0 crate1 hoist0 hoist1 hoist2 )
+(:init
+	(pallet pallet0)
+	(surface pallet0)
+	(at pallet0 depot0)
+	(clear crate1)
+	(pallet pallet1)
+	(surface pallet1)
+	(at pallet1 distributor0)
+	(clear crate0)
+	(pallet pallet2)
+	(surface pallet2)
+	(at pallet2 distributor1)
+	(clear pallet2)
+	(truck truck0)
+	(at truck0 distributor1)
+	(truck truck1)
+	(at truck1 depot0)
+	(hoist hoist0)
+	(at hoist0 depot0)
+	(available hoist0)
+	(hoist hoist1)
+	(at hoist1 distributor0)
+	(available hoist1)
+	(hoist hoist2)
+	(at hoist2 distributor1)
+	(available hoist2)
+	(crate crate0)
+	(surface crate0)
+	(at crate0 distributor0)
+	(on crate0 pallet1)
+	(crate crate1)
+	(surface crate1)
+	(at crate1 depot0)
+	(on crate1 pallet0)
+	(place depot0)
+	(place distributor0)
+	(place distributor1)
+)
+
+(:goal (and
+		(on crate0 pallet2)
+		(on crate1 pallet1)
+	)
+))

up_test_cases/builtin/classical/depots/pddl_files/depots_p01_invalid.plan

@@ -0,0 +1,5 @@
+(lift hoist1 crate0 pallet1 distributor0)
+(load hoist0 crate1 truck1 depot0)
+(drive truck1 depot0 distributor1)
+(unload hoist2 crate0 truck0 distributor1)
+(unload hoist1 crate1 truck1 distributor0)