From cca9d6ca61048fc7d257be83863a667a727d9907 Mon Sep 17 00:00:00 2001 From: Pierre Marieu Date: Wed, 21 Jul 2021 17:45:07 +0100 Subject: [PATCH 1/2] Send agent overhead values as int We are currently sending the `memory_usage_mb` and `timeInMs` fields in the agent overhead metadata, unfortunately the schema for those is INT which means they are currently discarded when the profile is validated at submission time. This does not prevent profiles from being accepted but we lose the overhead data which means we will not show it in the UI. This change converts them to int before we serialize the report. It would be nice for the backend to accept decimals at least for the memory since it is in MB and we may often be under 1MB. But at the moment it is better to align to current schema. --- codeguru_profiler_agent/agent_metadata/agent_metadata.py | 4 ++-- test/acceptance/test_end_to_end_profile_and_save_to_file.py | 4 ++-- test/unit/sdk_reporter/test_sdk_profile_encoder.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/codeguru_profiler_agent/agent_metadata/agent_metadata.py b/codeguru_profiler_agent/agent_metadata/agent_metadata.py index 006e7a7..4832515 100644 --- a/codeguru_profiler_agent/agent_metadata/agent_metadata.py +++ b/codeguru_profiler_agent/agent_metadata/agent_metadata.py @@ -83,7 +83,7 @@ def serialize_to_json(self, sample_weight, duration_ms, cpu_time_seconds, "version": self.agent_info.version }, "agentOverhead": { - "memory_usage_mb": memory_usage_mb + "memory_usage_mb": int(memory_usage_mb) }, "runtimeVersion": self.runtime_version, "cpuTimeInSeconds": cpu_time_seconds, @@ -93,5 +93,5 @@ def serialize_to_json(self, sample_weight, duration_ms, cpu_time_seconds, "numTimesSampled": total_sample_count } if overhead_ms != 0: - self.json_rep["agentOverhead"]["timeInMs"] = overhead_ms + self.json_rep["agentOverhead"]["timeInMs"] = int(overhead_ms) return self.json_rep diff --git a/test/acceptance/test_end_to_end_profile_and_save_to_file.py b/test/acceptance/test_end_to_end_profile_and_save_to_file.py index dde7586..a158e23 100644 --- a/test/acceptance/test_end_to_end_profile_and_save_to_file.py +++ b/test/acceptance/test_end_to_end_profile_and_save_to_file.py @@ -92,11 +92,11 @@ def assert_valid_agent_metadata(agent_metadata): assert agent_metadata["agentOverhead"] assert agent_metadata["durationInMs"] assert agent_metadata["sampleWeights"]["WALL_TIME"] - assert agent_metadata["agentOverhead"]["memory_usage_mb"] + assert type(agent_metadata["agentOverhead"]["memory_usage_mb"]) is int if platform.system() != "Windows": # Due to the issue mentioned on https://bugs.python.org/issue37859, we would skip checking agentOverhead for # Windows system as the agent is only run for very short period of time. We may improve the accuracy of # measuring the overhead by using time.perf_counter_ns for Windows in the future. - assert agent_metadata["agentOverhead"]["timeInMs"] + assert type(agent_metadata["agentOverhead"]["timeInMs"]) is int assert agent_metadata["cpuTimeInSeconds"] > 0 diff --git a/test/unit/sdk_reporter/test_sdk_profile_encoder.py b/test/unit/sdk_reporter/test_sdk_profile_encoder.py index cf6df1b..20a0799 100644 --- a/test/unit/sdk_reporter/test_sdk_profile_encoder.py +++ b/test/unit/sdk_reporter/test_sdk_profile_encoder.py @@ -31,7 +31,7 @@ def example_profile(): [Frame("bottom"), Frame("middle"), Frame("different_top")], [Frame("bottom"), Frame("middle")]], attempted_sample_threads_count=10, seen_threads_count=15)) profile.end = end_time - profile.set_overhead_ms(timedelta(milliseconds=256)) + profile.set_overhead_ms(timedelta(microseconds=256123)) if platform.system() == "Windows": # In Windows, as time.process stays constant if no cpu time was used (https://bugs.python.org/issue37859), we # would need to manually override the cpu_time_seconds to ensure the test runs as expected @@ -122,7 +122,7 @@ def test_it_includes_the_overhead_ms_in_the_agent_metadata(self): assert (self.decoded_json_result()["agentMetadata"]["agentOverhead"]["timeInMs"] == 256) def test_it_includes_the_memory_overhead_in_the_agent_metadata(self): - assert (self.decoded_json_result()["agentMetadata"]["agentOverhead"]["memory_usage_mb"] > 0) + assert (type(self.decoded_json_result()["agentMetadata"]["agentOverhead"]["memory_usage_mb"]) is int) def test_it_includes_the_num_times_sampled_in_the_agent_metadata(self): assert (self.decoded_json_result()["agentMetadata"]["numTimesSampled"] > 0) From b2a3e69611ab161218b5d85d81a8de26987c2988 Mon Sep 17 00:00:00 2001 From: Mirela Popoveniuc Date: Tue, 27 Jul 2021 12:06:57 +0100 Subject: [PATCH 2/2] Fix key for agent metadata based on schema: from 'memory_usage_mb' to 'memoryInMB'. --- codeguru_profiler_agent/agent_metadata/agent_metadata.py | 4 ++-- test/acceptance/test_end_to_end_profile_and_save_to_file.py | 2 +- test/unit/sdk_reporter/test_sdk_profile_encoder.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/codeguru_profiler_agent/agent_metadata/agent_metadata.py b/codeguru_profiler_agent/agent_metadata/agent_metadata.py index 4832515..2e62e90 100644 --- a/codeguru_profiler_agent/agent_metadata/agent_metadata.py +++ b/codeguru_profiler_agent/agent_metadata/agent_metadata.py @@ -69,7 +69,7 @@ def fleet_info(self): def serialize_to_json(self, sample_weight, duration_ms, cpu_time_seconds, average_num_threads, overhead_ms, memory_usage_mb, total_sample_count): """ - This needs to be compliant with agent profile schema. + This needs to be compliant with the AgentMetadata schema that is used on the service side. """ if self.json_rep is None: self.json_rep = { @@ -83,7 +83,7 @@ def serialize_to_json(self, sample_weight, duration_ms, cpu_time_seconds, "version": self.agent_info.version }, "agentOverhead": { - "memory_usage_mb": int(memory_usage_mb) + "memoryInMB": int(memory_usage_mb) }, "runtimeVersion": self.runtime_version, "cpuTimeInSeconds": cpu_time_seconds, diff --git a/test/acceptance/test_end_to_end_profile_and_save_to_file.py b/test/acceptance/test_end_to_end_profile_and_save_to_file.py index a158e23..fec3d90 100644 --- a/test/acceptance/test_end_to_end_profile_and_save_to_file.py +++ b/test/acceptance/test_end_to_end_profile_and_save_to_file.py @@ -92,7 +92,7 @@ def assert_valid_agent_metadata(agent_metadata): assert agent_metadata["agentOverhead"] assert agent_metadata["durationInMs"] assert agent_metadata["sampleWeights"]["WALL_TIME"] - assert type(agent_metadata["agentOverhead"]["memory_usage_mb"]) is int + assert type(agent_metadata["agentOverhead"]["memoryInMB"]) is int if platform.system() != "Windows": # Due to the issue mentioned on https://bugs.python.org/issue37859, we would skip checking agentOverhead for diff --git a/test/unit/sdk_reporter/test_sdk_profile_encoder.py b/test/unit/sdk_reporter/test_sdk_profile_encoder.py index 20a0799..3c12de1 100644 --- a/test/unit/sdk_reporter/test_sdk_profile_encoder.py +++ b/test/unit/sdk_reporter/test_sdk_profile_encoder.py @@ -122,7 +122,7 @@ def test_it_includes_the_overhead_ms_in_the_agent_metadata(self): assert (self.decoded_json_result()["agentMetadata"]["agentOverhead"]["timeInMs"] == 256) def test_it_includes_the_memory_overhead_in_the_agent_metadata(self): - assert (type(self.decoded_json_result()["agentMetadata"]["agentOverhead"]["memory_usage_mb"]) is int) + assert (type(self.decoded_json_result()["agentMetadata"]["agentOverhead"]["memoryInMB"]) is int) def test_it_includes_the_num_times_sampled_in_the_agent_metadata(self): assert (self.decoded_json_result()["agentMetadata"]["numTimesSampled"] > 0)