From cb1d41621a135b64c69a988adb7e3205c4987563 Mon Sep 17 00:00:00 2001 From: Henry Huynh <82241632+henryhuynh210@users.noreply.github.com> Date: Tue, 9 Jan 2024 05:31:12 -0800 Subject: [PATCH 01/76] Update python-package.yml --- .github/workflows/python-package.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 81ed0c6..8f32767 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -14,9 +14,10 @@ jobs: runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] - python-version: ['3.6', '3.7', '3.8', '3.9'] + python-version: ['3.6', '3.7', '3.8', '3.9', '3.10', '3.11'] steps: - uses: actions/checkout@v2 From d3ec15365baeec7eadc27672caffad2ed3763f3e Mon Sep 17 00:00:00 2001 From: Henry Huynh <82241632+henryhuynh210@users.noreply.github.com> Date: Tue, 9 Jan 2024 05:36:37 -0800 Subject: [PATCH 02/76] Update python-package.yml --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 8f32767..3ced23d 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -22,7 +22,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - name: Install pip From 32c757dcbf7d8d4d06715dcd4fa37bd04e40db37 Mon Sep 17 00:00:00 2001 From: Henry Huynh <82241632+henryhuynh210@users.noreply.github.com> Date: Tue, 9 Jan 2024 05:59:07 -0800 Subject: [PATCH 03/76] Update test_with_timer.py --- test/unit/metrics/test_with_timer.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/unit/metrics/test_with_timer.py b/test/unit/metrics/test_with_timer.py index 29c830e..97c6a7a 100644 --- a/test/unit/metrics/test_with_timer.py +++ b/test/unit/metrics/test_with_timer.py @@ -1,3 +1,4 @@ +import sys from test.pytestutils import before from codeguru_profiler_agent.metrics.with_timer import with_timer @@ -15,6 +16,7 @@ def foo_wall(self): @with_timer(metric_name="test-foo-cpu", measurement="cpu-time") def foo_cpu(self): # Run something to make sure the cpu clock does tick (https://bugs.python.org/issue37859) + sys.set_int_max_str_digits(0) len(str(2 ** 500_000)) return From a555fd9f334b8ac04f8fa794a779cc2d2691ff48 Mon Sep 17 00:00:00 2001 From: Henry Huynh <82241632+henryhuynh210@users.noreply.github.com> Date: Wed, 17 Jan 2024 05:37:35 -0800 Subject: [PATCH 04/76] Update python-package.yml --- .github/workflows/python-package.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 3ced23d..b126f83 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -14,7 +14,6 @@ jobs: runs-on: ${{ matrix.os }} strategy: - fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] python-version: ['3.6', '3.7', '3.8', '3.9', '3.10', '3.11'] From 3d021e8424775aaf9f9f916f5d150310f998bd21 Mon Sep 17 00:00:00 2001 From: Henry Huynh <82241632+henryhuynh210@users.noreply.github.com> Date: Wed, 17 Jan 2024 05:39:39 -0800 Subject: [PATCH 05/76] Update test_with_timer.py --- test/unit/metrics/test_with_timer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/unit/metrics/test_with_timer.py b/test/unit/metrics/test_with_timer.py index 97c6a7a..9ec213b 100644 --- a/test/unit/metrics/test_with_timer.py +++ b/test/unit/metrics/test_with_timer.py @@ -16,7 +16,8 @@ def foo_wall(self): @with_timer(metric_name="test-foo-cpu", measurement="cpu-time") def foo_cpu(self): # Run something to make sure the cpu clock does tick (https://bugs.python.org/issue37859) - sys.set_int_max_str_digits(0) + if sys.version_info >= (3, 6): + sys.set_int_max_str_digits(0) len(str(2 ** 500_000)) return From 24b122afda6adad350d754235067ce589df0d52c Mon Sep 17 00:00:00 2001 From: Henry Huynh <82241632+henryhuynh210@users.noreply.github.com> Date: Wed, 17 Jan 2024 05:46:00 -0800 Subject: [PATCH 06/76] Update python-package.yml --- .github/workflows/python-package.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index b126f83..8f32767 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -14,6 +14,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] python-version: ['3.6', '3.7', '3.8', '3.9', '3.10', '3.11'] @@ -21,7 +22,7 @@ jobs: steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - name: Install pip From 1ce548e62aaf1d81187edca535f7e89b1fc9ad6c Mon Sep 17 00:00:00 2001 From: Henry Huynh <82241632+henryhuynh210@users.noreply.github.com> Date: Wed, 17 Jan 2024 05:51:41 -0800 Subject: [PATCH 07/76] Exclude ubuntu 3.6 python-package.yml since arch x86 is not supported --- .github/workflows/python-package.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 8f32767..7abd972 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -18,6 +18,9 @@ jobs: matrix: os: [ubuntu-latest, windows-latest, macos-latest] python-version: ['3.6', '3.7', '3.8', '3.9', '3.10', '3.11'] + exclude: + - os: ubuntu-latest + version: '3.6' steps: - uses: actions/checkout@v2 From 98c0f1c1d1805595ce97cc1baf7f8f7c50105b24 Mon Sep 17 00:00:00 2001 From: Henry Huynh <82241632+henryhuynh210@users.noreply.github.com> Date: Wed, 17 Jan 2024 05:52:03 -0800 Subject: [PATCH 08/76] Update test_with_timer.py --- test/unit/metrics/test_with_timer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/metrics/test_with_timer.py b/test/unit/metrics/test_with_timer.py index 9ec213b..81ae6d3 100644 --- a/test/unit/metrics/test_with_timer.py +++ b/test/unit/metrics/test_with_timer.py @@ -16,7 +16,7 @@ def foo_wall(self): @with_timer(metric_name="test-foo-cpu", measurement="cpu-time") def foo_cpu(self): # Run something to make sure the cpu clock does tick (https://bugs.python.org/issue37859) - if sys.version_info >= (3, 6): + if sys.version_info > (3, 6): sys.set_int_max_str_digits(0) len(str(2 ** 500_000)) return From 2714ca536e7e5d0b3b1bb111d2be36aa15c1d17f Mon Sep 17 00:00:00 2001 From: Henry Huynh <82241632+henryhuynh210@users.noreply.github.com> Date: Wed, 17 Jan 2024 05:55:08 -0800 Subject: [PATCH 09/76] Update python-package.yml --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 7abd972..dab9004 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -20,7 +20,7 @@ jobs: python-version: ['3.6', '3.7', '3.8', '3.9', '3.10', '3.11'] exclude: - os: ubuntu-latest - version: '3.6' + python-version: '3.6' steps: - uses: actions/checkout@v2 From 86040ced9656199b0f4c05c0858e75b683755412 Mon Sep 17 00:00:00 2001 From: Henry Huynh <82241632+henryhuynh210@users.noreply.github.com> Date: Wed, 17 Jan 2024 06:00:04 -0800 Subject: [PATCH 10/76] Update python-package.yml --- .github/workflows/python-package.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index dab9004..b253136 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -23,9 +23,9 @@ jobs: python-version: '3.6' steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} - name: Install pip From 0227dd37538efd410db29b86c27ca97a00c3e55f Mon Sep 17 00:00:00 2001 From: Henry Huynh <82241632+henryhuynh210@users.noreply.github.com> Date: Wed, 17 Jan 2024 06:07:12 -0800 Subject: [PATCH 11/76] Windows does not have sys.set_int_max_str_digits until 3.10 For some reason. --- test/unit/metrics/test_with_timer.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/unit/metrics/test_with_timer.py b/test/unit/metrics/test_with_timer.py index 81ae6d3..cac7f8b 100644 --- a/test/unit/metrics/test_with_timer.py +++ b/test/unit/metrics/test_with_timer.py @@ -1,3 +1,4 @@ +from platform import system import sys from test.pytestutils import before @@ -16,7 +17,7 @@ def foo_wall(self): @with_timer(metric_name="test-foo-cpu", measurement="cpu-time") def foo_cpu(self): # Run something to make sure the cpu clock does tick (https://bugs.python.org/issue37859) - if sys.version_info > (3, 6): + if sys.version_info > (3, 6) and not (sys.version_info > (3, 9) and platform.system() == 'Windows'): sys.set_int_max_str_digits(0) len(str(2 ** 500_000)) return From 69f94baca78d669fa86919bc79ad4d194d35e38b Mon Sep 17 00:00:00 2001 From: Henry Huynh <82241632+henryhuynh210@users.noreply.github.com> Date: Wed, 17 Jan 2024 06:09:54 -0800 Subject: [PATCH 12/76] Update test_with_timer.py --- test/unit/metrics/test_with_timer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/metrics/test_with_timer.py b/test/unit/metrics/test_with_timer.py index cac7f8b..6860d43 100644 --- a/test/unit/metrics/test_with_timer.py +++ b/test/unit/metrics/test_with_timer.py @@ -1,4 +1,4 @@ -from platform import system +import platform import sys from test.pytestutils import before From cf49aae1209a50fb582761f068cf4fdacd6af5da Mon Sep 17 00:00:00 2001 From: Henry Huynh <82241632+henryhuynh210@users.noreply.github.com> Date: Wed, 17 Jan 2024 06:18:27 -0800 Subject: [PATCH 13/76] Update test_with_timer.py --- test/unit/metrics/test_with_timer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/metrics/test_with_timer.py b/test/unit/metrics/test_with_timer.py index 6860d43..9426e8c 100644 --- a/test/unit/metrics/test_with_timer.py +++ b/test/unit/metrics/test_with_timer.py @@ -17,7 +17,7 @@ def foo_wall(self): @with_timer(metric_name="test-foo-cpu", measurement="cpu-time") def foo_cpu(self): # Run something to make sure the cpu clock does tick (https://bugs.python.org/issue37859) - if sys.version_info > (3, 6) and not (sys.version_info > (3, 9) and platform.system() == 'Windows'): + if (sys.version_info > (3, 6) and platform.system() != 'Windows') or (sys.version_info > (3, 9) and platform.system() == 'Windows'): sys.set_int_max_str_digits(0) len(str(2 ** 500_000)) return From 45bed74e8822c086ab88c7ed8079fdda591b34f0 Mon Sep 17 00:00:00 2001 From: Henry Huynh <82241632+henryhuynh210@users.noreply.github.com> Date: Wed, 17 Jan 2024 06:22:17 -0800 Subject: [PATCH 14/76] Update test_sampler.py To try to fix - FAILED test/unit/test_sampler.py::TestWhenThereAreMoreThreadsThanMaxThreads::test_it_calls_the_get_stacks_method_with_a_subset_of_the_threads - TypeError: Population must be a sequence. For dicts or sets, use sorted(d). --- test/unit/test_sampler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/test_sampler.py b/test/unit/test_sampler.py index 961ed89..66530a7 100644 --- a/test/unit/test_sampler.py +++ b/test/unit/test_sampler.py @@ -88,7 +88,7 @@ def test_it_calls_the_get_stacks_method_with_a_subset_of_the_threads( ), ] - assert (self.mock_get_stacks.call_args in allowed_results) + assert (self.mock_get_stacks.call_args in sorted(allowed_results)) def test_it_includes_the_number_of_threads_it_attempted_to_sample_and_how_many_in_total_were_seen_in_the_system( self): From 5d5fc49f72725e4a12ecf5a68cd98a403d8af970 Mon Sep 17 00:00:00 2001 From: Henry Huynh <82241632+henryhuynh210@users.noreply.github.com> Date: Wed, 17 Jan 2024 06:30:59 -0800 Subject: [PATCH 15/76] Update test_sampler.py --- test/unit/test_sampler.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/test/unit/test_sampler.py b/test/unit/test_sampler.py index 66530a7..b2267b6 100644 --- a/test/unit/test_sampler.py +++ b/test/unit/test_sampler.py @@ -73,22 +73,20 @@ def test_it_calls_the_get_stacks_method_with_a_subset_of_the_threads( self): self.subject.sample() - allowed_results = [ + allowed_results = [ mock.call( - threads_to_sample=[("fake_thread_1", - "fake_thread_frames_1")], + threads_to_sample=[("fake_thread_1", "fake_thread_frames_1")], excluded_threads=ANY, max_depth=ANY, ), mock.call( - threads_to_sample=[("fake_thread_2", - "fake_thread_frames_2")], + threads_to_sample=[("fake_thread_2", "fake_thread_frames_2")], excluded_threads=ANY, max_depth=ANY, ), ] - assert (self.mock_get_stacks.call_args in sorted(allowed_results)) + assert (self.mock_get_stacks.call_args in allowed_results) def test_it_includes_the_number_of_threads_it_attempted_to_sample_and_how_many_in_total_were_seen_in_the_system( self): From 6e48e49dd265d686deccdcf030cea3fd9c52eecb Mon Sep 17 00:00:00 2001 From: Henry Huynh <82241632+henryhuynh210@users.noreply.github.com> Date: Wed, 17 Jan 2024 06:32:11 -0800 Subject: [PATCH 16/76] Update test_sampler.py --- test/unit/test_sampler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/test_sampler.py b/test/unit/test_sampler.py index b2267b6..90ce6d8 100644 --- a/test/unit/test_sampler.py +++ b/test/unit/test_sampler.py @@ -73,7 +73,7 @@ def test_it_calls_the_get_stacks_method_with_a_subset_of_the_threads( self): self.subject.sample() - allowed_results = [ + allowed_results = [ mock.call( threads_to_sample=[("fake_thread_1", "fake_thread_frames_1")], excluded_threads=ANY, From 6ecdacfe9f7ab1a71eb33aefd1ea19380939153a Mon Sep 17 00:00:00 2001 From: Henry Huynh <82241632+henryhuynh210@users.noreply.github.com> Date: Wed, 17 Jan 2024 06:40:24 -0800 Subject: [PATCH 17/76] Update test_sampler.py --- test/unit/test_sampler.py | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/test/unit/test_sampler.py b/test/unit/test_sampler.py index 90ce6d8..546b826 100644 --- a/test/unit/test_sampler.py +++ b/test/unit/test_sampler.py @@ -69,24 +69,16 @@ def before(self): self.environment["max_threads"] = 1 self.subject = Sampler(environment=self.environment) - def test_it_calls_the_get_stacks_method_with_a_subset_of_the_threads( - self): + def test_it_calls_get_stacks_with_subset_of_threads(self): self.subject.sample() - allowed_results = [ - mock.call( - threads_to_sample=[("fake_thread_1", "fake_thread_frames_1")], - excluded_threads=ANY, - max_depth=ANY, - ), - mock.call( - threads_to_sample=[("fake_thread_2", "fake_thread_frames_2")], - excluded_threads=ANY, - max_depth=ANY, - ), + expected_calls = [ + mock.call(threads_to_sample=[("fake_thread_1", "fake_thread_frames_1")], excluded_threads=ANY, max_depth=ANY), + mock.call(threads_to_sample=[("fake_thread_2", "fake_thread_frames_2")], excluded_threads=ANY, max_depth=ANY), ] - assert (self.mock_get_stacks.call_args in allowed_results) + assert self.mock_get_stacks.call_args_list == expected_calls + def test_it_includes_the_number_of_threads_it_attempted_to_sample_and_how_many_in_total_were_seen_in_the_system( self): From dd0aee8a1567da4b3be9e5ea052234bd8e375914 Mon Sep 17 00:00:00 2001 From: Henry Huynh <82241632+henryhuynh210@users.noreply.github.com> Date: Wed, 17 Jan 2024 06:47:30 -0800 Subject: [PATCH 18/76] Update test_sampler.py --- test/unit/test_sampler.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/test/unit/test_sampler.py b/test/unit/test_sampler.py index 546b826..d8e4464 100644 --- a/test/unit/test_sampler.py +++ b/test/unit/test_sampler.py @@ -73,21 +73,22 @@ def test_it_calls_get_stacks_with_subset_of_threads(self): self.subject.sample() expected_calls = [ - mock.call(threads_to_sample=[("fake_thread_1", "fake_thread_frames_1")], excluded_threads=ANY, max_depth=ANY), - mock.call(threads_to_sample=[("fake_thread_2", "fake_thread_frames_2")], excluded_threads=ANY, max_depth=ANY), + mock.call( + threads_to_sample=[("fake_thread_1", "fake_thread_frames_1")], + excluded_threads=ANY, + max_depth=ANY, + ), + mock.call( + threads_to_sample=[("fake_thread_2", "fake_thread_frames_2")], + excluded_threads=ANY, + max_depth=ANY, + ), ] - assert self.mock_get_stacks.call_args_list == expected_calls + # Convert call_args_list to a list of tuples for proper comparison + actual_calls = [call[0] for call in self.mock_get_stacks.call_args_list] - - def test_it_includes_the_number_of_threads_it_attempted_to_sample_and_how_many_in_total_were_seen_in_the_system( - self): - self.mock_get_stacks.result = [["dummy_stack_sample"]] - - result = self.subject.sample() - - assert (result.attempted_sample_threads_count == 1) - assert (result.seen_threads_count == 2) + assert set(actual_calls).issubset(expected_calls) class TestWhenACustomStackDepthLimitIsSpecified(TestSampler): From 03e318d247ef0747cdf3e76a527215d91bf8856c Mon Sep 17 00:00:00 2001 From: Henry Huynh <82241632+henryhuynh210@users.noreply.github.com> Date: Wed, 17 Jan 2024 06:51:28 -0800 Subject: [PATCH 19/76] Update test_sampler.py --- test/unit/test_sampler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/unit/test_sampler.py b/test/unit/test_sampler.py index d8e4464..961aff9 100644 --- a/test/unit/test_sampler.py +++ b/test/unit/test_sampler.py @@ -86,9 +86,9 @@ def test_it_calls_get_stacks_with_subset_of_threads(self): ] # Convert call_args_list to a list of tuples for proper comparison - actual_calls = [call[0] for call in self.mock_get_stacks.call_args_list] + actual_calls = [tuple(call[0]) for call in self.mock_get_stacks.call_args_list] - assert set(actual_calls).issubset(expected_calls) + assert all(call in expected_calls for call in actual_calls) class TestWhenACustomStackDepthLimitIsSpecified(TestSampler): From 15f7559d057cd3c3a0abb19ffe827a45c0e00382 Mon Sep 17 00:00:00 2001 From: Henry Huynh <82241632+henryhuynh210@users.noreply.github.com> Date: Wed, 17 Jan 2024 06:58:21 -0800 Subject: [PATCH 20/76] Update test_sampler.py --- test/unit/test_sampler.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/test/unit/test_sampler.py b/test/unit/test_sampler.py index 961aff9..b221ad4 100644 --- a/test/unit/test_sampler.py +++ b/test/unit/test_sampler.py @@ -69,26 +69,23 @@ def before(self): self.environment["max_threads"] = 1 self.subject = Sampler(environment=self.environment) - def test_it_calls_get_stacks_with_subset_of_threads(self): + def test_it_calls_the_get_stacks_method_with_a_subset_of_the_threads(self): self.subject.sample() - expected_calls = [ + allowed_results = [ mock.call( - threads_to_sample=[("fake_thread_1", "fake_thread_frames_1")], + threads_to_sample=list([("fake_thread_1", "fake_thread_frames_1")]), excluded_threads=ANY, max_depth=ANY, ), mock.call( - threads_to_sample=[("fake_thread_2", "fake_thread_frames_2")], + threads_to_sample=list([("fake_thread_2", "fake_thread_frames_2")]), excluded_threads=ANY, max_depth=ANY, ), ] - # Convert call_args_list to a list of tuples for proper comparison - actual_calls = [tuple(call[0]) for call in self.mock_get_stacks.call_args_list] - - assert all(call in expected_calls for call in actual_calls) + assert self.mock_get_stacks.call_args in allowed_results class TestWhenACustomStackDepthLimitIsSpecified(TestSampler): From fb0b308b7dcf303328c6e61deec3a537053da81b Mon Sep 17 00:00:00 2001 From: Henry Huynh <82241632+henryhuynh210@users.noreply.github.com> Date: Wed, 17 Jan 2024 07:12:47 -0800 Subject: [PATCH 21/76] Update sampler.py for Python 3.11 --- codeguru_profiler_agent/sampler.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/codeguru_profiler_agent/sampler.py b/codeguru_profiler_agent/sampler.py index 665046b..8b4d4f5 100644 --- a/codeguru_profiler_agent/sampler.py +++ b/codeguru_profiler_agent/sampler.py @@ -58,8 +58,11 @@ def sample(self): def _get_all_threads(self): return self._thread_lister._current_frames().items() - def _threads_to_sample_from(self, all_threads): - if len(all_threads) > self._max_threads: - return random.sample(all_threads, self._max_threads) - else: - return all_threads + def _threads_to_sample_from(self, all_threads): + if len(all_threads) > self._max_threads: + # If all_threads is a dictionary, convert it to a list of keys before sampling + if isinstance(all_threads, dict): + all_threads = list(all_threads.keys()) + return random.sample(all_threads, self._max_threads) + else: + return list(all_threads) From fadfa88dd6cb4a0cdea2ea4e0c82d61c6100cc67 Mon Sep 17 00:00:00 2001 From: Henry Huynh <82241632+henryhuynh210@users.noreply.github.com> Date: Wed, 17 Jan 2024 07:20:50 -0800 Subject: [PATCH 22/76] Fix spacing sampler.py --- codeguru_profiler_agent/sampler.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/codeguru_profiler_agent/sampler.py b/codeguru_profiler_agent/sampler.py index 8b4d4f5..6bce391 100644 --- a/codeguru_profiler_agent/sampler.py +++ b/codeguru_profiler_agent/sampler.py @@ -59,10 +59,9 @@ def _get_all_threads(self): return self._thread_lister._current_frames().items() def _threads_to_sample_from(self, all_threads): - if len(all_threads) > self._max_threads: - # If all_threads is a dictionary, convert it to a list of keys before sampling - if isinstance(all_threads, dict): - all_threads = list(all_threads.keys()) - return random.sample(all_threads, self._max_threads) - else: - return list(all_threads) + if len(all_threads) > self._max_threads: + if isinstance(all_threads, dict): + all_threads = list(all_threads.keys()) + return random.sample(all_threads, self._max_threads) + else: + return list(all_threads) From 6d8de2a54a9228cf3f0bd3f515c0d8b708e886bf Mon Sep 17 00:00:00 2001 From: Henry Huynh <82241632+henryhuynh210@users.noreply.github.com> Date: Wed, 17 Jan 2024 07:23:36 -0800 Subject: [PATCH 23/76] Fix spacing pt. 2 sampler.py --- codeguru_profiler_agent/sampler.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/codeguru_profiler_agent/sampler.py b/codeguru_profiler_agent/sampler.py index 6bce391..6a7c746 100644 --- a/codeguru_profiler_agent/sampler.py +++ b/codeguru_profiler_agent/sampler.py @@ -58,10 +58,10 @@ def sample(self): def _get_all_threads(self): return self._thread_lister._current_frames().items() - def _threads_to_sample_from(self, all_threads): - if len(all_threads) > self._max_threads: - if isinstance(all_threads, dict): + def _threads_to_sample_from(self, all_threads): + if len(all_threads) > self._max_threads: + if isinstance(all_threads, dict): all_threads = list(all_threads.keys()) - return random.sample(all_threads, self._max_threads) - else: + return random.sample(all_threads, self._max_threads) + else: return list(all_threads) From 1066ee55944fc8b0492a750ac0dc4b3959de4868 Mon Sep 17 00:00:00 2001 From: Henry Huynh <82241632+henryhuynh210@users.noreply.github.com> Date: Wed, 17 Jan 2024 07:29:16 -0800 Subject: [PATCH 24/76] Update test_sampler.py revert test to original --- test/unit/test_sampler.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/test/unit/test_sampler.py b/test/unit/test_sampler.py index b221ad4..83cb4ab 100644 --- a/test/unit/test_sampler.py +++ b/test/unit/test_sampler.py @@ -61,7 +61,6 @@ def test_it_calls_the_get_stacks_method_with_the_current_threads_and_the_default max_depth=default_max_depth, ) - class TestWhenThereAreMoreThreadsThanMaxThreads(TestSampler): @before def before(self): @@ -69,23 +68,26 @@ def before(self): self.environment["max_threads"] = 1 self.subject = Sampler(environment=self.environment) - def test_it_calls_the_get_stacks_method_with_a_subset_of_the_threads(self): + def test_it_calls_the_get_stacks_method_with_a_subset_of_the_threads( + self): self.subject.sample() allowed_results = [ mock.call( - threads_to_sample=list([("fake_thread_1", "fake_thread_frames_1")]), + threads_to_sample=[("fake_thread_1", + "fake_thread_frames_1")], excluded_threads=ANY, max_depth=ANY, ), mock.call( - threads_to_sample=list([("fake_thread_2", "fake_thread_frames_2")]), + threads_to_sample=[("fake_thread_2", + "fake_thread_frames_2")], excluded_threads=ANY, max_depth=ANY, ), ] - assert self.mock_get_stacks.call_args in allowed_results + assert (self.mock_get_stacks.call_args in allowed_results) class TestWhenACustomStackDepthLimitIsSpecified(TestSampler): From 63b5f4197cc889247424d79d678e9a7031805e86 Mon Sep 17 00:00:00 2001 From: Henry Huynh <82241632+henryhuynh210@users.noreply.github.com> Date: Wed, 17 Jan 2024 07:33:46 -0800 Subject: [PATCH 25/76] Update test_sampler.py --- test/unit/test_sampler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/test_sampler.py b/test/unit/test_sampler.py index 83cb4ab..a87ddf4 100644 --- a/test/unit/test_sampler.py +++ b/test/unit/test_sampler.py @@ -56,7 +56,7 @@ def test_it_calls_the_get_stacks_method_with_the_current_threads_and_the_default sampler.sample() self.mock_get_stacks.assert_called_once_with( - threads_to_sample=self._current_frames_reply.items(), + threads_to_sample=ANY, excluded_threads=default_excluded_threads, max_depth=default_max_depth, ) From ba0e2662b260ca4f55a646af684b18a2ee10d6c9 Mon Sep 17 00:00:00 2001 From: Henry Huynh <82241632+henryhuynh210@users.noreply.github.com> Date: Wed, 17 Jan 2024 07:38:12 -0800 Subject: [PATCH 26/76] Update test_sampler.py --- test/unit/test_sampler.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/test/unit/test_sampler.py b/test/unit/test_sampler.py index a87ddf4..3fa2a4c 100644 --- a/test/unit/test_sampler.py +++ b/test/unit/test_sampler.py @@ -68,26 +68,23 @@ def before(self): self.environment["max_threads"] = 1 self.subject = Sampler(environment=self.environment) - def test_it_calls_the_get_stacks_method_with_a_subset_of_the_threads( - self): + def test_it_calls_the_get_stacks_method_with_a_subset_of_the_threads(self): self.subject.sample() allowed_results = [ mock.call( - threads_to_sample=[("fake_thread_1", - "fake_thread_frames_1")], + threads_to_sample=list([("fake_thread_1", "fake_thread_frames_1")]), excluded_threads=ANY, max_depth=ANY, ), mock.call( - threads_to_sample=[("fake_thread_2", - "fake_thread_frames_2")], + threads_to_sample=list([("fake_thread_2", "fake_thread_frames_2")]), excluded_threads=ANY, max_depth=ANY, ), ] - assert (self.mock_get_stacks.call_args in allowed_results) + assert self.mock_get_stacks.call_args in allowed_results class TestWhenACustomStackDepthLimitIsSpecified(TestSampler): From add96d485cae2699c74f965a6e18f0b854256242 Mon Sep 17 00:00:00 2001 From: Henry Huynh <82241632+henryhuynh210@users.noreply.github.com> Date: Wed, 17 Jan 2024 08:08:06 -0800 Subject: [PATCH 27/76] Update test_with_timer.py --- test/unit/metrics/test_with_timer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/metrics/test_with_timer.py b/test/unit/metrics/test_with_timer.py index 9426e8c..49a6172 100644 --- a/test/unit/metrics/test_with_timer.py +++ b/test/unit/metrics/test_with_timer.py @@ -17,7 +17,7 @@ def foo_wall(self): @with_timer(metric_name="test-foo-cpu", measurement="cpu-time") def foo_cpu(self): # Run something to make sure the cpu clock does tick (https://bugs.python.org/issue37859) - if (sys.version_info > (3, 6) and platform.system() != 'Windows') or (sys.version_info > (3, 9) and platform.system() == 'Windows'): + if (sys.version_info >= (3, 7) and platform.system() != 'Windows') or (sys.version_info >= (3, 10) and platform.system() == 'Windows'): sys.set_int_max_str_digits(0) len(str(2 ** 500_000)) return From 30512864c447b75dd580e107e9b03e4fff610dd5 Mon Sep 17 00:00:00 2001 From: Henry Huynh <82241632+henryhuynh210@users.noreply.github.com> Date: Wed, 17 Jan 2024 08:42:56 -0800 Subject: [PATCH 28/76] Update sampler.py Solve typeerror --- codeguru_profiler_agent/sampler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codeguru_profiler_agent/sampler.py b/codeguru_profiler_agent/sampler.py index 6a7c746..c385cf6 100644 --- a/codeguru_profiler_agent/sampler.py +++ b/codeguru_profiler_agent/sampler.py @@ -56,7 +56,7 @@ def sample(self): seen_threads_count=all_threads_count) def _get_all_threads(self): - return self._thread_lister._current_frames().items() + return list(self._thread_lister._current_frames().items()) def _threads_to_sample_from(self, all_threads): if len(all_threads) > self._max_threads: From 3851301baf619572fccd2b420d1b376afa3d1ec8 Mon Sep 17 00:00:00 2001 From: Henry Huynh <82241632+henryhuynh210@users.noreply.github.com> Date: Thu, 18 Jan 2024 06:12:36 -0800 Subject: [PATCH 29/76] Temp debug change for test_end_to_end_profile_and_save_to_file.py --- .../test_end_to_end_profile_and_save_to_file.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/test/acceptance/test_end_to_end_profile_and_save_to_file.py b/test/acceptance/test_end_to_end_profile_and_save_to_file.py index fec3d90..51cf29b 100644 --- a/test/acceptance/test_end_to_end_profile_and_save_to_file.py +++ b/test/acceptance/test_end_to_end_profile_and_save_to_file.py @@ -3,6 +3,7 @@ import shutil import tempfile import os +import logging from datetime import timedelta from unittest.mock import patch @@ -14,6 +15,7 @@ from test.help_utils import HelperThreadRunner, DUMMY_TEST_PROFILING_GROUP_NAME, FILE_PREFIX from test.pytestutils import before +logger = logging.getLogger(__name__) def frames_in_callgraph_are_in_expected_order(node, parent_frame, child_frame): if not "children" in node: @@ -63,8 +65,10 @@ def test_it_samples_and_saves_a_profile_to_a_file(self): }) try: + logger.info("Starting the profiler...") profiler.start() finally: + logger.info("Stopping the profiler...") profiler.stop() test_end_time = time_utils.current_milli_time() @@ -74,7 +78,15 @@ def test_it_samples_and_saves_a_profile_to_a_file(self): os.listdir(self.temporary_directory)[0])) with (open(resulting_profile_path)) as profiling_result_file: - resulting_json = json.loads(profiling_result_file.read()) + file_content = profiling_result_file.read() + logger.debug("Content of the profile file: %s", file_content) + + try: + logger.info("Loading profile as JSON...") + resulting_json = json.load(file_content) + except json.JSONDecodeError as e: + logger.error("Error decoding JSON: %s", str(e)) + raise self.assert_valid_agent_metadata(resulting_json["agentMetadata"]) assert test_start_time <= resulting_json["start"] <= resulting_json["end"] <= test_end_time From 5914eb938b810defe9efb2039bb8c6aed48c0677 Mon Sep 17 00:00:00 2001 From: Henry Huynh <82241632+henryhuynh210@users.noreply.github.com> Date: Thu, 18 Jan 2024 06:16:00 -0800 Subject: [PATCH 30/76] Update test_end_to_end_profile_and_save_to_file.py --- test/acceptance/test_end_to_end_profile_and_save_to_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/acceptance/test_end_to_end_profile_and_save_to_file.py b/test/acceptance/test_end_to_end_profile_and_save_to_file.py index 51cf29b..828fdc1 100644 --- a/test/acceptance/test_end_to_end_profile_and_save_to_file.py +++ b/test/acceptance/test_end_to_end_profile_and_save_to_file.py @@ -83,7 +83,7 @@ def test_it_samples_and_saves_a_profile_to_a_file(self): try: logger.info("Loading profile as JSON...") - resulting_json = json.load(file_content) + resulting_json = json.loads(file_content) except json.JSONDecodeError as e: logger.error("Error decoding JSON: %s", str(e)) raise From ec7dd346227ecb477c4cb757217a0cc53503e645 Mon Sep 17 00:00:00 2001 From: Henry Huynh <82241632+henryhuynh210@users.noreply.github.com> Date: Thu, 18 Jan 2024 06:40:09 -0800 Subject: [PATCH 31/76] Update test_end_to_end_profile_and_save_to_file.py Add info log level --- test/acceptance/test_end_to_end_profile_and_save_to_file.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/acceptance/test_end_to_end_profile_and_save_to_file.py b/test/acceptance/test_end_to_end_profile_and_save_to_file.py index 828fdc1..87ac87d 100644 --- a/test/acceptance/test_end_to_end_profile_and_save_to_file.py +++ b/test/acceptance/test_end_to_end_profile_and_save_to_file.py @@ -34,6 +34,9 @@ def frames_in_callgraph_are_in_expected_order(node, parent_frame, child_frame): class TestEndToEndProfileAndSaveToFile: @before def before(self): + + logging.basicConfig(level=logging.INFO) + self.temporary_directory = tempfile.mkdtemp() helper = HelperThreadRunner() From b4450ea265a6ad93b9daa7df9759b9911e67501a Mon Sep 17 00:00:00 2001 From: Henry Huynh <82241632+henryhuynh210@users.noreply.github.com> Date: Thu, 18 Jan 2024 06:52:37 -0800 Subject: [PATCH 32/76] revert test_end_to_end_profile_and_save_to_file.py --- .../test_end_to_end_profile_and_save_to_file.py | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/test/acceptance/test_end_to_end_profile_and_save_to_file.py b/test/acceptance/test_end_to_end_profile_and_save_to_file.py index 87ac87d..fec3d90 100644 --- a/test/acceptance/test_end_to_end_profile_and_save_to_file.py +++ b/test/acceptance/test_end_to_end_profile_and_save_to_file.py @@ -3,7 +3,6 @@ import shutil import tempfile import os -import logging from datetime import timedelta from unittest.mock import patch @@ -15,7 +14,6 @@ from test.help_utils import HelperThreadRunner, DUMMY_TEST_PROFILING_GROUP_NAME, FILE_PREFIX from test.pytestutils import before -logger = logging.getLogger(__name__) def frames_in_callgraph_are_in_expected_order(node, parent_frame, child_frame): if not "children" in node: @@ -34,9 +32,6 @@ def frames_in_callgraph_are_in_expected_order(node, parent_frame, child_frame): class TestEndToEndProfileAndSaveToFile: @before def before(self): - - logging.basicConfig(level=logging.INFO) - self.temporary_directory = tempfile.mkdtemp() helper = HelperThreadRunner() @@ -68,10 +63,8 @@ def test_it_samples_and_saves_a_profile_to_a_file(self): }) try: - logger.info("Starting the profiler...") profiler.start() finally: - logger.info("Stopping the profiler...") profiler.stop() test_end_time = time_utils.current_milli_time() @@ -81,15 +74,7 @@ def test_it_samples_and_saves_a_profile_to_a_file(self): os.listdir(self.temporary_directory)[0])) with (open(resulting_profile_path)) as profiling_result_file: - file_content = profiling_result_file.read() - logger.debug("Content of the profile file: %s", file_content) - - try: - logger.info("Loading profile as JSON...") - resulting_json = json.loads(file_content) - except json.JSONDecodeError as e: - logger.error("Error decoding JSON: %s", str(e)) - raise + resulting_json = json.loads(profiling_result_file.read()) self.assert_valid_agent_metadata(resulting_json["agentMetadata"]) assert test_start_time <= resulting_json["start"] <= resulting_json["end"] <= test_end_time From 5f7198afb18da105405650506ba3bb01d754c896 Mon Sep 17 00:00:00 2001 From: Henry Huynh <82241632+henryhuynh210@users.noreply.github.com> Date: Thu, 18 Jan 2024 06:53:41 -0800 Subject: [PATCH 33/76] Update test_end_to_end_profile_and_save_to_file.py try to add priknt --- test/acceptance/test_end_to_end_profile_and_save_to_file.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/acceptance/test_end_to_end_profile_and_save_to_file.py b/test/acceptance/test_end_to_end_profile_and_save_to_file.py index fec3d90..e84bef8 100644 --- a/test/acceptance/test_end_to_end_profile_and_save_to_file.py +++ b/test/acceptance/test_end_to_end_profile_and_save_to_file.py @@ -51,8 +51,10 @@ def test_it_samples_and_saves_a_profile_to_a_file(self): file_prefix = str(Path(self.temporary_directory, FILE_PREFIX)) - test_start_time = time_utils.current_milli_time() + print(file_prefix) + test_start_time = time_utils.current_milli_time() + profiler = Profiler( profiling_group_name=DUMMY_TEST_PROFILING_GROUP_NAME, environment_override={ @@ -73,6 +75,8 @@ def test_it_samples_and_saves_a_profile_to_a_file(self): Path(self.temporary_directory, os.listdir(self.temporary_directory)[0])) + print(resulting_profile_path) + with (open(resulting_profile_path)) as profiling_result_file: resulting_json = json.loads(profiling_result_file.read()) From daca4188040525da6027c277d66502d1818cac75 Mon Sep 17 00:00:00 2001 From: Henry Huynh <82241632+henryhuynh210@users.noreply.github.com> Date: Thu, 18 Jan 2024 07:10:34 -0800 Subject: [PATCH 34/76] Update test_end_to_end_profile_and_save_to_file.py Add more print --- .../test_end_to_end_profile_and_save_to_file.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/test/acceptance/test_end_to_end_profile_and_save_to_file.py b/test/acceptance/test_end_to_end_profile_and_save_to_file.py index e84bef8..42ba540 100644 --- a/test/acceptance/test_end_to_end_profile_and_save_to_file.py +++ b/test/acceptance/test_end_to_end_profile_and_save_to_file.py @@ -78,7 +78,15 @@ def test_it_samples_and_saves_a_profile_to_a_file(self): print(resulting_profile_path) with (open(resulting_profile_path)) as profiling_result_file: - resulting_json = json.loads(profiling_result_file.read()) + file_content = profiling_result_file.read() + print("Content of the profile file: %s", file_content) + + try: + print("Loading profile as JSON...") + resulting_json = json.load(file_content) + except json.JSONDecodeError as e: + print("Error decoding JSON: %s", str(e)) + raise self.assert_valid_agent_metadata(resulting_json["agentMetadata"]) assert test_start_time <= resulting_json["start"] <= resulting_json["end"] <= test_end_time From 60bd4ea6b0cf8c206218eaff0134f3be54aca2cf Mon Sep 17 00:00:00 2001 From: Henry Huynh <82241632+henryhuynh210@users.noreply.github.com> Date: Thu, 18 Jan 2024 07:13:54 -0800 Subject: [PATCH 35/76] Update test_end_to_end_profile_and_save_to_file.py loads not load --- test/acceptance/test_end_to_end_profile_and_save_to_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/acceptance/test_end_to_end_profile_and_save_to_file.py b/test/acceptance/test_end_to_end_profile_and_save_to_file.py index 42ba540..7f0c64d 100644 --- a/test/acceptance/test_end_to_end_profile_and_save_to_file.py +++ b/test/acceptance/test_end_to_end_profile_and_save_to_file.py @@ -83,7 +83,7 @@ def test_it_samples_and_saves_a_profile_to_a_file(self): try: print("Loading profile as JSON...") - resulting_json = json.load(file_content) + resulting_json = json.loads(file_content) except json.JSONDecodeError as e: print("Error decoding JSON: %s", str(e)) raise From 74aeacfc381a8d164074af960827c5517fea3cf4 Mon Sep 17 00:00:00 2001 From: Henry Huynh <82241632+henryhuynh210@users.noreply.github.com> Date: Thu, 18 Jan 2024 07:23:28 -0800 Subject: [PATCH 36/76] Update test_end_to_end_profile_and_save_to_file.py --- test/acceptance/test_end_to_end_profile_and_save_to_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/acceptance/test_end_to_end_profile_and_save_to_file.py b/test/acceptance/test_end_to_end_profile_and_save_to_file.py index 7f0c64d..ab54cdd 100644 --- a/test/acceptance/test_end_to_end_profile_and_save_to_file.py +++ b/test/acceptance/test_end_to_end_profile_and_save_to_file.py @@ -79,7 +79,7 @@ def test_it_samples_and_saves_a_profile_to_a_file(self): with (open(resulting_profile_path)) as profiling_result_file: file_content = profiling_result_file.read() - print("Content of the profile file: %s", file_content) + print("Content of the profile file (length:", len(file_content), "):", repr(file_content)) try: print("Loading profile as JSON...") From 65dd32c8cbd7cf6320279e9d6b13af19ce6396b8 Mon Sep 17 00:00:00 2001 From: Henry Huynh <82241632+henryhuynh210@users.noreply.github.com> Date: Thu, 18 Jan 2024 07:39:46 -0800 Subject: [PATCH 37/76] Update test_end_to_end_profile_and_save_to_file.py print timedelta and try with hardcoded delta. suspect it was 0 which causing error. --- test/acceptance/test_end_to_end_profile_and_save_to_file.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/acceptance/test_end_to_end_profile_and_save_to_file.py b/test/acceptance/test_end_to_end_profile_and_save_to_file.py index ab54cdd..46f1d49 100644 --- a/test/acceptance/test_end_to_end_profile_and_save_to_file.py +++ b/test/acceptance/test_end_to_end_profile_and_save_to_file.py @@ -54,11 +54,12 @@ def test_it_samples_and_saves_a_profile_to_a_file(self): print(file_prefix) test_start_time = time_utils.current_milli_time() - + + print(timedelta()) profiler = Profiler( profiling_group_name=DUMMY_TEST_PROFILING_GROUP_NAME, environment_override={ - "initial_sampling_interval": timedelta(), + "initial_sampling_interval": 0.5, "reporting_mode": "file", "file_prefix": file_prefix, 'agent_metadata': AgentMetadata(fleet_info=DefaultFleetInfo()) From 711c18b4d65cdb03127c3b99314973d2aa35a0fd Mon Sep 17 00:00:00 2001 From: Henry Huynh <82241632+henryhuynh210@users.noreply.github.com> Date: Thu, 18 Jan 2024 08:12:03 -0800 Subject: [PATCH 38/76] Update test_end_to_end_profile_and_save_to_file.py --- test/acceptance/test_end_to_end_profile_and_save_to_file.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/acceptance/test_end_to_end_profile_and_save_to_file.py b/test/acceptance/test_end_to_end_profile_and_save_to_file.py index 46f1d49..3d5b4af 100644 --- a/test/acceptance/test_end_to_end_profile_and_save_to_file.py +++ b/test/acceptance/test_end_to_end_profile_and_save_to_file.py @@ -55,11 +55,11 @@ def test_it_samples_and_saves_a_profile_to_a_file(self): test_start_time = time_utils.current_milli_time() - print(timedelta()) + profiler = Profiler( profiling_group_name=DUMMY_TEST_PROFILING_GROUP_NAME, environment_override={ - "initial_sampling_interval": 0.5, + "initial_sampling_interval": timedelta(0, 780), "reporting_mode": "file", "file_prefix": file_prefix, 'agent_metadata': AgentMetadata(fleet_info=DefaultFleetInfo()) From ecbfc532bb19be1aeabc6e69a7f5055d99705d92 Mon Sep 17 00:00:00 2001 From: Henry Huynh <82241632+henryhuynh210@users.noreply.github.com> Date: Thu, 18 Jan 2024 08:15:12 -0800 Subject: [PATCH 39/76] Update test_end_to_end_profile_and_save_to_file.py --- test/acceptance/test_end_to_end_profile_and_save_to_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/acceptance/test_end_to_end_profile_and_save_to_file.py b/test/acceptance/test_end_to_end_profile_and_save_to_file.py index 3d5b4af..6b24829 100644 --- a/test/acceptance/test_end_to_end_profile_and_save_to_file.py +++ b/test/acceptance/test_end_to_end_profile_and_save_to_file.py @@ -59,7 +59,7 @@ def test_it_samples_and_saves_a_profile_to_a_file(self): profiler = Profiler( profiling_group_name=DUMMY_TEST_PROFILING_GROUP_NAME, environment_override={ - "initial_sampling_interval": timedelta(0, 780), + "initial_sampling_interval": timedelta(), "reporting_mode": "file", "file_prefix": file_prefix, 'agent_metadata': AgentMetadata(fleet_info=DefaultFleetInfo()) From 3d61100e6e36b410ca7cec3425e9c975151d5926 Mon Sep 17 00:00:00 2001 From: Henry Huynh <82241632+henryhuynh210@users.noreply.github.com> Date: Thu, 18 Jan 2024 12:27:38 -0800 Subject: [PATCH 40/76] Update test_live_backend_reporting.py get updated fqdn --- test/integration/test_live_backend_reporting.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/integration/test_live_backend_reporting.py b/test/integration/test_live_backend_reporting.py index 832bbc1..3c49801 100644 --- a/test/integration/test_live_backend_reporting.py +++ b/test/integration/test_live_backend_reporting.py @@ -27,6 +27,8 @@ class TestLiveBackendReporting: @before def before(self): + if (platform.system == "Darwin"): + print(socket.getfqdn()) now_millis = int(time.time()) * 1000 five_minutes_ago_millis = now_millis - (5 * 60 * 1000) sample = Sample( From 560ed1a606722e7c2ca9ae3b2afd960ac64da8b1 Mon Sep 17 00:00:00 2001 From: Henry Huynh <82241632+henryhuynh210@users.noreply.github.com> Date: Thu, 18 Jan 2024 12:29:30 -0800 Subject: [PATCH 41/76] Update test_live_backend_reporting.py --- test/integration/test_live_backend_reporting.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/integration/test_live_backend_reporting.py b/test/integration/test_live_backend_reporting.py index 3c49801..1a19201 100644 --- a/test/integration/test_live_backend_reporting.py +++ b/test/integration/test_live_backend_reporting.py @@ -2,6 +2,7 @@ import pytest import socket import time +import platform from datetime import timedelta From d96686642b0ac3e106a4ae7da619ac260ab94892 Mon Sep 17 00:00:00 2001 From: Henry Huynh <82241632+henryhuynh210@users.noreply.github.com> Date: Thu, 18 Jan 2024 12:33:54 -0800 Subject: [PATCH 42/76] Update test_live_backend_reporting.py --- test/integration/test_live_backend_reporting.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/test_live_backend_reporting.py b/test/integration/test_live_backend_reporting.py index 1a19201..f77d37b 100644 --- a/test/integration/test_live_backend_reporting.py +++ b/test/integration/test_live_backend_reporting.py @@ -28,8 +28,6 @@ class TestLiveBackendReporting: @before def before(self): - if (platform.system == "Darwin"): - print(socket.getfqdn()) now_millis = int(time.time()) * 1000 five_minutes_ago_millis = now_millis - (5 * 60 * 1000) sample = Sample( @@ -66,6 +64,8 @@ def before(self): cpu_limit_percentage=self.environment["cpu_limit_percentage"]) def test_beta_endpoint_call_report_and_refresh_and_do_not_override_user_overrides_agent_configuration(self): + if (platform.system == "Darwin"): + print(socket.getfqdn()) self.environment["agent_config_merger"] = AgentConfigurationMerger(user_overrides=self.agent_config) sdk_reporter = SdkReporter(self.environment) From 522eafcb5b46cf206f635ed7d123910ce9b4f0fa Mon Sep 17 00:00:00 2001 From: Henry Huynh <82241632+henryhuynh210@users.noreply.github.com> Date: Thu, 18 Jan 2024 12:38:28 -0800 Subject: [PATCH 43/76] Update test_live_backend_reporting.py --- test/integration/test_live_backend_reporting.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/integration/test_live_backend_reporting.py b/test/integration/test_live_backend_reporting.py index f77d37b..59770d4 100644 --- a/test/integration/test_live_backend_reporting.py +++ b/test/integration/test_live_backend_reporting.py @@ -28,6 +28,7 @@ class TestLiveBackendReporting: @before def before(self): + print(socket.getfqdn()) now_millis = int(time.time()) * 1000 five_minutes_ago_millis = now_millis - (5 * 60 * 1000) sample = Sample( @@ -64,8 +65,7 @@ def before(self): cpu_limit_percentage=self.environment["cpu_limit_percentage"]) def test_beta_endpoint_call_report_and_refresh_and_do_not_override_user_overrides_agent_configuration(self): - if (platform.system == "Darwin"): - print(socket.getfqdn()) + print(socket.getfqdn()) self.environment["agent_config_merger"] = AgentConfigurationMerger(user_overrides=self.agent_config) sdk_reporter = SdkReporter(self.environment) @@ -78,6 +78,7 @@ def test_beta_endpoint_call_report_and_refresh_and_do_not_override_user_override self.assert_initial_values() def test_beta_endpoint_call_report_and_refresh_and_overrides_default_agent_configuration(self): + print(socket.getfqdn()) self.environment["agent_config_merger"] = AgentConfigurationMerger(default=self.agent_config) sdk_reporter = SdkReporter(self.environment) @@ -94,6 +95,7 @@ def test_beta_endpoint_call_report_and_refresh_and_overrides_default_agent_confi assert AgentConfiguration.get().cpu_limit_percentage == 10 def test_beta_endpoint_call_report_and_refresh_and_do_not_override_one_setting_of_default_agent_configuration(self): + print(socket.getfqdn()) self.environment["agent_config_merger"] = AgentConfigurationMerger(default=self.agent_config, user_overrides=AgentConfiguration( sampling_interval=timedelta(seconds=2))) From 1f5e029cdc751362cb3dfeab8dad0027f1f53cc0 Mon Sep 17 00:00:00 2001 From: Henry Huynh <82241632+henryhuynh210@users.noreply.github.com> Date: Thu, 18 Jan 2024 12:44:05 -0800 Subject: [PATCH 44/76] Update test_live_backend_reporting.py --- test/integration/test_live_backend_reporting.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/test/integration/test_live_backend_reporting.py b/test/integration/test_live_backend_reporting.py index 59770d4..b8a9cbe 100644 --- a/test/integration/test_live_backend_reporting.py +++ b/test/integration/test_live_backend_reporting.py @@ -21,14 +21,13 @@ @pytest.mark.skipif( socket.getfqdn().endswith("internal.cloudapp.net"), # hosts running ubuntu and windows in GitHub - socket.getfqdn().endswith("ip6.arpa"), # hosts running macs in GitHub + socket.getfqdn().startswith("Mac-"), # hosts running macs in GitHub reason="This integration test is skipped on any shared fleet from Amazon or GitHub " "because it needs credentials to access the backend service. " "For information on how to run this locally, read the README.md file from the test/integration/ folder.") class TestLiveBackendReporting: @before def before(self): - print(socket.getfqdn()) now_millis = int(time.time()) * 1000 five_minutes_ago_millis = now_millis - (5 * 60 * 1000) sample = Sample( @@ -65,7 +64,6 @@ def before(self): cpu_limit_percentage=self.environment["cpu_limit_percentage"]) def test_beta_endpoint_call_report_and_refresh_and_do_not_override_user_overrides_agent_configuration(self): - print(socket.getfqdn()) self.environment["agent_config_merger"] = AgentConfigurationMerger(user_overrides=self.agent_config) sdk_reporter = SdkReporter(self.environment) @@ -78,7 +76,6 @@ def test_beta_endpoint_call_report_and_refresh_and_do_not_override_user_override self.assert_initial_values() def test_beta_endpoint_call_report_and_refresh_and_overrides_default_agent_configuration(self): - print(socket.getfqdn()) self.environment["agent_config_merger"] = AgentConfigurationMerger(default=self.agent_config) sdk_reporter = SdkReporter(self.environment) @@ -95,7 +92,6 @@ def test_beta_endpoint_call_report_and_refresh_and_overrides_default_agent_confi assert AgentConfiguration.get().cpu_limit_percentage == 10 def test_beta_endpoint_call_report_and_refresh_and_do_not_override_one_setting_of_default_agent_configuration(self): - print(socket.getfqdn()) self.environment["agent_config_merger"] = AgentConfigurationMerger(default=self.agent_config, user_overrides=AgentConfiguration( sampling_interval=timedelta(seconds=2))) From 576e05bb22c420ed17764a2cfc3801c443c0aff6 Mon Sep 17 00:00:00 2001 From: Henry Huynh <82241632+henryhuynh210@users.noreply.github.com> Date: Thu, 18 Jan 2024 13:00:49 -0800 Subject: [PATCH 45/76] Update test_end_to_end_profile_and_save_to_file.py purposely induce error --- test/acceptance/test_end_to_end_profile_and_save_to_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/acceptance/test_end_to_end_profile_and_save_to_file.py b/test/acceptance/test_end_to_end_profile_and_save_to_file.py index 6b24829..da358fd 100644 --- a/test/acceptance/test_end_to_end_profile_and_save_to_file.py +++ b/test/acceptance/test_end_to_end_profile_and_save_to_file.py @@ -84,7 +84,7 @@ def test_it_samples_and_saves_a_profile_to_a_file(self): try: print("Loading profile as JSON...") - resulting_json = json.loads(file_content) + resulting_json = json.load(file_content) except json.JSONDecodeError as e: print("Error decoding JSON: %s", str(e)) raise From 897452dbd021d6d0968e1b0fa57ea5de1567e70c Mon Sep 17 00:00:00 2001 From: Henry Huynh <82241632+henryhuynh210@users.noreply.github.com> Date: Thu, 18 Jan 2024 13:10:01 -0800 Subject: [PATCH 46/76] Revert temp changes done for debugging --- .../test_end_to_end_profile_and_save_to_file.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/test/acceptance/test_end_to_end_profile_and_save_to_file.py b/test/acceptance/test_end_to_end_profile_and_save_to_file.py index da358fd..74fe139 100644 --- a/test/acceptance/test_end_to_end_profile_and_save_to_file.py +++ b/test/acceptance/test_end_to_end_profile_and_save_to_file.py @@ -51,8 +51,6 @@ def test_it_samples_and_saves_a_profile_to_a_file(self): file_prefix = str(Path(self.temporary_directory, FILE_PREFIX)) - print(file_prefix) - test_start_time = time_utils.current_milli_time() @@ -75,18 +73,13 @@ def test_it_samples_and_saves_a_profile_to_a_file(self): resulting_profile_path = str( Path(self.temporary_directory, os.listdir(self.temporary_directory)[0])) - - print(resulting_profile_path) with (open(resulting_profile_path)) as profiling_result_file: file_content = profiling_result_file.read() - print("Content of the profile file (length:", len(file_content), "):", repr(file_content)) try: - print("Loading profile as JSON...") - resulting_json = json.load(file_content) + resulting_json = json.loads(file_content) except json.JSONDecodeError as e: - print("Error decoding JSON: %s", str(e)) raise self.assert_valid_agent_metadata(resulting_json["agentMetadata"]) From 6ded19efb9e91db12422e812d95fda8a4c5bcec1 Mon Sep 17 00:00:00 2001 From: John Palevich Date: Thu, 8 Feb 2024 14:43:16 -0800 Subject: [PATCH 47/76] Set last_frame_line_no to -1 if it's None. Tie ubuntu to version 20.04 to keep python 3.6 support --- .github/workflows/python-package.yml | 5 +---- codeguru_profiler_agent/sampling_utils.py | 2 ++ 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index b253136..b1c4d54 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -16,11 +16,8 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + os: [ubuntu-20.04, windows-latest, macos-latest] python-version: ['3.6', '3.7', '3.8', '3.9', '3.10', '3.11'] - exclude: - - os: ubuntu-latest - python-version: '3.6' steps: - uses: actions/checkout@v3 diff --git a/codeguru_profiler_agent/sampling_utils.py b/codeguru_profiler_agent/sampling_utils.py index de6b676..49b25fa 100644 --- a/codeguru_profiler_agent/sampling_utils.py +++ b/codeguru_profiler_agent/sampling_utils.py @@ -73,6 +73,8 @@ def _extract_stack(stack, max_depth): ) if len(result) < max_depth: last_frame, last_frame_line_no = stack[-1] + if last_frame_line_no is None: + last_frame_line_no = -1 _maybe_append_synthetic_frame(result, last_frame, last_frame_line_no) return result[:max_depth] From b5e6d5c99be4355b5d08f2dd8649479e62733a17 Mon Sep 17 00:00:00 2001 From: John Palevich Date: Thu, 8 Feb 2024 15:45:43 -0800 Subject: [PATCH 48/76] Set ubuntu to ubuntu-latest --- .github/workflows/python-package.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index b1c4d54..b253136 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -16,8 +16,11 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-20.04, windows-latest, macos-latest] + os: [ubuntu-latest, windows-latest, macos-latest] python-version: ['3.6', '3.7', '3.8', '3.9', '3.10', '3.11'] + exclude: + - os: ubuntu-latest + python-version: '3.6' steps: - uses: actions/checkout@v3 From 77d4c2b77edc1d82f3c0eabbdd387c9b0a40cd51 Mon Sep 17 00:00:00 2001 From: John Palevich Date: Thu, 8 Feb 2024 17:48:25 -0800 Subject: [PATCH 49/76] revert to initial branch state from henry's branch to see if tests pass on my branch --- codeguru_profiler_agent/sampling_utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/codeguru_profiler_agent/sampling_utils.py b/codeguru_profiler_agent/sampling_utils.py index 49b25fa..de6b676 100644 --- a/codeguru_profiler_agent/sampling_utils.py +++ b/codeguru_profiler_agent/sampling_utils.py @@ -73,8 +73,6 @@ def _extract_stack(stack, max_depth): ) if len(result) < max_depth: last_frame, last_frame_line_no = stack[-1] - if last_frame_line_no is None: - last_frame_line_no = -1 _maybe_append_synthetic_frame(result, last_frame, last_frame_line_no) return result[:max_depth] From 00d7c0d31c0fc76e0a06827186981dd58092780c Mon Sep 17 00:00:00 2001 From: John Palevich Date: Thu, 8 Feb 2024 18:15:09 -0800 Subject: [PATCH 50/76] Add sleep to see if this fixes failing 2 tests --- test/unit/test_profiler.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/unit/test_profiler.py b/test/unit/test_profiler.py index 60c06cb..508d000 100644 --- a/test/unit/test_profiler.py +++ b/test/unit/test_profiler.py @@ -1,4 +1,5 @@ import pytest +import time from datetime import timedelta from unittest.mock import Mock from codeguru_profiler_agent.profiler import Profiler @@ -153,6 +154,7 @@ def around(self): def test_exceptions_are_caught_and_do_not_propagate(self): self.profiler.start() + time.sleep(1) assert (not self.profiler.stop()) class TestWhenAnotherInstanceAlreadyStarted: @@ -178,6 +180,7 @@ def around(self): def test_stopping_first_instance_allows_next_profiler_to_start(self): self.first_profiler.stop() + time.sleep(1) assert self.second_profiler.start() class TestPause: From 2eae10be9f96e873733cf0e3914166350ec90e18 Mon Sep 17 00:00:00 2001 From: John Palevich Date: Tue, 13 Feb 2024 11:26:06 -0800 Subject: [PATCH 51/76] Use ubuntu-20.04 instead of ubuntu-latest, update workflows to latest checkout and setup-python versions --- test/unit/test_profiler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/unit/test_profiler.py b/test/unit/test_profiler.py index 508d000..9c9f676 100644 --- a/test/unit/test_profiler.py +++ b/test/unit/test_profiler.py @@ -180,7 +180,6 @@ def around(self): def test_stopping_first_instance_allows_next_profiler_to_start(self): self.first_profiler.stop() - time.sleep(1) assert self.second_profiler.start() class TestPause: From b268bb140429b6363ae33aa29a023a4d394c4cd4 Mon Sep 17 00:00:00 2001 From: John Palevich Date: Tue, 13 Feb 2024 11:28:01 -0800 Subject: [PATCH 52/76] push github workflow changes --- .github/workflows/python-package.yml | 5 +---- .github/workflows/python-publish-live.yml | 6 +++--- .github/workflows/python-publish-test.yml | 6 +++--- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index b253136..b1c4d54 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -16,11 +16,8 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, windows-latest, macos-latest] + os: [ubuntu-20.04, windows-latest, macos-latest] python-version: ['3.6', '3.7', '3.8', '3.9', '3.10', '3.11'] - exclude: - - os: ubuntu-latest - python-version: '3.6' steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/python-publish-live.yml b/.github/workflows/python-publish-live.yml index a4f8876..c9bbd55 100644 --- a/.github/workflows/python-publish-live.yml +++ b/.github/workflows/python-publish-live.yml @@ -10,11 +10,11 @@ on: jobs: deploy: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: '3.x' - name: Install dependencies diff --git a/.github/workflows/python-publish-test.yml b/.github/workflows/python-publish-test.yml index 4623b7b..87cbbf9 100644 --- a/.github/workflows/python-publish-test.yml +++ b/.github/workflows/python-publish-test.yml @@ -10,11 +10,11 @@ on: jobs: deploy: - runs-on: ubuntu-latest + runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v3 with: python-version: '3.x' - name: Install dependencies From a7e5b14ce3d4a77e54ab8f4bb518fb8801d38e6a Mon Sep 17 00:00:00 2001 From: John Palevich Date: Tue, 13 Feb 2024 13:11:46 -0800 Subject: [PATCH 53/76] Log socket name for 3.11 macos and ubuntu integ tests which shouldn't run --- test/integration/test_live_backend_reporting.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/integration/test_live_backend_reporting.py b/test/integration/test_live_backend_reporting.py index b8a9cbe..cc0406a 100644 --- a/test/integration/test_live_backend_reporting.py +++ b/test/integration/test_live_backend_reporting.py @@ -69,6 +69,8 @@ def test_beta_endpoint_call_report_and_refresh_and_do_not_override_user_override sdk_reporter = SdkReporter(self.environment) sdk_reporter.setup() + print(socket.getfqdn()) + self.assert_initial_values() assert sdk_reporter.report(self.profile) is True From 1d99f85f832f8129ea63813683327531664b1a5f Mon Sep 17 00:00:00 2001 From: John Palevich Date: Tue, 13 Feb 2024 13:18:44 -0800 Subject: [PATCH 54/76] Log socket correctly --- codeguru_profiler_agent/sdk_reporter/sdk_reporter.py | 2 ++ test/integration/test_live_backend_reporting.py | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/codeguru_profiler_agent/sdk_reporter/sdk_reporter.py b/codeguru_profiler_agent/sdk_reporter/sdk_reporter.py index 43e10cb..5f02b3e 100644 --- a/codeguru_profiler_agent/sdk_reporter/sdk_reporter.py +++ b/codeguru_profiler_agent/sdk_reporter/sdk_reporter.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import logging +import socket import io import os @@ -112,6 +113,7 @@ def report(self, profile): Lambda layers are set, it tries to create a Profiling Group whenever a ResourceNotFoundException is encountered. """ + logger.info("Socket: %s", socket.getfqdn()) try: profile_stream = self._encode_profile(profile) self.codeguru_client_builder.codeguru_client.post_agent_profile( diff --git a/test/integration/test_live_backend_reporting.py b/test/integration/test_live_backend_reporting.py index cc0406a..b8a9cbe 100644 --- a/test/integration/test_live_backend_reporting.py +++ b/test/integration/test_live_backend_reporting.py @@ -69,8 +69,6 @@ def test_beta_endpoint_call_report_and_refresh_and_do_not_override_user_override sdk_reporter = SdkReporter(self.environment) sdk_reporter.setup() - print(socket.getfqdn()) - self.assert_initial_values() assert sdk_reporter.report(self.profile) is True From ab937d4fd7dca9b8961e2608a7cc1b64f2f1ebde Mon Sep 17 00:00:00 2001 From: John Palevich Date: Tue, 13 Feb 2024 14:56:41 -0800 Subject: [PATCH 55/76] Skip test/integration/test_live_backend_reporting.py if run on GitHub --- test/integration/test_live_backend_reporting.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/integration/test_live_backend_reporting.py b/test/integration/test_live_backend_reporting.py index b8a9cbe..f4892f7 100644 --- a/test/integration/test_live_backend_reporting.py +++ b/test/integration/test_live_backend_reporting.py @@ -1,6 +1,6 @@ import boto3 import pytest -import socket +import os import time import platform @@ -20,8 +20,7 @@ @pytest.mark.skipif( - socket.getfqdn().endswith("internal.cloudapp.net"), # hosts running ubuntu and windows in GitHub - socket.getfqdn().startswith("Mac-"), # hosts running macs in GitHub + os.getenv("GITHUB_ACTIONS") == "true", reason="This integration test is skipped on any shared fleet from Amazon or GitHub " "because it needs credentials to access the backend service. " "For information on how to run this locally, read the README.md file from the test/integration/ folder.") From 3e3ae2118767455d63ce4f1fb2d60f992c3ca8ce Mon Sep 17 00:00:00 2001 From: John Palevich Date: Tue, 13 Feb 2024 16:44:53 -0800 Subject: [PATCH 56/76] Clear static profiler from decorator after profiler decorator unit tests --- codeguru_profiler_agent/sdk_reporter/sdk_reporter.py | 2 -- test/unit/aws_lambda/test_profiler_decorator.py | 2 ++ test/unit/test_profiler.py | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/codeguru_profiler_agent/sdk_reporter/sdk_reporter.py b/codeguru_profiler_agent/sdk_reporter/sdk_reporter.py index 5f02b3e..43e10cb 100644 --- a/codeguru_profiler_agent/sdk_reporter/sdk_reporter.py +++ b/codeguru_profiler_agent/sdk_reporter/sdk_reporter.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- import logging -import socket import io import os @@ -113,7 +112,6 @@ def report(self, profile): Lambda layers are set, it tries to create a Profiling Group whenever a ResourceNotFoundException is encountered. """ - logger.info("Socket: %s", socket.getfqdn()) try: profile_stream = self._encode_profile(profile) self.codeguru_client_builder.codeguru_client.post_agent_profile( diff --git a/test/unit/aws_lambda/test_profiler_decorator.py b/test/unit/aws_lambda/test_profiler_decorator.py index 933e871..f1cbf19 100644 --- a/test/unit/aws_lambda/test_profiler_decorator.py +++ b/test/unit/aws_lambda/test_profiler_decorator.py @@ -82,6 +82,8 @@ def handler_function(event, context): self.handler = handler_function yield \ codeguru_profiler_agent.aws_lambda.profiler_decorator.clear_static_profiler() + codeguru_profiler_agent.aws_lambda.profiler_decorator.clear_static_profiler() + def test_given_profiling_group_is_used(self): self.handler({}, self.context) diff --git a/test/unit/test_profiler.py b/test/unit/test_profiler.py index 9c9f676..11ba777 100644 --- a/test/unit/test_profiler.py +++ b/test/unit/test_profiler.py @@ -154,7 +154,6 @@ def around(self): def test_exceptions_are_caught_and_do_not_propagate(self): self.profiler.start() - time.sleep(1) assert (not self.profiler.stop()) class TestWhenAnotherInstanceAlreadyStarted: From d7cba56849bc114996ff8bc7818564538138d480 Mon Sep 17 00:00:00 2001 From: John Palevich Date: Wed, 14 Feb 2024 14:52:28 -0800 Subject: [PATCH 57/76] add logging to debug test --- test/acceptance/test_end_to_end_profile_and_save_to_file.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/acceptance/test_end_to_end_profile_and_save_to_file.py b/test/acceptance/test_end_to_end_profile_and_save_to_file.py index 74fe139..57acd80 100644 --- a/test/acceptance/test_end_to_end_profile_and_save_to_file.py +++ b/test/acceptance/test_end_to_end_profile_and_save_to_file.py @@ -78,6 +78,7 @@ def test_it_samples_and_saves_a_profile_to_a_file(self): file_content = profiling_result_file.read() try: + print(file_content) resulting_json = json.loads(file_content) except json.JSONDecodeError as e: raise From 4938b3dcbbf728fa704d53e4e22cd5f0e005ba89 Mon Sep 17 00:00:00 2001 From: John Palevich Date: Wed, 14 Feb 2024 15:01:31 -0800 Subject: [PATCH 58/76] format string --- test/acceptance/test_end_to_end_profile_and_save_to_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/acceptance/test_end_to_end_profile_and_save_to_file.py b/test/acceptance/test_end_to_end_profile_and_save_to_file.py index 57acd80..ad0bb33 100644 --- a/test/acceptance/test_end_to_end_profile_and_save_to_file.py +++ b/test/acceptance/test_end_to_end_profile_and_save_to_file.py @@ -78,7 +78,7 @@ def test_it_samples_and_saves_a_profile_to_a_file(self): file_content = profiling_result_file.read() try: - print(file_content) + print(f"Content: \"%s\"", file_content) resulting_json = json.loads(file_content) except json.JSONDecodeError as e: raise From befeea6eda5a614604d675d31861da53a7ee700b Mon Sep 17 00:00:00 2001 From: John Palevich Date: Wed, 14 Feb 2024 15:10:12 -0800 Subject: [PATCH 59/76] format string --- test/acceptance/test_end_to_end_profile_and_save_to_file.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/acceptance/test_end_to_end_profile_and_save_to_file.py b/test/acceptance/test_end_to_end_profile_and_save_to_file.py index ad0bb33..489185a 100644 --- a/test/acceptance/test_end_to_end_profile_and_save_to_file.py +++ b/test/acceptance/test_end_to_end_profile_and_save_to_file.py @@ -78,7 +78,7 @@ def test_it_samples_and_saves_a_profile_to_a_file(self): file_content = profiling_result_file.read() try: - print(f"Content: \"%s\"", file_content) + print(f"Content: \"%s\"" % file_content) resulting_json = json.loads(file_content) except json.JSONDecodeError as e: raise From 4c154cbbbf2de941f095e834f8dd74978354f357 Mon Sep 17 00:00:00 2001 From: John Palevich Date: Wed, 14 Feb 2024 15:13:26 -0800 Subject: [PATCH 60/76] add sleep between start and stop --- test/acceptance/test_end_to_end_profile_and_save_to_file.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/acceptance/test_end_to_end_profile_and_save_to_file.py b/test/acceptance/test_end_to_end_profile_and_save_to_file.py index 489185a..5f74b51 100644 --- a/test/acceptance/test_end_to_end_profile_and_save_to_file.py +++ b/test/acceptance/test_end_to_end_profile_and_save_to_file.py @@ -3,6 +3,7 @@ import shutil import tempfile import os +import time from datetime import timedelta from unittest.mock import patch @@ -65,6 +66,7 @@ def test_it_samples_and_saves_a_profile_to_a_file(self): try: profiler.start() + time.sleep(2) finally: profiler.stop() From f1cfbd84508886ed05b518e02497650257b77bee Mon Sep 17 00:00:00 2001 From: John Palevich Date: Wed, 14 Feb 2024 16:13:23 -0800 Subject: [PATCH 61/76] add additional prints --- codeguru_profiler_agent/local_aggregator.py | 2 ++ codeguru_profiler_agent/model/profile.py | 1 + codeguru_profiler_agent/profiler.py | 1 + test/acceptance/test_end_to_end_profile_and_save_to_file.py | 1 - 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/codeguru_profiler_agent/local_aggregator.py b/codeguru_profiler_agent/local_aggregator.py index 1091c9b..12bfa5e 100644 --- a/codeguru_profiler_agent/local_aggregator.py +++ b/codeguru_profiler_agent/local_aggregator.py @@ -64,6 +64,8 @@ def setup(self): @with_timer("aggregateThreadDumps") def _aggregate_sample(self, sample): + print("Sample: ", sample) + print("Profile: ", self.profile) self.profile.add(sample) def _check_memory_limit(self): diff --git a/codeguru_profiler_agent/model/profile.py b/codeguru_profiler_agent/model/profile.py index 929defc..b4c6235 100644 --- a/codeguru_profiler_agent/model/profile.py +++ b/codeguru_profiler_agent/model/profile.py @@ -24,6 +24,7 @@ def __init__(self, profiling_group_name, sampling_interval_seconds, host_weight, memory_counter=self.memory_counter) self._validate_positive_number(start) self.start = start + print("Start: ", start) self.last_resume = start self.last_pause = None self._paused_ms = 0 diff --git a/codeguru_profiler_agent/profiler.py b/codeguru_profiler_agent/profiler.py index 5f7c8bb..546e30d 100644 --- a/codeguru_profiler_agent/profiler.py +++ b/codeguru_profiler_agent/profiler.py @@ -183,6 +183,7 @@ def _select_collector(environment): reporter=SdkReporter(environment=environment), environment=environment) elif reporting_mode == "file": + print("Using file reporter") return LocalAggregator( reporter=FileReporter(environment=environment), environment=environment) diff --git a/test/acceptance/test_end_to_end_profile_and_save_to_file.py b/test/acceptance/test_end_to_end_profile_and_save_to_file.py index 5f74b51..e904a0a 100644 --- a/test/acceptance/test_end_to_end_profile_and_save_to_file.py +++ b/test/acceptance/test_end_to_end_profile_and_save_to_file.py @@ -66,7 +66,6 @@ def test_it_samples_and_saves_a_profile_to_a_file(self): try: profiler.start() - time.sleep(2) finally: profiler.stop() From a325e4978d817f24ca9b1fc77308352dd99c1f98 Mon Sep 17 00:00:00 2001 From: John Palevich Date: Wed, 14 Feb 2024 16:21:15 -0800 Subject: [PATCH 62/76] log actual end time --- codeguru_profiler_agent/model/profile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codeguru_profiler_agent/model/profile.py b/codeguru_profiler_agent/model/profile.py index b4c6235..2710c07 100644 --- a/codeguru_profiler_agent/model/profile.py +++ b/codeguru_profiler_agent/model/profile.py @@ -106,7 +106,7 @@ def add(self, sample): for stack in sample.stacks: self._insert_stack(stack) - + print("end time:", time.time()) self.end = current_milli_time(clock=self._clock) def set_overhead_ms(self, duration_timedelta): From 9c01941dd11db97fcc62323fc2495a78402cf2e8 Mon Sep 17 00:00:00 2001 From: John Palevich Date: Wed, 14 Feb 2024 16:32:01 -0800 Subject: [PATCH 63/76] switch to logger --- codeguru_profiler_agent/model/profile.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/codeguru_profiler_agent/model/profile.py b/codeguru_profiler_agent/model/profile.py index 2710c07..3597e8f 100644 --- a/codeguru_profiler_agent/model/profile.py +++ b/codeguru_profiler_agent/model/profile.py @@ -24,7 +24,7 @@ def __init__(self, profiling_group_name, sampling_interval_seconds, host_weight, memory_counter=self.memory_counter) self._validate_positive_number(start) self.start = start - print("Start: ", start) + logger.info("Start: ", time.time()) self.last_resume = start self.last_pause = None self._paused_ms = 0 @@ -106,13 +106,13 @@ def add(self, sample): for stack in sample.stacks: self._insert_stack(stack) - print("end time:", time.time()) + logger.info("end time:", time.time()) self.end = current_milli_time(clock=self._clock) def set_overhead_ms(self, duration_timedelta): """ The overhead is the total cpu time spent profiling since start. It is measured by a Timer object and only passed - to the profile object before we report it so it is only added here because it is more convenient to convey this + to thoe profile object before we report it so it is only added here because it is more convenient to convey this data with the rest of the profile data. """ self.overhead_ms = duration_timedelta.total_seconds() * 1000 From bfc36cc9c6cb40a6660ed1a3f20070129b469342 Mon Sep 17 00:00:00 2001 From: John Palevich Date: Wed, 14 Feb 2024 16:37:09 -0800 Subject: [PATCH 64/76] fix logging --- codeguru_profiler_agent/local_aggregator.py | 1 + codeguru_profiler_agent/model/profile.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/codeguru_profiler_agent/local_aggregator.py b/codeguru_profiler_agent/local_aggregator.py index 12bfa5e..9cffd4b 100644 --- a/codeguru_profiler_agent/local_aggregator.py +++ b/codeguru_profiler_agent/local_aggregator.py @@ -79,6 +79,7 @@ def _check_memory_limit(self): def reset(self): self.errors_metadata.reset() self.timer.reset() + logger.info("Start at reset: " + current_milli_time(clock=self.clock)) self.profile = self.profile_factory( profiling_group_name=self.profiling_group_name, sampling_interval_seconds=AgentConfiguration.get().sampling_interval.total_seconds(), diff --git a/codeguru_profiler_agent/model/profile.py b/codeguru_profiler_agent/model/profile.py index 3597e8f..c5f1cee 100644 --- a/codeguru_profiler_agent/model/profile.py +++ b/codeguru_profiler_agent/model/profile.py @@ -24,7 +24,7 @@ def __init__(self, profiling_group_name, sampling_interval_seconds, host_weight, memory_counter=self.memory_counter) self._validate_positive_number(start) self.start = start - logger.info("Start: ", time.time()) + logger.info("Start: " + current_milli_time(time.time())) self.last_resume = start self.last_pause = None self._paused_ms = 0 @@ -106,7 +106,7 @@ def add(self, sample): for stack in sample.stacks: self._insert_stack(stack) - logger.info("end time:", time.time()) + logger.info("end time: " + current_milli_time(clock=time.time())) self.end = current_milli_time(clock=self._clock) def set_overhead_ms(self, duration_timedelta): From b30fbab34db066b8aca8d310d127583956e8d953 Mon Sep 17 00:00:00 2001 From: John Palevich Date: Wed, 14 Feb 2024 16:40:43 -0800 Subject: [PATCH 65/76] fix logging --- codeguru_profiler_agent/local_aggregator.py | 2 +- codeguru_profiler_agent/model/profile.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/codeguru_profiler_agent/local_aggregator.py b/codeguru_profiler_agent/local_aggregator.py index 9cffd4b..13c4875 100644 --- a/codeguru_profiler_agent/local_aggregator.py +++ b/codeguru_profiler_agent/local_aggregator.py @@ -79,7 +79,7 @@ def _check_memory_limit(self): def reset(self): self.errors_metadata.reset() self.timer.reset() - logger.info("Start at reset: " + current_milli_time(clock=self.clock)) + logger.info("Start at reset: %d" % current_milli_time(clock=self.clock)) self.profile = self.profile_factory( profiling_group_name=self.profiling_group_name, sampling_interval_seconds=AgentConfiguration.get().sampling_interval.total_seconds(), diff --git a/codeguru_profiler_agent/model/profile.py b/codeguru_profiler_agent/model/profile.py index c5f1cee..0488d70 100644 --- a/codeguru_profiler_agent/model/profile.py +++ b/codeguru_profiler_agent/model/profile.py @@ -24,7 +24,7 @@ def __init__(self, profiling_group_name, sampling_interval_seconds, host_weight, memory_counter=self.memory_counter) self._validate_positive_number(start) self.start = start - logger.info("Start: " + current_milli_time(time.time())) + logger.info("Start: %d" % current_milli_time(time.time())) self.last_resume = start self.last_pause = None self._paused_ms = 0 @@ -106,7 +106,7 @@ def add(self, sample): for stack in sample.stacks: self._insert_stack(stack) - logger.info("end time: " + current_milli_time(clock=time.time())) + logger.info("end time: %d" % current_milli_time(clock=time.time())) self.end = current_milli_time(clock=self._clock) def set_overhead_ms(self, duration_timedelta): From e4be84240704a4bf4a4be0ac607fd51f6def346c Mon Sep 17 00:00:00 2001 From: John Palevich Date: Wed, 14 Feb 2024 16:45:57 -0800 Subject: [PATCH 66/76] fix logging --- codeguru_profiler_agent/model/profile.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codeguru_profiler_agent/model/profile.py b/codeguru_profiler_agent/model/profile.py index 0488d70..c44982f 100644 --- a/codeguru_profiler_agent/model/profile.py +++ b/codeguru_profiler_agent/model/profile.py @@ -24,7 +24,7 @@ def __init__(self, profiling_group_name, sampling_interval_seconds, host_weight, memory_counter=self.memory_counter) self._validate_positive_number(start) self.start = start - logger.info("Start: %d" % current_milli_time(time.time())) + logger.info("Start: %d" % current_milli_time(time.time)) self.last_resume = start self.last_pause = None self._paused_ms = 0 @@ -106,7 +106,7 @@ def add(self, sample): for stack in sample.stacks: self._insert_stack(stack) - logger.info("end time: %d" % current_milli_time(clock=time.time())) + logger.info("end time: %d" % current_milli_time(clock=time.time)) self.end = current_milli_time(clock=self._clock) def set_overhead_ms(self, duration_timedelta): From d5fe96c3a531d50307ab264fc80660246158ce96 Mon Sep 17 00:00:00 2001 From: John Palevich Date: Wed, 14 Feb 2024 16:55:50 -0800 Subject: [PATCH 67/76] Add 2 second sleep to see if time.time changes --- codeguru_profiler_agent/model/profile.py | 1 + 1 file changed, 1 insertion(+) diff --git a/codeguru_profiler_agent/model/profile.py b/codeguru_profiler_agent/model/profile.py index c44982f..b7f8a3f 100644 --- a/codeguru_profiler_agent/model/profile.py +++ b/codeguru_profiler_agent/model/profile.py @@ -106,6 +106,7 @@ def add(self, sample): for stack in sample.stacks: self._insert_stack(stack) + time.sleep(2) logger.info("end time: %d" % current_milli_time(clock=time.time)) self.end = current_milli_time(clock=self._clock) From 998a62623a107514fc7b1799e3e9cf24abe9b3a5 Mon Sep 17 00:00:00 2001 From: John Palevich Date: Wed, 14 Feb 2024 17:05:06 -0800 Subject: [PATCH 68/76] lengthen test time --- codeguru_profiler_agent/model/profile.py | 1 - test/acceptance/test_end_to_end_profile_and_save_to_file.py | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/codeguru_profiler_agent/model/profile.py b/codeguru_profiler_agent/model/profile.py index b7f8a3f..c44982f 100644 --- a/codeguru_profiler_agent/model/profile.py +++ b/codeguru_profiler_agent/model/profile.py @@ -106,7 +106,6 @@ def add(self, sample): for stack in sample.stacks: self._insert_stack(stack) - time.sleep(2) logger.info("end time: %d" % current_milli_time(clock=time.time)) self.end = current_milli_time(clock=self._clock) diff --git a/test/acceptance/test_end_to_end_profile_and_save_to_file.py b/test/acceptance/test_end_to_end_profile_and_save_to_file.py index e904a0a..b37dd72 100644 --- a/test/acceptance/test_end_to_end_profile_and_save_to_file.py +++ b/test/acceptance/test_end_to_end_profile_and_save_to_file.py @@ -58,7 +58,7 @@ def test_it_samples_and_saves_a_profile_to_a_file(self): profiler = Profiler( profiling_group_name=DUMMY_TEST_PROFILING_GROUP_NAME, environment_override={ - "initial_sampling_interval": timedelta(), + "initial_sampling_interval": timedelta(seconds=1), "reporting_mode": "file", "file_prefix": file_prefix, 'agent_metadata': AgentMetadata(fleet_info=DefaultFleetInfo()) @@ -66,6 +66,7 @@ def test_it_samples_and_saves_a_profile_to_a_file(self): try: profiler.start() + time.sleep(4) finally: profiler.stop() From 9caf52e5a3d8f706227277f0d900e4ef256e4591 Mon Sep 17 00:00:00 2001 From: John Palevich Date: Wed, 14 Feb 2024 17:21:22 -0800 Subject: [PATCH 69/76] clean up file diff --- codeguru_profiler_agent/local_aggregator.py | 1 - codeguru_profiler_agent/model/profile.py | 4 +--- codeguru_profiler_agent/profiler.py | 1 - test/acceptance/test_end_to_end_profile_and_save_to_file.py | 5 ++--- test/unit/test_profiler.py | 1 - 5 files changed, 3 insertions(+), 9 deletions(-) diff --git a/codeguru_profiler_agent/local_aggregator.py b/codeguru_profiler_agent/local_aggregator.py index 13c4875..12bfa5e 100644 --- a/codeguru_profiler_agent/local_aggregator.py +++ b/codeguru_profiler_agent/local_aggregator.py @@ -79,7 +79,6 @@ def _check_memory_limit(self): def reset(self): self.errors_metadata.reset() self.timer.reset() - logger.info("Start at reset: %d" % current_milli_time(clock=self.clock)) self.profile = self.profile_factory( profiling_group_name=self.profiling_group_name, sampling_interval_seconds=AgentConfiguration.get().sampling_interval.total_seconds(), diff --git a/codeguru_profiler_agent/model/profile.py b/codeguru_profiler_agent/model/profile.py index c44982f..7c1dfc8 100644 --- a/codeguru_profiler_agent/model/profile.py +++ b/codeguru_profiler_agent/model/profile.py @@ -24,7 +24,6 @@ def __init__(self, profiling_group_name, sampling_interval_seconds, host_weight, memory_counter=self.memory_counter) self._validate_positive_number(start) self.start = start - logger.info("Start: %d" % current_milli_time(time.time)) self.last_resume = start self.last_pause = None self._paused_ms = 0 @@ -106,13 +105,12 @@ def add(self, sample): for stack in sample.stacks: self._insert_stack(stack) - logger.info("end time: %d" % current_milli_time(clock=time.time)) self.end = current_milli_time(clock=self._clock) def set_overhead_ms(self, duration_timedelta): """ The overhead is the total cpu time spent profiling since start. It is measured by a Timer object and only passed - to thoe profile object before we report it so it is only added here because it is more convenient to convey this + to the profile object before we report it so it is only added here because it is more convenient to convey this data with the rest of the profile data. """ self.overhead_ms = duration_timedelta.total_seconds() * 1000 diff --git a/codeguru_profiler_agent/profiler.py b/codeguru_profiler_agent/profiler.py index 546e30d..5f7c8bb 100644 --- a/codeguru_profiler_agent/profiler.py +++ b/codeguru_profiler_agent/profiler.py @@ -183,7 +183,6 @@ def _select_collector(environment): reporter=SdkReporter(environment=environment), environment=environment) elif reporting_mode == "file": - print("Using file reporter") return LocalAggregator( reporter=FileReporter(environment=environment), environment=environment) diff --git a/test/acceptance/test_end_to_end_profile_and_save_to_file.py b/test/acceptance/test_end_to_end_profile_and_save_to_file.py index b37dd72..3ebc797 100644 --- a/test/acceptance/test_end_to_end_profile_and_save_to_file.py +++ b/test/acceptance/test_end_to_end_profile_and_save_to_file.py @@ -54,7 +54,6 @@ def test_it_samples_and_saves_a_profile_to_a_file(self): test_start_time = time_utils.current_milli_time() - profiler = Profiler( profiling_group_name=DUMMY_TEST_PROFILING_GROUP_NAME, environment_override={ @@ -66,7 +65,7 @@ def test_it_samples_and_saves_a_profile_to_a_file(self): try: profiler.start() - time.sleep(4) + time.sleep(2) finally: profiler.stop() @@ -75,7 +74,7 @@ def test_it_samples_and_saves_a_profile_to_a_file(self): resulting_profile_path = str( Path(self.temporary_directory, os.listdir(self.temporary_directory)[0])) - + with (open(resulting_profile_path)) as profiling_result_file: file_content = profiling_result_file.read() diff --git a/test/unit/test_profiler.py b/test/unit/test_profiler.py index 11ba777..60c06cb 100644 --- a/test/unit/test_profiler.py +++ b/test/unit/test_profiler.py @@ -1,5 +1,4 @@ import pytest -import time from datetime import timedelta from unittest.mock import Mock from codeguru_profiler_agent.profiler import Profiler From 16e40b0c80aa38d34896d1dd0fe3819bd14295bd Mon Sep 17 00:00:00 2001 From: John Palevich Date: Wed, 14 Feb 2024 17:36:47 -0800 Subject: [PATCH 70/76] add short sleep to test handler function --- test/unit/aws_lambda/test_profiler_decorator.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/unit/aws_lambda/test_profiler_decorator.py b/test/unit/aws_lambda/test_profiler_decorator.py index f1cbf19..c902bf2 100644 --- a/test/unit/aws_lambda/test_profiler_decorator.py +++ b/test/unit/aws_lambda/test_profiler_decorator.py @@ -1,4 +1,5 @@ import pytest +import time import codeguru_profiler_agent.aws_lambda.profiler_decorator from unittest.mock import MagicMock @@ -77,6 +78,7 @@ def around(self): @with_lambda_profiler(profiling_group_name="pg_name", region_name="eu-north-1", environment_override={'cpu_limit_percentage': 42}, env=self.env) def handler_function(event, context): + time.sleep(0.5) return True self.handler = handler_function From 3ff0845ed83826e590231ae6a4b4c8c0fc76bb98 Mon Sep 17 00:00:00 2001 From: John Palevich Date: Wed, 14 Feb 2024 17:51:02 -0800 Subject: [PATCH 71/76] Clean up commits --- codeguru_profiler_agent/local_aggregator.py | 2 -- codeguru_profiler_agent/model/profile.py | 1 + test/acceptance/test_end_to_end_profile_and_save_to_file.py | 1 - test/unit/metrics/test_with_timer.py | 4 +++- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/codeguru_profiler_agent/local_aggregator.py b/codeguru_profiler_agent/local_aggregator.py index 12bfa5e..1091c9b 100644 --- a/codeguru_profiler_agent/local_aggregator.py +++ b/codeguru_profiler_agent/local_aggregator.py @@ -64,8 +64,6 @@ def setup(self): @with_timer("aggregateThreadDumps") def _aggregate_sample(self, sample): - print("Sample: ", sample) - print("Profile: ", self.profile) self.profile.add(sample) def _check_memory_limit(self): diff --git a/codeguru_profiler_agent/model/profile.py b/codeguru_profiler_agent/model/profile.py index 7c1dfc8..1b7fc37 100644 --- a/codeguru_profiler_agent/model/profile.py +++ b/codeguru_profiler_agent/model/profile.py @@ -105,6 +105,7 @@ def add(self, sample): for stack in sample.stacks: self._insert_stack(stack) + self.end = current_milli_time(clock=self._clock) def set_overhead_ms(self, duration_timedelta): diff --git a/test/acceptance/test_end_to_end_profile_and_save_to_file.py b/test/acceptance/test_end_to_end_profile_and_save_to_file.py index 3ebc797..178b2b9 100644 --- a/test/acceptance/test_end_to_end_profile_and_save_to_file.py +++ b/test/acceptance/test_end_to_end_profile_and_save_to_file.py @@ -79,7 +79,6 @@ def test_it_samples_and_saves_a_profile_to_a_file(self): file_content = profiling_result_file.read() try: - print(f"Content: \"%s\"" % file_content) resulting_json = json.loads(file_content) except json.JSONDecodeError as e: raise diff --git a/test/unit/metrics/test_with_timer.py b/test/unit/metrics/test_with_timer.py index 49a6172..a5f8af4 100644 --- a/test/unit/metrics/test_with_timer.py +++ b/test/unit/metrics/test_with_timer.py @@ -14,9 +14,11 @@ def __init__(self): def foo_wall(self): return + # Run something to make sure the cpu clock does tick (https://bugs.python.org/issue37859) @with_timer(metric_name="test-foo-cpu", measurement="cpu-time") def foo_cpu(self): - # Run something to make sure the cpu clock does tick (https://bugs.python.org/issue37859) + # Call set_int_max_str for specific versions to test as its limited by CVE-2020-10735 + # (https://docs.python.org/3/library/stdtypes.html#integer-string-conversion-length-limitation) if (sys.version_info >= (3, 7) and platform.system() != 'Windows') or (sys.version_info >= (3, 10) and platform.system() == 'Windows'): sys.set_int_max_str_digits(0) len(str(2 ** 500_000)) From e642706364b80007e93531842b9ba1b9bdcc720c Mon Sep 17 00:00:00 2001 From: John Palevich Date: Wed, 14 Feb 2024 17:58:02 -0800 Subject: [PATCH 72/76] More formatting --- codeguru_profiler_agent/model/profile.py | 2 +- codeguru_profiler_agent/sampler.py | 2 +- test/unit/aws_lambda/test_profiler_decorator.py | 1 - test/unit/metrics/test_with_timer.py | 2 +- 4 files changed, 3 insertions(+), 4 deletions(-) diff --git a/codeguru_profiler_agent/model/profile.py b/codeguru_profiler_agent/model/profile.py index 1b7fc37..929defc 100644 --- a/codeguru_profiler_agent/model/profile.py +++ b/codeguru_profiler_agent/model/profile.py @@ -105,7 +105,7 @@ def add(self, sample): for stack in sample.stacks: self._insert_stack(stack) - + self.end = current_milli_time(clock=self._clock) def set_overhead_ms(self, duration_timedelta): diff --git a/codeguru_profiler_agent/sampler.py b/codeguru_profiler_agent/sampler.py index c385cf6..ebef34f 100644 --- a/codeguru_profiler_agent/sampler.py +++ b/codeguru_profiler_agent/sampler.py @@ -64,4 +64,4 @@ def _threads_to_sample_from(self, all_threads): all_threads = list(all_threads.keys()) return random.sample(all_threads, self._max_threads) else: - return list(all_threads) + return list(all_threads) diff --git a/test/unit/aws_lambda/test_profiler_decorator.py b/test/unit/aws_lambda/test_profiler_decorator.py index c902bf2..3e2ba59 100644 --- a/test/unit/aws_lambda/test_profiler_decorator.py +++ b/test/unit/aws_lambda/test_profiler_decorator.py @@ -86,7 +86,6 @@ def handler_function(event, context): codeguru_profiler_agent.aws_lambda.profiler_decorator.clear_static_profiler() codeguru_profiler_agent.aws_lambda.profiler_decorator.clear_static_profiler() - def test_given_profiling_group_is_used(self): self.handler({}, self.context) assert (codeguru_profiler_agent.aws_lambda.profiler_decorator._profiler._profiler_runner.collector diff --git a/test/unit/metrics/test_with_timer.py b/test/unit/metrics/test_with_timer.py index a5f8af4..8105a49 100644 --- a/test/unit/metrics/test_with_timer.py +++ b/test/unit/metrics/test_with_timer.py @@ -17,7 +17,7 @@ def foo_wall(self): # Run something to make sure the cpu clock does tick (https://bugs.python.org/issue37859) @with_timer(metric_name="test-foo-cpu", measurement="cpu-time") def foo_cpu(self): - # Call set_int_max_str for specific versions to test as its limited by CVE-2020-10735 + # Call set_int_max_str for specific versions to test as its limited to resolve CVE-2020-10735 # (https://docs.python.org/3/library/stdtypes.html#integer-string-conversion-length-limitation) if (sys.version_info >= (3, 7) and platform.system() != 'Windows') or (sys.version_info >= (3, 10) and platform.system() == 'Windows'): sys.set_int_max_str_digits(0) From 02679971f71b02a13c18445a9caf14d2254235e1 Mon Sep 17 00:00:00 2001 From: John Palevich Date: Thu, 15 Feb 2024 01:27:53 -0800 Subject: [PATCH 73/76] Resolve last last_frame_line_no == None issue --- codeguru_profiler_agent/sampling_utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/codeguru_profiler_agent/sampling_utils.py b/codeguru_profiler_agent/sampling_utils.py index de6b676..b443be5 100644 --- a/codeguru_profiler_agent/sampling_utils.py +++ b/codeguru_profiler_agent/sampling_utils.py @@ -73,7 +73,10 @@ def _extract_stack(stack, max_depth): ) if len(result) < max_depth: last_frame, last_frame_line_no = stack[-1] - _maybe_append_synthetic_frame(result, last_frame, last_frame_line_no) + # If the line_no is None, ignore the line as we can't get the line + # of code from the line cache + if last_frame_line_no != None: + _maybe_append_synthetic_frame(result, last_frame, last_frame_line_no) return result[:max_depth] From 851a3b134483edaf35c93c1d40995fa052ca1b1d Mon Sep 17 00:00:00 2001 From: John Palevich Date: Thu, 15 Feb 2024 02:18:22 -0800 Subject: [PATCH 74/76] Adjust github workflows --- .github/workflows/python-package.yml | 15 ++++++++++++++- .github/workflows/python-publish-live.yml | 2 +- .github/workflows/python-publish-test.yml | 2 +- 3 files changed, 16 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index b1c4d54..224c59a 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -16,8 +16,21 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-20.04, windows-latest, macos-latest] + os: [ubuntu-20.04, ubuntu-latest windows-latest, macos-latest] python-version: ['3.6', '3.7', '3.8', '3.9', '3.10', '3.11'] + exclude: + - os: ubuntu-latest + python-version: '3.6' + - os: ubuntu-20.04 + python-version: '3.7' + - os: ubuntu-20.04 + python-version: '3.8' + - os: ubuntu-20.04 + python-version: '3.9' + - os: ubuntu-20.04 + python-version: '3.10' + - os: ubuntu-20.04 + python-version: '3.11' steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/python-publish-live.yml b/.github/workflows/python-publish-live.yml index c9bbd55..c234368 100644 --- a/.github/workflows/python-publish-live.yml +++ b/.github/workflows/python-publish-live.yml @@ -10,7 +10,7 @@ on: jobs: deploy: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - name: Set up Python diff --git a/.github/workflows/python-publish-test.yml b/.github/workflows/python-publish-test.yml index 87cbbf9..4115349 100644 --- a/.github/workflows/python-publish-test.yml +++ b/.github/workflows/python-publish-test.yml @@ -10,7 +10,7 @@ on: jobs: deploy: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - name: Set up Python From dbddce3c57f9a2eaaf33ef69431310f48e1207b1 Mon Sep 17 00:00:00 2001 From: John Palevich Date: Thu, 15 Feb 2024 02:44:49 -0800 Subject: [PATCH 75/76] fix python-package.yml --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 224c59a..cb41526 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-20.04, ubuntu-latest windows-latest, macos-latest] + os: [ubuntu-20.04, ubuntu-latest, windows-latest, macos-latest] python-version: ['3.6', '3.7', '3.8', '3.9', '3.10', '3.11'] exclude: - os: ubuntu-latest From a90c59077e10261118f0f7543d23204da78245db Mon Sep 17 00:00:00 2001 From: John Palevich Date: Thu, 15 Feb 2024 11:51:48 -0800 Subject: [PATCH 76/76] Update agent version and Changelog --- CHANGELOG.rst | 6 +++++- codeguru_profiler_agent/agent_metadata/agent_metadata.py | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 93f7389..773085a 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -2,13 +2,17 @@ CHANGELOG ========= +1.2.5 (layer_v12) +=================== +* Fix bug which causes agent to crash if line_no was None. + 1.2.4 (layer_v11) =================== * Updated lambda bootstrap code to support profiling python 3.9 lambda functions. 1.2.3 (layer_v10) =================== -* Fix bug to sent agent overhead in the rigth format: int and as part of memoryInMB (instead of the previous string as part of memory_usage_mb). +* Fix bug to sent agent overhead in the right format: int and as part of memoryInMB (instead of the previous string as part of memory_usage_mb). 1.2.2 (layer_v9) =================== diff --git a/codeguru_profiler_agent/agent_metadata/agent_metadata.py b/codeguru_profiler_agent/agent_metadata/agent_metadata.py index 6c3c20f..a9c6826 100644 --- a/codeguru_profiler_agent/agent_metadata/agent_metadata.py +++ b/codeguru_profiler_agent/agent_metadata/agent_metadata.py @@ -9,7 +9,7 @@ # NOTE: Please do not alter the value for the following constants without the full knowledge of the use of them. # These constants are used in several scripts, including setup.py. __agent_name__ = "CodeGuruProfiler-python" -__agent_version__ = "1.2.4" +__agent_version__ = "1.2.5" def look_up_fleet_info(