Initial implementation of the agent duplicated and adapted from the i…

.github/workflows/python-package.yml

@@ -0,0 +1,58 @@
+# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: Python package
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        python-version: ['3.6', '3.7', '3.8', '3.9']
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install pip
+      run: |
+        python -m pip install --upgrade pip
+    - name: Install dependencies for running tests
+      run: |
+        python -m pip install flake8 pytest pytest-print
+        python -m pip install mock httpretty six pympler
+    - name: Install dependencies for additional checks
+      run: |
+        python -m pip install bandit
+    - name: Install dependencies from requirements
+      run: |
+        pip install -r requirements.txt
+    - name: Lint with flake8
+      run: |
+        # stop the build if there are Python syntax errors or undefined names
+        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+    - name: Run bandit
+      run: |
+        # run bandit to find common security issues
+        bandit -r codeguru_profiler_agent
+    - name: Run a specific test with logs
+      run: |
+        pytest -vv -o log_cli=true test/acceptance/test_live_profiling.py
+    - name: Run tests with pytest
+      run: |
+        pytest -vv
+        # For local testing, you can use pytest-html if you want a generated html report.
+        # python -m pip install pytest-html
+        # pytest -vv --html=pytest-report.html --self-contained-html

.gitignore

@@ -0,0 +1,2 @@
+__pycache__/
+pytest-report.html

CHANGELOG.rst

@@ -0,0 +1,15 @@
+=========
+CHANGELOG
+=========
+
+1.0.1 (layer_v3)
+===================
+
+* Fix bug for running agent in Windows; Update module_path_extractor to support Windows applications
+* Use json library instead of custom encoder for profile encoding for more reliable performance
+* Specify min version for boto3 in setup.py
+
+1.0.0 (layer_v1, layer_v2)
+==========================
+
+* Initial Release

README.md

@@ -1,11 +1,10 @@
-## My Project
+## Amazon CodeGuru Profiler Python Agent
 
-TODO: Fill this README out!
+For more details, check the documentation: https://docs.aws.amazon.com/codeguru/latest/profiler-ug/what-is-codeguru-profiler.html
 
-Be sure to:
+## Release to PyPI
 
-* Change the title in this README
-* Edit your repository description on GitHub
+Use the `setup.py` script to create the archive.
 
 ## Security
 

codeguru_profiler_agent/__init__.py

@@ -0,0 +1,12 @@
+"""
+Modules
+-------
+
+.. automodule:: codeguru_profiler_agent.profiler
+   :members:
+
+"""
+
+from . import *
+from .profiler import Profiler
+from .aws_lambda.profiler_decorator import with_lambda_profiler

codeguru_profiler_agent/__main__.py

@@ -0,0 +1,99 @@
+import os
+import sys
+import runpy
+import logging
+
+profiler = None
+
+
+def _start_profiler(options, env):
+    """
+    This will init the profiler object and start it.
+    :param options: options may contain profiling group name, region or credential profile if they are passed in command
+    :param env: the environment dict from which to search for variables (usually os.environ is passed)
+    :return: the profiler object
+    """
+    from codeguru_profiler_agent.profiler_builder import build_profiler
+    global profiler
+    profiler = build_profiler(pg_name=options.profiling_group_name, region_name=options.region,
+                              credential_profile=options.credential_profile, env=env)
+    if profiler is not None:
+        profiler.start()
+    return profiler
+
+
+def _set_log_level(log_level):
+    if log_level is None:
+        return
+    numeric_level = getattr(logging, log_level.upper(), None)
+    if isinstance(numeric_level, int):
+        logging.basicConfig(level=numeric_level)
+
+
+def main(input_args=sys.argv[1:], env=os.environ, start_profiler=_start_profiler):
+    from argparse import ArgumentParser
+    usage = 'python -m codeguru_profiler_agent [-p profilingGroupName] [-r region] [-c credentialProfileName]' \
+            ' [-m module | scriptfile.py] [arg]' \
+            + '...\nexample: python -m codeguru_profiler_agent -p myProfilingGroup hello_world.py'
+    parser = ArgumentParser(usage=usage)
+    parser.add_argument('-p', '--profiling-group-name', dest="profiling_group_name",
+                        help='Name of the profiling group to send profiles into')
+    parser.add_argument('-r', '--region', dest="region",
+                        help='Region in which you have created your profiling group. e.g. "us-west-2".'
+                             + ' Default depends on your configuration'
+                             + ' (see https://boto3.amazonaws.com/v1/documentation/api/latest/guide/configuration.html)')
+    parser.add_argument('-c', '--credential-profile-name', dest="credential_profile",
+                        help='Name of the profile created in shared credential file used for submitting profiles. '
+                             + '(see https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html#shared-credentials-file)')
+    parser.add_argument('-m', dest='module', action='store_true',
+                        help='Profile a library module', default=False)
+    parser.add_argument('--log', dest='log_level',
+                        help='Set log level, possible values: debug, info, warning, error and critical'
+                             + ' (default is warning)')
+    parser.add_argument('scriptfile')
+
+    (known_args, rest) = parser.parse_known_args(args=input_args)
+
+    # Set the sys arguments to the remaining arguments (the one needed by the client script) if they were set.
+    sys.argv = sys.argv[:1]
+    if len(rest) > 0:
+        sys.argv += rest
+
+    _set_log_level(known_args.log_level)
+
+    if known_args.module:
+        code = "run_module(modname, run_name='__main__')"
+        globs = {
+            'run_module': runpy.run_module,
+            'modname': known_args.scriptfile
+        }
+    else:
+        script_name = known_args.scriptfile
+        sys.path.insert(0, os.path.dirname(script_name))
+        with open(script_name, 'rb') as fp:
+            code = compile(fp.read(), script_name, 'exec')
+        globs = {
+            '__file__': script_name,
+            '__name__': '__main__',
+            '__package__': None,
+            '__cached__': None,
+        }
+
+    # now start and stop profile around executing the user's code
+    if not start_profiler(known_args, env):
+        parser.print_usage()
+    try:
+        # Skip issue reported by Bandit.
+        # Issue: [B102:exec_used] Use of exec detected.
+        # https://bandit.readthedocs.io/en/latest/plugins/b102_exec_used.html
+        # We need exec(..) here to run the code from the customer.
+        # Only the code from the customer's script is executed and only inside the customer's environment,
+        # so the customer's code cannot be altered before it is executed.
+        exec(code, globs, None)  # nosec
+    finally:
+        if profiler is not None:
+            profiler.stop()
+
+
+if __name__ == "__main__":
+    main()

codeguru_profiler_agent/agent_metadata/__init__.py

@@ -0,0 +1 @@
+from . import *

codeguru_profiler_agent/agent_metadata/agent_metadata.py

@@ -0,0 +1,96 @@
+import json
+from platform import python_version
+
+from codeguru_profiler_agent.agent_metadata.aws_ec2_instance import AWSEC2Instance
+from codeguru_profiler_agent.agent_metadata.aws_fargate_task import AWSFargateTask
+from codeguru_profiler_agent.agent_metadata.fleet_info import DefaultFleetInfo
+
+
+# NOTE: Please do not alter the value for the following constants without the full knowledge of the use of them.
+# These constants are used in several scripts, including setup.py.
+__agent_name__ = "CodeGuruProfiler-python"
+__agent_version__ = "1.0.1"
+
+
+def look_up_fleet_info(
+        platform_metadata_fetchers=(
+                AWSEC2Instance.look_up_metadata,
+                AWSFargateTask.look_up_metadata
+        )
+):
+    for metadata_fetcher in platform_metadata_fetchers:
+        fleet_info = metadata_fetcher()
+        if fleet_info is not None:
+            return fleet_info
+
+    return DefaultFleetInfo()
+
+
+class AgentInfo:
+    PYTHON_AGENT = __agent_name__
+    CURRENT_VERSION = __agent_version__
+
+    def __init__(self, agent_type=PYTHON_AGENT, version=CURRENT_VERSION):
+        self.agent_type = agent_type
+        self.version = version
+
+    @classmethod
+    def default_agent_info(cls):
+        return cls()
+
+    def __eq__(self, other):
+        if not isinstance(other, AgentInfo):
+            return False
+
+        return self.agent_type == other.agent_type and self.version == other.version
+
+
+class AgentMetadata:
+    """
+    This is once instantianted in the profiler.py file, marked as environment variable and reused in the other parts.
+    When needed to override for testing other components, update those components to allow a default parameter for
+    agent_metadata, or use the environment["agent_metadata"].
+    """
+    def __init__(self,
+                 fleet_info=None,
+                 agent_info=AgentInfo.default_agent_info(),
+                 runtime_version=python_version()):
+        self._fleet_info = fleet_info
+        self.agent_info = agent_info
+        self.runtime_version = runtime_version
+        self.json_rep = None
+
+    @property
+    def fleet_info(self):
+        if self._fleet_info is None:
+            self._fleet_info = look_up_fleet_info()
+        return self._fleet_info
+
+    def serialize_to_json(self, sample_weight, duration_ms, cpu_time_seconds,
+                          average_num_threads, overhead_ms, memory_usage_mb):
+        """
+        This needs to be compliant with agent profile schema.
+        """
+        if self.json_rep is None:
+            self.json_rep = {
+                "sampleWeights": {
+                    "WALL_TIME": sample_weight
+                },
+                "durationInMs": duration_ms,
+                "fleetInfo": self.fleet_info.serialize_to_map(),
+                "agentInfo": {
+                    "type": self.agent_info.agent_type,
+                    "version": self.agent_info.version
+                },
+                "agentOverhead": {
+                    "memory_usage_mb": memory_usage_mb
+                },
+                "runtimeVersion": self.runtime_version,
+                "cpuTimeInSeconds": cpu_time_seconds,
+                "metrics": {
+                    "numThreads": average_num_threads
+                }
+            }
+            if overhead_ms != 0:
+                self.json_rep["agentOverhead"]["timeInMs"] = overhead_ms
+        return self.json_rep

codeguru_profiler_agent/agent_metadata/aws_ec2_instance.py

@@ -0,0 +1,54 @@
+import logging
+from codeguru_profiler_agent.utils.log_exception import log_exception
+from codeguru_profiler_agent.agent_metadata.fleet_info import FleetInfo, http_get
+
+# Currently, there is not a utility function in boto3 to retrieve the instance metadata; hence we would need
+# get the metadata through URI.
+# See https://github.com/boto/boto3/issues/313 for tracking the work for supporting such function in boto3
+DEFAULT_EC2_METADATA_URI = "http://169.254.169.254/latest/meta-data/"
+EC2_HOST_NAME_URI = DEFAULT_EC2_METADATA_URI + "local-hostname"
+EC2_HOST_INSTANCE_TYPE_URI = DEFAULT_EC2_METADATA_URI + "instance-type"
+
+logger = logging.getLogger(__name__)
+
+class AWSEC2Instance(FleetInfo):
+    """
+    This class will get and parse the EC2 metadata if available.
+    See https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-data-retrieval.html.
+    """
+
+    def __init__(self, host_name, host_type):
+        super().__init__()
+        self.host_name = host_name
+        self.host_type = host_type
+
+    def get_fleet_instance_id(self):
+        return self.host_name
+
+    @classmethod
+    def __look_up_host_name(cls):
+        # The id of the fleet element. Eg. host name in ec2.
+        return http_get(url=EC2_HOST_NAME_URI).read().decode()
+
+    @classmethod
+    def __look_up_instance_type(cls):
+        return http_get(url=EC2_HOST_INSTANCE_TYPE_URI).read().decode()
+
+    @classmethod
+    def look_up_metadata(cls):
+        try:
+            return cls(
+                host_name=cls.__look_up_host_name(),
+                host_type=cls.__look_up_instance_type()
+            )
+        except Exception:
+            log_exception(logger, "Unable to get Ec2 instance metadata, this is normal when running in a different "
+                                  "environment (e.g. Fargate), profiler will still work")
+            return None
+
+    def serialize_to_map(self):
+        return {
+            "computeType": "aws_ec2_instance", 
+            "hostName": self.host_name, 
+            "hostType": self.host_type
+        }