Add RAPL energy measurements feature for the Linux Sensor with Intel …

…hardware
fvaleye · Nov 6, 2022 · 2e6d402 · 2e6d402
1 parent f5da71e
commit 2e6d402
Show file tree

Hide file tree

Showing 24 changed files with 315 additions and 177 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -11,7 +11,7 @@ jobs:
   format:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
 
       - name: Setup Python
         uses: actions/setup-python@v4
@@ -21,11 +21,6 @@ jobs:
       - name: Install Poetry
         uses: snok/install-poetry@v1
 
-      - name: Run pre-commit
-        run: |
-          pip install pre-commit
-          pre-commit run --all-files
-
       - name: Check Python
         run: |
           make init
@@ -45,7 +40,9 @@ jobs:
           - ubuntu-latest
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v1
+      - uses: actions/checkout@v3
+        with:
+          core-protectNTFS: false
 
       - name: Setup Python
         uses: actions/setup-python@v2

diff --git a/README.md b/README.md
@@ -27,11 +27,11 @@ pip install 'tracarbon[datadog]'
 ```
 
 ### 🔌 Devices: energy consumption
-| **Devices** |                                    **Description**                                    |
-|-------------|:-------------------------------------------------------------------------------------:|
-| Mac         |    ✅ Global energy consumption of your Mac (must be plugged into a wall adapter).     |
-| Linux       | ❌ Not yet implemented. See [#184](https://github.com/hubblo-org/scaphandre/pull/184). |
-| Windows     | ❌ Not yet implemented. See [#184](https://github.com/hubblo-org/scaphandre/pull/184). |
+| **Devices** |                                                             **Description**                                                             |
+|-------------|:---------------------------------------------------------------------------------------------------------------------------------------:|
+| Mac         |                             ✅ Global energy consumption of your Mac (must be plugged into a wall adapter).                              |
+| Linux       | ⚠️ Only with [RAPL](https://web.eece.maine.edu/~vweaver/projects/rapl/). See [#1](https://github.com/fvaleye/tracarbon/issues/1). |
+| Windows     |                          ❌ Not yet implemented. See [#184](https://github.com/hubblo-org/scaphandre/pull/184).                          |
 
 | **Cloud Provider** |                                                                                                  **Description**                                                                                                  |
 |--------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:|

diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -18,6 +18,7 @@ python = "^3.7"
 loguru = "^0.6.0"
 aiohttp = "^3.8.1"
 aiocache = "^0.11.1"
+aiofiles = "^22.1.0"
 psutil = "^5.9.1"
 ujson = "^5.3.0"
 msgpack = "^1.0.4"
@@ -32,21 +33,22 @@ python-dotenv = "^0.21.0"
 mypy = "^0.982"
 black = "^22.8.0"
 isort = "^5.10.1"
-pytest = "^7.1.3"
+pytest = "^7.2.0"
 pytest-mock = "^3.7.0"
-pytest-asyncio = "^0.19.0"
+pytest-asyncio = "^0.20.1"
 pytest-cov = "^4.0.0"
-pytest-xdist = "^2.5.0"
+pytest-xdist = "^3.0.2"
 pytest-clarity = "^1.0.1"
 sphinx = "^5.3.0"
-pydata-sphinx-theme = "^0.10.1"
+pydata-sphinx-theme = "^0.11.0"
 toml = "^0.10.2"
 types-ujson = "^5.4.0"
 datadog = "^0.44.0"
 prometheus-client = "^0.15.0"
 types-requests = "^2.28.10"
 bandit = "^1.7.4"
 radon = "^5.1.0"
+types-aiofiles = "^22.1.0"
 
 [tool.poetry.extras]
 datadog = ["datadog"]

diff --git a/tests/carbon_emissions/test_carbon_emissions.py b/tests/carbon_emissions/test_carbon_emissions.py
@@ -1,42 +1,7 @@
-import datetime
-
 import pytest
 
-from tracarbon import CarbonEmission, EnergyConsumption, MacEnergyConsumption
-from tracarbon.locations import Country, Location
-
-
-@pytest.mark.darwin
-def test_carbon_emission_should_convert_watt_hours_to_co2g():
-    co2g_per_kwh = 20.3
-    watt_hours = 10.1
-    carbon_emission = CarbonEmission(
-        location=Country(co2g_per_kwh=co2g_per_kwh),
-    )
-    co2g_expected = 0.20503
-
-    co2g = carbon_emission.co2g_from_watt_hours(
-        watt_hours=watt_hours, co2g_per_kwh=co2g_per_kwh
-    )
-
-    assert co2g == co2g_expected
-
-
-@pytest.mark.darwin
-def test_carbon_emission_should_convert_watt_hours_to_co2g():
-    wattage = 50
-    co2g_per_kwh = 20.3
-    watt_hours_expected = 0.833
-    name_alpha_iso_2 = "fr"
-    carbon_emission = CarbonEmission(
-        location=Country(name=name_alpha_iso_2, co2g_kwh=co2g_per_kwh),
-    )
-    one_minute_ago = datetime.datetime.now() - datetime.timedelta(seconds=60)
-    carbon_emission.previous_energy_consumption_time = one_minute_ago
-
-    watt_hours = carbon_emission.wattage_to_watt_hours(wattage=wattage)
-
-    assert round(watt_hours, 3) == watt_hours_expected
+from tracarbon import CarbonEmission, MacEnergyConsumption, Power
+from tracarbon.locations import Country
 
 
 @pytest.mark.asyncio

diff --git a/tests/exporters/test_promotheus_exporter.py b/tests/exporters/test_promotheus_exporter.py
@@ -1,5 +1,4 @@
 import psutil
-from prometheus_client import REGISTRY, Gauge
 
 from tracarbon import Country, HardwareInfo
 from tracarbon.exporters import Metric, PromotheusExporter, Tag

diff --git a/tests/hardwares/data/intel-rapl/intel-raplT0/energy_uj b/tests/hardwares/data/intel-rapl/intel-raplT0/energy_uj
@@ -0,0 +1 @@
+32000
diff --git a/tests/hardwares/data/intel-rapl/intel-raplT0/intel-raplT0T0/energy_uj b/tests/hardwares/data/intel-rapl/intel-raplT0/intel-raplT0T0/energy_uj
@@ -0,0 +1 @@
+7000
diff --git a/tests/hardwares/data/intel-rapl/intel-raplT0/intel-raplT0T1/energy_uj b/tests/hardwares/data/intel-rapl/intel-raplT0/intel-raplT0T1/energy_uj
@@ -0,0 +1 @@
+40000
diff --git a/tests/hardwares/data/intel-rapl/intel-raplT1/energy_uj b/tests/hardwares/data/intel-rapl/intel-raplT1/energy_uj
@@ -0,0 +1 @@
+20232
diff --git a/tests/hardwares/data/intel-rapl/intel-raplT1/intel-raplT1T0/energy_uj b/tests/hardwares/data/intel-rapl/intel-raplT1/intel-raplT1T0/energy_uj
@@ -0,0 +1 @@
+3
diff --git a/tests/hardwares/data/intel-rapl/intel-raplT1/intel-raplT1T1/energy_uj b/tests/hardwares/data/intel-rapl/intel-raplT1/intel-raplT1T1/energy_uj
@@ -0,0 +1 @@
+2433
diff --git a/tests/hardwares/test_energy.py b/tests/hardwares/test_energy.py
@@ -0,0 +1,37 @@
+import datetime
+
+import pytest
+
+from tracarbon.hardwares import Power
+
+
+def test_power_should_convert_watt_hours_to_co2g():
+    co2g_per_kwh = 20.3
+    watts_hour = 10.1
+    co2g_expected = 0.20503
+
+    co2g = Power.co2g_from_watts_hour(watts_hour=watts_hour, co2g_per_kwh=co2g_per_kwh)
+
+    assert co2g == co2g_expected
+
+
+def test_energy_should_convert_watt_hours_to_co2g():
+    watts = 50
+    watt_hours_expected = 0.833
+    one_minute_ago = datetime.datetime.now() - datetime.timedelta(seconds=60)
+    previous_energy_measurement_time = one_minute_ago
+
+    watt_hours = Power.watts_to_watt_hours(
+        watts=watts, previous_energy_measurement_time=previous_energy_measurement_time
+    )
+
+    assert round(watt_hours, 3) == watt_hours_expected
+
+
+def test_energy_should_convert_watts_from_microjoules():
+    microjoules = 43043430
+    watts_expected = 43.043
+
+    watts = Power.watts_from_microjoules(microjoules=microjoules)
+
+    assert round(watts, 3) == watts_expected
diff --git a/tests/hardwares/test_gpu.py b/tests/hardwares/test_gpu.py
@@ -1,7 +1,7 @@
 import pytest
 
 from tracarbon.exceptions import TracarbonException
-from tracarbon.hardwares.gpu import GPUInfo, NvidiaGPU
+from tracarbon.hardwares.gpu import NvidiaGPU
 
 
 def test_get_nvidia_gpu_power_usage(mocker):

diff --git a/tests/hardwares/test_hardware.py b/tests/hardwares/test_hardware.py
@@ -1,3 +1,4 @@
+import pathlib
 import platform
 
 import psutil
@@ -64,3 +65,26 @@ def test_get_gpu_power_usage_with_no_gpu():
     with pytest.raises(TracarbonException) as exception:
         HardwareInfo.get_gpu_power_usage()
     assert exception.value.args[0] == "No Nvidia GPU detected."
+
+
+def test_is_rapl_compatible(tmpdir):
+    assert HardwareInfo.is_rapl_compatible() is False
+
+    path = tmpdir.mkdir("intel-rapl")
+
+    assert HardwareInfo.is_rapl_compatible(path=path) is True
+
+
+@pytest.mark.asyncio
+@pytest.mark.linux
+@pytest.mark.darwin
+async def test_get_rapl_power_usage():
+    path = f"{pathlib.Path(__file__).parent.resolve()}/data/intel-rapl"
+    expected = 101668.0
+    rapl_separator_for_windows = "T"
+
+    results = await HardwareInfo.get_rapl_power_usage(
+        path=path, rapl_separator=rapl_separator_for_windows
+    )
+
+    assert results == expected
diff --git a/tests/locations/test_country.py b/tests/locations/test_country.py
@@ -35,7 +35,7 @@ async def test_france_location_should_return_latest_known(mocker):
 
 
 @pytest.mark.asyncio
-async def test_france_location_should_return_taux_co2(mocker):
+async def test_france_location_should_return_taux_co2():
     co2_expected = 51.1
     co2signal_api_key = ""
     country = Country(co2signal_api_key=co2signal_api_key, name="fr", co2g_kwh=51.1)

diff --git a/tests/test_sensors.py b/tests/test_sensors.py
@@ -1,14 +1,9 @@
 import pytest
 import requests
 
-from tracarbon import (
-    AWSEC2EnergyConsumption,
-    AWSSensorException,
-    EnergyConsumption,
-    TracarbonException,
-)
+from tracarbon import AWSEC2EnergyConsumption, EnergyConsumption, TracarbonException
 from tracarbon.hardwares import HardwareInfo
-from tracarbon.hardwares.cloud_providers import AWS, CloudProviders
+from tracarbon.hardwares.cloud_providers import AWS
 
 
 @pytest.mark.darwin

diff --git a/tracarbon/emissions/carbon_emissions.py b/tracarbon/emissions/carbon_emissions.py
@@ -3,7 +3,7 @@
 
 from loguru import logger
 
-from tracarbon.hardwares.sensors import EnergyConsumption, Sensor
+from tracarbon.hardwares import EnergyConsumption, Power, Sensor
 from tracarbon.locations import Country
 
 
@@ -15,8 +15,6 @@ class CarbonEmission(Sensor):
     location: Country
     energy_consumption: EnergyConsumption
     previous_energy_consumption_time: Optional[datetime] = None
-    WH_TO_KWH_FACTOR: int = 1000
-    SECONDS_TO_HOURS_FACTOR: int = 3600
 
     def __init__(self, **data: Any) -> None:
         if not "location" in data:
@@ -27,48 +25,22 @@ def __init__(self, **data: Any) -> None:
 
         super().__init__(**data)
 
-    def wattage_to_watt_hours(self, wattage: float) -> float:
-        """
-        Convert wattage in watts to watt-hours W/h.
-
-        :param wattage: the wattage in W
-        :return: watt-hours W/h
-        """
-        now = datetime.now()
-        if self.previous_energy_consumption_time:
-            time_difference_in_seconds = (
-                now - self.previous_energy_consumption_time
-            ).total_seconds()
-        else:
-            time_difference_in_seconds = 1
-        logger.debug(
-            f"Time difference with the previous energy consumption run: {time_difference_in_seconds}s"
-        )
-        self.previous_energy_consumption_time = now
-        return wattage * (time_difference_in_seconds / self.SECONDS_TO_HOURS_FACTOR)
-
-    def co2g_from_watt_hours(self, watt_hours: float, co2g_per_kwh: float) -> float:
-        """
-        Calculate the CO2g generated using watt hours and the CO2g/kwh.
-
-        :param watt_hours: the current wattage
-        :param co2g_per_kwh: the co2g emitted during one kwh
-        :return: the CO2g generated by the energy consumption
-        """
-        return (watt_hours / self.WH_TO_KWH_FACTOR) * co2g_per_kwh
-
     async def run(self) -> float:
         """
         Run the Carbon Emission sensor.
 
         :return: the CO2g/kwh generated.
         """
-        wattage = await self.energy_consumption.run()
-        logger.debug(f"Energy consumption run: {wattage}W")
-        watt_hours = self.wattage_to_watt_hours(wattage=wattage)
-        logger.debug(f"Energy consumption run: {watt_hours}W/h")
+        watts = await self.energy_consumption.run()
+        logger.debug(f"Energy consumption run: {watts}W")
+        watts_hour = Power.watts_to_watt_hours(
+            watts=watts,
+            previous_energy_measurement_time=self.previous_energy_consumption_time,
+        )
+        self.previous_energy_consumption_time = datetime.now()
+        logger.debug(f"Energy consumption run: {watts_hour}W/h")
         co2g_per_kwh = await self.location.get_latest_co2g_kwh()
         logger.debug(f"co2g_per_kwh of the location: {co2g_per_kwh}g CO2 eq/kWh")
-        return self.co2g_from_watt_hours(
-            watt_hours=watt_hours, co2g_per_kwh=co2g_per_kwh
+        return Power.co2g_from_watts_hour(
+            watts_hour=watts_hour, co2g_per_kwh=co2g_per_kwh
         )
diff --git a/tracarbon/exceptions.py b/tracarbon/exceptions.py
@@ -22,6 +22,12 @@ class AWSSensorException(TracarbonException):
     pass
 
 
+class HardwareRAPLException(TracarbonException):
+    """The hardware is not compatible with RAPL."""
+
+    pass
+
+
 class HardwareNoGPUDetectedException(TracarbonException):
     """The hardware does not have a GPU."""
 

diff --git a/tracarbon/exporters/json_exporter.py b/tracarbon/exporters/json_exporter.py
@@ -3,6 +3,7 @@
 from datetime import datetime
 from typing import Any
 
+import aiofiles
 import ujson
 
 from tracarbon.exporters.exporter import Exporter, Metric
@@ -36,22 +37,23 @@ async def launch(self, metric: Metric) -> None:
         :return:
         """
         file_exists = os.path.isfile(self.path)
-        with open(self.path, "a+") as file:
+        async with aiofiles.open(self.path, "a+") as file:
             if file_exists:
-                file.write(f",{os.linesep}")
+                await file.write(f",{os.linesep}")
             else:
-                file.write(f"[{os.linesep}")
-            ujson.dump(
-                {
-                    "timestamp": str(datetime.utcnow()),
-                    "metric_name": metric.format_name(
-                        metric_prefix_name=self.metric_prefix_name
-                    ),
-                    "metric_value": await metric.value(),
-                    "metric_tags": metric.format_tags(),
-                },
-                file,
-                indent=self.indent,
+                await file.write(f"[{os.linesep}")
+            await file.write(
+                ujson.dumps(
+                    {
+                        "timestamp": str(datetime.utcnow()),
+                        "metric_name": metric.format_name(
+                            metric_prefix_name=self.metric_prefix_name
+                        ),
+                        "metric_value": await metric.value(),
+                        "metric_tags": metric.format_tags(),
+                    },
+                    indent=self.indent,
+                )
             )
 
     @classmethod

diff --git a/tracarbon/hardwares/__init__.py b/tracarbon/hardwares/__init__.py
@@ -1,2 +1,3 @@
+from tracarbon.hardwares.energy import *
 from tracarbon.hardwares.hardware import *
 from tracarbon.hardwares.sensors import *