From 5db0a2779a7133840515a00858d374b1b727330a Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Mon, 22 Jul 2024 23:01:28 +0200 Subject: [PATCH 1/6] allow manual override of system type (#1) https://community.platformio.org/t/windows-on-arm64-problem-installing-xtensa-toolchain/25497 Co-authored-by: Chris <52449218+shadow578@users.noreply.github.com> --- platformio/util.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/platformio/util.py b/platformio/util.py index 54db102f55..ae4dfe90d7 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -15,6 +15,7 @@ import datetime import functools import math +import os import platform import re import shutil @@ -136,6 +137,11 @@ def get_instance(*args, **kwargs): def get_systype(): + # allow manual override, eg. for + # windows on arm64 systems with emulated x86 + if "PLATFORMIO_SYSTEM_TYPE" in os.environ: + return os.environ.get("PLATFORMIO_SYSTEM_TYPE") + system = platform.system().lower() arch = platform.machine().lower() if system == "windows": From eb6140926da312865339ecc11f521c7b5e59a45a Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Mon, 22 Jul 2024 23:48:17 +0200 Subject: [PATCH 2/6] remove telemetry --- README.rst | 8 +- platformio/__main__.py | 2 - platformio/app.py | 7 - platformio/debug/process/gdb.py | 5 - platformio/maintenance.py | 20 +- platformio/platform/_run.py | 3 +- platformio/project/helpers.py | 2 - platformio/remote/client/base.py | 2 - platformio/telemetry.py | 380 ------------------------------- 9 files changed, 5 insertions(+), 424 deletions(-) delete mode 100644 platformio/telemetry.py diff --git a/README.rst b/README.rst index df73525eab..b2baedcd26 100644 --- a/README.rst +++ b/README.rst @@ -82,13 +82,11 @@ Contributing See `contributing guidelines `_. -Telemetry / Privacy Policy --------------------------- +Telemetry +--------- -Share minimal diagnostics and usage information to help us make PlatformIO better. -It is enabled by default. For more information see: +Removed -* `Telemetry Setting `_ License ------- diff --git a/platformio/__main__.py b/platformio/__main__.py index 58cabe8b66..3b3430c1d8 100644 --- a/platformio/__main__.py +++ b/platformio/__main__.py @@ -106,7 +106,6 @@ def main(argv=None): exit_code = int(exc.code) except Exception as exc: # pylint: disable=broad-except if not isinstance(exc, exception.ReturnErrorCode): - maintenance.on_platformio_exception(exc) error_str = f"{exc.__class__.__name__}: " if isinstance(exc, exception.PlatformioException): error_str += str(exc) @@ -131,7 +130,6 @@ def main(argv=None): click.secho(error_str, fg="red", err=True) exit_code = int(str(exc)) if str(exc).isdigit() else 1 - maintenance.on_platformio_exit() sys.argv = prev_sys_argv return exit_code diff --git a/platformio/app.py b/platformio/app.py index 7d2f1ea9b7..5bb02d6a3f 100644 --- a/platformio/app.py +++ b/platformio/app.py @@ -46,10 +46,6 @@ def projects_dir_validate(projects_dir): "description": "Enable caching for HTTP API requests", "value": True, }, - "enable_telemetry": { - "description": ("Telemetry service (Yes/No)"), - "value": True, - }, "force_verbose": { "description": "Force verbose output when processing environments", "value": False, @@ -69,7 +65,6 @@ def projects_dir_validate(projects_dir): "command_ctx": None, "caller_id": None, "custom_project_conf": None, - "pause_telemetry": False, } @@ -274,8 +269,6 @@ def get_user_agent(): data.append("IDE/%s" % os.getenv("PLATFORMIO_IDE")) data.append("Python/%s" % platform.python_version()) data.append("Platform/%s" % platform.platform()) - if not get_setting("enable_telemetry"): - data.append("Telemetry/0") return " ".join(data) diff --git a/platformio/debug/process/gdb.py b/platformio/debug/process/gdb.py index 29a8ba3a23..7859531e68 100644 --- a/platformio/debug/process/gdb.py +++ b/platformio/debug/process/gdb.py @@ -16,7 +16,6 @@ import signal import time -from platformio import telemetry from platformio.compat import aio_get_running_loop, is_bytes from platformio.debug import helpers from platformio.debug.exception import DebugInitError @@ -130,7 +129,6 @@ def stdout_data_received(self, data): self._handle_error(data) # go to init break automatically if self.INIT_COMPLETED_BANNER.encode() in data: - telemetry.log_debug_started(self.debug_config) self._auto_exec_continue() def console_log(self, msg): @@ -175,7 +173,4 @@ def _handle_error(self, data): and b"Error in sourced" in self._errors_buffer ): return - telemetry.log_debug_exception( - DebugInitError(self._errors_buffer.decode()), self.debug_config - ) self.transport.close() diff --git a/platformio/maintenance.py b/platformio/maintenance.py index 45b4e5c95c..82e8509b3a 100644 --- a/platformio/maintenance.py +++ b/platformio/maintenance.py @@ -19,7 +19,7 @@ import click import semantic_version -from platformio import __version__, app, exception, fs, telemetry +from platformio import __version__, app, exception, fs from platformio.cache import cleanup_content_cache from platformio.cli import PlatformioCLI from platformio.commands.upgrade import get_latest_version @@ -32,7 +32,6 @@ def on_cmd_start(ctx, caller): app.set_session_var("command_ctx", ctx) set_caller(caller) - telemetry.on_cmd_start(ctx) if PlatformioCLI.in_silence(): return after_upgrade(ctx) @@ -57,14 +56,6 @@ def on_cmd_end(): ) -def on_platformio_exception(exc): - telemetry.log_exception(exc) - - -def on_platformio_exit(): - telemetry.on_exit() - - def set_caller(caller=None): caller = caller or os.getenv("PLATFORMIO_CALLER") if not caller: @@ -103,7 +94,6 @@ def _appstate_migration(_): state_path = app.resolve_state_path("core_dir", "appstate.json") if not os.path.isfile(state_path): return True - app.delete_state_item("telemetry") created_at = app.get_state_item("created_at", None) if not created_at: state_stat = os.stat(state_path) @@ -160,14 +150,6 @@ def after_upgrade(ctx): "PlatformIO has been successfully upgraded to %s!\n" % __version__, fg="green", ) - telemetry.log_event( - "pio_upgrade_core", - { - "label": "%s > %s" % (last_version_str, __version__), - "from_version": last_version_str, - "to_version": __version__, - }, - ) return print_welcome_banner() diff --git a/platformio/platform/_run.py b/platformio/platform/_run.py index 2912371dc9..1ff3822ea2 100644 --- a/platformio/platform/_run.py +++ b/platformio/platform/_run.py @@ -21,7 +21,7 @@ import click -from platformio import app, fs, proc, telemetry +from platformio import app, fs, proc from platformio.compat import hashlib_encode_data from platformio.package.manager.core import get_core_package_dir from platformio.platform.exception import BuildScriptNotFound @@ -60,7 +60,6 @@ def run( # pylint: disable=too-many-arguments if not os.path.isfile(variables["build_script"]): raise BuildScriptNotFound(variables["build_script"]) - telemetry.log_platform_run(self, self.config, variables["pioenv"], targets) result = self._run_scons(variables, targets, jobs) assert "returncode" in result diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index 4d153492e2..b9a6acafb8 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -174,9 +174,7 @@ def _load_build_metadata(project_dir, env_names, build_type=None): # args.extend(["--target", "__test"]) for name in env_names: args.extend(["-e", name]) - app.set_session_var("pause_telemetry", True) result = CliRunner().invoke(cmd_run, args) - app.set_session_var("pause_telemetry", False) if result.exit_code != 0 and not isinstance( result.exception, exception.ReturnErrorCode ): diff --git a/platformio/remote/client/base.py b/platformio/remote/client/base.py index cf5c240537..9e4ba22ec2 100644 --- a/platformio/remote/client/base.py +++ b/platformio/remote/client/base.py @@ -186,7 +186,5 @@ def cb_global_error(self, err): "a remote machine using `pio remote agent start` command.\n" "See http://docs.platformio.org/page/plus/pio-remote.html" ) - else: - maintenance.on_platformio_exception(Exception(err.type)) click.secho(msg, fg="red", err=True) self.disconnect(exit_code=1) diff --git a/platformio/telemetry.py b/platformio/telemetry.py deleted file mode 100644 index 5b0734c9a2..0000000000 --- a/platformio/telemetry.py +++ /dev/null @@ -1,380 +0,0 @@ -# Copyright (c) 2014-present PlatformIO -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import atexit -import os -import queue -import re -import sys -import threading -import time -import traceback -from collections import deque - -import requests - -from platformio import __title__, __version__, app, exception, fs, util -from platformio.cli import PlatformioCLI -from platformio.debug.config.base import DebugConfigBase -from platformio.http import HTTPSession -from platformio.proc import is_ci - -KEEP_MAX_REPORTS = 100 -SEND_MAX_EVENTS = 25 - - -class MeasurementProtocol: - def __init__(self, events=None): - self.client_id = app.get_cid() - self._events = events or [] - self._user_properties = {} - - self.set_user_property("systype", util.get_systype()) - created_at = app.get_state_item("created_at", None) - if created_at: - self.set_user_property("created_at", int(created_at)) - - @staticmethod - def event_to_dict(name, params, timestamp=None): - event = {"name": name, "params": params} - if timestamp is not None: - event["timestamp"] = timestamp - return event - - def set_user_property(self, name, value): - self._user_properties[name] = value - - def add_event(self, name, params): - self._events.append(self.event_to_dict(name, params)) - - def to_payload(self): - return { - "client_id": self.client_id, - "user_properties": self._user_properties, - "events": self._events, - } - - -@util.singleton -class TelemetryLogger: - def __init__(self): - self._events = deque() - - self._sender_thread = None - self._sender_queue = queue.Queue() - self._sender_terminated = False - - self._http_session = HTTPSession() - self._http_offline = False - - def close(self): - self._http_session.close() - - def log_event(self, name, params, timestamp=None, instant_sending=False): - if not app.get_setting("enable_telemetry") or app.get_session_var( - "pause_telemetry" - ): - return None - timestamp = timestamp or int(time.time()) - self._events.append( - MeasurementProtocol.event_to_dict(name, params, timestamp=timestamp) - ) - if self._http_offline: # if network is off-line - return False - if instant_sending: - self.send() - return True - - def send(self): - if not self._events or self._sender_terminated: - return - if not self._sender_thread: - self._sender_thread = threading.Thread( - target=self._sender_worker, daemon=True - ) - self._sender_thread.start() - while self._events: - events = [] - try: - while len(events) < SEND_MAX_EVENTS: - events.append(self._events.popleft()) - except IndexError: - pass - self._sender_queue.put(events) - - def _sender_worker(self): - while True: - if self._sender_terminated: - return - try: - events = self._sender_queue.get() - if not self._commit_events(events): - self._events.extend(events) - self._sender_queue.task_done() - except (queue.Empty, ValueError): - pass - - def _commit_events(self, events): - if self._http_offline: - return False - mp = MeasurementProtocol(events) - payload = mp.to_payload() - # print("_commit_payload", payload) - try: - r = self._http_session.post( - "https://collector.platformio.org/collect", - json=payload, - timeout=(2, 5), # connect, read - ) - r.raise_for_status() - return True - except requests.exceptions.HTTPError as exc: - # skip Bad Request - if exc.response.status_code >= 400 and exc.response.status_code < 500: - return True - except: # pylint: disable=bare-except - pass - self._http_offline = True - return False - - def terminate_sender(self): - self._sender_terminated = True - - def is_sending(self): - return self._sender_queue.unfinished_tasks - - def get_unsent_events(self): - result = list(self._events) - try: - while True: - result.extend(self._sender_queue.get_nowait()) - except queue.Empty: - pass - return result - - -def log_event(name, params, instant_sending=False): - TelemetryLogger().log_event(name, params, instant_sending=instant_sending) - - -def on_cmd_start(cmd_ctx): - process_postponed_logs() - log_command(cmd_ctx) - - -def on_exit(): - TelemetryLogger().send() - - -def log_command(ctx): - params = { - "path_args": PlatformioCLI.reveal_cmd_path_args(ctx), - } - if is_ci(): - params["ci_actor"] = resolve_ci_actor() or "Unknown" - log_event("cmd_run", params) - - -def resolve_ci_actor(): - known_cis = ( - "GITHUB_ACTIONS", - "TRAVIS", - "APPVEYOR", - "GITLAB_CI", - "CIRCLECI", - "SHIPPABLE", - "DRONE", - ) - for name in known_cis: - if os.getenv(name, "false").lower() == "true": - return name - return None - - -def dump_project_env_params(config, env, platform): - non_sensitive_data = [ - "platform", - "framework", - "board", - "upload_protocol", - "check_tool", - "debug_tool", - "test_framework", - ] - section = f"env:{env}" - params = { - option: config.get(section, option) - for option in non_sensitive_data - if config.has_option(section, option) - } - params["pid"] = app.get_project_id(os.path.dirname(config.path)) - params["platform_name"] = platform.name - params["platform_version"] = platform.version - return params - - -def log_platform_run(platform, project_config, project_env, targets=None): - params = dump_project_env_params(project_config, project_env, platform) - if targets: - params["targets"] = targets - log_event("platform_run", params, instant_sending=True) - - -def log_exception(exc): - skip_conditions = [ - isinstance(exc, cls) - for cls in ( - IOError, - exception.ReturnErrorCode, - exception.UserSideException, - ) - ] - skip_conditions.append(not isinstance(exc, Exception)) - if any(skip_conditions): - return - is_fatal = any( - [ - not isinstance(exc, exception.PlatformioException), - "Error" in exc.__class__.__name__, - ] - ) - - def _strip_module_path(match): - module_path = match.group(1).replace(fs.get_source_dir() + os.sep, "") - sp_folder_name = "site-packages" - sp_pos = module_path.find(sp_folder_name) - if sp_pos != -1: - module_path = module_path[sp_pos + len(sp_folder_name) + 1 :] - module_path = fs.to_unix_path(module_path) - return f'File "{module_path}",' - - trace = re.sub( - r'File "([^"]+)",', - _strip_module_path, - traceback.format_exc(), - flags=re.MULTILINE, - ) - - params = { - "name": exc.__class__.__name__, - "description": str(exc), - "traceback": trace, - "cmd_args": sys.argv[1:], - "is_fatal": is_fatal, - } - log_event("exception", params) - - -def log_debug_started(debug_config: DebugConfigBase): - log_event( - "debug_started", - dump_project_env_params( - debug_config.project_config, debug_config.env_name, debug_config.platform - ), - ) - - -def log_debug_exception(exc, debug_config: DebugConfigBase): - # cleanup sensitive information, such as paths - description = fs.to_unix_path(str(exc)) - description = re.sub( - r'(^|\s+|")(?:[a-z]\:)?((/[^"/]+)+)(\s+|"|$)', - lambda m: " %s " % os.path.join(*m.group(2).split("/")[-2:]), - description, - re.I | re.M, - ) - params = { - "name": exc.__class__.__name__, - "description": description.strip(), - } - params.update( - dump_project_env_params( - debug_config.project_config, debug_config.env_name, debug_config.platform - ) - ) - log_event("debug_exception", params) - - -@atexit.register -def _finalize(): - timeout = 1000 # msec - elapsed = 0 - telemetry = TelemetryLogger() - telemetry.terminate_sender() - try: - while elapsed < timeout: - if not telemetry.is_sending(): - break - time.sleep(0.2) - elapsed += 200 - except KeyboardInterrupt: - pass - postpone_events(telemetry.get_unsent_events()) - telemetry.close() - - -def load_postponed_events(): - state_path = app.resolve_state_path( - "cache_dir", "telemetry.json", ensure_dir_exists=False - ) - if not os.path.isfile(state_path): - return [] - with app.State(state_path) as state: - return state.get("events", []) - - -def save_postponed_events(events): - state_path = app.resolve_state_path("cache_dir", "telemetry.json") - if not events: - try: - if os.path.isfile(state_path): - os.remove(state_path) - except: # pylint: disable=bare-except - pass - return None - with app.State(state_path, lock=True) as state: - state["events"] = events - state.modified = True - return True - - -def postpone_events(events): - if not events: - return None - postponed_events = load_postponed_events() or [] - timestamp = int(time.time()) - for event in events: - if "timestamp" not in event: - event["timestamp"] = timestamp - postponed_events.append(event) - save_postponed_events(postponed_events[KEEP_MAX_REPORTS * -1 :]) - return True - - -def process_postponed_logs(): - events = load_postponed_events() - if not events: - return None - save_postponed_events([]) # clean - telemetry = TelemetryLogger() - for event in events: - if set(["name", "params", "timestamp"]) <= set(event.keys()): - telemetry.log_event( - event["name"], - event["params"], - timestamp=event["timestamp"], - instant_sending=False, - ) - telemetry.send() - return True From 1140331278c5d5c8287fcc3c3b58bb643805390a Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Tue, 23 Jul 2024 22:51:44 +0200 Subject: [PATCH 3/6] rm whitespaces --- platformio/util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/platformio/util.py b/platformio/util.py index ae4dfe90d7..3f8421c352 100644 --- a/platformio/util.py +++ b/platformio/util.py @@ -137,11 +137,11 @@ def get_instance(*args, **kwargs): def get_systype(): - # allow manual override, eg. for + # allow manual override, eg. for # windows on arm64 systems with emulated x86 if "PLATFORMIO_SYSTEM_TYPE" in os.environ: return os.environ.get("PLATFORMIO_SYSTEM_TYPE") - + system = platform.system().lower() arch = platform.machine().lower() if system == "windows": From d92dc19426f79cfad3a862838ceeaffe17ee9d56 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Tue, 23 Jul 2024 22:54:19 +0200 Subject: [PATCH 4/6] Update gdb.py --- platformio/debug/process/gdb.py | 1 - 1 file changed, 1 deletion(-) diff --git a/platformio/debug/process/gdb.py b/platformio/debug/process/gdb.py index 7859531e68..eef2ee2e26 100644 --- a/platformio/debug/process/gdb.py +++ b/platformio/debug/process/gdb.py @@ -18,7 +18,6 @@ from platformio.compat import aio_get_running_loop, is_bytes from platformio.debug import helpers -from platformio.debug.exception import DebugInitError from platformio.debug.process.client import DebugClientProcess From 302bf49101df67c4dff2b67311fa5bb93e60f280 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Tue, 23 Jul 2024 22:55:56 +0200 Subject: [PATCH 5/6] Update helpers.py --- platformio/project/helpers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/platformio/project/helpers.py b/platformio/project/helpers.py index b9a6acafb8..df46daad8e 100644 --- a/platformio/project/helpers.py +++ b/platformio/project/helpers.py @@ -164,7 +164,6 @@ def load_build_metadata(project_dir, env_or_envs, cache=False, build_type=None): def _load_build_metadata(project_dir, env_names, build_type=None): # pylint: disable=import-outside-toplevel - from platformio import app from platformio.run.cli import cli as cmd_run args = ["--project-dir", project_dir, "--target", "__idedata"] From 44d48b9a83477086da50f4f03dd961af09928462 Mon Sep 17 00:00:00 2001 From: Jason2866 <24528715+Jason2866@users.noreply.github.com> Date: Tue, 23 Jul 2024 22:57:58 +0200 Subject: [PATCH 6/6] Update base.py --- platformio/remote/client/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/platformio/remote/client/base.py b/platformio/remote/client/base.py index 9e4ba22ec2..f2a0c645ec 100644 --- a/platformio/remote/client/base.py +++ b/platformio/remote/client/base.py @@ -25,7 +25,7 @@ from twisted.spread import pb # pylint: disable=import-error from zope.interface import provider # pylint: disable=import-error -from platformio import __pioremote_endpoint__, __version__, app, exception, maintenance +from platformio import __pioremote_endpoint__, __version__, app, exception from platformio.remote.factory.client import RemoteClientFactory from platformio.remote.factory.ssl import SSLContextFactory