diff --git a/.markdownlint.yml b/.markdownlint.yml index 7e67849..e9dca8f 100644 --- a/.markdownlint.yml +++ b/.markdownlint.yml @@ -9,5 +9,4 @@ MD024: siblings_only: true MD033: false MD041: false -MD046: - style: fenced +MD046: false diff --git a/.vscode/settings.json b/.vscode/settings.json index 4a36493..b92c7f7 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -23,7 +23,8 @@ "yaml.customTags": [ "tag:yaml.org,2002:python/name:material.extensions.emoji.twemoji", "tag:yaml.org,2002:python/name:material.extensions.emoji.to_svg", - "tag:yaml.org,2002:python/name:pymdownx.superfences.fence_code_format" + "tag:yaml.org,2002:python/name:pymdownx.superfences.fence_code_format", + "tag:yaml.org,2002:python/object/apply:pymdownx.slugs.slugify" ], "yaml.schemas": { "diag_master/schema/config.schema.json": "diag_master/config/*.yml" diff --git a/README.md b/README.md index 17c9e4d..63d5882 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ The tools currently handle monitoring - local and remote PTP instances by implementing a PTP Management Client Documentation is available at [**sync-tooling.pages.dev**](https://sync-tooling.pages.dev/) :book: +(log in with your `@tier4.jp` email address). ## Ansible (Recommended) diff --git a/docs/css/mermaid-tweaks.css b/docs/css/mermaid-tweaks.css index 8e94bd3..aad80ba 100644 --- a/docs/css/mermaid-tweaks.css +++ b/docs/css/mermaid-tweaks.css @@ -1,4 +1,9 @@ .mermaid { display: flex; justify-content: center; +} + +.center-contents { + display: flex; + justify-content: center; } \ No newline at end of file diff --git a/docs/img/ptp_architectures.drawio b/docs/img/ptp_architectures.drawio new file mode 100644 index 0000000..203e7bc --- /dev/null +++ b/docs/img/ptp_architectures.drawio @@ -0,0 +1,527 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index bed9e42..5802b73 100644 --- a/docs/index.md +++ b/docs/index.md @@ -10,7 +10,12 @@ Takes the **pain** out of **p**recision time synchroniz**a**t**i**o**n**. * :simple-ros: plug-and-play compatible with ROS 2 `/diagnostics` * :octicons-browser-16: confirm system state live in the browser -Set up SYNC.TOOLING for your system: [Getting Started](getting-started.md). +## :fontawesome-solid-signs-post: First Steps + +* [PTP Architecture Guide](ptp-architecture-guide.md) - Set up PTP in a distributed system +* [Integrators' Guide](integrators-guide.md) - Set up SYNC.TOOLING for a Pilot.Auto based system +* [Installation Guide](installation.md) - Install SYNC.TOOLING manually +* [Usage Guide](usage.md) - Configure SYNC.TOOLING for your system ## :material-tools: SYNC.SETUP diff --git a/docs/getting-started.md b/docs/installation.md similarity index 97% rename from docs/getting-started.md rename to docs/installation.md index 0dcf783..e179137 100644 --- a/docs/getting-started.md +++ b/docs/installation.md @@ -5,11 +5,15 @@ software components necessary: - `diag-worker` which runs on every machine running LinuxPTP and provides updates to `diag-master` +!!! tip + If you are an integrator, or using Pilot.Auto, please refer to the + [Integrators Guide](integrators-guide.md) for automated setup instructions. + ## System Requirements - Ubuntu 22.04 - ROS 2 Humble with Python3.10 (the default version on Ubuntu 22.04) -- All machines are in the same IP subnet and can communicate via ROS 2 pub/sub +- All machines can communicate via ROS 2 pub/sub - `pip` must be available ## Installation diff --git a/docs/integrators-guide.md b/docs/integrators-guide.md index 6a3b8cc..791fb27 100644 --- a/docs/integrators-guide.md +++ b/docs/integrators-guide.md @@ -1,7 +1,69 @@ -This manual is for teams that want to integrate SYNC.DIAG as a part of their vehicle architecture. +This manual is for teams that want to integrate SYNC.DIAG into their vehicle architecture. -While this page covers the general process of integrating SYNC.DIAG, there are also specific -guides for integrating with certain vehicle platforms. +It is assumed that the vehicle is using [Pilot Auto][pilot-auto], is set up using Ansible, +and has access to [Autoware ECU System Setup][autoware-ecu-system-setup] Ansible roles. -!!! warning - This page is under construction. +[pilot-auto]: https://github.com/tier4/pilot-auto +[autoware-ecu-system-setup]: https://github.com/tier4/autoware_ecu_system_setup + + +## Pre-Requisites + +### Setup ECU IP Configuration and PTP + +The ECU, sensor and network architecture has been decided and set up. This includes things like +Netplan configuration and IP address assignments. + +The network architecture has to be compatible with the specific ECUs' and sensors' supported +PTP profiles. + +The [PTP Architecture Guide](ptp-architecture-guide.md) has been followed to set up the +PTP architecture, including all PTP4L and PHC2SYS configuration. + +### Setup Diagnostics in Sensor Driver + +Setup `nebula>=v0.2.8` and sensor configuration to output synchronization meta data on `/sync_diag/graph_updates`. + +Please check the configuration example in [this PR](https://github.com/tier4/aip_launcher/pull/529) + +## SYNC.DIAG + +Once a PTP architecture is set up, SYNC.DIAG can be configured to monitor synchronization +status and publish it to ROS 2. + +### Setup + +Install and configure SYNC.DIAG via the [autoware_ecu_system_setup.sync_tooling][role-sync-tooling] +role on every ECU that participates in PTP synchronization. + +The SYNC.DIAG Master is only required once, and should be run on an ECU that publishes or +processes ROS 2 `/diagnostics`. Usually, the main Autoware ECU is a good choice. + +SYNC.DIAG Workers are required on every ECU that participates in PTP synchronization, including +the one running the SYNC.DIAG Master. Each worker has to be configured with a list of +`ptp4l` and `phc2sys` instances to monitor. + +!!! bug + Again, make sure that, if there are multiple `ptp4l` instances, they have different + UDS addresses set via `--uds_address`. Otherwise, SYNC.TOOLING will not be able to + communicate with them. + +All ECUs running a SYNC.DIAG Worker and/or Master must be able to communicate over ROS 2, as +the `/sync_diag/graph_updates` topic is used to send status updates to the master. + +More information on requirements can be found in the [Manual Installation Guide](installation.md). + +More information on the required configuration file can be found in the [Usage Guide](usage.md). + +[role-sync-tooling]: https://github.com/tier4/autoware_ecu_system_setup/tree/590fabea4f21811a0a69e26793e4fff4f9b60bd1/roles/sync_tooling + +### Running SYNC.DIAG + +SYNC.DIAG is started automatically as a systemd service on every ECU that has been configured. +To make sure it is working correctly, check the output in ROS 2 `/diagnostics`. Also check +that none of the services crash or fail to start. + +### Running SYNC.DOCTOR + +To access the web interface SYNC.DOCTOR, the SYNC.DIAG Master has to be launched with the +`--web-ui` option. This will start a web server on port `5000` of the ECU running the master. diff --git a/docs/integrators-guide/x2gen2.md b/docs/integrators-guide/x2gen2.md deleted file mode 100644 index b2e1800..0000000 --- a/docs/integrators-guide/x2gen2.md +++ /dev/null @@ -1,4 +0,0 @@ -This guide covers the integration of SYNC.DIAG into the X2 gen2 vehicle platform. - -!!! warning - This page is under construction. diff --git a/docs/ptp-architecture-guide.md b/docs/ptp-architecture-guide.md new file mode 100644 index 0000000..40f8436 --- /dev/null +++ b/docs/ptp-architecture-guide.md @@ -0,0 +1,468 @@ +This guide provides an overview of time synchronization in automotive systems, and gives +pointers on how to set up and configure PTP (Precision Time Protocol). + +It is assumed that the vehicle is using [Pilot Auto][pilot-auto], is set up using Ansible, +and has access to [Autoware ECU System Setup][autoware-ecu-system-setup] Ansible roles. + +[pilot-auto]: https://github.com/tier4/pilot-auto +[autoware-ecu-system-setup]: https://github.com/tier4/autoware_ecu_system_setup + +## Overall Goal + +All devices in the vehicle, including all of their clocks if they have multiple, must be +synchronized in order for the system to function correctly. + +Otherwise, timestamps from different sensors might be far apart, and sensor fusion would be +impossible or inaccurate, log entries from different ECUs would be out of order, and so on. + +The overall synchronization architecture is usually tree-shaped: + +* One time source (NTP, GNSS, a dedicated grandmaster clock device) provides current UTC +* One ECU's system clock is synced to the time source +* All other clocks are directly or indirectly synced to that ECU's system clock +* Sensors and other hardware devices are synced via sensor-specific methods + +## Time Source + +To synchronize the vehicle with the outside world, a time source is required. + +Below are the most common options, in order of preference. + +### GNSS + +
+ ![Time Source: GNSS](img/ptp_architectures.drawio) +
+ +The vehicle usually has a GNSS receiver already, and that receiver already reports accurate +time obtained from satellites. To make use of this time, NTP clients like `chrony` can be set +up to sync to the time output by GNSS interface software like `gpsd`. + +See [this guide by Kovasky Buezo](https://kovasky.me/blogs/chronyd/) for setup details. + +### PTP Grandmaster Clock + +
+ ![Time Source: PTP Grandmaster](img/ptp_architectures.drawio) +
+ +This solution relies on a PTP Grandmaster Clock such as +[these](https://www.masterclock.com/master-clock-products.html) and +[these](https://www.microchip.com/en-us/products/clock-and-timing/systems/ptp-grandmaster-clocks). +The advantage is that no special software setup is needed, the clocks act as normal PTP instances +on the network. ECUs can then sync to the grandmaster clock as their PTP master. + +### NTP using `chrony` + +
+ ![Time Source: NTP (chrony)](img/ptp_architectures.drawio) +
+ +This is a simple, and usually accurate-enough solution that requires an internet connection. +The `chrony` software can query multiple NTP (network time protocol) servers, and behaves well +even in unstable networks. + +See the [guide by RedHat][redhat-ntp] for setup details. + +[redhat-ntp]: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/9/html/configuring_basic_system_settings/configuring-time-synchronization_configuring-basic-system-settings#chrony-intro_configuring-time-synchronization + +### NTP using `systemd-timesyncd` + +
+ ![Time Source: NTP (timesyncd)](img/ptp_architectures.drawio) +
+ +A simpler alternative to `chrony` is `systemd-timesyncd`, which is an SNTP (simple network time +protocol) client. It queries only one server and might have less accuracy in unstable networks. +Use this only if none of the other options are viable. + +## Sensor Synchronization + +The time synchronization mechanisms supported vary by sensor type and make, but mostly follow +the below pattern: + +### :fontawesome-solid-ethernet: Ethernet and Base-T1 Sensors + +Most LiDARs and imaging radars fall into this category. + +These usually support at least one PTP profile. See the PTP guide below. + +### :fontawesome-solid-camera: GMSL Sensors + +Most cameras fall into this category. + +These have sensor-specific timing characteristics and usually use trigger signals generated by +an ECU. Refer to the sensor's specific instructions. + +### :fontawesome-solid-microchip: CAN and CAN-FD Sensors + +Most radars, IMUs etc. fall into this category. + +While a standard for time synchronization over CAN(-FD) exists +([AutoSAR TimeSync over CAN](https://www.autosar.org/fileadmin/standards/R21-11/CP/AUTOSAR_SWS_TimeSyncOverCAN.pdf)), +this is not supported by many sensors, and not supported in Linux. For most CAN(-FD) based +sensors, there is no viable option to synchronize time. Confirm via measurements whether there +is a significant or variable time offset. + +## PTP Architecture + +While an exhaustive guidebook on PTP is hard to provide, due to the complexity of the topic, +this guide provides common pointers and pitfalls to avoid when setting up PTP. + +### PTP Software + +There are two common software stacks for PTP on Linux: `linuxptp` and `ptpd`. `linuxptp` +provides a more feature-rich implementation with its `ptp4l` and `phc2sys` programs and is the +recommended choice for most use cases. + +!!! warning + SYNC.TOOLING only supports `linuxptp`. `ptpd` is unsupported. + +!!! tip + Use `linuxptp` unless there is a strong reason not to. + +`ptp4l` participates in PTP synchronization over the network on one or more interfaces. +It is configured with one local clock and will synchronize it to the best available PTP clock +on the network (slave mode) or let other devices synchronize to it (master mode). + +`phc2sys` is used to synchronize multiple clocks on the same ECU, e.g. the system clock and +a network interface's hardware clock. + +Install and configure PTP4L and PHC2SYS via the [autoware_ecu_system_setup.ptp4l][role-ptp4l] +role, or via custom roles. For other setups, see the excellent [RedHat guide][redhat-ptp4l]. + +[redhat-ptp4l]: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/system_administrators_guide/ch-configuring_ptp_using_ptp4l +[role-ptp4l]: https://github.com/tier4/autoware_ecu_system_setup/tree/590fabea4f21811a0a69e26793e4fff4f9b60bd1/roles/ptp4l + +### Time Stamping Modes + +`ptp4l` can be configured with either the `-H` flag for hardware time stamping, or `-S` for +software time stamping. + +Hardware time stamping utilizes the network interface's hardware clock to timestamp packets +exactly when they are sent or received. This is the most accurate method, but requires hardware +support. Additionally, it has to be ensured that the hardware clock and system clock are +synchronized. This is done with `phc2sys`. + +Software time stamping uses the system clock to timestamp packets when they enter or leave the +kernel network stack. This is less accurate, but compatible with almost all hardware. + +To find out what time stamping modes are supported by a network interface (e.g. `eno1`), run + +```bash +ethtool -T eno1 +``` + +Example output: + +
+ + + + + + + + + + + +
Only Software Time Stamping SupportedHardware Time Stamping Supported
+ +```output +Time stamping parameters for eno1: +Capabilities: + software-receive + software-system-clock +PTP Hardware Clock: none +[...] +``` + + + +```output +Time stamping parameters for eno1: +Capabilities: + hardware-transmit + software-transmit + hardware-receive + software-receive + software-system-clock + hardware-raw-clock +PTP Hardware Clock: 0 +[...] +``` + +
+
+ +!!! warning + Some USB Ethernet adapters do not even support software time stamping and cannot be used + for PTP synchronization. + +### PTP Profiles + +There are three main PTP profiles: + +| Profile | Description | +| -------------- | -------------------------------------------------- | +| PTPv2 (1588v2) | The least precise, but most forgiving profile. | +| gPTP (802.1AS) | More precise, but highly restrictive requirements. | +| Automotive | A gPTP profile tuned for automotive use cases. | + +Which one to use depends on hardware support and network architecture. + +!!! bug + Do not mix PTP profiles in the same network segment, as this can lead to synchronization + faults. + +!!! bug + Never launch multiple `ptp4l` or `ptpd` instances on the same network interface, as this + can lead to synchronization faults. + +#### PTPv2 + +This is the most easily deployed profile. It can be used on bus-like network links with many +participants, and can use UDP/IP transport. + +PTPv2 + +* can run over UDPv4 (`-4`), UDPv6 (`-6`), or raw Ethernet/L2 (`-2`). +* supports time stamping in software (`-S`) or hardware (`-H`). +* can operate in either end-to-end (`-E`) or peer-to-peer (`-P`) mode. + +!!! bug + Do not mix software and hardware time stamping between a master and slave, as implicit + UTC-TAI conversions will lead to time offsets of roughly 37 seconds. + +!!! bug + Do not mix end-to-end and peer-to-peer mode in the same network segment, as this can lead + to synchronization faults. + +!!! tip + Use hardware time stamping (`-H`) where possible. Use UDPv4 (`-4`) for easy network setup. + Use peer-to-peer mode (`-P`) for best performance. If there are compatibility issues, + fall back to end-to-end mode (`-E`). + +#### gPTP, Automotive + +These profiles are highly restrictive. Network equipment like switches has to support the +profile, and all network links where the profile is used have to be 1-to-1. + +gPTP and Automotive PTP + +* can only run over raw Ethernet/L2 (`-2`) +* support time stamping in hardware (`-H`) only +* operate in peer-to-peer mode (`-P`) only + +Use these if they are required by the hardware, such as automotive sensors. +PTP4L provides configurations (`-f /path/to/file.conf`) for both profiles. These are typically +located in `/usr/share/doc/linuxptp/configs/`. + +!!! bug + Do not run gPTP or Automotive PTP on network links with more than two participants, + as this will lead to synchronization faults. + +As a case study, X2 gen2 is using a switch that does not support gPTP, so it was necessary +to configure the switch to forward PTP traffic in a way that simulates many 1-to-1 links. +See [X2 gen2 PTP troubleshooting](https://tier4.atlassian.net/wiki/x/jIAvwg) for details. + +### ECU PTP Architecture + +ECUs typically have multiple clocks: one system clock and multiple hardware clocks, usually +one per network card[^hw-clocks]. These have to be synchronized to each other (locally on the +ECU), so be sure to understand which clocks are available, e.g. using the `ethtool -T` command +detailed in the [Time Stamping Modes](#time-stamping-modes) section. + +[^hw-clocks]: Usually, one network interface card (NIC) has one or more hardware clocks (e.g. +`/dev/ptp0`), which are each assigned to one or more network interfaces (e.g. `eno1`). This is +vendor and model specific, so check the documentation of the NIC and use `ethtool -T` to +confirm. + +For every ECU, the system clock and all hardware clocks that are being used for PTP have to +be synchronized to each other. There are multiple patterns to achieve this. + +!!! info + By default, `ptp4l` participates as a server or client automatically based on the PTP peers. + In production, the role of each PTP instance should be explicitly configured, e.g. with the + `--clientOnly=1` and `--serverOnly=1` options (`--slaveOnly=1` and `--masterOnly=1` before + `v4.0`). Ensure that PHC2SYS source (`-s`) and destination (`-c`) clocks are configured + accordingly. + +#### Software vs Hardware Time Stamping + +The simplest architecture is to use software time stamping: + +=== "Manual (Master)" + + ```bash + ptp4l -i eno1 -S --serverOnly=1 + ``` + +=== "Manual (Slave)" + + ```bash + ptp4l -i eno1 -S --clientOnly=1 + ``` + +=== "Ansible (Master)" + + ```yaml + - role: autoware.ecu_system_setup.ptp4l_master + vars: + ptp4l_masters: + - name: eno1 + port_name: eno1 + config_name: tier4_normal_software_ptp.cfg + use_phc2sys: false + ``` + +=== "Ansible (Slave)" + + ```yaml + - role: autoware.ecu_system_setup.ptp4l + vars: + ptp4l_network_port: eno1 + ptp4l_use_software_stamp: true + ptp4l_synchronize_system_clock: false + ``` + +
+ ![HW PTP vs SW PTP: SW](img/ptp_architectures.drawio) +
+ +With hardware time stamping, PHC2SYS is required to synchronize the system and hardware clocks: + +=== "Manual (Master)" + + ```bash + ptp4l -i eno1 -H --serverOnly=1 + phc2sys -c /dev/ptp0 -s CLOCK_REALTIME -O0 + ``` + +=== "Manual (Slave)" + + ```bash + ptp4l -i eno1 -H --clientOnly=1 + phc2sys -c CLOCK_REALTIME -s /dev/ptp0 -O0 -w + ``` + +=== "Ansible (Master)" + + ```yaml + - role: autoware.ecu_system_setup.ptp4l_master + vars: + ptp4l_masters: + - name: eno1 + port_name: eno1 + config_name: tier4_normal_hardware_ptp.cfg + use_phc2sys: true + ``` + +=== "Ansible (Slave)" + + ```yaml + - role: autoware.ecu_system_setup.ptp4l + vars: + ptp4l_network_port: eno1 + ptp4l_use_software_stamp: false + ptp4l_synchronize_system_clock: true + ptp4l_phc2sys_wait_for_ptp: true + ``` + +
+ ![HW PTP vs SW PTP: HW](img/ptp_architectures.drawio) +
+ +#### Number of PTP4L/PHC2SYS Instances + +In general, one `ptp4l` instance can only operate on one clock. It can however operate on multiple +network interfaces, provided that both have the same hardware clock, or that software time +stamping is used. + +A single `phc2sys` instance can synchronize multiple clocks to a common source clock. + +`enp0s0f0` and `enp0s0f1` use the same hardware clock, `ptp0`, so only one `ptp4l` instance is +needed: + +=== "Manual (Master)" + + ```bash + ptp4l -i enp0s0f0 -i enp0s0f1 -H --serverOnly=1 + phc2sys -c /dev/ptp0 -s CLOCK_REALTIME -O0 + ``` + +=== "Ansible (Master)" + + !!! warning + The current Ansible role (`autoware.ecu_system_setup.ptp4l_master`) does not support + multiple interfaces in one `ptp4l` instance. With hardware time stamping, only one + `ptp4l` instance can use a given hardware clock. A custom role is required. + +
+ ![HW PTP: One Instance](img/ptp_architectures.drawio) +
+ +`enp0s0f0` and `enp1s0f0` have different hardware clocks, `ptp0` and `ptp1`, so two `ptp4l` +instances are needed. + +=== "Manual (Master)" + + ```bash + ptp4l -i enp0s0f0 -H --serverOnly=1 --uds_address /var/run/ptp4l@enp0s0f0 + ptp4l -i enp1s0f0 -H --serverOnly=1 --uds_address /var/run/ptp4l@enp1s0f0 + phc2sys -c /dev/ptp0 -c /dev/ptp1 -s CLOCK_REALTIME -O0 + ``` + +=== "Ansible (Master)" + + !!! info + The current Ansible role (`autoware.ecu_system_setup.ptp4l_master`) will create one + `phc2sys` instance per `ptp4l` instance, so this scenario is not supported. Instead, + see the scenario with multiple `phc2sys` instances below. + +
+ ![HW PTP: Multi Instance](img/ptp_architectures.drawio) +
+ +If it is undesirable to run only one `phc2sys` instance, e.g. due to different synchronization +settings for different clocks, multiple `phc2sys` instances can be used: + +=== "Manual (Master)" + + ```bash + ptp4l -i enp0s0f0 -H --serverOnly=1 --uds_address /var/run/ptp4l@enp0s0f0 + ptp4l -i enp1s0f0 -H --serverOnly=1 --uds_address /var/run/ptp4l@enp1s0f0 + phc2sys -c /dev/ptp0 -s CLOCK_REALTIME -O0 + phc2sys -c /dev/ptp1 -s CLOCK_REALTIME -O0 + ``` + +=== "Ansible (Master)" + + ```yaml + - role: autoware.ecu_system_setup.ptp4l_master + vars: + ptp4l_masters: + - name: enp0s0f0 + port_name: enp0s0f0 + config_name: tier4_normal_hardware_ptp.cfg + use_phc2sys: true + - name: enp1s0f0 + port_name: enp1s0f0 + config_name: tier4_normal_hardware_ptp.cfg + use_phc2sys: true + ``` + + !!! warning + The current Ansible role (`autoware.ecu_system_setup.ptp4l_master`) does not set + different UDS addresses for the `ptp4l` instances, so this role needs to be modified + to support this scenario. + +
+ ![HW PTP: Multi PHC2SYS](img/ptp_architectures.drawio) +
+ +!!! bug + When running multiple `ptp4l` instances, their local Unix Domain Sockets (UDS) must be + configured to be different, by applying the `--uds_address /var/run/ptp4l@` + option. Otherwise, PHC2SYS and SYNC.TOOLING will not be able to communicate with them. diff --git a/docs/usage.md b/docs/usage.md new file mode 100644 index 0000000..5bf2670 --- /dev/null +++ b/docs/usage.md @@ -0,0 +1,156 @@ +In case the software was installed via Ansible, the command lines of these programs have already +been configured in the generated systemd service files. + +## `diag-worker` command + +See available command line arguments with + +```shell +uv run diag-worker --help +``` + +By default, `diag-worker` will publish updates to the ROS 2 topic`/sync_diag/graph_updates`. +The monitored `ptp4l` and `phc2sys` units have to be specified with the `--ptp4l-units` and +`--phc2sys-units` arguments, respectively. + +!!! warning + There can only be one `diag-worker` per machine. + +!!! warning + If there are multiple `ptp4l` units on the same machine, their `uds_address`es must be + unique. We recommend, that for unit `ptp4l@xyz` the `uds_address` is set to + `/var/run/ptp4l@xyz`. Set either via `ptp4l`'s `uds_address` config option or via its + `--uds_address` command line argument. + +## `diag-master` command + +See available command line arguments with + +```shell +uv run diag-master --help +``` + +By default, `diag-master` will listen to updates from workers on the ROS 2 topic +`/sync_diag/graph_updates` and publish diagnostics to `/diagnostics`. +The web interface is launched on `0.0.0.0:5000` if launched with the `--web-ui` option. + +!!! warning + There can only be one `diag-master` per machine. In general, only one is needed per + distributed system (i.e. per vehicle). + +### Configuration + +A configuration file must be specified with the `--config` option. +This file contains the reference graph of the system, along with diagnostic thresholds and other +options. + +A reference graph is a tree of clocks, with each parent having a direct PTP or PHC2SYS link to +each of its children. Using the reference graph, SYNC.TOOLING can confirm if all clocks are +synchronized in the way the user intended. + +The config is specified in YAML format. An example is given below: + +```yaml +clock_tree: # (1)! + main_ecu.sys: # (2)! + main_ecu.ptp0: # (3)! + sensing_ecu.ptp0: + sensing_ecu.sys: # (4)! + sensor@lidar/left: # (5)! + sensor@lidar/right: # (6)! + main_ecu.ptp1: + sensor@radar/front: + sensor@radar/rear: +diagnostics: + diff_thresholds: # (7)! + ptp_master: # (8)! + warn: 100 + error: 1000 + unit: us + phc2sys: # (9)! + warn: 100 + error: 1000 + unit: us + measurement: # (10)! + warn: 1 + error: 20 + unit: ms +``` + +1. The root of the clock tree (= the reference graph) is the grandmaster clock of the system. +2. The reference graph has to be located under the `clock_tree` key. +3. The pattern `.ptp` is used to identify a hardware clock device of an ECU. +4. The pattern `.sys` is used to identify the system clock of an ECU. +5. The pattern `sensor@` is used to identify a sensor. +6. Even entries with no children need a `:` at the end. +7. These are customizable thresholds to diagnose reported time differences. +8. For a PTP link, the PTP master reports the time difference to a child clock, as calculated + by PTP. This is usually very accurate and low-variance. +9. For a PHC2SYS link, PHC2SYS reports the time difference between source and child clock. + This is usually very accurate and low-variance. +10. For a measurement link, the time difference is measured by SYNC.TOOLING. This is usually + less accurate and more variable than the other two. + +!!! warning + **Handling software time stamping** + + If a client-ECU is synchronizing in PTP4l using software time stamping with `-S`, + the tree structure would look like: (but check the [warning on mixing hardware-software stamping](./ptp-architecture-guide.md/#ptpv2)) + + ``` + main_ecu.ptp0: + client_ecu.sys + ``` + + **Handling hardware time stamping** + + The typical structure would be + + ``` + main_ecu.ptp0: + client_ecu.ptp1: + client_ecu.sys: + ``` + + **Hardware PTP clock Index** + + To determine what is the index `` for the pattern `.ptp` in hardware PTP, + find the clock e.g. using `ethtool -T `. E.g., if `ethtool -T eno1` prints + `[...] PTP Hardware Clock: 0 [...]`, the clock is `ptp0`. + +!!! tip + Refer to the + [example configuration](https://github.com/tier4/sync_tooling/blob/main/diag_master/config/example.yml) + for the most up-to-date information. + +## SYNC.DIAG + +The `diag-master` publishes diagnostics to the ROS 2 topic `/diagnostics`. +We label this functionality as SYNC.DIAG. + +This functionality is only fully enabled if a `--reference` graph has been provided to the +`diag-master`. + +## SYNC.DOCTOR + +The `diag-master` provides a web interface named SYNC.DOCTOR. +This interface allows for live or offline viewing of the synchronization state of the system. + +## Offline Analysis + +To record synchronization state for later analysis, run: + +```shell +ros2 bag record /sync_diag/graph_updates +``` + +The expected data rate is about `1 kB/s` per `ptp4l`, `phc2sys` or sensor instance. +For an architecture like X2 gen2, this results in a data rate of about `14 kB/s`. + +The bag can be replayed at a later time on any machine with only a `diag-master` running: + +```shell +ros2 bag play +``` + +Both SYNC.DIAG and SYNC.DOCTOR will respond live to the replay. diff --git a/mkdocs.yml b/mkdocs.yml index 27954fe..ae7c940 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -4,6 +4,7 @@ theme: name: material features: - content.code.annotate + - content.tabs.link palette: - scheme: default primary: white @@ -17,15 +18,21 @@ theme: name: Switch to light mode nav: - Home: index.md - - Getting Started: getting-started.md - - Integrators' Guide: - - integrators-guide.md - - X2 gen2: integrators-guide/x2gen2.md - Design: design.md + - Installation: installation.md + - Usage: usage.md + - PTP Architecture Guide: ptp-architecture-guide.md + - Integrators' Guide: integrators-guide.md markdown_extensions: - attr_list + - md_in_html - footnotes - admonition + - pymdownx.tabbed: + alternate_style: true + slugify: !!python/object/apply:pymdownx.slugs.slugify + kwds: + case: lower - pymdownx.emoji: emoji_index: !!python/name:material.extensions.emoji.twemoji emoji_generator: !!python/name:material.extensions.emoji.to_svg @@ -34,7 +41,6 @@ markdown_extensions: dedent_subsections: true - pymdownx.highlight: anchor_linenums: true - - pymdownx.superfences - pymdownx.tasklist: custom_checkbox: true - pymdownx.superfences: