-
-
Notifications
You must be signed in to change notification settings - Fork 50
/
0612-libxl-add-pcidevs-to-stubdomain-earlier.patch
173 lines (150 loc) · 6.86 KB
/
0612-libxl-add-pcidevs-to-stubdomain-earlier.patch
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
From af9bed0e0e40d28d8827d04bed64c554421456e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Marek=20Marczykowski-G=C3=B3recki?=
<marmarek@invisiblethingslab.com>
Date: Thu, 30 Dec 2021 21:12:00 +0100
Subject: [PATCH 12/26] libxl: add pcidevs to stubdomain earlier
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
When stubdomain is constructed, PCI devices were added during qemu
startup (via hotplug path). While it makes the code (slightly) simpler,
as it's more similar to no-stubdomain case, it also causes problems:
- race condition during stubdomain startup (QEMU trying to access sysfs
entries before xen-pcifront materialize them): QubesOS/qubes-issues#6203
- absent 0000:00:00.0 (host bridge) device during QEMU startup, which
breaks `igd-passthru=on` case: QubesOS/qubes-issues#7164
While the first issue has alternative solution (retrying on failure),
the second does not. In this case, QEMU tries to copy few parameters
from the host bridge (apparently necessary for some drivers, although
doesn't seem to affect Linux), even though only IGD is assigned. While
0000:00:00.0 in (PV) stubdomain is not even a host bridge, it is the
Intel graphics card (when it's assigned) and share the same values in
config space offsets that QEMU reads (see hw/pci-host/xen_igd_pt.c in
QEMU).
So, to make QEMU happy and fix both issues at the same time, assign PCI
devices before unpausing the stubdomain. This requires rather careful
handling, as some steps need to be done only once (assigning to
xen-pciback for example).
Signed-off-by: Marek Marczykowski-Górecki <marmarek@invisiblethingslab.com>
---
tools/libs/light/libxl_dm.c | 34 +++++++++++++++++++++++++++++--
tools/libs/light/libxl_internal.h | 3 +++
tools/libs/light/libxl_pci.c | 20 ++++++++++--------
3 files changed, 46 insertions(+), 11 deletions(-)
diff --git a/tools/libs/light/libxl_dm.c b/tools/libs/light/libxl_dm.c
index 98b900bd7b9d..f5d930bb94f0 100644
--- a/tools/libs/light/libxl_dm.c
+++ b/tools/libs/light/libxl_dm.c
@@ -2233,6 +2233,9 @@ static void spawn_stubdom_pvqemu_cb(libxl__egc *egc,
libxl__dm_spawn_state *stubdom_dmss,
int rc);
+static void spawn_stub_pcis_dm(libxl__egc *egc,
+ libxl__multidev *multidev, int ret);
+
static void spawn_stub_launch_dm(libxl__egc *egc,
libxl__multidev *aodevs, int ret);
@@ -2438,7 +2441,7 @@ retry_transaction:
goto retry_transaction;
libxl__multidev_begin(ao, &sdss->multidev);
- sdss->multidev.callback = spawn_stub_launch_dm;
+ sdss->multidev.callback = spawn_stub_pcis_dm;
libxl__add_disks(egc, ao, dm_domid, dm_config, &sdss->multidev);
libxl__multidev_prepared(egc, &sdss->multidev, 0);
@@ -2449,6 +2452,33 @@ out:
spawn_stubdom_pvqemu_cb(egc, &sdss->pvqemu, ret);
}
+static void spawn_stub_pcis_dm(libxl__egc *egc,
+ libxl__multidev *multidev, int ret)
+{
+ libxl__stub_dm_spawn_state *sdss = CONTAINER_OF(multidev, *sdss, multidev);
+ STATE_AO_GC(sdss->dm.spawn.ao);
+
+ /* convenience aliases */
+ libxl_domain_config *const guest_config = sdss->dm.guest_config;
+ const int guest_domid = sdss->dm.guest_domid;
+ uint32_t dm_domid = sdss->pvqemu.guest_domid;
+
+ if (ret) {
+ LOGD(ERROR, guest_domid, "error connecting disk devices");
+ goto out;
+ }
+
+ libxl__multidev_begin(ao, &sdss->multidev);
+ sdss->multidev.callback = spawn_stub_launch_dm;
+ libxl__add_pcis(egc, ao, dm_domid, guest_config, &sdss->multidev);
+ libxl__multidev_prepared(egc, &sdss->multidev, 0);
+ return;
+
+out:
+ assert(ret);
+ spawn_stubdom_pvqemu_cb(egc, &sdss->pvqemu, ret);
+}
+
static void spawn_stub_launch_dm(libxl__egc *egc,
libxl__multidev *multidev, int ret)
{
@@ -2468,7 +2498,7 @@ static void spawn_stub_launch_dm(libxl__egc *egc,
int need_qemu;
if (ret) {
- LOGD(ERROR, guest_domid, "error connecting disk devices");
+ LOGD(ERROR, guest_domid, "error connecting pci devices");
goto out;
}
diff --git a/tools/libs/light/libxl_internal.h b/tools/libs/light/libxl_internal.h
index a8297f23b08a..6df2452c6bd9 100644
--- a/tools/libs/light/libxl_internal.h
+++ b/tools/libs/light/libxl_internal.h
@@ -1732,6 +1732,9 @@ _hidden int libxl__device_pci_setdefault(libxl__gc *gc, uint32_t domid,
libxl_device_pci *pci, bool hotplug);
_hidden bool libxl__is_igd_vga_passthru(libxl__gc *gc,
const libxl_domain_config *d_config);
+_hidden void libxl__add_pcis(libxl__egc *egc, libxl__ao *ao, uint32_t domid,
+ libxl_domain_config *d_config,
+ libxl__multidev *multidev);
/* from libxl_dtdev */
diff --git a/tools/libs/light/libxl_pci.c b/tools/libs/light/libxl_pci.c
index b863da2f0d60..c9e01900ecfc 100644
--- a/tools/libs/light/libxl_pci.c
+++ b/tools/libs/light/libxl_pci.c
@@ -1434,10 +1434,6 @@ static void pci_add_dm_done(libxl__egc *egc,
if (rc) goto out;
- /* stubdomain is always running by now, even at create time */
- if (isstubdom)
- starting = false;
-
sysfs_path = GCSPRINTF(SYSFS_PCI_DEV"/"PCI_BDF"/resource", pci->domain,
pci->bus, pci->dev, pci->func);
f = fopen(sysfs_path, "r");
@@ -1670,6 +1666,13 @@ void libxl__device_pci_add(libxl__egc *egc, uint32_t domid,
rc = libxl__device_pci_setdefault(gc, domid, pci, !starting);
if (rc) goto out;
+ stubdomid = libxl_get_stubdom_id(ctx, domid);
+ if (stubdomid != 0 && starting) {
+ /* Initial work already done when attaching to the stubdom */
+ device_pci_add_stubdom_done(egc, pas, 0); /* must be last */
+ return;
+ }
+
if (pci->seize && !pciback_dev_is_assigned(gc, pci)) {
rc = libxl__device_pci_assignable_add(gc, pci, 1);
if ( rc )
@@ -1688,8 +1691,7 @@ void libxl__device_pci_add(libxl__egc *egc, uint32_t domid,
libxl__device_pci_reset(gc, pci->domain, pci->bus, pci->dev, pci->func);
- stubdomid = libxl_get_stubdom_id(ctx, domid);
- if (stubdomid != 0) {
+ if (stubdomid != 0 && !starting) {
pas->callback = device_pci_add_stubdom_wait;
do_pci_add(egc, stubdomid, pas); /* must be last */
@@ -1831,9 +1833,9 @@ typedef struct {
static void add_pcis_done(libxl__egc *, libxl__multidev *, int rc);
-static void libxl__add_pcis(libxl__egc *egc, libxl__ao *ao, uint32_t domid,
- libxl_domain_config *d_config,
- libxl__multidev *multidev)
+void libxl__add_pcis(libxl__egc *egc, libxl__ao *ao, uint32_t domid,
+ libxl_domain_config *d_config,
+ libxl__multidev *multidev)
{
AO_GC;
add_pcis_state *apds;
--
2.37.3