Skip to content

Commit 172546b

Browse files
Thiemo Seuferralfbaechle
Thiemo Seufer
authored and
ralfbaechle
committed
Fix race conditions for read_c0_entryhi. Remove broken ASID masks in
tlb-sb1.c. Make tlb-r4k.c and tlb-sb1.c more similiar and more efficient. Signed-off-by: Thiemo Seufer <ths@networkno.de> Signed-off-by: Ralf Baechle <ralf@linux-mips.org>
1 parent 202d038 commit 172546b

File tree

2 files changed

+63
-45
lines changed

2 files changed

+63
-45
lines changed

‎arch/mips/mm/tlb-r4k.c‎

Lines changed: 29 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,12 @@
2121

2222
extern void build_tlb_refill_handler(void);
2323

24+
/*
25+
* Make sure all entries differ. If they're not different
26+
* MIPS32 will take revenge ...
27+
*/
28+
#define UNIQUE_ENTRYHI(idx) (CKSEG0 + ((idx) << (PAGE_SHIFT + 1)))
29+
2430
/* CP0 hazard avoidance. */
2531
#define BARRIER __asm__ __volatile__(".set noreorder\n\t" \
2632
"nop; nop; nop; nop; nop; nop;\n\t" \
@@ -42,11 +48,8 @@ void local_flush_tlb_all(void)
4248

4349
/* Blast 'em all away. */
4450
while (entry < current_cpu_data.tlbsize) {
45-
/*
46-
* Make sure all entries differ. If they're not different
47-
* MIPS32 will take revenge ...
48-
*/
49-
write_c0_entryhi(CKSEG0 + (entry << (PAGE_SHIFT + 1)));
51+
/* Make sure all entries differ. */
52+
write_c0_entryhi(UNIQUE_ENTRYHI(entry));
5053
write_c0_index(entry);
5154
mtc0_tlbw_hazard();
5255
tlb_write_indexed();
@@ -57,12 +60,21 @@ void local_flush_tlb_all(void)
5760
local_irq_restore(flags);
5861
}
5962

63+
/* All entries common to a mm share an asid. To effectively flush
64+
these entries, we just bump the asid. */
6065
void local_flush_tlb_mm(struct mm_struct *mm)
6166
{
62-
int cpu = smp_processor_id();
67+
int cpu;
68+
69+
preempt_disable();
6370

64-
if (cpu_context(cpu, mm) != 0)
65-
drop_mmu_context(mm,cpu);
71+
cpu = smp_processor_id();
72+
73+
if (cpu_context(cpu, mm) != 0) {
74+
drop_mmu_context(mm, cpu);
75+
}
76+
77+
preempt_enable();
6678
}
6779

6880
void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
@@ -75,9 +87,9 @@ void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
7587
unsigned long flags;
7688
int size;
7789

78-
local_irq_save(flags);
7990
size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
8091
size = (size + 1) >> 1;
92+
local_irq_save(flags);
8193
if (size <= current_cpu_data.tlbsize/2) {
8294
int oldpid = read_c0_entryhi();
8395
int newpid = cpu_asid(cpu, mm);
@@ -99,8 +111,7 @@ void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
99111
if (idx < 0)
100112
continue;
101113
/* Make sure all entries differ. */
102-
write_c0_entryhi(CKSEG0 +
103-
(idx << (PAGE_SHIFT + 1)));
114+
write_c0_entryhi(UNIQUE_ENTRYHI(idx));
104115
mtc0_tlbw_hazard();
105116
tlb_write_indexed();
106117
}
@@ -118,9 +129,9 @@ void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
118129
unsigned long flags;
119130
int size;
120131

121-
local_irq_save(flags);
122132
size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
123133
size = (size + 1) >> 1;
134+
local_irq_save(flags);
124135
if (size <= current_cpu_data.tlbsize / 2) {
125136
int pid = read_c0_entryhi();
126137

@@ -142,7 +153,7 @@ void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
142153
if (idx < 0)
143154
continue;
144155
/* Make sure all entries differ. */
145-
write_c0_entryhi(CKSEG0 + (idx << (PAGE_SHIFT + 1)));
156+
write_c0_entryhi(UNIQUE_ENTRYHI(idx));
146157
mtc0_tlbw_hazard();
147158
tlb_write_indexed();
148159
}
@@ -176,7 +187,7 @@ void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
176187
if (idx < 0)
177188
goto finish;
178189
/* Make sure all entries differ. */
179-
write_c0_entryhi(CKSEG0 + (idx << (PAGE_SHIFT + 1)));
190+
write_c0_entryhi(UNIQUE_ENTRYHI(idx));
180191
mtc0_tlbw_hazard();
181192
tlb_write_indexed();
182193
tlbw_use_hazard();
@@ -197,8 +208,8 @@ void local_flush_tlb_one(unsigned long page)
197208
int oldpid, idx;
198209

199210
local_irq_save(flags);
200-
page &= (PAGE_MASK << 1);
201211
oldpid = read_c0_entryhi();
212+
page &= (PAGE_MASK << 1);
202213
write_c0_entryhi(page);
203214
mtc0_tlbw_hazard();
204215
tlb_probe();
@@ -208,7 +219,7 @@ void local_flush_tlb_one(unsigned long page)
208219
write_c0_entrylo1(0);
209220
if (idx >= 0) {
210221
/* Make sure all entries differ. */
211-
write_c0_entryhi(CKSEG0 + (idx << (PAGE_SHIFT + 1)));
222+
write_c0_entryhi(UNIQUE_ENTRYHI(idx));
212223
mtc0_tlbw_hazard();
213224
tlb_write_indexed();
214225
tlbw_use_hazard();
@@ -238,9 +249,9 @@ void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte)
238249
if (current->active_mm != vma->vm_mm)
239250
return;
240251

241-
pid = read_c0_entryhi() & ASID_MASK;
242-
243252
local_irq_save(flags);
253+
254+
pid = read_c0_entryhi() & ASID_MASK;
244255
address &= (PAGE_MASK << 1);
245256
write_c0_entryhi(address | pid);
246257
pgdp = pgd_offset(vma->vm_mm, address);
@@ -260,14 +271,12 @@ void __update_tlb(struct vm_area_struct * vma, unsigned long address, pte_t pte)
260271
write_c0_entrylo0(pte_val(*ptep++) >> 6);
261272
write_c0_entrylo1(pte_val(*ptep) >> 6);
262273
#endif
263-
write_c0_entryhi(address | pid);
264274
mtc0_tlbw_hazard();
265275
if (idx < 0)
266276
tlb_write_random();
267277
else
268278
tlb_write_indexed();
269279
tlbw_use_hazard();
270-
write_c0_entryhi(pid);
271280
local_irq_restore(flags);
272281
}
273282

‎arch/mips/mm/tlb-sb1.c‎

Lines changed: 34 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -94,7 +94,7 @@ void local_flush_tlb_all(void)
9494

9595
local_irq_save(flags);
9696
/* Save old context and create impossible VPN2 value */
97-
old_ctx = read_c0_entryhi() & ASID_MASK;
97+
old_ctx = read_c0_entryhi();
9898
write_c0_entrylo0(0);
9999
write_c0_entrylo1(0);
100100

@@ -144,17 +144,17 @@ void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
144144
unsigned long end)
145145
{
146146
struct mm_struct *mm = vma->vm_mm;
147-
unsigned long flags;
148-
int cpu;
147+
int cpu = smp_processor_id();
149148

150-
local_irq_save(flags);
151-
cpu = smp_processor_id();
152149
if (cpu_context(cpu, mm) != 0) {
150+
unsigned long flags;
153151
int size;
152+
154153
size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
155154
size = (size + 1) >> 1;
155+
local_irq_save(flags);
156156
if (size <= (current_cpu_data.tlbsize/2)) {
157-
int oldpid = read_c0_entryhi() & ASID_MASK;
157+
int oldpid = read_c0_entryhi();
158158
int newpid = cpu_asid(cpu, mm);
159159

160160
start &= (PAGE_MASK << 1);
@@ -169,17 +169,17 @@ void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
169169
idx = read_c0_index();
170170
write_c0_entrylo0(0);
171171
write_c0_entrylo1(0);
172-
write_c0_entryhi(UNIQUE_ENTRYHI(idx));
173172
if (idx < 0)
174173
continue;
174+
write_c0_entryhi(UNIQUE_ENTRYHI(idx));
175175
tlb_write_indexed();
176176
}
177177
write_c0_entryhi(oldpid);
178178
} else {
179179
drop_mmu_context(mm, cpu);
180180
}
181+
local_irq_restore(flags);
181182
}
182-
local_irq_restore(flags);
183183
}
184184

185185
void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
@@ -189,7 +189,6 @@ void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
189189

190190
size = (end - start + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
191191
size = (size + 1) >> 1;
192-
193192
local_irq_save(flags);
194193
if (size <= (current_cpu_data.tlbsize/2)) {
195194
int pid = read_c0_entryhi();
@@ -207,9 +206,9 @@ void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
207206
idx = read_c0_index();
208207
write_c0_entrylo0(0);
209208
write_c0_entrylo1(0);
210-
write_c0_entryhi(UNIQUE_ENTRYHI(idx));
211209
if (idx < 0)
212210
continue;
211+
write_c0_entryhi(UNIQUE_ENTRYHI(idx));
213212
tlb_write_indexed();
214213
}
215214
write_c0_entryhi(pid);
@@ -221,15 +220,16 @@ void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
221220

222221
void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
223222
{
224-
unsigned long flags;
225223
int cpu = smp_processor_id();
226224

227-
local_irq_save(flags);
228225
if (cpu_context(cpu, vma->vm_mm) != 0) {
226+
unsigned long flags;
229227
int oldpid, newpid, idx;
228+
230229
newpid = cpu_asid(cpu, vma->vm_mm);
231230
page &= (PAGE_MASK << 1);
232-
oldpid = read_c0_entryhi() & ASID_MASK;
231+
local_irq_save(flags);
232+
oldpid = read_c0_entryhi();
233233
write_c0_entryhi(page | newpid);
234234
tlb_probe();
235235
idx = read_c0_index();
@@ -240,10 +240,11 @@ void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
240240
/* Make sure all entries differ. */
241241
write_c0_entryhi(UNIQUE_ENTRYHI(idx));
242242
tlb_write_indexed();
243+
243244
finish:
244245
write_c0_entryhi(oldpid);
246+
local_irq_restore(flags);
245247
}
246-
local_irq_restore(flags);
247248
}
248249

249250
/*
@@ -255,18 +256,17 @@ void local_flush_tlb_one(unsigned long page)
255256
unsigned long flags;
256257
int oldpid, idx;
257258

258-
page &= (PAGE_MASK << 1);
259-
oldpid = read_c0_entryhi() & ASID_MASK;
260-
261259
local_irq_save(flags);
260+
oldpid = read_c0_entryhi();
261+
page &= (PAGE_MASK << 1);
262262
write_c0_entryhi(page);
263263
tlb_probe();
264264
idx = read_c0_index();
265+
write_c0_entrylo0(0);
266+
write_c0_entrylo1(0);
265267
if (idx >= 0) {
266268
/* Make sure all entries differ. */
267269
write_c0_entryhi(UNIQUE_ENTRYHI(idx));
268-
write_c0_entrylo0(0);
269-
write_c0_entrylo1(0);
270270
tlb_write_indexed();
271271
}
272272

@@ -297,6 +297,7 @@ void __update_tlb(struct vm_area_struct *vma, unsigned long address, pte_t pte)
297297
{
298298
unsigned long flags;
299299
pgd_t *pgdp;
300+
pud_t *pudp;
300301
pmd_t *pmdp;
301302
pte_t *ptep;
302303
int idx, pid;
@@ -311,19 +312,26 @@ void __update_tlb(struct vm_area_struct *vma, unsigned long address, pte_t pte)
311312

312313
pid = read_c0_entryhi() & ASID_MASK;
313314
address &= (PAGE_MASK << 1);
314-
write_c0_entryhi(address | (pid));
315+
write_c0_entryhi(address | pid);
315316
pgdp = pgd_offset(vma->vm_mm, address);
316317
tlb_probe();
317-
pmdp = pmd_offset(pgdp, address);
318+
pudp = pud_offset(pgdp, address);
319+
pmdp = pmd_offset(pudp, address);
318320
idx = read_c0_index();
319321
ptep = pte_offset_map(pmdp, address);
322+
323+
#if defined(CONFIG_64BIT_PHYS_ADDR) && defined(CONFIG_CPU_MIPS32)
324+
write_c0_entrylo0(ptep->pte_high);
325+
ptep++;
326+
write_c0_entrylo1(ptep->pte_high);
327+
#else
320328
write_c0_entrylo0(pte_val(*ptep++) >> 6);
321329
write_c0_entrylo1(pte_val(*ptep) >> 6);
322-
if (idx < 0) {
330+
#endif
331+
if (idx < 0)
323332
tlb_write_random();
324-
} else {
333+
else
325334
tlb_write_indexed();
326-
}
327335
local_irq_restore(flags);
328336
}
329337

@@ -336,7 +344,8 @@ void __init add_wired_entry(unsigned long entrylo0, unsigned long entrylo1,
336344
unsigned long old_ctx;
337345

338346
local_irq_save(flags);
339-
old_ctx = read_c0_entryhi() & 0xff;
347+
/* Save old context and create impossible VPN2 value */
348+
old_ctx = read_c0_entryhi();
340349
old_pagemask = read_c0_pagemask();
341350
wired = read_c0_wired();
342351
write_c0_wired(wired + 1);

0 commit comments

Comments
 (0)