diff --git a/arch/mips/include/asm/rjzcache.h b/arch/mips/include/asm/rjzcache.h index 7c3f0ead203d5b..1ab8b8afa2417b 100644 --- a/arch/mips/include/asm/rjzcache.h +++ b/arch/mips/include/asm/rjzcache.h @@ -591,6 +591,8 @@ static inline void blast_dcache_jz(void) } start += 32; }while(start < end); + + //SYNC_WB(); } static inline void blast_icache_jz(void) @@ -598,6 +600,7 @@ static inline void blast_icache_jz(void) unsigned long start = INDEX_BASE; unsigned long end = start + current_cpu_data.icache.waysize * current_cpu_data.icache.ways; unsigned long addr = start; + do { i_cache(JZ_FETCH_LOCK, addr, 0);i_cache(Hit_Invalidate_I,addr,0); addr += 32; @@ -608,6 +611,31 @@ static inline void blast_icache_jz(void) i_cache(JZ_FETCH_LOCK, addr, 0);i_cache(Hit_Invalidate_I,addr,0); addr += 32; } while (addr < end); + + do { + i_cache(Index_Load_Tag_I,start,0); + if(read_c0_dtaglo() & 1) { + i_cache(Index_Invalidate_I,start,0); + } + start += 32; + i_cache(Index_Load_Tag_I,start,0); + if(read_c0_dtaglo() & 1) { + i_cache(Index_Invalidate_I,start,0); + } + start += 32; + i_cache(Index_Load_Tag_I,start,0); + if(read_c0_dtaglo() & 1) { + i_cache(Index_Invalidate_I,start,0); + } + start += 32; + i_cache(Index_Load_Tag_I,start,0); + if(read_c0_dtaglo() & 1) { + i_cache(Index_Invalidate_I,start,0); + } + start += 32; + }while(start < end); + + INVALIDATE_BTB(); } static inline void blast_dcache32(void) @@ -799,7 +827,7 @@ static inline void protected_blast_icache_range(unsigned long start, unsigned long aend = (end - 1) & ~(lsize - 1); // K0_TO_K1(); - K0_TO_K1_CHECK(start,end); + //K0_TO_K1_CHECK(start,end); while (1) { protected_cache_op(Hit_Invalidate_I, addr); @@ -809,7 +837,7 @@ static inline void protected_blast_icache_range(unsigned long start, } INVALIDATE_BTB(); - K1_TO_K0(); + //K1_TO_K0(); } static inline void blast_dcache_range(unsigned long start, diff --git a/arch/mips/mm/c-jz.c b/arch/mips/mm/c-jz.c index 1da9a5970fd17c..1d773512ff4451 100755 --- a/arch/mips/mm/c-jz.c +++ b/arch/mips/mm/c-jz.c @@ -57,6 +57,15 @@ static inline void r4k_on_each_cpu(void (*func) (void *info), void *info) preempt_enable(); } +static inline void r4k_on_other_cpu(void (*func) (void *info), void *info) +{ +#if !defined(CONFIG_MIPS_MT_SMP) && !defined(CONFIG_MIPS_MT_SMTC) + preempt_disable(); + smp_call_function(func, info, 1); + preempt_enable(); +#endif +} + #if defined(CONFIG_MIPS_CMP) #define cpu_has_safe_index_cacheops 0 #else @@ -145,7 +154,8 @@ static void (* r4k_blast_dcache)(void); static void __cpuinit r4k_blast_dcache_setup(void) { unsigned long dc_lsize = cpu_dcache_line_size(); - r4k_blast_dcache_jz = blast_dcache_jz; + //r4k_blast_dcache_jz = blast_dcache_jz; + r4k_blast_dcache_jz = blast_dcache32; if (dc_lsize == 0) r4k_blast_dcache = (void *)cache_noop; else if (dc_lsize == 16) @@ -274,7 +284,8 @@ static void (* r4k_blast_icache)(void); static void __cpuinit r4k_blast_icache_setup(void) { unsigned long ic_lsize = cpu_icache_line_size(); - r4k_blast_icache_jz = blast_icache_jz; +// r4k_blast_icache_jz = blast_icache_jz; + r4k_blast_icache_jz = blast_icache32; if (ic_lsize == 0) r4k_blast_icache = (void *)cache_noop; else if (ic_lsize == 16) @@ -638,15 +649,15 @@ static inline void local_r4k_flush_icache_range(unsigned long start, unsigned lo { if (!cpu_has_ic_fills_f_dc) { if (end - start >= dcache_size) { - r4k_blast_dcache_jz(); + r4k_blast_dcache(); } else { R4600_HIT_CACHEOP_WAR_IMPL; protected_blast_dcache_range(start, end); } } - if (end - start > icache_size) - r4k_blast_icache_jz(); + if (end - start >= icache_size) + r4k_blast_icache(); else protected_blast_icache_range(start, end); } @@ -656,22 +667,134 @@ static inline void local_r4k_flush_icache_jz_ipi(void *args){ static inline void local_r4k_flush_dcache_jz_ipi(void *args){ r4k_blast_dcache_jz(); } + + +/* + The other CPU maybe running in different process(different ASID) whit different address mapping, + so flush dcache all with local_r4k_flush_dcache_ipi() is safe. + But local_r4k_flush_dcache_ipi() flush icache all by index, + both flushed L1 icache and L2 cache, it slowdown the machine performance. + so optimized the routine as following. + */ +static inline void protected_blast_other_cpu_dcache_range_ipi(void *args) +{ + unsigned long lsize; + unsigned long addr; + unsigned long aend; + struct flush_icache_range_args * addrp; + unsigned long start; + unsigned long end; + + addrp = (struct flush_icache_range_args *)args; + start = addrp->start; + end = addrp->end; + + lsize = cpu_dcache_line_size(); + addr = (start & ~(lsize - 1)) | INDEX_BASE; /* INDEX_BASE = 0x80000000 */ + aend = ((end - 1) & ~(lsize - 1)) | INDEX_BASE; + + //printk("protected_blast_other_cpu_dcache_range_ipi id=%d, start=%#x, size=%#x\n", smp_processor_id(), (unsigned int)start, (unsigned int)(end-start)); + + while (1) { + protected_cache_op(Index_Writeback_Inv_D, (addr + 0x0)); + protected_cache_op(Index_Writeback_Inv_D, (addr + 0x1000)); + protected_cache_op(Index_Writeback_Inv_D, (addr + 0x2000)); + protected_cache_op(Index_Writeback_Inv_D, (addr + 0x3000)); + protected_cache_op(Index_Writeback_Inv_D, (addr + 0x4000)); + protected_cache_op(Index_Writeback_Inv_D, (addr + 0x5000)); + protected_cache_op(Index_Writeback_Inv_D, (addr + 0x6000)); + protected_cache_op(Index_Writeback_Inv_D, (addr + 0x7000)); + if (addr == aend) + break; + addr += lsize; + } + + SYNC_WB(); +} + +/* + The other CPU maybe running in different process(different ASID) whit different address mapping, + so flush icache all with local_r4k_flush_icache_ipi() is safe. + But local_r4k_flush_icache_ipi() flush icache all by index, + both flushed L1 icache and L2 cache, it slowdown the machine performance. + so optimized the routine as following. + */ +static inline void protected_blast_other_cpu_icache_range_ipi(void *args) +{ + unsigned long lsize; + unsigned long addr; + unsigned long aend; + struct flush_icache_range_args * addrp; + unsigned long start; + unsigned long end; + + addrp = (struct flush_icache_range_args *)args; + start = addrp->start; + end = addrp->end; + + lsize = cpu_icache_line_size(); + addr = (start & ~(lsize - 1)) | INDEX_BASE; /* INDEX_BASE = 0x80000000 */ + aend = ((end - 1) & ~(lsize - 1)) | INDEX_BASE; + + //printk("protected_blast_other_cpu_icache_range_ipi id=%d, start=%#x, size=%#x\n", smp_processor_id(), (unsigned int)start, (unsigned int)(end-start)); + + while (1) { + protected_cache_op(Index_Invalidate_I, (addr + 0x0)); + protected_cache_op(Index_Invalidate_I, (addr + 0x1000)); + protected_cache_op(Index_Invalidate_I, (addr + 0x2000)); + protected_cache_op(Index_Invalidate_I, (addr + 0x3000)); + protected_cache_op(Index_Invalidate_I, (addr + 0x4000)); + protected_cache_op(Index_Invalidate_I, (addr + 0x5000)); + protected_cache_op(Index_Invalidate_I, (addr + 0x6000)); + protected_cache_op(Index_Invalidate_I, (addr + 0x7000)); + if (addr == aend) + break; + addr += lsize; + } + INVALIDATE_BTB(); +} + +/* + * Test result by 2013-10-25: + * Flush complete dcache and icache on other CPUs with local_r4k_flush_dcache_jz_ipi() + * and local_r4k_flush_icache_jz_ipi(), running flushtest is stable 4days. + * + * Flush dcache_range and icache_range by index on other CPUs, + * with protected_blast_other_cpu_dcache_range_ipi() protected_blast_other_cpu_icache_range_ipi(), + * fails running flushtest after hours. It should be update in the future. + */ static void r4k_flush_icache_range(unsigned long start, unsigned long end) { if (!cpu_has_ic_fills_f_dc) { if (end - start >= dcache_size) { - //r4k_blast_dcache_jz(); + /* Flush complete dcache on all CPUs */ r4k_on_each_cpu(local_r4k_flush_dcache_jz_ipi,0); } else { - R4600_HIT_CACHEOP_WAR_IMPL; + /* Flush dcache by address on this CPU */ protected_blast_dcache_range(start, end); } } - if (end - start > icache_size) + if (end - start >= icache_size) + /* Flush complete icache on all CPUs */ r4k_on_each_cpu(local_r4k_flush_icache_jz_ipi,0); - else + else { + preempt_disable(); + /* Flush icache by address on this CPU */ protected_blast_icache_range(start, end); + if (end-start <= PAGE_SIZE) { + /* Flush icache_range by index on other CPUs */ + struct flush_icache_range_args range_addr; + range_addr.start = start; + range_addr.end = end; + smp_call_function(protected_blast_other_cpu_icache_range_ipi, &range_addr,1); + } + else { + /* Flush complete icache on other CPUs */ + smp_call_function(local_r4k_flush_icache_jz_ipi,0,1); + } + preempt_enable(); + } } #endif diff --git a/kernel/timeconst.pl b/kernel/timeconst.pl index eb51d76e058a40..3f42652a6a3749 100644 --- a/kernel/timeconst.pl +++ b/kernel/timeconst.pl @@ -369,10 +369,8 @@ (@) die "Usage: $0 HZ\n"; } - @val = @{$canned_values{$hz}}; - if (!defined(@val)) { - @val = compute_values($hz); - } + $cv = $canned_values{$hz}; + @val = defined($cv) ? @$cv : compute_values($hz); output($hz, @val); } exit 0;