| Revisión | 403f290c0603f35f2d09c982bf5549b6d0803ec1 (tree) |
|---|---|
| Tiempo | 2018-10-19 11:46:53 |
| Autor | Emilio G. Cota <cota@braa...> |
| Commiter | Richard Henderson |
cputlb: read CPUTLBEntry.addr_write atomically
Updates can come from other threads, so readers that do not
take tlb_lock must use atomic_read to avoid undefined
behaviour (UB).
This completes the conversion to tlb_lock. This conversion results
on average in no performance loss, as the following experiments
(run on an Intel i7-6700K CPU @ 4.00GHz) show.
1. aarch64 bootup+shutdown test:
- Before:
- After:
2. SPEC06int:
Notes:
- tlb-lock-v2 corresponds to an implementation with a mutex.
- tlb-lock-v3 corresponds to the current implementation, i.e.
Signed-off-by: Emilio G. Cota <cota@braap.org>
Message-Id: <20181016153840.25877-1-cota@braap.org>
Signed-off-by: Richard Henderson <richard.henderson@linaro.org>
accel/tcg/cputlb.c (diff)
accel/tcg/softmmu_template.h (diff) include/exec/cpu_ldst.h (diff) include/exec/cpu_ldst_template.h (diff)| @@ -258,7 +258,7 @@ static inline bool tlb_hit_page_anyprot(CPUTLBEntry *tlb_entry, | ||
| 258 | 258 | target_ulong page) |
| 259 | 259 | { |
| 260 | 260 | return tlb_hit_page(tlb_entry->addr_read, page) || |
| 261 | - tlb_hit_page(tlb_entry->addr_write, page) || | |
| 261 | + tlb_hit_page(tlb_addr_write(tlb_entry), page) || | |
| 262 | 262 | tlb_hit_page(tlb_entry->addr_code, page); |
| 263 | 263 | } |
| 264 | 264 |
| @@ -855,7 +855,7 @@ static void io_writex(CPUArchState *env, CPUIOTLBEntry *iotlbentry, | ||
| 855 | 855 | tlb_fill(cpu, addr, size, MMU_DATA_STORE, mmu_idx, retaddr); |
| 856 | 856 | |
| 857 | 857 | entry = tlb_entry(env, mmu_idx, addr); |
| 858 | - tlb_addr = entry->addr_write; | |
| 858 | + tlb_addr = tlb_addr_write(entry); | |
| 859 | 859 | if (!(tlb_addr & ~(TARGET_PAGE_MASK | TLB_RECHECK))) { |
| 860 | 860 | /* RAM access */ |
| 861 | 861 | uintptr_t haddr = addr + entry->addend; |
| @@ -904,7 +904,14 @@ static bool victim_tlb_hit(CPUArchState *env, size_t mmu_idx, size_t index, | ||
| 904 | 904 | assert_cpu_is_self(ENV_GET_CPU(env)); |
| 905 | 905 | for (vidx = 0; vidx < CPU_VTLB_SIZE; ++vidx) { |
| 906 | 906 | CPUTLBEntry *vtlb = &env->tlb_v_table[mmu_idx][vidx]; |
| 907 | - target_ulong cmp = *(target_ulong *)((uintptr_t)vtlb + elt_ofs); | |
| 907 | + target_ulong cmp; | |
| 908 | + | |
| 909 | + /* elt_ofs might correspond to .addr_write, so use atomic_read */ | |
| 910 | +#if TCG_OVERSIZED_GUEST | |
| 911 | + cmp = *(target_ulong *)((uintptr_t)vtlb + elt_ofs); | |
| 912 | +#else | |
| 913 | + cmp = atomic_read((target_ulong *)((uintptr_t)vtlb + elt_ofs)); | |
| 914 | +#endif | |
| 908 | 915 | |
| 909 | 916 | if (cmp == page) { |
| 910 | 917 | /* Found entry in victim tlb, swap tlb and iotlb. */ |
| @@ -977,7 +984,7 @@ void probe_write(CPUArchState *env, target_ulong addr, int size, int mmu_idx, | ||
| 977 | 984 | uintptr_t index = tlb_index(env, mmu_idx, addr); |
| 978 | 985 | CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr); |
| 979 | 986 | |
| 980 | - if (!tlb_hit(entry->addr_write, addr)) { | |
| 987 | + if (!tlb_hit(tlb_addr_write(entry), addr)) { | |
| 981 | 988 | /* TLB entry is for a different page */ |
| 982 | 989 | if (!VICTIM_TLB_HIT(addr_write, addr)) { |
| 983 | 990 | tlb_fill(ENV_GET_CPU(env), addr, size, MMU_DATA_STORE, |
| @@ -995,7 +1002,7 @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr, | ||
| 995 | 1002 | size_t mmu_idx = get_mmuidx(oi); |
| 996 | 1003 | uintptr_t index = tlb_index(env, mmu_idx, addr); |
| 997 | 1004 | CPUTLBEntry *tlbe = tlb_entry(env, mmu_idx, addr); |
| 998 | - target_ulong tlb_addr = tlbe->addr_write; | |
| 1005 | + target_ulong tlb_addr = tlb_addr_write(tlbe); | |
| 999 | 1006 | TCGMemOp mop = get_memop(oi); |
| 1000 | 1007 | int a_bits = get_alignment_bits(mop); |
| 1001 | 1008 | int s_bits = mop & MO_SIZE; |
| @@ -1026,7 +1033,7 @@ static void *atomic_mmu_lookup(CPUArchState *env, target_ulong addr, | ||
| 1026 | 1033 | tlb_fill(ENV_GET_CPU(env), addr, 1 << s_bits, MMU_DATA_STORE, |
| 1027 | 1034 | mmu_idx, retaddr); |
| 1028 | 1035 | } |
| 1029 | - tlb_addr = tlbe->addr_write & ~TLB_INVALID_MASK; | |
| 1036 | + tlb_addr = tlb_addr_write(tlbe) & ~TLB_INVALID_MASK; | |
| 1030 | 1037 | } |
| 1031 | 1038 | |
| 1032 | 1039 | /* Notice an IO access or a needs-MMU-lookup access */ |
| @@ -280,7 +280,7 @@ void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val, | ||
| 280 | 280 | uintptr_t mmu_idx = get_mmuidx(oi); |
| 281 | 281 | uintptr_t index = tlb_index(env, mmu_idx, addr); |
| 282 | 282 | CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr); |
| 283 | - target_ulong tlb_addr = entry->addr_write; | |
| 283 | + target_ulong tlb_addr = tlb_addr_write(entry); | |
| 284 | 284 | unsigned a_bits = get_alignment_bits(get_memop(oi)); |
| 285 | 285 | uintptr_t haddr; |
| 286 | 286 |
| @@ -295,7 +295,7 @@ void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val, | ||
| 295 | 295 | tlb_fill(ENV_GET_CPU(env), addr, DATA_SIZE, MMU_DATA_STORE, |
| 296 | 296 | mmu_idx, retaddr); |
| 297 | 297 | } |
| 298 | - tlb_addr = entry->addr_write & ~TLB_INVALID_MASK; | |
| 298 | + tlb_addr = tlb_addr_write(entry) & ~TLB_INVALID_MASK; | |
| 299 | 299 | } |
| 300 | 300 | |
| 301 | 301 | /* Handle an IO access. */ |
| @@ -325,7 +325,7 @@ void helper_le_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val, | ||
| 325 | 325 | cannot evict the first. */ |
| 326 | 326 | page2 = (addr + DATA_SIZE) & TARGET_PAGE_MASK; |
| 327 | 327 | entry2 = tlb_entry(env, mmu_idx, page2); |
| 328 | - if (!tlb_hit_page(entry2->addr_write, page2) | |
| 328 | + if (!tlb_hit_page(tlb_addr_write(entry2), page2) | |
| 329 | 329 | && !VICTIM_TLB_HIT(addr_write, page2)) { |
| 330 | 330 | tlb_fill(ENV_GET_CPU(env), page2, DATA_SIZE, MMU_DATA_STORE, |
| 331 | 331 | mmu_idx, retaddr); |
| @@ -358,7 +358,7 @@ void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val, | ||
| 358 | 358 | uintptr_t mmu_idx = get_mmuidx(oi); |
| 359 | 359 | uintptr_t index = tlb_index(env, mmu_idx, addr); |
| 360 | 360 | CPUTLBEntry *entry = tlb_entry(env, mmu_idx, addr); |
| 361 | - target_ulong tlb_addr = entry->addr_write; | |
| 361 | + target_ulong tlb_addr = tlb_addr_write(entry); | |
| 362 | 362 | unsigned a_bits = get_alignment_bits(get_memop(oi)); |
| 363 | 363 | uintptr_t haddr; |
| 364 | 364 |
| @@ -373,7 +373,7 @@ void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val, | ||
| 373 | 373 | tlb_fill(ENV_GET_CPU(env), addr, DATA_SIZE, MMU_DATA_STORE, |
| 374 | 374 | mmu_idx, retaddr); |
| 375 | 375 | } |
| 376 | - tlb_addr = entry->addr_write & ~TLB_INVALID_MASK; | |
| 376 | + tlb_addr = tlb_addr_write(entry) & ~TLB_INVALID_MASK; | |
| 377 | 377 | } |
| 378 | 378 | |
| 379 | 379 | /* Handle an IO access. */ |
| @@ -403,7 +403,7 @@ void helper_be_st_name(CPUArchState *env, target_ulong addr, DATA_TYPE val, | ||
| 403 | 403 | cannot evict the first. */ |
| 404 | 404 | page2 = (addr + DATA_SIZE) & TARGET_PAGE_MASK; |
| 405 | 405 | entry2 = tlb_entry(env, mmu_idx, page2); |
| 406 | - if (!tlb_hit_page(entry2->addr_write, page2) | |
| 406 | + if (!tlb_hit_page(tlb_addr_write(entry2), page2) | |
| 407 | 407 | && !VICTIM_TLB_HIT(addr_write, page2)) { |
| 408 | 408 | tlb_fill(ENV_GET_CPU(env), page2, DATA_SIZE, MMU_DATA_STORE, |
| 409 | 409 | mmu_idx, retaddr); |
| @@ -126,6 +126,15 @@ extern __thread uintptr_t helper_retaddr; | ||
| 126 | 126 | /* The memory helpers for tcg-generated code need tcg_target_long etc. */ |
| 127 | 127 | #include "tcg.h" |
| 128 | 128 | |
| 129 | +static inline target_ulong tlb_addr_write(const CPUTLBEntry *entry) | |
| 130 | +{ | |
| 131 | +#if TCG_OVERSIZED_GUEST | |
| 132 | + return entry->addr_write; | |
| 133 | +#else | |
| 134 | + return atomic_read(&entry->addr_write); | |
| 135 | +#endif | |
| 136 | +} | |
| 137 | + | |
| 129 | 138 | /* Find the TLB index corresponding to the mmu_idx + address pair. */ |
| 130 | 139 | static inline uintptr_t tlb_index(CPUArchState *env, uintptr_t mmu_idx, |
| 131 | 140 | target_ulong addr) |
| @@ -439,7 +448,7 @@ static inline void *tlb_vaddr_to_host(CPUArchState *env, abi_ptr addr, | ||
| 439 | 448 | tlb_addr = tlbentry->addr_read; |
| 440 | 449 | break; |
| 441 | 450 | case 1: |
| 442 | - tlb_addr = tlbentry->addr_write; | |
| 451 | + tlb_addr = tlb_addr_write(tlbentry); | |
| 443 | 452 | break; |
| 444 | 453 | case 2: |
| 445 | 454 | tlb_addr = tlbentry->addr_code; |
| @@ -177,7 +177,7 @@ glue(glue(glue(cpu_st, SUFFIX), MEMSUFFIX), _ra)(CPUArchState *env, | ||
| 177 | 177 | addr = ptr; |
| 178 | 178 | mmu_idx = CPU_MMU_INDEX; |
| 179 | 179 | entry = tlb_entry(env, mmu_idx, addr); |
| 180 | - if (unlikely(entry->addr_write != | |
| 180 | + if (unlikely(tlb_addr_write(entry) != | |
| 181 | 181 | (addr & (TARGET_PAGE_MASK | (DATA_SIZE - 1))))) { |
| 182 | 182 | oi = make_memop_idx(SHIFT, mmu_idx); |
| 183 | 183 | glue(glue(helper_ret_st, SUFFIX), MMUSUFFIX)(env, addr, v, oi, |
Fork