127 lines · cpp
1//===-- tsan_vector_clock.cpp ---------------------------------------------===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8//9// This file is a part of ThreadSanitizer (TSan), a race detector.10//11//===----------------------------------------------------------------------===//12#include "tsan_vector_clock.h"13 14#include "sanitizer_common/sanitizer_placement_new.h"15#include "tsan_mman.h"16 17namespace __tsan {18 19#if TSAN_VECTORIZE20const uptr kVectorClockSize = kThreadSlotCount * sizeof(Epoch) / sizeof(m128);21#endif22 23VectorClock::VectorClock() { Reset(); }24 25void VectorClock::Reset() {26#if !TSAN_VECTORIZE27 for (uptr i = 0; i < kThreadSlotCount; i++)28 clk_[i] = kEpochZero;29#else30 m128 z = _mm_setzero_si128();31 m128* vclk = reinterpret_cast<m128*>(clk_);32 for (uptr i = 0; i < kVectorClockSize; i++) _mm_store_si128(&vclk[i], z);33#endif34}35 36void VectorClock::Acquire(const VectorClock* src) {37 if (!src)38 return;39#if !TSAN_VECTORIZE40 for (uptr i = 0; i < kThreadSlotCount; i++)41 clk_[i] = max(clk_[i], src->clk_[i]);42#else43 m128* __restrict vdst = reinterpret_cast<m128*>(clk_);44 m128 const* __restrict vsrc = reinterpret_cast<m128 const*>(src->clk_);45 for (uptr i = 0; i < kVectorClockSize; i++) {46 m128 s = _mm_load_si128(&vsrc[i]);47 m128 d = _mm_load_si128(&vdst[i]);48 m128 m = _mm_max_epu16(s, d);49 _mm_store_si128(&vdst[i], m);50 }51#endif52}53 54static VectorClock* AllocClock(VectorClock** dstp) {55 if (UNLIKELY(!*dstp))56 *dstp = New<VectorClock>();57 return *dstp;58}59 60void VectorClock::Release(VectorClock** dstp) const {61 VectorClock* dst = AllocClock(dstp);62 dst->Acquire(this);63}64 65void VectorClock::ReleaseStore(VectorClock** dstp) const {66 VectorClock* dst = AllocClock(dstp);67 *dst = *this;68}69 70VectorClock& VectorClock::operator=(const VectorClock& other) {71#if !TSAN_VECTORIZE72 for (uptr i = 0; i < kThreadSlotCount; i++)73 clk_[i] = other.clk_[i];74#else75 m128* __restrict vdst = reinterpret_cast<m128*>(clk_);76 m128 const* __restrict vsrc = reinterpret_cast<m128 const*>(other.clk_);77 for (uptr i = 0; i < kVectorClockSize; i++) {78 m128 s = _mm_load_si128(&vsrc[i]);79 _mm_store_si128(&vdst[i], s);80 }81#endif82 return *this;83}84 85void VectorClock::ReleaseStoreAcquire(VectorClock** dstp) {86 VectorClock* dst = AllocClock(dstp);87#if !TSAN_VECTORIZE88 for (uptr i = 0; i < kThreadSlotCount; i++) {89 Epoch tmp = dst->clk_[i];90 dst->clk_[i] = clk_[i];91 clk_[i] = max(clk_[i], tmp);92 }93#else94 m128* __restrict vdst = reinterpret_cast<m128*>(dst->clk_);95 m128* __restrict vclk = reinterpret_cast<m128*>(clk_);96 for (uptr i = 0; i < kVectorClockSize; i++) {97 m128 t = _mm_load_si128(&vdst[i]);98 m128 c = _mm_load_si128(&vclk[i]);99 m128 m = _mm_max_epu16(c, t);100 _mm_store_si128(&vdst[i], c);101 _mm_store_si128(&vclk[i], m);102 }103#endif104}105 106void VectorClock::ReleaseAcquire(VectorClock** dstp) {107 VectorClock* dst = AllocClock(dstp);108#if !TSAN_VECTORIZE109 for (uptr i = 0; i < kThreadSlotCount; i++) {110 dst->clk_[i] = max(dst->clk_[i], clk_[i]);111 clk_[i] = dst->clk_[i];112 }113#else114 m128* __restrict vdst = reinterpret_cast<m128*>(dst->clk_);115 m128* __restrict vclk = reinterpret_cast<m128*>(clk_);116 for (uptr i = 0; i < kVectorClockSize; i++) {117 m128 c = _mm_load_si128(&vclk[i]);118 m128 d = _mm_load_si128(&vdst[i]);119 m128 m = _mm_max_epu16(c, d);120 _mm_store_si128(&vdst[i], m);121 _mm_store_si128(&vclk[i], m);122 }123#endif124}125 126} // namespace __tsan127