220 lines · cpp
1//===-- lib/runtime/copy.cpp ------------------------------------*- C++ -*-===//2//3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.4// See https://llvm.org/LICENSE.txt for license information.5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception6//7//===----------------------------------------------------------------------===//8 9#include "copy.h"10#include "stack.h"11#include "flang-rt/runtime/descriptor.h"12#include "flang-rt/runtime/terminator.h"13#include "flang-rt/runtime/type-info.h"14#include "flang/Runtime/allocatable.h"15#include "flang/Runtime/freestanding-tools.h"16 17#include <cstring>18 19namespace Fortran::runtime {20namespace {21using StaticDescTy = StaticDescriptor<maxRank, true, 0>;22 23// A structure describing the data copy that needs to be done24// from one descriptor to another. It is a helper structure25// for CopyElement.26struct CopyDescriptor {27 // A constructor specifying all members explicitly.28 // The toAt and fromAt specify subscript storages that might be29 // external to CopyElement, and cannot be modified.30 // The copy descriptor only establishes toAtPtr_ and fromAtPtr_31 // pointers to point to these storages.32 RT_API_ATTRS CopyDescriptor(const Descriptor &to, const SubscriptValue toAt[],33 const Descriptor &from, const SubscriptValue fromAt[],34 std::size_t elements, bool usesStaticDescriptors = false)35 : to_(to), from_(from), elements_(elements),36 usesStaticDescriptors_(usesStaticDescriptors) {37 toAtPtr_ = toAt;38 fromAtPtr_ = fromAt;39 }40 // The number of elements to copy is initialized from the to descriptor.41 // The current element subscripts are initialized from the lower bounds42 // of the to and from descriptors.43 RT_API_ATTRS CopyDescriptor(const Descriptor &to, const Descriptor &from,44 bool usesStaticDescriptors = false)45 : to_(to), from_(from), elements_(to.Elements()),46 usesStaticDescriptors_(usesStaticDescriptors) {47 to.GetLowerBounds(toAt_);48 from.GetLowerBounds(fromAt_);49 }50 51 // Increment the toAt_ and fromAt_ subscripts to the next52 // element.53 RT_API_ATTRS void IncrementSubscripts(Terminator &terminator) {54 // This method must not be called for copy descriptors55 // using external non-modifiable subscript storage.56 RUNTIME_CHECK(terminator, toAt_ == toAtPtr_ && fromAt_ == fromAtPtr_);57 to_.IncrementSubscripts(toAt_);58 from_.IncrementSubscripts(fromAt_);59 }60 61 // Descriptor of the destination.62 const Descriptor &to_;63 // A subscript specifying the current element position to copy to.64 SubscriptValue toAt_[maxRank];65 // A pointer to the storage of the 'to' subscript.66 // It may point to toAt_ or to an external non-modifiable67 // subscript storage.68 const SubscriptValue *toAtPtr_{toAt_};69 // Descriptor of the source.70 const Descriptor &from_;71 // A subscript specifying the current element position to copy from.72 SubscriptValue fromAt_[maxRank];73 // A pointer to the storage of the 'from' subscript.74 // It may point to fromAt_ or to an external non-modifiable75 // subscript storage.76 const SubscriptValue *fromAtPtr_{fromAt_};77 // Number of elements left to copy.78 std::size_t elements_;79 // Must be true, if the to and from descriptors are allocated80 // by the CopyElement runtime. The allocated memory belongs81 // to a separate stack that needs to be popped in correspondence82 // with popping such a CopyDescriptor node.83 bool usesStaticDescriptors_;84};85 86// A pair of StaticDescTy elements.87struct StaticDescriptorsPair {88 StaticDescTy to;89 StaticDescTy from;90};91} // namespace92 93RT_OFFLOAD_API_GROUP_BEGIN94 95RT_API_ATTRS void CopyElement(const Descriptor &to, const SubscriptValue toAt[],96 const Descriptor &from, const SubscriptValue fromAt[],97 Terminator &terminator) {98 if (!to.Addendum()) {99 // Avoid the overhead of creating the work stacks below100 // for the simple non-derived type cases, because the overhead101 // might be noticeable over the total amount of work that102 // needs to be done for the copy.103 char *toPtr{to.Element<char>(toAt)};104 char *fromPtr{from.Element<char>(fromAt)};105 RUNTIME_CHECK(terminator, to.ElementBytes() == from.ElementBytes());106 runtime::memcpy(toPtr, fromPtr, to.ElementBytes());107 return;108 }109 110#if !defined(RT_DEVICE_COMPILATION)111 constexpr unsigned copyStackReserve{16};112 constexpr unsigned descriptorStackReserve{6};113#else114 // Always use dynamic allocation on the device to avoid115 // big stack sizes. This may be tuned as needed.116 constexpr unsigned copyStackReserve{0};117 constexpr unsigned descriptorStackReserve{0};118#endif119 // Keep a stack of CopyDescriptor's to avoid recursive calls.120 Stack<CopyDescriptor, copyStackReserve> copyStack{terminator};121 // Keep a separate stack of StaticDescTy pairs. These descriptors122 // may be used for representing copies of Component::Genre::Data123 // components (since they do not have their descriptors allocated124 // in memory).125 Stack<StaticDescriptorsPair, descriptorStackReserve> descriptorsStack{126 terminator};127 copyStack.emplace(to, toAt, from, fromAt, /*elements=*/std::size_t{1});128 129 while (!copyStack.empty()) {130 CopyDescriptor ¤tCopy{copyStack.top()};131 std::size_t &elements{currentCopy.elements_};132 if (elements == 0) {133 // This copy has been exhausted.134 if (currentCopy.usesStaticDescriptors_) {135 // Pop the static descriptors, if they were used136 // for the current copy.137 descriptorsStack.pop();138 }139 copyStack.pop();140 continue;141 }142 const Descriptor &curTo{currentCopy.to_};143 const SubscriptValue *curToAt{currentCopy.toAtPtr_};144 const Descriptor &curFrom{currentCopy.from_};145 const SubscriptValue *curFromAt{currentCopy.fromAtPtr_};146 char *toPtr{curTo.Element<char>(curToAt)};147 char *fromPtr{curFrom.Element<char>(curFromAt)};148 RUNTIME_CHECK(terminator, curTo.ElementBytes() == curFrom.ElementBytes());149 // TODO: the memcpy can be optimized when both to and from are contiguous.150 // Moreover, if we came here from an Component::Genre::Data component,151 // all the per-element copies are redundant, because the parent152 // has already been copied as a whole.153 runtime::memcpy(toPtr, fromPtr, curTo.ElementBytes());154 --elements;155 if (elements != 0) {156 currentCopy.IncrementSubscripts(terminator);157 }158 159 // Deep copy allocatable and automatic components if any.160 if (const auto *addendum{curTo.Addendum()}) {161 if (const auto *derived{addendum->derivedType()};162 derived && !derived->noDestructionNeeded()) {163 RUNTIME_CHECK(terminator,164 curFrom.Addendum() && derived == curFrom.Addendum()->derivedType());165 const Descriptor &componentDesc{derived->component()};166 const typeInfo::Component *component{167 componentDesc.OffsetElement<typeInfo::Component>()};168 std::size_t nComponents{componentDesc.Elements()};169 for (std::size_t j{0}; j < nComponents; ++j, ++component) {170 if (component->genre() == typeInfo::Component::Genre::Allocatable ||171 component->genre() ==172 typeInfo::Component::Genre::AllocatableDevice ||173 component->genre() == typeInfo::Component::Genre::Automatic) {174 Descriptor &toDesc{175 *reinterpret_cast<Descriptor *>(toPtr + component->offset())};176 if (toDesc.raw().base_addr != nullptr) {177 toDesc.set_base_addr(nullptr);178 RUNTIME_CHECK(terminator,179 toDesc.Allocate(/*asyncObject=*/nullptr) == CFI_SUCCESS);180 const Descriptor &fromDesc{*reinterpret_cast<const Descriptor *>(181 fromPtr + component->offset())};182 copyStack.emplace(toDesc, fromDesc);183 }184 } else if (component->genre() == typeInfo::Component::Genre::Data &&185 component->derivedType() &&186 !component->derivedType()->noDestructionNeeded()) {187 SubscriptValue extents[maxRank];188 const typeInfo::Value *bounds{component->bounds()};189 std::size_t elements{1};190 for (int dim{0}; dim < component->rank(); ++dim) {191 typeInfo::TypeParameterValue lb{192 bounds[2 * dim].GetValue(&curTo).value_or(0)};193 typeInfo::TypeParameterValue ub{194 bounds[2 * dim + 1].GetValue(&curTo).value_or(0)};195 extents[dim] = ub >= lb ? ub - lb + 1 : 0;196 elements *= extents[dim];197 }198 if (elements != 0) {199 const typeInfo::DerivedType &compType{*component->derivedType()};200 // Place a pair of static descriptors onto the descriptors stack.201 descriptorsStack.emplace();202 StaticDescriptorsPair &descs{descriptorsStack.top()};203 Descriptor &toCompDesc{descs.to.descriptor()};204 toCompDesc.Establish(compType, toPtr + component->offset(),205 component->rank(), extents);206 Descriptor &fromCompDesc{descs.from.descriptor()};207 fromCompDesc.Establish(compType, fromPtr + component->offset(),208 component->rank(), extents);209 copyStack.emplace(toCompDesc, fromCompDesc,210 /*usesStaticDescriptors=*/true);211 }212 }213 }214 }215 }216 }217}218RT_OFFLOAD_API_GROUP_END219} // namespace Fortran::runtime220