RetroSearch Browse

(

), HasCancel(HasCancel) {}

74

CGOpenMPRegionInfo(

const

CGOpenMPRegionKind RegionKind,

77

: CGCapturedStmtInfo(

CR_OpenMP

), RegionKind(RegionKind), CodeGen(CodeGen),

78 Kind

(

), HasCancel(HasCancel) {}

82 virtual const VarDecl

*getThreadIDVariable()

const

= 0;

93

CGOpenMPRegionKind getRegionKind()

const

{

return

RegionKind; }

97 bool

hasCancel()

const

{

return

HasCancel; }

99 static bool

classof(

const

CGCapturedStmtInfo *Info) {

103

~CGOpenMPRegionInfo()

override

default

;

106

CGOpenMPRegionKind RegionKind;

113class

CGOpenMPOutlinedRegionInfo final :

public

CGOpenMPRegionInfo {

118

StringRef HelperName)

119

: CGOpenMPRegionInfo(CS, ParallelOutlinedRegion, CodeGen,

121

ThreadIDVar(ThreadIDVar), HelperName(HelperName) {

122

assert(ThreadIDVar !=

nullptr

"No ThreadID in OpenMP region."

);

127 const VarDecl

*getThreadIDVariable()

const override

{

return

ThreadIDVar; }

130

StringRef getHelperName()

const override

{

return

HelperName; }

132 static bool

classof(

const

CGCapturedStmtInfo *Info) {

133 return

CGOpenMPRegionInfo::classof(Info) &&

134

cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==

135

ParallelOutlinedRegion;

142

StringRef HelperName;

146class

CGOpenMPTaskOutlinedRegionInfo final :

public

CGOpenMPRegionInfo {

152

llvm::SwitchInst *UntiedSwitch =

nullptr

;

155

UntiedTaskActionTy(

bool

Tied,

const VarDecl

*PartIDVar,

157

: Untied(!Tied), PartIDVar(PartIDVar), UntiedCodeGen(UntiedCodeGen) {}

167

UntiedSwitch = CGF.

.CreateSwitch(Res, DoneBB);

171

UntiedSwitch->addCase(CGF.

.getInt32(0),

172

CGF.

.GetInsertBlock());

173

emitUntiedSwitch(CGF);

184

CodeGenFunction::JumpDest CurPoint =

188

UntiedSwitch->addCase(CGF.

.getInt32(UntiedSwitch->getNumCases()),

189

CGF.

.GetInsertBlock());

194 unsigned

getNumberOfParts()

const

{

return

UntiedSwitch->getNumCases(); }

200 const

UntiedTaskActionTy &Action)

201

: CGOpenMPRegionInfo(CS, TaskOutlinedRegion, CodeGen,

, HasCancel),

202

ThreadIDVar(ThreadIDVar), Action(Action) {

203

assert(ThreadIDVar !=

nullptr

"No ThreadID in OpenMP region."

);

208 const VarDecl

*getThreadIDVariable()

const override

{

return

ThreadIDVar; }

214

StringRef getHelperName()

const override

{

return ".omp_outlined."

; }

217

Action.emitUntiedSwitch(CGF);

220 static bool

classof(

const

CGCapturedStmtInfo *Info) {

221 return

CGOpenMPRegionInfo::classof(Info) &&

222

cast<CGOpenMPRegionInfo>(Info)->getRegionKind() ==

231 const

UntiedTaskActionTy &Action;

236class

CGOpenMPInlinedRegionInfo :

public

CGOpenMPRegionInfo {

238

CGOpenMPInlinedRegionInfo(CodeGenFunction::CGCapturedStmtInfo *OldCSI,

241

: CGOpenMPRegionInfo(InlinedRegion, CodeGen,

, HasCancel),

243

OuterRegionInfo(dyn_cast_or_null<CGOpenMPRegionInfo>(OldCSI)) {}

246

llvm::Value *getContextValue()

const override

{

248 return

OuterRegionInfo->getContextValue();

249

llvm_unreachable(

"No context value for inlined OpenMP region"

);

252 void

setContextValue(llvm::Value *

)

override

{

253 if

(OuterRegionInfo) {

254

OuterRegionInfo->setContextValue(

);

257

llvm_unreachable(

"No context value for inlined OpenMP region"

);

263 return

OuterRegionInfo->lookup(VD);

269 FieldDecl

*getThisFieldDecl()

const override

{

271 return

OuterRegionInfo->getThisFieldDecl();

277 const VarDecl

*getThreadIDVariable()

const override

{

279 return

OuterRegionInfo->getThreadIDVariable();

286 return

OuterRegionInfo->getThreadIDVariableLValue(CGF);

287

llvm_unreachable(

"No LValue for inlined OpenMP construct"

);

291

StringRef getHelperName()

const override

{

292 if

(

auto

*OuterRegionInfo = getOldCSI())

293 return

OuterRegionInfo->getHelperName();

294

llvm_unreachable(

"No helper name for inlined OpenMP construct"

);

299

OuterRegionInfo->emitUntiedSwitch(CGF);

302

CodeGenFunction::CGCapturedStmtInfo *getOldCSI()

const

{

return

OldCSI; }

304 static bool

classof(

const

CGCapturedStmtInfo *Info) {

305 return

CGOpenMPRegionInfo::classof(Info) &&

306

cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == InlinedRegion;

309

~CGOpenMPInlinedRegionInfo()

override

default

;

313

CodeGenFunction::CGCapturedStmtInfo *OldCSI;

314

CGOpenMPRegionInfo *OuterRegionInfo;

322class

CGOpenMPTargetRegionInfo final :

public

CGOpenMPRegionInfo {

326

: CGOpenMPRegionInfo(CS, TargetRegion, CodeGen, OMPD_target,

328

HelperName(HelperName) {}

332 const VarDecl

*getThreadIDVariable()

const override

{

return nullptr

; }

335

StringRef getHelperName()

const override

{

return

HelperName; }

337 static bool

classof(

const

CGCapturedStmtInfo *Info) {

338 return

CGOpenMPRegionInfo::classof(Info) &&

339

cast<CGOpenMPRegionInfo>(Info)->getRegionKind() == TargetRegion;

343

StringRef HelperName;

347

llvm_unreachable(

"No codegen for expressions"

);

351class

CGOpenMPInnerExprInfo final :

public

CGOpenMPInlinedRegionInfo {

354

: CGOpenMPInlinedRegionInfo(CGF.CapturedStmtInfo, EmptyCodeGen,

361 for

(

const auto

: CS.

captures

()) {

362 if

.capturesVariable() && !

.capturesVariableByCopy())

365 const VarDecl

*VD =

.getCapturedVar();

375

(void)PrivScope.Privatize();

380 if

(

const FieldDecl

*FD = CGOpenMPInlinedRegionInfo::lookup(VD))

387

llvm_unreachable(

"No body for expressions"

);

392 const VarDecl

*getThreadIDVariable()

const override

{

393

llvm_unreachable(

"No thread id for expressions"

);

397

StringRef getHelperName()

const override

{

398

llvm_unreachable(

"No helper name for expressions"

);

401 static bool

classof(

const

CGCapturedStmtInfo *Info) {

return false

; }

405

CodeGenFunction::OMPPrivateScope PrivScope;

409class

InlinedOpenMPRegionRAII {

411

llvm::DenseMap<const ValueDecl *, FieldDecl *> LambdaCaptureFields;

412 FieldDecl

*LambdaThisCaptureField =

nullptr

;

414 bool

NoInheritance =

false

;

423 bool

NoInheritance =

true

)

424

: CGF(CGF), NoInheritance(NoInheritance) {

437

~InlinedOpenMPRegionRAII() {

454enum

OpenMPLocationFlags :

unsigned

{

456

OMP_IDENT_IMD = 0x01,

458

OMP_IDENT_KMPC = 0x02,

460

OMP_ATOMIC_REDUCE = 0x10,

462

OMP_IDENT_BARRIER_EXPL = 0x20,

464

OMP_IDENT_BARRIER_IMPL = 0x40,

466

OMP_IDENT_BARRIER_IMPL_FOR = 0x40,

468

OMP_IDENT_BARRIER_IMPL_SECTIONS = 0xC0,

470

OMP_IDENT_BARRIER_IMPL_SINGLE = 0x140,

472

OMP_IDENT_WORK_LOOP = 0x200,

474

OMP_IDENT_WORK_SECTIONS = 0x400,

476

OMP_IDENT_WORK_DISTRIBUTE = 0x800,

506enum

IdentFieldIndex {

508

IdentField_Reserved_1,

512

IdentField_Reserved_2,

514

IdentField_Reserved_3,

523enum

OpenMPSchedType {

526

OMP_sch_static_chunked = 33,

528

OMP_sch_dynamic_chunked = 35,

529

OMP_sch_guided_chunked = 36,

530

OMP_sch_runtime = 37,

533

OMP_sch_static_balanced_chunked = 45,

536

OMP_ord_static_chunked = 65,

538

OMP_ord_dynamic_chunked = 67,

539

OMP_ord_guided_chunked = 68,

540

OMP_ord_runtime = 69,

542

OMP_sch_default = OMP_sch_static,

544

OMP_dist_sch_static_chunked = 91,

545

OMP_dist_sch_static = 92,

548

OMP_sch_modifier_monotonic = (1 << 29),

550

OMP_sch_modifier_nonmonotonic = (1 << 30),

573

Callback(CodeGen, CGF, *PrePostAction);

576

Callback(CodeGen, CGF, Action);

584 if

(

const auto

*CE = dyn_cast<CallExpr>(ReductionOp))

585 if

(

const auto

*OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))

586 if

(

const auto

*DRE =

587

dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))

588 if

(

const auto

*DRD = dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl()))

599

std::pair<llvm::Function *, llvm::Function *>

Reduction

601 const auto

*CE = cast<CallExpr>(InitOp);

602 const auto

*OVE = cast<OpaqueValueExpr>(CE->getCallee());

606

cast<DeclRefExpr>(cast<UnaryOperator>(LHS)->getSubExpr());

608

cast<DeclRefExpr>(cast<UnaryOperator>(RHS)->getSubExpr());

609

CodeGenFunction::OMPPrivateScope PrivateScope(CGF);

610

PrivateScope.addPrivate(cast<VarDecl>(LHSDRE->getDecl()),

Private

);

611

PrivateScope.addPrivate(cast<VarDecl>(RHSDRE->getDecl()), Original);

612

(void)PrivateScope.Privatize();

614

CodeGenFunction::OpaqueValueMapping Map(CGF, OVE,

Func

);

619 auto

*GV =

new

llvm::GlobalVariable(

621

llvm::GlobalValue::PrivateLinkage,

, Name);

634

CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, LV);

641

CodeGenFunction::OpaqueValueMapping OpaqueMap(CGF, &OVE, InitRVal);

662

llvm::Value *NumElements = CGF.

emitArrayLength

(ArrayTy, ElementTy, DestAddr);

666

llvm::Value *SrcBegin =

nullptr

;

668

SrcBegin = SrcAddr.emitRawPointer(CGF);

671

llvm::Value *DestEnd =

676

llvm::Value *IsEmpty =

677

CGF.

.CreateICmpEQ(DestBegin, DestEnd,

"omp.arrayinit.isempty"

);

678

CGF.

.CreateCondBr(IsEmpty, DoneBB, BodyBB);

681

llvm::BasicBlock *EntryBB = CGF.

.GetInsertBlock();

686

llvm::PHINode *SrcElementPHI =

nullptr

;

689

SrcElementPHI = CGF.

.CreatePHI(SrcBegin->getType(), 2,

690 "omp.arraycpy.srcElementPast"

);

691

SrcElementPHI->addIncoming(SrcBegin, EntryBB);

693 Address

(SrcElementPHI, SrcAddr.getElementType(),

694

SrcAddr.getAlignment().alignmentOfArrayElement(ElementSize));

696

llvm::PHINode *DestElementPHI = CGF.

.CreatePHI(

697

DestBegin->getType(), 2,

"omp.arraycpy.destElementPast"

);

698

DestElementPHI->addIncoming(DestBegin, EntryBB);

705

CodeGenFunction::RunCleanupsScope InitScope(CGF);

706 if

(EmitDeclareReductionInit) {

708

SrcElementCurrent, ElementTy);

716

llvm::Value *SrcElementNext = CGF.

.CreateConstGEP1_32(

717

SrcAddr.getElementType(), SrcElementPHI,

718 "omp.arraycpy.dest.element"

);

719

SrcElementPHI->addIncoming(SrcElementNext, CGF.

.GetInsertBlock());

723

llvm::Value *DestElementNext = CGF.

.CreateConstGEP1_32(

725 "omp.arraycpy.dest.element"

);

728

CGF.

.CreateICmpEQ(DestElementNext, DestEnd,

"omp.arraycpy.done"

);

729

CGF.

.CreateCondBr(Done, DoneBB, BodyBB);

730

DestElementPHI->addIncoming(DestElementNext, CGF.

.GetInsertBlock());

742 if

(

const auto

*OASE = dyn_cast<ArraySectionExpr>(

))

747void

ReductionCodeGen::emitAggregateInitialization(

753 const auto

*PrivateVD =

754

cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].

Private

)->getDecl());

755 bool

EmitDeclareReductionInit =

758

EmitDeclareReductionInit,

759

EmitDeclareReductionInit ? ClausesData[N].ReductionOp

760

: PrivateVD->getInit(),

768

ClausesData.reserve(Shareds.size());

769

SharedAddresses.reserve(Shareds.size());

770

Sizes.reserve(Shareds.size());

771

BaseDecls.reserve(Shareds.size());

772 const auto

*IOrig = Origs.begin();

773 const auto

*IPriv = Privates.begin();

774 const auto

*IRed = ReductionOps.begin();

775 for

(

const Expr

*Ref : Shareds) {

776

ClausesData.emplace_back(Ref, *IOrig, *IPriv, *IRed);

777

std::advance(IOrig, 1);

778

std::advance(IPriv, 1);

779

std::advance(IRed, 1);

784

assert(SharedAddresses.size() == N && OrigAddresses.size() == N &&

785 "Number of generated lvalues must be exactly N."

);

786 LValue First

= emitSharedLValue(CGF, ClausesData[N].Shared);

787 LValue

Second = emitSharedLValueUB(CGF, ClausesData[N].Shared);

788

SharedAddresses.emplace_back(

First

, Second);

789 if

(ClausesData[N].Shared == ClausesData[N].Ref) {

790

OrigAddresses.emplace_back(

First

, Second);

792 LValue First

= emitSharedLValue(CGF, ClausesData[N].Ref);

793 LValue

Second = emitSharedLValueUB(CGF, ClausesData[N].Ref);

794

OrigAddresses.emplace_back(

First

, Second);

800 bool

AsArraySection = isa<ArraySectionExpr>(ClausesData[N].Ref);

803

CGF.

(OrigAddresses[N].first.getType().getNonReferenceType()),

808

llvm::Value *SizeInChars;

809 auto

*ElemType = OrigAddresses[N].first.getAddress().getElementType();

810 auto

*ElemSizeOf = llvm::ConstantExpr::getSizeOf(ElemType);

811 if

(AsArraySection) {

812

Size = CGF.

.CreatePtrDiff(ElemType,

813

OrigAddresses[N].second.getPointer(CGF),

814

OrigAddresses[N].first.getPointer(CGF));

815

Size = CGF.

.CreateNUWAdd(

816

Size, llvm::ConstantInt::get(Size->getType(),

1));

817

SizeInChars = CGF.

.CreateNUWMul(Size, ElemSizeOf);

820

CGF.

(OrigAddresses[N].first.getType().getNonReferenceType());

821

Size = CGF.

.CreateExactUDiv(SizeInChars, ElemSizeOf);

823

Sizes.emplace_back(SizeInChars, Size);

826

cast<OpaqueValueExpr>(

836

assert(!Size && !Sizes[N].second &&

837 "Size should be nullptr for non-variably modified reduction " 843

cast<OpaqueValueExpr>(

852

assert(SharedAddresses.size() > N &&

"No variable was generated"

);

853 const auto

*PrivateVD =

854

cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].

Private

)->getDecl());

859

(void)DefaultInit(CGF);

860

emitAggregateInitialization(CGF, N, PrivateAddr, SharedAddr, DRD);

861

}

else if

(DRD && (DRD->

getInitializer

() || !PrivateVD->hasInit())) {

862

(void)DefaultInit(CGF);

863 QualType

SharedType = SharedAddresses[N].first.getType();

865

PrivateAddr, SharedAddr, SharedType);

866

}

else if

(!DefaultInit(CGF) && PrivateVD->hasInit() &&

869

PrivateVD->

().getQualifiers(),

887

CGF.

pushDestroy

(DTorKind, PrivateAddr, PrivateType);

911 Address

OriginalBaseAddress, llvm::Value *Addr) {

935

Addr, OriginalBaseAddress.

());

940 const VarDecl

*OrigVD =

nullptr

;

941 if

(

const auto

*OASE = dyn_cast<ArraySectionExpr>(Ref)) {

942 const Expr

= OASE->getBase()->IgnoreParenImpCasts();

943 while

(

const auto

*TempOASE = dyn_cast<ArraySectionExpr>(

))

944 Base

= TempOASE->getBase()->IgnoreParenImpCasts();

945 while

(

const auto

*TempASE = dyn_cast<ArraySubscriptExpr>(

))

946 Base

= TempASE->getBase()->IgnoreParenImpCasts();

947

DE = cast<DeclRefExpr>(

);

948

OrigVD = cast<VarDecl>(DE->

());

949

}

else if

(

const auto

*ASE = dyn_cast<ArraySubscriptExpr>(Ref)) {

950 const Expr

= ASE->getBase()->IgnoreParenImpCasts();

951 while

(

const auto

*TempASE = dyn_cast<ArraySubscriptExpr>(

))

952 Base

= TempASE->getBase()->IgnoreParenImpCasts();

953

DE = cast<DeclRefExpr>(

);

954

OrigVD = cast<VarDecl>(DE->

());

963

BaseDecls.emplace_back(OrigVD);

966 loadToBegin

(CGF, OrigVD->getType(), SharedAddresses[N].first.getType(),

968 Address

SharedAddr = SharedAddresses[N].first.getAddress();

969

llvm::Value *Adjustment = CGF.

.CreatePtrDiff(

972

llvm::Value *PrivatePointer =

978

SharedAddresses[N].first.getType(),

981

BaseDecls.emplace_back(

982

cast<VarDecl>(cast<DeclRefExpr>(ClausesData[N].Ref)->getDecl()));

995

getThreadIDVariable()->getType()->castAs<PointerType>());

1013LValue

CGOpenMPTaskOutlinedRegionInfo::getThreadIDVariableLValue(

1016

getThreadIDVariable()->getType(),

1032

: CGM(CGM), OMPBuilder(CGM.getModule()) {

1034

llvm::OpenMPIRBuilderConfig Config(

1049 OMPBuilder

.Config.setHasRequiresUnifiedSharedMemory(

true

);

1057 if

.getValue().pointsToAliveValue())

1059 auto

*GV = dyn_cast<llvm::GlobalVariable>(

.getValue());

1062 if

(!GV->isDeclaration() || GV->getNumUses() > 0)

1064

GV->eraseFromParent();

1069 return OMPBuilder

.createPlatformSpecificName(Parts);

1072static

llvm::Function *

1074 const Expr

*CombinerInitializer,

const VarDecl

*In,

1075 const VarDecl

*Out,

bool

IsCombiner) {

1078 QualType

PtrTy =

.getPointerType(Ty).withRestrict();

1084

Args.push_back(&OmpOutParm);

1085

Args.push_back(&OmpInParm);

1090

{IsCombiner ?

"omp_combiner"

"omp_initializer"

""

});

1091 auto

*Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,

1095

Fn->removeFnAttr(llvm::Attribute::NoInline);

1096

Fn->removeFnAttr(llvm::Attribute::OptimizeNone);

1097

Fn->addFnAttr(llvm::Attribute::AlwaysInline);

1103

Out->getLocation());

1104

CodeGenFunction::OMPPrivateScope

(CGF);

1113

(void)

.Privatize();

1114 if

(!IsCombiner && Out->hasInit() &&

1117

Out->getType().getQualifiers(),

1120 if

(CombinerInitializer)

1122 Scope

.ForceCleanup();

1132 CGM

->getType(),

->getCombiner(),

1133

cast<VarDecl>(cast<DeclRefExpr>(

->getCombinerIn())->getDecl()),

1134

cast<VarDecl>(cast<DeclRefExpr>(

->getCombinerOut())->getDecl()),

1137 if

(

const Expr

->getInitializer()) {

1139 CGM

->getType(),

1142

cast<VarDecl>(cast<DeclRefExpr>(

->getInitOrig())->getDecl()),

1143

cast<VarDecl>(cast<DeclRefExpr>(

->getInitPriv())->getDecl()),

1151

std::pair<llvm::Function *, llvm::Function *>

1163struct

PushAndPopStackRAII {

1164

PushAndPopStackRAII(llvm::OpenMPIRBuilder *OMPBuilder,

CodeGenFunction

&CGF,

1165 bool

HasCancel, llvm::omp::Directive

)

1166

: OMPBuilder(OMPBuilder) {

1182 auto

FiniCB = [&CGF](llvm::OpenMPIRBuilder::InsertPointTy IP) {

1183

assert(IP.getBlock()->end() == IP.getPoint() &&

1184 "Clang CG should cause non-terminated block!"

);

1185

CGBuilderTy::InsertPointGuard IPG(CGF.

);

1190 return

llvm::Error::success();

1195

llvm::OpenMPIRBuilder::FinalizationInfo FI({FiniCB,

, HasCancel});

1196

OMPBuilder->pushFinalizationCB(std::move(FI));

1198

~PushAndPopStackRAII() {

1200

OMPBuilder->popFinalizationCB();

1202

llvm::OpenMPIRBuilder *OMPBuilder;

1211 "thread id variable must be of type kmp_int32 *"

);

1213 bool

HasCancel =

false

;

1214 if

(

const auto

*OPD = dyn_cast<OMPParallelDirective>(&

))

1215

HasCancel = OPD->hasCancel();

1216 else if

(

const auto

*OPD = dyn_cast<OMPTargetParallelDirective>(&

))

1217

HasCancel = OPD->hasCancel();

1218 else if

(

const auto

*OPSD = dyn_cast<OMPParallelSectionsDirective>(&

))

1219

HasCancel = OPSD->hasCancel();

1220 else if

(

const auto

*OPFD = dyn_cast<OMPParallelForDirective>(&

))

1221

HasCancel = OPFD->hasCancel();

1222 else if

(

const auto

*OPFD = dyn_cast<OMPTargetParallelForDirective>(&

))

1223

HasCancel = OPFD->hasCancel();

1224 else if

(

const auto

*OPFD = dyn_cast<OMPDistributeParallelForDirective>(&

))

1225

HasCancel = OPFD->hasCancel();

1226 else if

(

const auto

*OPFD =

1227

dyn_cast<OMPTeamsDistributeParallelForDirective>(&

))

1228

HasCancel = OPFD->hasCancel();

1229 else if

(

const auto

*OPFD =

1230

dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&

))

1231

HasCancel = OPFD->hasCancel();

1236

PushAndPopStackRAII PSR(&OMPBuilder, CGF, HasCancel, InnermostKind);

1237

CGOpenMPOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen, InnermostKind,

1238

HasCancel, OutlinedHelperName);

1239

CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);

1244

std::string Suffix =

({

"omp_outlined"

});

1245 return

(Name + Suffix).str();

1253

std::string Suffix =

({

"omp"

"reduction"

"reduction_func"

});

1254 return

(Name + Suffix).str();

1281 bool

Tied,

unsigned

&NumberOfParts) {

1286

llvm::Value *TaskArgs[] = {

1288

CGF.EmitLoadOfPointerLValue(CGF.GetAddrOfLocalVar(TaskTVar),

1291

CGF.EmitRuntimeCall(

.getOrCreateRuntimeFunction(

1295

CGOpenMPTaskOutlinedRegionInfo::UntiedTaskActionTy Action(Tied, PartIDVar,

1299 "thread id variable must be of type kmp_int32 for tasks"

);

1304 bool

HasCancel =

false

;

1305 if

(

const auto

*TD = dyn_cast<OMPTaskDirective>(&

))

1306

HasCancel = TD->hasCancel();

1307 else if

(

const auto

*TD = dyn_cast<OMPTaskLoopDirective>(&

))

1308

HasCancel = TD->hasCancel();

1309 else if

(

const auto

*TD = dyn_cast<OMPMasterTaskLoopDirective>(&

))

1310

HasCancel = TD->hasCancel();

1311 else if

(

const auto

*TD = dyn_cast<OMPParallelMasterTaskLoopDirective>(&

))

1312

HasCancel = TD->hasCancel();

1315

CGOpenMPTaskOutlinedRegionInfo CGInfo(*CS, ThreadIDVar, CodeGen,

1316

InnermostKind, HasCancel, Action);

1318

llvm::Function *Res = CGF.GenerateCapturedStmtFunction(*CS);

1320

NumberOfParts = Action.getNumberOfParts();

1325 bool

AtCurrentPoint) {

1327

assert(!Elem.ServiceInsertPt &&

"Insert point is set already."

);

1329

llvm::Value *Undef = llvm::UndefValue::get(CGF.

);

1330 if

(AtCurrentPoint) {

1331

Elem.ServiceInsertPt =

new

llvm::BitCastInst(Undef, CGF.

"svcpt"

1332

CGF.

.GetInsertBlock());

1334

Elem.ServiceInsertPt =

new

llvm::BitCastInst(Undef, CGF.

"svcpt"

);

1335

Elem.ServiceInsertPt->insertAfter(CGF.

AllocaInsertPt

->getIterator());

1341 if

(Elem.ServiceInsertPt) {

1342

llvm::Instruction *Ptr = Elem.ServiceInsertPt;

1343

Elem.ServiceInsertPt =

nullptr

;

1344

Ptr->eraseFromParent();

1351

llvm::raw_svector_ostream OS(Buffer);

1355 if

(

const auto

*FD = dyn_cast_or_null<FunctionDecl>(CGF.

CurFuncDecl

))

1356

OS << FD->getQualifiedNameAsString();

1363 unsigned

Flags,

bool

EmitLoc) {

1364

uint32_t SrcLocStrSize;

1365

llvm::Constant *SrcLocStr;

1367

llvm::codegenoptions::NoDebugInfo) ||

1369

SrcLocStr =

.getOrCreateDefaultSrcLocStr(SrcLocStrSize);

1371

std::string FunctionName;

1372 if

(

const auto

*FD = dyn_cast_or_null<FunctionDecl>(CGF.

CurFuncDecl

))

1373

FunctionName = FD->getQualifiedNameAsString();

1383

SrcLocStr, SrcLocStrSize, llvm::omp::IdentFlag(Flags), Reserved2Flags);

1388

assert(CGF.

"No function in current CodeGenFunction."

);

1394

uint32_t SrcLocStrSize;

1395 auto

*SrcLocStr =

.getOrCreateSrcLocStr(

1398 OMPBuilder

.getOrCreateIdent(SrcLocStr, SrcLocStrSize));

1401

llvm::Value *ThreadID =

nullptr

;

1406

ThreadID = I->second.ThreadID;

1407 if

(ThreadID !=

nullptr

)

1411 if

(

auto

*OMPRegionInfo =

1413 if

(OMPRegionInfo->getThreadIDVariable()) {

1415 LValue

LVal = OMPRegionInfo->getThreadIDVariableLValue(CGF);

1419

CGF.

.GetInsertBlock() == TopBlock ||

1420

!isa<llvm::Instruction>(LVal.

(CGF)) ||

1421

cast<llvm::Instruction>(LVal.

(CGF))->getParent() ==

1423

cast<llvm::Instruction>(LVal.

(CGF))->getParent() ==

1424

CGF.

.GetInsertBlock()) {

1428 if

(CGF.

.GetInsertBlock() == TopBlock)

1440 if

(!Elem.ServiceInsertPt)

1442

CGBuilderTy::InsertPointGuard IPG(CGF.

);

1443

CGF.

.SetInsertPoint(Elem.ServiceInsertPt);

1447

OMPRTL___kmpc_global_thread_num),

1450

Elem.ThreadID =

Call

;

1455

assert(CGF.

"No function in current CodeGenFunction."

);

1467 for

(

const auto

: I->second)

1482

llvm::Type *MicroParams[] = {llvm::PointerType::getUnqual(

1483

llvm::PointerType::getUnqual(

)};

1489static

llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseKind

1491

std::optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy =

1492

OMPDeclareTargetDeclAttr::getDeviceType(VD);

1494 return

llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNone;

1496 switch

((

int

)*DevTy) {

1497 case

OMPDeclareTargetDeclAttr::DT_Host:

1498 return

llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseHost;

1500 case

OMPDeclareTargetDeclAttr::DT_NoHost:

1501 return

llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNoHost;

1503 case

OMPDeclareTargetDeclAttr::DT_Any:

1504 return

llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseAny;

1507 return

llvm::OffloadEntriesInfoManager::OMPTargetDeviceClauseNone;

1512static

llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryKind

1514

std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> MapType =

1515

OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);

1517 return

llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryNone;

1518 switch

((

int

)*MapType) {

1519 case

OMPDeclareTargetDeclAttr::MapTypeTy::MT_To:

1520 return

llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryTo;

1522 case

OMPDeclareTargetDeclAttr::MapTypeTy::MT_Enter:

1523 return

llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryEnter;

1525 case

OMPDeclareTargetDeclAttr::MapTypeTy::MT_Link:

1526 return

llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryLink;

1529 return

llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryNone;

1538 auto

FileInfoCallBack = [&]() {

1542

llvm::sys::fs::UniqueID ID;

1543 if

(llvm::sys::fs::getUniqueID(PLoc.

getFilename

(), ID)) {

1544

PLoc =

.getPresumedLoc(BeginLoc,

false

);

1550 return

OMPBuilder.getTargetEntryUniqueInfo(FileInfoCallBack, ParentName);

1556 auto

LinkageForVariable = [&VD,

this

]() {

1560

std::vector<llvm::GlobalVariable *> GeneratedRefs;

1564

llvm::Constant *addr =

.getAddrOfDeclareTargetVar(

1572

LinkageForVariable);

1584

std::string Suffix =

({

"cache"

""

});

1585 return OMPBuilder

.getOrCreateInternalVariable(

1598

llvm::Value *Args[] = {

1606 CGM

(), OMPRTL___kmpc_threadprivate_cached),

1622

llvm::Value *Args[] = {

1625

Ctor, CopyCtor, Dtor};

1628 CGM

(), OMPRTL___kmpc_threadprivate_register),

1643

llvm::Value *Ctor =

nullptr

, *CopyCtor =

nullptr

, *Dtor =

nullptr

;

1653

Args.push_back(&Dst);

1658

std::string Name =

({

"__kmpc_global_ctor_"

""

});

1659

llvm::Function *Fn =

1685

Args.push_back(&Dst);

1690

std::string Name =

({

"__kmpc_global_dtor_"

""

});

1691

llvm::Function *Fn =

1716 if

(Ctor ==

nullptr

) {

1719 if

(Dtor ==

nullptr

) {

1723 auto

*InitFunctionTy =

1724

llvm::FunctionType::get(

VoidTy

false

);

1725

std::string Name =

({

"__omp_threadprivate_init_"

""

});

1735 return

InitFunction;

1743

llvm::GlobalValue *GV) {

1744

std::optional<OMPDeclareTargetDeclAttr *> ActiveAttr =

1745

OMPDeclareTargetDeclAttr::getActiveAttr(FD);

1748 if

(!ActiveAttr || !(*ActiveAttr)->getIndirect())

1755 OMPBuilder

.OffloadInfoManager.getTargetRegionEntryFnName(Name, EntryInfo);

1761

llvm::GlobalValue *Addr = GV;

1763

Addr =

new

llvm::GlobalVariable(

1765 true

, llvm::GlobalValue::ExternalLinkage, GV, Name,

1766 nullptr

, llvm::GlobalValue::NotThreadLocal,

1767 CGM

().getDataLayout().getDefaultGlobalsAddressSpace());

1768

Addr->setVisibility(llvm::GlobalValue::ProtectedVisibility);

1771 OMPBuilder

.OffloadInfoManager.registerDeviceGlobalVarEntryInfo(

1773

llvm::OffloadEntriesInfoManager::OMPTargetGlobalVarEntryIndirect,

1774

llvm::GlobalValue::WeakODRLinkage);

1780

std::string Suffix =

({

"artificial"

""

});

1782

llvm::GlobalVariable *GAddr =

.getOrCreateInternalVariable(

1783

VarLVType, Twine(Name).concat(Suffix).str());

1786

GAddr->setThreadLocal(

true

);

1787 return Address

(GAddr, GAddr->getValueType(),

1790

std::string CacheSuffix =

({

"cache"

""

});

1791

llvm::Value *Args[] = {

1799

Twine(Name).concat(Suffix).concat(CacheSuffix).str())};

1804 CGM

(), OMPRTL___kmpc_threadprivate_cached),

1850

llvm::Function *OutlinedFn,

1853

llvm::Value *NumThreads) {

1858 auto

&&ThenGen = [&M, OutlinedFn, CapturedVars, RTLoc,

1862

llvm::Value *Args[] = {

1864

CGF.

.getInt32(CapturedVars.size()),

1867

RealArgs.append(std::begin(Args), std::end(Args));

1868

RealArgs.append(CapturedVars.begin(), CapturedVars.end());

1870

llvm::FunctionCallee RTLFn =

1871 OMPBuilder

.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_fork_call);

1874 auto

&&ElseGen = [&M, OutlinedFn, CapturedVars, RTLoc,

1880

llvm::Value *Args[] = {RTLoc, ThreadID};

1882

M, OMPRTL___kmpc_serialized_parallel),

1889 ".bound.zero.addr"

);

1894

OutlinedFnArgs.push_back(ZeroAddrBound.

());

1895

OutlinedFnArgs.append(CapturedVars.begin(), CapturedVars.end());

1903

OutlinedFn->removeFnAttr(llvm::Attribute::AlwaysInline);

1904

OutlinedFn->addFnAttr(llvm::Attribute::NoInline);

1910

M, OMPRTL___kmpc_end_serialized_parallel),

1929 if

(

auto

*OMPRegionInfo =

1931 if

(OMPRegionInfo->getThreadIDVariable())

1932 return

OMPRegionInfo->getThreadIDVariableLValue(CGF).getAddress();

1941 return

ThreadIDTemp;

1945

std::string Prefix = Twine(

"gomp_critical_user_"

, CriticalName).str();

1946

std::string Name =

({Prefix,

"var"

});

1953

llvm::FunctionCallee EnterCallee;

1955

llvm::FunctionCallee ExitCallee;

1958

llvm::BasicBlock *ContBlock =

nullptr

;

1961

CommonActionTy(llvm::FunctionCallee EnterCallee,

1963

llvm::FunctionCallee ExitCallee,

1965

: EnterCallee(EnterCallee), EnterArgs(EnterArgs), ExitCallee(ExitCallee),

1966

ExitArgs(ExitArgs), Conditional(Conditional) {}

1968

llvm::Value *EnterRes = CGF.

EmitRuntimeCall

(EnterCallee, EnterArgs);

1970

llvm::Value *CallBool = CGF.

.CreateIsNotNull(EnterRes);

1974

CGF.

.CreateCondBr(CallBool, ThenBlock, ContBlock);

1990

StringRef CriticalName,

2004

EnterArgs.push_back(CGF.

.CreateIntCast(

2007

CommonActionTy Action(

2010

Hint ? OMPRTL___kmpc_critical_with_hint : OMPRTL___kmpc_critical),

2013

OMPRTL___kmpc_end_critical),

2030

CommonActionTy Action(

.getOrCreateRuntimeFunction(

2052

llvm::Value *FilterVal = Filter

2054

: llvm::ConstantInt::get(

0);

2059

CommonActionTy Action(

.getOrCreateRuntimeFunction(

2079

llvm::Value *Args[] = {

2081

llvm::ConstantInt::get(

true

)};

2087 if

(

auto

*Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.

CapturedStmtInfo

))

2088

Region->emitUntiedSwitch(CGF);

2101

CommonActionTy Action(

.getOrCreateRuntimeFunction(

2114 unsigned

Index,

const VarDecl

*Var) {

2135

Args.push_back(&LHSArg);

2136

Args.push_back(&RHSArg);

2142

llvm::GlobalValue::InternalLinkage, Name,

2145

Fn->setDoesNotRecurse();

2162 for

(

unsigned

I = 0,

= AssignmentOps.size(); I <

; ++I) {

2163 const auto

*DestVar =

2164

cast<VarDecl>(cast<DeclRefExpr>(DestExprs[I])->getDecl());

2167 const auto

*SrcVar =

2168

cast<VarDecl>(cast<DeclRefExpr>(SrcExprs[I])->getDecl());

2171 const auto

*VD = cast<DeclRefExpr>(CopyprivateVars[I])->getDecl();

2173

CGF.

EmitOMPCopy

(

Type

, DestAddr, SrcAddr, DestVar, SrcVar, AssignmentOps[I]);

2188

assert(CopyprivateVars.size() == SrcExprs.size() &&

2189

CopyprivateVars.size() == DstExprs.size() &&

2190

CopyprivateVars.size() == AssignmentOps.size());

2202 if

(!CopyprivateVars.empty()) {

2205 C

.getIntTypeForBitwidth(

32,

1);

2206

DidIt = CGF.

(KmpInt32Ty,

".omp.copyprivate.did_it"

);

2211

CommonActionTy Action(

.getOrCreateRuntimeFunction(

2228

llvm::APInt ArraySize(

32, CopyprivateVars.size());

2229 QualType

CopyprivateArrayTy =

.getConstantArrayType(

2234

CGF.

(CopyprivateArrayTy,

".omp.copyprivate.cpr_list"

);

2235 for

(

unsigned

I = 0,

= CopyprivateVars.size(); I <

; ++I) {

2247

SrcExprs, DstExprs, AssignmentOps,

);

2248

llvm::Value *BufSize = CGF.

(CopyprivateArrayTy);

2252

llvm::Value *Args[] = {

2277

CommonActionTy Action(

.getOrCreateRuntimeFunction(

2292 if

(Kind == OMPD_for)

2293

Flags = OMP_IDENT_BARRIER_IMPL_FOR;

2294 else if

(Kind == OMPD_sections)

2295

Flags = OMP_IDENT_BARRIER_IMPL_SECTIONS;

2296 else if

(Kind == OMPD_single)

2297

Flags = OMP_IDENT_BARRIER_IMPL_SINGLE;

2298 else if

(Kind == OMPD_barrier)

2299

Flags = OMP_IDENT_BARRIER_EXPL;

2301

Flags = OMP_IDENT_BARRIER_IMPL;

2313

ScheduleKind = OMPC_SCHEDULE_static;

2315

llvm::APInt ChunkSize(32, 1);

2325 bool

ForceSimpleCall) {

2327 auto

*OMPRegionInfo =

2330

llvm::OpenMPIRBuilder::InsertPointTy AfterIP =

2333

CGF.

.restoreIP(AfterIP);

2346 if

(OMPRegionInfo) {

2347 if

(!ForceSimpleCall && OMPRegionInfo->hasCancel()) {

2350

OMPRTL___kmpc_cancel_barrier),

2359

CGF.

.CreateCondBr(Cmp, ExitBB, ContBB);

2376 Expr

*ME,

bool

IsFatal) {

2379

: llvm::ConstantPointerNull::get(CGF.

);

2382

llvm::Value *Args[] = {

2384

llvm::ConstantInt::get(

, IsFatal ? 2 : 1),

2393 bool

Chunked,

bool

Ordered) {

2394 switch

(ScheduleKind) {

2395 case

OMPC_SCHEDULE_static:

2396 return

Chunked ? (Ordered ? OMP_ord_static_chunked : OMP_sch_static_chunked)

2397

: (Ordered ? OMP_ord_static : OMP_sch_static);

2398 case

OMPC_SCHEDULE_dynamic:

2399 return

Ordered ? OMP_ord_dynamic_chunked : OMP_sch_dynamic_chunked;

2400 case

OMPC_SCHEDULE_guided:

2401 return

Ordered ? OMP_ord_guided_chunked : OMP_sch_guided_chunked;

2402 case

OMPC_SCHEDULE_runtime:

2403 return

Ordered ? OMP_ord_runtime : OMP_sch_runtime;

2404 case

OMPC_SCHEDULE_auto:

2405 return

Ordered ? OMP_ord_auto : OMP_sch_auto;

2407

assert(!Chunked &&

"chunk was specified but schedule kind not known"

);

2408 return

Ordered ? OMP_ord_static : OMP_sch_static;

2410

llvm_unreachable(

"Unexpected runtime schedule"

);

2414static

OpenMPSchedType

2417 return

Chunked ? OMP_dist_sch_static_chunked : OMP_dist_sch_static;

2421 bool

Chunked)

const

{

2422

OpenMPSchedType Schedule =

2424 return

Schedule == OMP_sch_static;

2430 return

Schedule == OMP_dist_sch_static;

2434 bool

Chunked)

const

{

2435

OpenMPSchedType Schedule =

2437 return

Schedule == OMP_sch_static_chunked;

2443 return

Schedule == OMP_dist_sch_static_chunked;

2447

OpenMPSchedType Schedule =

2449

assert(Schedule != OMP_sch_static_chunked &&

"cannot be chunked here"

);

2450 return

Schedule != OMP_sch_static;

2458 case

OMPC_SCHEDULE_MODIFIER_monotonic:

2459

Modifier = OMP_sch_modifier_monotonic;

2461 case

OMPC_SCHEDULE_MODIFIER_nonmonotonic:

2462

Modifier = OMP_sch_modifier_nonmonotonic;

2464 case

OMPC_SCHEDULE_MODIFIER_simd:

2465 if

(Schedule == OMP_sch_static_chunked)

2466

Schedule = OMP_sch_static_balanced_chunked;

2473 case

OMPC_SCHEDULE_MODIFIER_monotonic:

2474

Modifier = OMP_sch_modifier_monotonic;

2476 case

OMPC_SCHEDULE_MODIFIER_nonmonotonic:

2477

Modifier = OMP_sch_modifier_nonmonotonic;

2479 case

OMPC_SCHEDULE_MODIFIER_simd:

2480 if

(Schedule == OMP_sch_static_chunked)

2481

Schedule = OMP_sch_static_balanced_chunked;

2493 if

(CGM.

().OpenMP >= 50 && Modifier == 0) {

2494 if

(!(Schedule == OMP_sch_static_chunked || Schedule == OMP_sch_static ||

2495

Schedule == OMP_sch_static_balanced_chunked ||

2496

Schedule == OMP_ord_static_chunked || Schedule == OMP_ord_static ||

2497

Schedule == OMP_dist_sch_static_chunked ||

2498

Schedule == OMP_dist_sch_static))

2499

Modifier = OMP_sch_modifier_nonmonotonic;

2501 return

Schedule | Modifier;

2511

ScheduleKind.

Schedule

, DispatchValues.

Chunk

nullptr

, Ordered);

2513

(Schedule != OMP_sch_static && Schedule != OMP_sch_static_chunked &&

2514

Schedule != OMP_ord_static && Schedule != OMP_ord_static_chunked &&

2515

Schedule != OMP_sch_static_balanced_chunked));

2522

llvm::Value *Chunk = DispatchValues.

Chunk

? DispatchValues.

Chunk 2523

: CGF.

.getIntN(IVSize, 1);

2524

llvm::Value *Args[] = {

2528 CGM

, Schedule, ScheduleKind.

, ScheduleKind.

)),

2529

DispatchValues.

2530

DispatchValues.

2531

CGF.

.getIntN(IVSize, 1),

2548 CodeGenFunction

&CGF, llvm::Value *UpdateLocation, llvm::Value *ThreadId,

2549

llvm::FunctionCallee ForStaticInitFunction, OpenMPSchedType Schedule,

2556

assert(Schedule == OMP_sch_static || Schedule == OMP_sch_static_chunked ||

2557

Schedule == OMP_sch_static_balanced_chunked ||

2558

Schedule == OMP_ord_static || Schedule == OMP_ord_static_chunked ||

2559

Schedule == OMP_dist_sch_static ||

2560

Schedule == OMP_dist_sch_static_chunked);

2567

llvm::Value *Chunk = Values.

Chunk

;

2568 if

(Chunk ==

nullptr

) {

2569

assert((Schedule == OMP_sch_static || Schedule == OMP_ord_static ||

2570

Schedule == OMP_dist_sch_static) &&

2571 "expected static non-chunked schedule"

);

2575

assert((Schedule == OMP_sch_static_chunked ||

2576

Schedule == OMP_sch_static_balanced_chunked ||

2577

Schedule == OMP_ord_static_chunked ||

2578

Schedule == OMP_dist_sch_static_chunked) &&

2579 "expected static chunked schedule"

);

2581

llvm::Value *Args[] = {

2604 "Expected loop-based or sections-based directive."

);

2607

? OMP_IDENT_WORK_LOOP

2608

: OMP_IDENT_WORK_SECTIONS);

2610

llvm::FunctionCallee StaticInitFunction =

2615

ScheduleNum, ScheduleKind.

, ScheduleKind.

, Values);

2622

OpenMPSchedType ScheduleNum =

2624

llvm::Value *UpdatedLocation =

2627

llvm::FunctionCallee StaticInitFunction;

2628 bool

isGPUDistribute =

2631

StaticInitFunction =

.createForStaticInitFunction(

2642

assert((DKind == OMPD_distribute || DKind == OMPD_for ||

2643

DKind == OMPD_sections) &&

2644 "Expected distribute, for, or sections directive kind"

);

2648

llvm::Value *Args[] = {

2651

(DKind == OMPD_target_teams_loop)

2652

? OMP_IDENT_WORK_DISTRIBUTE

2654

? OMP_IDENT_WORK_LOOP

2655

: OMP_IDENT_WORK_SECTIONS),

2663 CGM

(), OMPRTL___kmpc_distribute_static_fini),

2692

llvm::Value *Args[] = {

2700 OMPBuilder

.createDispatchNextFunction(IVSize, IVSigned), Args);

2707

llvm::Value *NumThreads,

2712

llvm::Value *Args[] = {

2721

ProcBindKind ProcBind,

2725

assert(ProcBind != OMP_PROC_BIND_unknown &&

"Unsupported proc_bind value."

);

2727

llvm::Value *Args[] = {

2729

llvm::ConstantInt::get(

unsigned

(ProcBind),

true

)};

2751enum

KmpTaskTFields {

2781

llvm::OpenMPIRBuilder::EmitMetadataErrorReportFunctionTy &&ErrorReportFn =

2782

[

this

](llvm::OpenMPIRBuilder::EmitMetadataErrorKind Kind,

2783 const

llvm::TargetRegionEntryInfo &EntryInfo) ->

void

{

2785 if

(Kind != llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_LINK_ERROR) {

2789 if

(I->getFirst().getUniqueID().getDevice() == EntryInfo.DeviceID &&

2790

I->getFirst().getUniqueID().getFile() == EntryInfo.FileID) {

2792

I->getFirst(), EntryInfo.Line, 1);

2798 case

llvm::OpenMPIRBuilder::EMIT_MD_TARGET_REGION_ERROR: {

2801 "%0 is incorrect: either the " 2802 "address or the ID is invalid."

);

2805 case

llvm::OpenMPIRBuilder::EMIT_MD_DECLARE_TARGET_ERROR: {

2808 "variable %0 is incorrect: the " 2809 "address is invalid."

);

2812 case

llvm::OpenMPIRBuilder::EMIT_MD_GLOBAL_VAR_LINK_ERROR: {

2815 "Offloading entry for declare target variable is incorrect: the " 2816 "address is invalid."

);

2822 OMPBuilder

.createOffloadEntriesAndInfoMetadata(ErrorReportFn);

2829 QualType

KmpRoutineEntryTyArgs[] = {KmpInt32Ty,

.VoidPtrTy};

2832 C

.getFunctionType(KmpInt32Ty, KmpRoutineEntryTyArgs, EPI));

2838struct

PrivateHelpersTy {

2839

PrivateHelpersTy(

const Expr

*OriginalRef,

const VarDecl

*Original,

2841

: OriginalRef(OriginalRef), Original(Original), PrivateCopy(PrivateCopy),

2842

PrivateElemInit(PrivateElemInit) {}

2843

PrivateHelpersTy(

const VarDecl

*Original) : Original(Original) {}

2844 const Expr

*OriginalRef =

nullptr

;

2845 const VarDecl

*Original =

nullptr

;

2846 const VarDecl

*PrivateCopy =

nullptr

;

2847 const VarDecl

*PrivateElemInit =

nullptr

;

2848 bool

isLocalPrivate()

const

{

2849 return

!OriginalRef && !PrivateCopy && !PrivateElemInit;

2852typedef

std::pair<

CharUnits

, PrivateHelpersTy> PrivateDataTy;

2857 if

(!CVD->

hasAttr

<OMPAllocateDeclAttr>())

2859 const auto

*AA = CVD->

getAttr

<OMPAllocateDeclAttr>();

2861 return

!(AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc &&

2862

!AA->getAllocator());

2867 if

(!Privates.empty()) {

2872 RecordDecl

*RD =

.buildImplicitRecord(

".kmp_privates.t"

);

2874 for

(

const auto

&Pair : Privates) {

2875 const VarDecl

*VD = Pair.second.Original;

2879 if

(Pair.second.isLocalPrivate()) {

2902 QualType

KmpRoutineEntryPointerQTy) {

2922 QualType

KmpCmplrdataTy =

.getRecordType(UD);

2923 RecordDecl

*RD =

.buildImplicitRecord(

"kmp_task_t"

);

2953 RecordDecl

*RD =

.buildImplicitRecord(

"kmp_task_t_with_privates"

);

2973static

llvm::Function *

2976 QualType

KmpTaskTWithPrivatesPtrQTy,

2978 QualType

SharedsPtrTy, llvm::Function *TaskFunction,

2979

llvm::Value *TaskPrivatesMap) {

2987

Args.push_back(&GtidArg);

2988

Args.push_back(&TaskTypeArg);

2989 const auto

&TaskEntryFnInfo =

2991

llvm::FunctionType *TaskEntryTy =

2994 auto

*TaskEntry = llvm::Function::Create(

2995

TaskEntryTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.

());

2997

TaskEntry->setDoesNotRecurse();

3012 const auto

*KmpTaskTWithPrivatesQTyRD =

3013

cast<RecordDecl>(KmpTaskTWithPrivatesQTy->

());

3016 const auto

*KmpTaskTQTyRD = cast<RecordDecl>(KmpTaskTQTy->

());

3017 auto

PartIdFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTPartId);

3019

llvm::Value *PartidParam = PartIdLVal.

(CGF);

3021 auto

SharedsFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds);

3027 auto

PrivatesFI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin(), 1);

3028

llvm::Value *PrivatesParam;

3029 if

(PrivatesFI != KmpTaskTWithPrivatesQTyRD->field_end()) {

3034

PrivatesParam = llvm::ConstantPointerNull::get(CGF.

);

3037

llvm::Value *CommonArgs[] = {

3038

GtidParam, PartidParam, PrivatesParam, TaskPrivatesMap,

3044

std::end(CommonArgs));

3046 auto

LBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound);

3049 auto

UBFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound);

3052 auto

StFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTStride);

3055 auto

LIFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTLastIter);

3058 auto

RFI = std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTReductions);

3061

CallArgs.push_back(LBParam);

3062

CallArgs.push_back(UBParam);

3063

CallArgs.push_back(StParam);

3064

CallArgs.push_back(LIParam);

3065

CallArgs.push_back(RParam);

3067

CallArgs.push_back(SharedsParam);

3080 QualType

KmpTaskTWithPrivatesPtrQTy,

3081 QualType

KmpTaskTWithPrivatesQTy) {

3089

Args.push_back(&GtidArg);

3090

Args.push_back(&TaskTypeArg);

3091 const auto

&DestructorFnInfo =

3093

llvm::FunctionType *DestructorFnTy =

3097 auto

*DestructorFn =

3098

llvm::Function::Create(DestructorFnTy, llvm::GlobalValue::InternalLinkage,

3102

DestructorFn->setDoesNotRecurse();

3110 const auto

*KmpTaskTWithPrivatesQTyRD =

3111

cast<RecordDecl>(KmpTaskTWithPrivatesQTy->

());

3112 auto

FI = std::next(KmpTaskTWithPrivatesQTyRD->field_begin());

3114 for

(

const auto

*Field :

3115

cast<RecordDecl>(FI->getType()->getAsTagDecl())->fields()) {

3117

Field->getType().isDestructedType()) {

3123 return

DestructorFn;

3143 C

nullptr

nullptr

3144 C

.getPointerType(PrivatesQTy).withConst().withRestrict(),

3146

Args.push_back(&TaskPrivatesArg);

3147

llvm::DenseMap<CanonicalDeclPtr<const VarDecl>,

unsigned

> PrivateVarsPos;

3148 unsigned

Counter = 1;

3149 for

(

const Expr

.PrivateVars) {

3151 C

nullptr

nullptr

3152 C

.getPointerType(

()))

3156 const auto

*VD = cast<VarDecl>(cast<DeclRefExpr>(

)->getDecl());

3157

PrivateVarsPos[VD] = Counter;

3160 for

(

const Expr

.FirstprivateVars) {

3162 C

nullptr

nullptr

3163 C

.getPointerType(

()))

3167 const auto

*VD = cast<VarDecl>(cast<DeclRefExpr>(

)->getDecl());

3168

PrivateVarsPos[VD] = Counter;

3171 for

(

const Expr

.LastprivateVars) {

3173 C

nullptr

nullptr

3174 C

.getPointerType(

()))

3178 const auto

*VD = cast<VarDecl>(cast<DeclRefExpr>(

)->getDecl());

3179

PrivateVarsPos[VD] = Counter;

3185

Ty =

.getPointerType(Ty);

3187

Ty =

.getPointerType(Ty);

3189 C

nullptr

nullptr

3190 C

.getPointerType(

.getPointerType(Ty)).withConst().withRestrict(),

3192

PrivateVarsPos[VD] = Counter;

3195 const auto

&TaskPrivatesMapFnInfo =

3197

llvm::FunctionType *TaskPrivatesMapTy =

3201 auto

*TaskPrivatesMap = llvm::Function::Create(

3202

TaskPrivatesMapTy, llvm::GlobalValue::InternalLinkage, Name,

3205

TaskPrivatesMapFnInfo);

3207

TaskPrivatesMap->removeFnAttr(llvm::Attribute::NoInline);

3208

TaskPrivatesMap->removeFnAttr(llvm::Attribute::OptimizeNone);

3209

TaskPrivatesMap->addFnAttr(llvm::Attribute::AlwaysInline);

3213

TaskPrivatesMapFnInfo, Args,

);

3219 const auto

*PrivatesQTyRD = cast<RecordDecl>(PrivatesQTy->

());

3221 for

(

const FieldDecl

*Field : PrivatesQTyRD->fields()) {

3223 const VarDecl

*VD = Args[PrivateVarsPos[Privates[Counter].second.Original]];

3232 return

TaskPrivatesMap;

3244 auto

FI = std::next(KmpTaskTWithPrivatesQTyRD->

());

3250

CodeGenFunction::CGCapturedStmtInfo CapturesInfo(CS);

3258 if

((!IsTargetTask && !

.FirstprivateVars.empty() && ForDup) ||

3259

(IsTargetTask && KmpTaskSharedsPtr.

isValid

())) {

3266

FI = cast<RecordDecl>(FI->getType()->getAsTagDecl())->field_begin();

3267 for

(

const

PrivateDataTy &Pair : Privates) {

3269 if

(Pair.second.isLocalPrivate()) {

3273 const VarDecl

*VD = Pair.second.PrivateCopy;

3275 if

(

&& (!ForDup || (isa<CXXConstructExpr>(

) &&

3278 if

(

const VarDecl

*Elem = Pair.second.PrivateElemInit) {

3279 const VarDecl

*OriginalVD = Pair.second.Original;

3284 const FieldDecl

*SharedField = CapturesInfo.lookup(OriginalVD);

3285 if

(IsTargetTask && !SharedField) {

3286

assert(isa<ImplicitParamDecl>(OriginalVD) &&

3289

->getNumParams() == 0 &&

3290

isa<TranslationUnitDecl>(

3292

->getDeclContext()) &&

3293 "Expected artificial target data variable."

);

3296

}

else if

(ForDup) {

3300 C

.getDeclAlign(OriginalVD)),

3304

Pair.second.Original->getCanonicalDecl()) > 0 ||

3306

SharedRefLValue = CGF.

EmitLValue

(Pair.second.OriginalRef);

3309

InlinedOpenMPRegionRAII Region(

3312

SharedRefLValue = CGF.

EmitLValue

(Pair.second.OriginalRef);

3324

[&CGF, Elem,

, &CapturesInfo](

DestElement,

3327

CodeGenFunction::OMPPrivateScope InitScope(CGF);

3328

InitScope.addPrivate(Elem, SrcElement);

3329

(void)InitScope.Privatize();

3331

CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(

3332

CGF, &CapturesInfo);

3333

CGF.EmitAnyExprToMem(Init, DestElement,

3334

Init->getType().getQualifiers(),

3339

CodeGenFunction::OMPPrivateScope InitScope(CGF);

3340

InitScope.addPrivate(Elem, SharedRefLValue.

getAddress

());

3341

(void)InitScope.Privatize();

3342

CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CapturesInfo);

3357 bool

InitRequired =

false

;

3358 for

(

const

PrivateDataTy &Pair : Privates) {

3359 if

(Pair.second.isLocalPrivate())

3361 const VarDecl

*VD = Pair.second.PrivateCopy;

3363

InitRequired = InitRequired || (isa_and_nonnull<CXXConstructExpr>(

) &&

3368 return

InitRequired;

3385 QualType

KmpTaskTWithPrivatesPtrQTy,

3393

KmpTaskTWithPrivatesPtrQTy,

3396

KmpTaskTWithPrivatesPtrQTy,

3400

Args.push_back(&DstArg);

3401

Args.push_back(&SrcArg);

3402

Args.push_back(&LastprivArg);

3403 const auto

&TaskDupFnInfo =

3407 auto

*TaskDup = llvm::Function::Create(

3408

TaskDupTy, llvm::GlobalValue::InternalLinkage, Name, &CGM.

());

3410

TaskDup->setDoesNotRecurse();

3420 auto

LIFI = std::next(KmpTaskTQTyRD->

(), KmpTaskTLastIter);

3422

TDBase, *KmpTaskTWithPrivatesQTyRD->

());

3430

assert(!Privates.empty());

3432 if

.FirstprivateVars.empty()) {

3437

TDBase, *KmpTaskTWithPrivatesQTyRD->

());

3445 emitPrivatesInit

(CGF,

, KmpTaskSharedsPtr, TDBase, KmpTaskTWithPrivatesQTyRD,

3446

SharedsTy, SharedsPtrTy,

, Privates,

true

);

3456 for

(

const

PrivateDataTy &

: Privates) {

3457 if

(

.second.isLocalPrivate())

3459 QualType

Ty =

.second.Original->getType().getNonReferenceType();

3468class

OMPIteratorGeneratorScope final

3469

public

CodeGenFunction::OMPPrivateScope {

3474

OMPIteratorGeneratorScope() =

delete

;

3475

OMPIteratorGeneratorScope(OMPIteratorGeneratorScope &) =

delete

;

3483 for

(

unsigned

I = 0, End =

->numOfIterators(); I < End; ++I) {

3485 const auto

*VD = cast<VarDecl>(

->getIteratorDecl(I));

3494 for

(

unsigned

I = 0, End =

->numOfIterators(); I < End; ++I) {

3503

CodeGenFunction::JumpDest &ContDest =

3505

CodeGenFunction::JumpDest &ExitDest =

3508

llvm::Value *N = Uppers[I];

3516

? CGF.

.CreateICmpSLT(CVal, N)

3517

: CGF.

.CreateICmpULT(CVal, N);

3519

CGF.

.CreateCondBr(Cmp, BodyBB, ExitDest.getBlock());

3526

~OMPIteratorGeneratorScope() {

3529 for

(

unsigned

I =

->numOfIterators(); I > 0; --I) {

3536

CGF.

EmitBlock

(ExitDests[I - 1].getBlock(),

I == 1);

3542static

std::pair<llvm::Value *, llvm::Value *>

3544 const auto

*OASE = dyn_cast<OMPArrayShapingExpr>(

);

3547 const Expr

= OASE->getBase();

3552

llvm::Value *SizeVal;

3555

SizeVal = CGF.

(OASE->getBase()->getType()->getPointeeType());

3556 for

(

const Expr

*SE : OASE->getDimensions()) {

3560

SizeVal = CGF.

.CreateNUWMul(SizeVal, Sz);

3562

}

else if

(

const auto

*ASE =

3566

llvm::Value *UpAddr = CGF.

.CreateConstGEP1_32(

3569

llvm::Value *LowIntPtr = CGF.

.CreatePtrToInt(Addr, CGF.

);

3570

llvm::Value *UpIntPtr = CGF.

.CreatePtrToInt(UpAddr, CGF.

);

3571

SizeVal = CGF.

.CreateNUWSub(UpIntPtr, LowIntPtr);

3575 return

std::make_pair(Addr, SizeVal);

3580 QualType

FlagsTy =

.getIntTypeForBitwidth(32,

false

);

3581 if

(KmpTaskAffinityInfoTy.

isNull

()) {

3583 C

.buildImplicitRecord(

"kmp_task_affinity_info_t"

);

3589

KmpTaskAffinityInfoTy =

.getRecordType(KmpAffinityInfoRD);

3596

llvm::Function *TaskFunction,

QualType

SharedsTy,

3601 const auto

*I =

.PrivateCopies.begin();

3602 for

(

const Expr

.PrivateVars) {

3603 const auto

*VD = cast<VarDecl>(cast<DeclRefExpr>(

)->getDecl());

3604

Privates.emplace_back(

3605 C

.getDeclAlign(VD),

3606

PrivateHelpersTy(

, VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),

3610

I =

.FirstprivateCopies.begin();

3611 const auto

*IElemInitRef =

.FirstprivateInits.begin();

3612 for

(

const Expr

.FirstprivateVars) {

3613 const auto

*VD = cast<VarDecl>(cast<DeclRefExpr>(

)->getDecl());

3614

Privates.emplace_back(

3615 C

.getDeclAlign(VD),

3617 E

, VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),

3618

cast<VarDecl>(cast<DeclRefExpr>(*IElemInitRef)->getDecl())));

3622

I =

.LastprivateCopies.begin();

3623 for

(

const Expr

.LastprivateVars) {

3624 const auto

*VD = cast<VarDecl>(cast<DeclRefExpr>(

)->getDecl());

3625

Privates.emplace_back(

3626 C

.getDeclAlign(VD),

3627

PrivateHelpersTy(

, VD, cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl()),

3635

Privates.emplace_back(

.getDeclAlign(VD), PrivateHelpersTy(VD));

3637

llvm::stable_sort(Privates,

3638

[](

const

PrivateDataTy &L,

const

PrivateDataTy &R) {

3639 return

L.first > R.first;

3641 QualType

KmpInt32Ty =

.getIntTypeForBitwidth(

32,

1);

3652

assert((

.getDirectiveKind() == OMPD_task ||

3655 "Expected taskloop, task or target directive"

);

3664 const RecordDecl

*KmpTaskTWithPrivatesQTyRD =

3666 QualType

KmpTaskTWithPrivatesQTy =

.getRecordType(KmpTaskTWithPrivatesQTyRD);

3667 QualType

KmpTaskTWithPrivatesPtrQTy =

3668 C

.getPointerType(KmpTaskTWithPrivatesQTy);

3669

llvm::Type *KmpTaskTWithPrivatesPtrTy = CGF.

.getPtrTy(0);

3670

llvm::Value *KmpTaskTWithPrivatesTySize =

3672 QualType

SharedsPtrTy =

.getPointerType(SharedsTy);

3675

llvm::Value *TaskPrivatesMap =

nullptr

;

3676

llvm::Type *TaskPrivatesMapTy =

3677

std::next(TaskFunction->arg_begin(), 3)->getType();

3678 if

(!Privates.empty()) {

3679 auto

FI = std::next(KmpTaskTWithPrivatesQTyRD->

());

3683

TaskPrivatesMap, TaskPrivatesMapTy);

3685

TaskPrivatesMap = llvm::ConstantPointerNull::get(

3686

cast<llvm::PointerType>(TaskPrivatesMapTy));

3691 CGM

.getDirectiveKind(), KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,

3692

KmpTaskTWithPrivatesQTy,

KmpTaskTQTy

, SharedsPtrTy, TaskFunction,

3704

DestructorsFlag = 0x8,

3705

PriorityFlag = 0x20,

3706

DetachableFlag = 0x40,

3708 unsigned

Flags =

.Tied ? TiedFlag : 0;

3709 bool

NeedsCleanup =

false

;

3710 if

(!Privates.empty()) {

3714

Flags = Flags | DestructorsFlag;

3716 if

(

.Priority.getInt())

3717

Flags = Flags | PriorityFlag;

3719

Flags = Flags | DetachableFlag;

3720

llvm::Value *TaskFlags =

3721 Data

.Final.getPointer()

3722

? CGF.

.CreateSelect(

.Final.getPointer(),

3723

CGF.

.getInt32(FinalFlag),

3725

: CGF.

.getInt32(

.Final.getInt() ? FinalFlag : 0);

3726

TaskFlags = CGF.

.CreateOr(TaskFlags, CGF.

.getInt32(Flags));

3727

llvm::Value *SharedsSize =

getSize

(

.getTypeSizeInChars(SharedsTy));

3732

llvm::Value *NewTask;

3739

llvm::Value *DeviceID;

3744

DeviceID = CGF.

.getInt64(OMP_DEVICEID_UNDEF);

3745

AllocArgs.push_back(DeviceID);

3748 CGM

(), OMPRTL___kmpc_omp_target_task_alloc),

3766

llvm::Value *Tid =

(CGF, DC->getBeginLoc());

3767

Tid = CGF.

.CreateIntCast(Tid, CGF.

false

);

3770 CGM

(), OMPRTL___kmpc_task_allow_completion_event),

3771

{Loc, Tid, NewTask});

3782

llvm::Value *NumOfElements =

nullptr

;

3783 unsigned

NumAffinities = 0;

3785 if

(

const Expr

*Modifier =

->getModifier()) {

3786 const auto

*IE = cast<OMPIteratorExpr>(Modifier->IgnoreParenImpCasts());

3787 for

(

unsigned

I = 0,

= IE->numOfIterators(); I <

; ++I) {

3791

NumOfElements ? CGF.

.CreateNUWMul(NumOfElements, Sz) : Sz;

3794

NumAffinities +=

->varlist_size();

3799 enum

RTLAffinityInfoFieldsTy { BaseAddr, Len, Flags };

3801 QualType

KmpTaskAffinityInfoArrayTy;

3802 if

(NumOfElements) {

3803

NumOfElements = CGF.

.CreateNUWAdd(

3804

llvm::ConstantInt::get(CGF.

, NumAffinities), NumOfElements);

3807 C

.getIntTypeForBitwidth(

.getTypeSize(

.getSizeType()),

0),

3811

KmpTaskAffinityInfoArrayTy =

.getVariableArrayType(

3819

NumOfElements = CGF.

.CreateIntCast(NumOfElements, CGF.

3822

KmpTaskAffinityInfoArrayTy =

.getConstantArrayType(

3824

llvm::APInt(

.getTypeSize(

.getSizeType()), NumAffinities),

nullptr

3827

CGF.

(KmpTaskAffinityInfoArrayTy,

".affs.arr.addr"

);

3829

NumOfElements = llvm::ConstantInt::get(

, NumAffinities,

3836 bool

HasIterator =

false

;

3838 if

(

->getModifier()) {

3839

HasIterator =

true

;

3842 for

(

const Expr

->varlist()) {

3851 Base

, *std::next(KmpAffinityInfoRD->field_begin(), BaseAddr));

3856 Base

, *std::next(KmpAffinityInfoRD->field_begin(), Len));

3870 const Expr

*Modifier =

->getModifier();

3873

OMPIteratorGeneratorScope IteratorScope(

3875 for

(

const Expr

->varlist()) {

3885 Base

, *std::next(KmpAffinityInfoRD->field_begin(), BaseAddr));

3890 Base

, *std::next(KmpAffinityInfoRD->field_begin(), Len));

3892

Idx = CGF.

.CreateNUWAdd(

3893

Idx, llvm::ConstantInt::get(Idx->getType(), 1));

3908 CGM

(), OMPRTL___kmpc_omp_reg_task_with_affinity),

3909

{LocRef, GTid, NewTask, NumOfElements, AffinListPtr});

3911

llvm::Value *NewTaskNewTaskTTy =

3913

NewTask, KmpTaskTWithPrivatesPtrTy);

3915

KmpTaskTWithPrivatesQTy);

3926

*std::next(KmpTaskTQTyRD->field_begin(), KmpTaskTShareds)),

3935 if

(!Privates.empty()) {

3937

SharedsTy, SharedsPtrTy,

, Privates,

3942 CGM

, KmpTaskTWithPrivatesPtrQTy, KmpTaskTWithPrivatesQTyRD,

3943

KmpTaskTQTyRD, SharedsTy, SharedsPtrTy,

, Privates,

3944

.LastprivateVars.empty());

3948 enum

{

Priority

= 0, Destructors = 1 };

3950 auto

FI = std::next(KmpTaskTQTyRD->field_begin(), Data1);

3952

(*FI)->getType()->getAsUnionType()->getDecl();

3955 CGM

, KmpInt32Ty, KmpTaskTWithPrivatesPtrQTy,

3956

KmpTaskTWithPrivatesQTy);

3959

Data1LV, *std::next(KmpCmplrdataUD->

(), Destructors));

3965 if

(

.Priority.getInt()) {

3967

TDBase, *std::next(KmpTaskTQTyRD->field_begin(), Data2));

3972 Result

.NewTask = NewTask;

3973 Result

.TaskEntry = TaskEntry;

3974 Result

.NewTaskNewTaskTTy = NewTaskNewTaskTTy;

3976 Result

.KmpTaskTQTyRD = KmpTaskTQTyRD;

3982

RTLDependenceKindTy DepKind;

3984 case

OMPC_DEPEND_in:

3985

DepKind = RTLDependenceKindTy::DepIn;

3988 case

OMPC_DEPEND_out:

3989 case

OMPC_DEPEND_inout:

3990

DepKind = RTLDependenceKindTy::DepInOut;

3992 case

OMPC_DEPEND_mutexinoutset:

3993

DepKind = RTLDependenceKindTy::DepMutexInOutSet;

3995 case

OMPC_DEPEND_inoutset:

3996

DepKind = RTLDependenceKindTy::DepInOutSet;

3998 case

OMPC_DEPEND_outallmemory:

3999

DepKind = RTLDependenceKindTy::DepOmpAllMem;

4001 case

OMPC_DEPEND_source:

4002 case

OMPC_DEPEND_sink:

4003 case

OMPC_DEPEND_depobj:

4004 case

OMPC_DEPEND_inoutallmemory:

4006

llvm_unreachable(

"Unknown task dependence type"

);

4014

FlagsTy =

.getIntTypeForBitwidth(

.getTypeSize(

.BoolTy),

false

);

4015 if

(KmpDependInfoTy.

isNull

()) {

4016 RecordDecl

*KmpDependInfoRD =

.buildImplicitRecord(

"kmp_depend_info"

);

4022

KmpDependInfoTy =

.getRecordType(KmpDependInfoRD);

4026

std::pair<llvm::Value *, LValue>

4040

CGF,

.getAddress(),

4041

llvm::ConstantInt::get(CGF.

, -1,

true

));

4048 static_cast<unsigned int>

(RTLDependInfoFields::BaseAddr)));

4050 return

std::make_pair(NumDeps,

);

4054

llvm::PointerUnion<unsigned *, LValue *> Pos,

4065

OMPIteratorGeneratorScope IteratorScope(

4066

CGF, cast_or_null<OMPIteratorExpr>(

4067 Data

.IteratorExpr ?

.IteratorExpr->IgnoreParenImpCasts()

4069 for

(

const Expr

.DepExprs) {

4078

Addr = llvm::ConstantInt::get(CGF.

, 0);

4079

Size = llvm::ConstantInt::get(CGF.

, 0);

4082 if

(

unsigned

= dyn_cast<unsigned *>(Pos)) {

4086

assert(

"Expected a non-null expression"

);

4087 LValue

&PosLVal = *cast<LValue *>(Pos);

4096 static_cast<unsigned int>

(RTLDependInfoFields::BaseAddr)));

4101 static_cast<unsigned int>

(RTLDependInfoFields::Len)));

4108 static_cast<unsigned int>

(RTLDependInfoFields::Flags)));

4110

llvm::ConstantInt::get(LLVMFlagsTy,

static_cast<unsigned int>

(DepKind)),

4112 if

(

unsigned

= dyn_cast<unsigned *>(Pos)) {

4115 LValue

&PosLVal = *cast<LValue *>(Pos);

4117

Idx = CGF.

.CreateNUWAdd(Idx,

4118

llvm::ConstantInt::get(Idx->getType(), 1));

4127

assert(

.DepKind == OMPC_DEPEND_depobj &&

4128 "Expected depobj dependency kind."

);

4133

OMPIteratorGeneratorScope IteratorScope(

4134

CGF, cast_or_null<OMPIteratorExpr>(

4135 Data

.IteratorExpr ?

.IteratorExpr->IgnoreParenImpCasts()

4137 for

(

const Expr

.DepExprs) {

4138

llvm::Value *NumDeps;

4141

std::tie(NumDeps,

) =

4145 C

.getUIntPtrType());

4149

llvm::Value *Add = CGF.

.CreateNUWAdd(PrevVal, NumDeps);

4151

SizeLVals.push_back(NumLVal);

4154 for

(

unsigned

I = 0,

= SizeLVals.size(); I <

; ++I) {

4157

Sizes.push_back(Size);

4167

assert(

.DepKind == OMPC_DEPEND_depobj &&

4168 "Expected depobj dependency kind."

);

4171

OMPIteratorGeneratorScope IteratorScope(

4172

CGF, cast_or_null<OMPIteratorExpr>(

4173 Data

.IteratorExpr ?

.IteratorExpr->IgnoreParenImpCasts()

4175 for

(

unsigned

I = 0, End =

.DepExprs.size(); I < End; ++I) {

4177

llvm::Value *NumDeps;

4180

std::tie(NumDeps,

) =

4184

llvm::Value *Size = CGF.

.CreateNUWMul(

4186

CGF.

.CreateIntCast(NumDeps, CGF.

false

));

4193

llvm::Value *Add = CGF.

.CreateNUWAdd(Pos, NumDeps);

4203 return D

.DepExprs.empty();

4209

llvm::Value *NumOfElements =

nullptr

;

4210 unsigned

NumDependencies = std::accumulate(

4211

Dependencies.begin(), Dependencies.end(), 0,

4213

return D.DepKind == OMPC_DEPEND_depobj

4215

: (V + (D.IteratorExpr ? 0 : D.DepExprs.size()));

4219 bool

HasDepobjDeps =

false

;

4220 bool

HasRegularWithIterators =

false

;

4221

llvm::Value *NumOfDepobjElements = llvm::ConstantInt::get(CGF.

, 0);

4222

llvm::Value *NumOfRegularWithIterators =

4223

llvm::ConstantInt::get(CGF.

, 0);

4227 if

(

.DepKind == OMPC_DEPEND_depobj) {

4230 for

(llvm::Value *Size : Sizes) {

4231

NumOfDepobjElements =

4232

CGF.

.CreateNUWAdd(NumOfDepobjElements, Size);

4234

HasDepobjDeps =

true

;

4239 if

(

const auto

*IE = cast_or_null<OMPIteratorExpr>(

.IteratorExpr)) {

4240

llvm::Value *ClauseIteratorSpace =

4241

llvm::ConstantInt::get(CGF.

, 1);

4242 for

(

unsigned

I = 0,

= IE->numOfIterators(); I <

; ++I) {

4245

ClauseIteratorSpace = CGF.

.CreateNUWMul(Sz, ClauseIteratorSpace);

4247

llvm::Value *NumClauseDeps = CGF.

.CreateNUWMul(

4248

ClauseIteratorSpace,

4249

llvm::ConstantInt::get(CGF.

.DepExprs.size()));

4250

NumOfRegularWithIterators =

4251

CGF.

.CreateNUWAdd(NumOfRegularWithIterators, NumClauseDeps);

4252

HasRegularWithIterators =

true

;

4258 if

(HasDepobjDeps || HasRegularWithIterators) {

4259

NumOfElements = llvm::ConstantInt::get(

, NumDependencies,

4261 if

(HasDepobjDeps) {

4263

CGF.

.CreateNUWAdd(NumOfDepobjElements, NumOfElements);

4265 if

(HasRegularWithIterators) {

4267

CGF.

.CreateNUWAdd(NumOfRegularWithIterators, NumOfElements);

4270 Loc

.getIntTypeForBitwidth(

64,

0),

4274

KmpDependInfoArrayTy =

4283

NumOfElements = CGF.

.CreateIntCast(NumOfElements, CGF.

4286

KmpDependInfoArrayTy =

.getConstantArrayType(

4290

CGF.

(KmpDependInfoArrayTy,

".dep.arr.addr"

);

4292

NumOfElements = llvm::ConstantInt::get(

, NumDependencies,

4296 for

(

unsigned

I = 0, End = Dependencies.size(); I < End; ++I) {

4297 if

(Dependencies[I].DepKind == OMPC_DEPEND_depobj ||

4298

Dependencies[I].IteratorExpr)

4305

CGF.

(

.getSizeType(),

"dep.counter.addr"

.getSizeType());

4307 for

(

unsigned

I = 0, End = Dependencies.size(); I < End; ++I) {

4308 if

(Dependencies[I].DepKind == OMPC_DEPEND_depobj ||

4309

!Dependencies[I].IteratorExpr)

4315 if

(HasDepobjDeps) {

4316 for

(

unsigned

I = 0, End = Dependencies.size(); I < End; ++I) {

4317 if

(Dependencies[I].DepKind != OMPC_DEPEND_depobj)

4325 return

std::make_pair(NumOfElements, DependenciesArray);

4331 if

(Dependencies.

DepExprs

.empty())

4336 unsigned

NumDependencies = Dependencies.

DepExprs

.size();

4347

llvm::Value *NumDepsVal;

4349 if

(

const auto

*IE =

4350

cast_or_null<OMPIteratorExpr>(Dependencies.

IteratorExpr

)) {

4351

NumDepsVal = llvm::ConstantInt::get(CGF.

, 1);

4352 for

(

unsigned

I = 0,

= IE->numOfIterators(); I <

; ++I) {

4355

NumDepsVal = CGF.

.CreateNUWMul(NumDepsVal, Sz);

4357

Size = CGF.

.CreateNUWAdd(llvm::ConstantInt::get(CGF.

, 1),

4361

llvm::Value *RecSize =

getSize

(SizeInBytes);

4362

Size = CGF.

.CreateNUWMul(Size, RecSize);

4366 QualType

KmpDependInfoArrayTy =

.getConstantArrayType(

4369 CharUnits

Sz =

.getTypeSizeInChars(KmpDependInfoArrayTy);

4371

NumDepsVal = llvm::ConstantInt::get(CGF.

, NumDependencies);

4376

llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.

);

4377

llvm::Value *Args[] = {ThreadID, Size, Allocator};

4382

Args,

".dep.arr.addr"

);

4385

Addr, CGF.

.getPtrTy(0));

4386

DependenciesArray =

(Addr, KmpDependInfoLlvmTy, Align);

4393 static_cast<unsigned int>

(RTLDependInfoFields::BaseAddr)));

4395

llvm::PointerUnion<unsigned *, LValue *> Pos;

4400

CGF.

(

.getSizeType(),

"iterator.counter.addr"

4412 return

DependenciesArray;

4428

llvm::ConstantInt::get(CGF.

, -1,

true

));

4433

llvm::Value *Allocator = llvm::ConstantPointerNull::get(CGF.

);

4434

llvm::Value *Args[] = {ThreadID, DepObjAddr, Allocator};

4451

llvm::Value *NumDeps;

4458 Begin

.emitRawPointer(CGF), NumDeps);

4462

llvm::BasicBlock *EntryBB = CGF.

.GetInsertBlock();

4464

llvm::PHINode *ElementPHI =

4465

CGF.

.CreatePHI(

Begin

.getType(), 2,

"omp.elementPast"

);

4466

ElementPHI->addIncoming(

Begin

.emitRawPointer(CGF), EntryBB);

4469 Base

.getTBAAInfo());

4474 static_cast<unsigned int>

(RTLDependInfoFields::Flags)));

4476

llvm::ConstantInt::get(LLVMFlagsTy,

static_cast<unsigned int>

(DepKind)),

4480

llvm::Value *ElementNext =

4483

ElementPHI->addIncoming(ElementNext, CGF.

.GetInsertBlock());

4484

llvm::Value *IsEmpty =

4485

CGF.

.CreateICmpEQ(ElementNext, End,

"omp.isempty"

);

4486

CGF.

.CreateCondBr(IsEmpty, DoneBB, BodyBB);

4493

llvm::Function *TaskFunction,

4502

llvm::Value *NewTask =

.NewTask;

4503

llvm::Function *TaskEntry =

.TaskEntry;

4504

llvm::Value *NewTaskNewTaskTTy =

.NewTaskNewTaskTTy;

4509

llvm::Value *NumOfElements;

4510

std::tie(NumOfElements, DependenciesArray) =

4521

llvm::Value *TaskArgs[] = { UpLoc, ThreadID, NewTask };

4522

llvm::Value *DepTaskArgs[7];

4523 if

.Dependences.empty()) {

4524

DepTaskArgs[0] = UpLoc;

4525

DepTaskArgs[1] = ThreadID;

4526

DepTaskArgs[2] = NewTask;

4527

DepTaskArgs[3] = NumOfElements;

4529

DepTaskArgs[5] = CGF.

.getInt32(0);

4530

DepTaskArgs[6] = llvm::ConstantPointerNull::get(CGF.

);

4532 auto

&&ThenCodeGen = [

this

, &

, TDBase, KmpTaskTQTyRD, &TaskArgs,

4535 auto

PartIdFI = std::next(KmpTaskTQTyRD->

(), KmpTaskTPartId);

4539 if

.Dependences.empty()) {

4552

Region->emitUntiedSwitch(CGF);

4555

llvm::Value *DepWaitTaskArgs[7];

4556 if

.Dependences.empty()) {

4557

DepWaitTaskArgs[0] = UpLoc;

4558

DepWaitTaskArgs[1] = ThreadID;

4559

DepWaitTaskArgs[2] = NumOfElements;

4561

DepWaitTaskArgs[4] = CGF.

.getInt32(0);

4562

DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.

);

4563

DepWaitTaskArgs[6] =

4564

llvm::ConstantInt::get(CGF.

.HasNowaitClause);

4567 auto

&&ElseCodeGen = [

this

, &M, &TaskArgs, ThreadID, NewTaskNewTaskTTy,

4568

TaskEntry, &

, &DepWaitTaskArgs,

4575 if

.Dependences.empty())

4577

M, OMPRTL___kmpc_omp_taskwait_deps_51),

4580 auto

&&CodeGen = [TaskEntry, ThreadID, NewTaskNewTaskTTy,

4583

llvm::Value *OutlinedFnArgs[] = {ThreadID, NewTaskNewTaskTTy};

4584

CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(CGF,

, TaskEntry,

4593

CommonActionTy Action(

.getOrCreateRuntimeFunction(

4594

M, OMPRTL___kmpc_omp_task_begin_if0),

4597

M, OMPRTL___kmpc_omp_task_complete_if0),

4613

llvm::Function *TaskFunction,

4633

IfVal = llvm::ConstantInt::getSigned(CGF.

1);

4638

*std::next(

.KmpTaskTQTyRD->field_begin(), KmpTaskTLowerBound));

4640

cast<VarDecl>(cast<DeclRefExpr>(

.getLowerBoundVariable())->getDecl());

4645

*std::next(

.KmpTaskTQTyRD->field_begin(), KmpTaskTUpperBound));

4647

cast<VarDecl>(cast<DeclRefExpr>(

.getUpperBoundVariable())->getDecl());

4652

*std::next(

.KmpTaskTQTyRD->field_begin(), KmpTaskTStride));

4654

cast<VarDecl>(cast<DeclRefExpr>(

.getStrideVariable())->getDecl());

4660

*std::next(

.KmpTaskTQTyRD->field_begin(), KmpTaskTReductions));

4661 if

(

.Reductions) {

4667 enum

{ NoSchedule = 0, Grainsize = 1, NumTasks = 2 };

4676

llvm::ConstantInt::getSigned(

4678

llvm::ConstantInt::getSigned(

4679

CGF.

.Schedule.getPointer()

4680

.Schedule.getInt() ? NumTasks : Grainsize

4682 Data

.Schedule.getPointer()

4685

: llvm::ConstantInt::get(CGF.

0)};

4686 if

(

.HasModifier)

4687

TaskArgs.push_back(llvm::ConstantInt::get(CGF.

, 1));

4689

TaskArgs.push_back(

.TaskDupFn

4692

: llvm::ConstantPointerNull::get(CGF.

));

4695

? OMPRTL___kmpc_taskloop_5

4696

: OMPRTL___kmpc_taskloop),

4713 const Expr

const Expr

*)> &RedOpGen,

4714 const Expr

*XExpr =

nullptr

const Expr

*EExpr =

nullptr

4715 const Expr

*UpExpr =

nullptr

) {

4723

llvm::Value *NumElements = CGF.

emitArrayLength

(ArrayTy, ElementTy, LHSAddr);

4728

llvm::Value *LHSEnd =

4733

llvm::Value *IsEmpty =

4734

CGF.

.CreateICmpEQ(LHSBegin, LHSEnd,

"omp.arraycpy.isempty"

);

4735

CGF.

.CreateCondBr(IsEmpty, DoneBB, BodyBB);

4738

llvm::BasicBlock *EntryBB = CGF.

.GetInsertBlock();

4743

llvm::PHINode *RHSElementPHI = CGF.

.CreatePHI(

4744

RHSBegin->getType(), 2,

"omp.arraycpy.srcElementPast"

);

4745

RHSElementPHI->addIncoming(RHSBegin, EntryBB);

4750

llvm::PHINode *LHSElementPHI = CGF.

.CreatePHI(

4751

LHSBegin->getType(), 2,

"omp.arraycpy.destElementPast"

);

4752

LHSElementPHI->addIncoming(LHSBegin, EntryBB);

4758

CodeGenFunction::OMPPrivateScope

(CGF);

4759 Scope

.addPrivate(LHSVar, LHSElementCurrent);

4760 Scope

.addPrivate(RHSVar, RHSElementCurrent);

4762

RedOpGen(CGF, XExpr, EExpr, UpExpr);

4763 Scope

.ForceCleanup();

4766

llvm::Value *LHSElementNext = CGF.

.CreateConstGEP1_32(

4768 "omp.arraycpy.dest.element"

);

4769

llvm::Value *RHSElementNext = CGF.

.CreateConstGEP1_32(

4771 "omp.arraycpy.src.element"

);

4774

CGF.

.CreateICmpEQ(LHSElementNext, LHSEnd,

"omp.arraycpy.done"

);

4775

CGF.

.CreateCondBr(Done, DoneBB, BodyBB);

4776

LHSElementPHI->addIncoming(LHSElementNext, CGF.

.GetInsertBlock());

4777

RHSElementPHI->addIncoming(RHSElementNext, CGF.

.GetInsertBlock());

4787 const Expr

*ReductionOp) {

4788 if

(

const auto

*CE = dyn_cast<CallExpr>(ReductionOp))

4789 if

(

const auto

*OVE = dyn_cast<OpaqueValueExpr>(CE->getCallee()))

4790 if

(

const auto

*DRE =

4791

dyn_cast<DeclRefExpr>(OVE->getSourceExpr()->IgnoreImpCasts()))

4792 if

(

const auto

*DRD =

4793

dyn_cast<OMPDeclareReductionDecl>(DRE->getDecl())) {

4794

std::pair<llvm::Function *, llvm::Function *>

Reduction

4797

CodeGenFunction::OpaqueValueMapping Map(CGF, OVE,

Func

);

4816

Args.push_back(&LHSArg);

4817

Args.push_back(&RHSArg);

4822

llvm::GlobalValue::InternalLinkage, Name,

4825

Fn->setDoesNotRecurse();

4844 const auto

*IPriv = Privates.begin();

4846 for

(

unsigned

I = 0,

= ReductionOps.size(); I <

; ++I, ++IPriv, ++Idx) {

4847 const auto

*RHSVar =

4848

cast<VarDecl>(cast<DeclRefExpr>(RHSExprs[I])->getDecl());

4850 const auto

*LHSVar =

4851

cast<VarDecl>(cast<DeclRefExpr>(LHSExprs[I])->getDecl());

4853 QualType

PrivTy = (*IPriv)->getType();

4861 const auto

*OVE = cast<OpaqueValueExpr>(VLA->

getSizeExpr

());

4868

IPriv = Privates.begin();

4869 const auto

*ILHS = LHSExprs.begin();

4870 const auto

*IRHS = RHSExprs.begin();

4871 for

(

const Expr

: ReductionOps) {

4872 if

((*IPriv)->getType()->isArrayType()) {

4874 const auto

*LHSVar = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());

4875 const auto

*RHSVar = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());

4877

CGF, (*IPriv)->getType(), LHSVar, RHSVar,

4879

emitReductionCombiner(CGF, E);

4889 Scope

.ForceCleanup();

4895 const Expr

*ReductionOp,

4896 const Expr

*PrivateRef,

4901 const auto

*LHSVar = cast<VarDecl>(LHS->

());

4902 const auto

*RHSVar = cast<VarDecl>(RHS->

());

4904

CGF, PrivateRef->

(), LHSVar, RHSVar,

4906

emitReductionCombiner(CGF, ReductionOp);

4923 bool

WithNowait = Options.WithNowait;

4924 bool

SimpleReduction = Options.SimpleReduction;

4963 if

(SimpleReduction) {

4965 const auto

*IPriv = Privates.begin();

4966 const auto

*ILHS = LHSExprs.begin();

4967 const auto

*IRHS = RHSExprs.begin();

4968 for

(

const Expr

: ReductionOps) {

4970

cast<DeclRefExpr>(*IRHS));

4980 auto

Size = RHSExprs.size();

4981 for

(

const Expr

: Privates) {

4986

llvm::APInt ArraySize(

32, Size);

4987 QualType

ReductionArrayTy =

.getConstantArrayType(

4991

CGF.

(ReductionArrayTy,

".omp.reduction.red_list"

);

4992 const auto

*IPriv = Privates.begin();

4994 for

(

unsigned

I = 0,

= RHSExprs.size(); I <

; ++I, ++IPriv, ++Idx) {

5000 if

((*IPriv)->getType()->isVariablyModifiedType()) {

5004

llvm::Value *Size = CGF.

.CreateIntCast(

5017

Privates, LHSExprs, RHSExprs, ReductionOps);

5020

std::string Name =

({

"reduction"

});

5027

llvm::Value *ReductionArrayTySize = CGF.

(ReductionArrayTy);

5030

llvm::Value *Args[] = {

5033

CGF.

.getInt32(RHSExprs.size()),

5034

ReductionArrayTySize,

5042

WithNowait ? OMPRTL___kmpc_reduce_nowait : OMPRTL___kmpc_reduce),

5046

llvm::BasicBlock *DefaultBB = CGF.

createBasicBlock

(

".omp.reduction.default"

);

5047

llvm::SwitchInst *SwInst =

5048

CGF.

.CreateSwitch(Res, DefaultBB,

2);

5056

llvm::BasicBlock *Case1BB = CGF.

createBasicBlock

(

".omp.reduction.case1"

);

5057

SwInst->addCase(CGF.

.getInt32(1), Case1BB);

5061

llvm::Value *EndArgs[] = {

5066 auto

&&CodeGen = [Privates, LHSExprs, RHSExprs, ReductionOps](

5069 const auto

*IPriv = Privates.begin();

5070 const auto

*ILHS = LHSExprs.begin();

5071 const auto

*IRHS = RHSExprs.begin();

5072 for

(

const Expr

: ReductionOps) {

5074

cast<DeclRefExpr>(*IRHS));

5081

CommonActionTy Action(

5084 CGM

(), WithNowait ? OMPRTL___kmpc_end_reduce_nowait

5085

: OMPRTL___kmpc_end_reduce),

5097

llvm::BasicBlock *Case2BB = CGF.

createBasicBlock

(

".omp.reduction.case2"

);

5098

SwInst->addCase(CGF.

.getInt32(2), Case2BB);

5101 auto

&&AtomicCodeGen = [

, Privates, LHSExprs, RHSExprs, ReductionOps](

5103 const auto

*ILHS = LHSExprs.begin();

5104 const auto

*IRHS = RHSExprs.begin();

5105 const auto

*IPriv = Privates.begin();

5106 for

(

const Expr

: ReductionOps) {

5107 const Expr

*XExpr =

nullptr

;

5108 const Expr

*EExpr =

nullptr

;

5109 const Expr

*UpExpr =

nullptr

;

5111 if

(

const auto

*BO = dyn_cast<BinaryOperator>(

)) {

5112 if

(BO->getOpcode() == BO_Assign) {

5113

XExpr = BO->getLHS();

5114

UpExpr = BO->getRHS();

5118 const Expr

*RHSExpr = UpExpr;

5121 if

(

const auto

*ACO = dyn_cast<AbstractConditionalOperator>(

5125

RHSExpr = ACO->getCond();

5127 if

(

const auto

*BORHS =

5129

EExpr = BORHS->getRHS();

5130

BO = BORHS->getOpcode();

5134 const auto

*VD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());

5135 auto

&&AtomicRedGen = [BO, VD,

5137 const Expr

*EExpr,

const Expr

*UpExpr) {

5138 LValue X

= CGF.EmitLValue(XExpr);

5141 E

= CGF.EmitAnyExpr(EExpr);

5142

CGF.EmitOMPAtomicSimpleUpdateExpr(

5144

llvm::AtomicOrdering::Monotonic,

5145

[&CGF, UpExpr, VD,

](

RValue

XRValue) {

5147 Address

LHSTemp = CGF.CreateMemTemp(VD->getType());

5148

CGF.emitOMPSimpleStore(

5149

CGF.MakeAddrLValue(LHSTemp, VD->

()), XRValue,

5150

VD->getType().getNonReferenceType(),

);

5153 return

CGF.EmitAnyExpr(UpExpr);

5156 if

((*IPriv)->getType()->isArrayType()) {

5158 const auto

*RHSVar =

5159

cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());

5161

AtomicRedGen, XExpr, EExpr, UpExpr);

5164

AtomicRedGen(CGF, XExpr, EExpr, UpExpr);

5171

std::string Name = RT.

({

"atomic_reduction"

});

5175

Action.

Enter

(CGF);

5180 if

((*IPriv)->getType()->isArrayType()) {

5181 const auto

*LHSVar =

5182

cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());

5183 const auto

*RHSVar =

5184

cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());

5188

CritRedGen(CGF,

nullptr

);

5199

llvm::Value *EndArgs[] = {

5204

CommonActionTy Action(

nullptr

, {},

5223

llvm::raw_svector_ostream Out(Buffer);

5227 D

= cast<VarDecl>(cast<DeclRefExpr>(Ref)->getDecl());

5231

Out << Prefix << Name <<

"_" 5233 return

std::string(Out.str());

5255

Args.emplace_back(&Param);

5256

Args.emplace_back(&ParamOrig);

5257 const auto

&FnInfo =

5261 auto

*Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,

5264

Fn->setDoesNotRecurse();

5270 C

.getPointerType(PrivateType)->castAs<

PointerType

>());

5271

llvm::Value *Size =

nullptr

;

5314 const Expr

*ReductionOp,

5316 const Expr

*PrivateRef) {

5318 const auto

*LHSVD = cast<VarDecl>(cast<DeclRefExpr>(LHS)->getDecl());

5319 const auto

*RHSVD = cast<VarDecl>(cast<DeclRefExpr>(RHS)->getDecl());

5325

Args.emplace_back(&ParamInOut);

5326

Args.emplace_back(&ParamIn);

5327 const auto

&FnInfo =

5331 auto

*Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,

5334

Fn->setDoesNotRecurse();

5337

llvm::Value *Size =

nullptr

;

5351

CodeGenFunction::OMPPrivateScope PrivateScope(CGF);

5352

PrivateScope.addPrivate(

5358 C

.getPointerType(LHSVD->getType())->castAs<

PointerType

>()));

5359

PrivateScope.addPrivate(

5365 C

.getPointerType(RHSVD->getType())->castAs<

PointerType

>()));

5366

PrivateScope.Privatize();

5371

CGF, ReductionOp, PrivateRef, cast<DeclRefExpr>(LHS),

5372

cast<DeclRefExpr>(RHS));

5394

Args.emplace_back(&Param);

5395 const auto

&FnInfo =

5399 auto

*Fn = llvm::Function::Create(FnTy, llvm::GlobalValue::InternalLinkage,

5402

Fn->setDoesNotRecurse();

5407

llvm::Value *Size =

nullptr

;

5442 RecordDecl

*RD =

.buildImplicitRecord(

"kmp_taskred_input_t"

);

5451 C

, RD,

.getIntTypeForBitwidth(

32,

false

));

5454 unsigned

Size =

.ReductionVars.size();

5455

llvm::APInt ArraySize(

64, Size);

5457 C

.getConstantArrayType(RDType, ArraySize,

nullptr

5462 Data

.ReductionCopies,

.ReductionOps);

5463 for

(

unsigned

Cnt = 0; Cnt < Size; ++Cnt) {

5465

llvm::Value *Idxs[] = {llvm::ConstantInt::get(

0),

5466

llvm::ConstantInt::get(

, Cnt)};

5469 false

false

5482

llvm::Value *SizeValInChars;

5483

llvm::Value *SizeVal;

5484

std::tie(SizeValInChars, SizeVal) = RCG.

getSizes

(Cnt);

5490 bool

DelayedCreation = !!SizeVal;

5491

SizeValInChars = CGF.

.CreateIntCast(SizeValInChars,

5502

llvm::Value *FiniAddr =

5503

Fini ? Fini : llvm::ConstantPointerNull::get(

);

5508 CGM

, RCG, Cnt,

.ReductionOps[Cnt], LHSExprs[Cnt],

5509

RHSExprs[Cnt],

.ReductionCopies[Cnt]);

5513 if

(DelayedCreation) {

5515

llvm::ConstantInt::get(

true

5520 if

(

.IsReductionWithTaskMod) {

5526

llvm::Value *Args[] = {

5528

llvm::ConstantInt::get(

.IsWorksharingReduction ? 1 : 0,

5530

llvm::ConstantInt::get(

, Size,

true

5535 CGM

(), OMPRTL___kmpc_taskred_modifier_init),

5539

llvm::Value *Args[] = {

5542

llvm::ConstantInt::get(

, Size,

true

5552 bool

IsWorksharingReduction) {

5558

llvm::Value *Args[] = {IdentTLoc, GTid,

5559

llvm::ConstantInt::get(

5560

IsWorksharingReduction ? 1 : 0,

5564 CGM

(), OMPRTL___kmpc_task_reduction_modifier_fini),

5576

llvm::Value *SizeVal = CGF.

.CreateIntCast(Sizes.second,

5587

llvm::Value *ReductionsPtr,

5600 CGM

(), OMPRTL___kmpc_task_reduction_get_th_data),

5618

llvm::Value *NumOfElements;

5619

std::tie(NumOfElements, DependenciesArray) =

5621 if

.Dependences.empty()) {

5622

llvm::Value *DepWaitTaskArgs[7];

5623

DepWaitTaskArgs[0] = UpLoc;

5624

DepWaitTaskArgs[1] = ThreadID;

5625

DepWaitTaskArgs[2] = NumOfElements;

5627

DepWaitTaskArgs[4] = CGF.

.getInt32(0);

5628

DepWaitTaskArgs[5] = llvm::ConstantPointerNull::get(CGF.

);

5629

DepWaitTaskArgs[6] =

5630

llvm::ConstantInt::get(CGF.

.HasNowaitClause);

5639

M, OMPRTL___kmpc_omp_taskwait_deps_51),

5646

llvm::Value *Args[] = {UpLoc, ThreadID};

5649 OMPBuilder

.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_omp_taskwait),

5654 if

(

auto

*Region = dyn_cast_or_null<CGOpenMPRegionInfo>(CGF.

CapturedStmtInfo

))

5655

Region->emitUntiedSwitch(CGF);

5664

InlinedOpenMPRegionRAII Region(CGF, CodeGen, InnerKind, HasCancel,

5665

InnerKind != OMPD_critical &&

5666

InnerKind != OMPD_master &&

5667

InnerKind != OMPD_masked);

5682

RTCancelKind CancelKind = CancelNoreq;

5683 if

(CancelRegion == OMPD_parallel)

5684

CancelKind = CancelParallel;

5685 else if

(CancelRegion == OMPD_for)

5686

CancelKind = CancelLoop;

5687 else if

(CancelRegion == OMPD_sections)

5688

CancelKind = CancelSections;

5690

assert(CancelRegion == OMPD_taskgroup);

5691

CancelKind = CancelTaskgroup;

5703 if

(

auto

*OMPRegionInfo =

5707 if

(CancelRegion == OMPD_taskgroup || OMPRegionInfo->hasCancel()) {

5708

llvm::Value *Args[] = {

5723

CGF.

.CreateCondBr(Cmp, ExitBB, ContBB);

5725 if

(CancelRegion == OMPD_parallel)

5744 if

(

auto

*OMPRegionInfo =

5746 auto

&&ThenGen = [

this

, &M,

, CancelRegion,

5749

llvm::Value *Args[] = {

5753

llvm::Value *

= CGF.EmitRuntimeCall(

5754 OMPBuilder

.getOrCreateRuntimeFunction(M, OMPRTL___kmpc_cancel), Args);

5759

llvm::BasicBlock *ExitBB = CGF.createBasicBlock(

".cancel.exit"

);

5760

llvm::BasicBlock *ContBB = CGF.createBasicBlock(

".cancel.continue"

);

5761

llvm::Value *Cmp = CGF.Builder.CreateIsNotNull(

);

5762

CGF.Builder.CreateCondBr(Cmp, ExitBB, ContBB);

5763

CGF.EmitBlock(ExitBB);

5764 if

(CancelRegion == OMPD_parallel)

5768

CGF.getOMPCancelDestination(OMPRegionInfo->getDirectiveKind());

5769

CGF.EmitBranchThroughCleanup(CancelDest);

5770

CGF.EmitBlock(ContBB,

true

);

5788

OMPUsesAllocatorsActionTy(

5789 ArrayRef

<std::pair<const Expr *, const Expr *>> Allocators)

5790

: Allocators(Allocators) {}

5794 for

(

const auto

&AllocatorData : Allocators) {

5796

CGF, AllocatorData.first, AllocatorData.second);

5802 for

(

const auto

&AllocatorData : Allocators) {

5804

AllocatorData.first);

5812

llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,

5814

assert(!ParentName.empty() &&

"Invalid target entry parent name!"

);

5818 for

(

unsigned

I = 0,

->getNumberOfAllocators(); I <

; ++I) {

5820 if

.AllocatorTraits)

5822

Allocators.emplace_back(

.Allocator,

.AllocatorTraits);

5825

OMPUsesAllocatorsActionTy UsesAllocatorAction(Allocators);

5826

CodeGen.

setAction

(UsesAllocatorAction);

5828

IsOffloadEntry, CodeGen);

5832 const Expr

*Allocator,

5833 const Expr

*AllocatorTraits) {

5834

llvm::Value *ThreadId =

(CGF, Allocator->getExprLoc());

5835

ThreadId = CGF.

.CreateIntCast(ThreadId, CGF.

true

);

5837

llvm::Value *MemSpaceHandle = llvm::ConstantPointerNull::get(CGF.

);

5838

llvm::Value *NumTraits = llvm::ConstantInt::get(

5839

CGF.

, cast<ConstantArrayType>(

5842

.getLimitedValue());

5851

llvm::Value *AllocatorVal =

5854

{ThreadId, MemSpaceHandle, NumTraits, Traits});

5857

cast<DeclRefExpr>(Allocator->IgnoreParenImpCasts())->getDecl()));

5861

Allocator->getType(), Allocator->getExprLoc());

5866 const Expr

*Allocator) {

5867

llvm::Value *ThreadId =

(CGF, Allocator->getExprLoc());

5868

ThreadId = CGF.

.CreateIntCast(ThreadId, CGF.

true

);

5870

llvm::Value *AllocatorVal =

5874

Allocator->getExprLoc());

5877

OMPRTL___kmpc_destroy_allocator),

5878

{ThreadId, AllocatorVal});

5883

llvm::OpenMPIRBuilder::TargetKernelDefaultAttrs &Attrs) {

5884

assert(Attrs.MaxTeams.size() == 1 && Attrs.MaxThreads.size() == 1 &&

5885 "invalid default attrs structure"

);

5886

int32_t &MaxTeamsVal = Attrs.MaxTeams.front();

5887

int32_t &MaxThreadsVal = Attrs.MaxThreads.front();

5894 for

(

auto

*A :

->getAttrs()) {

5895

int32_t AttrMinThreadsVal = 1, AttrMaxThreadsVal = -1;

5896

int32_t AttrMinBlocksVal = 1, AttrMaxBlocksVal = -1;

5897 if

(

auto

= dyn_cast<CUDALaunchBoundsAttr>(A))

5899

&AttrMinBlocksVal, &AttrMaxBlocksVal);

5900 else if

(

auto

= dyn_cast<AMDGPUFlatWorkGroupSizeAttr>(A))

5902 nullptr

nullptr

, &AttrMinThreadsVal,

5903

&AttrMaxThreadsVal);

5907

Attrs.MinThreads = std::max(Attrs.MinThreads, AttrMinThreadsVal);

5908 if

(AttrMaxThreadsVal > 0)

5909

MaxThreadsVal = MaxThreadsVal > 0

5910

? std::min(MaxThreadsVal, AttrMaxThreadsVal)

5911

: AttrMaxThreadsVal;

5912

Attrs.MinTeams = std::max(Attrs.MinTeams, AttrMinBlocksVal);

5913 if

(AttrMaxBlocksVal > 0)

5914

MaxTeamsVal = MaxTeamsVal > 0 ? std::min(MaxTeamsVal, AttrMaxBlocksVal)

5922

llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,

5925

llvm::TargetRegionEntryInfo EntryInfo =

5929

llvm::OpenMPIRBuilder::FunctionGenCallback &&GenerateOutlinedFunction =

5930

[&CGF, &

, &CodeGen](StringRef EntryFnName) {

5933

CGOpenMPTargetRegionInfo CGInfo(CS, CodeGen, EntryFnName);

5938

cantFail(

.emitTargetRegionFunction(

5939

EntryInfo, GenerateOutlinedFunction, IsOffloadEntry, OutlinedFn,

5948 for

(

auto

*A :

->getAttrs()) {

5949 if

(

auto

= dyn_cast<AMDGPUWavesPerEUAttr>(A))

5968 while

(

const auto

= dyn_cast_or_null<CompoundStmt>(Child)) {

5970 for

(

const Stmt

*S :

->body()) {

5971 if

(

const auto

= dyn_cast<Expr>(S)) {

5976 if

(isa<AsmStmt>(S) || isa<NullStmt>(S) || isa<OMPFlushDirective>(S) ||

5977

isa<OMPBarrierDirective>(S) || isa<OMPTaskyieldDirective>(S))

5980 if

(

const auto

*DS = dyn_cast<DeclStmt>(S)) {

5981 if

(llvm::all_of(DS->decls(), [](

const Decl

) {

5982

if (isa<EmptyDecl>(D) || isa<DeclContext>(D) ||

5983

isa<TypeDecl>(D) || isa<PragmaCommentDecl>(D) ||

5984

isa<PragmaDetectMismatchDecl>(D) || isa<UsingDecl>(D) ||

5985

isa<UsingDirectiveDecl>(D) ||

5986

isa<OMPDeclareReductionDecl>(D) ||

5987

isa<OMPThreadPrivateDecl>(D) || isa<OMPAllocateDecl>(D))

5989

const auto *VD = dyn_cast<VarDecl>(D);

5992

return VD->hasGlobalStorage() || !VD->isUsed();

6002

Child = Child->IgnoreContainers();

6009

int32_t &MaxTeamsVal) {

6013 "Expected target-based executable directive."

);

6014 switch

(DirectiveKind) {

6016 const auto

*CS =

.getInnermostCapturedStmt();

6019 const Stmt

*ChildStmt =

6021 if

(

const auto

*NestedDir =

6022

dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {

6029 if

(

auto

Constant =

6031

MinTeamsVal = MaxTeamsVal = Constant->getExtValue();

6034

MinTeamsVal = MaxTeamsVal = 0;

6037

MinTeamsVal = MaxTeamsVal = 1;

6041

MinTeamsVal = MaxTeamsVal = -1;

6044 case

OMPD_target_teams_loop:

6045 case

OMPD_target_teams:

6046 case

OMPD_target_teams_distribute:

6047 case

OMPD_target_teams_distribute_simd:

6048 case

OMPD_target_teams_distribute_parallel_for:

6049 case

OMPD_target_teams_distribute_parallel_for_simd: {

6051 const Expr

*NumTeams =

6055

MinTeamsVal = MaxTeamsVal = Constant->getExtValue();

6058

MinTeamsVal = MaxTeamsVal = 0;

6061 case

OMPD_target_parallel:

6062 case

OMPD_target_parallel_for:

6063 case

OMPD_target_parallel_for_simd:

6064 case

OMPD_target_parallel_loop:

6065 case

OMPD_target_simd:

6066

MinTeamsVal = MaxTeamsVal = 1;

6070 case

OMPD_parallel_for:

6071 case

OMPD_parallel_loop:

6072 case

OMPD_parallel_master:

6073 case

OMPD_parallel_sections:

6075 case

OMPD_parallel_for_simd:

6077 case

OMPD_cancellation_point:

6079 case

OMPD_threadprivate:

6090 case

OMPD_taskyield:

6093 case

OMPD_taskgroup:

6099 case

OMPD_target_data:

6100 case

OMPD_target_exit_data:

6101 case

OMPD_target_enter_data:

6102 case

OMPD_distribute:

6103 case

OMPD_distribute_simd:

6104 case

OMPD_distribute_parallel_for:

6105 case

OMPD_distribute_parallel_for_simd:

6106 case

OMPD_teams_distribute:

6107 case

OMPD_teams_distribute_simd:

6108 case

OMPD_teams_distribute_parallel_for:

6109 case

OMPD_teams_distribute_parallel_for_simd:

6110 case

OMPD_target_update:

6111 case

OMPD_declare_simd:

6112 case

OMPD_declare_variant:

6113 case

OMPD_begin_declare_variant:

6114 case

OMPD_end_declare_variant:

6115 case

OMPD_declare_target:

6116 case

OMPD_end_declare_target:

6117 case

OMPD_declare_reduction:

6118 case

OMPD_declare_mapper:

6120 case

OMPD_taskloop_simd:

6121 case

OMPD_master_taskloop:

6122 case

OMPD_master_taskloop_simd:

6123 case

OMPD_parallel_master_taskloop:

6124 case

OMPD_parallel_master_taskloop_simd:

6126 case

OMPD_metadirective:

6132

llvm_unreachable(

"Unexpected directive kind."

);

6137

assert(!CGF.

().OpenMPIsTargetDevice &&

6138 "Clauses associated with the teams directive expected to be emitted " 6139 "only for the host!"

);

6141

int32_t MinNT = -1, MaxNT = -1;

6142 const Expr

*NumTeams =

6144 if

(NumTeams !=

nullptr

) {

6147 switch

(DirectiveKind) {

6149 const auto

*CS =

.getInnermostCapturedStmt();

6150

CGOpenMPInnerExprInfo CGInfo(CGF, *CS);

6154 return

Bld.CreateIntCast(NumTeamsVal, CGF.

6157 case

OMPD_target_teams:

6158 case

OMPD_target_teams_distribute:

6159 case

OMPD_target_teams_distribute_simd:

6160 case

OMPD_target_teams_distribute_parallel_for:

6161 case

OMPD_target_teams_distribute_parallel_for_simd: {

6165 return

Bld.CreateIntCast(NumTeamsVal, CGF.

6173

assert(MinNT == MaxNT &&

"Num threads ranges require handling here."

);

6174 return

llvm::ConstantInt::get(CGF.

, MinNT);

6182 const Expr

, int32_t &UpperBound,

6183 bool

UpperBoundOnly, llvm::Value **CondVal) {

6186 const auto

*Dir = dyn_cast_or_null<OMPExecutableDirective>(Child);

6193 if

(CondVal && Dir->hasClausesOfKind<

OMPIfClause

>()) {

6194

CGOpenMPInnerExprInfo CGInfo(CGF, *CS);

6195

CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);

6197 for

(

const auto

: Dir->getClausesOfKind<

OMPIfClause

>()) {

6198 if

(

->getNameModifier() == OMPD_unknown ||

6199 C

->getNameModifier() == OMPD_parallel) {

6214 if

(

const auto

*PreInit =

6216 for

(

const auto

*I : PreInit->decls()) {

6217 if

(!I->hasAttr<OMPCaptureNoInitAttr>()) {

6220

CodeGenFunction::AutoVarEmission Emission =

6233

CGOpenMPInnerExprInfo CGInfo(CGF, *CS);

6234

CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGInfo);

6235 const auto

*NumThreadsClause =

6237 const Expr

*NTExpr = NumThreadsClause->getNumThreads();

6238 if

(NTExpr->isIntegerConstantExpr(CGF.

getContext

()))

6239 if

(

auto

Constant = NTExpr->getIntegerConstantExpr(CGF.

getContext

()))

6242

? Constant->getZExtValue()

6243

: std::min(UpperBound,

6244 static_cast<

int32_t

>

(Constant->getZExtValue()));

6247 if

(UpperBound == -1)

6251

CodeGenFunction::LexicalScope

(CGF, NTExpr->getSourceRange());

6252 if

(

const auto

*PreInit =

6253

cast_or_null<DeclStmt>(NumThreadsClause->getPreInitStmt())) {

6254 for

(

const auto

*I : PreInit->decls()) {

6255 if

(!I->hasAttr<OMPCaptureNoInitAttr>()) {

6258

CodeGenFunction::AutoVarEmission Emission =

6274 bool

UpperBoundOnly, llvm::Value **CondVal,

const Expr

**ThreadLimitExpr) {

6275

assert((!CGF.

().OpenMPIsTargetDevice || UpperBoundOnly) &&

6276 "Clauses associated with the teams directive expected to be emitted " 6277 "only for the host!"

);

6280 "Expected target-based executable directive."

);

6282 const Expr

*NT =

nullptr

;

6283 const Expr

**NTPtr = UpperBoundOnly ? nullptr : &NT;

6285 auto

CheckForConstExpr = [&](

const Expr

**EPtr) {

6288

UpperBound = UpperBound ? Constant->getZExtValue()

6289

: std::min(UpperBound,

6290

int32_t(Constant->getZExtValue()));

6294 if

(UpperBound == -1)

6300 auto

ReturnSequential = [&]() {

6305 switch

(DirectiveKind) {

6308 getNumThreads

(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);

6314 if

(

const auto

*Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) {

6316

ThreadLimitClause = TLC;

6317 if

(ThreadLimitExpr) {

6318

CGOpenMPInnerExprInfo CGInfo(CGF, *CS);

6322

ThreadLimitClause->getThreadLimit().front()->getSourceRange());

6323 if

(

const auto

*PreInit =

6324

cast_or_null<DeclStmt>(ThreadLimitClause->getPreInitStmt())) {

6325 for

(

const auto

*I : PreInit->decls()) {

6326 if

(!I->hasAttr<OMPCaptureNoInitAttr>()) {

6338 if

(ThreadLimitClause)

6339

CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(),

6341 if

(

const auto

*Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) {

6344

CS = Dir->getInnermostCapturedStmt();

6347

Dir = dyn_cast_or_null<OMPExecutableDirective>(Child);

6350

CS = Dir->getInnermostCapturedStmt();

6351 getNumThreads

(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);

6353 return

ReturnSequential();

6357 case

OMPD_target_teams: {

6361

CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(),

6365 getNumThreads

(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);

6368 if

(

const auto

*Dir = dyn_cast_or_null<OMPExecutableDirective>(Child)) {

6369 if

(Dir->getDirectiveKind() == OMPD_distribute) {

6370

CS = Dir->getInnermostCapturedStmt();

6371 getNumThreads

(CGF, CS, NTPtr, UpperBound, UpperBoundOnly, CondVal);

6376 case

OMPD_target_teams_distribute:

6380

CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(),

6383 getNumThreads

(CGF,

.getInnermostCapturedStmt(), NTPtr, UpperBound,

6384

UpperBoundOnly, CondVal);

6386 case

OMPD_target_teams_loop:

6387 case

OMPD_target_parallel_loop:

6388 case

OMPD_target_parallel:

6389 case

OMPD_target_parallel_for:

6390 case

OMPD_target_parallel_for_simd:

6391 case

OMPD_target_teams_distribute_parallel_for:

6392 case

OMPD_target_teams_distribute_parallel_for_simd: {

6395 for

(

const auto

.getClausesOfKind<

OMPIfClause

>()) {

6396 if

(

->getNameModifier() == OMPD_unknown ||

6397 C

->getNameModifier() == OMPD_parallel) {

6407 return

ReturnSequential();

6417

CheckForConstExpr(ThreadLimitClause->getThreadLimit().front(),

6423

CheckForConstExpr(NumThreadsClause->getNumThreads(),

nullptr

);

6424 return

NumThreadsClause->getNumThreads();

6428 case

OMPD_target_teams_distribute_simd:

6429 case

OMPD_target_simd:

6430 return

ReturnSequential();

6434

llvm_unreachable(

"Unsupported directive kind."

);

6439

llvm::Value *NumThreadsVal =

nullptr

;

6440

llvm::Value *CondVal =

nullptr

;

6441

llvm::Value *ThreadLimitVal =

nullptr

;

6442 const Expr

*ThreadLimitExpr =

nullptr

;

6443

int32_t UpperBound = -1;

6446

CGF,

, UpperBound,

false

, &CondVal,

6450 if

(ThreadLimitExpr) {

6453

ThreadLimitVal = CGF.

.CreateIntCast(ThreadLimitVal, CGF.

6458 if

(UpperBound == 1) {

6459

NumThreadsVal = CGF.

.getInt32(UpperBound);

6462

NumThreadsVal = CGF.

.CreateIntCast(NumThreadsVal, CGF.

6464

}

else if

(ThreadLimitVal) {

6467

NumThreadsVal = ThreadLimitVal;

6468

ThreadLimitVal =

nullptr

;

6471

assert(!ThreadLimitVal &&

"Default not applicable with thread limit value"

);

6472

NumThreadsVal = CGF.

.getInt32(0);

6479

NumThreadsVal = CGF.

.CreateSelect(CondVal, NumThreadsVal,

6485 if

(ThreadLimitVal) {

6486

NumThreadsVal = CGF.

.CreateSelect(

6487

CGF.

.CreateICmpULT(ThreadLimitVal, NumThreadsVal),

6488

ThreadLimitVal, NumThreadsVal);

6491 return

NumThreadsVal;

6501class

MappableExprsHandler {

6504 static unsigned

getFlagMemberOffset() {

6505 unsigned

Offset = 0;

6506 for

(uint64_t Remain =

6507 static_cast<

std::underlying_type_t<OpenMPOffloadMappingFlags>

>

(

6508

OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF);

6509

!(Remain & 1); Remain = Remain >> 1)

6516 class

MappingExprInfo {

6521 const Expr

*MapExpr =

nullptr

;

6524

MappingExprInfo(

const ValueDecl

*MapDecl,

const Expr

*MapExpr =

nullptr

)

6525

: MapDecl(MapDecl), MapExpr(MapExpr) {}

6527 const ValueDecl

*getMapDecl()

const

{

return

MapDecl; }

6528 const Expr

*getMapExpr()

const

{

return

MapExpr; }

6531 using

DeviceInfoTy = llvm::OpenMPIRBuilder::DeviceInfoTy;

6532 using

MapBaseValuesArrayTy = llvm::OpenMPIRBuilder::MapValuesArrayTy;

6533 using

MapValuesArrayTy = llvm::OpenMPIRBuilder::MapValuesArrayTy;

6534 using

MapFlagsArrayTy = llvm::OpenMPIRBuilder::MapFlagsArrayTy;

6535 using

MapDimArrayTy = llvm::OpenMPIRBuilder::MapDimArrayTy;

6536 using

MapNonContiguousArrayTy =

6537

llvm::OpenMPIRBuilder::MapNonContiguousArrayTy;

6544 struct

MapCombinedInfoTy : llvm::OpenMPIRBuilder::MapInfosTy {

6545

MapExprsArrayTy Exprs;

6546

MapValueDeclsArrayTy Mappers;

6547

MapValueDeclsArrayTy DevicePtrDecls;

6550 void

append(MapCombinedInfoTy &CurInfo) {

6551

Exprs.append(CurInfo.Exprs.begin(), CurInfo.Exprs.end());

6552

DevicePtrDecls.append(CurInfo.DevicePtrDecls.begin(),

6553

CurInfo.DevicePtrDecls.end());

6554

Mappers.append(CurInfo.Mappers.begin(), CurInfo.Mappers.end());

6555

llvm::OpenMPIRBuilder::MapInfosTy::append(CurInfo);

6563 struct

StructRangeInfoTy {

6564

MapCombinedInfoTy PreliminaryMapData;

6565

std::pair<

unsigned

> LowestElem = {

6567

std::pair<

unsigned

> HighestElem = {

6571 bool

IsArraySection =

false

;

6572 bool

HasCompleteRecord =

false

;

6582 bool

ReturnDevicePointer =

false

;

6583 bool

IsImplicit =

false

;

6585 const Expr

*VarRef =

nullptr

;

6586 bool

ForDeviceAddr =

false

;

6588

MapInfo() =

default

;

6594 bool

ReturnDevicePointer,

bool

IsImplicit,

6595 const ValueDecl

*Mapper =

nullptr

const Expr

*VarRef =

nullptr

6596 bool

ForDeviceAddr =

false

)

6597

: Components(Components), MapType(MapType), MapModifiers(MapModifiers),

6598

MotionModifiers(MotionModifiers),

6599

ReturnDevicePointer(ReturnDevicePointer), IsImplicit(IsImplicit),

6600

Mapper(Mapper), VarRef(VarRef), ForDeviceAddr(ForDeviceAddr) {}

6606 struct

DeferredDevicePtrEntryTy {

6607 const Expr

*IE =

nullptr

;

6609 bool

ForDeviceAddr =

false

;

6611

DeferredDevicePtrEntryTy(

const Expr

*IE,

const ValueDecl

*VD,

6613

: IE(IE), VD(VD), ForDeviceAddr(ForDeviceAddr) {}

6628

llvm::DenseMap<CanonicalDeclPtr<const VarDecl>,

bool

> FirstPrivateDecls;

6645

llvm::DenseMap<const ValueDecl *, const OMPMapClause *> LambdasMap;

6647

llvm::Value *getExprTypeSize(

const Expr

)

const

{

6651 if

(

const auto

*OAE = dyn_cast<OMPArrayShapingExpr>(

)) {

6652

llvm::Value *

6653

CGF.

(OAE->getBase()->getType()->getPointeeType());

6654 for

(

const Expr

*SE : OAE->getDimensions()) {

6671 if

(

const auto

*OAE = dyn_cast<ArraySectionExpr>(

)) {

6673

OAE->getBase()->IgnoreParenImpCasts())

6679 if

(!OAE->getLength() && OAE->getColonLocFirst().isValid() &&

6680

!OAE->getLowerBound())

6683

llvm::Value *ElemSize;

6685

ElemSize = CGF.

(PTy->getPointeeType().getCanonicalType());

6687 const auto

*ATy = cast<ArrayType>(BaseTy.

getTypePtr

());

6688

assert(ATy &&

"Expecting array type if not a pointer type."

);

6689

ElemSize = CGF.

(ATy->getElementType().getCanonicalType());

6694 if

(!OAE->getLength() && OAE->getColonLocFirst().isInvalid())

6697 if

(

const Expr

*LenExpr = OAE->getLength()) {

6701

LenExpr->getExprLoc());

6702 return

CGF.

.CreateNUWMul(LengthVal, ElemSize);

6704

assert(!OAE->getLength() && OAE->getColonLocFirst().isValid() &&

6705

OAE->getLowerBound() &&

"expected array_section[lb:]."

);

6707

llvm::Value *LengthVal = CGF.

(BaseTy);

6711

OAE->getLowerBound()->getExprLoc());

6712

LBVal = CGF.

.CreateNUWMul(LBVal, ElemSize);

6713

llvm::Value *Cmp = CGF.

.CreateICmpUGT(LengthVal, LBVal);

6714

llvm::Value *TrueVal = CGF.

.CreateNUWSub(LengthVal, LBVal);

6715

LengthVal = CGF.

.CreateSelect(

6716

Cmp, TrueVal, llvm::ConstantInt::get(CGF.

, 0));

6726

OpenMPOffloadMappingFlags getMapTypeBits(

6729 bool

AddPtrFlag,

bool

AddIsTargetParamFlag,

bool

IsNonContiguous)

const

{

6730

OpenMPOffloadMappingFlags Bits =

6731

IsImplicit ? OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT

6732

: OpenMPOffloadMappingFlags::OMP_MAP_NONE;

6734 case

OMPC_MAP_alloc:

6735 case

OMPC_MAP_release:

6742

Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TO;

6745

Bits |= OpenMPOffloadMappingFlags::OMP_MAP_FROM;

6747 case

OMPC_MAP_tofrom:

6748

Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TO |

6749

OpenMPOffloadMappingFlags::OMP_MAP_FROM;

6751 case

OMPC_MAP_delete:

6752

Bits |= OpenMPOffloadMappingFlags::OMP_MAP_DELETE;

6755

llvm_unreachable(

"Unexpected map type!"

);

6758

Bits |= OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ;

6759 if

(AddIsTargetParamFlag)

6760

Bits |= OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;

6761 if

(llvm::is_contained(MapModifiers, OMPC_MAP_MODIFIER_always))

6762

Bits |= OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS;

6763 if

(llvm::is_contained(MapModifiers, OMPC_MAP_MODIFIER_close))

6764

Bits |= OpenMPOffloadMappingFlags::OMP_MAP_CLOSE;

6765 if

(llvm::is_contained(MapModifiers, OMPC_MAP_MODIFIER_present) ||

6766

llvm::is_contained(MotionModifiers, OMPC_MOTION_MODIFIER_present))

6767

Bits |= OpenMPOffloadMappingFlags::OMP_MAP_PRESENT;

6768 if

(llvm::is_contained(MapModifiers, OMPC_MAP_MODIFIER_ompx_hold))

6769

Bits |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;

6770 if

(IsNonContiguous)

6771

Bits |= OpenMPOffloadMappingFlags::OMP_MAP_NON_CONTIG;

6777 bool

isFinalArraySectionExpression(

const Expr

)

const

{

6778 const auto

*OASE = dyn_cast<ArraySectionExpr>(

);

6785 if

(OASE->getColonLocFirst().isInvalid())

6788 const Expr

*Length = OASE->getLength();

6795

OASE->getBase()->IgnoreParenImpCasts())

6797 if

(

const auto

*ATy = dyn_cast<ConstantArrayType>(BaseQTy.

getTypePtr

()))

6798 return

ATy->getSExtSize() != 1;

6807 if

(!Length->EvaluateAsInt(Result, CGF.

getContext

()))

6810

llvm::APSInt ConstLength = Result.Val.getInt();

6811 return

ConstLength.getSExtValue() != 1;

6819 void

generateInfoForComponentList(

6823

MapCombinedInfoTy &CombinedInfo,

6824

MapCombinedInfoTy &StructBaseCombinedInfo,

6825

StructRangeInfoTy &PartialStruct,

bool

IsFirstComponentList,

6826 bool

IsImplicit,

bool

GenerateAllInfoForClauses,

6827 const ValueDecl

*Mapper =

nullptr

bool

ForDeviceAddr =

false

6828 const ValueDecl

*BaseDecl =

nullptr

const Expr

*MapExpr =

nullptr

6830

OverlappedElements = {},

6831 bool

AreBothBasePtrAndPteeMapped =

false

)

const

{

7013 bool

IsCaptureFirstInfo = IsFirstComponentList;

7017 bool

RequiresReference =

false

;

7020 auto

CI = Components.rbegin();

7021 auto

CE = Components.rend();

7026 bool

IsExpressionFirstInfo =

true

;

7027 bool

FirstPointerInComplexData =

false

;

7029 const Expr

*AssocExpr = I->getAssociatedExpression();

7030 const auto

*AE = dyn_cast<ArraySubscriptExpr>(AssocExpr);

7031 const auto

*OASE = dyn_cast<ArraySectionExpr>(AssocExpr);

7032 const auto

*OAShE = dyn_cast<OMPArrayShapingExpr>(AssocExpr);

7034 if

(AreBothBasePtrAndPteeMapped && std::next(I) == CE)

7036 if

(isa<MemberExpr>(AssocExpr)) {

7040

}

else if

((AE && isa<CXXThisExpr>(AE->getBase()->IgnoreParenImpCasts())) ||

7042

isa<CXXThisExpr>(OASE->getBase()->IgnoreParenImpCasts()))) {

7045

isa<CXXThisExpr>(OAShE->getBase()->IgnoreParenCasts())) {

7054 if

(

const auto

*VD =

7055

dyn_cast_or_null<VarDecl>(I->getAssociatedDeclaration())) {

7056 if

(std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =

7057

OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD)) {

7058 if

((*Res == OMPDeclareTargetDeclAttr::MT_Link) ||

7059

((*Res == OMPDeclareTargetDeclAttr::MT_To ||

7060

*Res == OMPDeclareTargetDeclAttr::MT_Enter) &&

7062

RequiresReference =

true

;

7072

I->getAssociatedDeclaration()->getType().getNonReferenceType();

7077 const auto

*VD = dyn_cast<VarDecl>(I->getAssociatedDeclaration());

7078 if

(!AreBothBasePtrAndPteeMapped &&

7083

FirstPointerInComplexData =

true

;

7102 bool

ShouldBeMemberOf =

false

;

7117 bool

IsNonContiguous = CombinedInfo.NonContigInfo.IsNonContiguous;

7118 bool

IsPrevMemberReference =

false

;

7120 bool

IsPartialMapped =

7121

!PartialStruct.PreliminaryMapData.BasePointers.empty();

7128 bool

IsMappingWholeStruct =

true

;

7129 if

(!GenerateAllInfoForClauses) {

7130

IsMappingWholeStruct =

false

;

7132 for

(

auto

TempI = I; TempI != CE; ++TempI) {

7134

dyn_cast<MemberExpr>(TempI->getAssociatedExpression());

7136

IsMappingWholeStruct =

false

;

7142 for

(; I != CE; ++I) {

7144 if

(!EncounteredME) {

7145

EncounteredME = dyn_cast<MemberExpr>(I->getAssociatedExpression());

7148 if

(EncounteredME) {

7149

ShouldBeMemberOf =

true

;

7152 if

(FirstPointerInComplexData) {

7154

->getAssociatedDeclaration()

7156

.getNonReferenceType();

7158

FirstPointerInComplexData =

false

;

7163 auto

Next = std::next(I);

7173 bool

IsFinalArraySection =

7175

isFinalArraySectionExpression(I->getAssociatedExpression());

7179 const ValueDecl

*MapDecl = (I->getAssociatedDeclaration())

7180

? I->getAssociatedDeclaration()

7182

MapExpr = (I->getAssociatedExpression()) ? I->getAssociatedExpression()

7189

dyn_cast<ArraySectionExpr>(I->getAssociatedExpression());

7191

dyn_cast<OMPArrayShapingExpr>(I->getAssociatedExpression());

7192 const auto

*UO = dyn_cast<UnaryOperator>(I->getAssociatedExpression());

7193 const auto

*BO = dyn_cast<BinaryOperator>(I->getAssociatedExpression());

7199

I->getAssociatedExpression()->getType()->isAnyPointerType();

7200 bool

IsMemberReference = isa<MemberExpr>(I->getAssociatedExpression()) &&

7203 bool

IsNonDerefPointer = IsPointer &&

7204

!(UO && UO->getOpcode() != UO_Deref) && !BO &&

7210 if

(Next == CE || IsMemberReference || IsNonDerefPointer ||

7211

IsFinalArraySection) {

7214

assert((Next == CE ||

7215

isa<MemberExpr>(Next->getAssociatedExpression()) ||

7216

isa<ArraySubscriptExpr>(Next->getAssociatedExpression()) ||

7217

isa<ArraySectionExpr>(Next->getAssociatedExpression()) ||

7218

isa<OMPArrayShapingExpr>(Next->getAssociatedExpression()) ||

7219

isa<UnaryOperator>(Next->getAssociatedExpression()) ||

7220

isa<BinaryOperator>(Next->getAssociatedExpression())) &&

7221 "Unexpected expression"

);

7227 const Expr

*BaseExpr =

->getBase();

7231 if

(

->isArrow()) {

7247

OAShE->getBase()->getType()->getPointeeType()),

7249

OAShE->getBase()->getType()));

7250

}

else if

(IsMemberReference) {

7251 const auto

*ME = cast<MemberExpr>(I->getAssociatedExpression());

7252 LValue

BaseLVal = EmitMemberExprBase(CGF, ME);

7254

BaseLVal, cast<FieldDecl>(MapDecl))

7267 bool

IsMemberPointerOrAddr =

7269

(((IsPointer || ForDeviceAddr) &&

7270

I->getAssociatedExpression() == EncounteredME) ||

7271

(IsPrevMemberReference && !IsPointer) ||

7272

(IsMemberReference && Next != CE &&

7273

!Next->getAssociatedExpression()->getType()->isPointerType()));

7274 if

(!OverlappedElements.empty() && Next == CE) {

7276

assert(!PartialStruct.Base.isValid() &&

"The base element is set."

);

7277

assert(!IsPointer &&

7278 "Unexpected base element with the pointer type."

);

7281

PartialStruct.LowestElem = {0, LowestElem};

7283

I->getAssociatedExpression()->getType());

7288

PartialStruct.HighestElem = {

7289

std::numeric_limits<

decltype

(

7290

PartialStruct.HighestElem.first)>

::max

(),

7292

PartialStruct.Base = BP;

7293

PartialStruct.LB = LB;

7295

PartialStruct.PreliminaryMapData.BasePointers.empty() &&

7296 "Overlapped elements must be used only once for the variable."

);

7297

std::swap(PartialStruct.PreliminaryMapData, CombinedInfo);

7299

OpenMPOffloadMappingFlags Flags =

7300

OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |

7301

getMapTypeBits(MapType, MapModifiers, MotionModifiers, IsImplicit,

7303 false

, IsNonContiguous);

7304

llvm::Value *

nullptr

;

7307

Component : OverlappedElements) {

7311 if

(

const ValueDecl

*VD = MC.getAssociatedDeclaration()) {

7312 const auto

*FD = dyn_cast<FieldDecl>(VD);

7313 if

(FD && FD->getType()->isLValueReferenceType()) {

7315

cast<MemberExpr>(MC.getAssociatedExpression());

7316 LValue

BaseLVal = EmitMemberExprBase(CGF, ME);

7332

assert(Size &&

"Failed to determine structure size"

);

7333

CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr);

7335

CombinedInfo.DevicePtrDecls.push_back(

nullptr

);

7336

CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);

7338

CombinedInfo.Sizes.push_back(CGF.

.CreateIntCast(

7340

CombinedInfo.Types.push_back(Flags);

7341

CombinedInfo.Mappers.push_back(

nullptr

);

7342

CombinedInfo.NonContigInfo.Dims.push_back(IsNonContiguous ? DimSize

7346

CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr);

7348

CombinedInfo.DevicePtrDecls.push_back(

nullptr

);

7349

CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);

7355

CombinedInfo.Sizes.push_back(

7357

CombinedInfo.Types.push_back(Flags);

7358

CombinedInfo.Mappers.push_back(

nullptr

);

7359

CombinedInfo.NonContigInfo.Dims.push_back(IsNonContiguous ? DimSize

7363

llvm::Value *

= getExprTypeSize(I->getAssociatedExpression());

7370 if

((!IsMemberPointerOrAddr && !IsPartialMapped) ||

7372 if

(!IsMappingWholeStruct) {

7373

CombinedInfo.Exprs.emplace_back(MapDecl, MapExpr);

7375

CombinedInfo.DevicePtrDecls.push_back(

nullptr

);

7376

CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);

7378

CombinedInfo.Sizes.push_back(CGF.

.CreateIntCast(

7380

CombinedInfo.NonContigInfo.Dims.push_back(IsNonContiguous ? DimSize

7383

StructBaseCombinedInfo.Exprs.emplace_back(MapDecl, MapExpr);

7384

StructBaseCombinedInfo.BasePointers.push_back(

7386

StructBaseCombinedInfo.DevicePtrDecls.push_back(

nullptr

);

7387

StructBaseCombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);

7388

StructBaseCombinedInfo.Pointers.push_back(LB.

emitRawPointer

(CGF));

7389

StructBaseCombinedInfo.Sizes.push_back(CGF.

.CreateIntCast(

7391

StructBaseCombinedInfo.NonContigInfo.Dims.push_back(

7392

IsNonContiguous ? DimSize : 1);

7396 bool

HasMapper = Mapper && Next == CE;

7397 if

(!IsMappingWholeStruct)

7398

CombinedInfo.Mappers.push_back(HasMapper ? Mapper :

nullptr

);

7400

StructBaseCombinedInfo.Mappers.push_back(HasMapper ? Mapper

7407

OpenMPOffloadMappingFlags Flags =

7408

getMapTypeBits(MapType, MapModifiers, MotionModifiers, IsImplicit,

7409

!IsExpressionFirstInfo || RequiresReference ||

7410

FirstPointerInComplexData || IsMemberReference,

7411

AreBothBasePtrAndPteeMapped ||

7412

(IsCaptureFirstInfo && !RequiresReference),

7415 if

(!IsExpressionFirstInfo || IsMemberReference) {

7418 if

(IsPointer || (IsMemberReference && Next != CE))

7419

Flags &= ~(OpenMPOffloadMappingFlags::OMP_MAP_TO |

7420

OpenMPOffloadMappingFlags::OMP_MAP_FROM |

7421

OpenMPOffloadMappingFlags::OMP_MAP_ALWAYS |

7422

OpenMPOffloadMappingFlags::OMP_MAP_DELETE |

7423

OpenMPOffloadMappingFlags::OMP_MAP_CLOSE);

7425 if

(ShouldBeMemberOf) {

7428

Flags |= OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF;

7431

ShouldBeMemberOf =

false

;

7435 if

(!IsMappingWholeStruct)

7436

CombinedInfo.Types.push_back(Flags);

7438

StructBaseCombinedInfo.Types.push_back(Flags);

7444 if

(EncounteredME) {

7445 const auto

*FD = cast<FieldDecl>(EncounteredME->

getMemberDecl

());

7446 unsigned

FieldIndex = FD->getFieldIndex();

7449 if

(!PartialStruct.Base.isValid()) {

7450

PartialStruct.LowestElem = {FieldIndex, LowestElem};

7451 if

(IsFinalArraySection) {

7455

PartialStruct.HighestElem = {FieldIndex, HB};

7457

PartialStruct.HighestElem = {FieldIndex, LowestElem};

7459

PartialStruct.Base = BP;

7460

PartialStruct.LB = BP;

7461

}

else if

(FieldIndex < PartialStruct.LowestElem.first) {

7462

PartialStruct.LowestElem = {FieldIndex, LowestElem};

7463

}

else if

(FieldIndex > PartialStruct.HighestElem.first) {

7464 if

(IsFinalArraySection) {

7468

PartialStruct.HighestElem = {FieldIndex, HB};

7470

PartialStruct.HighestElem = {FieldIndex, LowestElem};

7476 if

(IsFinalArraySection || IsNonContiguous)

7477

PartialStruct.IsArraySection =

true

;

7480 if

(IsFinalArraySection)

7485

BP = IsMemberReference ? LowestElem : LB;

7486 if

(!IsPartialMapped)

7487

IsExpressionFirstInfo =

false

;

7488

IsCaptureFirstInfo =

false

;

7489

FirstPointerInComplexData =

false

;

7490

IsPrevMemberReference = IsMemberReference;

7491

}

else if

(FirstPointerInComplexData) {

7493

->getAssociatedDeclaration()

7495

.getNonReferenceType();

7497

FirstPointerInComplexData =

false

;

7503

PartialStruct.HasCompleteRecord =

true

;

7505 if

(!IsNonContiguous)

7512

MapValuesArrayTy CurOffsets = {llvm::ConstantInt::get(CGF.

, 0)};

7513

MapValuesArrayTy CurCounts = {llvm::ConstantInt::get(CGF.

, 1)};

7514

MapValuesArrayTy CurStrides;

7515

MapValuesArrayTy DimSizes{llvm::ConstantInt::get(CGF.

, 1)};

7523 const Expr

*AssocExpr = Component.getAssociatedExpression();

7524 const auto

*OASE = dyn_cast<ArraySectionExpr>(AssocExpr);

7534

assert((VAT || CAT || &Component == &*Components.begin()) &&

7535 "Should be either ConstantArray or VariableArray if not the " 7536 "first Component"

);

7539 if

(CurStrides.empty()) {

7540 const Type

*ElementType =

nullptr

;

7542

ElementType = CAT->getElementType().getTypePtr();

7544

ElementType = VAT->getElementType().getTypePtr();

7546

assert(&Component == &*Components.begin() &&

7547 "Only expect pointer (non CAT or VAT) when this is the " 7548 "first Component"

);

7555 if

(&Component != &*Components.begin())

7559

CurStrides.push_back(

7560

llvm::ConstantInt::get(CGF.

, ElementTypeSize));

7565 if

(DimSizes.size() < Components.size() - 1) {

7568

llvm::ConstantInt::get(CGF.

, CAT->getZExtSize()));

7570

DimSizes.push_back(CGF.

.CreateIntCast(

7577 auto

*DI = DimSizes.begin() + 1;

7579

llvm::Value *DimProd =

7580

llvm::ConstantInt::get(CGF.

, ElementTypeSize);

7591 const Expr

*AssocExpr = Component.getAssociatedExpression();

7593 if

(

const auto

*AE = dyn_cast<ArraySubscriptExpr>(AssocExpr)) {

7594

llvm::Value *Offset = CGF.

.CreateIntCast(

7597

CurOffsets.push_back(Offset);

7598

CurCounts.push_back(llvm::ConstantInt::get(CGF.

1));

7599

CurStrides.push_back(CurStrides.back());

7603 const auto

*OASE = dyn_cast<ArraySectionExpr>(AssocExpr);

7609 const Expr

*OffsetExpr = OASE->getLowerBound();

7610

llvm::Value *Offset =

nullptr

;

7613

Offset = llvm::ConstantInt::get(CGF.

, 0);

7619

CurOffsets.push_back(Offset);

7622 const Expr

*CountExpr = OASE->getLength();

7623

llvm::Value *Count =

nullptr

;

7629 if

(!OASE->getColonLocFirst().isValid() &&

7630

!OASE->getColonLocSecond().isValid()) {

7631

Count = llvm::ConstantInt::get(CGF.

, 1);

7637 const Expr

*StrideExpr = OASE->getStride();

7638

llvm::Value *Stride =

7644

Count = CGF.

.CreateUDiv(

7645

CGF.

.CreateNUWSub(*DI, Offset), Stride);

7647

Count = CGF.

.CreateNUWSub(*DI, Offset);

7652

Count = CGF.

.CreateIntCast(Count, CGF.

false

);

7653

CurCounts.push_back(Count);

7662 const Expr

*StrideExpr = OASE->getStride();

7663

llvm::Value *Stride =

7668

DimProd = CGF.

.CreateNUWMul(DimProd, *(DI - 1));

7670

CurStrides.push_back(CGF.

.CreateNUWMul(DimProd, Stride));

7672

CurStrides.push_back(DimProd);

7673 if

(DI != DimSizes.end())

7677

CombinedInfo.NonContigInfo.Offsets.push_back(CurOffsets);

7678

CombinedInfo.NonContigInfo.Counts.push_back(CurCounts);

7679

CombinedInfo.NonContigInfo.Strides.push_back(CurStrides);

7685

OpenMPOffloadMappingFlags

7687

assert(Cap.

capturesVariable

() &&

"Expected capture by reference only!"

);

7694 return

OpenMPOffloadMappingFlags::OMP_MAP_TO |

7695

OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ;

7696 return

OpenMPOffloadMappingFlags::OMP_MAP_PRIVATE |

7697

OpenMPOffloadMappingFlags::OMP_MAP_TO;

7700 if

(I != LambdasMap.end())

7702 return

getMapTypeBits(

7703

I->getSecond()->getMapType(), I->getSecond()->getMapTypeModifiers(),

7704

{}, I->getSecond()->isImplicit(),

7708 return

OpenMPOffloadMappingFlags::OMP_MAP_TO |

7709

OpenMPOffloadMappingFlags::OMP_MAP_FROM;

7714 bool

AsBase)

const

{

7717

llvm::StructType *St =

7720 unsigned

NumElements = St->getNumElements();

7722

llvm::PointerUnion<const CXXRecordDecl *, const FieldDecl *>, 4>

7723

RecordLayout(NumElements);

7726 for

(

const auto

&I : RD->

bases

()) {

7741

RecordLayout[FieldIndex] =

;

7744 for

(

const auto

&I : RD->

vbases

()) {

7752 if

(RecordLayout[FieldIndex])

7754

RecordLayout[FieldIndex] =

;

7757

assert(!RD->

isUnion

() &&

"Unexpected union."

);

7758 for

(

const auto

*Field : RD->

fields

()) {

7761 if

Field

->isBitField() &&

7764

RecordLayout[FieldIndex] =

Field

;

7767 for

(

const

llvm::PointerUnion<const CXXRecordDecl *, const FieldDecl *>

7768

: RecordLayout) {

7769 if

(

.isNull())

7771 if

(

const auto

= dyn_cast<const CXXRecordDecl *>(

))

7772

getPlainLayout(

, Layout,

true

);

7774

Layout.push_back(cast<const FieldDecl *>(

));

7783 void

generateAllInfoForClauses(

7785

llvm::OpenMPIRBuilder &OMPBuilder,

7792

llvm::MapVector<CanonicalDeclPtr<const Decl>,

7799

[&Info, &SkipVarSet](

7805 bool

ReturnDevicePointer,

bool

IsImplicit,

const ValueDecl

*Mapper,

7806 const Expr

*VarRef =

nullptr

bool

ForDeviceAddr =

false

) {

7807 if

(SkipVarSet.contains(

))

7809 auto

It = Info.try_emplace(

, Total).first;

7810

It->second[

].emplace_back(

7811

L, MapType, MapModifiers, MotionModifiers, ReturnDevicePointer,

7812

IsImplicit, Mapper, VarRef, ForDeviceAddr);

7815 for

(

const auto

: Clauses) {

7816 const auto

= dyn_cast<OMPMapClause>(

);

7820 if

(llvm::is_contained(

->getMapTypeModifiers(),

7821

OMPC_MAP_MODIFIER_present))

7823 else if

(

->getMapType() == OMPC_MAP_alloc)

7825 const auto

*EI =

->getVarRefs().begin();

7826 for

(

const auto

L :

->component_lists()) {

7827 const Expr

= (

->getMapLoc().isValid()) ? *EI :

nullptr

;

7828

InfoGen(std::get<0>(L),

, std::get<1>(L),

->getMapType(),

7829 C

->getMapTypeModifiers(), {},

7830 false

->isImplicit(), std::get<2>(L),

7835 for

(

const auto

: Clauses) {

7836 const auto

= dyn_cast<OMPToClause>(

);

7840 if

(llvm::is_contained(

->getMotionModifiers(),

7841

OMPC_MOTION_MODIFIER_present))

7843 const auto

*EI =

->getVarRefs().begin();

7844 for

(

const auto

L :

->component_lists()) {

7845

InfoGen(std::get<0>(L),

, std::get<1>(L), OMPC_MAP_to, {},

7846 C

->getMotionModifiers(),

false

7847 C

->isImplicit(), std::get<2>(L), *EI);

7851 for

(

const auto

: Clauses) {

7852 const auto

= dyn_cast<OMPFromClause>(

);

7856 if

(llvm::is_contained(

->getMotionModifiers(),

7857

OMPC_MOTION_MODIFIER_present))

7859 const auto

*EI =

->getVarRefs().begin();

7860 for

(

const auto

L :

->component_lists()) {

7861

InfoGen(std::get<0>(L),

, std::get<1>(L), OMPC_MAP_from, {},

7862 C

->getMotionModifiers(),

7863 false

->isImplicit(), std::get<2>(L),

7876

llvm::MapVector<CanonicalDeclPtr<const Decl>,

7879

MapCombinedInfoTy UseDeviceDataCombinedInfo;

7881 auto

&&UseDeviceDataCombinedInfoGen =

7882

[&UseDeviceDataCombinedInfo](

const ValueDecl

*VD, llvm::Value *Ptr,

7884

UseDeviceDataCombinedInfo.Exprs.push_back(VD);

7885

UseDeviceDataCombinedInfo.BasePointers.emplace_back(Ptr);

7886

UseDeviceDataCombinedInfo.DevicePtrDecls.emplace_back(VD);

7887

UseDeviceDataCombinedInfo.DevicePointers.emplace_back(

7888

IsDevAddr ? DeviceInfoTy::Address : DeviceInfoTy::Pointer);

7889

UseDeviceDataCombinedInfo.Pointers.push_back(Ptr);

7890

UseDeviceDataCombinedInfo.Sizes.push_back(

7891

llvm::Constant::getNullValue(CGF.Int64Ty));

7892

UseDeviceDataCombinedInfo.Types.push_back(

7893

OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM);

7894

UseDeviceDataCombinedInfo.Mappers.push_back(

nullptr

);

7898

[&DeferredInfo, &UseDeviceDataCombinedInfoGen,

7902 bool

IsImplicit,

bool

IsDevAddr) {

7906 if

(isa<MemberExpr>(IE)) {

7915 false

, IsImplicit,

nullptr

7917

DeferredInfo[

nullptr

].emplace_back(IE, VD, IsDevAddr);

7921 if

(IE->isGLValue())

7922

Ptr = CGF.EmitLValue(IE).getPointer(CGF);

7924

Ptr = CGF.EmitScalarExpr(IE);

7926

Ptr = CGF.EmitLoadOfScalar(CGF.EmitLValue(IE), IE->getExprLoc());

7928

UseDeviceDataCombinedInfoGen(VD, Ptr, CGF, IsDevAddr);

7933 const Expr

*IE,

bool

IsDevAddr) ->

bool

{

7940 auto

It = Info.find(isa<MemberExpr>(IE) ? nullptr : VD);

7941 if

(It != Info.end()) {

7942 bool Found

false

;

7943 for

(

auto

: It->second) {

7944 auto

*CI = llvm::find_if(

, [VD](

const

MapInfo &MI) {

7945 return

MI.Components.back().getAssociatedDeclaration() == VD;

7953 if

(CI !=

.end()) {

7955

CI->ForDeviceAddr = IsDevAddr;

7956

CI->ReturnDevicePointer =

true

;

7960 auto

PrevCI = std::next(CI->Components.rbegin());

7961 const auto

*VarD = dyn_cast<VarDecl>(VD);

7963

isa<MemberExpr>(IE) ||

7964

!VD->getType().getNonReferenceType()->isPointerType() ||

7965

PrevCI == CI->Components.rend() ||

7966

isa<MemberExpr>(PrevCI->getAssociatedExpression()) || !VarD ||

7967

VarD->hasLocalStorage()) {

7968

CI->ForDeviceAddr = IsDevAddr;

7969

CI->ReturnDevicePointer =

true

;

7987 for

(

const auto

: Clauses) {

7988 const auto

= dyn_cast<OMPUseDevicePtrClause>(

);

7991 for

(

const auto

L :

->component_lists()) {

7994

assert(!Components.empty() &&

7995 "Not expecting empty list of components!"

);

7996 const ValueDecl

*VD = Components.back().getAssociatedDeclaration();

7998 const Expr

*IE = Components.back().getAssociatedExpression();

7999 if

(IsMapInfoExist(CGF, VD, IE,

false

))

8001

MapInfoGen(CGF, IE, VD, Components,

->isImplicit(),

8006

llvm::SmallDenseSet<CanonicalDeclPtr<const Decl>, 4> Processed;

8007 for

(

const auto

: Clauses) {

8008 const auto

= dyn_cast<OMPUseDeviceAddrClause>(

);

8011 for

(

const auto

L :

->component_lists()) {

8014

assert(!std::get<1>(L).empty() &&

8015 "Not expecting empty list of components!"

);

8016 const ValueDecl

*VD = std::get<1>(L).back().getAssociatedDeclaration();

8017 if

(!Processed.insert(VD).second)

8020 const Expr

*IE = std::get<1>(L).back().getAssociatedExpression();

8021 if

(IsMapInfoExist(CGF, VD, IE,

true

))

8023

MapInfoGen(CGF, IE, VD, Components,

->isImplicit(),

8028 for

(

const auto

: Info) {

8029

StructRangeInfoTy PartialStruct;

8031

MapCombinedInfoTy CurInfo;

8033

MapCombinedInfoTy StructBaseCurInfo;

8035 const ValueDecl

*VD = cast_or_null<ValueDecl>(

);

8036 bool

HasMapBasePtr =

false

;

8037 bool

HasMapArraySec =

false

;

8039 for

(

const auto

&M :

.second) {

8040

HasMapBasePtr = any_of(M, [](

const

MapInfo &L) {

8041 return

isa_and_present<DeclRefExpr>(L.VarRef);

8043

HasMapArraySec = any_of(M, [](

const

MapInfo &L) {

8044 return

isa_and_present<ArraySectionExpr, ArraySubscriptExpr>(

8047 if

(HasMapBasePtr && HasMapArraySec)

8051 for

(

const auto

&M :

.second) {

8052 for

(

const

MapInfo &L : M) {

8053

assert(!L.Components.empty() &&

8054 "Not expecting declaration with no component lists."

);

8057 unsigned

CurrentBasePointersIdx = CurInfo.BasePointers.size();

8058 unsigned

StructBasePointersIdx =

8059

StructBaseCurInfo.BasePointers.size();

8060

CurInfo.NonContigInfo.IsNonContiguous =

8061

L.Components.back().isNonContiguous();

8062

generateInfoForComponentList(

8063

L.MapType, L.MapModifiers, L.MotionModifiers, L.Components,

8064

CurInfo, StructBaseCurInfo, PartialStruct,

8065 false

, L.IsImplicit,

8066 true

, L.Mapper, L.ForDeviceAddr, VD,

8068

HasMapBasePtr && HasMapArraySec);

8072 if

(L.ReturnDevicePointer) {

8076

assert((CurrentBasePointersIdx < CurInfo.BasePointers.size() ||

8077

StructBasePointersIdx <

8078

StructBaseCurInfo.BasePointers.size()) &&

8079 "Unexpected number of mapped base pointers."

);

8083

L.Components.back().getAssociatedDeclaration();

8084

assert(RelevantVD &&

8085 "No relevant declaration related with device pointer??"

);

8092 if

(StructBasePointersIdx < StructBaseCurInfo.BasePointers.size()) {

8093

StructBaseCurInfo.DevicePtrDecls[StructBasePointersIdx] =

8095

StructBaseCurInfo.DevicePointers[StructBasePointersIdx] =

8096

L.ForDeviceAddr ? DeviceInfoTy::Address

8097

: DeviceInfoTy::Pointer;

8098

StructBaseCurInfo.Types[StructBasePointersIdx] |=

8099

OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM;

8101

CurInfo.DevicePtrDecls[CurrentBasePointersIdx] = RelevantVD;

8102

CurInfo.DevicePointers[CurrentBasePointersIdx] =

8103

L.ForDeviceAddr ? DeviceInfoTy::Address

8104

: DeviceInfoTy::Pointer;

8105

CurInfo.Types[CurrentBasePointersIdx] |=

8106

OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM;

8114 auto

CI = DeferredInfo.find(

.first);

8115 if

(CI != DeferredInfo.end()) {

8116 for

(

const

DeferredDevicePtrEntryTy &L : CI->second) {

8117

llvm::Value *BasePtr;

8119 if

(L.ForDeviceAddr) {

8120 if

(L.IE->isGLValue())

8128

CurInfo.Types.push_back(

8129

OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM |

8130

OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF);

8134

L.IE->getExprLoc());

8138

CurInfo.Types.push_back(

8139

OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |

8140

OpenMPOffloadMappingFlags::OMP_MAP_RETURN_PARAM |

8141

OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF);

8143

CurInfo.Exprs.push_back(L.VD);

8144

CurInfo.BasePointers.emplace_back(BasePtr);

8145

CurInfo.DevicePtrDecls.emplace_back(L.VD);

8146

CurInfo.DevicePointers.emplace_back(

8147

L.ForDeviceAddr ? DeviceInfoTy::Address : DeviceInfoTy::Pointer);

8148

CurInfo.Pointers.push_back(Ptr);

8149

CurInfo.Sizes.push_back(

8150

llvm::Constant::getNullValue(this->CGF.

));

8151

CurInfo.Mappers.push_back(

nullptr

);

8157

MapCombinedInfoTy UnionCurInfo;

8158

UnionCurInfo.append(StructBaseCurInfo);

8159

UnionCurInfo.append(CurInfo);

8163 if

(PartialStruct.Base.isValid()) {

8164

UnionCurInfo.NonContigInfo.Dims.push_back(0);

8166

emitCombinedEntry(CombinedInfo, UnionCurInfo.Types, PartialStruct,

8167

!VD, OMPBuilder, VD);

8171

CombinedInfo.append(UnionCurInfo);

8174

CombinedInfo.append(UseDeviceDataCombinedInfo);

8179

: CurDir(&Dir), CGF(CGF) {

8182 for

(

const auto

->varlist())

8183

FirstPrivateDecls.try_emplace(

8184

cast<VarDecl>(cast<DeclRefExpr>(

)->getDecl()),

->isImplicit());

8187 for

(

unsigned

I = 0,

->getNumberOfAllocators(); I <

; ++I) {

8189 if

(

const auto

*DRE = dyn_cast_or_null<DeclRefExpr>(

.AllocatorTraits))

8190

FirstPrivateDecls.try_emplace(cast<VarDecl>(DRE->getDecl()),

8192 else if

(

const auto

*VD = dyn_cast<VarDecl>(

8193

cast<DeclRefExpr>(

.Allocator->IgnoreParenImpCasts())

8195

FirstPrivateDecls.try_emplace(VD,

true

);

8200 for

(

auto

L :

->component_lists())

8201

DevPointersMap[std::get<0>(L)].push_back(std::get<1>(L));

8204 for

(

auto

L :

->component_lists())

8205

HasDevAddrsMap[std::get<0>(L)].push_back(std::get<1>(L));

8208 if

(

->getMapType() != OMPC_MAP_to)

8210 for

(

auto

L :

->component_lists()) {

8212 const auto

*RD = VD ? VD->

()

8218

LambdasMap.try_emplace(std::get<0>(L),

);

8225

: CurDir(&Dir), CGF(CGF) {}

8230 void

emitCombinedEntry(MapCombinedInfoTy &CombinedInfo,

8231

MapFlagsArrayTy &CurTypes,

8232 const

StructRangeInfoTy &PartialStruct,

bool

IsMapThis,

8233

llvm::OpenMPIRBuilder &OMPBuilder,

8235 bool

NotTargetParams =

true

)

const

{

8236 if

(CurTypes.size() == 1 &&

8237

((CurTypes.back() & OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF) !=

8238

OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF) &&

8239

!PartialStruct.IsArraySection)

8241 Address

LBAddr = PartialStruct.LowestElem.second;

8242 Address

HBAddr = PartialStruct.HighestElem.second;

8243 if

(PartialStruct.HasCompleteRecord) {

8244

LBAddr = PartialStruct.LB;

8245

HBAddr = PartialStruct.LB;

8247

CombinedInfo.Exprs.push_back(VD);

8249

CombinedInfo.BasePointers.push_back(PartialStruct.Base.emitRawPointer(CGF));

8250

CombinedInfo.DevicePtrDecls.push_back(

nullptr

);

8251

CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);

8257 bool

HasBaseClass = RD && IsMapThis ? RD->

getNumBases

() > 0 :

false

;

8267

CombinedInfo.Pointers.push_back(PartialStruct.Base.emitRawPointer(CGF));

8269

llvm::Value *

8272

CombinedInfo.Sizes.push_back(Size);

8274

CombinedInfo.Pointers.push_back(LB);

8277

llvm::Value *HAddr = CGF.

.CreateConstGEP1_32(

8279

llvm::Value *CLAddr = CGF.

.CreatePointerCast(LB, CGF.

);

8280

llvm::Value *CHAddr = CGF.

.CreatePointerCast(HAddr, CGF.

);

8281

llvm::Value *Diff = CGF.

.CreatePtrDiff(CGF.

Int8Ty

, CHAddr, CLAddr);

8284

CombinedInfo.Sizes.push_back(Size);

8286

CombinedInfo.Mappers.push_back(

nullptr

);

8288

CombinedInfo.Types.push_back(

8289

NotTargetParams ? OpenMPOffloadMappingFlags::OMP_MAP_NONE

8290

: !PartialStruct.PreliminaryMapData.BasePointers.empty()

8291

? OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ

8292

: OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM);

8295 if

(CurTypes.end() !=

8296

llvm::find_if(CurTypes, [](OpenMPOffloadMappingFlags

Type

) {

8297

return static_cast<std::underlying_type_t<OpenMPOffloadMappingFlags>>(

8298

Type & OpenMPOffloadMappingFlags::OMP_MAP_PRESENT);

8300

CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_PRESENT;

8302

(*CurTypes.begin()) &= ~OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;

8309 if

(CurTypes.end() !=

8310

llvm::find_if(CurTypes, [](OpenMPOffloadMappingFlags

Type

) {

8311 return static_cast<

std::underlying_type_t<OpenMPOffloadMappingFlags>

>

(

8312 Type

& OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD);

8314

CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;

8315 for

(

auto

&M : CurTypes)

8316

M |= OpenMPOffloadMappingFlags::OMP_MAP_OMPX_HOLD;

8322

OpenMPOffloadMappingFlags MemberOfFlag =

8323

OMPBuilder.getMemberOfFlag(CombinedInfo.BasePointers.size() - 1);

8324 for

(

auto

&M : CurTypes)

8325

OMPBuilder.setCorrectMemberOfFlag(M, MemberOfFlag);

8333 void

generateAllInfo(

8334

MapCombinedInfoTy &CombinedInfo, llvm::OpenMPIRBuilder &OMPBuilder,

8337

assert(isa<const OMPExecutableDirective *>(CurDir) &&

8338 "Expect a executable directive"

);

8339 const auto

*CurExecDir = cast<const OMPExecutableDirective *>(CurDir);

8340

generateAllInfoForClauses(CurExecDir->clauses(), CombinedInfo, OMPBuilder,

8347 void

generateAllInfoForMapper(MapCombinedInfoTy &CombinedInfo,

8348

llvm::OpenMPIRBuilder &OMPBuilder)

const

{

8349

assert(isa<const OMPDeclareMapperDecl *>(CurDir) &&

8350 "Expect a declare mapper directive"

);

8351 const auto

*CurMapperDir = cast<const OMPDeclareMapperDecl *>(CurDir);

8352

generateAllInfoForClauses(CurMapperDir->clauses(), CombinedInfo,

8357 void

generateInfoForLambdaCaptures(

8358 const ValueDecl

*VD, llvm::Value *Arg, MapCombinedInfoTy &CombinedInfo,

8359

llvm::DenseMap<llvm::Value *, llvm::Value *> &LambdaPointers)

const

{

8367

llvm::DenseMap<const ValueDecl *, FieldDecl *> Captures;

8374

LambdaPointers.try_emplace(ThisLVal.

(CGF),

8376

CombinedInfo.Exprs.push_back(VD);

8377

CombinedInfo.BasePointers.push_back(ThisLVal.

(CGF));

8378

CombinedInfo.DevicePtrDecls.push_back(

nullptr

);

8379

CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);

8380

CombinedInfo.Pointers.push_back(ThisLValVal.

(CGF));

8381

CombinedInfo.Sizes.push_back(

8384

CombinedInfo.Types.push_back(

8385

OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |

8386

OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |

8387

OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |

8388

OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);

8389

CombinedInfo.Mappers.push_back(

nullptr

);

8392 if

(!LC.capturesVariable())

8394 const VarDecl

*VD = cast<VarDecl>(LC.getCapturedVar());

8397 auto

It = Captures.find(VD);

8398

assert(It != Captures.end() &&

"Found lambda capture without field."

);

8400 if

(LC.getCaptureKind() ==

LCK_ByRef

) {

8402

LambdaPointers.try_emplace(VarLVal.

(CGF),

8404

CombinedInfo.Exprs.push_back(VD);

8405

CombinedInfo.BasePointers.push_back(VarLVal.

(CGF));

8406

CombinedInfo.DevicePtrDecls.push_back(

nullptr

);

8407

CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);

8408

CombinedInfo.Pointers.push_back(VarLValVal.

(CGF));

8409

CombinedInfo.Sizes.push_back(CGF.

.CreateIntCast(

8415

LambdaPointers.try_emplace(VarLVal.

(CGF),

8417

CombinedInfo.Exprs.push_back(VD);

8418

CombinedInfo.BasePointers.push_back(VarLVal.

(CGF));

8419

CombinedInfo.DevicePtrDecls.push_back(

nullptr

);

8420

CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);

8421

CombinedInfo.Pointers.push_back(VarRVal.

getScalarVal

());

8422

CombinedInfo.Sizes.push_back(llvm::ConstantInt::get(CGF.

, 0));

8424

CombinedInfo.Types.push_back(

8425

OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |

8426

OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |

8427

OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |

8428

OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);

8429

CombinedInfo.Mappers.push_back(

nullptr

);

8434 void

adjustMemberOfForLambdaCaptures(

8435

llvm::OpenMPIRBuilder &OMPBuilder,

8436 const

llvm::DenseMap<llvm::Value *, llvm::Value *> &LambdaPointers,

8437

MapBaseValuesArrayTy &BasePointers, MapValuesArrayTy &Pointers,

8438

MapFlagsArrayTy &Types)

const

{

8439 for

(

unsigned

I = 0,

= Types.size(); I <

; ++I) {

8441 if

(Types[I] != (OpenMPOffloadMappingFlags::OMP_MAP_PTR_AND_OBJ |

8442

OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |

8443

OpenMPOffloadMappingFlags::OMP_MAP_MEMBER_OF |

8444

OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT))

8446

llvm::Value *BasePtr = LambdaPointers.lookup(BasePointers[I]);

8447

assert(BasePtr &&

"Unable to find base lambda address."

);

8449 for

(

unsigned

J = I; J > 0; --J) {

8450 unsigned

Idx = J - 1;

8451 if

(Pointers[Idx] != BasePtr)

8456

assert(TgtIdx != -1 &&

"Unable to find parent lambda."

);

8460

OpenMPOffloadMappingFlags MemberOfFlag =

8461

OMPBuilder.getMemberOfFlag(TgtIdx);

8462

OMPBuilder.setCorrectMemberOfFlag(Types[I], MemberOfFlag);

8469

llvm::Value *Arg, MapCombinedInfoTy &CombinedInfo,

8470

StructRangeInfoTy &PartialStruct)

const

{

8472 "Not expecting to generate map info for a variable array type!"

);

8481 if

(LambdasMap.count(VD))

8487 if

(VD && (DevPointersMap.count(VD) || HasDevAddrsMap.count(VD))) {

8488

CombinedInfo.Exprs.push_back(VD);

8489

CombinedInfo.BasePointers.emplace_back(Arg);

8490

CombinedInfo.DevicePtrDecls.emplace_back(VD);

8491

CombinedInfo.DevicePointers.emplace_back(DeviceInfoTy::Pointer);

8492

CombinedInfo.Pointers.push_back(Arg);

8493

CombinedInfo.Sizes.push_back(CGF.

.CreateIntCast(

8496

CombinedInfo.Types.push_back(

8497

OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |

8498

OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM);

8499

CombinedInfo.Mappers.push_back(

nullptr

);

8511 auto

It = DevPointersMap.find(VD);

8512 if

(It != DevPointersMap.end())

8513 for

(

const auto

&MCL : It->second)

8514

DeclComponentLists.emplace_back(MCL, OMPC_MAP_to,

Unknown

8517 auto

I = HasDevAddrsMap.find(VD);

8518 if

(I != HasDevAddrsMap.end())

8519 for

(

const auto

&MCL : I->second)

8520

DeclComponentLists.emplace_back(MCL, OMPC_MAP_tofrom,

Unknown

8523

assert(isa<const OMPExecutableDirective *>(CurDir) &&

8524 "Expect a executable directive"

);

8525 const auto

*CurExecDir = cast<const OMPExecutableDirective *>(CurDir);

8526 bool

HasMapBasePtr =

false

;

8527 bool

HasMapArraySec =

false

;

8528 for

(

const auto

: CurExecDir->getClausesOfKind<

OMPMapClause

>()) {

8529 const auto

*EI =

->getVarRefs().begin();

8530 for

(

const auto

L :

->decl_component_lists(VD)) {

8533 const Expr

= (

->getMapLoc().isValid()) ? *EI :

nullptr

;

8535

std::tie(VDecl, Components, Mapper) = L;

8536

assert(VDecl == VD &&

"We got information for the wrong declaration??"

);

8537

assert(!Components.empty() &&

8538 "Not expecting declaration with no component lists."

);

8540

HasMapBasePtr =

true

;

8542

(isa<ArraySectionExpr>(

) || isa<ArraySubscriptExpr>(

)))

8543

HasMapArraySec =

true

;

8544

DeclComponentLists.emplace_back(Components,

->getMapType(),

8545 C

->getMapTypeModifiers(),

8546 C

->isImplicit(), Mapper,

);

8550

llvm::stable_sort(DeclComponentLists, [](

const

MapData &LHS,

8551 const

MapData &RHS) {

8555

llvm::is_contained(MapModifiers, clang::OMPC_MAP_MODIFIER_present);

8556 bool

HasAllocs = MapType == OMPC_MAP_alloc;

8557

MapModifiers = std::get<2>(RHS);

8558

MapType = std::get<1>(LHS);

8560

llvm::is_contained(MapModifiers, clang::OMPC_MAP_MODIFIER_present);

8561 bool

HasAllocsR = MapType == OMPC_MAP_alloc;

8562 return

(HasPresent && !HasPresentR) || (HasAllocs && !HasAllocsR);

8566

llvm::SmallDenseMap<

8573 for

(

const

MapData &L : DeclComponentLists) {

8580

std::tie(Components, MapType, MapModifiers, IsImplicit, Mapper, VarRef) =

8583 for

(

const

MapData &L1 :

ArrayRef

(DeclComponentLists).slice(Count)) {

8585

std::tie(Components1, MapType, MapModifiers, IsImplicit, Mapper,

8587 auto

CI = Components.rbegin();

8588 auto

CE = Components.rend();

8589 auto

SI = Components1.rbegin();

8590 auto

SE = Components1.rend();

8591 for

(; CI != CE && SI != SE; ++CI, ++SI) {

8592 if

(CI->getAssociatedExpression()->getStmtClass() !=

8593

SI->getAssociatedExpression()->getStmtClass())

8596 if

(CI->getAssociatedDeclaration() != SI->getAssociatedDeclaration())

8601 if

(CI == CE || SI == SE) {

8603 if

(CI == CE && SI == SE)

8605 const auto

It = (SI == SE) ? CI : SI;

8611 if

(!isa<MemberExpr>(It->getAssociatedExpression()) ||

8612

(std::prev(It)->getAssociatedDeclaration() &&

8614

->getAssociatedDeclaration()

8616

->isPointerType()) ||

8617

(It->getAssociatedDeclaration() &&

8618

It->getAssociatedDeclaration()->getType()->isPointerType() &&

8619

std::next(It) != CE && std::next(It) != SE))

8621 const

MapData &BaseData = CI == CE ? L : L1;

8623

SI == SE ? Components : Components1;

8624

OverlappedData[&BaseData].push_back(SubData);

8630 if

(!OverlappedData.empty()) {

8633 while

(BaseType != OrigType) {

8639

getPlainLayout(CRD, Layout,

false

);

8645 for

(

auto

&Pair : OverlappedData) {

8652

auto CI = First.rbegin();

8653

auto CE = First.rend();

8654

auto SI = Second.rbegin();

8655

auto SE = Second.rend();

8656

for (; CI != CE && SI != SE; ++CI, ++SI) {

8657

if (CI->getAssociatedExpression()->getStmtClass() !=

8658

SI->getAssociatedExpression()->getStmtClass())

8661

if (CI->getAssociatedDeclaration() !=

8662

SI->getAssociatedDeclaration())

8667 if

(CI == CE && SI == SE)

8671 if

(CI == CE || SI == SE)

8674 const auto

*FD1 = cast<FieldDecl>(CI->getAssociatedDeclaration());

8675 const auto

*FD2 = cast<FieldDecl>(SI->getAssociatedDeclaration());

8676 if

(FD1->getParent() == FD2->getParent())

8677 return

FD1->getFieldIndex() < FD2->getFieldIndex();

8679

llvm::find_if(Layout, [FD1, FD2](

const FieldDecl

*FD) {

8680 return

FD == FD1 || FD == FD2;

8688 bool

IsFirstComponentList =

true

;

8689

MapCombinedInfoTy StructBaseCombinedInfo;

8690 for

(

const auto

&Pair : OverlappedData) {

8691 const

MapData &L = *Pair.getFirst();

8698

std::tie(Components, MapType, MapModifiers, IsImplicit, Mapper, VarRef) =

8701

OverlappedComponents = Pair.getSecond();

8702

generateInfoForComponentList(

8703

MapType, MapModifiers, {}, Components, CombinedInfo,

8704

StructBaseCombinedInfo, PartialStruct, IsFirstComponentList,

8705

IsImplicit,

false

, Mapper,

8706 false

, VD, VarRef, OverlappedComponents);

8707

IsFirstComponentList =

false

;

8710 for

(

const

MapData &L : DeclComponentLists) {

8717

std::tie(Components, MapType, MapModifiers, IsImplicit, Mapper, VarRef) =

8719 auto

It = OverlappedData.find(&L);

8720 if

(It == OverlappedData.end())

8721

generateInfoForComponentList(

8722

MapType, MapModifiers, {}, Components, CombinedInfo,

8723

StructBaseCombinedInfo, PartialStruct, IsFirstComponentList,

8724

IsImplicit,

false

, Mapper,

8726

{}, HasMapBasePtr && HasMapArraySec);

8727

IsFirstComponentList =

false

;

8735

MapCombinedInfoTy &CombinedInfo)

const

{

8736 bool

IsImplicit =

true

;

8739

CombinedInfo.Exprs.push_back(

nullptr

);

8740

CombinedInfo.BasePointers.push_back(CV);

8741

CombinedInfo.DevicePtrDecls.push_back(

nullptr

);

8742

CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);

8743

CombinedInfo.Pointers.push_back(CV);

8745

CombinedInfo.Sizes.push_back(

8749

CombinedInfo.Types.push_back(OpenMPOffloadMappingFlags::OMP_MAP_TO |

8750

OpenMPOffloadMappingFlags::OMP_MAP_FROM);

8754

CombinedInfo.BasePointers.push_back(CV);

8755

CombinedInfo.DevicePtrDecls.push_back(

nullptr

);

8756

CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);

8757

CombinedInfo.Pointers.push_back(CV);

8761

CombinedInfo.Types.push_back(

8762

OpenMPOffloadMappingFlags::OMP_MAP_LITERAL);

8763

CombinedInfo.Sizes.push_back(CGF.

.CreateIntCast(

8768

CombinedInfo.Types.push_back(OpenMPOffloadMappingFlags::OMP_MAP_NONE);

8769

CombinedInfo.Sizes.push_back(llvm::Constant::getNullValue(CGF.

));

8771 auto

I = FirstPrivateDecls.find(VD);

8772 if

(I != FirstPrivateDecls.end())

8773

IsImplicit = I->getSecond();

8778

CombinedInfo.Sizes.push_back(CGF.

.CreateIntCast(

8783

CombinedInfo.Types.push_back(getMapModifiersForPrivateClauses(CI));

8785 auto

I = FirstPrivateDecls.find(VD);

8787

CombinedInfo.BasePointers.push_back(CV);

8788

CombinedInfo.DevicePtrDecls.push_back(

nullptr

);

8789

CombinedInfo.DevicePointers.push_back(DeviceInfoTy::None);

8796

CombinedInfo.Pointers.push_back(CV);

8798 if

(I != FirstPrivateDecls.end())

8799

IsImplicit = I->getSecond();

8802

CombinedInfo.Types.back() |=

8803

OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM;

8807

CombinedInfo.Types.back() |= OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT;

8810

CombinedInfo.Mappers.push_back(

nullptr

);

8822

dyn_cast<MemberExpr>(OASE->getBase()->IgnoreParenImpCasts()))

8823 return

ME->getMemberDecl();

8829static

llvm::Constant *

8831

MappableExprsHandler::MappingExprInfo &MapExprs) {

8833

uint32_t SrcLocStrSize;

8834 if

(!MapExprs.getMapDecl() && !MapExprs.getMapExpr())

8835 return

OMPBuilder.getOrCreateDefaultSrcLocStr(SrcLocStrSize);

8838 if

(!MapExprs.getMapDecl() && MapExprs.getMapExpr()) {

8842 Loc

= MapExprs.getMapExpr()->getExprLoc();

8844 Loc

= MapExprs.getMapDecl()->getLocation();

8847

std::string ExprName;

8848 if

(MapExprs.getMapExpr()) {

8850

llvm::raw_string_ostream OS(ExprName);

8851

MapExprs.getMapExpr()->printPretty(OS,

nullptr

);

8853

ExprName = MapExprs.getMapDecl()->getNameAsString();

8857 return

OMPBuilder.getOrCreateSrcLocStr(PLoc.

getFilename

(), ExprName,

8865 CodeGenFunction

&CGF, MappableExprsHandler::MapCombinedInfoTy &CombinedInfo,

8867 bool

IsNonContiguous =

false

bool

ForEndCall =

false

) {

8870 using

InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;

8873

InsertPointTy CodeGenIP(CGF.

.GetInsertBlock(),

8874

CGF.

.GetInsertPoint());

8876 auto

DeviceAddrCB = [&](

unsigned int

I, llvm::Value *NewDecl) {

8877 if

(

const ValueDecl

*DevVD = CombinedInfo.DevicePtrDecls[I]) {

8882 auto

CustomMapperCB = [&](

unsigned int

I) {

8883

llvm::Value *MFunc =

nullptr

;

8884 if

(CombinedInfo.Mappers[I]) {

8885

Info.HasMapper =

true

;

8887

cast<OMPDeclareMapperDecl>(CombinedInfo.Mappers[I]));

8891

OMPBuilder.emitOffloadingArraysAndArgs(

8892

AllocaIP, CodeGenIP, Info, Info.RTArgs, CombinedInfo, IsNonContiguous,

8893

ForEndCall, DeviceAddrCB, CustomMapperCB);

8899 const auto

*CS =

.getInnermostCapturedStmt();

8902 const Stmt

*ChildStmt =

8905 if

(

const auto

*NestedDir =

8906

dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {

8908 switch

(

.getDirectiveKind()) {

8914 if

(DKind == OMPD_teams) {

8915

Body = NestedDir->getInnermostCapturedStmt()->IgnoreContainers(

8920 if

(

const auto

*NND =

8921

dyn_cast_or_null<OMPExecutableDirective>(ChildStmt)) {

8922

DKind = NND->getDirectiveKind();

8928 case

OMPD_target_teams:

8932 case

OMPD_target_parallel:

8933 case

OMPD_target_simd:

8934 case

OMPD_target_parallel_for:

8935 case

OMPD_target_parallel_for_simd:

8937 case

OMPD_target_teams_distribute:

8938 case

OMPD_target_teams_distribute_simd:

8939 case

OMPD_target_teams_distribute_parallel_for:

8940 case

OMPD_target_teams_distribute_parallel_for_simd:

8943 case

OMPD_parallel_for:

8944 case

OMPD_parallel_master:

8945 case

OMPD_parallel_sections:

8947 case

OMPD_parallel_for_simd:

8949 case

OMPD_cancellation_point:

8951 case

OMPD_threadprivate:

8962 case

OMPD_taskyield:

8965 case

OMPD_taskgroup:

8971 case

OMPD_target_data:

8972 case

OMPD_target_exit_data:

8973 case

OMPD_target_enter_data:

8974 case

OMPD_distribute:

8975 case

OMPD_distribute_simd:

8976 case

OMPD_distribute_parallel_for:

8977 case

OMPD_distribute_parallel_for_simd:

8978 case

OMPD_teams_distribute:

8979 case

OMPD_teams_distribute_simd:

8980 case

OMPD_teams_distribute_parallel_for:

8981 case

OMPD_teams_distribute_parallel_for_simd:

8982 case

OMPD_target_update:

8983 case

OMPD_declare_simd:

8984 case

OMPD_declare_variant:

8985 case

OMPD_begin_declare_variant:

8986 case

OMPD_end_declare_variant:

8987 case

OMPD_declare_target:

8988 case

OMPD_end_declare_target:

8989 case

OMPD_declare_reduction:

8990 case

OMPD_declare_mapper:

8992 case

OMPD_taskloop_simd:

8993 case

OMPD_master_taskloop:

8994 case

OMPD_master_taskloop_simd:

8995 case

OMPD_parallel_master_taskloop:

8996 case

OMPD_parallel_master_taskloop_simd:

8998 case

OMPD_metadirective:

9001

llvm_unreachable(

"Unexpected directive."

);

9046 auto

*MapperVarDecl =

9047

cast<VarDecl>(cast<DeclRefExpr>(

->getMapperVarRef())->getDecl());

9048 CharUnits

ElementSize =

.getTypeSizeInChars(Ty);

9052

MappableExprsHandler::MapCombinedInfoTy CombinedInfo;

9053 auto

PrivatizeAndGenMapInfoCB =

9054

[&](llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP, llvm::Value *PtrPHI,

9055

llvm::Value *BeginArg) -> llvm::OpenMPIRBuilder::MapInfosTy & {

9056

MapperCGF.

.restoreIP(CodeGenIP);

9066 Scope

.addPrivate(MapperVarDecl, PtrCurrent);

9067

(void)

.Privatize();

9070

MappableExprsHandler MEHandler(*

, MapperCGF);

9071

MEHandler.generateAllInfoForMapper(CombinedInfo,

);

9073 auto

FillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) {

9077

llvm::codegenoptions::NoDebugInfo) {

9078

CombinedInfo.Names.resize(CombinedInfo.Exprs.size());

9079

llvm::transform(CombinedInfo.Exprs, CombinedInfo.Names.begin(),

9083 return

CombinedInfo;

9086 auto

CustomMapperCB = [&](

unsigned

I, llvm::Function **MapperFunc) {

9087 if

(CombinedInfo.Mappers[I]) {

9090

cast<OMPDeclareMapperDecl>(CombinedInfo.Mappers[I]));

9091

assert(*MapperFunc &&

"Expect a valid mapper function is available."

);

9098

llvm::raw_svector_ostream Out(TyStr);

9100

std::string Name =

({

"omp_mapper"

, TyStr,

->getName()});

9102 auto

*NewFn =

.emitUserDefinedMapper(PrivatizeAndGenMapInfoCB,

9103

ElemTy, Name, CustomMapperCB);

9104 UDMMap

.try_emplace(

, NewFn);

9128

Kind != OMPD_target_teams_loop)

9131 return

llvm::ConstantInt::get(CGF.

, 0);

9133 const auto

*LD = cast<OMPLoopDirective>(TD);

9134 if

(llvm::Value *NumIterations = SizeEmitter(CGF, *LD))

9135 return

NumIterations;

9136 return

llvm::ConstantInt::get(CGF.

, 0);

9145 if

(OffloadingMandatory) {

9146

CGF.

.CreateUnreachable();

9148 if

(RequiresOuterTask) {

9149

CapturedVars.clear();

9158

llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier>

9161

llvm::Value *DeviceID;

9162 if

(

.getPointer()) {

9164 Device

.getInt() == OMPC_DEVICE_device_num) &&

9165 "Expected device_num modifier."

);

9170

DeviceID = CGF.

.getInt64(OMP_DEVICEID_UNDEF);

9177

llvm::Value *DynCGroupMem = CGF.

.getInt32(0);

9180

CodeGenFunction::RunCleanupsScope DynCGroupMemScope(CGF);

9182

DynMemClause->getSize(),

true

);

9183

DynCGroupMem = CGF.

.CreateIntCast(DynCGroupMemVal, CGF.

9186 return

DynCGroupMem;

9191

llvm::OpenMPIRBuilder &OMPBuilder,

9193

MappableExprsHandler::MapCombinedInfoTy &CombinedInfo) {

9195

llvm::DenseMap<llvm::Value *, llvm::Value *> LambdaPointers;

9197 auto

*CV = CapturedVars.begin();

9200

CI != CE; ++CI, ++RI, ++CV) {

9201

MappableExprsHandler::MapCombinedInfoTy CurInfo;

9202

MappableExprsHandler::StructRangeInfoTy PartialStruct;

9207

CurInfo.Exprs.push_back(

nullptr

);

9208

CurInfo.BasePointers.push_back(*CV);

9209

CurInfo.DevicePtrDecls.push_back(

nullptr

);

9210

CurInfo.DevicePointers.push_back(

9211

MappableExprsHandler::DeviceInfoTy::None);

9212

CurInfo.Pointers.push_back(*CV);

9213

CurInfo.Sizes.push_back(CGF.

.CreateIntCast(

9216

CurInfo.Types.push_back(OpenMPOffloadMappingFlags::OMP_MAP_LITERAL |

9217

OpenMPOffloadMappingFlags::OMP_MAP_TARGET_PARAM |

9218

OpenMPOffloadMappingFlags::OMP_MAP_IMPLICIT);

9219

CurInfo.Mappers.push_back(

nullptr

);

9223

MEHandler.generateInfoForCapture(CI, *CV, CurInfo, PartialStruct);

9227

MappedVarSet.insert(

nullptr

);

9228 if

(CurInfo.BasePointers.empty() && !PartialStruct.Base.isValid())

9229

MEHandler.generateDefaultMapInfo(*CI, **RI, *CV, CurInfo);

9233

MEHandler.generateInfoForLambdaCaptures(CI->

getCapturedVar

(), *CV,

9234

CurInfo, LambdaPointers);

9237

assert((!CurInfo.BasePointers.empty() || PartialStruct.Base.isValid()) &&

9238 "Non-existing map pointer for capture!"

);

9239

assert(CurInfo.BasePointers.size() == CurInfo.Pointers.size() &&

9240

CurInfo.BasePointers.size() == CurInfo.Sizes.size() &&

9241

CurInfo.BasePointers.size() == CurInfo.Types.size() &&

9242

CurInfo.BasePointers.size() == CurInfo.Mappers.size() &&

9243 "Inconsistent map information sizes!"

);

9247 if

(PartialStruct.Base.isValid()) {

9248

CombinedInfo.append(PartialStruct.PreliminaryMapData);

9249

MEHandler.emitCombinedEntry(CombinedInfo, CurInfo.Types, PartialStruct,

9255

CombinedInfo.append(CurInfo);

9258

MEHandler.adjustMemberOfForLambdaCaptures(

9259

OMPBuilder, LambdaPointers, CombinedInfo.BasePointers,

9260

CombinedInfo.Pointers, CombinedInfo.Types);

9264

MappableExprsHandler::MapCombinedInfoTy &CombinedInfo,

9265

llvm::OpenMPIRBuilder &OMPBuilder,

9272

MEHandler.generateAllInfo(CombinedInfo, OMPBuilder, SkippedVarSet);

9274 auto

FillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) {

9278

llvm::codegenoptions::NoDebugInfo) {

9279

CombinedInfo.Names.resize(CombinedInfo.Exprs.size());

9280

llvm::transform(CombinedInfo.Exprs, CombinedInfo.Names.begin(),

9288

llvm::OpenMPIRBuilder &OMPBuilder,

9289

MappableExprsHandler::MapCombinedInfoTy &CombinedInfo) {

9291

MappableExprsHandler MEHandler(

, CGF);

9292

llvm::DenseSet<CanonicalDeclPtr<const Decl>> MappedVarSet;

9295

MappedVarSet, CombinedInfo);

9296 genMapInfo

(MEHandler, CGF, CombinedInfo, OMPBuilder, MappedVarSet);

9299template

typename

ClauseTy>

9304 const auto

.getSingleClause<ClauseTy>();

9305

assert(!

->varlist_empty() &&

9306 "ompx_bare requires explicit num_teams and thread_limit"

);

9307

CodeGenFunction::RunCleanupsScope

(CGF);

9308 for

(

auto

->varlist()) {

9320

llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier>

9321

llvm::Value *OutlinedFnID, CodeGenFunction::OMPTargetDataInfo &InputInfo,

9322

llvm::Value *&MapTypesArray, llvm::Value *&MapNamesArray,

9327

llvm::OpenMPIRBuilder &OMPBuilder = OMPRuntime->

getOMPBuilder

();

9330

MappableExprsHandler::MapCombinedInfoTy CombinedInfo;

9332 genMapInfo

(

, CGF, CS, CapturedVars, OMPBuilder, CombinedInfo);

9337

InputInfo.NumberOfTargetItems = Info.NumberOfPtrs;

9338

InputInfo.BasePointersArray =

(Info.RTArgs.BasePointersArray,

9340

InputInfo.PointersArray =

9342

InputInfo.SizesArray =

9344

InputInfo.MappersArray =

9346

MapTypesArray = Info.RTArgs.MapTypesArray;

9347

MapNamesArray = Info.RTArgs.MapNamesArray;

9349 auto

&&ThenGen = [&OMPRuntime, OutlinedFn, &

, &CapturedVars,

9350

RequiresOuterTask, &CS, OffloadingMandatory,

9351

OutlinedFnID, &InputInfo, &MapTypesArray, &MapNamesArray,

9353 bool

IsReverseOffloading =

.getInt() == OMPC_DEVICE_ancestor;

9355 if

(IsReverseOffloading) {

9361

RequiresOuterTask, CS, OffloadingMandatory, CGF);

9366 unsigned

NumTargetItems = InputInfo.NumberOfTargetItems;

9368

llvm::Value *BasePointersArray =

9369

InputInfo.BasePointersArray.emitRawPointer(CGF);

9370

llvm::Value *PointersArray = InputInfo.PointersArray.emitRawPointer(CGF);

9371

llvm::Value *SizesArray = InputInfo.SizesArray.emitRawPointer(CGF);

9372

llvm::Value *MappersArray = InputInfo.MappersArray.emitRawPointer(CGF);

9374 auto

&&EmitTargetCallFallbackCB =

9375

[&OMPRuntime, OutlinedFn, &

, &CapturedVars, RequiresOuterTask, &CS,

9376

OffloadingMandatory, &CGF](llvm::OpenMPIRBuilder::InsertPointTy IP)

9377

-> llvm::OpenMPIRBuilder::InsertPointTy {

9380

RequiresOuterTask, CS, OffloadingMandatory, CGF);

9381 return

CGF.

.saveIP();

9388

emitClauseForBareTargetDirective<OMPNumTeamsClause>(CGF,

, NumTeams);

9389

emitClauseForBareTargetDirective<OMPThreadLimitClause>(CGF,

9393

NumThreads.push_back(

9399

llvm::Value *NumIterations =

9402

llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(

9405

llvm::OpenMPIRBuilder::TargetDataRTArgs RTArgs(

9406

BasePointersArray, PointersArray, SizesArray, MapTypesArray,

9407 nullptr

, MappersArray, MapNamesArray);

9409

llvm::OpenMPIRBuilder::TargetKernelArgs Args(

9410

NumTargetItems, RTArgs, NumIterations, NumTeams, NumThreads,

9411

DynCGGroupMem, HasNoWait);

9413

llvm::OpenMPIRBuilder::InsertPointTy AfterIP =

9415

CGF.

, OutlinedFnID, EmitTargetCallFallbackCB, Args, DeviceID,

9417

CGF.

.restoreIP(AfterIP);

9420 if

(RequiresOuterTask)

9435

[&OMPRuntime, OutlinedFn, &

, &CapturedVars, RequiresOuterTask, &CS,

9438

RequiresOuterTask, CS, OffloadingMandatory, CGF);

9441 if

(RequiresOuterTask) {

9442

CodeGenFunction::OMPTargetDataInfo InputInfo;

9451

llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID,

const Expr

*IfCond,

9452

llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier>

9459 const bool

OffloadingMandatory = !

().OpenMPIsTargetDevice &&

9462

assert((OffloadingMandatory || OutlinedFn) &&

"Invalid outlined function!"

);

9464 const bool

RequiresOuterTask =

9480

llvm::Value *MapTypesArray =

nullptr

;

9481

llvm::Value *MapNamesArray =

nullptr

;

9483 auto

&&TargetThenGen = [

this

, OutlinedFn, &

, &CapturedVars,

9484

RequiresOuterTask, &CS, OffloadingMandatory,

9485

OutlinedFnID, &InputInfo, &MapTypesArray,

9489

RequiresOuterTask, CS, OffloadingMandatory,

9490 Device

, OutlinedFnID, InputInfo, MapTypesArray,

9491

MapNamesArray, SizeEmitter, CGF,

);

9494 auto

&&TargetElseGen =

9495

[

this

, OutlinedFn, &

, &CapturedVars, RequiresOuterTask, &CS,

9498

CS, OffloadingMandatory, CGF);

9507 emitIfClause

(CGF, IfCond, TargetThenGen, TargetElseGen);

9519

StringRef ParentName) {

9524 bool

RequiresDeviceCodegen =

9525

isa<OMPExecutableDirective>(S) &&

9527

cast<OMPExecutableDirective>(S)->getDirectiveKind());

9529 if

(RequiresDeviceCodegen) {

9530 const auto

= *cast<OMPExecutableDirective>(S);

9537 if

.OffloadInfoManager.hasTargetRegionEntryInfo(EntryInfo))

9540 switch

(

.getDirectiveKind()) {

9543

cast<OMPTargetDirective>(

));

9545 case

OMPD_target_parallel:

9547 CGM

, ParentName, cast<OMPTargetParallelDirective>(

));

9549 case

OMPD_target_teams:

9551 CGM

, ParentName, cast<OMPTargetTeamsDirective>(

));

9553 case

OMPD_target_teams_distribute:

9555 CGM

, ParentName, cast<OMPTargetTeamsDistributeDirective>(

));

9557 case

OMPD_target_teams_distribute_simd:

9559 CGM

, ParentName, cast<OMPTargetTeamsDistributeSimdDirective>(

));

9561 case

OMPD_target_parallel_for:

9563 CGM

, ParentName, cast<OMPTargetParallelForDirective>(

));

9565 case

OMPD_target_parallel_for_simd:

9567 CGM

, ParentName, cast<OMPTargetParallelForSimdDirective>(

));

9569 case

OMPD_target_simd:

9571 CGM

, ParentName, cast<OMPTargetSimdDirective>(

));

9573 case

OMPD_target_teams_distribute_parallel_for:

9576

cast<OMPTargetTeamsDistributeParallelForDirective>(

));

9578 case

OMPD_target_teams_distribute_parallel_for_simd:

9582

cast<OMPTargetTeamsDistributeParallelForSimdDirective>(

));

9584 case

OMPD_target_teams_loop:

9586 CGM

, ParentName, cast<OMPTargetTeamsGenericLoopDirective>(

));

9588 case

OMPD_target_parallel_loop:

9590 CGM

, ParentName, cast<OMPTargetParallelGenericLoopDirective>(

));

9594 case

OMPD_parallel_for:

9595 case

OMPD_parallel_master:

9596 case

OMPD_parallel_sections:

9598 case

OMPD_parallel_for_simd:

9600 case

OMPD_cancellation_point:

9602 case

OMPD_threadprivate:

9613 case

OMPD_taskyield:

9616 case

OMPD_taskgroup:

9622 case

OMPD_target_data:

9623 case

OMPD_target_exit_data:

9624 case

OMPD_target_enter_data:

9625 case

OMPD_distribute:

9626 case

OMPD_distribute_simd:

9627 case

OMPD_distribute_parallel_for:

9628 case

OMPD_distribute_parallel_for_simd:

9629 case

OMPD_teams_distribute:

9630 case

OMPD_teams_distribute_simd:

9631 case

OMPD_teams_distribute_parallel_for:

9632 case

OMPD_teams_distribute_parallel_for_simd:

9633 case

OMPD_target_update:

9634 case

OMPD_declare_simd:

9635 case

OMPD_declare_variant:

9636 case

OMPD_begin_declare_variant:

9637 case

OMPD_end_declare_variant:

9638 case

OMPD_declare_target:

9639 case

OMPD_end_declare_target:

9640 case

OMPD_declare_reduction:

9641 case

OMPD_declare_mapper:

9643 case

OMPD_taskloop_simd:

9644 case

OMPD_master_taskloop:

9645 case

OMPD_master_taskloop_simd:

9646 case

OMPD_parallel_master_taskloop:

9647 case

OMPD_parallel_master_taskloop_simd:

9649 case

OMPD_metadirective:

9652

llvm_unreachable(

"Unknown target directive for OpenMP device codegen."

);

9657 if

(

const auto

= dyn_cast<OMPExecutableDirective>(S)) {

9658 if

->hasAssociatedStmt() || !

->getAssociatedStmt())

9666 if

(

const auto

*L = dyn_cast<LambdaExpr>(S))

9670 for

(

const Stmt

*II : S->children())

9675

std::optional<OMPDeclareTargetDeclAttr::DevTypeTy> DevTy =

9676

OMPDeclareTargetDeclAttr::getDeviceType(VD);

9680 if

(!IsDevice && DevTy == OMPDeclareTargetDeclAttr::DT_NoHost)

9683 if

(IsDevice && DevTy == OMPDeclareTargetDeclAttr::DT_Host)

9692 if

(

const auto

*FD = dyn_cast<FunctionDecl>(GD.

()))

9701 if

(

const auto

*FD = dyn_cast<FunctionDecl>(VD)) {

9710 return

!OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD) &&

9728

StringRef ParentName =

9733

StringRef ParentName =

9740

std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =

9741

OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(

9742

cast<VarDecl>(GD.

()));

9743 if

(!Res || *Res == OMPDeclareTargetDeclAttr::MT_Link ||

9744

((*Res == OMPDeclareTargetDeclAttr::MT_To ||

9745

*Res == OMPDeclareTargetDeclAttr::MT_Enter) &&

9754

llvm::Constant *Addr) {

9759

std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =

9760

OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);

9764 if

(Res && *Res != OMPDeclareTargetDeclAttr::MT_Link &&

9779 auto

LinkageForVariable = [&VD,

this

]() {

9783

std::vector<llvm::GlobalVariable *> GeneratedRefs;

9796 for

(

auto

*ref : GeneratedRefs)

9801 if

(isa<FunctionDecl>(GD.

()) ||

9802

isa<OMPDeclareReductionDecl>(GD.

()))

9810

std::optional<OMPDeclareTargetDeclAttr::MapTypeTy> Res =

9811

OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(VD);

9814 if

((*Res == OMPDeclareTargetDeclAttr::MT_To ||

9815

*Res == OMPDeclareTargetDeclAttr::MT_Enter) &&

9819

assert((*Res == OMPDeclareTargetDeclAttr::MT_Link ||

9820

((*Res == OMPDeclareTargetDeclAttr::MT_To ||

9821

*Res == OMPDeclareTargetDeclAttr::MT_Enter) &&

9823 "Expected link clause or to clause with unified memory."

);

9832 " Expected target-based directive."

);

9836 for

(

const OMPClause

*Clause :

->clauselists()) {

9837 if

(Clause->getClauseKind() == OMPC_unified_shared_memory) {

9839 OMPBuilder

.Config.setHasRequiresUnifiedSharedMemory(

true

);

9840

}

else if

(

const auto

*AC =

9841

dyn_cast<OMPAtomicDefaultMemOrderClause>(Clause)) {

9842 switch

(AC->getAtomicDefaultMemOrderKind()) {

9843 case

OMPC_ATOMIC_DEFAULT_MEM_ORDER_acq_rel:

9846 case

OMPC_ATOMIC_DEFAULT_MEM_ORDER_seq_cst:

9849 case

OMPC_ATOMIC_DEFAULT_MEM_ORDER_relaxed:

9865 if

(!VD || !VD->

hasAttr

<OMPAllocateDeclAttr>())

9867 const auto

*A = VD->

getAttr

<OMPAllocateDeclAttr>();

9868 switch

(A->getAllocatorType()) {

9869 case

OMPAllocateDeclAttr::OMPNullMemAlloc:

9870 case

OMPAllocateDeclAttr::OMPDefaultMemAlloc:

9872 case

OMPAllocateDeclAttr::OMPLargeCapMemAlloc:

9873 case

OMPAllocateDeclAttr::OMPCGroupMemAlloc:

9874 case

OMPAllocateDeclAttr::OMPHighBWMemAlloc:

9875 case

OMPAllocateDeclAttr::OMPLowLatMemAlloc:

9876 case

OMPAllocateDeclAttr::OMPThreadMemAlloc:

9877 case

OMPAllocateDeclAttr::OMPConstMemAlloc:

9878 case

OMPAllocateDeclAttr::OMPPTeamMemAlloc:

9881 case

OMPAllocateDeclAttr::OMPUserDefinedMemAlloc:

9882

llvm_unreachable(

"Expected predefined allocator for the variables with the " 9883 "static storage."

);

9910 const auto

= cast<FunctionDecl>(GD.

());

9913 if

(OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(

)) {

9915 if

(

auto

*F = dyn_cast_or_null<llvm::Function>(

9917 return

!F->isDeclaration();

9929

llvm::Function *OutlinedFn,

9938

llvm::Value *Args[] = {

9940

CGF.

.getInt32(CapturedVars.size()),

9943

RealArgs.append(std::begin(Args), std::end(Args));

9944

RealArgs.append(CapturedVars.begin(), CapturedVars.end());

9946

llvm::FunctionCallee RTLFn =

.getOrCreateRuntimeFunction(

9952 const Expr

*NumTeams,

9953 const Expr

*ThreadLimit,

9960

llvm::Value *NumTeamsVal =

9966

llvm::Value *ThreadLimitVal =

9973

llvm::Value *PushNumTeamsArgs[] = {RTLoc,

(CGF,

), NumTeamsVal,

9981 const Expr

*ThreadLimit,

9984

llvm::Value *ThreadLimitVal =

9991

llvm::Value *ThreadLimitArgs[] = {RTLoc,

(CGF,

10009 using

InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;

10011

llvm::Value *IfCondVal =

nullptr

;

10016

llvm::Value *DeviceID =

nullptr

;

10021

DeviceID = CGF.

.getInt64(OMP_DEVICEID_UNDEF);

10025

MappableExprsHandler::MapCombinedInfoTy CombinedInfo;

10026 auto

GenMapInfoCB =

10027

[&](InsertPointTy CodeGenIP) -> llvm::OpenMPIRBuilder::MapInfosTy & {

10028

CGF.

.restoreIP(CodeGenIP);

10030

MappableExprsHandler MEHandler(

, CGF);

10031

MEHandler.generateAllInfo(CombinedInfo,

);

10033 auto

FillInfoMap = [&](MappableExprsHandler::MappingExprInfo &MapExpr) {

10037

llvm::codegenoptions::NoDebugInfo) {

10038

CombinedInfo.Names.resize(CombinedInfo.Exprs.size());

10039

llvm::transform(CombinedInfo.Exprs, CombinedInfo.Names.begin(),

10043 return

CombinedInfo;

10045 using

BodyGenTy = llvm::OpenMPIRBuilder::BodyGenTy;

10046 auto

BodyCB = [&](InsertPointTy CodeGenIP, BodyGenTy BodyGenType) {

10047

CGF.

.restoreIP(CodeGenIP);

10048 switch

(BodyGenType) {

10049 case

BodyGenTy::Priv:

10053 case

BodyGenTy::DupNoPriv:

10059 case

BodyGenTy::NoPriv:

10066 return

InsertPointTy(CGF.

.GetInsertBlock(),

10067

CGF.

.GetInsertPoint());

10070 auto

DeviceAddrCB = [&](

unsigned int

I, llvm::Value *NewDecl) {

10071 if

(

const ValueDecl

*DevVD = CombinedInfo.DevicePtrDecls[I]) {

10076 auto

CustomMapperCB = [&](

unsigned int

I) {

10077

llvm::Value *MFunc =

nullptr

;

10078 if

(CombinedInfo.Mappers[I]) {

10079

Info.HasMapper =

true

;

10081

cast<OMPDeclareMapperDecl>(CombinedInfo.Mappers[I]));

10091

InsertPointTy CodeGenIP(CGF.

.GetInsertBlock(),

10092

CGF.

.GetInsertPoint());

10093

llvm::OpenMPIRBuilder::LocationDescription OmpLoc(CodeGenIP);

10094

llvm::OpenMPIRBuilder::InsertPointTy AfterIP =

10096

OmpLoc, AllocaIP, CodeGenIP, DeviceID, IfCondVal, Info, GenMapInfoCB,

10097 nullptr

, BodyCB, DeviceAddrCB, CustomMapperCB, RTLoc));

10098

CGF.

.restoreIP(AfterIP);

10107

assert((isa<OMPTargetEnterDataDirective>(

) ||

10108

isa<OMPTargetExitDataDirective>(

) ||

10109

isa<OMPTargetUpdateDirective>(

)) &&

10110 "Expecting either target enter, exit data, or update directives."

);

10113

llvm::Value *MapTypesArray =

nullptr

;

10114

llvm::Value *MapNamesArray =

nullptr

;

10116 auto

&&ThenGen = [

this

, &

, &InputInfo, &MapTypesArray,

10119

llvm::Value *DeviceID =

nullptr

;

10124

DeviceID = CGF.

.getInt64(OMP_DEVICEID_UNDEF);

10128

llvm::Constant *PointerNum =

10135

{RTLoc, DeviceID, PointerNum,

10144

RuntimeFunction RTLFn;

10145 switch

(

.getDirectiveKind()) {

10146 case

OMPD_target_enter_data:

10147

RTLFn = HasNowait ? OMPRTL___tgt_target_data_begin_nowait_mapper

10148

: OMPRTL___tgt_target_data_begin_mapper;

10150 case

OMPD_target_exit_data:

10151

RTLFn = HasNowait ? OMPRTL___tgt_target_data_end_nowait_mapper

10152

: OMPRTL___tgt_target_data_end_mapper;

10154 case

OMPD_target_update:

10155

RTLFn = HasNowait ? OMPRTL___tgt_target_data_update_nowait_mapper

10156

: OMPRTL___tgt_target_data_update_mapper;

10158 case

OMPD_parallel:

10160 case

OMPD_parallel_for:

10161 case

OMPD_parallel_master:

10162 case

OMPD_parallel_sections:

10163 case

OMPD_for_simd:

10164 case

OMPD_parallel_for_simd:

10166 case

OMPD_cancellation_point:

10168 case

OMPD_threadprivate:

10169 case

OMPD_allocate:

10174 case

OMPD_sections:

10178 case

OMPD_critical:

10179 case

OMPD_taskyield:

10181 case

OMPD_taskwait:

10182 case

OMPD_taskgroup:

10188 case

OMPD_target_data:

10189 case

OMPD_distribute:

10190 case

OMPD_distribute_simd:

10191 case

OMPD_distribute_parallel_for:

10192 case

OMPD_distribute_parallel_for_simd:

10193 case

OMPD_teams_distribute:

10194 case

OMPD_teams_distribute_simd:

10195 case

OMPD_teams_distribute_parallel_for:

10196 case

OMPD_teams_distribute_parallel_for_simd:

10197 case

OMPD_declare_simd:

10198 case

OMPD_declare_variant:

10199 case

OMPD_begin_declare_variant:

10200 case

OMPD_end_declare_variant:

10201 case

OMPD_declare_target:

10202 case

OMPD_end_declare_target:

10203 case

OMPD_declare_reduction:

10204 case

OMPD_declare_mapper:

10205 case

OMPD_taskloop:

10206 case

OMPD_taskloop_simd:

10207 case

OMPD_master_taskloop:

10208 case

OMPD_master_taskloop_simd:

10209 case

OMPD_parallel_master_taskloop:

10210 case

OMPD_parallel_master_taskloop_simd:

10212 case

OMPD_target_simd:

10213 case

OMPD_target_teams_distribute:

10214 case

OMPD_target_teams_distribute_simd:

10215 case

OMPD_target_teams_distribute_parallel_for:

10216 case

OMPD_target_teams_distribute_parallel_for_simd:

10217 case

OMPD_target_teams:

10218 case

OMPD_target_parallel:

10219 case

OMPD_target_parallel_for:

10220 case

OMPD_target_parallel_for_simd:

10221 case

OMPD_requires:

10222 case

OMPD_metadirective:

10225

llvm_unreachable(

"Unexpected standalone target data directive."

);

10229

OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.

));

10230

OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.

));

10231

OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.

));

10232

OffloadingArgs.push_back(llvm::Constant::getNullValue(CGF.

));

10239 auto

&&TargetThenGen = [

this

, &ThenGen, &

, &InputInfo, &MapTypesArray,

10243

MappableExprsHandler::MapCombinedInfoTy CombinedInfo;

10245

MappableExprsHandler MEHandler(

, CGF);

10262

MapTypesArray = Info.RTArgs.MapTypesArray;

10263

MapNamesArray = Info.RTArgs.MapNamesArray;

10264 if

(RequiresOuterTask)

10290struct

ParamAttrTy {

10292

llvm::APSInt StrideOrArg;

10293

llvm::APSInt Alignment;

10294 bool

HasVarStride =

false

;

10327 unsigned

Offset = 0;

10328 if

(

const auto

*MD = dyn_cast<CXXMethodDecl>(FD)) {

10329 if

(ParamAttrs[Offset].Kind ==

Vector

)

10330

CDT =

.getPointerType(

.getRecordType(MD->

getParent

()));

10335 if

(ParamAttrs[I + Offset].Kind ==

Vector

) {

10347 return C

.getTypeSize(CDT);

10355

llvm::raw_svector_ostream Out(Buffer);

10356 for

(

const auto

&ParamAttr : ParamAttrs) {

10357 switch

(ParamAttr.Kind) {

10377 if

(ParamAttr.HasVarStride)

10378

Out <<

"s"

<< ParamAttr.StrideOrArg;

10379 else if

(ParamAttr.Kind == Linear || ParamAttr.Kind == LinearRef ||

10380

ParamAttr.Kind == LinearUVal || ParamAttr.Kind == LinearVal) {

10383 if

(ParamAttr.StrideOrArg < 0)

10384

Out <<

'n'

<< -ParamAttr.StrideOrArg;

10385 else if

(ParamAttr.StrideOrArg != 1)

10386

Out << ParamAttr.StrideOrArg;

10389 if

(!!ParamAttr.Alignment)

10390

Out <<

'a'

<< ParamAttr.Alignment;

10393 return

std::string(Out.str());

10398 const

llvm::APSInt &VLENVal,

10400

OMPDeclareSimdDeclAttr::BranchStateTy State) {

10403 unsigned

VecRegSize;

10405

ISADataTy ISAData[] = {

10421 case

OMPDeclareSimdDeclAttr::BS_Undefined:

10422

Masked.push_back(

'N'

);

10423

Masked.push_back(

'M'

);

10425 case

OMPDeclareSimdDeclAttr::BS_Notinbranch:

10426

Masked.push_back(

'N'

);

10428 case

OMPDeclareSimdDeclAttr::BS_Inbranch:

10429

Masked.push_back(

'M'

);

10432 for

(

char

Mask : Masked) {

10433 for

(

const

ISADataTy &

: ISAData) {

10435

llvm::raw_svector_ostream Out(Buffer);

10436

Out <<

"_ZGV"

.ISA << Mask;

10439

assert(NumElts &&

"Non-zero simdlen/cdtsize expected"

);

10440

Out << llvm::APSInt::getUnsigned(

.VecRegSize / NumElts);

10445

Out <<

'_'

<< Fn->getName();

10446

Fn->addFnAttr(Out.str());

10464 if

(Kind == ParamKindTy::Uniform)

10467 if

(Kind == ParamKindTy::LinearUVal || Kind == ParamKindTy::LinearRef)

10470 if

((Kind == ParamKindTy::Linear || Kind == ParamKindTy::LinearVal) &&

10480 unsigned

Size =

.getTypeSize(QT);

10483 if

(Size != 8 && Size != 16 && Size != 32 && Size != 64 && Size != 128)

10507 return C

.getTypeSize(PTy);

10510 return C

.getTypeSize(QT);

10512 return C

.getTypeSize(

.getUIntPtrType());

10518static

std::tuple<unsigned, unsigned, bool>

10524 bool

OutputBecomesInput =

false

;

10528

Sizes.push_back(

getAArch64LS

(RetType, ParamKindTy::Vector,

));

10530

OutputBecomesInput =

true

;

10534

Sizes.push_back(

getAArch64LS

(QT, ParamAttrs[I].Kind,

));

10537

assert(!Sizes.empty() &&

"Unable to determine NDS and WDS."

);

10540

assert(llvm::all_of(Sizes,

10541

[](

unsigned

Size) {

10542 return

Size == 8 || Size == 16 || Size == 32 ||

10543

Size == 64 || Size == 128;

10547 return

std::make_tuple(*std::min_element(std::begin(Sizes), std::end(Sizes)),

10548

*std::max_element(std::begin(Sizes), std::end(Sizes)),

10549

OutputBecomesInput);

10555template

typename

10557 char

ISA, StringRef ParSeq,

10558

StringRef MangledName,

bool

OutputBecomesInput,

10559

llvm::Function *Fn) {

10561

llvm::raw_svector_ostream Out(Buffer);

10562

Out << Prefix << ISA << LMask << VLEN;

10563 if

(OutputBecomesInput)

10565

Out << ParSeq <<

"_"

<< MangledName;

10566

Fn->addFnAttr(Out.str());

10572

StringRef Prefix,

char

ISA,

10573

StringRef ParSeq, StringRef MangledName,

10574 bool

OutputBecomesInput,

10575

llvm::Function *Fn) {

10579

OutputBecomesInput, Fn);

10581

OutputBecomesInput, Fn);

10585

OutputBecomesInput, Fn);

10587

OutputBecomesInput, Fn);

10591

OutputBecomesInput, Fn);

10593

OutputBecomesInput, Fn);

10598

OutputBecomesInput, Fn);

10601

llvm_unreachable(

"Scalar type is too wide."

);

10609

OMPDeclareSimdDeclAttr::BranchStateTy State, StringRef MangledName,

10610 char

ISA,

unsigned

VecRegSize, llvm::Function *Fn,

SourceLocation

SLoc) {

10614 const unsigned

NDS = std::get<0>(

);

10615 const unsigned

WDS = std::get<1>(

);

10616 const bool

OutputBecomesInput = std::get<2>(

);

10620 if

(UserVLEN == 1) {

10623 "The clause simdlen(1) has no effect when targeting aarch64."

);

10630 if

(ISA ==

'n'

&& UserVLEN && !llvm::isPowerOf2_32(UserVLEN)) {

10633 "power of 2 when targeting Advanced SIMD."

);

10640 if

(ISA ==

's'

&& UserVLEN != 0) {

10641 if

((UserVLEN * WDS > 2048) || (UserVLEN * WDS % 128 != 0)) {

10644 "lanes in the architectural constraints " 10645 "for SVE (min is 128-bit, max is " 10646 "2048-bit, by steps of 128-bit)"

);

10654

StringRef Prefix =

"_ZGV"

;

10660

OutputBecomesInput, Fn);

10662

assert(ISA ==

'n'

"Expected ISA either 's' or 'n'."

);

10666 case

OMPDeclareSimdDeclAttr::BS_Undefined:

10668

OutputBecomesInput, Fn);

10670

OutputBecomesInput, Fn);

10672 case

OMPDeclareSimdDeclAttr::BS_Notinbranch:

10674

OutputBecomesInput, Fn);

10676 case

OMPDeclareSimdDeclAttr::BS_Inbranch:

10678

OutputBecomesInput, Fn);

10688

OutputBecomesInput, Fn);

10690

assert(ISA ==

'n'

"Expected ISA either 's' or 'n'."

);

10695 case

OMPDeclareSimdDeclAttr::BS_Undefined:

10697

OutputBecomesInput, Fn);

10699

OutputBecomesInput, Fn);

10701 case

OMPDeclareSimdDeclAttr::BS_Notinbranch:

10703

OutputBecomesInput, Fn);

10705 case

OMPDeclareSimdDeclAttr::BS_Inbranch:

10707

OutputBecomesInput, Fn);

10715

llvm::Function *Fn) {

10720

llvm::DenseMap<const Decl *, unsigned> ParamPositions;

10721 if

(isa<CXXMethodDecl>(FD))

10722

ParamPositions.try_emplace(FD, 0);

10723 unsigned

ParamPos = ParamPositions.size();

10725

ParamPositions.try_emplace(

->getCanonicalDecl(), ParamPos);

10731 for

(

const Expr

->uniforms()) {

10734 if

(isa<CXXThisExpr>(

)) {

10735

Pos = ParamPositions[FD];

10737 const auto

*PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(

)->getDecl())

10738

->getCanonicalDecl();

10739 auto

It = ParamPositions.find(PVD);

10740

assert(It != ParamPositions.end() &&

"Function parameter not found"

);

10743

ParamAttrs[Pos].Kind = Uniform;

10746 auto

*NI =

->alignments_begin();

10747 for

(

const Expr

->aligneds()) {

10751 if

(isa<CXXThisExpr>(

)) {

10752

Pos = ParamPositions[FD];

10755 const auto

*PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(

)->getDecl())

10756

->getCanonicalDecl();

10757 auto

It = ParamPositions.find(PVD);

10758

assert(It != ParamPositions.end() &&

"Function parameter not found"

);

10760

ParmTy = PVD->getType();

10762

ParamAttrs[Pos].Alignment =

10764

? (*NI)->EvaluateKnownConstInt(

)

10765

: llvm::APSInt::getUnsigned(

10766 C

.toCharUnitsFromBits(

.getOpenMPDefaultSimdAlign(ParmTy))

10771 auto

*SI =

->steps_begin();

10772 auto

*MI =

->modifiers_begin();

10773 for

(

const Expr

->linears()) {

10776 bool

IsReferenceType =

false

;

10779 unsigned

PtrRescalingFactor = 1;

10780 if

(isa<CXXThisExpr>(

)) {

10781

Pos = ParamPositions[FD];

10782 auto

= cast<PointerType>(

());

10787 const auto

*PVD = cast<ParmVarDecl>(cast<DeclRefExpr>(

)->getDecl())

10788

->getCanonicalDecl();

10789 auto

It = ParamPositions.find(PVD);

10790

assert(It != ParamPositions.end() &&

"Function parameter not found"

);

10792 if

(

auto

= dyn_cast<PointerType>(PVD->getType()))

10796 else if

(PVD->getType()->isReferenceType()) {

10797

IsReferenceType =

true

;

10798

PtrRescalingFactor =

10804

ParamAttrTy &ParamAttr = ParamAttrs[Pos];

10805 if

(*MI == OMPC_LINEAR_ref)

10806

ParamAttr.Kind = LinearRef;

10807 else if

(*MI == OMPC_LINEAR_uval)

10808

ParamAttr.Kind = LinearUVal;

10809 else if

(IsReferenceType)

10810

ParamAttr.Kind = LinearVal;

10812

ParamAttr.Kind = Linear;

10814

ParamAttr.StrideOrArg = llvm::APSInt::getUnsigned(1);

10818 if

(

const auto

*DRE =

10819

cast<DeclRefExpr>((*SI)->IgnoreParenImpCasts())) {

10820 if

(

const auto

*StridePVD =

10821

dyn_cast<ParmVarDecl>(DRE->getDecl())) {

10822

ParamAttr.HasVarStride =

true

;

10823 auto

It = ParamPositions.find(StridePVD->getCanonicalDecl());

10824

assert(It != ParamPositions.end() &&

10825 "Function parameter not found"

);

10826

ParamAttr.StrideOrArg = llvm::APSInt::getUnsigned(It->second);

10830

ParamAttr.StrideOrArg =

.Val.getInt();

10836 if

(!ParamAttr.HasVarStride &&

10837

(ParamAttr.Kind == Linear || ParamAttr.Kind == LinearRef))

10838

ParamAttr.StrideOrArg = ParamAttr.StrideOrArg * PtrRescalingFactor;

10842

llvm::APSInt VLENVal;

10844 const Expr

*VLENExpr =

->getSimdlen();

10849

OMPDeclareSimdDeclAttr::BranchStateTy State =

->getBranchState();

10852

}

else if

(

getTriple

().getArch() == llvm::Triple::aarch64) {

10853 unsigned

VLEN = VLENVal.getExtValue();

10854

StringRef MangledName = Fn->getName();

10857

MangledName,

's'

, 128, Fn, ExprLoc);

10860

MangledName,

'n'

, 128, Fn, ExprLoc);

10871 static const int

DoacrossFinArgs = 2;

10874

llvm::FunctionCallee RTLFn;

10875

llvm::Value *Args[DoacrossFinArgs];

10878

DoacrossCleanupTy(llvm::FunctionCallee RTLFn,

10881

assert(CallArgs.size() == DoacrossFinArgs);

10882

std::copy(CallArgs.begin(), CallArgs.end(), std::begin(Args));

10899 QualType

Int64Ty =

.getIntTypeForBitwidth(

64,

true

);

10907

RD =

.buildImplicitRecord(

"kmp_dim"

);

10917

llvm::APInt Size(

32, NumIterations.size());

10923 enum

{ LowerFD = 0, UpperFD, StrideFD };

10925 for

(

unsigned

I = 0,

= NumIterations.size(); I <

; ++I) {

10930

DimsLVal, *std::next(RD->

(), UpperFD));

10932

CGF.

EmitScalarExpr

(NumIterations[I]), NumIterations[I]->getType(),

10933

Int64Ty, NumIterations[I]->getExprLoc());

10937

DimsLVal, *std::next(RD->

(), StrideFD));

10944

llvm::Value *Args[] = {

10947

llvm::ConstantInt::getSigned(

, NumIterations.size()),

10952

llvm::FunctionCallee RTLFn =

.getOrCreateRuntimeFunction(

10955

llvm::Value *FiniArgs[DoacrossCleanupTy::DoacrossFinArgs] = {

10957

llvm::FunctionCallee FiniRTLFn =

.getOrCreateRuntimeFunction(

10963template

typename

10965 const T

, llvm::Value *ULoc,

10966

llvm::Value *ThreadID) {

10969

llvm::APInt Size(

32,

->getNumLoops());

10973 for

(

unsigned

I = 0,

->getNumLoops(); I <

; ++I) {

10974 const Expr

*CounterVal =

->getLoopData(I);

10975

assert(CounterVal);

10982

llvm::Value *Args[] = {

10985

llvm::FunctionCallee RTLFn;

10987

OMPDoacrossKind<T> ODK;

10988 if

(ODK.isSource(

)) {

10990

OMPRTL___kmpc_doacross_post);

10992

assert(ODK.isSink(

) &&

"Expect sink modifier."

);

10994

OMPRTL___kmpc_doacross_wait);

11001 return

EmitDoacrossOrdered<OMPDependClause>(

11008 return

EmitDoacrossOrdered<OMPDoacrossClause>(

11014

llvm::FunctionCallee Callee,

11016

assert(

isValid

() &&

"Outlined function call location must be valid."

);

11019 if

(

auto

*Fn = dyn_cast<llvm::Function>(Callee.getCallee())) {

11020 if

(Fn->doesNotThrow()) {

11035 if

(

const auto

*FD = dyn_cast<FunctionDecl>(

))

11036 if

(OMPDeclareTargetDeclAttr::isDeclareTargetDeclaration(FD))

11042 const VarDecl

*TargetParam)

const

{

11049 const Expr

*Allocator) {

11050

llvm::Value *AllocVal;

11057

Allocator->getExprLoc());

11060

AllocVal = llvm::Constant::getNullValue(

11070 if

(!AllocateAlignment)

11073 return

llvm::ConstantInt::get(

, AllocateAlignment->getQuantity());

11086 auto

I = UntiedData.find(VD);

11087 if

(I != UntiedData.end()) {

11088

UntiedAddr = I->second.first;

11089

UntiedRealAddr = I->second.second;

11093 if

(CVD->

hasAttr

<OMPAllocateDeclAttr>()) {

11102

Size = CGF.

.CreateNUWAdd(

11111 const auto

*AA = CVD->

getAttr

<OMPAllocateDeclAttr>();

11112 const Expr

*Allocator = AA->getAllocator();

11116

Args.push_back(ThreadID);

11118

Args.push_back(Alignment);

11119

Args.push_back(Size);

11120

Args.push_back(AllocVal);

11121

llvm::omp::RuntimeFunction FnID =

11122

Alignment ? OMPRTL___kmpc_aligned_alloc : OMPRTL___kmpc_alloc;

11126

llvm::FunctionCallee FiniRTLFn =

.getOrCreateRuntimeFunction(

11136

llvm::FunctionCallee RTLFn;

11139 const Expr

*AllocExpr;

11142

OMPAllocateCleanupTy(llvm::FunctionCallee RTLFn,

11144 const Expr

*AllocExpr)

11145

: RTLFn(RTLFn), LocEncoding(LocEncoding), Addr(Addr),

11146

AllocExpr(AllocExpr) {}

11150

llvm::Value *Args[3];

11156

Args[2] = AllocVal;

11164

CGF.

EHStack

.pushCleanup<OMPAllocateCleanupTy>(

11166

VDAddr, Allocator);

11167 if

(UntiedRealAddr.

isValid

())

11168 if

(

auto

*Region =

11170

Region->emitUntiedSwitch(CGF);

11187

assert(CGM.

().OpenMP &&

"Not in OpenMP mode."

);

11193 for

(

const Stmt

*Ref :

->private_refs()) {

11194 const auto

*SimpleRefExpr = cast<Expr>(Ref)->IgnoreParenImpCasts();

11196 if

(

const auto

*DRE = dyn_cast<DeclRefExpr>(SimpleRefExpr)) {

11197

VD = DRE->getDecl();

11199 const auto

*ME = cast<MemberExpr>(SimpleRefExpr);

11200

assert((ME->isImplicitCXXThis() ||

11201

isa<CXXThisExpr>(ME->getBase()->IgnoreParenImpCasts())) &&

11202 "Expected member of current class."

);

11203

VD = ME->getMemberDecl();

11219

std::pair<Address, Address>> &LocalVars)

11220

(CGF.

), NeedToPush(!LocalVars.empty()) {

11237 return

llvm::any_of(

11242void

CGOpenMPRuntime::LastprivateConditionalRAII::tryToDisableInnerAnalysis(

11246

llvm::DenseSet<CanonicalDeclPtr<const Decl>> NeedToCheckForLPCs;

11252 const CapturedStmt

*CS = S.getCapturedStmt(CaptureRegions.front());

11260 for

(

const Expr

*Ref :

->varlist()) {

11261 if

(!Ref->getType()->isScalarType())

11263 const auto

*DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());

11266

NeedToCheckForLPCs.insert(DRE->getDecl());

11270 for

(

const Expr

*Ref :

->varlist()) {

11271 if

(!Ref->getType()->isScalarType())

11273 const auto

*DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());

11276

NeedToCheckForLPCs.insert(DRE->getDecl());

11280 for

(

const Expr

*Ref :

->varlist()) {

11281 if

(!Ref->getType()->isScalarType())

11283 const auto

*DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());

11286

NeedToCheckForLPCs.insert(DRE->getDecl());

11290 for

(

const Expr

*Ref :

->varlist()) {

11291 if

(!Ref->getType()->isScalarType())

11293 const auto

*DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());

11296

NeedToCheckForLPCs.insert(DRE->getDecl());

11300 for

(

const Expr

*Ref :

->varlist()) {

11301 if

(!Ref->getType()->isScalarType())

11303 const auto

*DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());

11306

NeedToCheckForLPCs.insert(DRE->getDecl());

11309 for

(

const Decl

*VD : NeedToCheckForLPCs) {

11310 for

(

const

LastprivateConditionalData &

11312 if

(

.DeclToUniqueName.count(VD) > 0) {

11313 if

.Disabled)

11314

NeedToAddForLPCsAsDisabled.insert(VD);

11321

CGOpenMPRuntime::LastprivateConditionalRAII::LastprivateConditionalRAII(

11324

Action((

.getLangOpts().OpenMP >= 50 &&

11327 return C

->getKind() ==

11328

OMPC_LASTPRIVATE_conditional;

11330

? ActionToDo::PushAsLastprivateConditional

11331

: ActionToDo::DoNotPush) {

11333 if

(

().OpenMP < 50 || Action == ActionToDo::DoNotPush)

11335

assert(Action == ActionToDo::PushAsLastprivateConditional &&

11336 "Expected a push action."

);

11340 if

(

->getKind() != OMPC_LASTPRIVATE_conditional)

11343 for

(

const Expr

*Ref :

->varlist()) {

11344 Data

.DeclToUniqueName.insert(std::make_pair(

11345

cast<DeclRefExpr>(Ref->IgnoreParenImpCasts())->getDecl(),

11349 Data

.IVLVal = IVLVal;

11353

CGOpenMPRuntime::LastprivateConditionalRAII::LastprivateConditionalRAII(

11355

(CGF.

), Action(ActionToDo::DoNotPush) {

11356

assert(CGM.

().OpenMP &&

"Not in OpenMP mode."

);

11359

llvm::DenseSet<CanonicalDeclPtr<const Decl>> NeedToAddForLPCsAsDisabled;

11360

tryToDisableInnerAnalysis(S, NeedToAddForLPCsAsDisabled);

11361 if

(!NeedToAddForLPCsAsDisabled.empty()) {

11362

Action = ActionToDo::DisableLastprivateConditional;

11363

LastprivateConditionalData &

11365 for

(

const Decl

*VD : NeedToAddForLPCsAsDisabled)

11368 Data

.Disabled =

true

;

11381 if

(Action == ActionToDo::DisableLastprivateConditional) {

11383 "Expected list of disabled private vars."

);

11386 if

(Action == ActionToDo::PushAsLastprivateConditional) {

11389 "Expected list of lastprivate conditional vars."

);

11402 auto

VI = I->getSecond().find(VD);

11403 if

(VI == I->getSecond().end()) {

11404 RecordDecl

*RD =

.buildImplicitRecord(

"lasprivate.conditional"

);

11409

NewType =

.getRecordType(RD);

11412

I->getSecond().try_emplace(VD, NewType, VDField, FiredField, BaseLVal);

11414

NewType = std::get<0>(VI->getSecond());

11415

VDField = std::get<1>(VI->getSecond());

11416

FiredField = std::get<2>(VI->getSecond());

11417

BaseLVal = std::get<3>(VI->getSecond());

11429class

LastprivateConditionalRefChecker final

11432 const Expr

*FoundE =

nullptr

;

11433 const Decl

*FoundD =

nullptr

;

11434

StringRef UniqueDeclName;

11436

llvm::Function *FoundFn =

nullptr

;

11442

llvm::reverse(LPM)) {

11443 auto

It =

.DeclToUniqueName.find(

->getDecl());

11444 if

(It ==

.DeclToUniqueName.end())

11449

FoundD =

->getDecl()->getCanonicalDecl();

11450

UniqueDeclName = It->second;

11451

IVLVal =

.IVLVal;

11455 return

FoundE ==

;

11458 if

(!CodeGenFunction::IsWrappedCXXThis(

->getBase()))

11461

llvm::reverse(LPM)) {

11462 auto

It =

.DeclToUniqueName.find(

->getMemberDecl());

11463 if

(It ==

.DeclToUniqueName.end())

11468

FoundD =

->getMemberDecl()->getCanonicalDecl();

11469

UniqueDeclName = It->second;

11470

IVLVal =

.IVLVal;

11474 return

FoundE ==

;

11476 bool

VisitStmt(

const Stmt

*S) {

11477 for

(

const Stmt

*Child : S->children()) {

11480 if

(

const auto

= dyn_cast<Expr>(Child))

11488 explicit

LastprivateConditionalRefChecker(

11491

std::tuple<const Expr *, const Decl *, StringRef, LValue, llvm::Function *>

11492

getFoundData()

const

{

11493 return

std::make_tuple(FoundE, FoundD, UniqueDeclName, IVLVal, FoundFn);

11500

StringRef UniqueDeclName,

11506

llvm::Constant *LastIV =

.getOrCreateInternalVariable(

11507

LLIVTy,

({UniqueDeclName,

"iv"

}));

11508

cast<llvm::GlobalVariable>(LastIV)->setAlignment(

11515

llvm::GlobalVariable *

Last

.getOrCreateInternalVariable(

11517

cast<llvm::GlobalVariable>(

Last

)->setAlignment(

11531 auto

&&CodeGen = [&LastIVLVal, &IVLVal, IVVal, &LVal, &LastLVal,

11537

llvm::Value *CmpRes;

11539

CmpRes = CGF.

.CreateICmpSLE(LastIVVal, IVVal);

11542 "Loop iteration variable must be integer."

);

11543

CmpRes = CGF.

.CreateICmpULE(LastIVVal, IVVal);

11547

CGF.

.CreateCondBr(CmpRes, ThenBB, ExitBB);

11568 "Aggregates are not supported in lastprivate conditional."

);

11591 if

(!Checker.Visit(LHS))

11593 const Expr

*FoundE;

11594 const Decl

*FoundD;

11595

StringRef UniqueDeclName;

11597

llvm::Function *FoundFn;

11598

std::tie(FoundE, FoundD, UniqueDeclName, IVLVal, FoundFn) =

11599

Checker.getFoundData();

11600 if

(FoundFn != CGF.

) {

11605 "Lastprivate conditional is not found in outer region."

);

11606 QualType

StructTy = std::get<0>(It->getSecond());

11607 const FieldDecl

* FiredDecl = std::get<2>(It->getSecond());

11618

FiredLVal, llvm::AtomicOrdering::Unordered,

11636 auto

It = llvm::find_if(

11638 if

(It ==

Range

.end() || It->Fn != CGF.

)

11642 "Lastprivates must be registered already."

);

11645 const CapturedStmt

*CS =

.getCapturedStmt(CaptureRegions.back());

11646 for

(

const auto

&Pair : It->DeclToUniqueName) {

11647 const auto

*VD = cast<VarDecl>(Pair.first->getCanonicalDecl());

11650 auto

I = LPCI->getSecond().find(Pair.first);

11651

assert(I != LPCI->getSecond().end() &&

11652 "Lastprivate must be rehistered already."

);

11654 LValue

BaseLVal = std::get<3>(I->getSecond());

11658

llvm::Value *Cmp = CGF.

.CreateIsNotNull(Res);

11662

CGF.

.CreateCondBr(Cmp, ThenBB, DoneBB);

11687 "Unknown lastprivate conditional variable."

);

11688

StringRef UniqueName = It->second;

11689

llvm::GlobalVariable *GV =

().getNamedGlobal(UniqueName);

11703

llvm_unreachable(

"Not supported in SIMD-only mode"

);

11710

llvm_unreachable(

"Not supported in SIMD-only mode"

);

11717 bool

Tied,

unsigned

&NumberOfParts) {

11718

llvm_unreachable(

"Not supported in SIMD-only mode"

);

11723

llvm::Function *OutlinedFn,

11725 const Expr

*IfCond,

11726

llvm::Value *NumThreads) {

11727

llvm_unreachable(

"Not supported in SIMD-only mode"

);

11733 const Expr

*Hint) {

11734

llvm_unreachable(

"Not supported in SIMD-only mode"

);

11740

llvm_unreachable(

"Not supported in SIMD-only mode"

);

11746 const Expr

*Filter) {

11747

llvm_unreachable(

"Not supported in SIMD-only mode"

);

11752

llvm_unreachable(

"Not supported in SIMD-only mode"

);

11758

llvm_unreachable(

"Not supported in SIMD-only mode"

);

11766

llvm_unreachable(

"Not supported in SIMD-only mode"

);

11773

llvm_unreachable(

"Not supported in SIMD-only mode"

);

11780 bool

ForceSimpleCall) {

11781

llvm_unreachable(

"Not supported in SIMD-only mode"

);

11788

llvm_unreachable(

"Not supported in SIMD-only mode"

);

11793

llvm_unreachable(

"Not supported in SIMD-only mode"

);

11799

llvm_unreachable(

"Not supported in SIMD-only mode"

);

11805

llvm_unreachable(

"Not supported in SIMD-only mode"

);

11812

llvm_unreachable(

"Not supported in SIMD-only mode"

);

11818

llvm_unreachable(

"Not supported in SIMD-only mode"

);

11823 unsigned

IVSize,

bool

IVSigned,

11826

llvm_unreachable(

"Not supported in SIMD-only mode"

);

11830

llvm::Value *NumThreads,

11832

llvm_unreachable(

"Not supported in SIMD-only mode"

);

11836

ProcBindKind ProcBind,

11838

llvm_unreachable(

"Not supported in SIMD-only mode"

);

11845

llvm_unreachable(

"Not supported in SIMD-only mode"

);

11851

llvm_unreachable(

"Not supported in SIMD-only mode"

);

11856

llvm_unreachable(

"Not supported in SIMD-only mode"

);

11862

llvm::AtomicOrdering AO) {

11863

llvm_unreachable(

"Not supported in SIMD-only mode"

);

11868

llvm::Function *TaskFunction,

11870 const Expr

*IfCond,

11872

llvm_unreachable(

"Not supported in SIMD-only mode"

);

11879

llvm_unreachable(

"Not supported in SIMD-only mode"

);

11886

assert(Options.SimpleReduction &&

"Only simple reduction is expected."

);

11888

ReductionOps, Options);

11894

llvm_unreachable(

"Not supported in SIMD-only mode"

);

11899 bool

IsWorksharingReduction) {

11900

llvm_unreachable(

"Not supported in SIMD-only mode"

);

11907

llvm_unreachable(

"Not supported in SIMD-only mode"

);

11912

llvm::Value *ReductionsPtr,

11914

llvm_unreachable(

"Not supported in SIMD-only mode"

);

11920

llvm_unreachable(

"Not supported in SIMD-only mode"

);

11926

llvm_unreachable(

"Not supported in SIMD-only mode"

);

11932

llvm_unreachable(

"Not supported in SIMD-only mode"

);

11937

llvm::Function *&OutlinedFn, llvm::Constant *&OutlinedFnID,

11939

llvm_unreachable(

"Not supported in SIMD-only mode"

);

11944

llvm::Function *OutlinedFn, llvm::Value *OutlinedFnID,

const Expr

*IfCond,

11945

llvm::PointerIntPair<const Expr *, 2, OpenMPDeviceClauseModifier>