Home - News ( United States | United Kingdom | Italy | Germany ) - Football scores

Showing content from https://clang.llvm.org/doxygen/CGStmtOpenMP_8cpp_source.html below:

clang: lib/CodeGen/CGStmtOpenMP.cpp Source File

;

CodeGen;

llvm::omp;

OMPLexicalScope :

CodeGenFunction::LexicalScope {

(

*

: S.clauses()) {

(

*PreInit =

cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {

(

*I : PreInit->decls()) {

(!I->hasAttr<OMPCaptureNoInitAttr>()) {

CodeGenFunction::AutoVarEmission Emission =

CodeGenFunction::OMPPrivateScope InlinedShareds;

cast<BlockDecl>(CGF.

)->capturesVariable(VD));

std::optional<OpenMPDirectiveKind> CapturedRegion = std::nullopt,

EmitPreInitStmt =

)

emitPreInitStmt(CGF, S);

assert(S.hasAssociatedStmt() &&

);

*CS = S.getCapturedStmt(*CapturedRegion);

(

&

: CS->

()) {

(

.capturesVariable() ||

.capturesVariableByCopy()) {

*VD =

.getCapturedVar();

);

InlinedShareds.isGlobalVarCaptured(VD)),

(void)InlinedShareds.Privatize();

OMPParallelScope final :

OMPLexicalScope {

: OMPLexicalScope(CGF, S,

::nullopt,

EmitPreInitStmt(S)) {}

OMPTeamsScope final :

OMPLexicalScope {

: OMPLexicalScope(CGF, S,

::nullopt,

EmitPreInitStmt(S)) {}

OMPLoopScope :

CodeGenFunction::RunCleanupsScope {

*PreInits;

CodeGenFunction::OMPMapVars PreCondVars;

(

*LD = dyn_cast<OMPLoopDirective>(&S)) {

llvm::DenseSet<const VarDecl *> EmittedAsPrivate;

(

*

: LD->counters()) {

*VD = cast<VarDecl>(cast<DeclRefExpr>(

)->getDecl());

(void)PreCondVars.setVarAddr(

(

*IRef :

->varlist()) {

cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl());

(EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {

OrigVDTy = OrigVD->getType().getNonReferenceType();

(void)PreCondVars.setVarAddr(

(void)PreCondVars.apply(CGF);

LD->getInnermostCapturedStmt()->getCapturedStmt(),

, LD->getLoopsNumber(),

[&CGF](

Cnt,

*CurStmt) {

(

*CXXFor = dyn_cast<CXXForRangeStmt>(CurStmt)) {

(

*

= CXXFor->getInit())

CGF.

(CXXFor->getRangeStmt());

CGF.

(CXXFor->getEndStmt());

PreInits = LD->getPreInits();

}

(

*

= dyn_cast<OMPTileDirective>(&S)) {

PreInits =

->getPreInits();

}

(

*Unroll = dyn_cast<OMPUnrollDirective>(&S)) {

PreInits = Unroll->getPreInits();

}

(

*Reverse = dyn_cast<OMPReverseDirective>(&S)) {

PreInits = Reverse->getPreInits();

}

(

*Interchange =

dyn_cast<OMPInterchangeDirective>(&S)) {

PreInits = Interchange->getPreInits();

llvm_unreachable(

);

(

*PreInitCompound = dyn_cast<CompoundStmt>(PreInits))

llvm::append_range(PreInitStmts, PreInitCompound->body());

PreInitStmts.push_back(PreInits);

(

*S : PreInitStmts) {

(

*PreInitDecl = dyn_cast<DeclStmt>(S)) {

(

*I : PreInitDecl->decls())

PreCondVars.restore(CGF);

emitPreInitStmt(CGF, S);

OMPSimdLexicalScope :

CodeGenFunction::LexicalScope {

CodeGenFunction::OMPPrivateScope InlinedShareds;

cast<BlockDecl>(CGF.

)->capturesVariable(VD));

InlinedShareds(CGF) {

(

*

: S.clauses()) {

(

*PreInit =

cast_or_null<DeclStmt>(CPI->getPreInitStmt())) {

(

*I : PreInit->decls()) {

(!I->hasAttr<OMPCaptureNoInitAttr>()) {

CodeGenFunction::AutoVarEmission Emission =

}

(

*UDP = dyn_cast<OMPUseDevicePtrClause>(

)) {

(

*

: UDP->varlist()) {

*

= cast<DeclRefExpr>(

)->getDecl();

(

*OED = dyn_cast<OMPCapturedExprDecl>(

))

}

(

*UDP = dyn_cast<OMPUseDeviceAddrClause>(

)) {

(

*

: UDP->varlist()) {

(

*OED = dyn_cast<OMPCapturedExprDecl>(

))

(

*TG = dyn_cast<OMPTaskgroupDirective>(&S)) {

(

*

= TG->getReductionRef())

CGF.

(*cast<VarDecl>(cast<DeclRefExpr>(

)->getDecl()));

llvm::DenseSet<CanonicalDeclPtr<const Decl>> CopyArrayTemps;

(

->getModifier() != OMPC_REDUCTION_inscan)

(

*

:

->copy_array_temps())

CopyArrayTemps.insert(cast<DeclRefExpr>(

)->getDecl());

*CS = cast_or_null<CapturedStmt>(S.getAssociatedStmt());

(

.capturesVariable() ||

.capturesVariableByCopy()) {

*VD =

.getCapturedVar();

(CopyArrayTemps.contains(VD))

);

isCapturedVar(CGF, VD) ||

InlinedShareds.isGlobalVarCaptured(VD)),

(void)InlinedShareds.Privatize();

(Kind != OMPD_loop)

BindKind =

->getBindKind();

OMPC_BIND_parallel:

OMPC_BIND_teams:

OMPD_distribute;

OMPC_BIND_thread:

CodeGenFunction::EmitOMPSharedLValue(

*

) {

(

*OrigDRE = dyn_cast<DeclRefExpr>(

)) {

(

*OrigVD = dyn_cast<VarDecl>(OrigDRE->getDecl())) {

OrigVD = OrigVD->getCanonicalDecl();

llvm::Value *

=

;

SizeInChars =

.getTypeSizeInChars(Ty);

(SizeInChars.isZero()) {

?

.CreateNUWMul(Size, VlaSize.NumElts) : VlaSize.NumElts;

SizeInChars =

.getTypeSizeInChars(Ty);

(SizeInChars.isZero())

llvm::ConstantInt::get(

,

0);

*RD = S.getCapturedRecordDecl();

CurCap = S.captures().begin();

= S.capture_init_end();

I !=

; ++I, ++CurField, ++CurCap) {

(CurField->hasCapturedVLAType()) {

llvm::Value *Val = VLASizeMap[VAT->

()];

CapturedVars.push_back(Val);

}

(CurCap->capturesThis()) {

CapturedVars.push_back(CXXThisValue);

}

(CurCap->capturesVariableByCopy()) {

(!CurField->getType()->isAnyPointerType()) {

Twine(CurCap->getCapturedVar()->getName(),

));

Ctx.

(CurField->getType()), CurCap->getLocation());

CapturedVars.push_back(CV);

assert(CurCap->capturesVariable() &&

);

.getLValueReferenceType(

(

*VLA = dyn_cast<VariableArrayType>(A))

(!A->isVariablyModifiedType())

.getCanonicalType(

);

.getCanonicalParamType(

);

FunctionOptions {

UIntPtrCastRequired =

;

RegisterCastedArgsOnly =

;

StringRef FunctionName;

FunctionOptions(

*S,

UIntPtrCastRequired,

RegisterCastedArgsOnly, StringRef FunctionName,

: S(S), UIntPtrCastRequired(UIntPtrCastRequired),

RegisterCastedArgsOnly(UIntPtrCastRequired && RegisterCastedArgsOnly),

FunctionName(FunctionName),

(

) {}

llvm::MapVector<

*, std::pair<const VarDecl *, Address>>

llvm::DenseMap<

*, std::pair<const Expr *, llvm::Value *>>

llvm::Value *&CXXThisValue,

FunctionOptions &FO) {

*RD = FO.S->getCapturedRecordDecl();

assert(CD->

() &&

);

CXXThisValue =

;

I = FO.S->captures().begin();

(!FO.UIntPtrCastRequired) {

(FO.UIntPtrCastRequired &&

I->capturesVariableArrayType()))

(I->capturesVariable() || I->capturesVariableByCopy()) {

CapVar = I->getCapturedVar();

}

(I->capturesThis()) {

assert(I->capturesVariableArrayType());

}

(DebugFunctionDecl && (CapVar || I->capturesThis())) {

Ctx, DebugFunctionDecl,

CapVar ? CapVar->

() : FD->getBeginLoc(),

CapVar ? CapVar->

() : FD->getLocation(), II, ArgType,

Args.emplace_back(Arg);

TargetArgs.emplace_back(

FO.UIntPtrCastRequired

llvm::Function::Create(FuncLLVMTy, llvm::GlobalValue::InternalLinkage,

F->setDoesNotThrow();

F->setDoesNotRecurse();

F->removeFnAttr(llvm::Attribute::NoInline);

F->addFnAttr(llvm::Attribute::AlwaysInline);

FO.UIntPtrCastRequired ? FO.Loc : FO.S->getBeginLoc(),

FO.UIntPtrCastRequired ? FO.Loc

I = FO.S->captures().begin();

(!FO.UIntPtrCastRequired && Args[Cnt] != TargetArgs[Cnt]) {

(I->capturesVariableByCopy() && FD->getType()->isAnyPointerType()) {

*CurVD = I->getCapturedVar();

(!FO.RegisterCastedArgsOnly)

LocalAddrs.insert({Args[Cnt], {CurVD, LocalAddr}});

(FD->hasCapturedVLAType()) {

(FO.UIntPtrCastRequired) {

Args[Cnt]->getName(), ArgLVal),

VLASizes.try_emplace(Args[Cnt], VAT->

(), ExprArg);

}

(I->capturesVariable()) {

*Var = I->getCapturedVar();

(!FO.RegisterCastedArgsOnly) {

}

(I->capturesVariableByCopy()) {

assert(!FD->getType()->isAnyPointerType() &&

);

*Var = I->getCapturedVar();

LocalAddrs.insert({Args[Cnt],

{Var, FO.UIntPtrCastRequired

CGF, I->getLocation(), FD->getType(),

Args[Cnt]->getName(), ArgLVal)

assert(I->capturesThis());

LocalAddrs.insert({Args[Cnt], {

, ArgLVal.

()}});

);

NeedWrapperFunction =

llvm::MapVector<const Decl *, std::pair<const VarDecl *, Address>> LocalAddrs,

llvm::DenseMap<const Decl *, std::pair<const Expr *, llvm::Value *>> VLASizes,

llvm::raw_svector_ostream Out(Buffer);

llvm::Function *WrapperF =

;

(NeedWrapperFunction) {

FunctionOptions WrapperFO(&S,

,

WrapperCGF.CXXThisValue, WrapperFO);

FunctionOptions FO(&S, !NeedWrapperFunction,

,

*

, WrapperArgs, WrapperLocalAddrs, WrapperVLASizes, CXXThisValue, FO);

CodeGenFunction::OMPPrivateScope LocalScope(*

);

(

&LocalAddrPair : WrapperLocalAddrs) {

(LocalAddrPair.second.first) {

LocalScope.addPrivate(LocalAddrPair.second.first,

LocalAddrPair.second.second);

(void)LocalScope.Privatize();

(

&VLASizePair : WrapperVLASizes)

VLASizeMap[VLASizePair.second.first] = VLASizePair.second.second;

(void)LocalScope.ForceCleanup();

(!NeedWrapperFunction)

WrapperF->removeFromParent();

F->getParent()->getFunctionList().insertAfter(F->getIterator(), WrapperF);

*PI = F->arg_begin();

(

*Arg : Args) {

I = LocalAddrs.find(Arg);

(I != LocalAddrs.end()) {

LV = WrapperCGF.MakeAddrLValue(

I->second.first ? I->second.first->getType() : Arg->getType(),

= WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());

EI = VLASizes.find(Arg);

(EI != VLASizes.end()) {

WrapperCGF.MakeAddrLValue(WrapperCGF.GetAddrOfLocalVar(Arg),

= WrapperCGF.EmitLoadOfScalar(LV, S.getBeginLoc());

CallArgs.emplace_back(WrapperCGF.EmitFromMemory(

, Arg->

()));

WrapperCGF.FinishFunction();

llvm::Value *NumElements =

(ArrayTy, ElementTy, DestAddr);

DestBegin, NumElements);

llvm::Value *IsEmpty =

.CreateICmpEQ(DestBegin, DestEnd,

);

.CreateCondBr(IsEmpty, DoneBB, BodyBB);

llvm::BasicBlock *EntryBB =

.GetInsertBlock();

llvm::PHINode *SrcElementPHI =

.CreatePHI(SrcBegin->getType(), 2,

);

SrcElementPHI->addIncoming(SrcBegin, EntryBB);

llvm::PHINode *DestElementPHI =

.CreatePHI(

DestBegin->getType(), 2,

);

DestElementPHI->addIncoming(DestBegin, EntryBB);

CopyGen(DestElementCurrent, SrcElementCurrent);

llvm::Value *DestElementNext =

1,

);

llvm::Value *SrcElementNext =

1,

);

.CreateICmpEQ(DestElementNext, DestEnd,

);

.CreateCondBr(Done, DoneBB, BodyBB);

DestElementPHI->addIncoming(DestElementNext,

.GetInsertBlock());

SrcElementPHI->addIncoming(SrcElementNext,

.GetInsertBlock());

*BO = dyn_cast<BinaryOperator>(

);

(BO && BO->getOpcode() == BO_Assign) {

DestAddr, SrcAddr, OriginalType,

CodeGenFunction::OMPPrivateScope Remap(*

);

Remap.addPrivate(DestVD, DestElement);

Remap.addPrivate(SrcVD, SrcElement);

(void)Remap.Privatize();

CodeGenFunction::OMPPrivateScope Remap(*

);

Remap.addPrivate(SrcVD, SrcAddr);

Remap.addPrivate(DestVD, DestAddr);

(void)Remap.Privatize();

OMPPrivateScope &PrivateScope) {

DeviceConstTarget =

().OpenMPIsTargetDevice &&

FirstprivateIsLastprivate =

;

llvm::DenseMap<const VarDecl *, OpenMPLastprivateModifier> Lastprivates;

(

*

:

->varlist())

Lastprivates.try_emplace(

llvm::DenseSet<const VarDecl *> EmittedAsFirstprivate;

MustEmitFirstprivateCopy =

CaptureRegions.size() == 1 && CaptureRegions.back() == OMPD_unknown;

*IRef =

->varlist_begin();

*InitsRef =

->inits().begin();

(

*IInit :

->private_copies()) {

*OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());

ThisFirstprivateIsLastprivate =

Lastprivates.count(OrigVD->getCanonicalDecl()) > 0;

*VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());

(!MustEmitFirstprivateCopy && !ThisFirstprivateIsLastprivate && FD &&

(!VD || !VD->

<OMPAllocateDeclAttr>())) {

EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());

(DeviceConstTarget && OrigVD->getType().isConstant(

()) &&

(!VD || !VD->

<OMPAllocateDeclAttr>())) {

EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl());

FirstprivateIsLastprivate =

FirstprivateIsLastprivate || ThisFirstprivateIsLastprivate;

(EmittedAsFirstprivate.insert(OrigVD->getCanonicalDecl()).second) {

cast<VarDecl>(cast<DeclRefExpr>(*InitsRef)->getDecl());

(CE && !CE.isReference()) {

(CE && CE.isReference()) {

OriginalLVal = CE.getReferenceLValue(*

, &DRE);

assert(!CE &&

);

Emission.getAllocatedAddress(), OriginalLVal.

(),

,

RunCleanupsScope InitScope(*this);

setAddrOfLocalVar(VDInit, SrcElement);

EmitAnyExprToMem(Init, DestElement,

Init->getType().getQualifiers(),

LocalDeclMap.erase(VDInit);

PrivateScope.addPrivate(OrigVD, Emission.getAllocatedAddress());

setAddrOfLocalVar(VDInit, OriginalAddr);

LocalDeclMap.erase(VDInit);

(ThisFirstprivateIsLastprivate &&

Lastprivates[OrigVD->getCanonicalDecl()] ==

OMPC_LASTPRIVATE_conditional) {

(*IRef)->getExprLoc());

LocalDeclMap.erase(VD);

setAddrOfLocalVar(VD, VDAddr);

IsRegistered = PrivateScope.addPrivate(OrigVD, VDAddr);

assert(IsRegistered &&

);

FirstprivateIsLastprivate && !EmittedAsFirstprivate.empty();

CodeGenFunction::OMPPrivateScope &PrivateScope) {

llvm::DenseSet<const VarDecl *> EmittedAsPrivate;

IRef =

->varlist_begin();

(

*IInit :

->private_copies()) {

*OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());

(EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {

*VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());

assert(IsRegistered &&

);

llvm::DenseSet<const VarDecl *> CopiedVars;

llvm::BasicBlock *CopyBegin =

, *CopyEnd =

;

IRef =

->varlist_begin();

ISrcRef =

->source_exprs().begin();

IDestRef =

->destination_exprs().begin();

(

*AssignOp :

->assignment_ops()) {

*VD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());

().getTargetInfo().isTLSSupported()) {

);

LocalDeclMap.erase(VD);

(CopiedVars.size() == 1) {

*MasterAddrInt =

.CreatePtrToInt(

*PrivateAddrInt =

.CreatePtrToInt(

.CreateICmpNE(MasterAddrInt, PrivateAddrInt), CopyBegin,

cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());

*DestVD =

cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());

(

, PrivateAddr, MasterAddr, DestVD, SrcVD, AssignOp);

HasAtLeastOneLastprivate =

;

llvm::DenseSet<const VarDecl *> SIMDLCVs;

*LoopDirective = cast<OMPLoopDirective>(&

);

(

*

: LoopDirective->counters()) {

llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;

HasAtLeastOneLastprivate =

;

*IRef =

->varlist_begin();

*IDestRef =

->destination_exprs().begin();

(

*IInit :

->private_copies()) {

*OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());

(AlreadyEmittedVars.insert(OrigVD->getCanonicalDecl()).second) {

*DestVD =

cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());

PrivateScope.addPrivate(DestVD,

(&DRE).getAddress());

(IInit && !SIMDLCVs.count(OrigVD->getCanonicalDecl())) {

*VD = cast<VarDecl>(cast<DeclRefExpr>(IInit)->getDecl());

(

->getKind() == OMPC_LASTPRIVATE_conditional) {

setAddrOfLocalVar(VD, VDAddr);

IsRegistered = PrivateScope.addPrivate(OrigVD, VDAddr);

assert(IsRegistered &&

);

HasAtLeastOneLastprivate;

llvm::Value *IsLastIterCond) {

llvm::BasicBlock *ThenBB =

;

llvm::BasicBlock *DoneBB =

;

(IsLastIterCond) {

return C->getKind() == OMPC_LASTPRIVATE_conditional;

.CreateCondBr(IsLastIterCond, ThenBB, DoneBB);

llvm::DenseSet<const VarDecl *> AlreadyEmittedVars;

llvm::DenseMap<const VarDecl *, const Expr *> LoopCountersAndUpdates;

(

*LoopDirective = dyn_cast<OMPLoopDirective>(&

)) {

IC = LoopDirective->counters().begin();

(

*F : LoopDirective->finals()) {

AlreadyEmittedVars.insert(

);

LoopCountersAndUpdates[

] = F;

IRef =

->varlist_begin();

ISrcRef =

->source_exprs().begin();

IDestRef =

->destination_exprs().begin();

(

*AssignOp :

->assignment_ops()) {

*PrivateVD =

cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());

*CanonicalVD = PrivateVD->getCanonicalDecl();

(AlreadyEmittedVars.insert(CanonicalVD).second) {

(

*FinalExpr = LoopCountersAndUpdates.lookup(CanonicalVD))

cast<VarDecl>(cast<DeclRefExpr>(*ISrcRef)->getDecl());

*DestVD =

cast<VarDecl>(cast<DeclRefExpr>(*IDestRef)->getDecl());

(

*RefTy = PrivateVD->getType()->getAs<

>())

(

->getKind() == OMPC_LASTPRIVATE_conditional)

*

,

(PrivateAddr, (*IRef)->getType()), PrivateVD,

(*IRef)->getExprLoc());

(

, OriginalAddr, PrivateAddr, DestVD, SrcVD, AssignOp);

(

*PostUpdate =

->getPostUpdateExpr())

CodeGenFunction::OMPPrivateScope &PrivateScope,

ForInscan) {

(ForInscan != (

->getModifier() == OMPC_REDUCTION_inscan))

Shareds.append(

->varlist_begin(),

->varlist_end());

Privates.append(

->privates().begin(),

->privates().end());

ReductionOps.append(

->reduction_ops().begin(),

->reduction_ops().end());

LHSs.append(

->lhs_exprs().begin(),

->lhs_exprs().end());

RHSs.append(

->rhs_exprs().begin(),

->rhs_exprs().end());

(

->getModifier() == OMPC_REDUCTION_task) {

.ReductionVars.append(

->privates().begin(),

->privates().end());

.ReductionOrigs.append(

->varlist_begin(),

->varlist_end());

.ReductionCopies.append(

->privates().begin(),

->privates().end());

.ReductionOps.append(

->reduction_ops().begin(),

->reduction_ops().end());

TaskLHSs.append(

->lhs_exprs().begin(),

->lhs_exprs().end());

TaskRHSs.append(

->rhs_exprs().begin(),

->rhs_exprs().end());

*ILHS = LHSs.begin();

*IRHS = RHSs.begin();

*IPriv = Privates.begin();

(

*IRef : Shareds) {

*PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*IPriv)->getDecl());

RedCG.emitSharedOrigLValue(*

, Count);

RedCG.emitAggregateType(*

, Count);

RedCG.emitInitialization(*

, Count, Emission.getAllocatedAddress(),

RedCG.getSharedLValue(Count).getAddress(),

CGF.EmitAutoVarInit(Emission);

BaseAddr = RedCG.adjustPrivateAddress(

*

, Count, Emission.getAllocatedAddress());

PrivateScope.addPrivate(RedCG.getBaseDecl(Count), BaseAddr);

assert(IsRegistered &&

);

*LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());

*RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());

isaOMPArraySectionExpr = isa<ArraySectionExpr>(IRef);

PrivateScope.addPrivate(LHSVD, RedCG.getSharedLValue(Count).getAddress());

isa<ArraySubscriptExpr>(IRef)) {

PrivateScope.addPrivate(LHSVD, RedCG.getSharedLValue(Count).getAddress());

PrivateScope.addPrivate(RHSVD,

OriginalAddr = RedCG.getSharedLValue(Count).getAddress();

PrivateScope.addPrivate(LHSVD, OriginalAddr);

PrivateScope.addPrivate(

(!

.ReductionVars.empty()) {

.IsReductionWithTaskMod =

;

*TaskRedRef =

;

TaskRedRef = cast<OMPParallelDirective>(

).getTaskReductionRefExpr();

TaskRedRef = cast<OMPForDirective>(

).getTaskReductionRefExpr();

TaskRedRef = cast<OMPSectionsDirective>(

).getTaskReductionRefExpr();

OMPD_parallel_for:

TaskRedRef = cast<OMPParallelForDirective>(

).getTaskReductionRefExpr();

OMPD_parallel_master:

cast<OMPParallelMasterDirective>(

).getTaskReductionRefExpr();

OMPD_parallel_sections:

cast<OMPParallelSectionsDirective>(

).getTaskReductionRefExpr();

OMPD_target_parallel:

cast<OMPTargetParallelDirective>(

).getTaskReductionRefExpr();

OMPD_target_parallel_for:

cast<OMPTargetParallelForDirective>(

).getTaskReductionRefExpr();

OMPD_distribute_parallel_for:

cast<OMPDistributeParallelForDirective>(

).getTaskReductionRefExpr();

OMPD_teams_distribute_parallel_for:

TaskRedRef = cast<OMPTeamsDistributeParallelForDirective>(

)

.getTaskReductionRefExpr();

OMPD_target_teams_distribute_parallel_for:

TaskRedRef = cast<OMPTargetTeamsDistributeParallelForDirective>(

)

.getTaskReductionRefExpr();

OMPD_parallel_for_simd:

OMPD_taskyield:

OMPD_taskgroup:

OMPD_cancellation_point:

OMPD_target_data:

OMPD_target_enter_data:

OMPD_target_exit_data:

OMPD_taskloop_simd:

OMPD_master_taskloop:

OMPD_master_taskloop_simd:

OMPD_parallel_master_taskloop:

OMPD_parallel_master_taskloop_simd:

OMPD_distribute:

OMPD_target_update:

OMPD_distribute_parallel_for_simd:

OMPD_distribute_simd:

OMPD_target_parallel_for_simd:

OMPD_target_simd:

OMPD_teams_distribute:

OMPD_teams_distribute_simd:

OMPD_teams_distribute_parallel_for_simd:

OMPD_target_teams:

OMPD_target_teams_distribute:

OMPD_target_teams_distribute_parallel_for_simd:

OMPD_target_teams_distribute_simd:

OMPD_declare_target:

OMPD_end_declare_target:

OMPD_threadprivate:

OMPD_declare_reduction:

OMPD_declare_mapper:

OMPD_declare_simd:

OMPD_declare_variant:

OMPD_begin_declare_variant:

OMPD_end_declare_variant:

llvm_unreachable(

);

*VD = cast<VarDecl>(cast<DeclRefExpr>(TaskRedRef)->getDecl());

, TaskRedRef->

());

HasAtLeastOneReduction =

;

IsReductionWithTaskMod =

;

(

->getModifier() == OMPC_REDUCTION_inscan)

HasAtLeastOneReduction =

;

Privates.append(

->privates().begin(),

->privates().end());

LHSExprs.append(

->lhs_exprs().begin(),

->lhs_exprs().end());

RHSExprs.append(

->rhs_exprs().begin(),

->rhs_exprs().end());

ReductionOps.append(

->reduction_ops().begin(),

->reduction_ops().end());

IsReductionWithTaskMod =

IsReductionWithTaskMod ||

->getModifier() == OMPC_REDUCTION_task;

(HasAtLeastOneReduction) {

(IsReductionWithTaskMod) {

TeamsLoopCanBeParallel =

;

(

*TTLD = dyn_cast<OMPTargetTeamsGenericLoopDirective>(&

))

TeamsLoopCanBeParallel = TTLD->canBeParallelFor();

TeamsLoopCanBeParallel || ReductionKind == OMPD_simd;

SimpleReduction = ReductionKind == OMPD_simd;

*

,

.

(), Privates, LHSExprs, RHSExprs, ReductionOps,

{WithNowait, SimpleReduction, ReductionKind});

llvm::function_ref<llvm::Value *(

&)> CondGen) {

llvm::BasicBlock *DoneBB =

;

(

*PostUpdate =

->getPostUpdateExpr()) {

(llvm::Value *Cond = CondGen(CGF)) {

CGF.

.CreateCondBr(Cond, ThenBB, DoneBB);

CodeGenBoundParametersTy;

llvm::DenseSet<CanonicalDeclPtr<const VarDecl>> PrivateDecls;

(

*Ref :

->varlist()) {

(!Ref->getType()->isScalarType())

*DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());

PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));

(

*Ref :

->varlist()) {

(!Ref->getType()->isScalarType())

*DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());

PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));

(

*Ref :

->varlist()) {

(!Ref->getType()->isScalarType())

*DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());

PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));

(

*Ref :

->varlist()) {

(!Ref->getType()->isScalarType())

*DRE = dyn_cast<DeclRefExpr>(Ref->IgnoreParenImpCasts());

PrivateDecls.insert(cast<VarDecl>(DRE->getDecl()));

CGF, S, PrivateDecls);

CodeGenBoundParametersTy &CodeGenBoundParameters) {

*CS = S.getCapturedStmt(OMPD_parallel);

llvm::Value *NumThreads =

;

llvm::Function *OutlinedFn =

CodeGenFunction::RunCleanupsScope NumThreadsScope(CGF);

NumThreads = CGF.

(NumThreadsClause->getNumThreads(),

CGF, NumThreads, NumThreadsClause->getBeginLoc());

CodeGenFunction::RunCleanupsScope ProcBindScope(CGF);

CGF, ProcBindClause->getProcBindKind(), ProcBindClause->getBeginLoc());

*IfCond =

;

(

*

: S.getClausesOfKind<

>()) {

(

->getNameModifier() == OMPD_unknown ||

->getNameModifier() == OMPD_parallel) {

IfCond =

->getCondition();

OMPParallelScope

(CGF, S);

CodeGenBoundParameters(CGF, S, CapturedVars);

CapturedVars, IfCond, NumThreads);

(!CVD->

<OMPAllocateDeclAttr>())

*AA = CVD->

<OMPAllocateDeclAttr>();

!((AA->getAllocatorType() == OMPAllocateDeclAttr::OMPDefaultMemAlloc ||

AA->getAllocatorType() == OMPAllocateDeclAttr::OMPNullMemAlloc) &&

!AA->getAllocator());

CGF, S.getBeginLoc(), OMPD_unknown,

,

Size = CGF.

.CreateNUWAdd(

*AA = CVD->

<OMPAllocateDeclAttr>();

assert(AA->getAllocator() &&

);

llvm::Value *Allocator = CGF.

(AA->getAllocator());

(Allocator->getType()->isIntegerTy())

(Allocator->getType()->isPointerTy())

llvm::Value *Addr = OMPBuilder.createOMPAlloc(

CGF.

, Size, Allocator,

llvm::CallInst *FreeCI =

OMPBuilder.createOMPFree(CGF.

, Addr, Allocator);

llvm::Value *

=

std::string Suffix = getNameWithSeparators({

,

});

llvm::CallInst *ThreadPrivateCacheCall =

OMPBuilder.createCachedThreadPrivate(CGF.

,

, Size, CacheName);

llvm::raw_svector_ostream OS(Buffer);

StringRef Sep = FirstSeparator;

(StringRef Part : Parts) {

OS.str().str();

.restoreIP(CodeGenIP);

llvm::BasicBlock *FiniBB = splitBBWithSuffix(

,

,

+ RegionName +

);

(

.saveIP().isSet())

.restoreIP(CodeGenIP);

llvm::BasicBlock *FiniBB = splitBBWithSuffix(

,

,

+ RegionName +

);

(

.saveIP().isSet())

llvm::Value *IfCond =

;

(

*

= S.getSingleClause<

>())

llvm::Value *NumThreads =

;

ProcBindKind ProcBind = OMP_PROC_BIND_default;

ProcBind = ProcBindClause->getProcBindKind();

InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;

FiniCB = [

](InsertPointTy IP) {

llvm::Error::success();

PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,

llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {

*CS = S.getCapturedStmt(OMPD_parallel);

BodyGenCB = [&,

](InsertPointTy AllocaIP,

InsertPointTy CodeGenIP) {

*

, ParallelRegionBodyStmt, AllocaIP, CodeGenIP,

);

llvm::Error::success();

CGCapturedStmtInfo CGSI(*CS,

);

CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(*

, &CGSI);

llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(

llvm::OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(

OMPBuilder.createParallel(

, AllocaIP, BodyGenCB, PrivCB, FiniCB,

IfCond, NumThreads, ProcBind, S.hasCancel()));

OMPPrivateScope PrivateScope(CGF);

(void)PrivateScope.Privatize();

CGF.

(S.getCapturedStmt(OMPD_parallel)->getCapturedStmt());

OMPTransformDirectiveScopeRAII {

OMPLoopScope *

=

;

CodeGenFunction::CGCapturedStmtInfo *CGSI =

;

CodeGenFunction::CGCapturedStmtRAII *CapInfoRAII =

;

OMPTransformDirectiveScopeRAII(

OMPTransformDirectiveScopeRAII &) =

OMPTransformDirectiveScopeRAII &

operator=(

OMPTransformDirectiveScopeRAII &) =

;

(

*Dir = dyn_cast<OMPLoopBasedDirective>(S)) {

=

OMPLoopScope(CGF, *Dir);

CGSI =

CodeGenFunction::CGCapturedStmtInfo(

);

CapInfoRAII =

CodeGenFunction::CGCapturedStmtRAII(CGF, CGSI);

~OMPTransformDirectiveScopeRAII() {

MaxLevel,

Level = 0) {

assert(Level < MaxLevel &&

);

*SimplifiedS = S->IgnoreContainers();

(

*CS = dyn_cast<CompoundStmt>(SimplifiedS)) {

);

CodeGenFunction::LexicalScope

(CGF, S->getSourceRange());

(

*CurStmt : CS->body())

(CGF, CurStmt, NextLoop, MaxLevel, Level);

(SimplifiedS == NextLoop) {

(

*Dir = dyn_cast<OMPLoopTransformationDirective>(SimplifiedS))

SimplifiedS = Dir->getTransformedStmt();

(

*CanonLoop = dyn_cast<OMPCanonicalLoop>(SimplifiedS))

SimplifiedS = CanonLoop->getLoopStmt();

(

*For = dyn_cast<ForStmt>(SimplifiedS)) {

assert(isa<CXXForRangeStmt>(SimplifiedS) &&

);

*CXXFor = cast<CXXForRangeStmt>(SimplifiedS);

CGF.

(CXXFor->getLoopVarStmt());

S = CXXFor->getBody();

(Level + 1 < MaxLevel) {

(CGF, S, NextLoop, MaxLevel, Level + 1);

RunCleanupsScope BodyScope(*

);

(

*UE :

.updates())

(

*UE :

->updates())

BreakContinueStack.push_back(BreakContinue(

, Continue));

(

*

:

.finals_conditions()) {

OMPPrivateScope InscanScope(*

);

IsInscanRegion = InscanScope.Privatize();

(IsInscanRegion) {

(EKind != OMPD_simd && !

().OpenMPSimd)

.getInnermostCapturedStmt()->getCapturedStmt()->IgnoreContainers();

.getLoopsNumber());

BreakContinueStack.pop_back();

std::unique_ptr<CodeGenFunction::CGCapturedStmtInfo> CSI =

std::make_unique<CodeGenFunction::CGCapturedStmtInfo>(*S);

CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, CSI.get());

llvm::CallInst *

EffectiveArgs.reserve(Args.size() + 1);

llvm::append_range(EffectiveArgs, Args);

EffectiveArgs.push_back(Cap.second);

llvm::CanonicalLoopInfo *

assert(Depth == 1 &&

);

assert(

.size() >= (

)Depth &&

);

*SyntacticalLoop = S->getLoopStmt();

LexicalScope ForScope(*

, S->getSourceRange());

*BodyStmt;

(

*For = dyn_cast<ForStmt>(SyntacticalLoop)) {

(

*InitStmt = For->getInit())

BodyStmt = For->getBody();

}

(

*RangeFor =

dyn_cast<CXXForRangeStmt>(SyntacticalLoop)) {

(

*RangeStmt = RangeFor->getRangeStmt())

(

*BeginStmt = RangeFor->getBeginStmt())

(

*EndStmt = RangeFor->getEndStmt())

(

*LoopVarStmt = RangeFor->getLoopVarStmt())

BodyStmt = RangeFor->getBody();

llvm_unreachable(

);

*DistanceFunc = S->getDistanceFunc();

BodyGen = [&,

](llvm::OpenMPIRBuilder::InsertPointTy CodeGenIP,

llvm::Value *IndVar) {

.restoreIP(CodeGenIP);

*LoopVarRef = S->getLoopVarRef();

RunCleanupsScope BodyScope(*

);

llvm::Error::success();

llvm::CanonicalLoopInfo *CL =

cantFail(OMPBuilder.createCanonicalLoop(

, BodyGen, DistVal));

.restoreIP(CL->getAfterIP());

ForScope.ForceCleanup();

*IncExpr,

&OMPED = cast<OMPExecutableDirective>(S);

*ICS = OMPED.getInnermostCapturedStmt();

llvm::BasicBlock *ExitBlock =

.getBlock();

(RequiresCleanup)

(ExitBlock !=

.getBlock()) {

BreakContinueStack.push_back(BreakContinue(

, Continue));

PostIncGen(*

);

BreakContinueStack.pop_back();

HasLinears =

;

(

*

:

->inits()) {

*VD = cast<VarDecl>(cast<DeclRefExpr>(

)->getDecl());

(

*Ref =

*OrigVD = cast<VarDecl>(Ref->getDecl());

(

*CS = cast_or_null<BinaryOperator>(

->getCalcStep()))

(

*SaveRef = cast<DeclRefExpr>(CS->getLHS())) {

llvm::function_ref<llvm::Value *(

&)> CondGen) {

llvm::BasicBlock *DoneBB =

;

IC =

->varlist_begin();

(

*F :

->finals()) {

(llvm::Value *Cond = CondGen(*

)) {

.CreateCondBr(Cond, ThenBB, DoneBB);

*OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IC)->getDecl());

CodeGenFunction::OMPPrivateScope VarScope(*

);

VarScope.addPrivate(OrigVD, OrigAddr);

(void)VarScope.Privatize();

(

*PostUpdate =

->getPostUpdateExpr())

llvm::APInt ClauseAlignment(64, 0);

(

*AlignmentExpr = Clause->getAlignment()) {

ClauseAlignment = AlignmentCI->getValue();

(

*

: Clause->varlist()) {

llvm::APInt Alignment(ClauseAlignment);

(Alignment == 0) {

assert((Alignment == 0 || Alignment.isPowerOf2()) &&

);

(Alignment != 0) {

I = S.private_counters().begin();

(

*

: S.counters()) {

*VD = cast<VarDecl>(cast<DeclRefExpr>(

)->getDecl());

*PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(*I)->getDecl());

LocalDeclMap.erase(PrivateVD);

(void)LoopScope.addPrivate(VD, VarEmission.getAllocatedAddress());

(void)LoopScope.addPrivate(PrivateVD, VarEmission.getAllocatedAddress());

(!

->getNumForLoops())

(

I = S.getLoopsNumber(),

=

->getLoopNumIterations().size();

*DRE = cast<DeclRefExpr>(

->getLoopCounter(I));

*VD = cast<VarDecl>(DRE->getDecl());

(DRE->refersToEnclosingVariableOrCapture()) {

(void)LoopScope.addPrivate(

*Cond, llvm::BasicBlock *TrueBlock,

llvm::BasicBlock *FalseBlock, uint64_t TrueCount) {

CodeGenFunction::OMPPrivateScope PreCondScope(CGF);

(void)PreCondScope.Privatize();

(

*I : S.inits()) {

CodeGenFunction::OMPMapVars PreCondVars;

(

*

: S.dependent_counters()) {

);

*VD = cast<VarDecl>(cast<DeclRefExpr>(

)->getDecl());

(void)PreCondVars.setVarAddr(CGF, VD, CounterAddr);

(void)PreCondVars.apply(CGF);

(

*

: S.dependent_inits()) {

PreCondVars.restore(CGF);

llvm::DenseSet<const VarDecl *> SIMDLCVs;

*LoopDirective = cast<OMPLoopDirective>(&

);

(

*

: LoopDirective->counters()) {

CurPrivate =

->privates().begin();

(

*

:

->varlist()) {

*VD = cast<VarDecl>(cast<DeclRefExpr>(

)->getDecl());

*PrivateVD =

cast<VarDecl>(cast<DeclRefExpr>(*CurPrivate)->getDecl());

assert(IsRegistered &&

);

*Val = cast<llvm::ConstantInt>(Len.

());

*Val = cast<llvm::ConstantInt>(Len.

());

(

->getKind() == OMPC_ORDER_concurrent)

((EKind == OMPD_simd ||

return C->getModifier() == OMPC_REDUCTION_inscan;

llvm::function_ref<llvm::Value *(

&)> CondGen) {

llvm::BasicBlock *DoneBB =

;

IC =

.counters().begin();

IPC =

.private_counters().begin();

(

*F :

.finals()) {

*OrigVD = cast<VarDecl>(cast<DeclRefExpr>((*IC))->getDecl());

*PrivateVD = cast<VarDecl>(cast<DeclRefExpr>((*IPC))->getDecl());

*CED = dyn_cast<OMPCapturedExprDecl>(OrigVD);

OrigVD->hasGlobalStorage() || CED) {

(llvm::Value *Cond = CondGen(*

)) {

.CreateCondBr(Cond, ThenBB, DoneBB);

OMPPrivateScope VarScope(*

);

VarScope.addPrivate(OrigVD, OrigAddr);

(void)VarScope.Privatize();

CodeGenFunction::JumpDest

) {

VDecl = cast<VarDecl>(Helper->

());

&&ThenGen = [&S, &SimdInitGen, &BodyCodeGen](

&CGF,

CodeGenFunction::OMPLocalDeclMapRAII

(CGF);

CodeGenFunction::OMPLocalDeclMapRAII

(CGF);

*IfCond =

;

(

*

: S.getClausesOfKind<

>()) {

(

->getNameModifier() == OMPD_unknown ||

->getNameModifier() == OMPD_simd)) {

IfCond =

->getCondition();

Action.

(CGF);

OMPLoopScope PreInitScope(CGF, S);

(void)

(CGF, cast<DeclRefExpr>(S.getLowerBoundVariable()));

(void)

(CGF, cast<DeclRefExpr>(S.getUpperBoundVariable()));

llvm::BasicBlock *ContBlock =

;

(CGF, S, S.getPreCond(), ThenBlock, ContBlock,

*IVExpr = S.getIterationVariable();

*IVDecl = cast<VarDecl>(cast<DeclRefExpr>(IVExpr)->getDecl());

(

*LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {

CGF.

(*cast<VarDecl>(LIExpr->getDecl()));

CodeGenFunction::OMPPrivateScope LoopScope(CGF);

CGF, S, CGF.

(S.getIterationVariable()));

(void)LoopScope.Privatize();

S, LoopScope.requiresCleanups(), S.getCond(), S.getInc(),

emitOMPLoopBodyWithStopPoint(CGF, S,

CodeGenFunction::JumpDest());

(HasLastprivateClause)

LoopScope.restoreMap();

(!(isa<OMPSimdlenClause>(

) || isa<OMPSafelenClause>(

) ||

isa<OMPOrderClause>(

) || isa<OMPAlignedClause>(

)))

(

*CanonLoop = dyn_cast<OMPCanonicalLoop>(S.getRawStmt())) {

(

*SyntacticalLoop = CanonLoop->getLoopStmt()) {

(

*SubStmt : SyntacticalLoop->

()) {

(

*CS = dyn_cast<CompoundStmt>(SubStmt)) {

(isa<OMPOrderedDirective>(CSSubStmt)) {

llvm::MapVector<llvm::Value *, llvm::Value *>

llvm::MapVector<llvm::Value *, llvm::Value *> AlignedVars;

llvm::APInt ClauseAlignment(64, 0);

(

*AlignmentExpr = Clause->getAlignment()) {

ClauseAlignment = AlignmentCI->getValue();

(

*

: Clause->varlist()) {

llvm::APInt Alignment(ClauseAlignment);

(Alignment == 0) {

assert((Alignment == 0 || Alignment.isPowerOf2()) &&

);

AlignedVars[PtrValue] = CGF.

.getInt64(Alignment.getSExtValue());

UseOMPIRBuilder =

(UseOMPIRBuilder) {

(UseOMPIRBuilder) {

llvm::MapVector<llvm::Value *, llvm::Value *> AlignedVars =

*Inner = S.getRawStmt();

llvm::CanonicalLoopInfo *CLI =

CGF.EmitOMPCollapsedCanonicalLoopNest(Inner, 1);

llvm::OpenMPIRBuilder &OMPBuilder =

llvm::ConstantInt *Simdlen =

;

*Val = cast<llvm::ConstantInt>(Len.

());

llvm::ConstantInt *Safelen =

;

*Val = cast<llvm::ConstantInt>(Len.

());

llvm::omp::OrderKind Order = llvm::omp::OrderKind::OMP_ORDER_unknown;

(

->getKind() == OpenMPOrderClauseKind::OMPC_ORDER_concurrent) {

Order = llvm::omp::OrderKind::OMP_ORDER_concurrent;

OMPBuilder.applySimd(CLI, AlignedVars,

, Order, Simdlen, Safelen);

OMPLexicalScope

(CGF, S, OMPD_unknown);

CodeGenFunction::ParentLoopDirectiveForScanRegion ScanRegion(CGF, S);

OMPLexicalScope

(CGF, S, OMPD_unknown);

OMPTransformDirectiveScopeRAII TileScope(*

, &S);

OMPTransformDirectiveScopeRAII ReverseScope(*

, &S);

OMPTransformDirectiveScopeRAII InterchangeScope(*

, &S);

(UseOMPIRBuilder) {

*Inner = S.getRawStmt();

llvm::CanonicalLoopInfo *UnrolledCLI =

;

OMPBuilder.unrollLoopFull(DL, CLI);

}

(

*PartialClause = S.getSingleClause<

>()) {

(

*FactorExpr = PartialClause->getFactor()) {

Factor = FactorExpr->EvaluateKnownConstInt(

()).getZExtValue();

assert(Factor >= 1 &&

);

OMPBuilder.unrollLoopPartial(DL, CLI, Factor,

NeedsUnrolledCLI ? &UnrolledCLI :

);

OMPBuilder.unrollLoopHeuristic(DL, CLI);

assert((!NeedsUnrolledCLI || UnrolledCLI) &&

);

}

(

*PartialClause = S.getSingleClause<

>()) {

(

*FactorExpr = PartialClause->getFactor()) {

FactorExpr->EvaluateKnownConstInt(

()).getZExtValue();

assert(Factor >= 1 &&

);

CodeGenFunction::EmitOMPOuterLoop(

CodeGenFunction::OMPPrivateScope &LoopScope,

CodeGenFunction::OMPLoopArguments &LoopArgs,

*IVExpr = S.getIterationVariable();

llvm::Value *BoolCondVal =

;

(!DynamicOrOrdered) {

RT.

(*

, S.getBeginLoc(), IVSize, IVSigned, LoopArgs.IL,

LoopArgs.LB, LoopArgs.UB, LoopArgs.ST);

llvm::BasicBlock *ExitBlock =

.getBlock();

(LoopScope.requiresCleanups())

.CreateCondBr(BoolCondVal, LoopBody, ExitBlock);

(ExitBlock !=

.getBlock()) {

(DynamicOrOrdered)

BreakContinueStack.push_back(BreakContinue(

, Continue));

(

->getKind() == OMPC_ORDER_concurrent)

[&S, &LoopArgs,

, &CodeGenLoop, IVSize, IVSigned, &CodeGenOrdered,

CGF.EmitOMPInnerLoop(

S, LoopScope.requiresCleanups(), LoopArgs.Cond, LoopArgs.IncExpr,

CodeGenLoop(CGF, S, LoopExit);

CodeGenOrdered(CGF, Loc, IVSize, IVSigned);

BreakContinueStack.pop_back();

(!DynamicOrOrdered) {

&&CodeGen = [DynamicOrOrdered, &S, &LoopArgs](

&CGF) {

(!DynamicOrOrdered)

CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),

OMPCancelStack.emitExit(*

, EKind, CodeGen);

CodeGenFunction::EmitOMPForOuterLoop(

OMPLoopArguments &LoopArgs,

DynamicOrOrdered = Ordered || RT.

(ScheduleKind.

);

LoopArgs.Chunk !=

)) &&

);

*IVExpr = S.getIterationVariable();

(DynamicOrOrdered) {

std::pair<llvm::Value *, llvm::Value *> DispatchBounds =

CGDispatchBounds(*

, S, LoopArgs.LB, LoopArgs.UB);

llvm::Value *LBVal = DispatchBounds.first;

llvm::Value *UBVal = DispatchBounds.second;

IVSigned, Ordered, DipatchRTInputValues);

IVSize, IVSigned, Ordered, LoopArgs.IL, LoopArgs.LB, LoopArgs.UB,

LoopArgs.ST, LoopArgs.Chunk);

IVSize,

IVSigned) {

OMPLoopArguments OuterLoopArgs(LoopArgs.LB, LoopArgs.UB, LoopArgs.ST,

LoopArgs.IL, LoopArgs.Chunk, LoopArgs.EUB);

OuterLoopArgs.IncExpr = S.getInc();

OuterLoopArgs.Init = S.getInit();

OuterLoopArgs.Cond = S.getCond();

OuterLoopArgs.NextLB = S.getNextLowerBound();

OuterLoopArgs.NextUB = S.getNextUpperBound();

OuterLoopArgs.DKind = LoopArgs.DKind;

EmitOMPOuterLoop(DynamicOrOrdered, IsMonotonic, S, LoopScope, OuterLoopArgs,

(DynamicOrOrdered) {

IVSize,

IVSigned) {}

CodeGenFunction::EmitOMPDistributeOuterLoop(

OMPPrivateScope &LoopScope,

OMPLoopArguments &LoopArgs,

*IVExpr = S.getIterationVariable();

IVSize, IVSigned,

, LoopArgs.IL, LoopArgs.LB,

LoopArgs.UB, LoopArgs.ST, LoopArgs.Chunk);

IncExpr = S.getDistInc();

IncExpr = S.getInc();

OMPLoopArguments OuterLoopArgs;

OuterLoopArgs.LB = LoopArgs.LB;

OuterLoopArgs.UB = LoopArgs.UB;

OuterLoopArgs.ST = LoopArgs.ST;

OuterLoopArgs.IL = LoopArgs.IL;

OuterLoopArgs.Chunk = LoopArgs.Chunk;

? S.getCombinedEnsureUpperBound()

: S.getEnsureUpperBound();

OuterLoopArgs.IncExpr = IncExpr;

? S.getCombinedInit()

? S.getCombinedCond()

? S.getCombinedNextLowerBound()

: S.getNextLowerBound();

? S.getCombinedNextUpperBound()

: S.getNextUpperBound();

OuterLoopArgs.DKind = OMPD_distribute;

EmitOMPOuterLoop(

,

, S,

LoopScope, OuterLoopArgs, CodeGenLoopContent,

std::pair<LValue, LValue>

std::pair<llvm::Value *, llvm::Value *>

llvm::Value *LBVal =

llvm::Value *UBVal =

{LBVal, UBVal};

&Dir = cast<OMPLoopDirective>(S);

CGF.

(cast<DeclRefExpr>(Dir.getCombinedLowerBoundVariable()));

llvm::Value *LBCast = CGF.

.CreateIntCast(

CapturedVars.push_back(LBCast);

CGF.

(cast<DeclRefExpr>(Dir.getCombinedUpperBoundVariable()));

llvm::Value *UBCast = CGF.

.CreateIntCast(

CapturedVars.push_back(UBCast);

CodeGenFunction::JumpDest

) {

HasCancel =

;

(

*

= dyn_cast<OMPTeamsDistributeParallelForDirective>(&S))

HasCancel =

->hasCancel();

(

*

= dyn_cast<OMPDistributeParallelForDirective>(&S))

HasCancel =

->hasCancel();

(

*

=

dyn_cast<OMPTargetTeamsDistributeParallelForDirective>(&S))

HasCancel =

->hasCancel();

CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, EKind, HasCancel);

CGInlinedWorksharingLoop,

OMPLexicalScope

(*

, S, OMPD_parallel);

OMPLexicalScope

(*

, S, OMPD_parallel);

OMPLexicalScope

(*

, S, OMPD_unknown);

llvm::Function *

;

llvm::Constant *Addr;

S, ParentName, Fn, Addr,

, CodeGen);

assert(Fn && Addr &&

);

ScheduleKindModifiersTy {

*IVExpr = cast<DeclRefExpr>(S.getIterationVariable());

*IVDecl = cast<VarDecl>(IVExpr->getDecl());

(

*LIExpr = dyn_cast<DeclRefExpr>(S.getLastIteration())) {

HasLastprivateClause;

OMPLoopScope PreInitScope(*

, S);

llvm::BasicBlock *ContBlock =

;

(*

, S, S.getPreCond(), ThenBlock, ContBlock,

RunCleanupsScope DoacrossCleanupScope(*

);

Ordered =

;

(

*OrderedClause = S.getSingleClause<

>()) {

(OrderedClause->getNumForLoops())

llvm::DenseSet<const Expr *> EmittedFinals;

std::pair<LValue, LValue> Bounds = CodeGenLoopBounds(*

, S);

LB = Bounds.first;

UB = Bounds.second;

OMPPrivateScope LoopScope(*

);

*

, S.getBeginLoc(), OMPD_unknown,

,

*

, S,

(S.getIterationVariable()));

(void)LoopScope.Privatize();

*ChunkExpr =

;

ScheduleKind.

=

->getScheduleKind();

ScheduleKind.

=

->getFirstScheduleModifier();

ScheduleKind.

=

->getSecondScheduleModifier();

ChunkExpr =

->getChunkSize();

*

, S, ScheduleKind.

, ChunkExpr);

HasChunkSizeOne =

;

llvm::Value *Chunk =

;

S.getIterationVariable()->getType(),

llvm::APSInt EvaluatedChunk =

.Val.getInt();

HasChunkSizeOne = (EvaluatedChunk.getLimitedValue() == 1);

StaticChunkedOne =

Chunk !=

) &&

(ScheduleKind.

== OMPC_SCHEDULE_static &&

!(ScheduleKind.

== OMPC_SCHEDULE_MODIFIER_nonmonotonic ||

ScheduleKind.

== OMPC_SCHEDULE_MODIFIER_nonmonotonic)) ||

ScheduleKind.

== OMPC_SCHEDULE_MODIFIER_monotonic ||

ScheduleKind.

== OMPC_SCHEDULE_MODIFIER_monotonic;

Chunk !=

) ||

StaticChunkedOne) &&

if (C->getKind() == OMPC_ORDER_concurrent)

CGF.LoopStack.setParallel(

true);

[IVSize, IVSigned, Ordered, IL, LB, UB, ST, StaticChunkedOne, Chunk,

&S, ScheduleKind,

, EKind,

IVSize, IVSigned, Ordered, IL.getAddress(), LB.getAddress(),

UB.getAddress(), ST.getAddress(),

StaticChunkedOne ? Chunk :

);

CGF.CGM.getOpenMPRuntime().emitForStaticInit(

CGF, S.getBeginLoc(), EKind, ScheduleKind, StaticInit);

(!StaticChunkedOne)

CGF.EmitIgnoredExpr(S.getEnsureUpperBound());

CGF.EmitIgnoredExpr(S.getInit());

CGF.EmitOMPInnerLoop(

S, LoopScope.requiresCleanups(),

StaticChunkedOne ? S.getCombinedParForInDistCond()

StaticChunkedOne ? S.getDistInc() : S.getInc(),

emitOMPLoopBodyWithStopPoint(CGF, S, LoopExit);

CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),

OMPCancelStack.emitExit(*

, EKind, CodeGen);

OMPLoopArguments LoopArguments(LB.getAddress(), UB.getAddress(),

ST.getAddress(), IL.getAddress(), Chunk,

LoopArguments.DKind = OMPD_for;

EmitOMPForOuterLoop(ScheduleKind, IsMonotonic, S, LoopScope, Ordered,

LoopArguments, CGDispatchBounds);

CGF.

.CreateIsNotNull(

?

OMPD_parallel_for_simd

CGF.

.CreateIsNotNull(

(HasLastprivateClause)

LoopScope.restoreMap();

CGF.

.CreateIsNotNull(

DoacrossCleanupScope.ForceCleanup();

HasLastprivateClause;

std::pair<LValue, LValue>

&LS = cast<OMPLoopDirective>(S);

std::pair<llvm::Value *, llvm::Value *>

&LS = cast<OMPLoopDirective>(S);

*IVExpr = LS.getIterationVariable();

llvm::Value *LBVal = CGF.

.getIntN(IVSize, 0);

llvm::Value *UBVal = CGF.

(LS.getLastIteration());

{LBVal, UBVal};

llvm::function_ref<llvm::Value *(

&)> NumIteratorsGen) {

llvm::Value *OMPScanNumIterations = CGF.

.CreateIntCast(

NumIteratorsGen(CGF), CGF.

,

);

assert(

->getModifier() == OMPC_REDUCTION_inscan &&

);

Shareds.append(

->varlist_begin(),

->varlist_end());

Privates.append(

->privates().begin(),

->privates().end());

ReductionOps.append(

->reduction_ops().begin(),

->reduction_ops().end());

CopyArrayTemps.append(

->copy_array_temps().begin(),

->copy_array_temps().end());

*ITA = CopyArrayTemps.begin();

(

*IRef : Privates) {

*PrivateVD = cast<VarDecl>(cast<DeclRefExpr>(IRef)->getDecl());

(PrivateVD->getType()->isVariablyModifiedType()) {

CodeGenFunction::OpaqueValueMapping DimMapping(

cast<OpaqueValueExpr>(

cast<VariableArrayType>((*ITA)->getType()->getAsArrayTypeUnsafe())

CGF.

(*cast<VarDecl>(cast<DeclRefExpr>(*ITA)->getDecl()));

llvm::function_ref<llvm::Value *(

&)> NumIteratorsGen) {

llvm::Value *OMPScanNumIterations = CGF.

.CreateIntCast(

NumIteratorsGen(CGF), CGF.

,

);

assert(

->getModifier() == OMPC_REDUCTION_inscan &&

);

Shareds.append(

->varlist_begin(),

->varlist_end());

LHSs.append(

->lhs_exprs().begin(),

->lhs_exprs().end());

RHSs.append(

->rhs_exprs().begin(),

->rhs_exprs().end());

Privates.append(

->privates().begin(),

->privates().end());

CopyOps.append(

->copy_ops().begin(),

->copy_ops().end());

CopyArrayElems.append(

->copy_array_elems().begin(),

->copy_array_elems().end());

llvm::Value *OMPLast = CGF.

.CreateNSWSub(

OMPScanNumIterations,

llvm::ConstantInt::get(CGF.

, 1,

));

(

I = 0,

= CopyArrayElems.size(); I <

; ++I) {

*PrivateExpr = Privates[I];

*OrigExpr = Shareds[I];

*CopyArrayElem = CopyArrayElems[I];

CodeGenFunction::OpaqueValueMapping IdxMapping(

cast<OpaqueValueExpr>(

cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),

cast<VarDecl>(cast<DeclRefExpr>(LHSs[I])->getDecl()),

cast<VarDecl>(cast<DeclRefExpr>(RHSs[I])->getDecl()), CopyOps[I]);

llvm::function_ref<llvm::Value *(

&)> NumIteratorsGen,

llvm::Value *OMPScanNumIterations = CGF.

.CreateIntCast(

NumIteratorsGen(CGF), CGF.

,

);

assert(

->getModifier() == OMPC_REDUCTION_inscan &&

);

Privates.append(

->privates().begin(),

->privates().end());

ReductionOps.append(

->reduction_ops().begin(),

->reduction_ops().end());

LHSs.append(

->lhs_exprs().begin(),

->lhs_exprs().end());

RHSs.append(

->rhs_exprs().begin(),

->rhs_exprs().end());

CopyArrayElems.append(

->copy_array_elems().begin(),

->copy_array_elems().end());

CodeGenFunction::ParentLoopDirectiveForScanRegion ScanRegion(CGF, S);

CodeGenFunction::OMPLocalDeclMapRAII

(CGF);

&&CodeGen = [&S, OMPScanNumIterations, &LHSs, &RHSs, &CopyArrayElems,

llvm::BasicBlock *InputBB = CGF.Builder.GetInsertBlock();

llvm::BasicBlock *LoopBB = CGF.createBasicBlock(

);

llvm::BasicBlock *ExitBB = CGF.createBasicBlock(

);

CGF.CGM.getIntrinsic(llvm::Intrinsic::log2, CGF.DoubleTy);

CGF.Builder.CreateUIToFP(OMPScanNumIterations, CGF.DoubleTy);

llvm::Value *LogVal = CGF.EmitNounwindRuntimeCall(F, Arg);

F = CGF.CGM.getIntrinsic(llvm::Intrinsic::ceil, CGF.DoubleTy);

LogVal = CGF.EmitNounwindRuntimeCall(F, LogVal);

LogVal = CGF.Builder.CreateFPToUI(LogVal, CGF.IntTy);

llvm::Value *NMin1 = CGF.Builder.CreateNUWSub(

OMPScanNumIterations, llvm::ConstantInt::get(CGF.SizeTy, 1));

CGF.EmitBlock(LoopBB);

*Counter = CGF.Builder.CreatePHI(CGF.IntTy, 2);

*Pow2K = CGF.Builder.CreatePHI(CGF.SizeTy, 2);

Counter->addIncoming(llvm::ConstantInt::get(CGF.IntTy, 0), InputBB);

Pow2K->addIncoming(llvm::ConstantInt::get(CGF.SizeTy, 1), InputBB);

llvm::BasicBlock *InnerLoopBB =

CGF.createBasicBlock(

);

llvm::BasicBlock *InnerExitBB =

CGF.createBasicBlock(

);

llvm::Value *CmpI = CGF.Builder.CreateICmpUGE(NMin1, Pow2K);

CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);

CGF.EmitBlock(InnerLoopBB);

*IVal = CGF.Builder.CreatePHI(CGF.SizeTy, 2);

IVal->addIncoming(NMin1, LoopBB);

CodeGenFunction::OMPPrivateScope PrivScope(CGF);

*ILHS = LHSs.begin();

*IRHS = RHSs.begin();

(

*CopyArrayElem : CopyArrayElems) {

*LHSVD = cast<VarDecl>(cast<DeclRefExpr>(*ILHS)->getDecl());

*RHSVD = cast<VarDecl>(cast<DeclRefExpr>(*IRHS)->getDecl());

CodeGenFunction::OpaqueValueMapping IdxMapping(

cast<OpaqueValueExpr>(

cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),

LHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress();

PrivScope.addPrivate(LHSVD, LHSAddr);

llvm::Value *OffsetIVal = CGF.Builder.CreateNUWSub(IVal, Pow2K);

CodeGenFunction::OpaqueValueMapping IdxMapping(

cast<OpaqueValueExpr>(

cast<ArraySubscriptExpr>(CopyArrayElem)->getIdx()),

RHSAddr = CGF.EmitLValue(CopyArrayElem).getAddress();

PrivScope.addPrivate(RHSVD, RHSAddr);

PrivScope.Privatize();

CGF.CGM.getOpenMPRuntime().emitReduction(

CGF, S.getEndLoc(), Privates, LHSs, RHSs, ReductionOps,

{

true,

true, OMPD_unknown});

llvm::Value *NextIVal =

CGF.Builder.CreateNUWSub(IVal, llvm::ConstantInt::get(CGF.SizeTy, 1));

IVal->addIncoming(NextIVal, CGF.Builder.GetInsertBlock());

CmpI = CGF.Builder.CreateICmpUGE(NextIVal, Pow2K);

CGF.Builder.CreateCondBr(CmpI, InnerLoopBB, InnerExitBB);

CGF.EmitBlock(InnerExitBB);

CGF.Builder.CreateNUWAdd(Counter, llvm::ConstantInt::get(CGF.IntTy, 1));

Counter->addIncoming(Next, CGF.Builder.GetInsertBlock());

llvm::Value *NextPow2K =

CGF.Builder.CreateShl(Pow2K, 1,

,

);

Pow2K->addIncoming(NextPow2K, CGF.Builder.GetInsertBlock());

llvm::Value *Cmp = CGF.Builder.CreateICmpNE(Next, LogVal);

CGF.Builder.CreateCondBr(Cmp, LoopBB, ExitBB);

CGF.EmitBlock(ExitBB);

CGF.CGM.getOpenMPRuntime().emitMasterRegion(CGF, CodeGen, S.getBeginLoc());

CGF.CGM.getOpenMPRuntime().emitBarrierCall(

CGF, S.getBeginLoc(), OMPD_unknown,

,

CGF.OMPFirstScanLoop =

;

HasLastprivates;

return C->getModifier() == OMPC_REDUCTION_inscan;

CodeGenFunction::OMPLocalDeclMapRAII

(CGF);

OMPLoopScope LoopScope(CGF, S);

&&FirstGen = [&S, HasCancel, EKind](

&CGF) {

CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, EKind, HasCancel);

&&SecondGen = [&S, HasCancel, EKind,

CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, EKind, HasCancel);

CodeGenFunction::OMPCancelStackRAII CancelRegion(CGF, EKind, HasCancel);

HasLastprivates;

(isa<OMPNowaitClause, OMPBindClause>(

))

(

*SC = dyn_cast<OMPScheduleClause>(

)) {

(SC->getScheduleKind()) {

OMPC_SCHEDULE_auto:

OMPC_SCHEDULE_dynamic:

OMPC_SCHEDULE_runtime:

OMPC_SCHEDULE_guided:

OMPC_SCHEDULE_static:

llvm::omp::ScheduleKind

(ScheduleClauseKind) {

llvm::omp::OMP_SCHEDULE_Default;

OMPC_SCHEDULE_auto:

llvm::omp::OMP_SCHEDULE_Auto;

OMPC_SCHEDULE_dynamic:

llvm::omp::OMP_SCHEDULE_Dynamic;

OMPC_SCHEDULE_guided:

llvm::omp::OMP_SCHEDULE_Guided;

OMPC_SCHEDULE_runtime:

llvm::omp::OMP_SCHEDULE_Runtime;

OMPC_SCHEDULE_static:

llvm::omp::OMP_SCHEDULE_Static;

llvm_unreachable(

);

HasLastprivates =

;

&&CodeGen = [&S, &

, HasCancel, &HasLastprivates,

(UseOMPIRBuilder) {

llvm::omp::ScheduleKind SchedKind = llvm::omp::OMP_SCHEDULE_Default;

llvm::Value *ChunkSize =

;

(

*ChunkSizeExpr = SchedClause->getChunkSize())

*Inner = S.getRawStmt();

llvm::CanonicalLoopInfo *CLI =

llvm::OpenMPIRBuilder &OMPBuilder =

llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(

cantFail(OMPBuilder.applyWorkshareLoop(

CGF.

.getCurrentDebugLocation(), CLI, AllocaIP, NeedsBarrier,

SchedKind, ChunkSize,

,

OMPLexicalScope

(CGF, S, OMPD_unknown);

(!UseOMPIRBuilder) {

HasLastprivates =

;

OMPLexicalScope

(*

, S, OMPD_unknown);

llvm::Value *

=

) {

*

= S.getInnermostCapturedStmt()->getCapturedStmt();

*CS = dyn_cast<CompoundStmt>(

);

HasLastprivates =

;

.getIntTypeForBitwidth(

32,

1);

CGF.Builder.getInt32(0));

llvm::ConstantInt *GlobalUBVal = CS !=

? CGF.Builder.getInt32(CS->size() - 1)

: CGF.Builder.getInt32(0);

CGF.Builder.getInt32(1));

CGF.Builder.getInt32(0));

CodeGenFunction::OpaqueValueMapping OpaqueIV(CGF, &IVRefExpr, IV);

CodeGenFunction::OpaqueValueMapping OpaqueUB(CGF, &UBRefExpr, UB);

llvm::BasicBlock *ExitBB = CGF.createBasicBlock(

);

CGF.Builder.CreateSwitch(CGF.EmitLoadOfScalar(IV, S.getBeginLoc()),

ExitBB, CS ==

? 1 : CS->size());

CaseNumber = 0;

CaseBB = CGF.createBasicBlock(

);

CGF.EmitBlock(CaseBB);

->addCase(CGF.Builder.getInt32(CaseNumber), CaseBB);

CGF.EmitStmt(SubStmt);

CGF.EmitBranch(ExitBB);

llvm::BasicBlock *CaseBB = CGF.createBasicBlock(

);

CGF.EmitBlock(CaseBB);

->addCase(CGF.Builder.getInt32(0), CaseBB);

CGF.EmitBranch(ExitBB);

CGF.EmitBlock(ExitBB,

);

CodeGenFunction::OMPPrivateScope LoopScope(CGF);

(CGF.EmitOMPFirstprivateClause(S, LoopScope)) {

CGF.CGM.getOpenMPRuntime().emitBarrierCall(

CGF, S.getBeginLoc(), OMPD_unknown,

,

CGF.EmitOMPPrivateClause(S, LoopScope);

HasLastprivates = CGF.EmitOMPLastprivateClauseInit(S, LoopScope);

CGF.EmitOMPReductionClauseInit(S, LoopScope);

(void)LoopScope.Privatize();

CGF.CGM.getOpenMPRuntime().adjustTargetSpecificDataForLambdas(CGF, S);

ScheduleKind.

= OMPC_SCHEDULE_static;

CGF.CGM.getOpenMPRuntime().emitForStaticInit(CGF, S.getBeginLoc(), EKind,

ScheduleKind, StaticInit);

llvm::Value *UBVal = CGF.EmitLoadOfScalar(UB, S.getBeginLoc());

llvm::Value *MinUBGlobalUB = CGF.Builder.CreateSelect(

CGF.Builder.CreateICmpSLT(UBVal, GlobalUBVal), UBVal, GlobalUBVal);

CGF.EmitStoreOfScalar(MinUBGlobalUB, UB);

CGF.EmitStoreOfScalar(CGF.EmitLoadOfScalar(LB, S.getBeginLoc()), IV);

CGF.EmitOMPInnerLoop(S,

, Cond, Inc, BodyGen,

CGF.CGM.getOpenMPRuntime().emitForStaticFinish(CGF, S.getEndLoc(),

CGF.OMPCancelStack.emitExit(CGF, EKind, CodeGen);

CGF.EmitOMPReductionClauseFinal(S,

OMPD_parallel);

CGF.

.CreateIsNotNull(

(HasLastprivates)

CGF.

.CreateIsNotNull(

HasCancel =

;

(

*OSD = dyn_cast<OMPSectionsDirective>(&S))

HasCancel = OSD->hasCancel();

(

*OPSD = dyn_cast<OMPParallelSectionsDirective>(&S))

HasCancel = OPSD->hasCancel();

OMPCancelStackRAII CancelRegion(*

, EKind, HasCancel);

OMPPrivateScope PrivateScope(CGF);

(void)PrivateScope.Privatize();

CGF.

(S.getInnermostCapturedStmt()->getCapturedStmt());

OMPLexicalScope

(*

, S, OMPD_unknown);

InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;

BodyGenCallbackTy = llvm::OpenMPIRBuilder::StorableBodyGenCallbackTy;

FiniCB = [

](InsertPointTy IP) {

llvm::Error::success();

*ICS = S.getInnermostCapturedStmt();

*

= S.getInnermostCapturedStmt()->getCapturedStmt();

*CS = dyn_cast<CompoundStmt>(

);

SectionCB = [

, SubStmt](InsertPointTy AllocaIP,

InsertPointTy CodeGenIP) {

*

, SubStmt, AllocaIP, CodeGenIP,

);

llvm::Error::success();

SectionCBVector.push_back(SectionCB);

SectionCB = [

,

](InsertPointTy AllocaIP,

InsertPointTy CodeGenIP) {

*

,

, AllocaIP, CodeGenIP,

);

llvm::Error::success();

SectionCBVector.push_back(SectionCB);

PrivCB = [](InsertPointTy AllocaIP, InsertPointTy CodeGenIP,

llvm::Value &, llvm::Value &Val, llvm::Value *&ReplVal) {

CGCapturedStmtInfo CGSI(*ICS,

);

CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(*

, &CGSI);

llvm::OpenMPIRBuilder::InsertPointTy AllocaIP(

llvm::OpenMPIRBuilder::InsertPointTy AfterIP =

cantFail(OMPBuilder.createSections(

, AllocaIP, SectionCBVector, PrivCB, FiniCB, S.hasCancel(),

OMPLexicalScope

(*

, S, OMPD_unknown);

InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;

*SectionRegionBodyStmt = S.getAssociatedStmt();

FiniCB = [

](InsertPointTy IP) {

llvm::Error::success();

BodyGenCB = [SectionRegionBodyStmt,

](InsertPointTy AllocaIP,

InsertPointTy CodeGenIP) {

*

, SectionRegionBodyStmt, AllocaIP, CodeGenIP,

);

llvm::Error::success();

LexicalScope

(*

, S.getSourceRange());

llvm::OpenMPIRBuilder::InsertPointTy AfterIP =

cantFail(OMPBuilder.createSection(

, BodyGenCB, FiniCB));

LexicalScope

(*

, S.getSourceRange());

CopyprivateVars.append(

->varlist_begin(),

->varlist_end());

DestExprs.append(

->destination_exprs().begin(),

->destination_exprs().end());

SrcExprs.append(

->source_exprs().begin(),

->source_exprs().end());

AssignmentOps.append(

->assignment_ops().begin(),

->assignment_ops().end());

CGF.

(S.getInnermostCapturedStmt()->getCapturedStmt());

OMPLexicalScope

(*

, S, OMPD_unknown);

CopyprivateVars, DestExprs,

SrcExprs, AssignmentOps);

(!S.getSingleClause<

>() && CopyprivateVars.empty()) {

*

, S.getBeginLoc(),

InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;

*MasterRegionBodyStmt = S.getAssociatedStmt();

FiniCB = [

](InsertPointTy IP) {

llvm::Error::success();

BodyGenCB = [MasterRegionBodyStmt,

](InsertPointTy AllocaIP,

InsertPointTy CodeGenIP) {

*

, MasterRegionBodyStmt, AllocaIP, CodeGenIP,

);

llvm::Error::success();

LexicalScope

(*

, S.getSourceRange());

llvm::OpenMPIRBuilder::InsertPointTy AfterIP =

cantFail(OMPBuilder.createMaster(

, BodyGenCB, FiniCB));

LexicalScope

(*

, S.getSourceRange());

*Filter =

;

(

*FilterClause = S.getSingleClause<

>())

Filter = FilterClause->getThreadID();

InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;

*MaskedRegionBodyStmt = S.getAssociatedStmt();

(

*FilterClause = S.getSingleClause<

>())

= FilterClause->getThreadID();

llvm::Value *FilterVal =

FiniCB = [

](InsertPointTy IP) {

llvm::Error::success();

BodyGenCB = [MaskedRegionBodyStmt,

](InsertPointTy AllocaIP,

InsertPointTy CodeGenIP) {

*

, MaskedRegionBodyStmt, AllocaIP, CodeGenIP,

);

llvm::Error::success();

LexicalScope

(*

, S.getSourceRange());

llvm::OpenMPIRBuilder::InsertPointTy AfterIP = cantFail(

OMPBuilder.createMasked(

, BodyGenCB, FiniCB, FilterVal));

LexicalScope

(*

, S.getSourceRange());

InsertPointTy = llvm::OpenMPIRBuilder::InsertPointTy;

*CriticalRegionBodyStmt = S.getAssociatedStmt();

*Hint =

;

(

*HintClause = S.getSingleClause<

>())

Hint = HintClause->getHint();

llvm::Value *HintInst =

;

FiniCB = [

](InsertPointTy IP) {

llvm::Error::success();

BodyGenCB = [CriticalRegionBodyStmt,

](InsertPointTy AllocaIP,

InsertPointTy CodeGenIP) {

*

, CriticalRegionBodyStmt, AllocaIP, CodeGenIP,

);

llvm::Error::success();

LexicalScope

(*

, S.getSourceRange());

llvm::OpenMPIRBuilder::InsertPointTy AfterIP =

cantFail(OMPBuilder.createCritical(

, BodyGenCB, FiniCB,

S.getDirectiveName().getAsString(),

CGF.

(S.getAssociatedStmt());

*Hint =

;

(

*HintClause = S.getSingleClause<

>())

Hint = HintClause->getHint();

LexicalScope

(*

, S.getSourceRange());

S.getDirectiveName().getAsString(),

CodeGen, S.getBeginLoc(), Hint);

CodeGenFunction::OMPLocalDeclMapRAII

(CGF);

CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI);

OMPLoopScope LoopScope(CGF, S);

return C->getModifier() == OMPC_REDUCTION_inscan;

CodeGenFunction::OMPLocalDeclMapRAII

(CGF);

CodeGenFunction::CGCapturedStmtRAII CapInfoRAII(CGF, &CGSI);

OMPLoopScope LoopScope(CGF, S);

return C->getModifier() == OMPC_REDUCTION_inscan;

OMPPrivateScope PrivateScope(CGF);

(void)PrivateScope.Privatize();

OMPPrivateScope PrivateScope(CGF);

(void)PrivateScope.Privatize();

CGF.EmitSections(S);

CheckVarsEscapingUntiedTaskDeclContext final

CheckVarsEscapingUntiedTaskDeclContext() =

;

~CheckVarsEscapingUntiedTaskDeclContext() =

;

VisitDeclStmt(

*S) {

(

*

: S->decls()) {

(

*VD = dyn_cast_or_null<VarDecl>(

))

PrivateDecls.push_back(VD);

VisitBlockExpr(

*) {}

VisitStmt(

*S) {

(

*Child : S->children())

OmpAllMemory =

;

return C->getDependencyKind() == OMPC_DEPEND_outallmemory ||

C->getDependencyKind() == OMPC_DEPEND_inoutallmemory;

OmpAllMemory =

;

.Dependences.emplace_back(OMPC_DEPEND_outallmemory,

DD.

.push_back(

);

(Kind == OMPC_DEPEND_outallmemory || Kind == OMPC_DEPEND_inoutallmemory)

(OmpAllMemory && (Kind == OMPC_DEPEND_out || Kind == OMPC_DEPEND_inout))

.Dependences.emplace_back(

->getDependencyKind(),

->getModifier());

DD.

.append(

->varlist_begin(),

->varlist_end());

*CS = S.getCapturedStmt(CapturedRegion);

PartId = std::next(I);

TaskT = std::next(I, 4);

(

*Clause = S.getSingleClause<

>()) {

*Cond = Clause->getCondition();

.Final.setInt(CondConstant);

.Final.setInt(

);

*Prio = Clause->getPriority();

.Priority.setInt(

);

llvm::DenseSet<const VarDecl *> EmittedAsPrivate;

IRef =

->varlist_begin();

(

*IInit :

->private_copies()) {

*OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());

(EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {

.PrivateVars.push_back(*IRef);

.PrivateCopies.push_back(IInit);

EmittedAsPrivate.clear();

IRef =

->varlist_begin();

IElemInitRef =

->inits().begin();

(

*IInit :

->private_copies()) {

*OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());

(EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {

.FirstprivateVars.push_back(*IRef);

.FirstprivateCopies.push_back(IInit);

.FirstprivateInits.push_back(*IElemInitRef);

llvm::MapVector<const VarDecl *, const DeclRefExpr *> LastprivateDstsOrigs;

IRef =

->varlist_begin();

=

->destination_exprs().begin();

(

*IInit :

->private_copies()) {

*OrigVD = cast<VarDecl>(cast<DeclRefExpr>(*IRef)->getDecl());

(EmittedAsPrivate.insert(OrigVD->getCanonicalDecl()).second) {

.LastprivateVars.push_back(*IRef);

.LastprivateCopies.push_back(IInit);

LastprivateDstsOrigs.insert(

std::make_pair(cast<VarDecl>(cast<DeclRefExpr>(*ID)->getDecl()),

cast<DeclRefExpr>(*IRef)));

.ReductionVars.append(

->varlist_begin(),

->varlist_end());

.ReductionOrigs.append(

->varlist_begin(),

->varlist_end());

.ReductionCopies.append(

->privates().begin(),

->privates().end());

.ReductionOps.append(

->reduction_ops().begin(),

->reduction_ops().end());

LHSs.append(

->lhs_exprs().begin(),

->lhs_exprs().end());

RHSs.append(

->rhs_exprs().begin(),

->rhs_exprs().end());

*

, S.getBeginLoc(), LHSs, RHSs,

);

CheckVarsEscapingUntiedTaskDeclContext Checker;

Checker.Visit(S.getInnermostCapturedStmt()->getCapturedStmt());

.PrivateLocals.append(Checker.getPrivateDecls().begin(),

Checker.getPrivateDecls().end());

&&CodeGen = [&

, &S, CS, &BodyGen, &LastprivateDstsOrigs,

llvm::MapVector<CanonicalDeclPtr<const VarDecl>,

std::pair<Address, Address>>

OMPPrivateScope

(CGF);

(

*DI = CGF.getDebugInfo()) {

llvm::SmallDenseMap<const VarDecl *, FieldDecl *> CaptureFields =

CGF.CapturedStmtInfo->getCaptureFields();

llvm::Value *ContextValue = CGF.CapturedStmtInfo->getContextValue();

(CaptureFields.size() && ContextValue) {

CharWidth = CGF.getContext().getCharWidth();

(

It = CaptureFields.begin(); It != CaptureFields.end(); ++It) {

*SharedVar = It->first;

CGF.getContext().getASTRecordLayout(CaptureRecord);

Layout.

(It->second->getFieldIndex()) / CharWidth;

(CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())

(void)DI->EmitDeclareOfAutoVariable(SharedVar, ContextValue,

CGF.Builder,

);

UpdateExpr = [](llvm::LLVMContext &Ctx,

*

,

Ops.push_back(llvm::dwarf::DW_OP_plus_uconst);

Ops.push_back(Offset);

Ops.push_back(llvm::dwarf::DW_OP_deref);

->setExpression(llvm::DIExpression::get(Ctx, Ops));

llvm::Instruction &

= CGF.Builder.GetInsertBlock()->back();

(

DDI = dyn_cast<llvm::DbgVariableIntrinsic>(&

))

UpdateExpr(DDI->getContext(), DDI, Offset);

assert(!

.isTerminator() &&

);

CGF.Builder.GetInsertBlock()->getTrailingDbgRecords()) {

(llvm::DbgVariableRecord &DVR : llvm::reverse(

llvm::filterDbgVars(Marker->getDbgRecordRange()))) {

UpdateExpr(

.getContext(), &DVR, Offset);

(!

.PrivateVars.empty() || !

.FirstprivateVars.empty() ||

!

.LastprivateVars.empty() || !

.PrivateLocals.empty()) {

{ PrivatesParam = 2, CopyFnParam = 3 };

llvm::Value *CopyFn = CGF.Builder.CreateLoad(

CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));

llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(

CS->getCapturedDecl()->getParam(PrivatesParam)));

CallArgs.push_back(PrivatesPtr);

ParamTypes.push_back(PrivatesPtr->getType());

(

*

:

.PrivateVars) {

*VD = cast<VarDecl>(cast<DeclRefExpr>(

)->getDecl());

CGF.getContext().getPointerType(

->

()),

);

PrivatePtrs.emplace_back(VD, PrivatePtr);

CallArgs.push_back(PrivatePtr.

());

ParamTypes.push_back(PrivatePtr.

());

(

*

:

.FirstprivateVars) {

*VD = cast<VarDecl>(cast<DeclRefExpr>(

)->getDecl());

CGF.CreateMemTemp(CGF.getContext().getPointerType(

->

()),

);

PrivatePtrs.emplace_back(VD, PrivatePtr);

FirstprivatePtrs.emplace_back(VD, PrivatePtr);

CallArgs.push_back(PrivatePtr.

());

ParamTypes.push_back(PrivatePtr.

());

(

*

:

.LastprivateVars) {

*VD = cast<VarDecl>(cast<DeclRefExpr>(

)->getDecl());

CGF.CreateMemTemp(CGF.getContext().getPointerType(

->

()),

);

PrivatePtrs.emplace_back(VD, PrivatePtr);

CallArgs.push_back(PrivatePtr.

());

ParamTypes.push_back(PrivatePtr.

());

Ty = CGF.getContext().getPointerType(Ty);

Ty = CGF.getContext().getPointerType(Ty);

CGF.getContext().getPointerType(Ty),

);

= UntiedLocalVars.insert(

(

.second ==

)

*

.first = std::make_pair(

CallArgs.push_back(PrivatePtr.

());

ParamTypes.push_back(PrivatePtr.

());

*CopyFnTy = llvm::FunctionType::get(CGF.Builder.getVoidTy(),

ParamTypes,

);

CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(

CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);

(

&Pair : LastprivateDstsOrigs) {

*OrigVD = cast<VarDecl>(Pair.second->getDecl());

CGF.CapturedStmtInfo->lookup(OrigVD) !=

,

Pair.second->getExprLoc());

.addPrivate(Pair.first, CGF.EmitLValue(&DRE).getAddress());

(

&Pair : PrivatePtrs) {

CGF.Builder.CreateLoad(Pair.second),

CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),

CGF.getContext().getDeclAlign(Pair.first));

.addPrivate(Pair.first, Replacement);

(

*DI = CGF.getDebugInfo())

(CGF.CGM.getCodeGenOpts().hasReducedDebugInfo())

(void)DI->EmitDeclareOfAutoVariable(

Pair.first, Pair.second.getBasePointer(), CGF.Builder,

(

&Pair : UntiedLocalVars) {

VDType = Pair.first->getType().getNonReferenceType();

(Pair.first->getType()->isLValueReferenceType())

VDType = CGF.getContext().getPointerType(VDType);

llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first);

CGF.ConvertTypeForMem(CGF.getContext().getPointerType(VDType)),

CGF.getPointerAlign());

Pair.second.first = Replacement;

Ptr = CGF.Builder.CreateLoad(Replacement);

Replacement =

(Ptr, CGF.ConvertTypeForMem(VDType),

CGF.getContext().getDeclAlign(Pair.first));

Pair.second.second = Replacement;

llvm::Value *Ptr = CGF.Builder.CreateLoad(Pair.second.first);

Replacement(Ptr, CGF.ConvertTypeForMem(VDType),

CGF.getContext().getDeclAlign(Pair.first));

Pair.second.first = Replacement;

(

.Reductions) {

OMPPrivateScope FirstprivateScope(CGF);

(

&Pair : FirstprivatePtrs) {

CGF.Builder.CreateLoad(Pair.second),

CGF.ConvertTypeForMem(Pair.first->getType().getNonReferenceType()),

CGF.getContext().getDeclAlign(Pair.first));

FirstprivateScope.addPrivate(Pair.first, Replacement);

(void)FirstprivateScope.Privatize();

OMPLexicalScope LexScope(CGF, S, CapturedRegion);

.ReductionCopies,

.ReductionOps);

llvm::Value *ReductionsPtr = CGF.Builder.CreateLoad(

CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(9)));

(

Cnt = 0,

=

.ReductionVars.size(); Cnt <

; ++Cnt) {

RedCG.emitSharedOrigLValue(CGF, Cnt);

RedCG.emitAggregateType(CGF, Cnt);

CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),

Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(

CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));

CGF.EmitScalarConversion(Replacement.emitRawPointer(CGF),

CGF.getContext().VoidPtrTy,

CGF.getContext().getPointerType(

.ReductionCopies[Cnt]->getType()),

.ReductionCopies[Cnt]->getExprLoc()),

CGF.ConvertTypeForMem(

.ReductionCopies[Cnt]->getType()),

Replacement.getAlignment());

Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);

.addPrivate(RedCG.getBaseDecl(Cnt), Replacement);

(void)

.Privatize();

IPriv =

->privates().begin();

IRed =

->reduction_ops().begin();

ITD =

->taskgroup_descriptors().begin();

(

*Ref :

->varlist()) {

InRedVars.emplace_back(Ref);

InRedPrivs.emplace_back(*IPriv);

InRedOps.emplace_back(*IRed);

TaskgroupDescriptors.emplace_back(*ITD);

std::advance(IPriv, 1);

std::advance(IRed, 1);

std::advance(ITD, 1);

OMPPrivateScope InRedScope(CGF);

(!InRedVars.empty()) {

(

Cnt = 0,

= InRedVars.size(); Cnt <

; ++Cnt) {

RedCG.emitSharedOrigLValue(CGF, Cnt);

RedCG.emitAggregateType(CGF, Cnt);

CGF.CGM.getOpenMPRuntime().emitTaskReductionFixups(CGF, S.getBeginLoc(),

llvm::Value *ReductionsPtr;

(

*TRExpr = TaskgroupDescriptors[Cnt]) {

ReductionsPtr = CGF.EmitLoadOfScalar(CGF.EmitLValue(TRExpr),

TRExpr->getExprLoc());

ReductionsPtr = llvm::ConstantPointerNull::get(CGF.VoidPtrTy);

Replacement = CGF.CGM.getOpenMPRuntime().getTaskReductionItem(

CGF, S.getBeginLoc(), ReductionsPtr, RedCG.getSharedLValue(Cnt));

CGF.EmitScalarConversion(

Replacement.emitRawPointer(CGF), CGF.getContext().VoidPtrTy,

CGF.getContext().getPointerType(InRedPrivs[Cnt]->getType()),

InRedPrivs[Cnt]->getExprLoc()),

CGF.ConvertTypeForMem(InRedPrivs[Cnt]->getType()),

Replacement.getAlignment());

Replacement = RedCG.adjustPrivateAddress(CGF, Cnt, Replacement);

InRedScope.addPrivate(RedCG.getBaseDecl(Cnt), Replacement);

(void)InRedScope.Privatize();

S, *I, *PartId, *TaskT, EKind, CodeGen,

.Tied,

.NumberOfParts);

OMPLexicalScope

(*

, S, std::nullopt,

TaskGen(*

, OutlinedFn,

);

ElemType =

.getBaseElementType(Ty);

.FirstprivateVars.emplace_back(OrigRef);

.FirstprivateCopies.emplace_back(PrivateRef);

.FirstprivateInits.emplace_back(InitRef);

OMPTargetDataInfo &InputInfo) {

PartId = std::next(I);

TaskT = std::next(I, 4);

.Final.setInt(

);

IRef =

->varlist_begin();

IElemInitRef =

->inits().begin();

(

*IInit :

->private_copies()) {

.FirstprivateVars.push_back(*IRef);

.FirstprivateCopies.push_back(IInit);

.FirstprivateInits.push_back(*IElemInitRef);

.ReductionVars.append(

->varlist_begin(),

->varlist_end());

.ReductionOrigs.append(

->varlist_begin(),

->varlist_end());

.ReductionCopies.append(

->privates().begin(),

->privates().end());

.ReductionOps.append(

->reduction_ops().begin(),

->reduction_ops().end());

LHSs.append(

->lhs_exprs().begin(),

->lhs_exprs().end());

RHSs.append(

->rhs_exprs().begin(),

->rhs_exprs().end());

OMPPrivateScope TargetScope(*

);

(InputInfo.NumberOfTargetItems > 0) {

llvm::APInt ArrSize(

32, InputInfo.NumberOfTargetItems);

(),

, BaseAndPointerAndMapperType, CD, S.getBeginLoc());

(),

, BaseAndPointerAndMapperType, CD, S.getBeginLoc());

TargetScope.addPrivate(BPVD, InputInfo.BasePointersArray);

TargetScope.addPrivate(PVD, InputInfo.PointersArray);

TargetScope.addPrivate(SVD, InputInfo.SizesArray);

(!isa_and_nonnull<llvm::ConstantPointerNull>(

InputInfo.MappersArray.emitRawPointer(*

))) {

(),

, BaseAndPointerAndMapperType, CD, S.getBeginLoc());

TargetScope.addPrivate(MVD, InputInfo.MappersArray);

(void)TargetScope.Privatize();

&&CodeGen = [&

, &S, CS, &BodyGen, BPVD, PVD, SVD, MVD, EKind,

OMPPrivateScope

(CGF);

(!

.FirstprivateVars.empty()) {

{ PrivatesParam = 2, CopyFnParam = 3 };

llvm::Value *CopyFn = CGF.Builder.CreateLoad(

CGF.GetAddrOfLocalVar(CS->getCapturedDecl()->getParam(CopyFnParam)));

llvm::Value *PrivatesPtr = CGF.Builder.CreateLoad(CGF.GetAddrOfLocalVar(

CS->getCapturedDecl()->getParam(PrivatesParam)));

CallArgs.push_back(PrivatesPtr);

ParamTypes.push_back(PrivatesPtr->getType());

(

*

:

.FirstprivateVars) {

*VD = cast<VarDecl>(cast<DeclRefExpr>(

)->getDecl());

CGF.CreateMemTemp(CGF.getContext().getPointerType(

->

()),

);

PrivatePtrs.emplace_back(VD, PrivatePtr);

CallArgs.push_back(PrivatePtr.

());

ParamTypes.push_back(PrivatePtr.

());

*CopyFnTy = llvm::FunctionType::get(CGF.Builder.getVoidTy(),

ParamTypes,

);

CGF.CGM.getOpenMPRuntime().emitOutlinedFunctionCall(

CGF, S.getBeginLoc(), {CopyFnTy, CopyFn}, CallArgs);

(

&Pair : PrivatePtrs) {

CGF.Builder.CreateLoad(Pair.second),