Home - News ( United States | United Kingdom | Italy | Germany ) - Football scores

Showing content from https://clang.llvm.org/doxygen/Tokens_8cpp_source.html below:

clang: lib/Tooling/Syntax/Tokens.cpp Source File

;

.isBeforeInTranslationUnit(

.location(), R.

());

!

.isBeforeInTranslationUnit(R.

(),

.location());

{

, End};

assert(

.getSLocEntry(TargetFile).isFile());

(

.isMacroID() &&

.isMacroID()) {

DecFirst =

.getDecomposedLoc(

);

DecLast =

.getDecomposedLoc(

);

&ExpFirst =

.getSLocEntry(DecFirst.first).getExpansion();

&ExpLast =

.getSLocEntry(DecLast.first).getExpansion();

(!ExpFirst.isMacroArgExpansion() || !ExpLast.isMacroArgExpansion())

(ExpFirst.getExpansionLocStart() != ExpLast.getExpansionLocStart())

ExpFileID =

.getFileID(ExpFirst.getExpansionLocStart());

(ExpFileID == TargetFile)

= ExpFirst.getSpellingLoc().getLocWithOffset(DecFirst.second);

= ExpLast.getSpellingLoc().getLocWithOffset(DecLast.second);

DecFirst =

.getDecomposedExpansionLoc(Candidate.

());

DecLast =

.getDecomposedExpansionLoc(Candidate.

());

(Candidate.

() || DecFirst.first != TargetFile ||

DecLast.first != TargetFile)

=

.getDecomposedLoc(

.getExpansionRange(Prev).getBegin());

(

.first != DecFirst.first ||

.second >= DecFirst.second)

(Next.isValid()) {

=

.getDecomposedLoc(

.getExpansionRange(Next).getEnd());

(

.first != DecLast.first ||

.second <= DecLast.second)

: Location(Location), Length(Length), Kind(Kind) {

assert(!

.isAnnotation());

*Start =

.getCharacterData(location(), &

);

llvm::StringRef(Start,

());

assert(location().isFileID() &&

);

StartOffset;

std::tie(

, StartOffset) =

.getDecomposedLoc(location());

L =

.range(

);

assert(F.file() == L.file() &&

);

assert((F == L || F.endOffset() <= L.beginOffset()) &&

);

(F.file(), F.beginOffset(), L.endOffset());

OS <<

.str();

assert(

.isValid());

assert(BeginOffset <= EndOffset);

std::tie(File,

) =

.getDecomposedLoc(BeginLoc);

End =

+ Length;

assert(

.getFileID(BeginLoc) ==

.getFileID(EndLoc));

assert(

.getFileOffset(BeginLoc) <=

.getFileOffset(EndLoc));

std::tie(File,

) =

.getDecomposedLoc(BeginLoc);

End =

.getFileOffset(EndLoc);

OS << llvm::formatv(

,

assert(End <=

.size());

(!ExpandedTokIndex.empty())

ExpandedTokIndex.reserve(ExpandedTokens.size());

(

I = 0,

= ExpandedTokens.size(); I !=

; ++I) {

ExpandedTokIndex[

] = I;

(!ExpandedTokIndex.empty()) {

B = ExpandedTokIndex.find(R.

());

= ExpandedTokIndex.find(R.

());

(B != ExpandedTokIndex.end() &&

!= ExpandedTokIndex.end()) {

*L = ExpandedTokens.data() + B->getSecond();

*R = ExpandedTokens.data() +

->getSecond() + 1;

std::pair<const syntax::Token *, const TokenBuffer::Mapping *>

TokenBuffer::spelledForExpandedToken(

*Expanded)

{

assert(ExpandedTokens.data() <= Expanded &&

Expanded < ExpandedTokens.data() + ExpandedTokens.size());

FileIt = Files.find(

assert(FileIt != Files.end() &&

);

MarkedFile &

= FileIt->second;

ExpandedIndex = Expanded - ExpandedTokens.data();

It = llvm::partition_point(

.Mappings, [&](

Mapping &M) {

return M.BeginExpanded <= ExpandedIndex;

(It ==

.Mappings.begin()) {

{&

.SpelledTokens[ExpandedIndex -

.BeginExpanded],

(ExpandedIndex < It->EndExpanded)

{&

.SpelledTokens[It->BeginSpelled],

&*It};

&

.SpelledTokens[It->EndSpelled + (ExpandedIndex - It->EndExpanded)],

TokenBuffer::Mapping *

TokenBuffer::mappingStartingBeforeSpelled(

MarkedFile &F,

assert(F.SpelledTokens.data() <= Spelled);

SpelledI = Spelled - F.SpelledTokens.data();

assert(SpelledI < F.SpelledTokens.size());

It = llvm::partition_point(F.Mappings, [SpelledI](

Mapping &M) {

return M.BeginSpelled <= SpelledI;

(It == F.Mappings.begin())

&

= fileForSpelled(Spelled);

*FrontMapping = mappingStartingBeforeSpelled(

, &Spelled.front());

SpelledFrontI = &Spelled.front() -

.SpelledTokens.data();

assert(SpelledFrontI <

.SpelledTokens.size());

ExpandedBegin;

ExpandedBegin =

.BeginExpanded + SpelledFrontI;

}

(SpelledFrontI < FrontMapping->EndSpelled) {

(SpelledFrontI != FrontMapping->BeginSpelled) {

ExpandedBegin = FrontMapping->BeginExpanded;

FrontMapping->EndExpanded + (SpelledFrontI - FrontMapping->EndSpelled);

*BackMapping = mappingStartingBeforeSpelled(

, &Spelled.back());

SpelledBackI = &Spelled.back() -

.SpelledTokens.data();

ExpandedEnd;

ExpandedEnd =

.BeginExpanded + SpelledBackI + 1;

}

(SpelledBackI < BackMapping->EndSpelled) {

(SpelledBackI + 1 != BackMapping->EndSpelled) {

ExpandedEnd = BackMapping->EndExpanded;

BackMapping->EndExpanded + (SpelledBackI - BackMapping->EndSpelled) + 1;

assert(ExpandedBegin < ExpandedTokens.size());

assert(ExpandedEnd < ExpandedTokens.size());

(ExpandedBegin == ExpandedEnd)

ExpandedTokens.data() + ExpandedEnd)};

It = Files.find(FID);

assert(It != Files.end());

It->second.SpelledTokens;

*Tok = llvm::partition_point(

[&](

&Tok) { return Tok.endLocation() <= Loc; });

(!Tok || Loc < Tok->location())

std::string TokenBuffer::Mapping::str()

{

llvm::formatv(

,

BeginSpelled, EndSpelled, BeginExpanded, EndExpanded));

std::optional<llvm::ArrayRef<syntax::Token>>

(!Expanded.empty() && Expanded.back().kind() == tok::eof) {

Expanded = Expanded.drop_back();

(Expanded.empty())

[FirstSpelled, FirstMapping] = spelledForExpandedToken(

);

[LastSpelled, LastMapping] = spelledForExpandedToken(

);

(FID != SourceMgr->

(LastSpelled->location()))

MarkedFile &

= Files.find(FID)->second;

(FirstMapping && FirstMapping == LastMapping &&

: (

- 1)->location();

: (

+ 1)->location();

->location(),

->location(), Prev, Next, FID, *SourceMgr);

getTokensCovering(

.SpelledTokens,

, *SourceMgr);

FirstExpanded = Expanded.begin() - ExpandedTokens.data();

LastExpanded = Expanded.end() - ExpandedTokens.data();

(FirstMapping && FirstExpanded != FirstMapping->BeginExpanded)

(LastMapping && LastMapping->EndExpanded != LastExpanded)

FirstMapping ?

.SpelledTokens.data() + FirstMapping->BeginSpelled

LastMapping ?

.SpelledTokens.data() + LastMapping->EndSpelled

Mapping &M)

{

F.SpelledTokens.data() + M.EndSpelled);

ExpandedTokens.data() + M.EndExpanded);

TokenBuffer::MarkedFile &

assert(!Spelled.empty());

assert(Spelled.front().location().isFileID() &&

);

FileIt = Files.find(SourceMgr->

(Spelled.front().location()));

assert(FileIt != Files.end() &&

);

&

= FileIt->second;

assert(

.SpelledTokens.data() <= Spelled.data() &&

(

.SpelledTokens.data() +

.SpelledTokens.size()) &&

);

T1 = Spelled.back().location();

T2 =

.SpelledTokens.back().location();

assert(T1 == T2 ||

().isBeforeInTranslationUnit(T1, T2));

std::optional<TokenBuffer::Expansion>

&

= fileForSpelled(*Spelled);

SpelledIndex = Spelled -

.SpelledTokens.data();

M = llvm::partition_point(

.Mappings, [&](

Mapping &M) {

return M.BeginSpelled < SpelledIndex;

(M ==

.Mappings.end() || M->BeginSpelled != SpelledIndex)

makeExpansion(

, *M);

&

= fileForSpelled(Spelled);

SpelledBeginIndex = Spelled.begin() -

.SpelledTokens.data();

SpelledEndIndex = Spelled.end() -

.SpelledTokens.data();

M = llvm::partition_point(

.Mappings, [&](

Mapping &M) {

return M.EndSpelled <= SpelledBeginIndex;

std::vector<TokenBuffer::Expansion> Expansions;

(; M !=

.Mappings.end() && M->BeginSpelled < SpelledEndIndex; ++M)

Expansions.push_back(makeExpansion(

, *M));

*Right = llvm::partition_point(

AcceptRight = Right != Tokens.end() && Right->location() <=

;

Right != Tokens.begin() && (Right - 1)->endLocation() >=

;

Right + (AcceptRight ? 1 : 0));

, Tokens.spelledTokens(Tokens.sourceManager().getFileID(

)));

(Tok.kind() == tok::identifier)

, Tokens.spelledTokens(Tokens.sourceManager().getFileID(

)));

std::vector<const syntax::Token *>

FileIt = Files.find(FID);

assert(FileIt != Files.end() &&

);

&

= FileIt->second;

std::vector<const syntax::Token *> Expansions;

&Spelled =

.SpelledTokens;

(

Mapping :

.Mappings) {

(

->

() == tok::TokenKind::identifier)

Expansions.push_back(

);

std::vector<syntax::Token> Tokens;

(

.getKind() == tok::raw_identifier && !

.needsCleaning() &&

.setIdentifierInfo(&II);

SrcBuffer =

.getBufferData(FR.

());

L(

.getLocForStartOfFile(FR.

()), LO, SrcBuffer.data(),

SrcBuffer.data() + SrcBuffer.size());

(

.getFileOffset(

.getLocation()) < FR.

())

&

= Collector->PP.getSourceManager();

(LastExpansionEnd.isValid() &&

!

.isBeforeInTranslationUnit(LastExpansionEnd,

.

()))

);

(

.isAnnotation())

DEBUG_WITH_TYPE(

, llvm::dbgs()

CB = std::make_unique<CollectPPExpansions>(*

);

this->Collector = CB.get();

(std::vector<syntax::Token> Expanded, PPExpansions CollectedExpansions,

: Result(

), CollectedExpansions(

::move(CollectedExpansions)),

(

),

Result.ExpandedTokens = std::move(Expanded);

assert(!Result.ExpandedTokens.empty());

assert(Result.ExpandedTokens.back().kind() == tok::eof);

buildSpelledTokens();

(NextExpanded < Result.ExpandedTokens.size() - 1

) {

OldPosition = NextExpanded;

(NextExpanded == OldPosition)

diagnoseAdvanceFailure();

(

&

: Result.Files)

discard(

.first);

(

&pair : Result.Files) {

&mappings = pair.second.Mappings;

assert(llvm::is_sorted(mappings, [](

TokenBuffer::Mapping &M1,

TokenBuffer::Mapping &M2) {

M1.BeginSpelled < M2.BeginSpelled &&

M1.EndSpelled < M2.EndSpelled &&

M1.BeginExpanded < M2.BeginExpanded &&

M1.EndExpanded < M2.EndExpanded;

std::move(Result);

discard(std::optional<FileID> Drain = std::nullopt) {

Drain ?

.getLocForEndOfFile(*Drain)

:

.getExpansionLoc(

Result.ExpandedTokens[NextExpanded].location());

&SpelledTokens = Result.Files[

].SpelledTokens;

&NextSpelled = this->NextSpelled[

];

TokenBuffer::Mapping Mapping;

Mapping.BeginSpelled = NextSpelled;

Mapping.BeginExpanded = Mapping.EndExpanded =

Drain ? Result.Files[*Drain].EndExpanded : NextExpanded;

FlushMapping = [&,

] {

Mapping.EndSpelled = NextSpelled;

(Mapping.BeginSpelled != Mapping.EndSpelled)

Result.Files[

].Mappings.push_back(Mapping);

Mapping.BeginSpelled = NextSpelled;

(NextSpelled < SpelledTokens.size() &&

SpelledTokens[NextSpelled].location() <

) {

CollectedExpansions.lookup(SpelledTokens[NextSpelled].location());

(NextSpelled < SpelledTokens.size() &&

SpelledTokens[NextSpelled].location() <= KnownEnd)

&Tok = Result.ExpandedTokens[NextExpanded];

&SpelledTokens = Result.Files[

].SpelledTokens;

&NextSpelled = this->NextSpelled[

];

(NextSpelled < SpelledTokens.size() &&

NextExpanded < Result.ExpandedTokens.size() &&

SpelledTokens[NextSpelled].location() ==

Result.ExpandedTokens[NextExpanded].location()) {

End = CollectedExpansions.lookup(Expansion);

assert(End.isValid() &&

);

TokenBuffer::Mapping Mapping;

Mapping.BeginExpanded = NextExpanded;

Mapping.BeginSpelled = NextSpelled;

(NextSpelled < SpelledTokens.size() &&

SpelledTokens[NextSpelled].location() <= End)

(NextExpanded < Result.ExpandedTokens.size() &&

Result.ExpandedTokens[NextExpanded].location()) == Expansion)

Mapping.EndExpanded = NextExpanded;

Mapping.EndSpelled = NextSpelled;

Result.Files[

].Mappings.push_back(Mapping);

diagnoseAdvanceFailure() {

(

I = (NextExpanded < 10) ? 0 : NextExpanded - 10;

I < NextExpanded + 5 && I < Result.ExpandedTokens.size(); ++I) {

(I == NextExpanded) ?

: (I < NextExpanded) ?

:

;

llvm::errs() << L << Result.ExpandedTokens[I].dumpForTests(

) <<

;

llvm_unreachable(

);

buildSpelledTokens() {

(

I = 0; I < Result.ExpandedTokens.size(); ++I) {

&Tok = Result.ExpandedTokens[I];

FID =

.getFileID(

.getExpansionLoc(Tok.

()));

It = Result.Files.try_emplace(FID);

TokenBuffer::MarkedFile &

= It.first->second;

.EndExpanded = Tok.

() == tok::eof ? I : I + 1;

.BeginExpanded = I;

NextExpanded = 0;

llvm::DenseMap<FileID, unsigned> NextSpelled;

PPExpansions CollectedExpansions;

PP.setTokenWatcher(

);

Collector->disable();

Builder(std::move(Expanded), std::move(Expansions),

PP.getSourceManager(), PP.getLangOpts())

std::string(llvm::formatv(

,

std::string(llvm::formatv(

, text(

),

PrintToken = [

](

&

) -> std::string {

(

.kind() == tok::eof)

std::string(

.text(*SourceMgr));

DumpTokens = [

, &PrintToken](llvm::raw_ostream &OS,

(Tokens.empty()) {

OS << Tokens[0].text(*SourceMgr);

(

I = 1; I < Tokens.size(); ++I) {

(Tokens[I].kind() == tok::eof)

OS <<

<< PrintToken(Tokens[I]);

llvm::raw_string_ostream OS(Dump);

OS <<

std::vector<FileID> Keys;

(

&F : Files)

Keys.push_back(F.first);

MarkedFile &

= Files.find(ID)->second;

std::string

= llvm::sys::path::convert_to_slash(Entry->getName());

OS << llvm::formatv(

,

) <<

DumpTokens(OS,

.SpelledTokens);

(

.Mappings.empty()) {

OS <<

;

OS <<

;

(

&M :

.Mappings) {

,

PrintToken(

.SpelledTokens[M.BeginSpelled]), M.BeginSpelled,

M.EndSpelled ==

.SpelledTokens.size()

: PrintToken(

.SpelledTokens[M.EndSpelled]),

M.EndSpelled, PrintToken(ExpandedTokens[M.BeginExpanded]),

M.BeginExpanded, PrintToken(ExpandedTokens[M.EndExpanded]),

Defines the Diagnostic-related interfaces.

static Decl::Kind getKind(const Decl *D)

Defines the clang::IdentifierInfo, clang::IdentifierTable, and clang::Selector interfaces.

Forward-declares and imports various common LLVM datatypes that clang wants to use unqualified.

Defines the clang::LangOptions interface.

llvm::MachO::Target Target

Defines the PPCallbacks interface.

static ParseState advance(ParseState S, size_t N)

Defines the clang::Preprocessor interface.

Defines the clang::SourceLocation class and associated facilities.

Defines the SourceManager interface.

Defines the clang::TokenKind enum and support functions.

Builds mappings and spelled tokens in the TokenBuffer based on the expanded token stream.

Builder(std::vector< syntax::Token > Expanded, PPExpansions CollectedExpansions, const SourceManager &SM, const LangOptions &LangOpts)

Records information reqired to construct mappings for the token buffer that we are collecting.

CollectPPExpansions(TokenCollector &C)

void disable()

Disabled instance will stop reporting anything to TokenCollector.

void MacroExpands(const clang::Token &MacroNameTok, const MacroDefinition &MD, SourceRange Range, const MacroArgs *Args) override

Called by Preprocessor::HandleMacroExpandedIdentifier when a macro invocation is found.

Represents a character-granular source range.

An opaque identifier used by SourceManager which refers to a source file (MemoryBuffer) along with it...

unsigned getHashValue() const

One of these records is kept for each identifier that is lexed.

tok::TokenKind getTokenID() const

If this is a source-language token (e.g.

Implements an efficient mapping from strings to IdentifierInfo nodes.

IdentifierInfo & get(StringRef Name)

Return the identifier token info for the specified named identifier.

Keeps track of the various options that can be enabled, which controls the dialect of C or C++ that i...

Lexer - This provides a simple interface that turns a text buffer into a stream of tokens.

bool LexFromRawLexer(Token &Result)

LexFromRawLexer - Lex a token from a designated raw lexer (one with no associated preprocessor object...

unsigned getCurrentBufferOffset()

Returns the current lexing offset.

MacroArgs - An instance of this class captures information about the formal arguments specified to a ...

A description of the current definition of a macro.

This interface provides a way to observe the actions of the preprocessor as it does its thing.

Engages in a tight little dance with the lexer to efficiently preprocess tokens.

void addPPCallbacks(std::unique_ptr< PPCallbacks > C)

SourceManager & getSourceManager() const

void setTokenWatcher(llvm::unique_function< void(const clang::Token &)> F)

Register a function that would be called on each token in the final expanded token stream.

Encodes a location in the source.

bool isValid() const

Return true if this is a valid SourceLocation object.

This class handles loading and caching of source files into memory.

FileID getFileID(SourceLocation SpellingLoc) const

Return the FileID for a SourceLocation.

OptionalFileEntryRef getFileEntryRefForID(FileID FID) const

Returns the FileEntryRef for the provided FileID.

bool isMacroArgExpansion(SourceLocation Loc, SourceLocation *StartLoc=nullptr) const

Tests whether the given source location represents a macro argument's expansion into the function-lik...

SourceLocation getExpansionLoc(SourceLocation Loc) const

Given a SourceLocation object Loc, return the expansion location referenced by the ID.

A trivial tuple used to represent a source range.

void setBegin(SourceLocation b)

SourceLocation getEnd() const

SourceLocation getBegin() const

Token - This structure provides full information about a lexed token.

A list of tokens obtained by preprocessing a text buffer and operations to map between the expanded a...

const syntax::Token * spelledTokenContaining(SourceLocation Loc) const

Returns the spelled Token containing the Loc, if there are no such tokens returns nullptr.

const SourceManager & sourceManager() const

void indexExpandedTokens()

Builds a cache to make future calls to expandedToken(SourceRange) faster.

llvm::SmallVector< llvm::ArrayRef< syntax::Token >, 1 > expandedForSpelled(llvm::ArrayRef< syntax::Token > Spelled) const

Find the subranges of expanded tokens, corresponding to Spelled.

llvm::ArrayRef< syntax::Token > expandedTokens() const

All tokens produced by the preprocessor after all macro replacements, directives, etc.

std::string dumpForTests() const

std::optional< llvm::ArrayRef< syntax::Token > > spelledForExpanded(llvm::ArrayRef< syntax::Token > Expanded) const

Returns the subrange of spelled tokens corresponding to AST node spanning Expanded.

std::vector< Expansion > expansionsOverlapping(llvm::ArrayRef< syntax::Token > Spelled) const

Returns all expansions (partially) expanded from the specified tokens.

std::optional< Expansion > expansionStartingAt(const syntax::Token *Spelled) const

If Spelled starts a mapping (e.g.

llvm::ArrayRef< syntax::Token > spelledTokens(FileID FID) const

Lexed tokens of a file before preprocessing.

std::vector< const syntax::Token * > macroExpansions(FileID FID) const

Get all tokens that expand a macro in FID.

Collects tokens for the main file while running the frontend action.

TokenBuffer consume() &&

Finalizes token collection.

TokenCollector(Preprocessor &P)

Adds the hooks to collect the tokens.

A token coming directly from a file or from a macro invocation.

std::string str() const

For debugging purposes.

llvm::StringRef text(const SourceManager &SM) const

Get the substring covered by the token.

tok::TokenKind kind() const

FileRange range(const SourceManager &SM) const

Gets a range of this token.

Token(SourceLocation Location, unsigned Length, tok::TokenKind Kind)

std::string dumpForTests(const SourceManager &SM) const

SourceLocation location() const

Location of the first character of a token.

bool Dec(InterpState &S, CodePtr OpPC)

1) Pops a pointer from the stack 2) Load the value from the pointer 3) Writes the value decreased by ...

const syntax::Token * spelledIdentifierTouching(SourceLocation Loc, llvm::ArrayRef< syntax::Token > Tokens)

The identifier token that overlaps or touches a spelling location Loc.

std::vector< syntax::Token > tokenize(FileID FID, const SourceManager &SM, const LangOptions &LO)

Lex the text buffer, corresponding to FID, in raw mode and record the resulting spelled tokens.

raw_ostream & operator<<(raw_ostream &OS, NodeKind K)

For debugging purposes.

llvm::ArrayRef< syntax::Token > spelledTokensTouching(SourceLocation Loc, const syntax::TokenBuffer &Tokens)

The spelled tokens that overlap or touch a spelling location Loc.

const char * getTokenName(TokenKind Kind) LLVM_READNONE

Determines the name of a token as used within the front end.

TokenKind

Provides a simple uniform namespace for tokens from all C languages.

The JSON file list parser is used to communicate input to InstallAPI.

const FunctionProtoType * T

float __ovld __cnfn length(float)

Return the length of vector p, i.e., sqrt(p.x2 + p.y 2 + ...)

A half-open character range inside a particular file, the start offset is included and the end offset...

CharSourceRange toCharRange(const SourceManager &SM) const

Convert to the clang range.

FileRange(FileID File, unsigned BeginOffset, unsigned EndOffset)

EXPECTS: File.isValid() && Begin <= End.

unsigned beginOffset() const

Start is a start offset (inclusive) in the corresponding file.

llvm::StringRef text(const SourceManager &SM) const

Gets the substring that this FileRange refers to.

unsigned endOffset() const

End offset (exclusive) in the corresponding file.

An expansion produced by the preprocessor, includes macro expansions and preprocessor directives.

RetroSearch is an open source project built by @garambo | Open a GitHub Issue

Search and Browse the WWW like it's 1997 | Search results from DuckDuckGo

HTML: 3.2 | Encoding: UTF-8 | Version: 0.7.4