+14
-4
lines changedFilter options
+14
-4
lines changed Original file line number Diff line number Diff line change
@@ -22,14 +22,17 @@ public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
22 22
StringBuilder sb = sourceCode.getCodeBuffer();
23 23
try (BufferedReader reader = new BufferedReader(new CharArrayReader(sb.toString().toCharArray()))) {
24 24
int lineNumber = 1;
25 +
int colNumber = 1;
25 26
String line = reader.readLine();
26 27
while (line != null) {
27 28
StringTokenizer tokenizer = new StringTokenizer(line, TOKENS, true);
28 29
while (tokenizer.hasMoreTokens()) {
29 30
String token = tokenizer.nextToken();
31 +
int endCol = colNumber + token.length() - 1; // -1 because inclusive
30 32
if (!" ".equals(token) && !"\t".equals(token)) {
31 -
tokenEntries.add(new TokenEntry(token, sourceCode.getFileName(), lineNumber));
33 +
tokenEntries.add(new TokenEntry(token, sourceCode.getFileName(), lineNumber, colNumber, endCol));
32 34
}
35 +
colNumber = endCol + 1;
33 36
}
34 37
// advance iteration variables
35 38
line = reader.readLine();
@@ -48,7 +48,10 @@ private TokenEntry() {
48 48
* @param image
49 49
* @param tokenSrcID
50 50
* @param beginLine the linenumber, 1-based.
51 +
*
52 +
* @deprecated Use {@link #TokenEntry(String, String, int, int, int)}, don't be lazy
51 53
*/
54 +
@Deprecated
52 55
public TokenEntry(String image, String tokenSrcID, int beginLine) {
53 56
this(image, tokenSrcID, beginLine, -1, -1);
54 57
}
@@ -62,7 +65,7 @@ public TokenEntry(String image, String tokenSrcID, int beginLine) {
62 65
* @param endColumn the column number, 1-based
63 66
*/
64 67
public TokenEntry(String image, String tokenSrcID, int beginLine, int beginColumn, int endColumn) {
65 -
assert beginLine >= 1 && beginColumn >= 1 && endColumn >= 1 : "Coordinates are 1-based";
68 +
assert isOk(beginLine) && isOk(beginColumn) && isOk(endColumn) : "Coordinates are 1-based";
66 69
setImage(image);
67 70
this.tokenSrcID = tokenSrcID;
68 71
this.beginLine = beginLine;
@@ -71,6 +74,10 @@ public TokenEntry(String image, String tokenSrcID, int beginLine, int beginColum
71 74
this.index = TOKEN_COUNT.get().getAndIncrement();
72 75
}
73 76 77 +
public boolean isOk(int coord) {
78 +
return coord >= 1 || coord == -1;
79 +
}
80 + 74 81
public static TokenEntry getEOF() {
75 82
TOKEN_COUNT.get().getAndIncrement();
76 83
return EOF;
@@ -40,8 +40,8 @@ public void testColumns() {
40 40
final int beginLine = 1;
41 41
final int beginColumn = 2;
42 42
final int endColumn = 3;
43 -
final TokenEntry token = new TokenEntry("public", "/var/Foo.java", 1, beginColumn, 0);
44 -
final TokenEntry endToken = new TokenEntry("}", "/var/Foo.java", 5, 0, endColumn);
43 +
final TokenEntry token = new TokenEntry("public", "/var/Foo.java", 1, beginColumn, beginColumn + "public".length());
44 +
final TokenEntry endToken = new TokenEntry("}", "/var/Foo.java", 5, endColumn - 1, endColumn);
45 45 46 46
final Mark mark = new Mark(token);
47 47
final int lineCount = 10;
You can’t perform that action at this time.
RetroSearch is an open source project built by @garambo | Open a GitHub Issue
Search and Browse the WWW like it's 1997 | Search results from DuckDuckGo
HTML:
3.2
| Encoding:
UTF-8
| Version:
0.7.4