1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 package org.sonar.plugins.scala.cpd;
21
22 import java.util.List;
23
24 import net.sourceforge.pmd.cpd.SourceCode;
25 import net.sourceforge.pmd.cpd.TokenEntry;
26 import net.sourceforge.pmd.cpd.Tokenizer;
27 import net.sourceforge.pmd.cpd.Tokens;
28 import net.sourceforge.pmd.lang.ast.TokenMgrError;
29
30 import org.sonar.plugins.scala.compiler.Lexer;
31 import org.sonar.plugins.scala.compiler.Token;
32
33
34
35
36
37
38 public final class ScalaTokenizer implements Tokenizer {
39
40 public void tokenize(SourceCode source, Tokens cpdTokens) {
41 String filename = source.getFileName();
42
43 try {
44 Lexer lexer = new Lexer();
45 List<Token> tokens = lexer.getTokensOfFile(filename);
46 for (Token token : tokens) {
47 String tokenVal =
48 token.tokenVal() != null ? token.tokenVal() : Integer.toString(token.tokenType());
49
50 TokenEntry cpdToken = new TokenEntry(tokenVal, filename, token.line());
51 cpdTokens.add(cpdToken);
52 }
53 cpdTokens.add(TokenEntry.getEOF());
54 } catch (RuntimeException e) {
55 e.printStackTrace();
56
57
58 throw new TokenMgrError(
59 "Lexical error in file " + filename + ". The scala tokenizer exited with error: " + e.getMessage(),
60 TokenMgrError.LEXICAL_ERROR);
61 }
62 }
63
64 }