1
2
3
4 package net.sourceforge.pmd.cpd;
5
6 import java.io.BufferedReader;
7 import java.io.IOException;
8 import java.io.Reader;
9 import java.io.StringReader;
10 import java.util.Properties;
11
12 import net.sourceforge.pmd.PMD;
13 import net.sourceforge.pmd.lang.LanguageRegistry;
14 import net.sourceforge.pmd.lang.LanguageVersionHandler;
15 import net.sourceforge.pmd.lang.TokenManager;
16 import net.sourceforge.pmd.lang.ast.TokenMgrError;
17 import net.sourceforge.pmd.lang.cpp.CppLanguageModule;
18 import net.sourceforge.pmd.lang.cpp.ast.Token;
19 import net.sourceforge.pmd.util.IOUtil;
20
21 import org.apache.commons.io.IOUtils;
22
23
24
25
26 public class CPPTokenizer implements Tokenizer {
27
28 private boolean skipBlocks = true;
29 private String skipBlocksStart;
30 private String skipBlocksEnd;
31
32
33
34
35
36
37
38 public void setProperties(Properties properties) {
39 skipBlocks = Boolean.parseBoolean(properties.getProperty(OPTION_SKIP_BLOCKS, Boolean.TRUE.toString()));
40 if (skipBlocks) {
41 String skipBlocksPattern = properties.getProperty(OPTION_SKIP_BLOCKS_PATTERN, DEFAULT_SKIP_BLOCKS_PATTERN);
42 String[] split = skipBlocksPattern.split("\\|", 2);
43 skipBlocksStart = split[0];
44 if (split.length == 1) {
45 skipBlocksEnd = split[0];
46 } else {
47 skipBlocksEnd = split[1];
48 }
49 }
50 }
51
52 @Override
53 public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
54 StringBuilder buffer = sourceCode.getCodeBuffer();
55 Reader reader = null;
56 try {
57 LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(CppLanguageModule.NAME)
58 .getDefaultVersion().getLanguageVersionHandler();
59 reader = new StringReader(maybeSkipBlocks(buffer.toString()));
60 reader = IOUtil.skipBOM(reader);
61 TokenManager tokenManager = languageVersionHandler.getParser(
62 languageVersionHandler.getDefaultParserOptions()).getTokenManager(sourceCode.getFileName(), reader);
63 Token currentToken = (Token) tokenManager.getNextToken();
64 while (currentToken.image.length() > 0) {
65 tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine));
66 currentToken = (Token) tokenManager.getNextToken();
67 }
68 tokenEntries.add(TokenEntry.getEOF());
69 System.err.println("Added " + sourceCode.getFileName());
70 } catch (TokenMgrError err) {
71 err.printStackTrace();
72 System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error");
73 tokenEntries.add(TokenEntry.getEOF());
74 } catch (IOException e) {
75 e.printStackTrace();
76 System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error");
77 tokenEntries.add(TokenEntry.getEOF());
78 } finally {
79 IOUtils.closeQuietly(reader);
80 }
81 }
82
83 private String maybeSkipBlocks(String test) throws IOException {
84 if (!skipBlocks) {
85 return test;
86 }
87
88 BufferedReader reader = new BufferedReader(new StringReader(test));
89 StringBuilder filtered = new StringBuilder(test.length());
90 String line;
91 boolean skip = false;
92 while ((line = reader.readLine()) != null) {
93 if (skipBlocksStart.equalsIgnoreCase(line.trim())) {
94 skip = true;
95 } else if (skip && skipBlocksEnd.equalsIgnoreCase(line.trim())) {
96 skip = false;
97 }
98 if (!skip) {
99 filtered.append(line);
100 }
101 filtered.append(PMD.EOL);
102 }
103 return filtered.toString();
104 }
105 }