1
2
3
4 package net.sourceforge.pmd.cpd;
5
6 import java.io.Reader;
7 import java.io.StringReader;
8
9 import net.sourceforge.pmd.lang.LanguageRegistry;
10 import net.sourceforge.pmd.lang.LanguageVersionHandler;
11 import net.sourceforge.pmd.lang.TokenManager;
12 import net.sourceforge.pmd.lang.ast.TokenMgrError;
13 import net.sourceforge.pmd.lang.objectivec.ObjectiveCLanguageModule;
14 import net.sourceforge.pmd.lang.objectivec.ast.Token;
15
16 import org.apache.commons.io.IOUtils;
17
18
19
20
21 public class ObjectiveCTokenizer implements Tokenizer {
22
23 @Override
24 public void tokenize(SourceCode sourceCode, Tokens tokenEntries) {
25 StringBuilder buffer = sourceCode.getCodeBuffer();
26 Reader reader = null;
27 try {
28 LanguageVersionHandler languageVersionHandler = LanguageRegistry.getLanguage(ObjectiveCLanguageModule.NAME)
29 .getDefaultVersion().getLanguageVersionHandler();
30 reader = new StringReader(buffer.toString());
31 TokenManager tokenManager = languageVersionHandler.getParser(
32 languageVersionHandler.getDefaultParserOptions()).getTokenManager(sourceCode.getFileName(), reader);
33 Token currentToken = (Token) tokenManager.getNextToken();
34 while (currentToken.image.length() > 0) {
35 tokenEntries.add(new TokenEntry(currentToken.image, sourceCode.getFileName(), currentToken.beginLine));
36 currentToken = (Token) tokenManager.getNextToken();
37 }
38 tokenEntries.add(TokenEntry.getEOF());
39 System.err.println("Added " + sourceCode.getFileName());
40 } catch (TokenMgrError err) {
41 err.printStackTrace();
42 System.err.println("Skipping " + sourceCode.getFileName() + " due to parse error");
43 tokenEntries.add(TokenEntry.getEOF());
44 } finally {
45 IOUtils.closeQuietly(reader);
46 }
47 }
48 }