View Javadoc
1   /**
2    * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
3    */
4   package net.sourceforge.pmd.cpd;
5   
6   import static org.junit.Assert.assertEquals;
7   import static org.junit.Assert.assertTrue;
8   
9   import java.util.Properties;
10  
11  import net.sourceforge.pmd.PMD;
12  
13  import org.apache.commons.io.IOUtils;
14  import org.junit.Test;
15  
16  public class CPPTokenizerTest {
17  
18      @Test
19      public void testUTFwithBOM() {
20          Tokens tokens = parse("\ufeffint start()\n{ int ret = 1;\nreturn ret;\n}\n");
21          assertTrue(TokenEntry.getEOF() != tokens.getTokens().get(0));
22          assertEquals(15, tokens.size());
23      }
24  
25      @Test
26      public void testUnicodeSupport() {
27          String code = "\ufeff" +
28                  "#include <iostream>\n" +
29                  "#include <string>\n" +
30                  "\n" +
31                  "// example\n" +
32                  "\n" +
33                  "int main()\n" +
34                  "{\n" +
35                  "    std::string text(\"ąęćśźńó\");\n" +
36                  "    std::cout << text;\n" +
37                  "    return 0;\n" +
38                  "}\n";
39          Tokens tokens = parse(code);
40          assertTrue(TokenEntry.getEOF() != tokens.getTokens().get(0));
41          assertEquals(24, tokens.size());
42      }
43  
44      @Test
45      public void testMultiLineMacros() throws Throwable {
46          Tokens tokens = parse(TEST1);
47          assertEquals(7, tokens.size());
48      }
49  
50      @Test
51      public void testDollarSignInIdentifier() {
52          parse(TEST2);
53      }
54  
55      @Test
56      public void testDollarSignStartingIdentifier() {
57          parse(TEST3);
58      }
59  
60      @Test
61      public void testWideCharacters() {
62          parse(TEST4);
63      }
64  
65      @Test
66      public void testContinuation_IntraToken() {
67      	Tokens tokens = parse(TEST5);
68          assertEquals(7, tokens.size());
69      }
70      
71      @Test
72      public void testContinuation_InterToken() {
73      	Tokens tokens = parse(TEST6);
74      	assertEquals(17, tokens.size());
75      }
76  
77      @Test
78      public void testTokenizerWithSkipBlocks() throws Exception {
79          String test = IOUtils.toString(CPPTokenizerTest.class.getResourceAsStream("cpp/cpp_with_asm.cpp"));
80          Tokens tokens = parse(test, true);
81          assertEquals(19, tokens.size());
82      }
83  
84      @Test
85      public void testTokenizerWithSkipBlocksPattern() throws Exception {
86          String test = IOUtils.toString(CPPTokenizerTest.class.getResourceAsStream("cpp/cpp_with_asm.cpp"));
87          Tokens tokens = parse(test, true, "#if debug|#endif");
88          assertEquals(31, tokens.size());
89      }
90  
91      @Test
92      public void testTokenizerWithoutSkipBlocks() throws Exception {
93          String test = IOUtils.toString(CPPTokenizerTest.class.getResourceAsStream("cpp/cpp_with_asm.cpp"));
94          Tokens tokens = parse(test, false);
95          assertEquals(37, tokens.size());
96      }
97  
98      @Test
99      //ASM code containing the '@' character
100     public void testAsmWithAtSign() {
101         Tokens tokens = parse(TEST7);
102         assertEquals(22, tokens.size());
103     }
104 
105     @Test
106     public void testEOLCommentInPreprocessingDirective() {
107         parse("#define LSTFVLES_CPP  //*" + PMD.EOL);
108     }
109 
110     @Test
111     public void testEmptyCharacter() {
112         Tokens tokens = parse("std::wstring wsMessage( sMessage.length(), L'');" + PMD.EOL);
113         assertEquals(15, tokens.size());
114     }
115 
116     @Test
117     public void testHexCharacter() {
118         Tokens tokens = parse("if (*pbuf == '\\0x05')" + PMD.EOL);
119         assertEquals(8, tokens.size());
120     }
121 
122     @Test
123     public void testWhiteSpaceEscape() {
124         Tokens tokens = parse("szPath = m_sdcacheDir + _T(\"\\    oMedia\");" + PMD.EOL);
125         assertEquals(10, tokens.size());
126     }
127 
128     private Tokens parse(String snippet) {
129         return parse(snippet, false);
130     }
131     private Tokens parse(String snippet, boolean skipBlocks) {
132         return parse(snippet, skipBlocks, null);
133     }
134     private Tokens parse(String snippet, boolean skipBlocks, String skipPattern) {
135         Properties properties = new Properties();
136         properties.setProperty(Tokenizer.OPTION_SKIP_BLOCKS, Boolean.toString(skipBlocks));
137         if (skipPattern != null) {
138             properties.setProperty(Tokenizer.OPTION_SKIP_BLOCKS_PATTERN, skipPattern);
139         }
140 
141         CPPTokenizer tokenizer = new CPPTokenizer();
142         tokenizer.setProperties(properties);
143 
144         SourceCode code = new SourceCode(new SourceCode.StringCodeLoader(snippet));
145         Tokens tokens = new Tokens();
146         tokenizer.tokenize(code, tokens);
147         return tokens;
148     }
149 
150     private static final String TEST1 =
151             "#define FOO a +\\" + PMD.EOL +
152             "            b +\\" + PMD.EOL +
153             "            c +\\" + PMD.EOL +
154             "            d +\\" + PMD.EOL +
155             "            e +\\" + PMD.EOL +
156             "            f +\\" + PMD.EOL +
157             "            g" + PMD.EOL +
158             " void main() {}";
159 
160     private static final String TEST2 =
161             " void main() { int x$y = 42; }";
162 
163     private static final String TEST3 =
164             " void main() { int $x = 42; }";
165 
166     private static final String TEST4 =
167             " void main() { char x = L'a'; }";
168     
169     private static final String TEST5 =
170             "v\\" + PMD.EOL +
171             "o\\" + PMD.EOL +
172             "i\\" + PMD.EOL +
173             "d\\" + PMD.EOL +
174             " \\" + PMD.EOL +
175             "m\\" + PMD.EOL +
176             "a\\" + PMD.EOL +
177             "i\\" + PMD.EOL +
178             "n\\" + PMD.EOL +
179             "(\\" + PMD.EOL +
180             ")\\" + PMD.EOL +
181             " \\" + PMD.EOL +
182             "{\\" + PMD.EOL +
183             " \\" + PMD.EOL +
184             "}\\" + PMD.EOL;
185     
186     private static final String TEST6 =
187             "#include <iostream>" + PMD.EOL +
188             PMD.EOL +
189             "int main()" + PMD.EOL +
190             "{" + PMD.EOL +
191             "   std::cout << \"Hello, \" \\" + PMD.EOL +
192             "                \"world!\\n\";" + PMD.EOL +
193             "   return 0;" + PMD.EOL +
194             "}";
195 
196     private static final String TEST7 =
197             "asm void eSPI_boot()" + PMD.EOL +
198             "{" + PMD.EOL +
199             "  // setup stack pointer" + PMD.EOL +
200             "  lis r1, _stack_addr@h" + PMD.EOL +
201             "  ori r1, r1, _stack_addr@l"  + PMD.EOL +
202             "}";
203 
204     public static junit.framework.Test suite() {
205         return new junit.framework.JUnit4TestAdapter(CPPTokenizerTest.class);
206     }
207 }