1
2
3
4 package net.sourceforge.pmd.cpd;
5
6 import static org.junit.Assert.assertEquals;
7 import static org.junit.Assert.assertTrue;
8
9 import java.util.Properties;
10
11 import net.sourceforge.pmd.PMD;
12
13 import org.apache.commons.io.IOUtils;
14 import org.junit.Test;
15
16 public class CPPTokenizerTest {
17
18 @Test
19 public void testUTFwithBOM() {
20 Tokens tokens = parse("\ufeffint start()\n{ int ret = 1;\nreturn ret;\n}\n");
21 assertTrue(TokenEntry.getEOF() != tokens.getTokens().get(0));
22 assertEquals(15, tokens.size());
23 }
24
25 @Test
26 public void testUnicodeSupport() {
27 String code = "\ufeff" +
28 "#include <iostream>\n" +
29 "#include <string>\n" +
30 "\n" +
31 "// example\n" +
32 "\n" +
33 "int main()\n" +
34 "{\n" +
35 " std::string text(\"ąęćśźńó\");\n" +
36 " std::cout << text;\n" +
37 " return 0;\n" +
38 "}\n";
39 Tokens tokens = parse(code);
40 assertTrue(TokenEntry.getEOF() != tokens.getTokens().get(0));
41 assertEquals(24, tokens.size());
42 }
43
44 @Test
45 public void testMultiLineMacros() throws Throwable {
46 Tokens tokens = parse(TEST1);
47 assertEquals(7, tokens.size());
48 }
49
50 @Test
51 public void testDollarSignInIdentifier() {
52 parse(TEST2);
53 }
54
55 @Test
56 public void testDollarSignStartingIdentifier() {
57 parse(TEST3);
58 }
59
60 @Test
61 public void testWideCharacters() {
62 parse(TEST4);
63 }
64
65 @Test
66 public void testContinuation_IntraToken() {
67 Tokens tokens = parse(TEST5);
68 assertEquals(7, tokens.size());
69 }
70
71 @Test
72 public void testContinuation_InterToken() {
73 Tokens tokens = parse(TEST6);
74 assertEquals(17, tokens.size());
75 }
76
77 @Test
78 public void testTokenizerWithSkipBlocks() throws Exception {
79 String test = IOUtils.toString(CPPTokenizerTest.class.getResourceAsStream("cpp/cpp_with_asm.cpp"));
80 Tokens tokens = parse(test, true);
81 assertEquals(19, tokens.size());
82 }
83
84 @Test
85 public void testTokenizerWithSkipBlocksPattern() throws Exception {
86 String test = IOUtils.toString(CPPTokenizerTest.class.getResourceAsStream("cpp/cpp_with_asm.cpp"));
87 Tokens tokens = parse(test, true, "#if debug|#endif");
88 assertEquals(31, tokens.size());
89 }
90
91 @Test
92 public void testTokenizerWithoutSkipBlocks() throws Exception {
93 String test = IOUtils.toString(CPPTokenizerTest.class.getResourceAsStream("cpp/cpp_with_asm.cpp"));
94 Tokens tokens = parse(test, false);
95 assertEquals(37, tokens.size());
96 }
97
98 @Test
99
100 public void testAsmWithAtSign() {
101 Tokens tokens = parse(TEST7);
102 assertEquals(22, tokens.size());
103 }
104
105 @Test
106 public void testEOLCommentInPreprocessingDirective() {
107 parse("#define LSTFVLES_CPP //*" + PMD.EOL);
108 }
109
110 @Test
111 public void testEmptyCharacter() {
112 Tokens tokens = parse("std::wstring wsMessage( sMessage.length(), L'');" + PMD.EOL);
113 assertEquals(15, tokens.size());
114 }
115
116 @Test
117 public void testHexCharacter() {
118 Tokens tokens = parse("if (*pbuf == '\\0x05')" + PMD.EOL);
119 assertEquals(8, tokens.size());
120 }
121
122 @Test
123 public void testWhiteSpaceEscape() {
124 Tokens tokens = parse("szPath = m_sdcacheDir + _T(\"\\ oMedia\");" + PMD.EOL);
125 assertEquals(10, tokens.size());
126 }
127
128 private Tokens parse(String snippet) {
129 return parse(snippet, false);
130 }
131 private Tokens parse(String snippet, boolean skipBlocks) {
132 return parse(snippet, skipBlocks, null);
133 }
134 private Tokens parse(String snippet, boolean skipBlocks, String skipPattern) {
135 Properties properties = new Properties();
136 properties.setProperty(Tokenizer.OPTION_SKIP_BLOCKS, Boolean.toString(skipBlocks));
137 if (skipPattern != null) {
138 properties.setProperty(Tokenizer.OPTION_SKIP_BLOCKS_PATTERN, skipPattern);
139 }
140
141 CPPTokenizer tokenizer = new CPPTokenizer();
142 tokenizer.setProperties(properties);
143
144 SourceCode code = new SourceCode(new SourceCode.StringCodeLoader(snippet));
145 Tokens tokens = new Tokens();
146 tokenizer.tokenize(code, tokens);
147 return tokens;
148 }
149
150 private static final String TEST1 =
151 "#define FOO a +\\" + PMD.EOL +
152 " b +\\" + PMD.EOL +
153 " c +\\" + PMD.EOL +
154 " d +\\" + PMD.EOL +
155 " e +\\" + PMD.EOL +
156 " f +\\" + PMD.EOL +
157 " g" + PMD.EOL +
158 " void main() {}";
159
160 private static final String TEST2 =
161 " void main() { int x$y = 42; }";
162
163 private static final String TEST3 =
164 " void main() { int $x = 42; }";
165
166 private static final String TEST4 =
167 " void main() { char x = L'a'; }";
168
169 private static final String TEST5 =
170 "v\\" + PMD.EOL +
171 "o\\" + PMD.EOL +
172 "i\\" + PMD.EOL +
173 "d\\" + PMD.EOL +
174 " \\" + PMD.EOL +
175 "m\\" + PMD.EOL +
176 "a\\" + PMD.EOL +
177 "i\\" + PMD.EOL +
178 "n\\" + PMD.EOL +
179 "(\\" + PMD.EOL +
180 ")\\" + PMD.EOL +
181 " \\" + PMD.EOL +
182 "{\\" + PMD.EOL +
183 " \\" + PMD.EOL +
184 "}\\" + PMD.EOL;
185
186 private static final String TEST6 =
187 "#include <iostream>" + PMD.EOL +
188 PMD.EOL +
189 "int main()" + PMD.EOL +
190 "{" + PMD.EOL +
191 " std::cout << \"Hello, \" \\" + PMD.EOL +
192 " \"world!\\n\";" + PMD.EOL +
193 " return 0;" + PMD.EOL +
194 "}";
195
196 private static final String TEST7 =
197 "asm void eSPI_boot()" + PMD.EOL +
198 "{" + PMD.EOL +
199 " // setup stack pointer" + PMD.EOL +
200 " lis r1, _stack_addr@h" + PMD.EOL +
201 " ori r1, r1, _stack_addr@l" + PMD.EOL +
202 "}";
203
204 public static junit.framework.Test suite() {
205 return new junit.framework.JUnit4TestAdapter(CPPTokenizerTest.class);
206 }
207 }