View Javadoc
1   /**
2    * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
3    */
4   
5   package net.sourceforge.pmd.cpd;
6   
7   import static org.junit.Assert.assertEquals;
8   
9   import java.util.List;
10  
11  import org.junit.Before;
12  import org.junit.Test;
13  
14  public class CsTokenizerTest {
15  
16      private CsTokenizer tokenizer = new CsTokenizer();
17  
18      private Tokens tokens;
19  
20      @Before
21      public void init() {
22  	tokens = new Tokens();
23  	TokenEntry.clearImages();
24      }
25  
26      @Test
27      public void testSimpleClass() {
28  	tokenizer.tokenize(toSourceCode("class Foo {}"), tokens);
29  	assertEquals(5, tokens.size());
30      }
31  
32      @Test
33      public void testSimpleClassDuplicatedTokens() {
34  	tokenizer.tokenize(toSourceCode("class Foo { class Foo { } }"), tokens);
35  	assertEquals(9, tokens.size());
36  	List<TokenEntry> tokenList = tokens.getTokens();
37  	assertEquals(tokenList.get(0).getIdentifier(), tokenList.get(3).getIdentifier());
38  	assertEquals(tokenList.get(1).getIdentifier(), tokenList.get(4).getIdentifier());
39  	assertEquals(tokenList.get(2).getIdentifier(), tokenList.get(5).getIdentifier());
40  	assertEquals(tokenList.get(6).getIdentifier(), tokenList.get(7).getIdentifier());
41      }
42  
43      @Test
44      public void testSimpleClassMethodMultipleLines() {
45  	tokenizer.tokenize(toSourceCode(
46  		"class Foo {\n"
47  			+ "  public String foo(int a) {\n"
48  			+ "    int i = a;\n"
49  			+ "    return \"x\" + a;\n"
50  			+ "  }\n"
51  			+ "}"), tokens);
52  	assertEquals(22, tokens.size());
53  	List<TokenEntry> tokenList = tokens.getTokens();
54  	assertEquals(1, tokenList.get(0).getBeginLine());
55  	assertEquals(2, tokenList.get(4).getBeginLine());
56  	assertEquals(3, tokenList.get(11).getBeginLine());
57      }
58  
59      @Test
60      public void testStrings() {
61  	tokenizer.tokenize(toSourceCode("String s =\"aaa \\\"b\\n\";"), tokens);
62  	assertEquals(5, tokens.size());
63      }
64  
65      @Test
66      public void testOpenString() {
67  	tokenizer.tokenize(toSourceCode("String s =\"aaa \\\"b\\"), tokens);
68  	assertEquals(5, tokens.size());
69      }
70  
71  
72      @Test
73      public void testCommentsIgnored1() {
74  	tokenizer.tokenize(toSourceCode("class Foo { /* class * ** X */ }"), tokens);
75  	assertEquals(5, tokens.size());
76      }
77  
78      @Test
79      public void testCommentsIgnored2() {
80  	tokenizer.tokenize(toSourceCode("class Foo { // class X /* aaa */ \n }"), tokens);
81  	assertEquals(5, tokens.size());
82      }
83  
84      @Test
85      public void testCommentsIgnored3() {
86  	tokenizer.tokenize(toSourceCode("class Foo { /// class X /* aaa */ \n }"), tokens);
87  	assertEquals(5, tokens.size());
88      }
89  
90      @Test
91      public void testMoreTokens() {
92  	tokenizer.tokenize(toSourceCode(
93  		"class Foo {\n"
94  			+ "  void bar() {\n"
95  			+ "    int a = 1 >> 2; \n"
96  			+ "    a += 1; \n"
97  			+ "    a++; \n"
98  			+ "    a /= 3e2; \n"
99  			+ "    float f = -3.1; \n"
100 			+ "    f *= 2; \n"
101 			+ "    bool b = ! (f == 2.0 || f >= 1.0 && f <= 2.0) \n"
102 			+ "  }\n"
103 			+ "}"
104 		), tokens);
105 	assertEquals(50, tokens.size());
106     }
107 
108     @Test
109     public void testLineNumberAfterMultilineComment() {
110 	tokenizer.tokenize(toSourceCode(
111 		"/* This is a multiline comment \n"
112 			+ " * \n"
113 			+ " * Lorem ipsum dolor sit amet, \n"
114 			+ " * consectetur adipiscing elit \n"
115 			+ " */\n"
116 			+ "\n"
117 			+ "class Foo {\n"
118 			+ "\n"
119 			+ "}"
120 		), tokens);
121 	assertEquals(5, tokens.size());
122 	assertEquals(7, tokens.getTokens().get(0).getBeginLine());
123     }
124 
125     @Test
126     public void testLineNumberAfterMultilineString() {
127 	tokenizer.tokenize(toSourceCode(
128 		"class Foo {\n"
129 			+ "  void bar() {\n"
130 			+ "    String query = \n"
131 			+ "      @\"SELECT foo, bar\n"
132 			+ "         FROM table \n"
133 			+ "         WHERE id = 42\"; \n"
134 			+ "  }\n"
135 			+ "}"
136 		), tokens);
137 	assertEquals(16, tokens.size());
138 	assertEquals(8, tokens.getTokens().get(14).getBeginLine());
139     }
140 
141     private SourceCode toSourceCode(String source) {
142 	return new SourceCode(new SourceCode.StringCodeLoader(source));
143     }
144 }