View Javadoc
1   /**
2    * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
3    */
4   
5   package net.sourceforge.pmd.cpd;
6   
7   import static org.junit.Assert.assertEquals;
8   import static org.junit.Assert.assertNotEquals;
9   
10  import java.util.List;
11  
12  import org.junit.Before;
13  import org.junit.Test;
14  
15  public class CsTokenizerTest {
16  
17      private CsTokenizer tokenizer;
18  
19      private Tokens tokens;
20  
21      @Before
22      public void init() {
23          tokenizer = new CsTokenizer();
24          tokens = new Tokens();
25          TokenEntry.clearImages();
26      }
27  
28      @Test
29      public void testSimpleClass() {
30          tokenizer.tokenize(toSourceCode("class Foo {}"), tokens);
31          assertEquals(5, tokens.size());
32      }
33  
34      @Test
35      public void testSimpleClassDuplicatedTokens() {
36          tokenizer.tokenize(toSourceCode("class Foo { class Foo { } }"), tokens);
37          assertEquals(9, tokens.size());
38          List<TokenEntry> tokenList = tokens.getTokens();
39          assertEquals(tokenList.get(0).getIdentifier(), tokenList.get(3).getIdentifier());
40          assertEquals(tokenList.get(1).getIdentifier(), tokenList.get(4).getIdentifier());
41          assertEquals(tokenList.get(2).getIdentifier(), tokenList.get(5).getIdentifier());
42          assertEquals(tokenList.get(6).getIdentifier(), tokenList.get(7).getIdentifier());
43      }
44  
45      @Test
46      public void testSimpleClassMethodMultipleLines() {
47          tokenizer.tokenize(toSourceCode(
48                  "class Foo {\n"
49                          + "  public String foo(int a) {\n"
50                          + "    int i = a;\n"
51                          + "    return \"x\" + a;\n"
52                          + "  }\n"
53                          + "}"), tokens);
54          assertEquals(22, tokens.size());
55          List<TokenEntry> tokenList = tokens.getTokens();
56          assertEquals(1, tokenList.get(0).getBeginLine());
57          assertEquals(2, tokenList.get(4).getBeginLine());
58          assertEquals(3, tokenList.get(11).getBeginLine());
59      }
60  
61      @Test
62      public void testStrings() {
63          tokenizer.tokenize(toSourceCode("String s =\"aaa \\\"b\\n\";"), tokens);
64          assertEquals(5, tokens.size());
65      }
66  
67      @Test
68      public void testOpenString() {
69          tokenizer.tokenize(toSourceCode("String s =\"aaa \\\"b\\"), tokens);
70          assertEquals(5, tokens.size());
71      }
72  
73  
74      @Test
75      public void testCommentsIgnored1() {
76          tokenizer.tokenize(toSourceCode("class Foo { /* class * ** X */ }"), tokens);
77          assertEquals(5, tokens.size());
78      }
79  
80      @Test
81      public void testCommentsIgnored2() {
82          tokenizer.tokenize(toSourceCode("class Foo { // class X /* aaa */ \n }"), tokens);
83          assertEquals(5, tokens.size());
84      }
85  
86      @Test
87      public void testCommentsIgnored3() {
88          tokenizer.tokenize(toSourceCode("class Foo { /// class X /* aaa */ \n }"), tokens);
89          assertEquals(5, tokens.size());
90      }
91  
92      @Test
93      public void testMoreTokens() {
94          tokenizer.tokenize(toSourceCode(
95                  "class Foo {\n"
96                          + "  void bar() {\n"
97                          + "    int a = 1 >> 2; \n"
98                          + "    a += 1; \n"
99                          + "    a++; \n"
100                         + "    a /= 3e2; \n"
101                         + "    float f = -3.1; \n"
102                         + "    f *= 2; \n"
103                         + "    bool b = ! (f == 2.0 || f >= 1.0 && f <= 2.0) \n"
104                         + "  }\n"
105                         + "}"
106                 ), tokens);
107         assertEquals(50, tokens.size());
108     }
109 
110     @Test
111     public void testLineNumberAfterMultilineComment() {
112         tokenizer.tokenize(toSourceCode(
113                 "/* This is a multiline comment \n"
114                         + " * \n"
115                         + " * Lorem ipsum dolor sit amet, \n"
116                         + " * consectetur adipiscing elit \n"
117                         + " */\n"
118                         + "\n"
119                         + "class Foo {\n"
120                         + "\n"
121                         + "}"
122                 ), tokens);
123         assertEquals(5, tokens.size());
124         assertEquals(7, tokens.getTokens().get(0).getBeginLine());
125     }
126 
127     @Test
128     public void testLineNumberAfterMultilineString() {
129         tokenizer.tokenize(toSourceCode(
130                 "class Foo {\n"
131                         + "  void bar() {\n"
132                         + "    String query = \n"
133                         + "      @\"SELECT foo, bar\n"
134                         + "         FROM table \n"
135                         + "         WHERE id = 42\"; \n"
136                         + "  }\n"
137                         + "}"
138                 ), tokens);
139         assertEquals(16, tokens.size());
140         assertEquals(8, tokens.getTokens().get(14).getBeginLine());
141     }
142 
143     @Test
144     public void testIgnoreUsingDirectives() {
145         tokenizer.setIgnoreUsings(true);
146         tokenizer.tokenize(toSourceCode("using System.Text;\n"), tokens);
147         assertNotEquals("using", tokens.getTokens().get(0).toString());
148         assertEquals(2, tokens.size());
149     }
150 
151     @Test
152     public void testUsingStatementsAreNotIgnored() {
153         tokenizer.setIgnoreUsings(true);
154         tokenizer.tokenize(toSourceCode(
155                 "using (Font font1 = new Font(\"Arial\", 10.0f)) {\n"
156                         + "  byte charset = font1.GdiCharSet;\n"
157                         + "}\n"
158                 ), tokens);
159         assertEquals("using", tokens.getTokens().get(0).toString());
160     }
161 
162     private SourceCode toSourceCode(String source) {
163         return new SourceCode(new SourceCode.StringCodeLoader(source));
164     }
165 }