View Javadoc
1   /**
2    * BSD-style license; for more info see http://pmd.sourceforge.net/license.html
3    */
4   package net.sourceforge.pmd.cpd;
5   
6   import static org.junit.Assert.assertEquals;
7   import static org.junit.Assert.assertTrue;
8   
9   import java.util.Properties;
10  
11  import net.sourceforge.pmd.PMD;
12  
13  import org.apache.commons.io.IOUtils;
14  import org.junit.Test;
15  
16  public class CPPTokenizerTest {
17  
18      @Test
19      public void testUTFwithBOM() {
20          Tokens tokens = parse("\ufeffint start()\n{ int ret = 1;\nreturn ret;\n}\n");
21          assertTrue(TokenEntry.getEOF() != tokens.getTokens().get(0));
22          assertEquals(15, tokens.size());
23      }
24  
25      @Test
26      public void testUnicodeSupport() {
27          String code = "\ufeff" +
28                  "#include <iostream>\n" +
29                  "#include <string>\n" +
30                  "\n" +
31                  "// example\n" +
32                  "\n" +
33                  "int main()\n" +
34                  "{\n" +
35                  "    std::string text(\"ąęćśźńó\");\n" +
36                  "    std::cout << text;\n" +
37                  "    return 0;\n" +
38                  "}\n";
39          Tokens tokens = parse(code);
40          assertTrue(TokenEntry.getEOF() != tokens.getTokens().get(0));
41          assertEquals(24, tokens.size());
42      }
43  
44      @Test
45      public void testMultiLineMacros() throws Throwable {
46          Tokens tokens = parse(TEST1);
47          assertEquals(7, tokens.size());
48      }
49  
50      @Test
51      public void testDollarSignInIdentifier() {
52          parse(TEST2);
53      }
54  
55      @Test
56      public void testDollarSignStartingIdentifier() {
57          parse(TEST3);
58      }
59  
60      @Test
61      public void testWideCharacters() {
62          parse(TEST4);
63      }
64  
65      @Test
66      public void testContinuation_IntraToken() {
67      	Tokens tokens = parse(TEST5);
68          assertEquals(7, tokens.size());
69      }
70      
71      @Test
72      public void testContinuation_InterToken() {
73      	Tokens tokens = parse(TEST6);
74      	assertEquals(17, tokens.size());
75      }
76  
77      @Test
78      public void testTokenizerWithSkipBlocks() throws Exception {
79          String test = IOUtils.toString(CPPTokenizerTest.class.getResourceAsStream("cpp/cpp_with_asm.cpp"));
80          Tokens tokens = parse(test, true);
81          assertEquals(19, tokens.size());
82      }
83  
84      @Test
85      public void testTokenizerWithSkipBlocksPattern() throws Exception {
86          String test = IOUtils.toString(CPPTokenizerTest.class.getResourceAsStream("cpp/cpp_with_asm.cpp"));
87          Tokens tokens = parse(test, true, "#if debug|#endif");
88          assertEquals(31, tokens.size());
89      }
90  
91      @Test
92      public void testTokenizerWithoutSkipBlocks() throws Exception {
93          String test = IOUtils.toString(CPPTokenizerTest.class.getResourceAsStream("cpp/cpp_with_asm.cpp"));
94          Tokens tokens = parse(test, false);
95          assertEquals(37, tokens.size());
96      }
97  
98      @Test
99      //ASM code containing the '@' character
100     public void testAsmWithAtSign() {
101         Tokens tokens = parse(TEST7);
102         assertEquals(22, tokens.size());
103     }
104 
105     @Test
106     public void testEOLCommentInPreprocessingDirective() {
107         parse("#define LSTFVLES_CPP  //*" + PMD.EOL);
108     }
109 
110     @Test
111     public void testEmptyCharacter() {
112         Tokens tokens = parse("std::wstring wsMessage( sMessage.length(), L'');" + PMD.EOL);
113         assertEquals(15, tokens.size());
114     }
115 
116     @Test
117     public void testHexCharacter() {
118         Tokens tokens = parse("if (*pbuf == '\\0x05')" + PMD.EOL);
119         assertEquals(8, tokens.size());
120     }
121 
122     @Test
123     public void testWhiteSpaceEscape() {
124         Tokens tokens = parse("szPath = m_sdcacheDir + _T(\"\\    oMedia\");" + PMD.EOL);
125         assertEquals(10, tokens.size());
126     }
127     
128     @Test
129     public void testRawStringLiteral() {
130         String code =
131                 "const char* const KDefaultConfig = R\"(\n" +
132                 "    [Sinks.1]\n" +
133                 "    Destination=Console\n" +
134                 "    AutoFlush=true\n" +
135                 "    Format=\"[%TimeStamp%] %ThreadId% %QueryIdHigh% %QueryIdLow% %LoggerFile%:%Line% (%Severity%) - %Message%\"\n" +
136                 "    Filter=\"%Severity% >= WRN\"\n" +
137                 ")\";\n";
138         Tokens tokens = parse(code);
139         assertTrue(TokenEntry.getEOF() != tokens.getTokens().get(0));
140         assertEquals(9, tokens.size());
141     }
142 
143     private Tokens parse(String snippet) {
144         return parse(snippet, false);
145     }
146     private Tokens parse(String snippet, boolean skipBlocks) {
147         return parse(snippet, skipBlocks, null);
148     }
149     private Tokens parse(String snippet, boolean skipBlocks, String skipPattern) {
150         Properties properties = new Properties();
151         properties.setProperty(Tokenizer.OPTION_SKIP_BLOCKS, Boolean.toString(skipBlocks));
152         if (skipPattern != null) {
153             properties.setProperty(Tokenizer.OPTION_SKIP_BLOCKS_PATTERN, skipPattern);
154         }
155 
156         CPPTokenizer tokenizer = new CPPTokenizer();
157         tokenizer.setProperties(properties);
158 
159         SourceCode code = new SourceCode(new SourceCode.StringCodeLoader(snippet));
160         Tokens tokens = new Tokens();
161         tokenizer.tokenize(code, tokens);
162         return tokens;
163     }
164 
165     private static final String TEST1 =
166             "#define FOO a +\\" + PMD.EOL +
167             "            b +\\" + PMD.EOL +
168             "            c +\\" + PMD.EOL +
169             "            d +\\" + PMD.EOL +
170             "            e +\\" + PMD.EOL +
171             "            f +\\" + PMD.EOL +
172             "            g" + PMD.EOL +
173             " void main() {}";
174 
175     private static final String TEST2 =
176             " void main() { int x$y = 42; }";
177 
178     private static final String TEST3 =
179             " void main() { int $x = 42; }";
180 
181     private static final String TEST4 =
182             " void main() { char x = L'a'; }";
183     
184     private static final String TEST5 =
185             "v\\" + PMD.EOL +
186             "o\\" + PMD.EOL +
187             "i\\" + PMD.EOL +
188             "d\\" + PMD.EOL +
189             " \\" + PMD.EOL +
190             "m\\" + PMD.EOL +
191             "a\\" + PMD.EOL +
192             "i\\" + PMD.EOL +
193             "n\\" + PMD.EOL +
194             "(\\" + PMD.EOL +
195             ")\\" + PMD.EOL +
196             " \\" + PMD.EOL +
197             "{\\" + PMD.EOL +
198             " \\" + PMD.EOL +
199             "}\\" + PMD.EOL;
200     
201     private static final String TEST6 =
202             "#include <iostream>" + PMD.EOL +
203             PMD.EOL +
204             "int main()" + PMD.EOL +
205             "{" + PMD.EOL +
206             "   std::cout << \"Hello, \" \\" + PMD.EOL +
207             "                \"world!\\n\";" + PMD.EOL +
208             "   return 0;" + PMD.EOL +
209             "}";
210 
211     private static final String TEST7 =
212             "asm void eSPI_boot()" + PMD.EOL +
213             "{" + PMD.EOL +
214             "  // setup stack pointer" + PMD.EOL +
215             "  lis r1, _stack_addr@h" + PMD.EOL +
216             "  ori r1, r1, _stack_addr@l"  + PMD.EOL +
217             "}";
218 
219     public static junit.framework.Test suite() {
220         return new junit.framework.JUnit4TestAdapter(CPPTokenizerTest.class);
221     }
222 }