1
2
3
4 package net.sourceforge.pmd.cpd;
5
6 import static org.junit.Assert.assertEquals;
7 import static org.junit.Assert.assertTrue;
8
9 import java.util.Properties;
10
11 import net.sourceforge.pmd.PMD;
12
13 import org.apache.commons.io.IOUtils;
14 import org.junit.Test;
15
16 public class CPPTokenizerTest {
17
18 @Test
19 public void testUTFwithBOM() {
20 Tokens tokens = parse("\ufeffint start()\n{ int ret = 1;\nreturn ret;\n}\n");
21 assertTrue(TokenEntry.getEOF() != tokens.getTokens().get(0));
22 assertEquals(15, tokens.size());
23 }
24
25 @Test
26 public void testUnicodeSupport() {
27 String code = "\ufeff" +
28 "#include <iostream>\n" +
29 "#include <string>\n" +
30 "\n" +
31 "// example\n" +
32 "\n" +
33 "int main()\n" +
34 "{\n" +
35 " std::string text(\"ąęćśźńó\");\n" +
36 " std::cout << text;\n" +
37 " return 0;\n" +
38 "}\n";
39 Tokens tokens = parse(code);
40 assertTrue(TokenEntry.getEOF() != tokens.getTokens().get(0));
41 assertEquals(24, tokens.size());
42 }
43
44 @Test
45 public void testMultiLineMacros() throws Throwable {
46 Tokens tokens = parse(TEST1);
47 assertEquals(7, tokens.size());
48 }
49
50 @Test
51 public void testDollarSignInIdentifier() {
52 parse(TEST2);
53 }
54
55 @Test
56 public void testDollarSignStartingIdentifier() {
57 parse(TEST3);
58 }
59
60 @Test
61 public void testWideCharacters() {
62 parse(TEST4);
63 }
64
65 @Test
66 public void testContinuation_IntraToken() {
67 Tokens tokens = parse(TEST5);
68 assertEquals(7, tokens.size());
69 }
70
71 @Test
72 public void testContinuation_InterToken() {
73 Tokens tokens = parse(TEST6);
74 assertEquals(17, tokens.size());
75 }
76
77 @Test
78 public void testTokenizerWithSkipBlocks() throws Exception {
79 String test = IOUtils.toString(CPPTokenizerTest.class.getResourceAsStream("cpp/cpp_with_asm.cpp"));
80 Tokens tokens = parse(test, true);
81 assertEquals(19, tokens.size());
82 }
83
84 @Test
85 public void testTokenizerWithSkipBlocksPattern() throws Exception {
86 String test = IOUtils.toString(CPPTokenizerTest.class.getResourceAsStream("cpp/cpp_with_asm.cpp"));
87 Tokens tokens = parse(test, true, "#if debug|#endif");
88 assertEquals(31, tokens.size());
89 }
90
91 @Test
92 public void testTokenizerWithoutSkipBlocks() throws Exception {
93 String test = IOUtils.toString(CPPTokenizerTest.class.getResourceAsStream("cpp/cpp_with_asm.cpp"));
94 Tokens tokens = parse(test, false);
95 assertEquals(37, tokens.size());
96 }
97
98 @Test
99
100 public void testAsmWithAtSign() {
101 Tokens tokens = parse(TEST7);
102 assertEquals(22, tokens.size());
103 }
104
105 @Test
106 public void testEOLCommentInPreprocessingDirective() {
107 parse("#define LSTFVLES_CPP //*" + PMD.EOL);
108 }
109
110 @Test
111 public void testEmptyCharacter() {
112 Tokens tokens = parse("std::wstring wsMessage( sMessage.length(), L'');" + PMD.EOL);
113 assertEquals(15, tokens.size());
114 }
115
116 @Test
117 public void testHexCharacter() {
118 Tokens tokens = parse("if (*pbuf == '\\0x05')" + PMD.EOL);
119 assertEquals(8, tokens.size());
120 }
121
122 @Test
123 public void testWhiteSpaceEscape() {
124 Tokens tokens = parse("szPath = m_sdcacheDir + _T(\"\\ oMedia\");" + PMD.EOL);
125 assertEquals(10, tokens.size());
126 }
127
128 @Test
129 public void testRawStringLiteral() {
130 String code =
131 "const char* const KDefaultConfig = R\"(\n" +
132 " [Sinks.1]\n" +
133 " Destination=Console\n" +
134 " AutoFlush=true\n" +
135 " Format=\"[%TimeStamp%] %ThreadId% %QueryIdHigh% %QueryIdLow% %LoggerFile%:%Line% (%Severity%) - %Message%\"\n" +
136 " Filter=\"%Severity% >= WRN\"\n" +
137 ")\";\n";
138 Tokens tokens = parse(code);
139 assertTrue(TokenEntry.getEOF() != tokens.getTokens().get(0));
140 assertEquals(9, tokens.size());
141 }
142
143 private Tokens parse(String snippet) {
144 return parse(snippet, false);
145 }
146 private Tokens parse(String snippet, boolean skipBlocks) {
147 return parse(snippet, skipBlocks, null);
148 }
149 private Tokens parse(String snippet, boolean skipBlocks, String skipPattern) {
150 Properties properties = new Properties();
151 properties.setProperty(Tokenizer.OPTION_SKIP_BLOCKS, Boolean.toString(skipBlocks));
152 if (skipPattern != null) {
153 properties.setProperty(Tokenizer.OPTION_SKIP_BLOCKS_PATTERN, skipPattern);
154 }
155
156 CPPTokenizer tokenizer = new CPPTokenizer();
157 tokenizer.setProperties(properties);
158
159 SourceCode code = new SourceCode(new SourceCode.StringCodeLoader(snippet));
160 Tokens tokens = new Tokens();
161 tokenizer.tokenize(code, tokens);
162 return tokens;
163 }
164
165 private static final String TEST1 =
166 "#define FOO a +\\" + PMD.EOL +
167 " b +\\" + PMD.EOL +
168 " c +\\" + PMD.EOL +
169 " d +\\" + PMD.EOL +
170 " e +\\" + PMD.EOL +
171 " f +\\" + PMD.EOL +
172 " g" + PMD.EOL +
173 " void main() {}";
174
175 private static final String TEST2 =
176 " void main() { int x$y = 42; }";
177
178 private static final String TEST3 =
179 " void main() { int $x = 42; }";
180
181 private static final String TEST4 =
182 " void main() { char x = L'a'; }";
183
184 private static final String TEST5 =
185 "v\\" + PMD.EOL +
186 "o\\" + PMD.EOL +
187 "i\\" + PMD.EOL +
188 "d\\" + PMD.EOL +
189 " \\" + PMD.EOL +
190 "m\\" + PMD.EOL +
191 "a\\" + PMD.EOL +
192 "i\\" + PMD.EOL +
193 "n\\" + PMD.EOL +
194 "(\\" + PMD.EOL +
195 ")\\" + PMD.EOL +
196 " \\" + PMD.EOL +
197 "{\\" + PMD.EOL +
198 " \\" + PMD.EOL +
199 "}\\" + PMD.EOL;
200
201 private static final String TEST6 =
202 "#include <iostream>" + PMD.EOL +
203 PMD.EOL +
204 "int main()" + PMD.EOL +
205 "{" + PMD.EOL +
206 " std::cout << \"Hello, \" \\" + PMD.EOL +
207 " \"world!\\n\";" + PMD.EOL +
208 " return 0;" + PMD.EOL +
209 "}";
210
211 private static final String TEST7 =
212 "asm void eSPI_boot()" + PMD.EOL +
213 "{" + PMD.EOL +
214 " // setup stack pointer" + PMD.EOL +
215 " lis r1, _stack_addr@h" + PMD.EOL +
216 " ori r1, r1, _stack_addr@l" + PMD.EOL +
217 "}";
218
219 public static junit.framework.Test suite() {
220 return new junit.framework.JUnit4TestAdapter(CPPTokenizerTest.class);
221 }
222 }