NewLang Project
Yet another programm language
Loading...
Searching...
No Matches
lexer_test.cpp
Go to the documentation of this file.
1#ifdef BUILD_UNITTEST
2
3#include "warning_push.h"
4#include <gtest/gtest.h>
5#include "warning_pop.h"
6
7#include "term.h"
8#include "lexer.h"
9#include "parser.h"
10#include "macro.h"
11
12
13using namespace newlang;
14
15class Lexer : public ::testing::Test {
16protected:
17
18 std::vector<TermPtr> tokens;
19
20 void SetUp() {
21 }
22
23 void TearDown() {
24 }
25
26 int64_t TokenParse(const char *str, bool ignore_space = true, bool ignore_indent = true, bool ignore_comment = true, bool ignore_crlf = true) {
27 std::istringstream strstr(str);
28
29 Scanner lexer(&strstr);
30
31 lexer.m_ignore_indent = ignore_indent;
32 lexer.m_ignore_space = ignore_space;
33 lexer.m_ignore_comment = ignore_comment;
34 lexer.m_ignore_crlf = ignore_crlf;
35
36 tokens.clear();
37 TermPtr tok;
38 parser::location_type loc;
39 while (lexer.lex(&tok, &loc) != parser::token::END) {
40 tokens.push_back(tok);
41 }
42 return tokens.size();
43 }
44
45 int Count(TermID token_id) {
46 int result = 0;
47 for (size_t i = 0; i < tokens.size(); i++) {
48 if (tokens[i]->getTermID() == token_id) {
49 result++;
50 }
51 }
52 return result;
53 }
54
55 std::string Dump() {
56 std::string result;
57 for (int i = 0; i < tokens.size(); i++) {
58 result += tokens[i]->m_text;
59 result += ":";
60 result += toString(tokens[i]->m_id);
61 result += " ";
62 }
63 return result;
64 }
65
66};
67
68TEST_F(Lexer, Word) {
69 ASSERT_EQ(1, TokenParse("alpha "));
70 EXPECT_EQ(1, Count(TermID::NAME));
71 EXPECT_STREQ("alpha", tokens[0]->getText().c_str());
72
73
74 ASSERT_EQ(2, TokenParse("буквы ещёЁ_99"));
75 EXPECT_EQ(2, Count(TermID::NAME));
76 EXPECT_STREQ("буквы", tokens[0]->getText().c_str());
77 EXPECT_STREQ("ещёЁ_99", tokens[1]->getText().c_str());
78
79 ASSERT_EQ(3, TokenParse("one two \t three"));
80 EXPECT_EQ(3, Count(TermID::NAME));
81
82 EXPECT_STREQ("one", tokens[0]->getText().c_str()) << tokens[0]->getText().c_str();
83 EXPECT_STREQ("two", tokens[1]->getText().c_str()) << tokens[1]->getText().c_str();
84 EXPECT_STREQ("three", tokens[2]->getText().c_str()) << tokens[2]->getText().c_str();
85}
86
87TEST_F(Lexer, Template) {
88 ASSERT_EQ(1, TokenParse("\"\"\"\"\"\""));
89 EXPECT_EQ(1, Count(TermID::TEMPLATE));
90 EXPECT_STREQ("", tokens[0]->getText().c_str()) << tokens[0]->getText();
91}
92
93TEST_F(Lexer, Template2) {
94 ASSERT_EQ(1, TokenParse("\"\"\" ${123} \n \"\"\""));
95 EXPECT_EQ(1, Count(TermID::TEMPLATE));
96 EXPECT_STREQ(" ${123} \n ", tokens[0]->getText().c_str()) << tokens[0]->getText();
97}
98
99TEST_F(Lexer, Template3) {
100 ASSERT_EQ(1, TokenParse("''' ${123} \n\t '''"));
101 EXPECT_EQ(1, Count(TermID::TEMPLATE));
102 EXPECT_STREQ(" ${123} \n\t ", tokens[0]->getText().c_str()) << tokens[0]->getText();
103}
104
105TEST_F(Lexer, StringEmpty) {
106 ASSERT_EQ(0, TokenParse(""));
107 ASSERT_EQ(1, TokenParse("''"));
108 EXPECT_EQ(1, Count(TermID::STRCHAR));
109 EXPECT_STREQ("", tokens[0]->getText().c_str()) << tokens[0]->getText();
110}
111
112TEST_F(Lexer, StringEmpty2) {
113 ASSERT_EQ(0, TokenParse(""));
114 ASSERT_EQ(1, TokenParse("\"\""));
115 EXPECT_EQ(1, Count(TermID::STRWIDE));
116 EXPECT_STREQ("", tokens[0]->getText().c_str()) << tokens[0]->getText();
117}
118
119TEST_F(Lexer, StringSimple) {
120 ASSERT_EQ(1, TokenParse("' '"));
121 EXPECT_EQ(1, Count(TermID::STRCHAR));
122 EXPECT_STREQ(" ", tokens[0]->getText().c_str()) << tokens[0]->getText();
123}
124
125TEST_F(Lexer, StringSimple2) {
126 ASSERT_EQ(1, TokenParse("\" \""));
127 EXPECT_EQ(1, Count(TermID::STRWIDE));
128 EXPECT_STREQ(" ", tokens[0]->getText().c_str()) << tokens[0]->getText();
129}
130
131TEST_F(Lexer, FullString) {
132 ASSERT_EQ(1, TokenParse("' \t \xFF\r\\''"));
133 EXPECT_EQ(1, Count(TermID::STRCHAR));
134 EXPECT_STREQ(" \t \xFF\r'", tokens[0]->getText().c_str()) << tokens[0]->getText();
135}
136
137TEST_F(Lexer, FullString2) {
138 ASSERT_EQ(1, TokenParse("\" \t \xFF\r\\\"\""));
139 EXPECT_EQ(1, Count(TermID::STRWIDE));
140 EXPECT_STREQ(" \t \xFF\r\"", tokens[0]->getText().c_str()) << tokens[0]->getText();
141}
142
143TEST_F(Lexer, Integer) {
144 ASSERT_EQ(1, TokenParse("123456"));
145 EXPECT_EQ(1, Count(TermID::INTEGER)) << newlang::toString(tokens[0]->getTermID());
146
147 EXPECT_STREQ("123456", tokens[0]->getText().c_str());
148
149 ASSERT_EQ(3, TokenParse("123456 * 123"));
150 EXPECT_EQ(1, Count(TermID::SYMBOL)) << Dump();
151 EXPECT_EQ(2, Count(TermID::INTEGER)) << Dump();
152
153 EXPECT_STREQ("123456", tokens[0]->getText().c_str()) << tokens[0]->getText();
154 EXPECT_STREQ("*", tokens[1]->getText().c_str()) << tokens[1]->getText();
155 EXPECT_STREQ("123", tokens[2]->getText().c_str()) << tokens[2]->getText();
156}
157
158TEST_F(Lexer, Float) {
159 ASSERT_EQ(1, TokenParse("1.e10"));
160 EXPECT_EQ(1, Count(TermID::NUMBER));
161 EXPECT_STREQ("1.e10", tokens[0]->getText().c_str());
162}
163
164TEST_F(Lexer, Complex0) {
165 if (1 != TokenParse("1j")) {
166 for (auto elem : tokens) {
167 std::cout << newlang::toString(elem->m_id) << " " << elem->m_text << "\n";
168 }
169 }
170 ASSERT_EQ(1, tokens.size());
171 EXPECT_EQ(1, Count(TermID::COMPLEX)) << newlang::toString(tokens[0]->m_id);
172 EXPECT_STREQ("1j", tokens[0]->getText().c_str());
173
174 if (1 != TokenParse("-1i")) {
175 for (auto elem : tokens) {
176 std::cout << newlang::toString(elem->m_id) << " " << elem->m_text << "\n";
177 }
178 }
179 ASSERT_EQ(1, tokens.size());
180 EXPECT_EQ(1, Count(TermID::COMPLEX)) << newlang::toString(tokens[0]->m_id);
181 EXPECT_STREQ("-1i", tokens[0]->getText().c_str());
182
183 if (1 != TokenParse("-1j-0.2")) {
184 for (auto elem : tokens) {
185 std::cout << newlang::toString(elem->m_id) << " " << elem->m_text << "\n";
186 }
187 }
188 ASSERT_EQ(1, tokens.size());
189 EXPECT_EQ(1, Count(TermID::COMPLEX)) << newlang::toString(tokens[0]->m_id);
190 EXPECT_STREQ("-1j-0.2", tokens[0]->getText().c_str());
191
192 if (1 != TokenParse("1i+2")) {
193 for (auto elem : tokens) {
194 std::cout << newlang::toString(elem->m_id) << " " << elem->m_text << "\n";
195 }
196 }
197 ASSERT_EQ(1, tokens.size());
198 EXPECT_EQ(1, Count(TermID::COMPLEX)) << newlang::toString(tokens[0]->m_id);
199 EXPECT_STREQ("1i+2", tokens[0]->getText().c_str());
200
201 if (1 != TokenParse("1j+0.2")) {
202 for (auto elem : tokens) {
203 std::cout << newlang::toString(elem->m_id) << " " << elem->m_text << "\n";
204 }
205 }
206 ASSERT_EQ(1, tokens.size());
207 EXPECT_EQ(1, Count(TermID::COMPLEX)) << newlang::toString(tokens[0]->m_id);
208 EXPECT_STREQ("1j+0.2", tokens[0]->getText().c_str());
209}
210
211TEST_F(Lexer, Complex1) {
212 if (1 != TokenParse("1.333+0.e10j")) {
213 for (auto elem : tokens) {
214 std::cout << newlang::toString(elem->m_id) << " " << elem->m_text << "\n";
215 }
216 }
217 ASSERT_EQ(1, tokens.size());
218 EXPECT_EQ(1, Count(TermID::COMPLEX));
219 EXPECT_STREQ("1.333+0.e10j", tokens[0]->getText().c_str());
220}
221
222TEST_F(Lexer, Term) {
223
224 if (1 != TokenParse("$alpha ")) {
225 for (auto elem : tokens) {
226 std::cout << newlang::toString(elem->m_id) << " " << elem->m_text << "\n";
227 }
228
229 }
230 ASSERT_EQ(1, tokens.size());
231 EXPECT_EQ(1, Count(TermID::LOCAL)) << Dump();
232 EXPECT_STREQ("$alpha", tokens[0]->getText().c_str());
233
234
235 ASSERT_EQ(2, TokenParse("буквы ещёЁ_99"));
236 EXPECT_EQ(2, Count(TermID::NAME));
237 EXPECT_STREQ("буквы", tokens[0]->getText().c_str());
238 EXPECT_STREQ("ещёЁ_99", tokens[1]->getText().c_str());
239
240 ASSERT_EQ(5, TokenParse("one \\two \\\\two \t $three @four")) << Dump();
241 EXPECT_EQ(1, Count(TermID::NAME)) << Dump();
242 EXPECT_EQ(1, Count(TermID::LOCAL)) << Dump();
243 EXPECT_EQ(1, Count(TermID::MACRO)) << Dump();
244 EXPECT_EQ(2, Count(TermID::MODULE)) << Dump();
245
246 EXPECT_STREQ("one", tokens[0]->getText().c_str()) << tokens[0]->getText().c_str();
247 EXPECT_STREQ("\\two", tokens[1]->getText().c_str()) << tokens[1]->getText().c_str();
248 EXPECT_STREQ("\\\\two", tokens[2]->getText().c_str()) << tokens[2]->getText().c_str();
249 EXPECT_STREQ("$three", tokens[3]->getText().c_str()) << tokens[3]->getText().c_str();
250 EXPECT_STREQ("@four", tokens[4]->getText().c_str()) << tokens[4]->getText().c_str();
251}
252
253TEST_F(Lexer, AssignEq) {
254 ASSERT_EQ(3, TokenParse("token=ssssssss"));
255 EXPECT_EQ(2, Count(TermID::NAME));
256 EXPECT_EQ(1, Count(TermID::SYMBOL));
257
258 EXPECT_STREQ("token", tokens[0]->getText().c_str()) << tokens[0]->getText();
259 EXPECT_STREQ("ssssssss", tokens[2]->getText().c_str()) << tokens[2]->getText();
260
261 ASSERT_EQ(3, TokenParse("token:=\"ssssssss\""));
262 EXPECT_EQ(1, Count(TermID::NAME));
263 EXPECT_EQ(1, Count(TermID::CREATE_USE));
264 EXPECT_EQ(1, Count(TermID::STRWIDE));
265
266 EXPECT_STREQ("token", tokens[0]->getText().c_str()) << tokens[0]->getText();
267 EXPECT_STREQ("ssssssss", tokens[2]->getText().c_str()) << tokens[2]->getText();
268
269 ASSERT_EQ(3, TokenParse(" token \t ::= 'ssssssss' "));
270 EXPECT_EQ(1, Count(TermID::NAME));
271 EXPECT_EQ(1, Count(TermID::CREATE_NEW));
272 EXPECT_EQ(1, Count(TermID::STRCHAR));
273
274 EXPECT_STREQ("token", tokens[0]->getText().c_str()) << tokens[0]->getText();
275 EXPECT_STREQ("ssssssss", tokens[2]->getText().c_str()) << tokens[2]->getText();
276}
277
278TEST_F(Lexer, CodeInner) {
279 ASSERT_EQ(3, TokenParse("{%if(){%} {%}else{%} {%} %}"));
280 EXPECT_EQ(3, Count(TermID::EMBED));
281 EXPECT_STREQ("if(){", tokens[0]->getText().c_str()) << tokens[0]->getText().c_str();
282 EXPECT_STREQ("}else{", tokens[1]->getText().c_str()) << tokens[1]->getText().c_str();
283 EXPECT_STREQ("} ", tokens[2]->getText().c_str()) << tokens[2]->getText().c_str();
284
285 ASSERT_EQ(5, TokenParse("{ {%if(){%} {%}else{%} {%} %} }"));
286 EXPECT_EQ(2, Count(TermID::SYMBOL));
287 EXPECT_EQ(3, Count(TermID::EMBED));
288 EXPECT_STREQ("{", tokens[0]->getText().c_str()) << tokens[0]->getText().c_str();
289 EXPECT_STREQ("if(){", tokens[1]->getText().c_str()) << tokens[1]->getText().c_str();
290 EXPECT_STREQ("}else{", tokens[2]->getText().c_str()) << tokens[2]->getText().c_str();
291 EXPECT_STREQ("} ", tokens[3]->getText().c_str()) << tokens[3]->getText().c_str();
292 EXPECT_STREQ("}", tokens[4]->getText().c_str()) << tokens[4]->getText().c_str();
293}
294
295TEST_F(Lexer, Code) {
296 ASSERT_EQ(2, TokenParse("{ }"));
297 EXPECT_EQ(2, Count(TermID::SYMBOL));
298 EXPECT_STREQ("{", tokens[0]->getText().c_str()) << tokens[0]->getText().c_str();
299 EXPECT_STREQ("}", tokens[1]->getText().c_str()) << tokens[1]->getText().c_str();
300
301 ASSERT_EQ(4, TokenParse("{ { } }"));
302 EXPECT_EQ(4, Count(TermID::SYMBOL));
303}
304
305TEST_F(Lexer, CodeSource) {
306 ASSERT_EQ(1, TokenParse("{%%}"));
307 EXPECT_EQ(1, Count(TermID::EMBED));
308 EXPECT_STREQ("", tokens[0]->getText().c_str()) << tokens[0]->getText().c_str();
309
310 ASSERT_EQ(1, TokenParse("{% % %}"));
311 ASSERT_EQ(1, Count(TermID::EMBED));
312 ASSERT_STREQ(" % ", tokens[0]->getText().c_str()) << tokens[0]->getText().c_str();
313}
314
315TEST_F(Lexer, Assign) {
316 ASSERT_EQ(5, TokenParse(":= :- ::= ::- ="));
317 EXPECT_EQ(1, Count(TermID::CREATE_NEW));
318 EXPECT_EQ(1, Count(TermID::CREATE_USE));
319 EXPECT_EQ(1, Count(TermID::PURE_USE));
320 EXPECT_EQ(1, Count(TermID::PURE_NEW));
321}
322
323TEST_F(Lexer, Function) {
324 ASSERT_EQ(1, TokenParse("\\name")) << Dump();
325 EXPECT_EQ(1, Count(TermID::MODULE)) << toString(tokens[0]->getTermID());
326 EXPECT_STREQ("\\name", tokens[0]->getText().c_str()) << tokens[0]->getText().c_str();
327
328 ASSERT_EQ(1, TokenParse("\\\\name"));
329 EXPECT_EQ(1, Count(TermID::MODULE)) << toString(tokens[0]->getTermID());
330 EXPECT_STREQ("\\\\name", tokens[0]->getText().c_str()) << tokens[0]->getText().c_str();
331
332 ASSERT_EQ(1, TokenParse("$name"));
333 EXPECT_EQ(1, Count(TermID::LOCAL)) << toString(tokens[0]->getTermID());
334 EXPECT_STREQ("$name", tokens[0]->getText().c_str()) << tokens[0]->getText().c_str();
335
336 ASSERT_EQ(2, TokenParse("%native"));
337 EXPECT_EQ(1, Count(TermID::SYMBOL)) << toString(tokens[0]->getTermID());
338 EXPECT_EQ(1, Count(TermID::NAME)) << toString(tokens[1]->getTermID());
339 EXPECT_STREQ("%", tokens[0]->getText().c_str()) << tokens[0]->getText().c_str();
340 EXPECT_STREQ("native", tokens[1]->getText().c_str()) << tokens[1]->getText().c_str();
341
342 ASSERT_EQ(1, TokenParse("@name"));
343 EXPECT_EQ(1, Count(TermID::MACRO));
344 EXPECT_STREQ("@name", tokens[0]->getText().c_str()) << tokens[0]->getText().c_str();
345
346 ASSERT_EQ(1, TokenParse("@функция_alpha_ёЁ"));
347 EXPECT_EQ(1, Count(TermID::MACRO));
348 EXPECT_STREQ("@функция_alpha_ёЁ", tokens[0]->getText().c_str()) << tokens[0]->getText().c_str();
349}
350
351TEST_F(Lexer, Sentence) {
352 ASSERT_EQ(2, TokenParse("token."));
353 EXPECT_EQ(1, Count(TermID::NAME));
354 ASSERT_EQ(2, TokenParse("token;"));
355 EXPECT_EQ(1, Count(TermID::NAME));
356}
357
358TEST_F(Lexer, Comment) {
359 ASSERT_EQ(0, TokenParse("# lskdafj ldsjf ldkjfa l;sdj fl;k"));
360 ASSERT_EQ(0, TokenParse("#! lskdafj ldsjf ldkjfa l;sdj fl;k\n "));
361
362 ASSERT_EQ(1, TokenParse("/***** lskdafj\n\n\n\n ldsjf ldkjfa l;sdj fl;k*****/ "));
363 EXPECT_EQ(1, Count(TermID::DOC_BEFORE));
364
365 ASSERT_EQ(1, TokenParse("/// lskdafj"));
366 EXPECT_EQ(1, Count(TermID::DOC_BEFORE));
367 ASSERT_EQ(1, TokenParse("/// lskdafj\n"));
368 EXPECT_EQ(1, Count(TermID::DOC_BEFORE));
369 ASSERT_EQ(1, TokenParse("///< lskdafj"));
370 EXPECT_EQ(1, Count(TermID::DOC_AFTER));
371 ASSERT_EQ(1, TokenParse("///< lskdafj\n"));
372 EXPECT_EQ(1, Count(TermID::DOC_AFTER));
373}
374
375TEST_F(Lexer, Comment2) {
376 ASSERT_EQ(2, TokenParse("#!22\n#1\nterm;\n"));
377 // EXPECT_EQ(2, Count(TermID::COMMENT));
378 EXPECT_EQ(1, Count(TermID::SYMBOL));
379 EXPECT_EQ(1, Count(TermID::NAME));
380
381 ASSERT_EQ(1, TokenParse("\n\n/* \n\n*/\n\n term"));
382 // EXPECT_EQ(1, Count(TermID::COMMENT));
383 EXPECT_EQ(1, Count(TermID::NAME));
384 // EXPECT_STREQ(" \n\n", tokens[0]->getText().c_str()) << tokens[0]->getText().c_str();
385 // EXPECT_EQ(5, tokens[0]->m_line);
386 // EXPECT_EQ(1, tokens[0]->m_col);
387 EXPECT_STREQ("term", tokens[0]->getText().c_str()) << tokens[1]->getText().c_str();
388 EXPECT_EQ(7, tokens[0]->m_line) << Dump();
389 EXPECT_EQ(7, tokens[0]->m_col) << Dump();
390}
391
392TEST_F(Lexer, Paren) {
393 ASSERT_EQ(3, TokenParse("\\name()")) << Dump();
394 EXPECT_EQ(1, Count(TermID::MODULE));
395 EXPECT_EQ(2, Count(TermID::SYMBOL));
396
397
398 ASSERT_EQ(4, TokenParse("%функция_alpha_ёЁ ()"));
399 EXPECT_EQ(1, Count(TermID::NAME));
400 EXPECT_EQ(3, Count(TermID::SYMBOL));
401
402}
403
404TEST_F(Lexer, Module) {
405 ASSERT_EQ(1, TokenParse("\\name")) << Dump();
406 EXPECT_EQ(1, Count(TermID::MODULE));
407
408 ASSERT_EQ(1, TokenParse("\\\\dir\\module"));
409 EXPECT_EQ(1, Count(TermID::MODULE));
410
411 ASSERT_EQ(1, TokenParse("\\dir\\dir\\module"));
412 EXPECT_EQ(1, Count(TermID::MODULE));
413
414 ASSERT_EQ(3, TokenParse("\\name::var")) << Dump();
415 EXPECT_EQ(1, Count(TermID::MODULE));
416
417 ASSERT_EQ(5, TokenParse("\\\\dir\\module::var.filed")) << Dump();
418 EXPECT_EQ(1, Count(TermID::MODULE));
419
420 ASSERT_EQ(5, TokenParse("\\dir\\dir\\module::var.filed")) << Dump();
421 EXPECT_EQ(1, Count(TermID::MODULE));
422}
423
424TEST_F(Lexer, Arg) {
425 ASSERT_EQ(7, TokenParse("term(name=value);"));
426 EXPECT_EQ(3, Count(TermID::NAME));
427 EXPECT_EQ(4, Count(TermID::SYMBOL));
428}
429
430TEST_F(Lexer, Args) {
431 ASSERT_EQ(11, TokenParse("$0 $1 $22 $333 $4sss $sss1 -- ++ $* $^ ")) << Dump();
432 EXPECT_EQ(5, Count(TermID::ARGUMENT)) << Dump();
433 EXPECT_EQ(2, Count(TermID::ARGS)) << Dump();
434 EXPECT_EQ(1, Count(TermID::INT_PLUS)) << Dump();
435 EXPECT_EQ(1, Count(TermID::INT_MINUS)) << Dump();
436 EXPECT_EQ(1, Count(TermID::NAME)) << Dump();
437 EXPECT_EQ(1, Count(TermID::LOCAL)) << Dump();
438}
439
440TEST_F(Lexer, MutArgs) {
441 ASSERT_EQ(11, TokenParse("$0^ $1^ $22 $333 $4sss^ $sss1^ -- ++ $* $^ ")) << Dump();
442 EXPECT_EQ(5, Count(TermID::ARGUMENT)) << Dump();
443 EXPECT_EQ(2, Count(TermID::ARGS)) << Dump();
444 EXPECT_EQ(1, Count(TermID::INT_PLUS)) << Dump();
445 EXPECT_EQ(1, Count(TermID::INT_MINUS)) << Dump();
446 EXPECT_EQ(1, Count(TermID::NAME)) << Dump();
447 EXPECT_EQ(1, Count(TermID::LOCAL)) << Dump();
448}
449
450//TEST_F(Lexer, MutLink) {
451// ASSERT_EQ(11, TokenParse("&0^ &1^ &22 &333 &4sss^ &sss1^ -- ++ &* &^ ")) << Dump();
452// EXPECT_EQ(5, Count(TermID::ARGUMENT)) << Dump();
453// EXPECT_EQ(2, Count(TermID::ARGS)) << Dump();
454// EXPECT_EQ(1, Count(TermID::INT_PLUS)) << Dump();
455// EXPECT_EQ(1, Count(TermID::INT_MINUS)) << Dump();
456// EXPECT_EQ(1, Count(TermID::NAME)) << Dump();
457// EXPECT_EQ(1, Count(TermID::LOCAL)) << Dump();
458//}
459
460/* @{ }@
461 * {@ @}
462 *
463 * SyncNone
464 * SyncTimedMutex
465 * SyncTimedShared
466 * SyncSingleThread
467 *
468 * {@ &0 ::= SyncNone @}; # *
469 * {@ &1 ::= SyncSingleThread @}; # &
470 * {@ &2 ::= SyncTimedShared @}; # &&
471 * {@ &3 ::= SyncTimedSharedRecursive @}; # &*
472 *
473 * [@ unused @]
474 * [@ unsafe(on) @]
475 * [@ warning(no=100, message="ddddddddd") @]
476 *
477 * {@ func.unused ::= 1 @}; # &*
478 *
479 */
480TEST_F(Lexer, UTF8) {
481 ASSERT_EQ(7, TokenParse("термин(имя=значение);"));
482 EXPECT_EQ(3, Count(TermID::NAME)) << Dump();
483 EXPECT_EQ(4, Count(TermID::SYMBOL)) << Dump();
484}
485
486TEST_F(Lexer, MutUTF8) {
487 ASSERT_EQ(7, TokenParse("термин^(имя^=значение);"));
488 EXPECT_EQ(3, Count(TermID::NAME)) << Dump();
489 EXPECT_EQ(4, Count(TermID::SYMBOL)) << Dump();
490 ASSERT_EQ(7, TokenParse("термин^(имя^=значение);"));
491 EXPECT_EQ(3, Count(TermID::NAME)) << Dump();
492 EXPECT_EQ(4, Count(TermID::SYMBOL)) << Dump();
493}
494
495TEST_F(Lexer, ELLIPSIS) {
496 ASSERT_EQ(2, TokenParse("... ...")) << Dump();
497 EXPECT_EQ(2, Count(TermID::ELLIPSIS)) << Dump();
498}
499
500TEST_F(Lexer, Alias) {
501 ASSERT_EQ(5, TokenParse("+>:<-")) << Dump();
502 EXPECT_EQ(5, Count(TermID::SYMBOL)) << Dump();
503
504 ASSERT_EQ(4, TokenParse("@alias := @ALIAS;")) << Dump();
505 EXPECT_EQ(2, Count(TermID::MACRO)) << Dump();
506
507 ASSERT_EQ(7, TokenParse("/** Comment */@@ alias2 @@ ALIAS2@@///< Комментарий")) << Dump();
508 EXPECT_EQ(1, Count(TermID::DOC_BEFORE));
509 EXPECT_EQ(1, Count(TermID::DOC_AFTER));
510 EXPECT_EQ(2, Count(TermID::NAME));
511 EXPECT_EQ(1, tokens[0]->m_line) << Dump();
512 EXPECT_EQ(15, tokens[0]->m_col) << Dump();
513 EXPECT_EQ(1, tokens[1]->m_line) << Dump();
514 EXPECT_EQ(17, tokens[1]->m_col) << Dump();
515
516 ASSERT_EQ(2, TokenParse("/** Русские символы */name")) << Dump();
517 EXPECT_EQ(1, Count(TermID::DOC_BEFORE));
518 EXPECT_EQ(1, Count(TermID::NAME));
519 EXPECT_EQ(1, tokens[0]->m_line);
520 EXPECT_EQ(37, tokens[0]->m_col);
521 EXPECT_EQ(1, tokens[1]->m_line);
522 EXPECT_EQ(23 + 14 + 4, tokens[1]->m_col);
523}
524
525TEST_F(Lexer, Macro) {
526
527 ASSERT_EQ(1, TokenParse("@$arg")) << Dump();
528 EXPECT_EQ(1, Count(TermID::MACRO_ARGNAME)) << Dump();
529
530 ASSERT_EQ(1, TokenParse("@$1")) << Dump();
531 EXPECT_EQ(1, Count(TermID::MACRO_ARGPOS)) << Dump();
532
533 // ASSERT_EQ(1, TokenParse("@$name(*)")) << Dump();
534 // EXPECT_EQ(1, Count(TermID::MACRO_ARGUMENT));
535 // ASSERT_EQ(1, TokenParse("@$name[*]")) << Dump();
536 // EXPECT_EQ(1, Count(TermID::MACRO_ARGUMENT));
537 // ASSERT_EQ(1, TokenParse("@$name<*>")) << Dump();
538 // EXPECT_EQ(1, Count(TermID::MACRO_ARGUMENT));
539 //
540 // ASSERT_EQ(1, TokenParse("@$name(#)")) << Dump();
541 // EXPECT_EQ(1, Count(TermID::MACRO_ARGCOUNT));
542 // ASSERT_EQ(1, TokenParse("@$name[#]")) << Dump();
543 // EXPECT_EQ(1, Count(TermID::MACRO_ARGCOUNT));
544 // ASSERT_EQ(1, TokenParse("@$name<#>")) << Dump();
545 // EXPECT_EQ(1, Count(TermID::MACRO_ARGCOUNT));
546
547
548 ASSERT_EQ(1, TokenParse("@#")) << Dump();
549 EXPECT_EQ(1, Count(TermID::MACRO_TOSTR));
550
551 ASSERT_EQ(1, TokenParse("@#'")) << Dump();
552 EXPECT_EQ(1, Count(TermID::MACRO_TOSTR));
553 ASSERT_EQ(1, TokenParse("@#\"")) << Dump();
554 EXPECT_EQ(1, Count(TermID::MACRO_TOSTR));
555
556 ASSERT_EQ(1, TokenParse("@##")) << Dump();
557 EXPECT_EQ(1, Count(TermID::MACRO_CONCAT));
558
559 ASSERT_EQ(1, TokenParse("@$...")) << Dump();
560 EXPECT_EQ(1, Count(TermID::MACRO_ARGUMENT));
561 ASSERT_EQ(1, TokenParse("@$*")) << Dump();
562 EXPECT_EQ(1, Count(TermID::MACRO_ARGUMENT));
563 ASSERT_EQ(1, TokenParse("@$#")) << Dump();
564 EXPECT_EQ(1, Count(TermID::MACRO_ARGCOUNT));
565
566 ASSERT_EQ(7, TokenParse("@macro := @@123 ... 456@@")) << Dump();
567 EXPECT_EQ(1, Count(TermID::MACRO)) << Dump();
568 EXPECT_EQ(2, Count(TermID::MACRO_SEQ)) << Dump();
569
570 ASSERT_EQ(3, TokenParse("@macro := @@@123 ... 456@@@"));
571 EXPECT_EQ(1, Count(TermID::MACRO));
572 EXPECT_EQ(1, Count(TermID::MACRO_STR));
573 EXPECT_STREQ("@macro", tokens[0]->m_text.c_str());
574 EXPECT_STREQ("123 ... 456", tokens[2]->m_text.c_str());
575 EXPECT_EQ(1, tokens[0]->m_line) << Dump();
576 EXPECT_EQ(7, tokens[0]->m_col) << Dump();
577 EXPECT_EQ(1, tokens[2]->m_line) << Dump();
578 EXPECT_EQ(28, tokens[2]->m_col) << Dump();
579
580 ASSERT_EQ(6, TokenParse("@macro (name) := @@@123 \n \n ... 456@@@ # Комментарий"));
581 EXPECT_EQ(1, Count(TermID::NAME));
582 EXPECT_EQ(1, Count(TermID::MACRO));
583 EXPECT_EQ(2, Count(TermID::SYMBOL));
584 EXPECT_EQ(1, Count(TermID::MACRO_STR));
585 EXPECT_STREQ("@macro", tokens[0]->m_text.c_str());
586 EXPECT_STREQ("123 \n \n ... 456", tokens[5]->m_text.c_str());
587 EXPECT_EQ(1, tokens[0]->m_line);
588 EXPECT_EQ(7, tokens[0]->m_col);
589 EXPECT_EQ(3, tokens[5]->m_line);
590 EXPECT_EQ(12, tokens[5]->m_col);
591
592 ASSERT_EQ(11, TokenParse("@if($args) := @@ [@$args] --> @@")) << Dump();
593 EXPECT_EQ(1, Count(TermID::MACRO));
594 EXPECT_EQ(4, Count(TermID::SYMBOL));
595 EXPECT_EQ(1, Count(TermID::CREATE_USE));
596 EXPECT_EQ(2, Count(TermID::MACRO_SEQ));
597 EXPECT_EQ(1, Count(TermID::FOLLOW));
598 EXPECT_EQ(1, Count(TermID::MACRO_ARGNAME)) << Dump();
599}
600
601TEST_F(Lexer, Ignore) {
602 ASSERT_EQ(1, TokenParse("\\\\ ")) << Dump();
603 ASSERT_EQ(1, Count(TermID::MODULE)) << Dump();
604 EXPECT_STREQ("\\\\", tokens[0]->m_text.c_str());
605
606 ASSERT_EQ(2, TokenParse("\\\\ ", false)) << Dump();
607 ASSERT_EQ(1, Count(TermID::MODULE)) << Dump();
608 ASSERT_EQ(1, Count(TermID::SPACE)) << Dump();
609 EXPECT_STREQ("\\\\", tokens[0]->m_text.c_str());
610 EXPECT_STREQ(" ", tokens[1]->m_text.c_str());
611
612 ASSERT_EQ(2, TokenParse("\\\\ \t ", false)) << Dump();
613 ASSERT_EQ(1, Count(TermID::MODULE)) << Dump();
614 ASSERT_EQ(1, Count(TermID::SPACE)) << Dump();
615 EXPECT_STREQ("\\\\", tokens[0]->m_text.c_str());
616 EXPECT_STREQ(" \t ", tokens[1]->m_text.c_str());
617
618
619 ASSERT_EQ(2, TokenParse(" \\\\ \t \n", false)) << Dump();
620 ASSERT_EQ(1, Count(TermID::MODULE)) << Dump();
621 ASSERT_EQ(1, Count(TermID::SPACE)) << Dump();
622 EXPECT_STREQ("\\\\", tokens[0]->m_text.c_str());
623 EXPECT_STREQ(" \t ", tokens[1]->m_text.c_str());
624
625
626 ASSERT_EQ(5, TokenParse(" \\\\ \t \n\t\t", false, false, false, false)) << Dump();
627 ASSERT_EQ(1, Count(TermID::MODULE)) << Dump();
628 ASSERT_EQ(1, Count(TermID::SPACE)) << Dump();
629 ASSERT_EQ(2, Count(TermID::INDENT)) << Dump();
630 ASSERT_EQ(1, Count(TermID::CRLF)) << Dump();
631 EXPECT_STREQ(" ", tokens[0]->m_text.c_str());
632 EXPECT_STREQ("\\\\", tokens[1]->m_text.c_str());
633 EXPECT_STREQ(" \t ", tokens[2]->m_text.c_str());
634 EXPECT_STREQ("\n", tokens[3]->m_text.c_str());
635 EXPECT_STREQ("\t\t", tokens[4]->m_text.c_str());
636
637 ASSERT_EQ(4, TokenParse("/* /* */ */ # \n", false, false, false, false)) << Dump();
638 ASSERT_EQ(1, Count(TermID::SPACE)) << Dump();
639 ASSERT_EQ(2, Count(TermID::COMMENT)) << Dump();
640 ASSERT_EQ(1, Count(TermID::CRLF)) << Dump();
641 EXPECT_STREQ("/* /* */ */", tokens[0]->m_text.c_str());
642 EXPECT_STREQ(" ", tokens[1]->m_text.c_str());
643 EXPECT_STREQ("# ", tokens[2]->m_text.c_str());
644 EXPECT_STREQ("\n", tokens[3]->m_text.c_str());
645
646}
647
648TEST_F(Lexer, Mangled) {
649 ASSERT_EQ(1, TokenParse("_$$_123$")) << Dump();
650 ASSERT_EQ(1, Count(TermID::MANGLED)) << Dump();
651
652 ASSERT_EQ(1, TokenParse("_$name_$_123$")) << Dump();
653 ASSERT_EQ(1, Count(TermID::MANGLED)) << Dump();
654
655 ASSERT_EQ(1, TokenParse("_$na12me_$_name$$$")) << Dump();
656 ASSERT_EQ(1, Count(TermID::MANGLED)) << Dump();
657
658 ASSERT_EQ(1, TokenParse("_$na$_12me_$_name$$$")) << Dump();
659 ASSERT_EQ(1, Count(TermID::MANGLED)) << Dump();
660}
661
662TEST_F(Lexer, ParseLexem) {
663
664 BlockType arr = Scanner::ParseLexem("1 2 3 4 5");
665
666 ASSERT_EQ(5, arr.size()) << Macro::DumpText(arr).c_str();
667 ASSERT_STREQ("1 2 3 4 5", Macro::DumpText(arr).c_str());
668
669 arr = Scanner::ParseLexem("macro @test(1,2,3,...):type; next \n; # sssssss\n @only lexem((((;; ;");
670 ASSERT_STREQ("macro @test ( 1 , 2 , 3 , ... ) : type ; next ; @only lexem ( ( ( ( ; ; ;", Macro::DumpText(arr).c_str());
671}
672
673#endif // UNITTEST
result
int result
Definition lexer.l:367
newlang
Definition nlc.h:59
newlang::TermID
TermID
Definition term.h:119
newlang::TermPtr
std::shared_ptr< Term > TermPtr
Definition variable.h:33
newlang::BlockType
std::vector< TermPtr > BlockType
Definition types.h:239
newlang::toString
const char * toString(TermID type)
Definition term.h:126
newlang::Dump
std::string Dump(const T &iterable)
Definition logger.h:306
warning_pop.h
warning_push.h
Generated by doxygen 1.11.0