test/zserio/JsonTokenizerTest.cpp
Line | Count | Source |
1 | | #include <sstream> |
2 | | |
3 | | #include "gtest/gtest.h" |
4 | | #include "zserio/JsonTokenizer.h" |
5 | | |
6 | | namespace zserio |
7 | | { |
8 | | |
9 | | using JsonTokenizer = BasicJsonTokenizer<>; |
10 | | |
11 | | TEST(JsonTokenizerTest, tokens) |
12 | 1 | { |
13 | 1 | std::stringstream str("{\"array\":\n[\n{\"key\":\n10}]}"); |
14 | 1 | auto tokenizer = JsonTokenizer(str, std::allocator<uint8_t>()); |
15 | 1 | ASSERT_EQ(JsonToken::BEGIN_OBJECT, tokenizer.next()); |
16 | 1 | ASSERT_EQ('{', tokenizer.getValue().get<char>()); |
17 | 1 | ASSERT_EQ(JsonToken::VALUE, tokenizer.next()); |
18 | 1 | ASSERT_EQ("array", tokenizer.getValue().get<string<>>()); |
19 | 1 | ASSERT_EQ(JsonToken::KEY_SEPARATOR, tokenizer.next()); |
20 | 1 | ASSERT_EQ(':', tokenizer.getValue().get<char>()); |
21 | 1 | ASSERT_EQ(JsonToken::BEGIN_ARRAY, tokenizer.next()); |
22 | 1 | ASSERT_EQ('[', tokenizer.getValue().get<char>()); |
23 | 1 | ASSERT_EQ(JsonToken::BEGIN_OBJECT, tokenizer.next()); |
24 | 1 | ASSERT_EQ('{', tokenizer.getValue().get<char>()); |
25 | 1 | ASSERT_EQ(JsonToken::VALUE, tokenizer.next()); |
26 | 1 | ASSERT_EQ("key", tokenizer.getValue().get<string<>>()); |
27 | 1 | ASSERT_EQ(JsonToken::KEY_SEPARATOR, tokenizer.next()); |
28 | 1 | ASSERT_EQ(':', tokenizer.getValue().get<char>()); |
29 | 1 | ASSERT_EQ(JsonToken::VALUE, tokenizer.next()); |
30 | 1 | ASSERT_EQ(10, tokenizer.getValue().get<uint64_t>()); |
31 | 1 | ASSERT_EQ(JsonToken::END_OBJECT, tokenizer.next()); |
32 | 1 | ASSERT_EQ('}', tokenizer.getValue().get<char>()); |
33 | 1 | ASSERT_EQ(JsonToken::END_ARRAY, tokenizer.next()); |
34 | 1 | ASSERT_EQ(']', tokenizer.getValue().get<char>()); |
35 | 1 | ASSERT_EQ(JsonToken::END_OBJECT, tokenizer.next()); |
36 | 1 | ASSERT_EQ('}', tokenizer.getValue().get<char>()); |
37 | 1 | ASSERT_EQ(JsonToken::END_OF_FILE, tokenizer.next()); |
38 | 1 | } |
39 | | |
40 | | TEST(JsonTokenizerTest, lineColumn) |
41 | 1 | { |
42 | 1 | std::stringstream str("\n\t{\r \"key\" \r\n\t :\n10}\r"); |
43 | 1 | JsonTokenizer tokenizer(str, std::allocator<uint8_t>()); |
44 | | |
45 | 1 | ASSERT_EQ(JsonToken::BEGIN_OBJECT, tokenizer.next()); |
46 | 1 | ASSERT_EQ('{', tokenizer.getValue().get<char>()); |
47 | 1 | ASSERT_EQ(2, tokenizer.getLine()); |
48 | 1 | ASSERT_EQ(2, tokenizer.getColumn()); |
49 | | |
50 | 1 | ASSERT_EQ(JsonToken::VALUE, tokenizer.next()); |
51 | 1 | ASSERT_EQ("key", tokenizer.getValue().get<string<>>()); |
52 | 1 | ASSERT_EQ(3, tokenizer.getLine()); |
53 | 1 | ASSERT_EQ(4, tokenizer.getColumn()); |
54 | | |
55 | 1 | ASSERT_EQ(JsonToken::KEY_SEPARATOR, tokenizer.next()); |
56 | 1 | ASSERT_EQ(':', tokenizer.getValue().get<char>()); |
57 | 1 | ASSERT_EQ(4, tokenizer.getLine()); |
58 | 1 | ASSERT_EQ(3, tokenizer.getColumn()); |
59 | | |
60 | 1 | ASSERT_EQ(JsonToken::VALUE, tokenizer.next()); |
61 | 1 | ASSERT_EQ(10, tokenizer.getValue().get<uint64_t>()); |
62 | 1 | ASSERT_EQ(5, tokenizer.getLine()); |
63 | 1 | ASSERT_EQ(1, tokenizer.getColumn()); |
64 | | |
65 | 1 | ASSERT_EQ(JsonToken::END_OBJECT, tokenizer.next()); |
66 | 1 | ASSERT_EQ('}', tokenizer.getValue().get<char>()); |
67 | 1 | ASSERT_EQ(5, tokenizer.getLine()); |
68 | 1 | ASSERT_EQ(3, tokenizer.getColumn()); |
69 | | |
70 | 1 | ASSERT_EQ(JsonToken::END_OF_FILE, tokenizer.next()); |
71 | 1 | ASSERT_FALSE(tokenizer.getValue().hasValue()); |
72 | 1 | ASSERT_EQ(5, tokenizer.getLine()); |
73 | 1 | ASSERT_EQ(4, tokenizer.getColumn()); |
74 | 1 | } |
75 | | |
76 | | TEST(JsonTokenizerTest, longInputSplitInNumber) |
77 | 1 | { |
78 | 1 | std::stringstream str; |
79 | 1 | str << "{\n"; // 2 chars |
80 | 4.00k | for (size_t i = 0; i < 4000; ++i4.00k ) // 20 x 4000 > 65534 to check reading by chunks |
81 | 4.00k | { |
82 | | // BUFFER_SIZE is 65536, thus 65534 % 20 gives position within the string below |
83 | | // where the buffer will be split => 14, which is somewhere in the middle of the number |
84 | | // |-> <-| |
85 | 4.00k | str << " \"key\": 100000000,\n"; // 20 chars |
86 | 4.00k | } |
87 | 1 | str << " \"key\": 100000000\n"; |
88 | 1 | str << '}'; |
89 | | |
90 | 1 | JsonTokenizer tokenizer(str, std::allocator<uint8_t>()); |
91 | | |
92 | 1 | ASSERT_EQ(JsonToken::BEGIN_OBJECT, tokenizer.next()); |
93 | 1 | ASSERT_EQ('{', tokenizer.getValue().get<char>()); |
94 | 1 | ASSERT_EQ(1, tokenizer.getLine()); |
95 | 1 | ASSERT_EQ(1, tokenizer.getColumn()); |
96 | | |
97 | 1 | size_t number = 0; |
98 | 4.00k | for (; number < 4000; ++number4.00k ) |
99 | 4.00k | { |
100 | 8.00k | ASSERT_EQ(JsonToken::VALUE, tokenizer.next()) << "i=" << number; |
101 | 8.00k | ASSERT_EQ("key", tokenizer.getValue().get<string<>>()) << "i=" << number; |
102 | 8.00k | ASSERT_EQ(1 + number + 1, tokenizer.getLine()) << "i=" << number; |
103 | 8.00k | ASSERT_EQ(3, tokenizer.getColumn()) << "i=" << number; |
104 | | |
105 | 8.00k | ASSERT_EQ(JsonToken::KEY_SEPARATOR, tokenizer.next()) << "i=" << number; |
106 | 8.00k | ASSERT_EQ(':', tokenizer.getValue().get<char>()) << "i=" << number; |
107 | 8.00k | ASSERT_EQ(1 + number + 1, tokenizer.getLine()) << "i=" << number; |
108 | 8.00k | ASSERT_EQ(8, tokenizer.getColumn()) << "i=" << number; |
109 | | |
110 | 8.00k | ASSERT_EQ(JsonToken::VALUE, tokenizer.next()) << "i=" << number; |
111 | 8.00k | ASSERT_EQ(100000000, tokenizer.getValue().get<uint64_t>()) << "i=" << number; |
112 | 8.00k | ASSERT_EQ(1 + number + 1, tokenizer.getLine()) << "i=" << number; |
113 | 8.00k | ASSERT_EQ(10, tokenizer.getColumn()) << "i=" << number; |
114 | | |
115 | 8.00k | ASSERT_EQ(JsonToken::ITEM_SEPARATOR, tokenizer.next()) << "i=" << number; |
116 | 8.00k | ASSERT_EQ(',', tokenizer.getValue().get<char>()) << "i=" << number; |
117 | 8.00k | ASSERT_EQ(1 + number + 1, tokenizer.getLine()) << "i=" << number; |
118 | 8.00k | ASSERT_EQ(19, tokenizer.getColumn()) << "i=" << number; |
119 | 4.00k | } |
120 | | |
121 | 2 | ASSERT_EQ(JsonToken::VALUE, tokenizer.next()) << "i=" << number; |
122 | 2 | ASSERT_EQ("key", tokenizer.getValue().get<string<>>()) << "i=" << number; |
123 | 2 | ASSERT_EQ(1 + number + 1, tokenizer.getLine()) << "i=" << number; |
124 | 2 | ASSERT_EQ(3, tokenizer.getColumn()) << "i=" << number; |
125 | | |
126 | 2 | ASSERT_EQ(JsonToken::KEY_SEPARATOR, tokenizer.next()) << "i=" << number; |
127 | 2 | ASSERT_EQ(':', tokenizer.getValue().get<char>()) << "i=" << number; |
128 | 2 | ASSERT_EQ(1 + number + 1, tokenizer.getLine()) << "i=" << number; |
129 | 2 | ASSERT_EQ(8, tokenizer.getColumn()) << "i=" << number; |
130 | | |
131 | 2 | ASSERT_EQ(JsonToken::VALUE, tokenizer.next()) << "i=" << number; |
132 | 2 | ASSERT_EQ(100000000, tokenizer.getValue().get<uint64_t>()) << "i=" << number; |
133 | 2 | ASSERT_EQ(1 + number + 1, tokenizer.getLine()) << "i=" << number; |
134 | 2 | ASSERT_EQ(10, tokenizer.getColumn()) << "i=" << number; |
135 | | |
136 | 1 | ASSERT_EQ(JsonToken::END_OBJECT, tokenizer.next()); |
137 | 1 | ASSERT_EQ(1 + number + 2, tokenizer.getLine()); |
138 | 1 | ASSERT_EQ(1, tokenizer.getColumn()); |
139 | 1 | } |
140 | | |
141 | | TEST(JsonTokenizerTest, longInputSplitInString) |
142 | 1 | { |
143 | 1 | std::stringstream str; |
144 | 1 | str << "{\n"; // 2 chars |
145 | 4.00k | for (size_t i = 0; i < 4000; ++i4.00k ) // 20 x 4000 > 65534 to check reading by chunks |
146 | 4.00k | { |
147 | | // BUFFER_SIZE is 65536, thus 65534 % 20 gives position within the string below |
148 | | // where the buffer will be split => 14, which is somewhere in the middle of the number |
149 | | // |-> <-| |
150 | 4.00k | str << " \"key\": \"1000000\",\n"; // 20 chars |
151 | 4.00k | } |
152 | 1 | str << " \"key\": \"1000000\"\n"; |
153 | 1 | str << '}'; |
154 | | |
155 | 1 | JsonTokenizer tokenizer(str, std::allocator<uint8_t>()); |
156 | | |
157 | 1 | ASSERT_EQ(JsonToken::BEGIN_OBJECT, tokenizer.next()); |
158 | 1 | ASSERT_EQ('{', tokenizer.getValue().get<char>()); |
159 | 1 | ASSERT_EQ(1, tokenizer.getLine()); |
160 | 1 | ASSERT_EQ(1, tokenizer.getColumn()); |
161 | | |
162 | 1 | size_t i = 0; |
163 | 4.00k | for (; i < 4000; ++i4.00k ) |
164 | 4.00k | { |
165 | 8.00k | ASSERT_EQ(JsonToken::VALUE, tokenizer.next()) << "i=" << i; |
166 | 8.00k | ASSERT_EQ("key", tokenizer.getValue().get<string<>>()) << "i=" << i; |
167 | 8.00k | ASSERT_EQ(1 + i + 1, tokenizer.getLine()) << "i=" << i; |
168 | 8.00k | ASSERT_EQ(3, tokenizer.getColumn()) << "i=" << i; |
169 | | |
170 | 8.00k | ASSERT_EQ(JsonToken::KEY_SEPARATOR, tokenizer.next()) << "i=" << i; |
171 | 8.00k | ASSERT_EQ(':', tokenizer.getValue().get<char>()) << "i=" << i; |
172 | 8.00k | ASSERT_EQ(1 + i + 1, tokenizer.getLine()) << "i=" << i; |
173 | 8.00k | ASSERT_EQ(8, tokenizer.getColumn()) << "i=" << i; |
174 | | |
175 | 8.00k | ASSERT_EQ(JsonToken::VALUE, tokenizer.next()) << "i=" << i; |
176 | 8.00k | ASSERT_EQ("1000000", tokenizer.getValue().get<string<>>()) << "i=" << i; |
177 | 8.00k | ASSERT_EQ(1 + i + 1, tokenizer.getLine()) << "i=" << i; |
178 | 8.00k | ASSERT_EQ(10, tokenizer.getColumn()) << "i=" << i; |
179 | | |
180 | 8.00k | ASSERT_EQ(JsonToken::ITEM_SEPARATOR, tokenizer.next()) << "i=" << i; |
181 | 8.00k | ASSERT_EQ(',', tokenizer.getValue().get<char>()) << "i=" << i; |
182 | 8.00k | ASSERT_EQ(1 + i + 1, tokenizer.getLine()) << "i=" << i; |
183 | 8.00k | ASSERT_EQ(19, tokenizer.getColumn()) << "i=" << i; |
184 | 4.00k | } |
185 | | |
186 | 2 | ASSERT_EQ(JsonToken::VALUE, tokenizer.next()) << "i=" << i; |
187 | 2 | ASSERT_EQ("key", tokenizer.getValue().get<string<>>()) << "i=" << i; |
188 | 2 | ASSERT_EQ(1 + i + 1, tokenizer.getLine()) << "i=" << i; |
189 | 2 | ASSERT_EQ(3, tokenizer.getColumn()) << "i=" << i; |
190 | | |
191 | 2 | ASSERT_EQ(JsonToken::KEY_SEPARATOR, tokenizer.next()) << "i=" << i; |
192 | 2 | ASSERT_EQ(':', tokenizer.getValue().get<char>()) << "i=" << i; |
193 | 2 | ASSERT_EQ(1 + i + 1, tokenizer.getLine()) << "i=" << i; |
194 | 2 | ASSERT_EQ(8, tokenizer.getColumn()) << "i=" << i; |
195 | | |
196 | 2 | ASSERT_EQ(JsonToken::VALUE, tokenizer.next()) << "i=" << i; |
197 | 2 | ASSERT_EQ("1000000", tokenizer.getValue().get<string<>>()) << "i=" << i; |
198 | 2 | ASSERT_EQ(1 + i + 1, tokenizer.getLine()) << "i=" << i; |
199 | 2 | ASSERT_EQ(10, tokenizer.getColumn()) << "i=" << i; |
200 | | |
201 | 1 | ASSERT_EQ(JsonToken::END_OBJECT, tokenizer.next()); |
202 | 1 | ASSERT_EQ(1 + i + 2, tokenizer.getLine()); |
203 | 2 | ASSERT_EQ(1, tokenizer.getColumn()) << "i=" << i; |
204 | 1 | } |
205 | | |
206 | | TEST(JsonTokenizerTest, longInputSplitInDoubleAfterE) |
207 | 1 | { |
208 | 1 | std::stringstream str; |
209 | 1 | str << "{\n"; // 2 chars |
210 | 4.00k | for (size_t i = 0; i < 4000; ++i4.00k ) // 20 x 4000 > 65534 to check reading by chunks |
211 | 4.00k | { |
212 | | // BUFFER_SIZE is 65536, thus 65534 % 20 gives position within the string below |
213 | | // where the buffer will be split => 14, which is somewhere in the middle of the number |
214 | | // |-> <-| |
215 | 4.00k | str << " \"key\": 1e5 ,\n"; // 20 chars |
216 | 4.00k | } |
217 | 1 | str << " \"key\": 1e5 \n"; |
218 | 1 | str << '}'; |
219 | | |
220 | 1 | JsonTokenizer tokenizer(str, std::allocator<uint8_t>()); |
221 | | |
222 | 1 | ASSERT_EQ(JsonToken::BEGIN_OBJECT, tokenizer.next()); |
223 | 1 | ASSERT_EQ('{', tokenizer.getValue().get<char>()); |
224 | 1 | ASSERT_EQ(1, tokenizer.getLine()); |
225 | 1 | ASSERT_EQ(1, tokenizer.getColumn()); |
226 | | |
227 | 1 | size_t i = 0; |
228 | 4.00k | for (; i < 4000; ++i4.00k ) |
229 | 4.00k | { |
230 | 8.00k | ASSERT_EQ(JsonToken::VALUE, tokenizer.next()) << "i=" << i; |
231 | 8.00k | ASSERT_EQ("key", tokenizer.getValue().get<string<>>()) << "i=" << i; |
232 | 8.00k | ASSERT_EQ(1 + i + 1, tokenizer.getLine()) << "i=" << i; |
233 | 8.00k | ASSERT_EQ(3, tokenizer.getColumn()) << "i=" << i; |
234 | | |
235 | 8.00k | ASSERT_EQ(JsonToken::KEY_SEPARATOR, tokenizer.next()) << "i=" << i; |
236 | 8.00k | ASSERT_EQ(':', tokenizer.getValue().get<char>()) << "i=" << i; |
237 | 8.00k | ASSERT_EQ(1 + i + 1, tokenizer.getLine()) << "i=" << i; |
238 | 8.00k | ASSERT_EQ(8, tokenizer.getColumn()) << "i=" << i; |
239 | | |
240 | 8.00k | ASSERT_EQ(JsonToken::VALUE, tokenizer.next()) << "i=" << i; |
241 | 8.00k | ASSERT_EQ(1e5, tokenizer.getValue().get<double>()) << "i=" << i; |
242 | 8.00k | ASSERT_EQ(1 + i + 1, tokenizer.getLine()) << "i=" << i; |
243 | 8.00k | ASSERT_EQ(13, tokenizer.getColumn()) << "i=" << i; |
244 | | |
245 | 8.00k | ASSERT_EQ(JsonToken::ITEM_SEPARATOR, tokenizer.next()) << "i=" << i; |
246 | 8.00k | ASSERT_EQ(',', tokenizer.getValue().get<char>()) << "i=" << i; |
247 | 8.00k | ASSERT_EQ(1 + i + 1, tokenizer.getLine()) << "i=" << i; |
248 | 8.00k | ASSERT_EQ(19, tokenizer.getColumn()) << "i=" << i; |
249 | 4.00k | } |
250 | | |
251 | 2 | ASSERT_EQ(JsonToken::VALUE, tokenizer.next()) << "i=" << i; |
252 | 2 | ASSERT_EQ("key", tokenizer.getValue().get<string<>>()) << "i=" << i; |
253 | 2 | ASSERT_EQ(1 + i + 1, tokenizer.getLine()) << "i=" << i; |
254 | 2 | ASSERT_EQ(3, tokenizer.getColumn()) << "i=" << i; |
255 | | |
256 | 2 | ASSERT_EQ(JsonToken::KEY_SEPARATOR, tokenizer.next()) << "i=" << i; |
257 | 2 | ASSERT_EQ(':', tokenizer.getValue().get<char>()) << "i=" << i; |
258 | 2 | ASSERT_EQ(1 + i + 1, tokenizer.getLine()) << "i=" << i; |
259 | 2 | ASSERT_EQ(8, tokenizer.getColumn()) << "i=" << i; |
260 | | |
261 | 2 | ASSERT_EQ(JsonToken::VALUE, tokenizer.next()) << "i=" << i; |
262 | 2 | ASSERT_EQ(1e5, tokenizer.getValue().get<double>()) << "i=" << i; |
263 | 2 | ASSERT_EQ(1 + i + 1, tokenizer.getLine()) << "i=" << i; |
264 | 2 | ASSERT_EQ(13, tokenizer.getColumn()) << "i=" << i; |
265 | | |
266 | 1 | ASSERT_EQ(JsonToken::END_OBJECT, tokenizer.next()); |
267 | 1 | ASSERT_EQ(1 + i + 2, tokenizer.getLine()); |
268 | 2 | ASSERT_EQ(1, tokenizer.getColumn()) << "i=" << i; |
269 | 1 | } |
270 | | |
271 | | TEST(JsonTokenizerTest, unknownToken) |
272 | 1 | { |
273 | 1 | std::stringstream str("\\\n"); |
274 | 1 | JsonTokenizer tokenizer(str, std::allocator<uint8_t>()); |
275 | 1 | ASSERT_THROW( |
276 | 1 | { |
277 | 1 | try |
278 | 1 | { |
279 | 1 | tokenizer.next(); |
280 | 1 | } |
281 | 1 | catch (const JsonParserException& e) |
282 | 1 | { |
283 | 1 | ASSERT_STREQ("JsonTokenizer:1:1: Unknown token!", e.what()); |
284 | 1 | throw; |
285 | 1 | } |
286 | 1 | }, |
287 | 1 | JsonParserException); |
288 | 1 | } |
289 | | |
290 | | TEST(JsonTokenizerTest, cppRuntimeExceptionOperator) |
291 | 1 | { |
292 | 1 | ASSERT_STREQ("UNKNOWN", (JsonParserException() << (JsonToken::UNKNOWN)).what()); |
293 | 1 | ASSERT_STREQ("BEGIN_OF_FILE", (JsonParserException() << (JsonToken::BEGIN_OF_FILE)).what()); |
294 | 1 | ASSERT_STREQ("END_OF_FILE", (JsonParserException() << (JsonToken::END_OF_FILE)).what()); |
295 | 1 | ASSERT_STREQ("BEGIN_OBJECT", (JsonParserException() << (JsonToken::BEGIN_OBJECT)).what()); |
296 | 1 | ASSERT_STREQ("END_OBJECT", (JsonParserException() << (JsonToken::END_OBJECT)).what()); |
297 | 1 | ASSERT_STREQ("BEGIN_ARRAY", (JsonParserException() << (JsonToken::BEGIN_ARRAY)).what()); |
298 | 1 | ASSERT_STREQ("END_ARRAY", (JsonParserException() << (JsonToken::END_ARRAY)).what()); |
299 | 1 | ASSERT_STREQ("KEY_SEPARATOR", (JsonParserException() << (JsonToken::KEY_SEPARATOR)).what()); |
300 | 1 | ASSERT_STREQ("ITEM_SEPARATOR", (JsonParserException() << (JsonToken::ITEM_SEPARATOR)).what()); |
301 | 1 | ASSERT_STREQ("VALUE", (JsonParserException() << (JsonToken::VALUE)).what()); |
302 | 1 | } |
303 | | |
304 | | } // namespace zserio |