Coverage Report

Created: 2024-04-30 09:35

test/zserio/JsonTokenizerTest.cpp
Line
Count
Source
1
#include <sstream>
2
3
#include "gtest/gtest.h"
4
#include "zserio/JsonTokenizer.h"
5
6
namespace zserio
7
{
8
9
using JsonTokenizer = BasicJsonTokenizer<>;
10
11
TEST(JsonTokenizerTest, tokens)
12
1
{
13
1
    std::stringstream str("{\"array\":\n[\n{\"key\":\n10}]}");
14
1
    auto tokenizer = JsonTokenizer(str, std::allocator<uint8_t>());
15
1
    ASSERT_EQ(JsonToken::BEGIN_OBJECT, tokenizer.next());
16
1
    ASSERT_EQ('{', tokenizer.getValue().get<char>());
17
1
    ASSERT_EQ(JsonToken::VALUE, tokenizer.next());
18
1
    ASSERT_EQ("array", tokenizer.getValue().get<string<>>());
19
1
    ASSERT_EQ(JsonToken::KEY_SEPARATOR, tokenizer.next());
20
1
    ASSERT_EQ(':', tokenizer.getValue().get<char>());
21
1
    ASSERT_EQ(JsonToken::BEGIN_ARRAY, tokenizer.next());
22
1
    ASSERT_EQ('[', tokenizer.getValue().get<char>());
23
1
    ASSERT_EQ(JsonToken::BEGIN_OBJECT, tokenizer.next());
24
1
    ASSERT_EQ('{', tokenizer.getValue().get<char>());
25
1
    ASSERT_EQ(JsonToken::VALUE, tokenizer.next());
26
1
    ASSERT_EQ("key", tokenizer.getValue().get<string<>>());
27
1
    ASSERT_EQ(JsonToken::KEY_SEPARATOR, tokenizer.next());
28
1
    ASSERT_EQ(':', tokenizer.getValue().get<char>());
29
1
    ASSERT_EQ(JsonToken::VALUE, tokenizer.next());
30
1
    ASSERT_EQ(10, tokenizer.getValue().get<uint64_t>());
31
1
    ASSERT_EQ(JsonToken::END_OBJECT, tokenizer.next());
32
1
    ASSERT_EQ('}', tokenizer.getValue().get<char>());
33
1
    ASSERT_EQ(JsonToken::END_ARRAY, tokenizer.next());
34
1
    ASSERT_EQ(']', tokenizer.getValue().get<char>());
35
1
    ASSERT_EQ(JsonToken::END_OBJECT, tokenizer.next());
36
1
    ASSERT_EQ('}', tokenizer.getValue().get<char>());
37
1
    ASSERT_EQ(JsonToken::END_OF_FILE, tokenizer.next());
38
1
}
39
40
TEST(JsonTokenizerTest, lineColumn)
41
1
{
42
1
    std::stringstream str("\n\t{\r   \"key\"  \r\n\t :\n10}\r");
43
1
    JsonTokenizer tokenizer(str, std::allocator<uint8_t>());
44
45
1
    ASSERT_EQ(JsonToken::BEGIN_OBJECT, tokenizer.next());
46
1
    ASSERT_EQ('{', tokenizer.getValue().get<char>());
47
1
    ASSERT_EQ(2, tokenizer.getLine());
48
1
    ASSERT_EQ(2, tokenizer.getColumn());
49
50
1
    ASSERT_EQ(JsonToken::VALUE, tokenizer.next());
51
1
    ASSERT_EQ("key", tokenizer.getValue().get<string<>>());
52
1
    ASSERT_EQ(3, tokenizer.getLine());
53
1
    ASSERT_EQ(4, tokenizer.getColumn());
54
55
1
    ASSERT_EQ(JsonToken::KEY_SEPARATOR, tokenizer.next());
56
1
    ASSERT_EQ(':', tokenizer.getValue().get<char>());
57
1
    ASSERT_EQ(4, tokenizer.getLine());
58
1
    ASSERT_EQ(3, tokenizer.getColumn());
59
60
1
    ASSERT_EQ(JsonToken::VALUE, tokenizer.next());
61
1
    ASSERT_EQ(10, tokenizer.getValue().get<uint64_t>());
62
1
    ASSERT_EQ(5, tokenizer.getLine());
63
1
    ASSERT_EQ(1, tokenizer.getColumn());
64
65
1
    ASSERT_EQ(JsonToken::END_OBJECT, tokenizer.next());
66
1
    ASSERT_EQ('}', tokenizer.getValue().get<char>());
67
1
    ASSERT_EQ(5, tokenizer.getLine());
68
1
    ASSERT_EQ(3, tokenizer.getColumn());
69
70
1
    ASSERT_EQ(JsonToken::END_OF_FILE, tokenizer.next());
71
1
    ASSERT_FALSE(tokenizer.getValue().hasValue());
72
1
    ASSERT_EQ(5, tokenizer.getLine());
73
1
    ASSERT_EQ(4, tokenizer.getColumn());
74
1
}
75
76
TEST(JsonTokenizerTest, longInputSplitInNumber)
77
1
{
78
1
    std::stringstream str;
79
1
    str << "{\n"; // 2 chars
80
4.00k
    for (size_t i = 0; i < 4000; 
++i4.00k
) // 20 x 4000 > 65534 to check reading by chunks
81
4.00k
    {
82
        // BUFFER_SIZE is 65536, thus 65534 % 20 gives position within the string below
83
        // where the buffer will be split => 14, which is somewhere in the middle of the number
84
        //     |->            <-|
85
4.00k
        str << "  \"key\": 100000000,\n"; // 20 chars
86
4.00k
    }
87
1
    str << "  \"key\": 100000000\n";
88
1
    str << '}';
89
90
1
    JsonTokenizer tokenizer(str, std::allocator<uint8_t>());
91
92
1
    ASSERT_EQ(JsonToken::BEGIN_OBJECT, tokenizer.next());
93
1
    ASSERT_EQ('{', tokenizer.getValue().get<char>());
94
1
    ASSERT_EQ(1, tokenizer.getLine());
95
1
    ASSERT_EQ(1, tokenizer.getColumn());
96
97
1
    size_t number = 0;
98
4.00k
    for (; number < 4000; 
++number4.00k
)
99
4.00k
    {
100
8.00k
        ASSERT_EQ(JsonToken::VALUE, tokenizer.next()) << "i=" << number;
101
8.00k
        ASSERT_EQ("key", tokenizer.getValue().get<string<>>()) << "i=" << number;
102
8.00k
        ASSERT_EQ(1 + number + 1, tokenizer.getLine()) << "i=" << number;
103
8.00k
        ASSERT_EQ(3, tokenizer.getColumn()) << "i=" << number;
104
105
8.00k
        ASSERT_EQ(JsonToken::KEY_SEPARATOR, tokenizer.next()) << "i=" << number;
106
8.00k
        ASSERT_EQ(':', tokenizer.getValue().get<char>()) << "i=" << number;
107
8.00k
        ASSERT_EQ(1 + number + 1, tokenizer.getLine()) << "i=" << number;
108
8.00k
        ASSERT_EQ(8, tokenizer.getColumn()) << "i=" << number;
109
110
8.00k
        ASSERT_EQ(JsonToken::VALUE, tokenizer.next()) << "i=" << number;
111
8.00k
        ASSERT_EQ(100000000, tokenizer.getValue().get<uint64_t>()) << "i=" << number;
112
8.00k
        ASSERT_EQ(1 + number + 1, tokenizer.getLine()) << "i=" << number;
113
8.00k
        ASSERT_EQ(10, tokenizer.getColumn()) << "i=" << number;
114
115
8.00k
        ASSERT_EQ(JsonToken::ITEM_SEPARATOR, tokenizer.next()) << "i=" << number;
116
8.00k
        ASSERT_EQ(',', tokenizer.getValue().get<char>()) << "i=" << number;
117
8.00k
        ASSERT_EQ(1 + number + 1, tokenizer.getLine()) << "i=" << number;
118
8.00k
        ASSERT_EQ(19, tokenizer.getColumn()) << "i=" << number;
119
4.00k
    }
120
121
2
    ASSERT_EQ(JsonToken::VALUE, tokenizer.next()) << "i=" << number;
122
2
    ASSERT_EQ("key", tokenizer.getValue().get<string<>>()) << "i=" << number;
123
2
    ASSERT_EQ(1 + number + 1, tokenizer.getLine()) << "i=" << number;
124
2
    ASSERT_EQ(3, tokenizer.getColumn()) << "i=" << number;
125
126
2
    ASSERT_EQ(JsonToken::KEY_SEPARATOR, tokenizer.next()) << "i=" << number;
127
2
    ASSERT_EQ(':', tokenizer.getValue().get<char>()) << "i=" << number;
128
2
    ASSERT_EQ(1 + number + 1, tokenizer.getLine()) << "i=" << number;
129
2
    ASSERT_EQ(8, tokenizer.getColumn()) << "i=" << number;
130
131
2
    ASSERT_EQ(JsonToken::VALUE, tokenizer.next()) << "i=" << number;
132
2
    ASSERT_EQ(100000000, tokenizer.getValue().get<uint64_t>()) << "i=" << number;
133
2
    ASSERT_EQ(1 + number + 1, tokenizer.getLine()) << "i=" << number;
134
2
    ASSERT_EQ(10, tokenizer.getColumn()) << "i=" << number;
135
136
1
    ASSERT_EQ(JsonToken::END_OBJECT, tokenizer.next());
137
1
    ASSERT_EQ(1 + number + 2, tokenizer.getLine());
138
1
    ASSERT_EQ(1, tokenizer.getColumn());
139
1
}
140
141
TEST(JsonTokenizerTest, longInputSplitInString)
142
1
{
143
1
    std::stringstream str;
144
1
    str << "{\n"; // 2 chars
145
4.00k
    for (size_t i = 0; i < 4000; 
++i4.00k
) // 20 x 4000 > 65534 to check reading by chunks
146
4.00k
    {
147
        // BUFFER_SIZE is 65536, thus 65534 % 20 gives position within the string below
148
        // where the buffer will be split => 14, which is somewhere in the middle of the number
149
        //     |->             <-|
150
4.00k
        str << "  \"key\": \"1000000\",\n"; // 20 chars
151
4.00k
    }
152
1
    str << "  \"key\": \"1000000\"\n";
153
1
    str << '}';
154
155
1
    JsonTokenizer tokenizer(str, std::allocator<uint8_t>());
156
157
1
    ASSERT_EQ(JsonToken::BEGIN_OBJECT, tokenizer.next());
158
1
    ASSERT_EQ('{', tokenizer.getValue().get<char>());
159
1
    ASSERT_EQ(1, tokenizer.getLine());
160
1
    ASSERT_EQ(1, tokenizer.getColumn());
161
162
1
    size_t i = 0;
163
4.00k
    for (; i < 4000; 
++i4.00k
)
164
4.00k
    {
165
8.00k
        ASSERT_EQ(JsonToken::VALUE, tokenizer.next()) << "i=" << i;
166
8.00k
        ASSERT_EQ("key", tokenizer.getValue().get<string<>>()) << "i=" << i;
167
8.00k
        ASSERT_EQ(1 + i + 1, tokenizer.getLine()) << "i=" << i;
168
8.00k
        ASSERT_EQ(3, tokenizer.getColumn()) << "i=" << i;
169
170
8.00k
        ASSERT_EQ(JsonToken::KEY_SEPARATOR, tokenizer.next()) << "i=" << i;
171
8.00k
        ASSERT_EQ(':', tokenizer.getValue().get<char>()) << "i=" << i;
172
8.00k
        ASSERT_EQ(1 + i + 1, tokenizer.getLine()) << "i=" << i;
173
8.00k
        ASSERT_EQ(8, tokenizer.getColumn()) << "i=" << i;
174
175
8.00k
        ASSERT_EQ(JsonToken::VALUE, tokenizer.next()) << "i=" << i;
176
8.00k
        ASSERT_EQ("1000000", tokenizer.getValue().get<string<>>()) << "i=" << i;
177
8.00k
        ASSERT_EQ(1 + i + 1, tokenizer.getLine()) << "i=" << i;
178
8.00k
        ASSERT_EQ(10, tokenizer.getColumn()) << "i=" << i;
179
180
8.00k
        ASSERT_EQ(JsonToken::ITEM_SEPARATOR, tokenizer.next()) << "i=" << i;
181
8.00k
        ASSERT_EQ(',', tokenizer.getValue().get<char>()) << "i=" << i;
182
8.00k
        ASSERT_EQ(1 + i + 1, tokenizer.getLine()) << "i=" << i;
183
8.00k
        ASSERT_EQ(19, tokenizer.getColumn()) << "i=" << i;
184
4.00k
    }
185
186
2
    ASSERT_EQ(JsonToken::VALUE, tokenizer.next()) << "i=" << i;
187
2
    ASSERT_EQ("key", tokenizer.getValue().get<string<>>()) << "i=" << i;
188
2
    ASSERT_EQ(1 + i + 1, tokenizer.getLine()) << "i=" << i;
189
2
    ASSERT_EQ(3, tokenizer.getColumn()) << "i=" << i;
190
191
2
    ASSERT_EQ(JsonToken::KEY_SEPARATOR, tokenizer.next()) << "i=" << i;
192
2
    ASSERT_EQ(':', tokenizer.getValue().get<char>()) << "i=" << i;
193
2
    ASSERT_EQ(1 + i + 1, tokenizer.getLine()) << "i=" << i;
194
2
    ASSERT_EQ(8, tokenizer.getColumn()) << "i=" << i;
195
196
2
    ASSERT_EQ(JsonToken::VALUE, tokenizer.next()) << "i=" << i;
197
2
    ASSERT_EQ("1000000", tokenizer.getValue().get<string<>>()) << "i=" << i;
198
2
    ASSERT_EQ(1 + i + 1, tokenizer.getLine()) << "i=" << i;
199
2
    ASSERT_EQ(10, tokenizer.getColumn()) << "i=" << i;
200
201
1
    ASSERT_EQ(JsonToken::END_OBJECT, tokenizer.next());
202
1
    ASSERT_EQ(1 + i + 2, tokenizer.getLine());
203
2
    ASSERT_EQ(1, tokenizer.getColumn()) << "i=" << i;
204
1
}
205
206
TEST(JsonTokenizerTest, longInputSplitInDoubleAfterE)
207
1
{
208
1
    std::stringstream str;
209
1
    str << "{\n"; // 2 chars
210
4.00k
    for (size_t i = 0; i < 4000; 
++i4.00k
) // 20 x 4000 > 65534 to check reading by chunks
211
4.00k
    {
212
        // BUFFER_SIZE is 65536, thus 65534 % 20 gives position within the string below
213
        // where the buffer will be split => 14, which is somewhere in the middle of the number
214
        //     |->            <-|
215
4.00k
        str << "  \"key\":    1e5   ,\n"; // 20 chars
216
4.00k
    }
217
1
    str << "  \"key\":    1e5  \n";
218
1
    str << '}';
219
220
1
    JsonTokenizer tokenizer(str, std::allocator<uint8_t>());
221
222
1
    ASSERT_EQ(JsonToken::BEGIN_OBJECT, tokenizer.next());
223
1
    ASSERT_EQ('{', tokenizer.getValue().get<char>());
224
1
    ASSERT_EQ(1, tokenizer.getLine());
225
1
    ASSERT_EQ(1, tokenizer.getColumn());
226
227
1
    size_t i = 0;
228
4.00k
    for (; i < 4000; 
++i4.00k
)
229
4.00k
    {
230
8.00k
        ASSERT_EQ(JsonToken::VALUE, tokenizer.next()) << "i=" << i;
231
8.00k
        ASSERT_EQ("key", tokenizer.getValue().get<string<>>()) << "i=" << i;
232
8.00k
        ASSERT_EQ(1 + i + 1, tokenizer.getLine()) << "i=" << i;
233
8.00k
        ASSERT_EQ(3, tokenizer.getColumn()) << "i=" << i;
234
235
8.00k
        ASSERT_EQ(JsonToken::KEY_SEPARATOR, tokenizer.next()) << "i=" << i;
236
8.00k
        ASSERT_EQ(':', tokenizer.getValue().get<char>()) << "i=" << i;
237
8.00k
        ASSERT_EQ(1 + i + 1, tokenizer.getLine()) << "i=" << i;
238
8.00k
        ASSERT_EQ(8, tokenizer.getColumn()) << "i=" << i;
239
240
8.00k
        ASSERT_EQ(JsonToken::VALUE, tokenizer.next()) << "i=" << i;
241
8.00k
        ASSERT_EQ(1e5, tokenizer.getValue().get<double>()) << "i=" << i;
242
8.00k
        ASSERT_EQ(1 + i + 1, tokenizer.getLine()) << "i=" << i;
243
8.00k
        ASSERT_EQ(13, tokenizer.getColumn()) << "i=" << i;
244
245
8.00k
        ASSERT_EQ(JsonToken::ITEM_SEPARATOR, tokenizer.next()) << "i=" << i;
246
8.00k
        ASSERT_EQ(',', tokenizer.getValue().get<char>()) << "i=" << i;
247
8.00k
        ASSERT_EQ(1 + i + 1, tokenizer.getLine()) << "i=" << i;
248
8.00k
        ASSERT_EQ(19, tokenizer.getColumn()) << "i=" << i;
249
4.00k
    }
250
251
2
    ASSERT_EQ(JsonToken::VALUE, tokenizer.next()) << "i=" << i;
252
2
    ASSERT_EQ("key", tokenizer.getValue().get<string<>>()) << "i=" << i;
253
2
    ASSERT_EQ(1 + i + 1, tokenizer.getLine()) << "i=" << i;
254
2
    ASSERT_EQ(3, tokenizer.getColumn()) << "i=" << i;
255
256
2
    ASSERT_EQ(JsonToken::KEY_SEPARATOR, tokenizer.next()) << "i=" << i;
257
2
    ASSERT_EQ(':', tokenizer.getValue().get<char>()) << "i=" << i;
258
2
    ASSERT_EQ(1 + i + 1, tokenizer.getLine()) << "i=" << i;
259
2
    ASSERT_EQ(8, tokenizer.getColumn()) << "i=" << i;
260
261
2
    ASSERT_EQ(JsonToken::VALUE, tokenizer.next()) << "i=" << i;
262
2
    ASSERT_EQ(1e5, tokenizer.getValue().get<double>()) << "i=" << i;
263
2
    ASSERT_EQ(1 + i + 1, tokenizer.getLine()) << "i=" << i;
264
2
    ASSERT_EQ(13, tokenizer.getColumn()) << "i=" << i;
265
266
1
    ASSERT_EQ(JsonToken::END_OBJECT, tokenizer.next());
267
1
    ASSERT_EQ(1 + i + 2, tokenizer.getLine());
268
2
    ASSERT_EQ(1, tokenizer.getColumn()) << "i=" << i;
269
1
}
270
271
TEST(JsonTokenizerTest, unknownToken)
272
1
{
273
1
    std::stringstream str("\\\n");
274
1
    JsonTokenizer tokenizer(str, std::allocator<uint8_t>());
275
1
    ASSERT_THROW(
276
1
            {
277
1
                try
278
1
                {
279
1
                    tokenizer.next();
280
1
                }
281
1
                catch (const JsonParserException& e)
282
1
                {
283
1
                    ASSERT_STREQ("JsonTokenizer:1:1: Unknown token!", e.what());
284
1
                    throw;
285
1
                }
286
1
            },
287
1
            JsonParserException);
288
1
}
289
290
TEST(JsonTokenizerTest, cppRuntimeExceptionOperator)
291
1
{
292
1
    ASSERT_STREQ("UNKNOWN", (JsonParserException() << (JsonToken::UNKNOWN)).what());
293
1
    ASSERT_STREQ("BEGIN_OF_FILE", (JsonParserException() << (JsonToken::BEGIN_OF_FILE)).what());
294
1
    ASSERT_STREQ("END_OF_FILE", (JsonParserException() << (JsonToken::END_OF_FILE)).what());
295
1
    ASSERT_STREQ("BEGIN_OBJECT", (JsonParserException() << (JsonToken::BEGIN_OBJECT)).what());
296
1
    ASSERT_STREQ("END_OBJECT", (JsonParserException() << (JsonToken::END_OBJECT)).what());
297
1
    ASSERT_STREQ("BEGIN_ARRAY", (JsonParserException() << (JsonToken::BEGIN_ARRAY)).what());
298
1
    ASSERT_STREQ("END_ARRAY", (JsonParserException() << (JsonToken::END_ARRAY)).what());
299
1
    ASSERT_STREQ("KEY_SEPARATOR", (JsonParserException() << (JsonToken::KEY_SEPARATOR)).what());
300
1
    ASSERT_STREQ("ITEM_SEPARATOR", (JsonParserException() << (JsonToken::ITEM_SEPARATOR)).what());
301
1
    ASSERT_STREQ("VALUE", (JsonParserException() << (JsonToken::VALUE)).what());
302
1
}
303
304
} // namespace zserio