1 #ifndef ZSERIO_JSON_TOKENIZER_H_INC
2 #define ZSERIO_JSON_TOKENIZER_H_INC
55 template <
typename ALLOC = std::allocator<u
int8_t>>
69 m_decoderResult(0, allocator),
70 m_content(readContent(allocator)),
124 return m_tokenColumnNumber;
131 bool skipWhitespaces();
133 template <
typename T>
134 void setToken(
JsonToken token, T&& value);
137 void setPosition(
size_t newPos,
size_t newColumnNumber);
138 void setTokenValue();
140 static constexpr
size_t BUFFER_SIZE = 64 * 1024;
141 std::array<char, BUFFER_SIZE> m_buffer;
147 size_t m_lineNumber = 1;
148 size_t m_columnNumber = 1;
149 size_t m_tokenColumnNumber = 1;
155 template <
typename ALLOC>
158 while (!decodeNext())
160 string<ALLOC> newContent = readContent(m_content.get_allocator());
161 if (newContent.empty())
165 m_tokenColumnNumber = m_columnNumber;
176 m_content = m_content.substr(m_pos) + newContent;
183 template <
typename ALLOC>
186 const size_t count =
static_cast<size_t>(m_in.rdbuf()->sgetn(m_buffer.data(), BUFFER_SIZE));
190 template <
typename ALLOC>
191 bool BasicJsonTokenizer<ALLOC>::decodeNext()
193 if (!skipWhitespaces())
196 m_tokenColumnNumber = m_columnNumber;
198 const char nextChar = m_content[m_pos];
203 setPosition(m_pos + 1, m_columnNumber + 1);
207 setPosition(m_pos + 1, m_columnNumber + 1);
211 setPosition(m_pos + 1, m_columnNumber + 1);
215 setPosition(m_pos + 1, m_columnNumber + 1);
219 setPosition(m_pos + 1, m_columnNumber + 1);
223 setPosition(m_pos + 1, m_columnNumber + 1);
226 m_decoderResult = m_decoder.decodeValue(
StringView(m_content.data()).
substr(m_pos));
227 if (m_pos + m_decoderResult.numReadChars >= m_content.size())
237 template <
typename ALLOC>
238 bool BasicJsonTokenizer<ALLOC>::skipWhitespaces()
242 if (m_pos >= m_content.size())
248 const char nextChar = m_content[m_pos];
253 setPosition(m_pos + 1, m_columnNumber + 1);
257 setPosition(m_pos + 1, 1);
260 if (m_pos + 1 >= m_content.size())
266 setPosition(m_pos + (m_content[m_pos + 1] ==
'\n' ? 2 : 1), 1);
274 template <
typename ALLOC>
275 template <
typename T>
276 void BasicJsonTokenizer<ALLOC>::setToken(
JsonToken token, T&& value)
279 m_value.set(std::forward<T>(value));
282 template <
typename ALLOC>
283 void BasicJsonTokenizer<ALLOC>::setToken(
JsonToken token, AnyHolder<ALLOC>&& value)
286 m_value = std::move(value);
289 template <
typename ALLOC>
290 void BasicJsonTokenizer<ALLOC>::setToken(
JsonToken token)
296 template <
typename ALLOC>
297 void BasicJsonTokenizer<ALLOC>::setPosition(
size_t newPos,
size_t newColumnNumber)
300 m_columnNumber = newColumnNumber;
303 template <
typename ALLOC>
304 void BasicJsonTokenizer<ALLOC>::setTokenValue()
306 if (!m_decoderResult.value.hasValue())
308 throw JsonParserException(
"JsonTokenizer:")
309 << m_lineNumber <<
":" << m_tokenColumnNumber <<
": "
310 << (m_decoderResult.integerOverflow ?
"Value is outside of the 64-bit integer range!"
315 setPosition(m_pos + m_decoderResult.numReadChars, m_columnNumber + m_decoderResult.numReadChars);
JsonToken getToken() const
const AnyHolder< ALLOC > & getValue() const
BasicJsonTokenizer(std::istream &in, const ALLOC &allocator)
BasicStringView substr(size_type pos=0, size_type count=npos) const
CppRuntimeException(const char *message="")
BasicStringView< char, std::char_traits< char > > StringView
std::basic_string< char, std::char_traits< char >, RebindAlloc< ALLOC, char > > string
CppRuntimeException & operator<<(CppRuntimeException &exception, const BasicBitBuffer< ALLOC > &bitBuffer)