1 #ifndef ZSERIO_JSON_TOKENIZER_H_INC 2 #define ZSERIO_JSON_TOKENIZER_H_INC 56 template <
typename ALLOC = std::allocator<u
int8_t>>
67 m_buffer(), m_in(in), m_decoder(allocator), m_decoderResult(0, allocator),
68 m_content(readContent(allocator)), m_value(allocator)
103 size_t getLine()
const {
return m_lineNumber; }
110 size_t getColumn()
const {
return m_tokenColumnNumber; }
116 bool skipWhitespaces();
118 template <
typename T>
119 void setToken(
JsonToken token, T&& value);
122 void setPosition(
size_t newPos,
size_t newColumnNumber);
123 void setTokenValue();
125 static constexpr
size_t BUFFER_SIZE = 64 * 1024;
126 std::array<char, BUFFER_SIZE> m_buffer;
132 size_t m_lineNumber = 1;
133 size_t m_columnNumber = 1;
134 size_t m_tokenColumnNumber = 1;
140 template <
typename ALLOC>
143 while (!decodeNext())
145 string<ALLOC> newContent = readContent(m_content.get_allocator());
146 if (newContent.empty())
150 m_tokenColumnNumber = m_columnNumber;
161 m_content = m_content.substr(m_pos) + newContent;
168 template <
typename ALLOC>
171 const size_t count =
static_cast<size_t>(m_in.rdbuf()->sgetn(m_buffer.data(), BUFFER_SIZE));
175 template <
typename ALLOC>
178 if (!skipWhitespaces())
181 m_tokenColumnNumber = m_columnNumber;
183 const char nextChar = m_content[m_pos];
188 setPosition(m_pos + 1, m_columnNumber + 1);
192 setPosition(m_pos + 1, m_columnNumber + 1);
196 setPosition(m_pos + 1, m_columnNumber + 1);
200 setPosition(m_pos + 1, m_columnNumber + 1);
204 setPosition(m_pos + 1, m_columnNumber + 1);
208 setPosition(m_pos + 1, m_columnNumber + 1);
211 m_decoderResult = m_decoder.decodeValue(
StringView(m_content.data()).substr(m_pos));
212 if (m_pos + m_decoderResult.numReadChars >= m_content.size())
222 template <
typename ALLOC>
227 if (m_pos >= m_content.size())
233 const char nextChar = m_content[m_pos];
238 setPosition(m_pos + 1, m_columnNumber + 1);
242 setPosition(m_pos + 1, 1);
245 if (m_pos + 1 >= m_content.size())
251 setPosition(m_pos + (m_content[m_pos + 1] ==
'\n' ? 2 : 1), 1);
259 template <
typename ALLOC>
260 template <
typename T>
264 m_value.set(std::forward<T>(value));
267 template <
typename ALLOC>
271 m_value = std::move(value);
274 template <
typename ALLOC>
281 template <
typename ALLOC>
285 m_columnNumber = newColumnNumber;
288 template <
typename ALLOC>
291 if (!m_decoderResult.value.hasValue())
293 throw JsonParserException(
"JsonTokenizer:") << m_lineNumber <<
":" << m_tokenColumnNumber <<
": " <<
294 (m_decoderResult.integerOverflow
295 ?
"Value is outside of the 64-bit integer range!" 300 setPosition(m_pos + m_decoderResult.numReadChars, m_columnNumber + m_decoderResult.numReadChars);
305 #endif // ZSERIO_JSON_TOKENIZER_H_INC
BasicJsonTokenizer(std::istream &in, const ALLOC &allocator)
const AnyHolder< ALLOC > & getValue() const
BasicStringView< char, std::char_traits< char >> StringView
std::basic_string< char, std::char_traits< char >, RebindAlloc< ALLOC, char >> string
JsonToken getToken() const
CppRuntimeException(const char *message="")
CppRuntimeException & operator<<(CppRuntimeException &exception, const BasicBitBuffer< ALLOC > &bitBuffer)