encodings.hxx

Go to the documentation of this file.

 
#ifndef PQXX_INTERNAL_ENCODINGS_HXX
#define PQXX_INTERNAL_ENCODINGS_HXX
 
#include <cassert>
#include <iomanip>
#include <string>
#include <string_view>
 
#include "pqxx/encoding_group.hxx"
#include "pqxx/strconv.hxx"
 
 
namespace pqxx
{
PQXX_DECLARE_ENUM_CONVERSION(encoding_group);
} // namespace pqxx
 
 
namespace pqxx::internal
{
PQXX_PURE char const *name_encoding(int encoding_id) noexcept;
 
PQXX_PURE PQXX_LIBEXPORT encoding_group
enc_group(int /* libpq encoding ID */, sl);
 
 
 
PQXX_PURE PQXX_INLINE_ONLY constexpr inline unsigned char
get_byte(std::string_view buffer, std::size_t offset) noexcept
{
  assert(offset < std::size(buffer));
  return static_cast<unsigned char>(buffer[offset]);
}
 
 
 
[[noreturn]] PQXX_COLD PQXX_LIBEXPORT PQXX_ZARGS void throw_for_encoding_error(
  char const *encoding_name, std::string_view buffer, std::size_t start,
  std::size_t count, sl loc);
 
 
 
[[noreturn]] PQXX_COLD PQXX_LIBEXPORT PQXX_ZARGS void
throw_for_truncated_character(
  char const *encoding_name, std::string_view buffer, std::size_t start,
  sl loc);
 
 
 
PQXX_PURE PQXX_INLINE_ONLY constexpr inline bool
between_inc(unsigned char value, unsigned bottom, unsigned top) noexcept
{
  return value >= bottom and value <= top;
}
 
 
 
template<encoding_group> struct glyph_scanner final
{
 
  static constexpr inline std::size_t
  call(std::string_view, std::size_t start, sl);
};
 
 
 
template<encoding_group ENC, char... NEEDLE>
PQXX_INLINE_COV inline constexpr std::size_t
find_ascii_char(std::string_view haystack, std::size_t here, sl loc)
{
  // We only know how to search for ASCII characters.  It's an optimisation
  // assumption in the code below.
  static_assert((... and ((NEEDLE & 0x80) == 0)));
 
  auto const sz{std::size(haystack)};
  auto const data{std::data(haystack)};
  while (here < sz)
  {
    // Look up the next character boundary.  This can be quite costly, so we
    // desperately want the call inlined.
    auto next{glyph_scanner<ENC>::call(haystack, here, loc)};
    PQXX_ASSUME(next > here);
 
    // (For some reason gcc had a problem with a right-fold here.  But clang
    // was fine.)
    //
    // In all supported encodings, if a character's first byte is in the ASCII
    // range, that means it's a single-byte character.  It follows that when we
    // find a match at a position that's the beginning of a character, we do
    // not need to check that we're in a single-byte character.  We are.
    //
    // So, we only ever need to check each character's first byte, and if it
    // doesn't match, move on to the next character.
    //
    // As an optimisation for "ASCII-safe" encodings however, we just check
    // every byte in the text.  It's going to be faster than finding character
    // boundaries first.  In these encodings, a multichar byte never contains
    // any bytes in the ASCII range at all.
    if ((... or (data[here] == NEEDLE)))
      return here;
 
    // Nope, no hit.  Move on.
    here = next;
  }
  return sz;
}
 
 
 
template<> struct glyph_scanner<encoding_group::ascii_safe> final
{
  PQXX_INLINE_ONLY PQXX_PURE static constexpr std::size_t
  call(std::string_view, std::size_t start, sl) noexcept
  {
    return start + 1;
  }
};
 
 
 
template<> struct glyph_scanner<encoding_group::two_tier> final
{
  PQXX_INLINE_ONLY static constexpr std::size_t
  call(std::string_view buffer, std::size_t start, sl loc)
  {
    auto const byte1{get_byte(buffer, start)};
    if (byte1 < 0x80)
    {
      // Single-byte ASCII subset.
      return start + 1;
    }
    else if (start + 2 <= std::size(buffer))
    {
      // Two-byte character.  Not all combinations are valid, but that's not
      // our concern.  All that matters to libpqxx is that it not mistake an
      // ASCII-like value in the second byte for a special character, or vice
      // versa.
      return start + 2;
    }
    else
    {
      // We do need to ensure that the string does not end in the middle of
      // a character, or an attacker could "steal" a special ASCII character
      // that comes directly after the end of the input, and escape the bounds
      // of the text that way.
      [[unlikely]] throw_for_truncated_character(
        "variable-width two-byte encoding", buffer, start, loc);
    }
  }
};
 
 
 
template<> struct glyph_scanner<encoding_group::gb18030> final
{
  PQXX_INLINE_ONLY static constexpr std::size_t
  call(std::string_view buffer, std::size_t start, sl loc)
  {
    auto const byte1{get_byte(buffer, start)};
    if (byte1 < 0x80)
      return start + 1;
    auto const sz{std::size(buffer)};
    if (byte1 == 0x80)
      throw_for_encoding_error("GB18030", buffer, start, sz - start, loc);
 
    if (start + 2 > sz) [[unlikely]]
      throw_for_truncated_character("GB18030", buffer, start, loc);
 
    auto const byte2{get_byte(buffer, start + 1)};
    if (between_inc(byte2, 0x40, 0xfe))
    {
      if (byte2 == 0x7f) [[unlikely]]
        throw_for_encoding_error("GB18030", buffer, start, 2, loc);
 
      return start + 2;
    }
 
    if (start + 4 > sz) [[unlikely]]
      throw_for_truncated_character("GB18030", buffer, start, loc);
 
    if (
      between_inc(byte2, 0x30, 0x39) and
      between_inc(get_byte(buffer, start + 2), 0x81, 0xfe) and
      between_inc(get_byte(buffer, start + 3), 0x30, 0x39))
      return start + 4;
 
    [[unlikely]] throw_for_encoding_error("GB18030", buffer, start, 4, loc);
  }
};
 
 
 
template<> struct glyph_scanner<encoding_group::sjis> final
{
  PQXX_INLINE_ONLY static constexpr std::size_t
  call(std::string_view buffer, std::size_t start, sl loc)
  {
    auto const byte1{get_byte(buffer, start)};
    if (byte1 < 0x80)
      // ASCII subset (though some characters changed).
      return start + 1;
    if (between_inc(byte1, 0xa1, 0xdf))
      // Katakana, also single-byte characters.
      return start + 1;
 
    // We're a bit strict at checking the first byte, because this is a
    // relatively complex encoding.  We don't want to get fooled by some
    // extension we don't know about.  An error and a user complaint is still
    // better than a lurking bug.
    if (
      not between_inc(byte1, 0x81, 0x9f) and
      not between_inc(byte1, 0xe0, 0xfc)) [[unlikely]]
      throw_for_encoding_error("SJIS", buffer, start, 1, loc);
 
    if (start + 2 > std::size(buffer)) [[unlikely]]
      throw_for_truncated_character("SJIS", buffer, start, loc);
 
    return start + 2;
  }
};
 
 
 
template<char... NEEDLE>
PQXX_PURE
  PQXX_RETURNS_NONNULL PQXX_INLINE_COV constexpr inline char_finder_func *
  get_char_finder(encoding_group enc, sl loc)
{
  // All characters in NEEDLE must be ASCII.
  static_assert((... and (static_cast<unsigned char>(NEEDLE) < 0x80)));
 
  // We don't support searching for a NEEDLE that's a letter.  This allows us
  // to lump UHC in with the more efficient ASCII-safe group.
  static_assert((... and not between_inc(NEEDLE, 'A', 'Z')));
  static_assert((... and not between_inc(NEEDLE, 'a', 'z')));
 
  switch (enc)
  {
  case encoding_group::unknown:
    throw pqxx::argument_error{
      "Tried to read text without knowing its encoding.", loc};
 
  case encoding_group::ascii_safe:
    return pqxx::internal::find_ascii_char<
      encoding_group::ascii_safe, NEEDLE...>;
  case encoding_group::two_tier:
    return pqxx::internal::find_ascii_char<
      encoding_group::two_tier, NEEDLE...>;
  case encoding_group::gb18030:
    return pqxx::internal::find_ascii_char<encoding_group::gb18030, NEEDLE...>;
  case encoding_group::sjis:
    return pqxx::internal::find_ascii_char<encoding_group::sjis, NEEDLE...>;
 
  default:
    throw pqxx::internal_error{
      std::format("Unexpected encoding group: {}.", to_string(enc)), loc};
  }
}
} // namespace pqxx::internal
#endif