Open History to see the last commit
1#include "Utf16ToUtf8.hpp"
2
3#include <cstdint>
4#include <cstring>
5
6namespace margelo::nitro::t3highlights {
7
8void appendUtf16AsUtf8(std::string& out, const char16_t* data, size_t length) {
9 const size_t start = out.size();
10 out.resize(start + length * 3);
11 auto* const begin = reinterpret_cast<uint8_t*>(out.data());
12 auto* write = begin + start;
13 size_t at = 0;
14 while (at < length) {
15 // Runs of four ASCII code units are the common case in source code.
16 if (at + 4 <= length) {
17 uint64_t chunk;
18 std::memcpy(&chunk, data + at, sizeof(chunk));
19 if ((chunk & 0xff80ff80ff80ff80ULL) == 0) {
20 write[0] = static_cast<uint8_t>(data[at]);
21 write[1] = static_cast<uint8_t>(data[at + 1]);
22 write[2] = static_cast<uint8_t>(data[at + 2]);
23 write[3] = static_cast<uint8_t>(data[at + 3]);
24 write += 4;
25 at += 4;
26 continue;
27 }
28 }
29 uint32_t unit = data[at++];
30 if (unit < 0x80) {
31 *write++ = static_cast<uint8_t>(unit);
32 } else if (unit < 0x800) {
33 *write++ = static_cast<uint8_t>(0xc0 | (unit >> 6));
34 *write++ = static_cast<uint8_t>(0x80 | (unit & 0x3f));
35 } else {
36 uint32_t codePoint = unit;
37 if (unit >= 0xd800 && unit <= 0xdfff) {
38 codePoint = 0xfffd;
39 if (unit <= 0xdbff && at < length && data[at] >= 0xdc00 && data[at] <= 0xdfff) {
40 codePoint = 0x10000 + ((unit - 0xd800) << 10) + (data[at++] - 0xdc00);
41 }
42 }
43 if (codePoint < 0x10000) {
44 *write++ = static_cast<uint8_t>(0xe0 | (codePoint >> 12));
45 *write++ = static_cast<uint8_t>(0x80 | ((codePoint >> 6) & 0x3f));
46 *write++ = static_cast<uint8_t>(0x80 | (codePoint & 0x3f));
47 } else {
48 *write++ = static_cast<uint8_t>(0xf0 | (codePoint >> 18));
49 *write++ = static_cast<uint8_t>(0x80 | ((codePoint >> 12) & 0x3f));
50 *write++ = static_cast<uint8_t>(0x80 | ((codePoint >> 6) & 0x3f));
51 *write++ = static_cast<uint8_t>(0x80 | (codePoint & 0x3f));
52 }
53 }
54 }
55 out.resize(static_cast<size_t>(write - begin));
56}
57
58} // namespace margelo::nitro::t3highlights