stdbuiltinasserttestingmemnative_endianAllocatorreplacement_characterreplacement_character_utf8utf8CodepointSequenceLengthutf8ByteSequenceLengthutf8EncodeSurrogatesutf8EncodeImplutf8EncodeComptimeUtf8DecodeErrorutf8DecodeUtf8Decode2Errorutf8Decode2Utf8Decode3Errorutf8Decode3Utf8Decode3AllowSurrogateHalfErrorutf8Decode3AllowSurrogateHalfUtf8Decode4Errorutf8Decode4utf8ValidCodepointutf8CountCodepointsutf8ValidateSliceutf8ValidateSliceImplUtf8ViewUtf8Iteratorutf16IsHighSurrogateutf16IsLowSurrogateutf16CodepointSequenceLengthutf16CodeUnitSequenceLengthutf16DecodeSurrogatePairUtf16LeIteratorutf16CountCodepointstestUtf16CountCodepointstestUtf8EncodetestUtf8EncodeErrortestErrorEncodetestUtf8IteratorOnAsciitestUtf8ViewBadtestUtf8ViewOktestValidateSlicetestValidUtf8testInvalidUtf8ContinuationBytestestOverlongUtf8CodepointtestMiscInvalidUtf8testUtf8PeekingtestUtf8PeekCodepointtestErrortestValidtestDecodeformatUtf8fmtUtf8utf16LeToUtf8ArrayListImplUtf16LeToUtf8AllocErrorutf16LeToUtf8ArrayListutf16LeToUtf8Allocutf16LeToUtf8AllocZUtf16LeToUtf8Errorutf16LeToUtf8Implutf16LeToUtf8utf8ToUtf16LeArrayListImplutf8ToUtf16LeArrayListutf8ToUtf16LeAllocutf8ToUtf16LeAllocZutf8ToUtf16Leutf8ToUtf16LeImplutf8ToUtf16LeStringLiteralImplutf8ToUtf16LeStringLiteralwtf8ToWtf16LeStringLiteralcalcUtf16LeLenImplCalcUtf16LeLenErrorcalcUtf16LeLenCalcWtf16LeLenErrorcalcWtf16LeLentestCalcUtf16LeLenImplformatUtf16LefmtUtf16LetestUtf8ToUtf16LeStringLiteraltestUtf8CountCodepointstestUtf8ValidCodepointisSurrogateCodepointwtf8EncodeWtf8DecodeErrorwtf8Decodewtf8ValidateSlicetestValidateWtf8SliceWtf8ViewWtf8Iteratorwtf16LeToWtf8ArrayListwtf16LeToWtf8Allocwtf16LeToWtf8AllocZwtf16LeToWtf8wtf8ToWtf16LeArrayListwtf8ToWtf16LeAllocwtf8ToWtf16LeAllocZwtf8ToWtf16Lewtf8ToUtf8Lossywtf8ToUtf8LossyAllocwtf8ToUtf8LossyAllocZWtf16LeIteratortestRoundtripWtf8testRoundtripWtf16calcWtf8LentestCalcWtf8Len
std.unicode.Allocator
别名
别名目标 std.mem.Allocator
std.unicode.assert
别名
别名目标 std.debug.assert
std.unicode.builtin
常量
std.unicode.calcUtf16LeLen
函数
以 []u16 的长度形式返回 UTF-8 切片的 UTF-16LE 长度。 []u8 中的长度为 2*len16。
pub fn calcUtf16LeLen(utf8: []const u8) CalcUtf16LeLenError!usize
参数
utf8: []const u8
错误
- Utf8CodepointTooLarge Utf8Decode4Error
- Utf8EncodesSurrogateHalf Utf8Decode3Error
- Utf8ExpectedContinuation Utf8Decode2Error
- Utf8InvalidStartByte
- Utf8OverlongEncoding Utf8Decode2Error
用法示例
test calcUtf16LeLen {
try testCalcUtf16LeLenImpl(calcUtf16LeLen);
try comptime testCalcUtf16LeLenImpl(calcUtf16LeLen);
}
std.unicode.CalcUtf16LeLenError
错误集合
错误
- Utf8CodepointTooLarge Utf8Decode4Error
- Utf8EncodesSurrogateHalf Utf8Decode3Error
- Utf8ExpectedContinuation Utf8Decode2Error
- Utf8InvalidStartByte
- Utf8OverlongEncoding Utf8Decode2Error
std.unicode.calcUtf16LeLenImpl
函数
pub fn calcUtf16LeLenImpl(utf8: []const u8, comptime surrogates: Surrogates) !usize
参数
utf8: []const u8
surrogates: Surrogates
std.unicode.calcWtf16LeLen
函数
返回 WTF-8 切片转换为 WTF-16LE 后的长度,按 []u16 元素数计。 按 []u8 计的长度为 2*len16。
pub fn calcWtf16LeLen(wtf8: []const u8) CalcWtf16LeLenError!usize
参数
wtf8: []const u8
错误
- Utf8CodepointTooLarge Utf8Decode4Error
- Utf8ExpectedContinuation Utf8Decode2Error
- Utf8InvalidStartByte
- Utf8OverlongEncoding Utf8Decode2Error
用法示例
test calcWtf16LeLen {
try testCalcUtf16LeLenImpl(calcWtf16LeLen);
try comptime testCalcUtf16LeLenImpl(calcWtf16LeLen);
}
std.unicode.CalcWtf16LeLenError
错误集合
错误
- Utf8CodepointTooLarge Utf8Decode4Error
- Utf8ExpectedContinuation Utf8Decode2Error
- Utf8InvalidStartByte
- Utf8OverlongEncoding Utf8Decode2Error
std.unicode.calcWtf8Len
函数
返回将给定 WTF-16 LE 切片编码为 WTF-8 所需的字节数。
pub fn calcWtf8Len(wtf16le: []const u16) usize
参数
wtf16le: []const u16
std.unicode.fmtUtf16Le
函数
返回用于 UTF-16 LE 字符串(可能格式不正确)的 Formatter, 格式化时将其转换为 UTF-8。 未配对代理码元会替换为替换字符(U+FFFD)。
pub fn fmtUtf16Le(utf16le: []const u16) std.fmt.Alt([]const u16, formatUtf16Le)
参数
utf16le: []const u16
用法示例
test fmtUtf16Le {
const expectFmt = testing.expectFmt;
try expectFmt("", "{f}", .{fmtUtf16Le(utf8ToUtf16LeStringLiteral(""))});
try expectFmt("", "{f}", .{fmtUtf16Le(wtf8ToWtf16LeStringLiteral(""))});
try expectFmt("foo", "{f}", .{fmtUtf16Le(utf8ToUtf16LeStringLiteral("foo"))});
try expectFmt("foo", "{f}", .{fmtUtf16Le(wtf8ToWtf16LeStringLiteral("foo"))});
try expectFmt("𐐷", "{f}", .{fmtUtf16Le(wtf8ToWtf16LeStringLiteral("𐐷"))});
try expectFmt("", "{f}", .{fmtUtf16Le(&[_]u16{mem.readInt(u16, "\xff\xd7", native_endian)})});
try expectFmt("�", "{f}", .{fmtUtf16Le(&[_]u16{mem.readInt(u16, "\x00\xd8", native_endian)})});
try expectFmt("�", "{f}", .{fmtUtf16Le(&[_]u16{mem.readInt(u16, "\xff\xdb", native_endian)})});
try expectFmt("�", "{f}", .{fmtUtf16Le(&[_]u16{mem.readInt(u16, "\x00\xdc", native_endian)})});
try expectFmt("�", "{f}", .{fmtUtf16Le(&[_]u16{mem.readInt(u16, "\xff\xdf", native_endian)})});
try expectFmt("", "{f}", .{fmtUtf16Le(&[_]u16{mem.readInt(u16, "\x00\xe0", native_endian)})});
}
std.unicode.fmtUtf8
函数
为可能结构不良的 UTF-8 字符串返回 Formatter。 根据 Unicode 标准第 3 章“用 U+FFFD 替换最大子部分”及 https://encoding.spec.whatwg.org/#utf-8-decoder 的规定,将结构不良的 UTF-8 字节序列替换为替换字符 U+FFFD。
pub fn fmtUtf8(utf8: []const u8) std.fmt.Alt([]const u8, formatUtf8)
参数
utf8: []const u8
用法示例
test fmtUtf8 {
const expectFmt = testing.expectFmt;
try expectFmt("", "{f}", .{fmtUtf8("")});
try expectFmt("foo", "{f}", .{fmtUtf8("foo")});
try expectFmt("𐐷", "{f}", .{fmtUtf8("𐐷")});
// 表 3-8:用于非最短形式序列的 U+FFFD。
try expectFmt("��������A", "{f}", .{fmtUtf8("\xC0\xAF\xE0\x80\xBF\xF0\x81\x82A")});
// 表 3-9:用于代理码点格式错误序列的 U+FFFD。
try expectFmt("��������A", "{f}", .{fmtUtf8("\xED\xA0\x80\xED\xBF\xBF\xED\xAFA")});
// 表 3-10。其他格式错误序列对应的 U+FFFD
try expectFmt("�����A��B", "{f}", .{fmtUtf8("\xF4\x91\x92\x93\xFFA\x80\xBFB")});
// 表 3-11。截断序列使用 U+FFFD
try expectFmt("����A", "{f}", .{fmtUtf8("\xE1\x80\xE2\xF0\x91\x92\xF1\xBFA")});
}
std.unicode.formatUtf16Le
函数
将给定 utf16le 字符串编码为 UTF-8 字节并打印。
未配对代理项会被替换字符 (U+FFFD) 替换。
fn formatUtf16Le(utf16le: []const u16, writer: *std.Io.Writer) std.Io.Writer.Error!void
参数
错误
- WriteFailed
详细诊断信息见
Writer实现。
std.unicode.formatUtf8
函数
打印给定 utf8 字符串,按 UTF-8 字节编码。
根据 Unicode 标准第 3 章“用 U+FFFD 替换最大子部分”及 https://encoding.spec.whatwg.org/#utf-8-decoder 的规定,将结构不良的 UTF-8 字节序列替换为替换字符 U+FFFD。
fn formatUtf8(utf8: []const u8, writer: *std.Io.Writer) std.Io.Writer.Error!void
参数
错误
- WriteFailed
详细诊断信息见
Writer实现。
std.unicode.isSurrogateCodepoint
函数
若码点为代理码点(U+DC00 到 U+DFFF),则返回 true
pub fn isSurrogateCodepoint(c: u21) bool
参数
c: u21
std.unicode.mem
别名
别名目标 std.mem
std.unicode.native_endian
常量
std.unicode.replacement_character
常量
用此字符替换未知、无法识别或无法表示的字符。
另见:https://en.wikipedia.org/wiki/Specials_(Unicode_block)#Replacement_character
u21
std.unicode.replacement_character_utf8
常量
[3]u8
std.unicode.std
别名
别名目标 std
std.unicode.Surrogates
容器
字段
cannot_encode_surrogate_half
can_encode_surrogate_half
std.unicode.testCalcUtf16LeLenImpl
函数
fn testCalcUtf16LeLenImpl(calcUtf16LeLenImpl_: anytype) !void
std.unicode.testCalcWtf8Len
函数
fn testCalcWtf8Len() !void
std.unicode.testDecode
函数
fn testDecode(bytes: []const u8) !u21
参数
bytes: []const u8
std.unicode.testError
函数
fn testError(bytes: []const u8, expected_err: anyerror) !void
参数
bytes: []const u8
expected_err: anyerror
std.unicode.testErrorEncode
函数
fn testErrorEncode(codePoint: u21, array: []u8, expectedErr: anyerror) !void
参数
codePoint: u21
array: []u8
expectedErr: anyerror
std.unicode.testing
别名
别名目标 std.testing
std.unicode.testInvalidUtf8ContinuationBytes
函数
fn testInvalidUtf8ContinuationBytes() !void
std.unicode.testMiscInvalidUtf8
函数
fn testMiscInvalidUtf8() !void
std.unicode.testOverlongUtf8Codepoint
函数
fn testOverlongUtf8Codepoint() !void
std.unicode.testRoundtripWtf16
函数
fn testRoundtripWtf16(wtf16le: []const u16) !void
参数
wtf16le: []const u16
std.unicode.testRoundtripWtf8
函数
fn testRoundtripWtf8(wtf8: []const u8) !void
参数
wtf8: []const u8
std.unicode.testUtf16CountCodepoints
函数
fn testUtf16CountCodepoints() !void
std.unicode.testUtf8CountCodepoints
函数
fn testUtf8CountCodepoints() !void
std.unicode.testUtf8Encode
函数
fn testUtf8Encode() !void
std.unicode.testUtf8EncodeError
函数
fn testUtf8EncodeError() !void
std.unicode.testUtf8IteratorOnAscii
函数
fn testUtf8IteratorOnAscii() !void
std.unicode.testUtf8PeekCodepoint
函数
fn testUtf8PeekCodepoint() !void
std.unicode.testUtf8Peeking
函数
fn testUtf8Peeking() !void
std.unicode.testUtf8ToUtf16LeStringLiteral
函数
fn testUtf8ToUtf16LeStringLiteral(utf8ToUtf16LeStringLiteral_: anytype) !void
std.unicode.testUtf8ValidCodepoint
函数
fn testUtf8ValidCodepoint() !void
std.unicode.testUtf8ViewBad
函数
fn testUtf8ViewBad() !void
std.unicode.testUtf8ViewOk
函数
fn testUtf8ViewOk() !void
std.unicode.testValid
函数
fn testValid(bytes: []const u8, expected_codepoint: u21) !void
参数
bytes: []const u8
expected_codepoint: u21
std.unicode.testValidateSlice
函数
fn testValidateSlice() !void
std.unicode.testValidateWtf8Slice
函数
fn testValidateWtf8Slice() !void
std.unicode.testValidUtf8
函数
fn testValidUtf8() !void
std.unicode.utf16CodepointSequenceLength
函数
返回给定码点的 UTF-16 表示所需的 代码单元数量。
pub fn utf16CodepointSequenceLength(c: u21) !u2
参数
c: u21
用法示例
test utf16CodepointSequenceLength {
try testing.expectEqual(@as(u2, 1), try utf16CodepointSequenceLength('a'));
try testing.expectEqual(@as(u2, 1), try utf16CodepointSequenceLength(0xFFFF));
try testing.expectEqual(@as(u2, 2), try utf16CodepointSequenceLength(0x10000));
try testing.expectEqual(@as(u2, 2), try utf16CodepointSequenceLength(0x10FFFF));
try testing.expectError(error.CodepointTooLarge, utf16CodepointSequenceLength(0x110000));
}
std.unicode.utf16CodeUnitSequenceLength
函数
给定 UTF-16 码点的第一个码元,返回 1 至 2 之间的数字, 表示该码点包含的 UTF-16 码元总数。 如果此码元不符合 UTF-16 起始码元的格式,则返回 Utf16InvalidStartCodeUnit。
pub fn utf16CodeUnitSequenceLength(first_code_unit: u16) !u2
参数
first_code_unit: u16
用法示例
test utf16CodeUnitSequenceLength {
try testing.expectEqual(@as(u2, 1), try utf16CodeUnitSequenceLength('a'));
try testing.expectEqual(@as(u2, 1), try utf16CodeUnitSequenceLength(0xFFFF));
try testing.expectEqual(@as(u2, 2), try utf16CodeUnitSequenceLength(0xDBFF));
try testing.expectError(error.Utf16InvalidStartCodeUnit, utf16CodeUnitSequenceLength(0xDFFF));
}
std.unicode.utf16CountCodepoints
函数
以 Unicode 码点数为单位,返回所提供 UTF-16 字符串字面量的长度。
pub fn utf16CountCodepoints(utf16le: []const u16) !usize
参数
utf16le: []const u16
std.unicode.utf16DecodeSurrogatePair
函数
解码给定的一对 UTF-16 码元所编码的码点。
断言 surrogate_pair.len >= 2,且第一个码元为高位代理项。
若第二个码元不是低位代理项,返回 error.ExpectedSecondSurrogateHalf。
pub fn utf16DecodeSurrogatePair(surrogate_pair: []const u16) !u21
参数
surrogate_pair: []const u16
std.unicode.utf16IsHighSurrogate
函数
pub fn utf16IsHighSurrogate(c: u16) bool
参数
c: u16
std.unicode.utf16IsLowSurrogate
函数
pub fn utf16IsLowSurrogate(c: u16) bool
参数
c: u16
std.unicode.Utf16LeIterator
容器
字段
bytes: []const u8
i: usize
std.unicode.Utf16LeIterator.init
函数
pub fn init(s: []const u16) Utf16LeIterator
参数
s: []const u16
std.unicode.Utf16LeIterator.nextCodepoint
函数
pub fn nextCodepoint(it: *Utf16LeIterator) NextCodepointError!?u21
参数
it: *Utf16LeIterator
错误
- DanglingSurrogateHalf
- ExpectedSecondSurrogateHalf
- UnexpectedSecondSurrogateHalf
std.unicode.Utf16LeIterator.NextCodepointError
错误集合
错误
- DanglingSurrogateHalf
- ExpectedSecondSurrogateHalf
- UnexpectedSecondSurrogateHalf
std.unicode.utf16LeToUtf8
函数
pub fn utf16LeToUtf8(utf8: []u8, utf16le: []const u16) Utf16LeToUtf8Error!usize
参数
utf8: []u8
utf16le: []const u16
错误
- DanglingSurrogateHalf NextCodepointError
- ExpectedSecondSurrogateHalf NextCodepointError
- UnexpectedSecondSurrogateHalf NextCodepointError
用法示例
test utf16LeToUtf8 {
var utf16le: [2]u16 = undefined;
const utf16le_as_bytes = mem.sliceAsBytes(utf16le[0..]);
{
mem.writeInt(u16, utf16le_as_bytes[0..2], 'A', .little);
mem.writeInt(u16, utf16le_as_bytes[2..4], 'a', .little);
const utf8 = try utf16LeToUtf8Alloc(testing.allocator, &utf16le);
defer testing.allocator.free(utf8);
try testing.expect(mem.eql(u8, utf8, "Aa"));
}
{
mem.writeInt(u16, utf16le_as_bytes[0..2], 0x80, .little);
mem.writeInt(u16, utf16le_as_bytes[2..4], 0xffff, .little);
const utf8 = try utf16LeToUtf8Alloc(testing.allocator, &utf16le);
defer testing.allocator.free(utf8);
try testing.expect(mem.eql(u8, utf8, "\xc2\x80" ++ "\xef\xbf\xbf"));
}
{
// 刚好位于代理半区范围之外的值
mem.writeInt(u16, utf16le_as_bytes[0..2], 0xd7ff, .little);
mem.writeInt(u16, utf16le_as_bytes[2..4], 0xe000, .little);
const utf8 = try utf16LeToUtf8Alloc(testing.allocator, &utf16le);
defer testing.allocator.free(utf8);
try testing.expect(mem.eql(u8, utf8, "\xed\x9f\xbf" ++ "\xee\x80\x80"));
}
{
// 最小的代理对
mem.writeInt(u16, utf16le_as_bytes[0..2], 0xd800, .little);
mem.writeInt(u16, utf16le_as_bytes[2..4], 0xdc00, .little);
const utf8 = try utf16LeToUtf8Alloc(testing.allocator, &utf16le);
defer testing.allocator.free(utf8);
try testing.expect(mem.eql(u8, utf8, "\xf0\x90\x80\x80"));
}
{
// 最大的代理对
mem.writeInt(u16, utf16le_as_bytes[0..2], 0xdbff, .little);
mem.writeInt(u16, utf16le_as_bytes[2..4], 0xdfff, .little);
const utf8 = try utf16LeToUtf8Alloc(testing.allocator, &utf16le);
defer testing.allocator.free(utf8);
try testing.expect(mem.eql(u8, utf8, "\xf4\x8f\xbf\xbf"));
}
{
mem.writeInt(u16, utf16le_as_bytes[0..2], 0xdbff, .little);
mem.writeInt(u16, utf16le_as_bytes[2..4], 0xdc00, .little);
const utf8 = try utf16LeToUtf8Alloc(testing.allocator, &utf16le);
defer testing.allocator.free(utf8);
try testing.expect(mem.eql(u8, utf8, "\xf4\x8f\xb0\x80"));
}
{
mem.writeInt(u16, utf16le_as_bytes[0..2], 0xdcdc, .little);
mem.writeInt(u16, utf16le_as_bytes[2..4], 0xdcdc, .little);
const result = utf16LeToUtf8Alloc(testing.allocator, &utf16le);
try testing.expectError(error.UnexpectedSecondSurrogateHalf, result);
}
}
std.unicode.utf16LeToUtf8Alloc
函数
调用方拥有返回内存的所有权。
pub fn utf16LeToUtf8Alloc(allocator: Allocator, utf16le: []const u16) Utf16LeToUtf8AllocError![]u8
参数
allocator: Allocator
utf16le: []const u16
错误
- DanglingSurrogateHalf NextCodepointError
- ExpectedSecondSurrogateHalf NextCodepointError
- OutOfMemory Error
- UnexpectedSecondSurrogateHalf NextCodepointError
std.unicode.Utf16LeToUtf8AllocError
错误集合
错误
- DanglingSurrogateHalf NextCodepointError
- ExpectedSecondSurrogateHalf NextCodepointError
- OutOfMemory Error
- UnexpectedSecondSurrogateHalf NextCodepointError
std.unicode.utf16LeToUtf8AllocZ
函数
调用方拥有返回内存的所有权。
pub fn utf16LeToUtf8AllocZ(allocator: Allocator, utf16le: []const u16) Utf16LeToUtf8AllocError![:0]u8
参数
allocator: Allocator
utf16le: []const u16
错误
- DanglingSurrogateHalf NextCodepointError
- ExpectedSecondSurrogateHalf NextCodepointError
- OutOfMemory Error
- UnexpectedSecondSurrogateHalf NextCodepointError
std.unicode.utf16LeToUtf8ArrayList
函数
pub fn utf16LeToUtf8ArrayList(result: *std.array_list.Managed(u8), utf16le: []const u16) Utf16LeToUtf8AllocError!void
参数
result: *std.array_list.Managed(u8)
utf16le: []const u16
错误
- DanglingSurrogateHalf NextCodepointError
- ExpectedSecondSurrogateHalf NextCodepointError
- OutOfMemory Error
- UnexpectedSecondSurrogateHalf NextCodepointError
std.unicode.utf16LeToUtf8ArrayListImpl
函数
fn utf16LeToUtf8ArrayListImpl( result: *std.array_list.Managed(u8), utf16le: []const u16, comptime surrogates: Surrogates, ) (switch (surrogates) { .cannot_encode_surrogate_half => Utf16LeToUtf8AllocError, .can_encode_surrogate_half => Allocator.Error, })!void
参数
std.unicode.Utf16LeToUtf8Error
别名
std.unicode.utf16LeToUtf8Impl
函数
断言输出缓冲区足够大。 返回 utf8 中的结束字节索引。
fn utf16LeToUtf8Impl(utf8: []u8, utf16le: []const u16, comptime surrogates: Surrogates) (switch (surrogates) { .cannot_encode_surrogate_half => Utf16LeToUtf8Error, .can_encode_surrogate_half => error{}, })!usize
参数
std.unicode.utf8ByteSequenceLength
函数
给定 UTF-8 码点的第一个字节, 返回 1 至 4 之间的数字,表示该码点的总字节长度。 如果此字节不符合 UTF-8 起始字节的格式,则返回 Utf8InvalidStartByte。
pub fn utf8ByteSequenceLength(first_byte: u8) !u3
参数
first_byte: u8
std.unicode.utf8CodepointSequenceLength
函数
返回给定码点的 UTF-8 表示 所需的字节数。
pub fn utf8CodepointSequenceLength(c: u21) !u3
参数
c: u21
std.unicode.utf8CountCodepoints
函数
返回给定 UTF-8 字符串字面量所含的 Unicode 码点数量。
pub fn utf8CountCodepoints(s: []const u8) !usize
参数
s: []const u8
std.unicode.utf8Decode
函数
已弃用。此函数的 API 不够直观,容易误用。
pub fn utf8Decode(bytes: []const u8) Utf8DecodeError!u21
参数
bytes: []const u8
错误
- Utf8CodepointTooLarge Utf8Decode4Error
- Utf8EncodesSurrogateHalf Utf8Decode3Error
- Utf8ExpectedContinuation Utf8Decode2Error
- Utf8OverlongEncoding Utf8Decode2Error
std.unicode.utf8Decode2
函数
pub fn utf8Decode2(bytes: [2]u8) Utf8Decode2Error!u21
参数
bytes: [2]u8
错误
- Utf8ExpectedContinuation
- Utf8OverlongEncoding
std.unicode.Utf8Decode2Error
错误集合
错误
- Utf8ExpectedContinuation
- Utf8OverlongEncoding
std.unicode.utf8Decode3
函数
pub fn utf8Decode3(bytes: [3]u8) Utf8Decode3Error!u21
参数
bytes: [3]u8
错误
- Utf8EncodesSurrogateHalf
- Utf8ExpectedContinuation Utf8Decode3AllowSurrogateHalfError
- Utf8OverlongEncoding Utf8Decode3AllowSurrogateHalfError
std.unicode.utf8Decode3AllowSurrogateHalf
函数
pub fn utf8Decode3AllowSurrogateHalf(bytes: [3]u8) Utf8Decode3AllowSurrogateHalfError!u21
参数
bytes: [3]u8
错误
- Utf8ExpectedContinuation
- Utf8OverlongEncoding
std.unicode.Utf8Decode3AllowSurrogateHalfError
错误集合
错误
- Utf8ExpectedContinuation
- Utf8OverlongEncoding
std.unicode.Utf8Decode3Error
错误集合
错误
- Utf8EncodesSurrogateHalf
- Utf8ExpectedContinuation Utf8Decode3AllowSurrogateHalfError
- Utf8OverlongEncoding Utf8Decode3AllowSurrogateHalfError
std.unicode.utf8Decode4
函数
pub fn utf8Decode4(bytes: [4]u8) Utf8Decode4Error!u21
参数
bytes: [4]u8
错误
- Utf8CodepointTooLarge
- Utf8ExpectedContinuation
- Utf8OverlongEncoding
std.unicode.Utf8Decode4Error
错误集合
错误
- Utf8CodepointTooLarge
- Utf8ExpectedContinuation
- Utf8OverlongEncoding
std.unicode.Utf8DecodeError
错误集合
错误
- Utf8CodepointTooLarge Utf8Decode4Error
- Utf8EncodesSurrogateHalf Utf8Decode3Error
- Utf8ExpectedContinuation Utf8Decode2Error
- Utf8OverlongEncoding Utf8Decode2Error
std.unicode.utf8Encode
函数
将给定码点编码为 UTF-8 字节序列。 c:码点。 out:用于写入的输出缓冲区。长度必须 >= utf8CodepointSequenceLength(c)。 错误:c 无法编码为 UTF-8 时返回错误。 返回值:写入 out 的字节数。
pub fn utf8Encode(c: u21, out: []u8) error{ Utf8CannotEncodeSurrogateHalf, CodepointTooLarge }!u3
参数
c: u21
out: []u8
错误
- CodepointTooLarge
- Utf8CannotEncodeSurrogateHalf
std.unicode.utf8EncodeComptime
函数
pub inline fn utf8EncodeComptime(comptime c: u21) [ utf8CodepointSequenceLength(c) catch |err| @compileError(@errorName(err)) ]u8
参数
c: u21
std.unicode.utf8EncodeImpl
函数
fn utf8EncodeImpl(c: u21, out: []u8, comptime surrogates: Surrogates) !u3
参数
std.unicode.Utf8Iterator
容器
字段
bytes: []const u8
i: usize
std.unicode.Utf8Iterator.nextCodepoint
函数
pub fn nextCodepoint(it: *Utf8Iterator) ?u21
参数
it: *Utf8Iterator
std.unicode.Utf8Iterator.nextCodepointSlice
函数
pub fn nextCodepointSlice(it: *Utf8Iterator) ?[]const u8
参数
it: *Utf8Iterator
std.unicode.Utf8Iterator.peek
函数
向前查看接下来的 n 个码点,而不推进迭代器。 如果可用码点不足 n 个,则返回字符串的剩余部分。
pub fn peek(it: *Utf8Iterator, n: usize) []const u8
参数
it: *Utf8Iterator
n: usize
std.unicode.Utf8Iterator.peekCodepoint
函数
向前查看下一个码点,而不推进迭代器。 若不存在码点,则返回 null。
pub fn peekCodepoint(it: *Utf8Iterator) ?u21
参数
it: *Utf8Iterator
std.unicode.utf8ToUtf16Le
函数
返回下一个字符的索引。若恰好填满,返回索引等于输出切片长度。 假定输出空间足够。
pub fn utf8ToUtf16Le(utf16le: []u16, utf8: []const u8) error{InvalidUtf8}!usize
参数
utf16le: []u16
utf8: []const u8
错误
- InvalidUtf8
用法示例
test utf8ToUtf16Le {
var utf16le: [128]u16 = undefined;
{
const length = try utf8ToUtf16Le(utf16le[0..], "𐐷");
try testing.expectEqualSlices(u8, "\x01\xd8\x37\xdc", mem.sliceAsBytes(utf16le[0..length]));
}
{
const length = try utf8ToUtf16Le(utf16le[0..], "\u{10FFFF}");
try testing.expectEqualSlices(u8, "\xff\xdb\xff\xdf", mem.sliceAsBytes(utf16le[0..length]));
}
{
const result = utf8ToUtf16Le(utf16le[0..], "\xf4\x90\x80\x80");
try testing.expectError(error.InvalidUtf8, result);
}
{
const length = try utf8ToUtf16Le(utf16le[0..], "This string has been designed to test the vectorized implementat" ++
"ion by beginning with one hundred twenty-seven ASCII characters¡");
try testing.expectEqualSlices(u8, &.{
'T', 0, 'h', 0, 'i', 0, 's', 0, ' ', 0, 's', 0, 't', 0, 'r', 0, 'i', 0, 'n', 0, 'g', 0, ' ', 0, 'h', 0, 'a', 0, 's', 0, ' ', 0,
'b', 0, 'e', 0, 'e', 0, 'n', 0, ' ', 0, 'd', 0, 'e', 0, 's', 0, 'i', 0, 'g', 0, 'n', 0, 'e', 0, 'd', 0, ' ', 0, 't', 0, 'o', 0,
' ', 0, 't', 0, 'e', 0, 's', 0, 't', 0, ' ', 0, 't', 0, 'h', 0, 'e', 0, ' ', 0, 'v', 0, 'e', 0, 'c', 0, 't', 0, 'o', 0, 'r', 0,
'i', 0, 'z', 0, 'e', 0, 'd', 0, ' ', 0, 'i', 0, 'm', 0, 'p', 0, 'l', 0, 'e', 0, 'm', 0, 'e', 0, 'n', 0, 't', 0, 'a', 0, 't', 0,
'i', 0, 'o', 0, 'n', 0, ' ', 0, 'b', 0, 'y', 0, ' ', 0, 'b', 0, 'e', 0, 'g', 0, 'i', 0, 'n', 0, 'n', 0, 'i', 0, 'n', 0, 'g', 0,
' ', 0, 'w', 0, 'i', 0, 't', 0, 'h', 0, ' ', 0, 'o', 0, 'n', 0, 'e', 0, ' ', 0, 'h', 0, 'u', 0, 'n', 0, 'd', 0, 'r', 0, 'e', 0,
'd', 0, ' ', 0, 't', 0, 'w', 0, 'e', 0, 'n', 0, 't', 0, 'y', 0, '-', 0, 's', 0, 'e', 0, 'v', 0, 'e', 0, 'n', 0, ' ', 0, 'A', 0,
'S', 0, 'C', 0, 'I', 0, 'I', 0, ' ', 0, 'c', 0, 'h', 0, 'a', 0, 'r', 0, 'a', 0, 'c', 0, 't', 0, 'e', 0, 'r', 0, 's', 0,
'¡',
0,
}, mem.sliceAsBytes(utf16le[0..length]));
}
}
std.unicode.utf8ToUtf16LeAlloc
函数
pub fn utf8ToUtf16LeAlloc(allocator: Allocator, utf8: []const u8) error{ InvalidUtf8, OutOfMemory }![]u16
参数
allocator: Allocator
utf8: []const u8
错误
- InvalidUtf8
- OutOfMemory
用法示例
test utf8ToUtf16LeAlloc {
{
const utf16 = try utf8ToUtf16LeAlloc(testing.allocator, "𐐷");
defer testing.allocator.free(utf16);
try testing.expectEqualSlices(u8, "\x01\xd8\x37\xdc", mem.sliceAsBytes(utf16[0..]));
}
{
const utf16 = try utf8ToUtf16LeAlloc(testing.allocator, "\u{10FFFF}");
defer testing.allocator.free(utf16);
try testing.expectEqualSlices(u8, "\xff\xdb\xff\xdf", mem.sliceAsBytes(utf16[0..]));
}
{
const result = utf8ToUtf16LeAlloc(testing.allocator, "\xf4\x90\x80\x80");
try testing.expectError(error.InvalidUtf8, result);
}
}
std.unicode.utf8ToUtf16LeAllocZ
函数
pub fn utf8ToUtf16LeAllocZ(allocator: Allocator, utf8: []const u8) error{ InvalidUtf8, OutOfMemory }![:0]u16
参数
allocator: Allocator
utf8: []const u8
错误
- InvalidUtf8
- OutOfMemory
用法示例
test utf8ToUtf16LeAllocZ {
{
const utf16 = try utf8ToUtf16LeAllocZ(testing.allocator, "𐐷");
defer testing.allocator.free(utf16);
try testing.expectEqualSlices(u8, "\x01\xd8\x37\xdc", mem.sliceAsBytes(utf16));
try testing.expect(utf16[2] == 0);
}
{
const utf16 = try utf8ToUtf16LeAllocZ(testing.allocator, "\u{10FFFF}");
defer testing.allocator.free(utf16);
try testing.expectEqualSlices(u8, "\xff\xdb\xff\xdf", mem.sliceAsBytes(utf16));
try testing.expect(utf16[2] == 0);
}
{
const result = utf8ToUtf16LeAllocZ(testing.allocator, "\xf4\x90\x80\x80");
try testing.expectError(error.InvalidUtf8, result);
}
{
const utf16 = try utf8ToUtf16LeAllocZ(testing.allocator, "This string has been designed to test the vectorized implementat" ++
"ion by beginning with one hundred twenty-seven ASCII characters¡");
defer testing.allocator.free(utf16);
try testing.expectEqualSlices(u8, &.{
'T', 0, 'h', 0, 'i', 0, 's', 0, ' ', 0, 's', 0, 't', 0, 'r', 0, 'i', 0, 'n', 0, 'g', 0, ' ', 0, 'h', 0, 'a', 0, 's', 0, ' ', 0,
'b', 0, 'e', 0, 'e', 0, 'n', 0, ' ', 0, 'd', 0, 'e', 0, 's', 0, 'i', 0, 'g', 0, 'n', 0, 'e', 0, 'd', 0, ' ', 0, 't', 0, 'o', 0,
' ', 0, 't', 0, 'e', 0, 's', 0, 't', 0, ' ', 0, 't', 0, 'h', 0, 'e', 0, ' ', 0, 'v', 0, 'e', 0, 'c', 0, 't', 0, 'o', 0, 'r', 0,
'i', 0, 'z', 0, 'e', 0, 'd', 0, ' ', 0, 'i', 0, 'm', 0, 'p', 0, 'l', 0, 'e', 0, 'm', 0, 'e', 0, 'n', 0, 't', 0, 'a', 0, 't', 0,
'i', 0, 'o', 0, 'n', 0, ' ', 0, 'b', 0, 'y', 0, ' ', 0, 'b', 0, 'e', 0, 'g', 0, 'i', 0, 'n', 0, 'n', 0, 'i', 0, 'n', 0, 'g', 0,
' ', 0, 'w', 0, 'i', 0, 't', 0, 'h', 0, ' ', 0, 'o', 0, 'n', 0, 'e', 0, ' ', 0, 'h', 0, 'u', 0, 'n', 0, 'd', 0, 'r', 0, 'e', 0,
'd', 0, ' ', 0, 't', 0, 'w', 0, 'e', 0, 'n', 0, 't', 0, 'y', 0, '-', 0, 's', 0, 'e', 0, 'v', 0, 'e', 0, 'n', 0, ' ', 0, 'A', 0,
'S', 0, 'C', 0, 'I', 0, 'I', 0, ' ', 0, 'c', 0, 'h', 0, 'a', 0, 'r', 0, 'a', 0, 'c', 0, 't', 0, 'e', 0, 'r', 0, 's', 0,
'¡',
0,
}, mem.sliceAsBytes(utf16));
}
}
std.unicode.utf8ToUtf16LeArrayList
函数
pub fn utf8ToUtf16LeArrayList(result: *std.array_list.Managed(u16), utf8: []const u8) error{ InvalidUtf8, OutOfMemory }!void
参数
result: *std.array_list.Managed(u16)
utf8: []const u8
错误
- InvalidUtf8
- OutOfMemory
用法示例
test utf8ToUtf16LeArrayList {
{
var list = std.array_list.Managed(u16).init(testing.allocator);
defer list.deinit();
try utf8ToUtf16LeArrayList(&list, "𐐷");
try testing.expectEqualSlices(u8, "\x01\xd8\x37\xdc", mem.sliceAsBytes(list.items));
}
{
var list = std.array_list.Managed(u16).init(testing.allocator);
defer list.deinit();
try utf8ToUtf16LeArrayList(&list, "\u{10FFFF}");
try testing.expectEqualSlices(u8, "\xff\xdb\xff\xdf", mem.sliceAsBytes(list.items));
}
{
var list = std.array_list.Managed(u16).init(testing.allocator);
defer list.deinit();
const result = utf8ToUtf16LeArrayList(&list, "\xf4\x90\x80\x80");
try testing.expectError(error.InvalidUtf8, result);
}
}
std.unicode.utf8ToUtf16LeArrayListImpl
函数
fn utf8ToUtf16LeArrayListImpl(result: *std.array_list.Managed(u16), utf8: []const u8, comptime surrogates: Surrogates) !void
参数
std.unicode.utf8ToUtf16LeImpl
函数
pub fn utf8ToUtf16LeImpl(utf16le: []u16, utf8: []const u8, comptime surrogates: Surrogates) !usize
参数
std.unicode.utf8ToUtf16LeStringLiteral
函数
将 UTF-8 字符串字面量转换为 UTF-16LE 字符串字面量。
pub fn utf8ToUtf16LeStringLiteral(comptime utf8: []const u8) *const [calcUtf16LeLen(utf8) catch |err| @compileError(err):0]u16
参数
utf8: []const u8
用法示例
test utf8ToUtf16LeStringLiteral {
try testUtf8ToUtf16LeStringLiteral(utf8ToUtf16LeStringLiteral);
}
std.unicode.utf8ToUtf16LeStringLiteralImpl
函数
fn utf8ToUtf16LeStringLiteralImpl(comptime utf8: []const u8, comptime surrogates: Surrogates) *const [calcUtf16LeLenImpl(utf8, surrogates) catch |err| @compileError(err):0]u16
参数
utf8: []const u8
surrogates: Surrogates
std.unicode.utf8ValidateSlice
函数
若输入完全由 UTF-8 码点组成,则返回 true
pub fn utf8ValidateSlice(input: []const u8) bool
参数
input: []const u8
std.unicode.utf8ValidateSliceImpl
函数
fn utf8ValidateSliceImpl(input: []const u8, comptime surrogates: Surrogates) bool
参数
input: []const u8
surrogates: Surrogates
std.unicode.utf8ValidCodepoint
函数
若给定 Unicode 码点可编码为 UTF-8,则返回 true。
pub fn utf8ValidCodepoint(value: u21) bool
参数
value: u21
std.unicode.Utf8View
容器
Utf8View 遍历 UTF-8 编码字符串的码点。
var utf8 = (try std.unicode.Utf8View.init("hi there")).iterator();
while (utf8.nextCodepointSlice()) |codepoint| {
std.debug.print("got codepoint {s}\n", .{codepoint});
}
字段
bytes: []const u8
std.unicode.Utf8View.init
函数
pub fn init(s: []const u8) !Utf8View
参数
s: []const u8
std.unicode.Utf8View.initComptime
函数
pub inline fn initComptime(comptime s: []const u8) Utf8View
参数
s: []const u8
std.unicode.Utf8View.initUnchecked
函数
pub fn initUnchecked(s: []const u8) Utf8View
参数
s: []const u8
std.unicode.Utf8View.iterator
函数
pub fn iterator(s: Utf8View) Utf8Iterator
参数
s: Utf8View
std.unicode.Wtf16LeIterator
容器
字段
bytes: []const u8
i: usize
std.unicode.Wtf16LeIterator.init
函数
pub fn init(s: []const u16) Wtf16LeIterator
参数
s: []const u16
std.unicode.Wtf16LeIterator.nextCodepoint
函数
若下一个码点由代理对编码, 则返回代理对所表示的码点。 若下一个码点是未配对代理码点, 则返回该未配对代理码点本身。
pub fn nextCodepoint(it: *Wtf16LeIterator) ?u21
参数
it: *Wtf16LeIterator
std.unicode.wtf16LeToWtf8
函数
pub fn wtf16LeToWtf8(wtf8: []u8, wtf16le: []const u16) usize
参数
wtf8: []u8
wtf16le: []const u16
std.unicode.wtf16LeToWtf8Alloc
函数
调用者必须释放返回的内存。
pub fn wtf16LeToWtf8Alloc(allocator: Allocator, wtf16le: []const u16) Allocator.Error![]u8
参数
allocator: Allocator
wtf16le: []const u16
错误
- OutOfMemory
std.unicode.wtf16LeToWtf8AllocZ
函数
调用者必须释放返回的内存。
pub fn wtf16LeToWtf8AllocZ(allocator: Allocator, wtf16le: []const u16) Allocator.Error![:0]u8
参数
allocator: Allocator
wtf16le: []const u16
错误
- OutOfMemory
std.unicode.wtf16LeToWtf8ArrayList
函数
pub fn wtf16LeToWtf8ArrayList(result: *std.array_list.Managed(u8), utf16le: []const u16) Allocator.Error!void
参数
result: *std.array_list.Managed(u8)
utf16le: []const u16
错误
- OutOfMemory
std.unicode.wtf8Decode
函数
已弃用。此函数的 API 不够直观,容易误用。
pub fn wtf8Decode(bytes: []const u8) Wtf8DecodeError!u21
参数
bytes: []const u8
错误
- Utf8CodepointTooLarge Utf8Decode4Error
- Utf8ExpectedContinuation Utf8Decode2Error
- Utf8OverlongEncoding Utf8Decode2Error
std.unicode.Wtf8DecodeError
错误集合
错误
- Utf8CodepointTooLarge Utf8Decode4Error
- Utf8ExpectedContinuation Utf8Decode2Error
- Utf8OverlongEncoding Utf8Decode2Error
std.unicode.wtf8Encode
函数
将给定码点编码为 WTF-8 字节序列。 c:码点。 out:写入的输出缓冲区,长度必须 >= utf8CodepointSequenceLength(c)。 错误:c 无法编码为 WTF-8 时。 返回:写入 out 的字节数。
pub fn wtf8Encode(c: u21, out: []u8) error{CodepointTooLarge}!u3
参数
c: u21
out: []u8
错误
- CodepointTooLarge
std.unicode.Wtf8Iterator
容器
断言 bytes 是有效的 WTF-8
字段
bytes: []const u8
i: usize
std.unicode.Wtf8Iterator.nextCodepoint
函数
pub fn nextCodepoint(it: *Wtf8Iterator) ?u21
参数
it: *Wtf8Iterator
std.unicode.Wtf8Iterator.nextCodepointSlice
函数
pub fn nextCodepointSlice(it: *Wtf8Iterator) ?[]const u8
参数
it: *Wtf8Iterator
std.unicode.Wtf8Iterator.peek
函数
向前查看接下来的 n 个码点,而不推进迭代器。 如果可用码点不足 n 个,则返回字符串的剩余部分。
pub fn peek(it: *Wtf8Iterator, n: usize) []const u8
参数
it: *Wtf8Iterator
n: usize
std.unicode.Wtf8Iterator.peekCodepoint
函数
向前查看下一个码点,而不推进迭代器。 若不存在码点,则返回 null。
pub fn peekCodepoint(it: *Wtf8Iterator) ?u21
参数
it: *Wtf8Iterator
std.unicode.wtf8ToUtf8Lossy
函数
将代理码点(U+D800 至 U+DFFF)替换为 Unicode 替换字符(U+FFFD)。
所有代理码点与替换字符均编码为三个字节,因此输入与输出切片的长度始终相同。
当 utf8 和 wtf8 指向同一切片时,支持原地转换。
注意:若 wtf8 完全由结构良好的 UTF-8 组成,则无需转换。
可使用 utf8ValidateSlice 检查是否值得执行有损转换。
若 wtf8 不是有效 WTF-8,则返回 error.InvalidWtf8。
pub fn wtf8ToUtf8Lossy(utf8: []u8, wtf8: []const u8) error{InvalidWtf8}!void
参数
utf8: []u8
wtf8: []const u8
错误
- InvalidWtf8
用法示例
test wtf8ToUtf8Lossy {
var buf: [32]u8 = undefined;
const invalid_utf8 = "\xff";
try testing.expectError(error.InvalidWtf8, wtf8ToUtf8Lossy(&buf, invalid_utf8));
const ascii = "abcd";
try wtf8ToUtf8Lossy(&buf, ascii);
try testing.expectEqualStrings("abcd", buf[0..ascii.len]);
const high_surrogate_half = "ab\xed\xa0\xbdcd";
try wtf8ToUtf8Lossy(&buf, high_surrogate_half);
try testing.expectEqualStrings("ab\u{FFFD}cd", buf[0..high_surrogate_half.len]);
const low_surrogate_half = "ab\xed\xb2\xa9cd";
try wtf8ToUtf8Lossy(&buf, low_surrogate_half);
try testing.expectEqualStrings("ab\u{FFFD}cd", buf[0..low_surrogate_half.len]);
// 若 WTF-8 格式不良,则每个代理码元分别转换为一个
// 替换字符,而不是解释为代理项对。
const encoded_surrogate_pair = "ab\xed\xa0\xbd\xed\xb2\xa9cd";
try wtf8ToUtf8Lossy(&buf, encoded_surrogate_pair);
try testing.expectEqualStrings("ab\u{FFFD}\u{FFFD}cd", buf[0..encoded_surrogate_pair.len]);
// 原地
@memcpy(buf[0..low_surrogate_half.len], low_surrogate_half);
const slice = buf[0..low_surrogate_half.len];
try wtf8ToUtf8Lossy(slice, slice);
try testing.expectEqualStrings("ab\u{FFFD}cd", slice);
}
std.unicode.wtf8ToUtf8LossyAlloc
函数
pub fn wtf8ToUtf8LossyAlloc(allocator: Allocator, wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }![]u8
参数
allocator: Allocator
wtf8: []const u8
错误
- InvalidWtf8
- OutOfMemory
用法示例
test wtf8ToUtf8LossyAlloc {
const invalid_utf8 = "\xff";
try testing.expectError(error.InvalidWtf8, wtf8ToUtf8LossyAlloc(testing.allocator, invalid_utf8));
{
const ascii = "abcd";
const utf8 = try wtf8ToUtf8LossyAlloc(testing.allocator, ascii);
defer testing.allocator.free(utf8);
try testing.expectEqualStrings("abcd", utf8);
}
{
const surrogate_half = "ab\xed\xa0\xbdcd";
const utf8 = try wtf8ToUtf8LossyAlloc(testing.allocator, surrogate_half);
defer testing.allocator.free(utf8);
try testing.expectEqualStrings("ab\u{FFFD}cd", utf8);
}
{
// 若 WTF-8 格式不良,则每个代理码元分别转换为一个
// 替换字符,而不是解释为代理项对。
const encoded_surrogate_pair = "ab\xed\xa0\xbd\xed\xb2\xa9cd";
const utf8 = try wtf8ToUtf8LossyAlloc(testing.allocator, encoded_surrogate_pair);
defer testing.allocator.free(utf8);
try testing.expectEqualStrings("ab\u{FFFD}\u{FFFD}cd", utf8);
}
}
std.unicode.wtf8ToUtf8LossyAllocZ
函数
pub fn wtf8ToUtf8LossyAllocZ(allocator: Allocator, wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }![:0]u8
参数
allocator: Allocator
wtf8: []const u8
错误
- InvalidWtf8
- OutOfMemory
用法示例
test wtf8ToUtf8LossyAllocZ {
const invalid_utf8 = "\xff";
try testing.expectError(error.InvalidWtf8, wtf8ToUtf8LossyAllocZ(testing.allocator, invalid_utf8));
{
const ascii = "abcd";
const utf8 = try wtf8ToUtf8LossyAllocZ(testing.allocator, ascii);
defer testing.allocator.free(utf8);
try testing.expectEqualStrings("abcd", utf8);
}
{
const surrogate_half = "ab\xed\xa0\xbdcd";
const utf8 = try wtf8ToUtf8LossyAllocZ(testing.allocator, surrogate_half);
defer testing.allocator.free(utf8);
try testing.expectEqualStrings("ab\u{FFFD}cd", utf8);
}
{
// 若 WTF-8 格式不良,则每个代理码元分别转换为一个
// 替换字符,而不是解释为代理项对。
const encoded_surrogate_pair = "ab\xed\xa0\xbd\xed\xb2\xa9cd";
const utf8 = try wtf8ToUtf8LossyAllocZ(testing.allocator, encoded_surrogate_pair);
defer testing.allocator.free(utf8);
try testing.expectEqualStrings("ab\u{FFFD}\u{FFFD}cd", utf8);
}
}
std.unicode.wtf8ToWtf16Le
函数
返回下一个字符的索引。若恰好填满,返回索引等于输出切片长度。 假定输出空间足够。
pub fn wtf8ToWtf16Le(wtf16le: []u16, wtf8: []const u8) error{InvalidWtf8}!usize
参数
wtf16le: []u16
wtf8: []const u8
错误
- InvalidWtf8
std.unicode.wtf8ToWtf16LeAlloc
函数
pub fn wtf8ToWtf16LeAlloc(allocator: Allocator, wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }![]u16
参数
allocator: Allocator
wtf8: []const u8
错误
- InvalidWtf8
- OutOfMemory
std.unicode.wtf8ToWtf16LeAllocZ
函数
pub fn wtf8ToWtf16LeAllocZ(allocator: Allocator, wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }![:0]u16
参数
allocator: Allocator
wtf8: []const u8
错误
- InvalidWtf8
- OutOfMemory
std.unicode.wtf8ToWtf16LeArrayList
函数
pub fn wtf8ToWtf16LeArrayList(result: *std.array_list.Managed(u16), wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }!void
参数
result: *std.array_list.Managed(u16)
wtf8: []const u8
错误
- InvalidWtf8
- OutOfMemory
std.unicode.wtf8ToWtf16LeStringLiteral
函数
将 WTF-8 字符串字面量转换为 WTF-16LE 字符串字面量。
pub fn wtf8ToWtf16LeStringLiteral(comptime wtf8: []const u8) *const [calcWtf16LeLen(wtf8) catch |err| @compileError(err):0]u16
参数
wtf8: []const u8
用法示例
test wtf8ToWtf16LeStringLiteral {
try testUtf8ToUtf16LeStringLiteral(wtf8ToWtf16LeStringLiteral);
}
std.unicode.wtf8ValidateSlice
函数
如果输入完全由 WTF-8 码点组成,则返回 true (与 UTF-8 限制相同,但允许代理码点 U+D800 到 U+DFFF)。 不检查 WTF-8 是否格式良好,也就是说, 此函数不检查所有代理码元是否均未配对。
pub fn wtf8ValidateSlice(input: []const u8) bool
参数
input: []const u8
std.unicode.Wtf8View
容器
Wtf8View 遍历 WTF-8 编码字符串中的码点,包括未配对代理项。
var wtf8 = (try std.unicode.Wtf8View.init("hi there")).iterator();
while (wtf8.nextCodepointSlice()) |codepoint| {
// 注意:码点可能是代理项,不属于有效 UTF-8;
// 避免直接打印、发送或输出它
}
字段
bytes: []const u8
std.unicode.Wtf8View.init
函数
pub fn init(s: []const u8) error{InvalidWtf8}!Wtf8View
参数
s: []const u8
错误
- InvalidWtf8
std.unicode.Wtf8View.initComptime
函数
pub inline fn initComptime(comptime s: []const u8) Wtf8View
参数
s: []const u8
std.unicode.Wtf8View.initUnchecked
函数
pub fn initUnchecked(s: []const u8) Wtf8View
参数
s: []const u8
std.unicode.Wtf8View.iterator
函数
pub fn iterator(s: Wtf8View) Wtf8Iterator
参数
s: Wtf8View
源码
std/unicode.zig
const std = @import("./std.zig");
const builtin = @import("builtin");
const assert = std.debug.assert;
const testing = std.testing;
const mem = std.mem;
const native_endian = builtin.cpu.arch.endian();
const Allocator = std.mem.Allocator;
/// 用此字符替换未知、无法识别或无法表示的字符。
///
/// 另见:https://en.wikipedia.org/wiki/Specials_(Unicode_block)#Replacement_character
pub const replacement_character: u21 = 0xFFFD;
pub const replacement_character_utf8: [3]u8 = utf8EncodeComptime(replacement_character);
/// 返回给定码点的 UTF-8 表示
/// 所需的字节数。
pub fn utf8CodepointSequenceLength(c: u21) !u3 {
if (c < 0x80) return @as(u3, 1);
if (c < 0x800) return @as(u3, 2);
if (c < 0x10000) return @as(u3, 3);
if (c < 0x110000) return @as(u3, 4);
return error.CodepointTooLarge;
}
/// 给定 UTF-8 码点的第一个字节,
/// 返回 1 至 4 之间的数字,表示该码点的总字节长度。
/// 如果此字节不符合 UTF-8 起始字节的格式,则返回 Utf8InvalidStartByte。
pub fn utf8ByteSequenceLength(first_byte: u8) !u3 {
// switch 的优化效果远优于使用 @clz 的“巧妙”方案
return switch (first_byte) {
0b0000_0000...0b0111_1111 => 1,
0b1100_0000...0b1101_1111 => 2,
0b1110_0000...0b1110_1111 => 3,
0b1111_0000...0b1111_0111 => 4,
else => error.Utf8InvalidStartByte,
};
}
/// 将给定码点编码为 UTF-8 字节序列。
/// c:码点。
/// out:用于写入的输出缓冲区。长度必须 >= utf8CodepointSequenceLength(c)。
/// 错误:c 无法编码为 UTF-8 时返回错误。
/// 返回值:写入 out 的字节数。
pub fn utf8Encode(c: u21, out: []u8) error{ Utf8CannotEncodeSurrogateHalf, CodepointTooLarge }!u3 {
return utf8EncodeImpl(c, out, .cannot_encode_surrogate_half);
}
const Surrogates = enum {
cannot_encode_surrogate_half,
can_encode_surrogate_half,
};
fn utf8EncodeImpl(c: u21, out: []u8, comptime surrogates: Surrogates) !u3 {
const length = try utf8CodepointSequenceLength(c);
assert(out.len >= length);
switch (length) {
// 每个模式均相同:
// - 每次将初始移位增加 6
// - 首次之后,每次将移位后的值
// 缩短为最多 0b111111(63)
1 => out[0] = @as(u8, @intCast(c)), // 初始范围可直接计算 0 + codepoint
2 => {
out[0] = @as(u8, @intCast(0b11000000 | (c >> 6)));
out[1] = @as(u8, @intCast(0b10000000 | (c & 0b111111)));
},
3 => {
if (surrogates == .cannot_encode_surrogate_half and isSurrogateCodepoint(c)) {
return error.Utf8CannotEncodeSurrogateHalf;
}
out[0] = @as(u8, @intCast(0b11100000 | (c >> 12)));
out[1] = @as(u8, @intCast(0b10000000 | ((c >> 6) & 0b111111)));
out[2] = @as(u8, @intCast(0b10000000 | (c & 0b111111)));
},
4 => {
out[0] = @as(u8, @intCast(0b11110000 | (c >> 18)));
out[1] = @as(u8, @intCast(0b10000000 | ((c >> 12) & 0b111111)));
out[2] = @as(u8, @intCast(0b10000000 | ((c >> 6) & 0b111111)));
out[3] = @as(u8, @intCast(0b10000000 | (c & 0b111111)));
},
else => unreachable,
}
return length;
}
pub inline fn utf8EncodeComptime(comptime c: u21) [
utf8CodepointSequenceLength(c) catch |err|
@compileError(@errorName(err))
]u8 {
comptime var result: [
utf8CodepointSequenceLength(c) catch
unreachable
]u8 = undefined;
comptime assert((utf8Encode(c, &result) catch |err|
@compileError(@errorName(err))) == result.len);
return result;
}
const Utf8DecodeError = Utf8Decode2Error || Utf8Decode3Error || Utf8Decode4Error;
/// 已弃用。此函数的 API 不够直观,容易误用。
pub fn utf8Decode(bytes: []const u8) Utf8DecodeError!u21 {
return switch (bytes.len) {
1 => bytes[0],
2 => utf8Decode2(bytes[0..2].*),
3 => utf8Decode3(bytes[0..3].*),
4 => utf8Decode4(bytes[0..4].*),
else => unreachable,
};
}
const Utf8Decode2Error = error{
Utf8ExpectedContinuation,
Utf8OverlongEncoding,
};
pub fn utf8Decode2(bytes: [2]u8) Utf8Decode2Error!u21 {
assert(bytes[0] & 0b11100000 == 0b11000000);
var value: u21 = bytes[0] & 0b00011111;
if (bytes[1] & 0b11000000 != 0b10000000) return error.Utf8ExpectedContinuation;
value <<= 6;
value |= bytes[1] & 0b00111111;
if (value < 0x80) return error.Utf8OverlongEncoding;
return value;
}
const Utf8Decode3Error = Utf8Decode3AllowSurrogateHalfError || error{
Utf8EncodesSurrogateHalf,
};
pub fn utf8Decode3(bytes: [3]u8) Utf8Decode3Error!u21 {
const value = try utf8Decode3AllowSurrogateHalf(bytes);
if (0xd800 <= value and value <= 0xdfff) return error.Utf8EncodesSurrogateHalf;
return value;
}
const Utf8Decode3AllowSurrogateHalfError = error{
Utf8ExpectedContinuation,
Utf8OverlongEncoding,
};
pub fn utf8Decode3AllowSurrogateHalf(bytes: [3]u8) Utf8Decode3AllowSurrogateHalfError!u21 {
assert(bytes[0] & 0b11110000 == 0b11100000);
var value: u21 = bytes[0] & 0b00001111;
if (bytes[1] & 0b11000000 != 0b10000000) return error.Utf8ExpectedContinuation;
value <<= 6;
value |= bytes[1] & 0b00111111;
if (bytes[2] & 0b11000000 != 0b10000000) return error.Utf8ExpectedContinuation;
value <<= 6;
value |= bytes[2] & 0b00111111;
if (value < 0x800) return error.Utf8OverlongEncoding;
return value;
}
const Utf8Decode4Error = error{
Utf8ExpectedContinuation,
Utf8OverlongEncoding,
Utf8CodepointTooLarge,
};
pub fn utf8Decode4(bytes: [4]u8) Utf8Decode4Error!u21 {
assert(bytes[0] & 0b11111000 == 0b11110000);
var value: u21 = bytes[0] & 0b00000111;
if (bytes[1] & 0b11000000 != 0b10000000) return error.Utf8ExpectedContinuation;
value <<= 6;
value |= bytes[1] & 0b00111111;
if (bytes[2] & 0b11000000 != 0b10000000) return error.Utf8ExpectedContinuation;
value <<= 6;
value |= bytes[2] & 0b00111111;
if (bytes[3] & 0b11000000 != 0b10000000) return error.Utf8ExpectedContinuation;
value <<= 6;
value |= bytes[3] & 0b00111111;
if (value < 0x10000) return error.Utf8OverlongEncoding;
if (value > 0x10FFFF) return error.Utf8CodepointTooLarge;
return value;
}
/// 若给定 Unicode 码点可编码为 UTF-8,则返回 true。
pub fn utf8ValidCodepoint(value: u21) bool {
return switch (value) {
0xD800...0xDFFF => false, // 代理项范围
0x110000...0x1FFFFF => false, // 超过最大码点值
else => true,
};
}
/// 返回给定 UTF-8 字符串字面量所含的 Unicode
/// 码点数量。
pub fn utf8CountCodepoints(s: []const u8) !usize {
var len: usize = 0;
const N = @sizeOf(usize);
const MASK = 0x80 * (std.math.maxInt(usize) / 0xff);
var i: usize = 0;
while (i < s.len) {
// ASCII 序列的快速路径
while (i + N <= s.len) : (i += N) {
const v = mem.readInt(usize, s[i..][0..N], native_endian);
if (v & MASK != 0) break;
len += N;
}
if (i < s.len) {
const n = try utf8ByteSequenceLength(s[i]);
if (i + n > s.len) return error.TruncatedInput;
switch (n) {
1 => {}, // ASCII,无须验证
else => _ = try utf8Decode(s[i..][0..n]),
}
i += n;
len += 1;
}
}
return len;
}
/// 若输入完全由 UTF-8 码点组成,则返回 true
pub fn utf8ValidateSlice(input: []const u8) bool {
return utf8ValidateSliceImpl(input, .cannot_encode_surrogate_half);
}
fn utf8ValidateSliceImpl(input: []const u8, comptime surrogates: Surrogates) bool {
var remaining = input;
if (std.simd.suggestVectorLength(u8)) |chunk_len| {
const Chunk = @Vector(chunk_len, u8);
// 快速路径。检查并跳过输入开头的 ASCII 字符。
while (remaining.len >= chunk_len) {
const chunk: Chunk = remaining[0..chunk_len].*;
const mask: Chunk = @splat(0x80);
if (@reduce(.Or, chunk & mask == mask)) {
// 发现非 ASCII 字节
break;
}
remaining = remaining[chunk_len..];
}
}
// 默认的最低和最高续接字节
const lo_cb = 0b10000000;
const hi_cb = 0b10111111;
const min_non_ascii_codepoint = 0x80;
// 首个半字节用于确定可接受的续接字节范围。
// 第二个半字节表示大小。
const xx = 0xF1; // 无效:大小为 1
const as = 0xF0; // ASCII:大小为 1
const s1 = 0x02; // 接受 0,大小为 2
const s2 = switch (surrogates) {
.cannot_encode_surrogate_half => 0x13, // 接受 1,大小 3
.can_encode_surrogate_half => 0x03, // 接受 0,大小为 3
};
const s3 = 0x03; // 接受 0,大小为 3
const s4 = switch (surrogates) {
.cannot_encode_surrogate_half => 0x23, // 接受 2,大小为 3
.can_encode_surrogate_half => 0x03, // 接受 0,大小为 3
};
const s5 = 0x34; // 接受 3,大小 4
const s6 = 0x04; // 接受 0,大小为 4
const s7 = 0x44; // 接受 4,大小为 4
// 关于 UTF-8 序列第一个字节的信息。
const first = comptime first: {
const a: [128]u8 = @splat(as);
const b: [64]u8 = @splat(xx);
const c: [64]u8 = .{
xx, xx, s1, s1, s1, s1, s1, s1, s1, s1, s1, s1, s1, s1, s1, s1,
s1, s1, s1, s1, s1, s1, s1, s1, s1, s1, s1, s1, s1, s1, s1, s1,
s2, s3, s3, s3, s3, s3, s3, s3, s3, s3, s3, s3, s3, s4, s3, s3,
s5, s6, s6, s6, s7, xx, xx, xx, xx, xx, xx, xx, xx, xx, xx, xx,
};
break :first a ++ b ++ c;
};
const n = remaining.len;
var i: usize = 0;
while (i < n) {
const first_byte = remaining[i];
if (first_byte < min_non_ascii_codepoint) {
i += 1;
continue;
}
const info = first[first_byte];
if (info == xx) {
return false; // 非法的起始字节。
}
const size = info & 7;
if (i + size > n) {
return false; // 长度不足或无效。
}
// 从默认值开始,确定可接受的后续字节
// 上下界。
var accept_lo: u8 = lo_cb;
var accept_hi: u8 = hi_cb;
switch (info >> 4) {
0 => {},
1 => accept_lo = 0xA0,
2 => accept_hi = 0x9F,
3 => accept_lo = 0x90,
4 => accept_hi = 0x8F,
else => unreachable,
}
const c1 = remaining[i + 1];
if (c1 < accept_lo or accept_hi < c1) {
return false;
}
switch (size) {
2 => i += 2,
3 => {
const c2 = remaining[i + 2];
if (c2 < lo_cb or hi_cb < c2) {
return false;
}
i += 3;
},
4 => {
const c2 = remaining[i + 2];
if (c2 < lo_cb or hi_cb < c2) {
return false;
}
const c3 = remaining[i + 3];
if (c3 < lo_cb or hi_cb < c3) {
return false;
}
i += 4;
},
else => unreachable,
}
}
return true;
}
/// Utf8View 遍历 UTF-8 编码字符串的码点。
///
/// ```
/// var utf8 = (try std.unicode.Utf8View.init("hi there")).iterator();
/// while (utf8.nextCodepointSlice()) |codepoint| {
/// std.debug.print("got codepoint {s}\n", .{codepoint});
/// }
/// ```
pub const Utf8View = struct {
bytes: []const u8,
pub fn init(s: []const u8) !Utf8View {
if (!utf8ValidateSlice(s)) {
return error.InvalidUtf8;
}
return initUnchecked(s);
}
pub fn initUnchecked(s: []const u8) Utf8View {
return Utf8View{ .bytes = s };
}
pub inline fn initComptime(comptime s: []const u8) Utf8View {
return comptime if (init(s)) |r| r else |err| switch (err) {
error.InvalidUtf8 => {
@compileError("invalid utf8");
},
};
}
pub fn iterator(s: Utf8View) Utf8Iterator {
return Utf8Iterator{
.bytes = s.bytes,
.i = 0,
};
}
};
pub const Utf8Iterator = struct {
bytes: []const u8,
i: usize,
pub fn nextCodepointSlice(it: *Utf8Iterator) ?[]const u8 {
if (it.i >= it.bytes.len) {
return null;
}
const cp_len = utf8ByteSequenceLength(it.bytes[it.i]) catch unreachable;
it.i += cp_len;
return it.bytes[it.i - cp_len .. it.i];
}
pub fn nextCodepoint(it: *Utf8Iterator) ?u21 {
const slice = it.nextCodepointSlice() orelse return null;
return utf8Decode(slice) catch unreachable;
}
/// 向前查看接下来的 n 个码点,而不推进迭代器。
/// 如果可用码点不足 n 个,则返回字符串的剩余部分。
pub fn peek(it: *Utf8Iterator, n: usize) []const u8 {
const original_i = it.i;
defer it.i = original_i;
var end_ix = original_i;
var found: usize = 0;
while (found < n) : (found += 1) {
const next_codepoint = it.nextCodepointSlice() orelse return it.bytes[original_i..];
end_ix += next_codepoint.len;
}
return it.bytes[original_i..end_ix];
}
/// 向前查看下一个码点,而不推进迭代器。
/// 若不存在码点,则返回 null。
pub fn peekCodepoint(it: *Utf8Iterator) ?u21 {
const original_i = it.i;
defer it.i = original_i;
return it.nextCodepoint();
}
};
pub fn utf16IsHighSurrogate(c: u16) bool {
return c & ~@as(u16, 0x03ff) == 0xd800;
}
pub fn utf16IsLowSurrogate(c: u16) bool {
return c & ~@as(u16, 0x03ff) == 0xdc00;
}
/// 返回给定码点的 UTF-16 表示所需的
/// 代码单元数量。
pub fn utf16CodepointSequenceLength(c: u21) !u2 {
if (c <= 0xFFFF) return 1;
if (c <= 0x10FFFF) return 2;
return error.CodepointTooLarge;
}
test utf16CodepointSequenceLength {
try testing.expectEqual(@as(u2, 1), try utf16CodepointSequenceLength('a'));
try testing.expectEqual(@as(u2, 1), try utf16CodepointSequenceLength(0xFFFF));
try testing.expectEqual(@as(u2, 2), try utf16CodepointSequenceLength(0x10000));
try testing.expectEqual(@as(u2, 2), try utf16CodepointSequenceLength(0x10FFFF));
try testing.expectError(error.CodepointTooLarge, utf16CodepointSequenceLength(0x110000));
}
/// 给定 UTF-16 码点的第一个码元,返回 1 至 2 之间的数字,
/// 表示该码点包含的 UTF-16 码元总数。
/// 如果此码元不符合 UTF-16 起始码元的格式,则返回 Utf16InvalidStartCodeUnit。
pub fn utf16CodeUnitSequenceLength(first_code_unit: u16) !u2 {
if (utf16IsHighSurrogate(first_code_unit)) return 2;
if (utf16IsLowSurrogate(first_code_unit)) return error.Utf16InvalidStartCodeUnit;
return 1;
}
test utf16CodeUnitSequenceLength {
try testing.expectEqual(@as(u2, 1), try utf16CodeUnitSequenceLength('a'));
try testing.expectEqual(@as(u2, 1), try utf16CodeUnitSequenceLength(0xFFFF));
try testing.expectEqual(@as(u2, 2), try utf16CodeUnitSequenceLength(0xDBFF));
try testing.expectError(error.Utf16InvalidStartCodeUnit, utf16CodeUnitSequenceLength(0xDFFF));
}
/// 解码给定的一对 UTF-16 码元所编码的码点。
/// 断言 `surrogate_pair.len >= 2`,且第一个码元为高位代理项。
/// 若第二个码元不是低位代理项,返回 error.ExpectedSecondSurrogateHalf。
pub fn utf16DecodeSurrogatePair(surrogate_pair: []const u16) !u21 {
assert(surrogate_pair.len >= 2);
assert(utf16IsHighSurrogate(surrogate_pair[0]));
const high_half: u21 = surrogate_pair[0];
const low_half = surrogate_pair[1];
if (!utf16IsLowSurrogate(low_half)) return error.ExpectedSecondSurrogateHalf;
return 0x10000 + ((high_half & 0x03ff) << 10) | (low_half & 0x03ff);
}
pub const Utf16LeIterator = struct {
bytes: []const u8,
i: usize,
pub fn init(s: []const u16) Utf16LeIterator {
return Utf16LeIterator{
.bytes = mem.sliceAsBytes(s),
.i = 0,
};
}
pub const NextCodepointError = error{ DanglingSurrogateHalf, ExpectedSecondSurrogateHalf, UnexpectedSecondSurrogateHalf };
pub fn nextCodepoint(it: *Utf16LeIterator) NextCodepointError!?u21 {
assert(it.i <= it.bytes.len);
if (it.i == it.bytes.len) return null;
var code_units: [2]u16 = undefined;
code_units[0] = mem.readInt(u16, it.bytes[it.i..][0..2], .little);
it.i += 2;
if (utf16IsHighSurrogate(code_units[0])) {
// 代理对
if (it.i >= it.bytes.len) return error.DanglingSurrogateHalf;
code_units[1] = mem.readInt(u16, it.bytes[it.i..][0..2], .little);
const codepoint = try utf16DecodeSurrogatePair(&code_units);
it.i += 2;
return codepoint;
} else if (utf16IsLowSurrogate(code_units[0])) {
return error.UnexpectedSecondSurrogateHalf;
} else {
return code_units[0];
}
}
};
/// 以 Unicode 码点数为单位,返回所提供 UTF-16 字符串字面量的长度。
///
pub fn utf16CountCodepoints(utf16le: []const u16) !usize {
var len: usize = 0;
var it = Utf16LeIterator.init(utf16le);
while (try it.nextCodepoint()) |_| len += 1;
return len;
}
fn testUtf16CountCodepoints() !void {
try testing.expectEqual(
@as(usize, 1),
try utf16CountCodepoints(utf8ToUtf16LeStringLiteral("a")),
);
try testing.expectEqual(
@as(usize, 10),
try utf16CountCodepoints(utf8ToUtf16LeStringLiteral("abcdefghij")),
);
try testing.expectEqual(
@as(usize, 10),
try utf16CountCodepoints(utf8ToUtf16LeStringLiteral("äåéëþüúíóö")),
);
try testing.expectEqual(
@as(usize, 5),
try utf16CountCodepoints(utf8ToUtf16LeStringLiteral("こんにちは")),
);
}
test "utf16 count codepoints" {
@setEvalBranchQuota(2000);
try testUtf16CountCodepoints();
try comptime testUtf16CountCodepoints();
}
test "utf8 encode" {
try comptime testUtf8Encode();
try testUtf8Encode();
}
fn testUtf8Encode() !void {
// 部分取自维基百科,部分取自其他来源
var array: [4]u8 = undefined;
try testing.expect((try utf8Encode(try utf8Decode("€"), array[0..])) == 3);
try testing.expect(array[0] == 0b11100010);
try testing.expect(array[1] == 0b10000010);
try testing.expect(array[2] == 0b10101100);
try testing.expect((try utf8Encode(try utf8Decode("$"), array[0..])) == 1);
try testing.expect(array[0] == 0b00100100);
try testing.expect((try utf8Encode(try utf8Decode("¢"), array[0..])) == 2);
try testing.expect(array[0] == 0b11000010);
try testing.expect(array[1] == 0b10100010);
try testing.expect((try utf8Encode(try utf8Decode("𐍈"), array[0..])) == 4);
try testing.expect(array[0] == 0b11110000);
try testing.expect(array[1] == 0b10010000);
try testing.expect(array[2] == 0b10001101);
try testing.expect(array[3] == 0b10001000);
}
test "utf8 encode comptime" {
try testing.expectEqualSlices(u8, "€", &utf8EncodeComptime('€'));
try testing.expectEqualSlices(u8, "$", &utf8EncodeComptime('$'));
try testing.expectEqualSlices(u8, "¢", &utf8EncodeComptime('¢'));
try testing.expectEqualSlices(u8, "𐍈", &utf8EncodeComptime('𐍈'));
}
test "utf8 encode error" {
try comptime testUtf8EncodeError();
try testUtf8EncodeError();
}
fn testUtf8EncodeError() !void {
var array: [4]u8 = undefined;
try testErrorEncode(0xd800, array[0..], error.Utf8CannotEncodeSurrogateHalf);
try testErrorEncode(0xdfff, array[0..], error.Utf8CannotEncodeSurrogateHalf);
try testErrorEncode(0x110000, array[0..], error.CodepointTooLarge);
try testErrorEncode(0x1fffff, array[0..], error.CodepointTooLarge);
}
fn testErrorEncode(codePoint: u21, array: []u8, expectedErr: anyerror) !void {
try testing.expectError(expectedErr, utf8Encode(codePoint, array));
}
test "utf8 iterator on ascii" {
try comptime testUtf8IteratorOnAscii();
try testUtf8IteratorOnAscii();
}
fn testUtf8IteratorOnAscii() !void {
const s = Utf8View.initComptime("abc");
var it1 = s.iterator();
try testing.expect(mem.eql(u8, "a", it1.nextCodepointSlice().?));
try testing.expect(mem.eql(u8, "b", it1.nextCodepointSlice().?));
try testing.expect(mem.eql(u8, "c", it1.nextCodepointSlice().?));
try testing.expect(it1.nextCodepointSlice() == null);
var it2 = s.iterator();
try testing.expect(it2.nextCodepoint().? == 'a');
try testing.expect(it2.nextCodepoint().? == 'b');
try testing.expect(it2.nextCodepoint().? == 'c');
try testing.expect(it2.nextCodepoint() == null);
}
test "utf8 view bad" {
try comptime testUtf8ViewBad();
try testUtf8ViewBad();
}
fn testUtf8ViewBad() !void {
// 编译错误。
// const s3 = Utf8View.initComptime("\xfe\xf2");
try testing.expectError(error.InvalidUtf8, Utf8View.init("hel\xadlo"));
}
test "utf8 view ok" {
try comptime testUtf8ViewOk();
try testUtf8ViewOk();
}
fn testUtf8ViewOk() !void {
const s = Utf8View.initComptime("東京市");
var it1 = s.iterator();
try testing.expect(mem.eql(u8, "東", it1.nextCodepointSlice().?));
try testing.expect(mem.eql(u8, "京", it1.nextCodepointSlice().?));
try testing.expect(mem.eql(u8, "市", it1.nextCodepointSlice().?));
try testing.expect(it1.nextCodepointSlice() == null);
var it2 = s.iterator();
try testing.expect(it2.nextCodepoint().? == 0x6771);
try testing.expect(it2.nextCodepoint().? == 0x4eac);
try testing.expect(it2.nextCodepoint().? == 0x5e02);
try testing.expect(it2.nextCodepoint() == null);
}
test "validate slice" {
try comptime testValidateSlice();
try testValidateSlice();
// 按可变大小的块(基于建议的向量大小)跳过
// ASCII 字符。确保分块方式正确。
const str = @as([550]u8, @splat('a')) ++ "\xc0";
for (0..str.len - 3) |i| {
try testing.expect(!utf8ValidateSlice(str[i..]));
}
}
fn testValidateSlice() !void {
try testing.expect(utf8ValidateSlice("abc"));
try testing.expect(utf8ValidateSlice("abc\xdf\xbf"));
try testing.expect(utf8ValidateSlice(""));
try testing.expect(utf8ValidateSlice("a"));
try testing.expect(utf8ValidateSlice("abc"));
try testing.expect(utf8ValidateSlice("Ж"));
try testing.expect(utf8ValidateSlice("ЖЖ"));
try testing.expect(utf8ValidateSlice("брэд-ЛГТМ"));
try testing.expect(utf8ValidateSlice("☺☻☹"));
try testing.expect(utf8ValidateSlice("a\u{fffdb}"));
try testing.expect(utf8ValidateSlice("\xf4\x8f\xbf\xbf"));
try testing.expect(utf8ValidateSlice("abc\xdf\xbf"));
try testing.expect(!utf8ValidateSlice("abc\xc0"));
try testing.expect(!utf8ValidateSlice("abc\xc0abc"));
try testing.expect(!utf8ValidateSlice("aa\xe2"));
try testing.expect(!utf8ValidateSlice("\x42\xfa"));
try testing.expect(!utf8ValidateSlice("\x42\xfa\x43"));
try testing.expect(!utf8ValidateSlice("abc\xc0"));
try testing.expect(!utf8ValidateSlice("abc\xc0abc"));
try testing.expect(!utf8ValidateSlice("\xf4\x90\x80\x80"));
try testing.expect(!utf8ValidateSlice("\xf7\xbf\xbf\xbf"));
try testing.expect(!utf8ValidateSlice("\xfb\xbf\xbf\xbf\xbf"));
try testing.expect(!utf8ValidateSlice("\xc0\x80"));
try testing.expect(!utf8ValidateSlice("\xed\xa0\x80"));
try testing.expect(!utf8ValidateSlice("\xed\xbf\xbf"));
}
test "valid utf8" {
try comptime testValidUtf8();
try testValidUtf8();
}
fn testValidUtf8() !void {
try testValid("\x00", 0x0);
try testValid("\x20", 0x20);
try testValid("\x7f", 0x7f);
try testValid("\xc2\x80", 0x80);
try testValid("\xdf\xbf", 0x7ff);
try testValid("\xe0\xa0\x80", 0x800);
try testValid("\xe1\x80\x80", 0x1000);
try testValid("\xef\xbf\xbf", 0xffff);
try testValid("\xf0\x90\x80\x80", 0x10000);
try testValid("\xf1\x80\x80\x80", 0x40000);
try testValid("\xf3\xbf\xbf\xbf", 0xfffff);
try testValid("\xf4\x8f\xbf\xbf", 0x10ffff);
}
test "invalid utf8 continuation bytes" {
try comptime testInvalidUtf8ContinuationBytes();
try testInvalidUtf8ContinuationBytes();
}
fn testInvalidUtf8ContinuationBytes() !void {
// 意外的续字节
try testError("\x80", error.Utf8InvalidStartByte);
try testError("\xbf", error.Utf8InvalidStartByte);
// 前导 1 过多
try testError("\xf8", error.Utf8InvalidStartByte);
try testError("\xff", error.Utf8InvalidStartByte);
// 双字节序列的预期后续字节
try testError("\xc2", error.UnexpectedEof);
try testError("\xc2\x00", error.Utf8ExpectedContinuation);
try testError("\xc2\xc0", error.Utf8ExpectedContinuation);
// 3 字节序列的预期续接字节
try testError("\xe0", error.UnexpectedEof);
try testError("\xe0\x00", error.UnexpectedEof);
try testError("\xe0\xc0", error.UnexpectedEof);
try testError("\xe0\xa0", error.UnexpectedEof);
try testError("\xe0\xa0\x00", error.Utf8ExpectedContinuation);
try testError("\xe0\xa0\xc0", error.Utf8ExpectedContinuation);
// 四字节序列的预期后续字节
try testError("\xf0", error.UnexpectedEof);
try testError("\xf0\x00", error.UnexpectedEof);
try testError("\xf0\xc0", error.UnexpectedEof);
try testError("\xf0\x90\x00", error.UnexpectedEof);
try testError("\xf0\x90\xc0", error.UnexpectedEof);
try testError("\xf0\x90\x80\x00", error.Utf8ExpectedContinuation);
try testError("\xf0\x90\x80\xc0", error.Utf8ExpectedContinuation);
}
test "overlong utf8 codepoint" {
try comptime testOverlongUtf8Codepoint();
try testOverlongUtf8Codepoint();
}
fn testOverlongUtf8Codepoint() !void {
try testError("\xc0\x80", error.Utf8OverlongEncoding);
try testError("\xc1\xbf", error.Utf8OverlongEncoding);
try testError("\xe0\x80\x80", error.Utf8OverlongEncoding);
try testError("\xe0\x9f\xbf", error.Utf8OverlongEncoding);
try testError("\xf0\x80\x80\x80", error.Utf8OverlongEncoding);
try testError("\xf0\x8f\xbf\xbf", error.Utf8OverlongEncoding);
}
test "misc invalid utf8" {
try comptime testMiscInvalidUtf8();
try testMiscInvalidUtf8();
}
fn testMiscInvalidUtf8() !void {
// 码点超出边界
try testError("\xf4\x90\x80\x80", error.Utf8CodepointTooLarge);
try testError("\xf7\xbf\xbf\xbf", error.Utf8CodepointTooLarge);
// 代理项的两半
try testValid("\xed\x9f\xbf", 0xd7ff);
try testError("\xed\xa0\x80", error.Utf8EncodesSurrogateHalf);
try testError("\xed\xbf\xbf", error.Utf8EncodesSurrogateHalf);
try testValid("\xee\x80\x80", 0xe000);
}
test "utf8 iterator peeking" {
try comptime testUtf8Peeking();
try testUtf8Peeking();
comptime try testUtf8PeekCodepoint();
try testUtf8PeekCodepoint();
}
fn testUtf8Peeking() !void {
const s = Utf8View.initComptime("noël");
var it = s.iterator();
try testing.expect(mem.eql(u8, "n", it.nextCodepointSlice().?));
try testing.expect(mem.eql(u8, "o", it.peek(1)));
try testing.expect(mem.eql(u8, "oë", it.peek(2)));
try testing.expect(mem.eql(u8, "oël", it.peek(3)));
try testing.expect(mem.eql(u8, "oël", it.peek(4)));
try testing.expect(mem.eql(u8, "oël", it.peek(10)));
try testing.expect(mem.eql(u8, "o", it.nextCodepointSlice().?));
try testing.expect(mem.eql(u8, "ë", it.nextCodepointSlice().?));
try testing.expect(mem.eql(u8, "l", it.nextCodepointSlice().?));
try testing.expect(it.nextCodepointSlice() == null);
try testing.expect(mem.eql(u8, &[_]u8{}, it.peek(1)));
}
fn testUtf8PeekCodepoint() !void {
const s = Utf8View.initComptime("東京市");
var it = s.iterator();
try testing.expect(it.peekCodepoint().? == 0x6771);
try testing.expect(it.peekCodepoint().? == 0x6771);
_ = it.nextCodepoint();
try testing.expect(it.peekCodepoint().? == 0x4eac);
_ = it.nextCodepoint();
try testing.expect(it.peekCodepoint().? == 0x5e02);
_ = it.nextCodepoint();
try testing.expect(it.peekCodepoint() == null);
}
fn testError(bytes: []const u8, expected_err: anyerror) !void {
try testing.expectError(expected_err, testDecode(bytes));
}
fn testValid(bytes: []const u8, expected_codepoint: u21) !void {
try testing.expect((testDecode(bytes) catch unreachable) == expected_codepoint);
}
fn testDecode(bytes: []const u8) !u21 {
const length = try utf8ByteSequenceLength(bytes[0]);
if (bytes.len < length) return error.UnexpectedEof;
try testing.expect(bytes.len == length);
return utf8Decode(bytes);
}
/// 打印给定 `utf8` 字符串,按 UTF-8 字节编码。
/// 根据 Unicode 标准第 3 章“用 U+FFFD 替换最大子部分”及 https://encoding.spec.whatwg.org/#utf-8-decoder 的规定,将结构不良的 UTF-8 字节序列替换为替换字符 U+FFFD。
///
///
fn formatUtf8(utf8: []const u8, writer: *std.Io.Writer) std.Io.Writer.Error!void {
var buf: [300]u8 = undefined; // 只是一个任意大小
var u8len: usize = 0;
// 此实现基于以下规范:
// https://encoding.spec.whatwg.org/#utf-8-decoder
var codepoint: u21 = 0;
var cont_bytes_seen: u3 = 0;
var cont_bytes_needed: u3 = 0;
var lower_boundary: u8 = 0x80;
var upper_boundary: u8 = 0xBF;
var i: usize = 0;
while (i < utf8.len) {
const byte = utf8[i];
if (cont_bytes_needed == 0) {
switch (byte) {
0x00...0x7F => {
buf[u8len] = byte;
u8len += 1;
},
0xC2...0xDF => {
cont_bytes_needed = 1;
codepoint = byte & 0b00011111;
},
0xE0...0xEF => {
if (byte == 0xE0) lower_boundary = 0xA0;
if (byte == 0xED) upper_boundary = 0x9F;
cont_bytes_needed = 2;
codepoint = byte & 0b00001111;
},
0xF0...0xF4 => {
if (byte == 0xF0) lower_boundary = 0x90;
if (byte == 0xF4) upper_boundary = 0x8F;
cont_bytes_needed = 3;
codepoint = byte & 0b00000111;
},
else => {
u8len += utf8Encode(replacement_character, buf[u8len..]) catch unreachable;
},
}
// 消耗该字节
i += 1;
} else if (byte < lower_boundary or byte > upper_boundary) {
codepoint = 0;
cont_bytes_needed = 0;
cont_bytes_seen = 0;
lower_boundary = 0x80;
upper_boundary = 0xBF;
u8len += utf8Encode(replacement_character, buf[u8len..]) catch unreachable;
// 不要消费当前字节;现在应将它视为可能的起始字节
} else {
lower_boundary = 0x80;
upper_boundary = 0xBF;
codepoint <<= 6;
codepoint |= byte & 0b00111111;
cont_bytes_seen += 1;
// 消耗该字节
i += 1;
if (cont_bytes_seen == cont_bytes_needed) {
const codepoint_len = cont_bytes_seen + 1;
const codepoint_start_i = i - codepoint_len;
@memcpy(buf[u8len..][0..codepoint_len], utf8[codepoint_start_i..][0..codepoint_len]);
u8len += codepoint_len;
codepoint = 0;
cont_bytes_needed = 0;
cont_bytes_seen = 0;
}
}
// 确保始终有足够空间容纳另一个最大长度的 UTF-8 码点
if (u8len + 4 > buf.len) {
try writer.writeAll(buf[0..u8len]);
u8len = 0;
}
}
if (cont_bytes_needed != 0) {
// 已知空间足够,因为缓冲区剩余空间
// 少于 4 字节时总会刷新。
u8len += utf8Encode(replacement_character, buf[u8len..]) catch unreachable;
}
try writer.writeAll(buf[0..u8len]);
}
/// 为可能结构不良的 UTF-8 字符串返回 Formatter。
/// 根据 Unicode 标准第 3 章“用 U+FFFD 替换最大子部分”及 https://encoding.spec.whatwg.org/#utf-8-decoder 的规定,将结构不良的 UTF-8 字节序列替换为替换字符 U+FFFD。
///
///
pub fn fmtUtf8(utf8: []const u8) std.fmt.Alt([]const u8, formatUtf8) {
return .{ .data = utf8 };
}
test fmtUtf8 {
const expectFmt = testing.expectFmt;
try expectFmt("", "{f}", .{fmtUtf8("")});
try expectFmt("foo", "{f}", .{fmtUtf8("foo")});
try expectFmt("𐐷", "{f}", .{fmtUtf8("𐐷")});
// 表 3-8:用于非最短形式序列的 U+FFFD。
try expectFmt("��������A", "{f}", .{fmtUtf8("\xC0\xAF\xE0\x80\xBF\xF0\x81\x82A")});
// 表 3-9:用于代理码点格式错误序列的 U+FFFD。
try expectFmt("��������A", "{f}", .{fmtUtf8("\xED\xA0\x80\xED\xBF\xBF\xED\xAFA")});
// 表 3-10。其他格式错误序列对应的 U+FFFD
try expectFmt("�����A��B", "{f}", .{fmtUtf8("\xF4\x91\x92\x93\xFFA\x80\xBFB")});
// 表 3-11。截断序列使用 U+FFFD
try expectFmt("����A", "{f}", .{fmtUtf8("\xE1\x80\xE2\xF0\x91\x92\xF1\xBFA")});
}
fn utf16LeToUtf8ArrayListImpl(
result: *std.array_list.Managed(u8),
utf16le: []const u16,
comptime surrogates: Surrogates,
) (switch (surrogates) {
.cannot_encode_surrogate_half => Utf16LeToUtf8AllocError,
.can_encode_surrogate_half => Allocator.Error,
})!void {
assert(result.unusedCapacitySlice().len >= utf16le.len);
var remaining = utf16le;
vectorized: {
const chunk_len = std.simd.suggestVectorLength(u16) orelse break :vectorized;
const Chunk = @Vector(chunk_len, u16);
// 快速路径:检查并编码输入开头的 ASCII 字符。
while (remaining.len >= chunk_len) {
const chunk: Chunk = remaining[0..chunk_len].*;
const mask: Chunk = @splat(mem.nativeToLittle(u16, 0x7F));
if (@reduce(.Or, chunk | mask != mask)) {
// 发现非 ASCII 码元
break;
}
const ascii_chunk: @Vector(chunk_len, u8) = @truncate(mem.nativeToLittle(Chunk, chunk));
// 已分配足够空间,将每个 UTF-16 码元编码为 ASCII,
// 因此若整个字符串都是 ASCII,
// 就保证分配的空间足够。
result.addManyAsArrayAssumeCapacity(chunk_len).* = ascii_chunk;
remaining = remaining[chunk_len..];
}
}
switch (surrogates) {
.cannot_encode_surrogate_half => {
var it = Utf16LeIterator.init(remaining);
while (try it.nextCodepoint()) |codepoint| {
const utf8_len = utf8CodepointSequenceLength(codepoint) catch unreachable;
assert((utf8Encode(codepoint, try result.addManyAsSlice(utf8_len)) catch unreachable) == utf8_len);
}
},
.can_encode_surrogate_half => {
var it = Wtf16LeIterator.init(remaining);
while (it.nextCodepoint()) |codepoint| {
const utf8_len = utf8CodepointSequenceLength(codepoint) catch unreachable;
assert((wtf8Encode(codepoint, try result.addManyAsSlice(utf8_len)) catch unreachable) == utf8_len);
}
},
}
}
pub const Utf16LeToUtf8AllocError = Allocator.Error || Utf16LeToUtf8Error;
pub fn utf16LeToUtf8ArrayList(result: *std.array_list.Managed(u8), utf16le: []const u16) Utf16LeToUtf8AllocError!void {
try result.ensureUnusedCapacity(utf16le.len);
return utf16LeToUtf8ArrayListImpl(result, utf16le, .cannot_encode_surrogate_half);
}
/// 调用方拥有返回内存的所有权。
pub fn utf16LeToUtf8Alloc(allocator: Allocator, utf16le: []const u16) Utf16LeToUtf8AllocError![]u8 {
// 乐观地假设它们全都是 ASCII 字符。
var result = try std.array_list.Managed(u8).initCapacity(allocator, utf16le.len);
errdefer result.deinit();
try utf16LeToUtf8ArrayListImpl(&result, utf16le, .cannot_encode_surrogate_half);
return result.toOwnedSlice();
}
/// 调用方拥有返回内存的所有权。
pub fn utf16LeToUtf8AllocZ(allocator: Allocator, utf16le: []const u16) Utf16LeToUtf8AllocError![:0]u8 {
// 乐观地假定全部为 ascii(并为空终止符分配空间)
var result = try std.array_list.Managed(u8).initCapacity(allocator, utf16le.len + 1);
errdefer result.deinit();
try utf16LeToUtf8ArrayListImpl(&result, utf16le, .cannot_encode_surrogate_half);
return result.toOwnedSliceSentinel(0);
}
pub const Utf16LeToUtf8Error = Utf16LeIterator.NextCodepointError;
/// 断言输出缓冲区足够大。
/// 返回 utf8 中的结束字节索引。
fn utf16LeToUtf8Impl(utf8: []u8, utf16le: []const u16, comptime surrogates: Surrogates) (switch (surrogates) {
.cannot_encode_surrogate_half => Utf16LeToUtf8Error,
.can_encode_surrogate_half => error{},
})!usize {
var dest_index: usize = 0;
var remaining = utf16le;
vectorized: {
const chunk_len = std.simd.suggestVectorLength(u16) orelse break :vectorized;
const Chunk = @Vector(chunk_len, u16);
// 快速路径:检查并编码输入开头的 ASCII 字符。
while (remaining.len >= chunk_len) {
const chunk: Chunk = remaining[0..chunk_len].*;
const mask: Chunk = @splat(mem.nativeToLittle(u16, 0x7F));
if (@reduce(.Or, chunk | mask != mask)) {
// 发现非 ASCII 码元
break;
}
const ascii_chunk: @Vector(chunk_len, u8) = @truncate(mem.nativeToLittle(Chunk, chunk));
utf8[dest_index..][0..chunk_len].* = ascii_chunk;
dest_index += chunk_len;
remaining = remaining[chunk_len..];
}
}
switch (surrogates) {
.cannot_encode_surrogate_half => {
var it = Utf16LeIterator.init(remaining);
while (try it.nextCodepoint()) |codepoint| {
dest_index += utf8Encode(codepoint, utf8[dest_index..]) catch |err| switch (err) {
// UTF-16 可编码的最大码点为 U+10FFFF,
// 位于有效码点范围内。
error.CodepointTooLarge => unreachable,
// 已知该码点在 UTF-16 中有效,
// 即它不是未配对的代理码点。
error.Utf8CannotEncodeSurrogateHalf => unreachable,
};
}
},
.can_encode_surrogate_half => {
var it = Wtf16LeIterator.init(remaining);
while (it.nextCodepoint()) |codepoint| {
dest_index += wtf8Encode(codepoint, utf8[dest_index..]) catch |err| switch (err) {
// UTF-16 可编码的最大码点为 U+10FFFF,
// 位于有效码点范围内。
error.CodepointTooLarge => unreachable,
};
}
},
}
return dest_index;
}
pub fn utf16LeToUtf8(utf8: []u8, utf16le: []const u16) Utf16LeToUtf8Error!usize {
return utf16LeToUtf8Impl(utf8, utf16le, .cannot_encode_surrogate_half);
}
test utf16LeToUtf8 {
var utf16le: [2]u16 = undefined;
const utf16le_as_bytes = mem.sliceAsBytes(utf16le[0..]);
{
mem.writeInt(u16, utf16le_as_bytes[0..2], 'A', .little);
mem.writeInt(u16, utf16le_as_bytes[2..4], 'a', .little);
const utf8 = try utf16LeToUtf8Alloc(testing.allocator, &utf16le);
defer testing.allocator.free(utf8);
try testing.expect(mem.eql(u8, utf8, "Aa"));
}
{
mem.writeInt(u16, utf16le_as_bytes[0..2], 0x80, .little);
mem.writeInt(u16, utf16le_as_bytes[2..4], 0xffff, .little);
const utf8 = try utf16LeToUtf8Alloc(testing.allocator, &utf16le);
defer testing.allocator.free(utf8);
try testing.expect(mem.eql(u8, utf8, "\xc2\x80" ++ "\xef\xbf\xbf"));
}
{
// 刚好位于代理半区范围之外的值
mem.writeInt(u16, utf16le_as_bytes[0..2], 0xd7ff, .little);
mem.writeInt(u16, utf16le_as_bytes[2..4], 0xe000, .little);
const utf8 = try utf16LeToUtf8Alloc(testing.allocator, &utf16le);
defer testing.allocator.free(utf8);
try testing.expect(mem.eql(u8, utf8, "\xed\x9f\xbf" ++ "\xee\x80\x80"));
}
{
// 最小的代理对
mem.writeInt(u16, utf16le_as_bytes[0..2], 0xd800, .little);
mem.writeInt(u16, utf16le_as_bytes[2..4], 0xdc00, .little);
const utf8 = try utf16LeToUtf8Alloc(testing.allocator, &utf16le);
defer testing.allocator.free(utf8);
try testing.expect(mem.eql(u8, utf8, "\xf0\x90\x80\x80"));
}
{
// 最大的代理对
mem.writeInt(u16, utf16le_as_bytes[0..2], 0xdbff, .little);
mem.writeInt(u16, utf16le_as_bytes[2..4], 0xdfff, .little);
const utf8 = try utf16LeToUtf8Alloc(testing.allocator, &utf16le);
defer testing.allocator.free(utf8);
try testing.expect(mem.eql(u8, utf8, "\xf4\x8f\xbf\xbf"));
}
{
mem.writeInt(u16, utf16le_as_bytes[0..2], 0xdbff, .little);
mem.writeInt(u16, utf16le_as_bytes[2..4], 0xdc00, .little);
const utf8 = try utf16LeToUtf8Alloc(testing.allocator, &utf16le);
defer testing.allocator.free(utf8);
try testing.expect(mem.eql(u8, utf8, "\xf4\x8f\xb0\x80"));
}
{
mem.writeInt(u16, utf16le_as_bytes[0..2], 0xdcdc, .little);
mem.writeInt(u16, utf16le_as_bytes[2..4], 0xdcdc, .little);
const result = utf16LeToUtf8Alloc(testing.allocator, &utf16le);
try testing.expectError(error.UnexpectedSecondSurrogateHalf, result);
}
}
fn utf8ToUtf16LeArrayListImpl(result: *std.array_list.Managed(u16), utf8: []const u8, comptime surrogates: Surrogates) !void {
assert(result.unusedCapacitySlice().len >= utf8.len);
var remaining = utf8;
vectorized: {
const chunk_len = std.simd.suggestVectorLength(u16) orelse break :vectorized;
const Chunk = @Vector(chunk_len, u8);
// 快速路径:检查并编码输入开头的 ASCII 字符。
while (remaining.len >= chunk_len) {
const chunk: Chunk = remaining[0..chunk_len].*;
const mask: Chunk = @splat(0x80);
if (@reduce(.Or, chunk & mask == mask)) {
// 发现非 ASCII 码元
break;
}
const utf16_chunk = mem.nativeToLittle(@Vector(chunk_len, u16), chunk);
result.addManyAsArrayAssumeCapacity(chunk_len).* = utf16_chunk;
remaining = remaining[chunk_len..];
}
}
const view = switch (surrogates) {
.cannot_encode_surrogate_half => try Utf8View.init(remaining),
.can_encode_surrogate_half => try Wtf8View.init(remaining),
};
var it = view.iterator();
while (it.nextCodepoint()) |codepoint| {
if (codepoint < 0x10000) {
try result.append(mem.nativeToLittle(u16, @intCast(codepoint)));
} else {
const high = @as(u16, @intCast((codepoint - 0x10000) >> 10)) + 0xD800;
const low = @as(u16, @intCast(codepoint & 0x3FF)) + 0xDC00;
try result.appendSlice(&.{ mem.nativeToLittle(u16, high), mem.nativeToLittle(u16, low) });
}
}
}
pub fn utf8ToUtf16LeArrayList(result: *std.array_list.Managed(u16), utf8: []const u8) error{ InvalidUtf8, OutOfMemory }!void {
try result.ensureUnusedCapacity(utf8.len);
return utf8ToUtf16LeArrayListImpl(result, utf8, .cannot_encode_surrogate_half);
}
pub fn utf8ToUtf16LeAlloc(allocator: Allocator, utf8: []const u8) error{ InvalidUtf8, OutOfMemory }![]u16 {
// 乐观地假定不需要代理对。
var result = try std.array_list.Managed(u16).initCapacity(allocator, utf8.len);
errdefer result.deinit();
try utf8ToUtf16LeArrayListImpl(&result, utf8, .cannot_encode_surrogate_half);
return result.toOwnedSlice();
}
pub fn utf8ToUtf16LeAllocZ(allocator: Allocator, utf8: []const u8) error{ InvalidUtf8, OutOfMemory }![:0]u16 {
// 乐观地假定不需要代理对。
var result = try std.array_list.Managed(u16).initCapacity(allocator, utf8.len + 1);
errdefer result.deinit();
try utf8ToUtf16LeArrayListImpl(&result, utf8, .cannot_encode_surrogate_half);
return result.toOwnedSliceSentinel(0);
}
/// 返回下一个字符的索引。若恰好填满,返回索引等于输出切片长度。
/// 假定输出空间足够。
pub fn utf8ToUtf16Le(utf16le: []u16, utf8: []const u8) error{InvalidUtf8}!usize {
return utf8ToUtf16LeImpl(utf16le, utf8, .cannot_encode_surrogate_half);
}
pub fn utf8ToUtf16LeImpl(utf16le: []u16, utf8: []const u8, comptime surrogates: Surrogates) !usize {
var dest_index: usize = 0;
var remaining = utf8;
vectorized: {
const chunk_len = std.simd.suggestVectorLength(u16) orelse break :vectorized;
const Chunk = @Vector(chunk_len, u8);
// 快速路径:检查并编码输入开头的 ASCII 字符。
while (remaining.len >= chunk_len) {
const chunk: Chunk = remaining[0..chunk_len].*;
const mask: Chunk = @splat(0x80);
if (@reduce(.Or, chunk & mask == mask)) {
// 发现非 ASCII 码元
break;
}
const utf16_chunk = mem.nativeToLittle(@Vector(chunk_len, u16), chunk);
utf16le[dest_index..][0..chunk_len].* = utf16_chunk;
dest_index += chunk_len;
remaining = remaining[chunk_len..];
}
}
const view = switch (surrogates) {
.cannot_encode_surrogate_half => try Utf8View.init(remaining),
.can_encode_surrogate_half => try Wtf8View.init(remaining),
};
var it = view.iterator();
while (it.nextCodepoint()) |codepoint| {
if (codepoint < 0x10000) {
utf16le[dest_index] = mem.nativeToLittle(u16, @intCast(codepoint));
dest_index += 1;
} else {
const high = @as(u16, @intCast((codepoint - 0x10000) >> 10)) + 0xD800;
const low = @as(u16, @intCast(codepoint & 0x3FF)) + 0xDC00;
utf16le[dest_index..][0..2].* = .{ mem.nativeToLittle(u16, high), mem.nativeToLittle(u16, low) };
dest_index += 2;
}
}
return dest_index;
}
test utf8ToUtf16Le {
var utf16le: [128]u16 = undefined;
{
const length = try utf8ToUtf16Le(utf16le[0..], "𐐷");
try testing.expectEqualSlices(u8, "\x01\xd8\x37\xdc", mem.sliceAsBytes(utf16le[0..length]));
}
{
const length = try utf8ToUtf16Le(utf16le[0..], "\u{10FFFF}");
try testing.expectEqualSlices(u8, "\xff\xdb\xff\xdf", mem.sliceAsBytes(utf16le[0..length]));
}
{
const result = utf8ToUtf16Le(utf16le[0..], "\xf4\x90\x80\x80");
try testing.expectError(error.InvalidUtf8, result);
}
{
const length = try utf8ToUtf16Le(utf16le[0..], "This string has been designed to test the vectorized implementat" ++
"ion by beginning with one hundred twenty-seven ASCII characters¡");
try testing.expectEqualSlices(u8, &.{
'T', 0, 'h', 0, 'i', 0, 's', 0, ' ', 0, 's', 0, 't', 0, 'r', 0, 'i', 0, 'n', 0, 'g', 0, ' ', 0, 'h', 0, 'a', 0, 's', 0, ' ', 0,
'b', 0, 'e', 0, 'e', 0, 'n', 0, ' ', 0, 'd', 0, 'e', 0, 's', 0, 'i', 0, 'g', 0, 'n', 0, 'e', 0, 'd', 0, ' ', 0, 't', 0, 'o', 0,
' ', 0, 't', 0, 'e', 0, 's', 0, 't', 0, ' ', 0, 't', 0, 'h', 0, 'e', 0, ' ', 0, 'v', 0, 'e', 0, 'c', 0, 't', 0, 'o', 0, 'r', 0,
'i', 0, 'z', 0, 'e', 0, 'd', 0, ' ', 0, 'i', 0, 'm', 0, 'p', 0, 'l', 0, 'e', 0, 'm', 0, 'e', 0, 'n', 0, 't', 0, 'a', 0, 't', 0,
'i', 0, 'o', 0, 'n', 0, ' ', 0, 'b', 0, 'y', 0, ' ', 0, 'b', 0, 'e', 0, 'g', 0, 'i', 0, 'n', 0, 'n', 0, 'i', 0, 'n', 0, 'g', 0,
' ', 0, 'w', 0, 'i', 0, 't', 0, 'h', 0, ' ', 0, 'o', 0, 'n', 0, 'e', 0, ' ', 0, 'h', 0, 'u', 0, 'n', 0, 'd', 0, 'r', 0, 'e', 0,
'd', 0, ' ', 0, 't', 0, 'w', 0, 'e', 0, 'n', 0, 't', 0, 'y', 0, '-', 0, 's', 0, 'e', 0, 'v', 0, 'e', 0, 'n', 0, ' ', 0, 'A', 0,
'S', 0, 'C', 0, 'I', 0, 'I', 0, ' ', 0, 'c', 0, 'h', 0, 'a', 0, 'r', 0, 'a', 0, 'c', 0, 't', 0, 'e', 0, 'r', 0, 's', 0,
'¡',
0,
}, mem.sliceAsBytes(utf16le[0..length]));
}
}
test utf8ToUtf16LeArrayList {
{
var list = std.array_list.Managed(u16).init(testing.allocator);
defer list.deinit();
try utf8ToUtf16LeArrayList(&list, "𐐷");
try testing.expectEqualSlices(u8, "\x01\xd8\x37\xdc", mem.sliceAsBytes(list.items));
}
{
var list = std.array_list.Managed(u16).init(testing.allocator);
defer list.deinit();
try utf8ToUtf16LeArrayList(&list, "\u{10FFFF}");
try testing.expectEqualSlices(u8, "\xff\xdb\xff\xdf", mem.sliceAsBytes(list.items));
}
{
var list = std.array_list.Managed(u16).init(testing.allocator);
defer list.deinit();
const result = utf8ToUtf16LeArrayList(&list, "\xf4\x90\x80\x80");
try testing.expectError(error.InvalidUtf8, result);
}
}
test utf8ToUtf16LeAlloc {
{
const utf16 = try utf8ToUtf16LeAlloc(testing.allocator, "𐐷");
defer testing.allocator.free(utf16);
try testing.expectEqualSlices(u8, "\x01\xd8\x37\xdc", mem.sliceAsBytes(utf16[0..]));
}
{
const utf16 = try utf8ToUtf16LeAlloc(testing.allocator, "\u{10FFFF}");
defer testing.allocator.free(utf16);
try testing.expectEqualSlices(u8, "\xff\xdb\xff\xdf", mem.sliceAsBytes(utf16[0..]));
}
{
const result = utf8ToUtf16LeAlloc(testing.allocator, "\xf4\x90\x80\x80");
try testing.expectError(error.InvalidUtf8, result);
}
}
test utf8ToUtf16LeAllocZ {
{
const utf16 = try utf8ToUtf16LeAllocZ(testing.allocator, "𐐷");
defer testing.allocator.free(utf16);
try testing.expectEqualSlices(u8, "\x01\xd8\x37\xdc", mem.sliceAsBytes(utf16));
try testing.expect(utf16[2] == 0);
}
{
const utf16 = try utf8ToUtf16LeAllocZ(testing.allocator, "\u{10FFFF}");
defer testing.allocator.free(utf16);
try testing.expectEqualSlices(u8, "\xff\xdb\xff\xdf", mem.sliceAsBytes(utf16));
try testing.expect(utf16[2] == 0);
}
{
const result = utf8ToUtf16LeAllocZ(testing.allocator, "\xf4\x90\x80\x80");
try testing.expectError(error.InvalidUtf8, result);
}
{
const utf16 = try utf8ToUtf16LeAllocZ(testing.allocator, "This string has been designed to test the vectorized implementat" ++
"ion by beginning with one hundred twenty-seven ASCII characters¡");
defer testing.allocator.free(utf16);
try testing.expectEqualSlices(u8, &.{
'T', 0, 'h', 0, 'i', 0, 's', 0, ' ', 0, 's', 0, 't', 0, 'r', 0, 'i', 0, 'n', 0, 'g', 0, ' ', 0, 'h', 0, 'a', 0, 's', 0, ' ', 0,
'b', 0, 'e', 0, 'e', 0, 'n', 0, ' ', 0, 'd', 0, 'e', 0, 's', 0, 'i', 0, 'g', 0, 'n', 0, 'e', 0, 'd', 0, ' ', 0, 't', 0, 'o', 0,
' ', 0, 't', 0, 'e', 0, 's', 0, 't', 0, ' ', 0, 't', 0, 'h', 0, 'e', 0, ' ', 0, 'v', 0, 'e', 0, 'c', 0, 't', 0, 'o', 0, 'r', 0,
'i', 0, 'z', 0, 'e', 0, 'd', 0, ' ', 0, 'i', 0, 'm', 0, 'p', 0, 'l', 0, 'e', 0, 'm', 0, 'e', 0, 'n', 0, 't', 0, 'a', 0, 't', 0,
'i', 0, 'o', 0, 'n', 0, ' ', 0, 'b', 0, 'y', 0, ' ', 0, 'b', 0, 'e', 0, 'g', 0, 'i', 0, 'n', 0, 'n', 0, 'i', 0, 'n', 0, 'g', 0,
' ', 0, 'w', 0, 'i', 0, 't', 0, 'h', 0, ' ', 0, 'o', 0, 'n', 0, 'e', 0, ' ', 0, 'h', 0, 'u', 0, 'n', 0, 'd', 0, 'r', 0, 'e', 0,
'd', 0, ' ', 0, 't', 0, 'w', 0, 'e', 0, 'n', 0, 't', 0, 'y', 0, '-', 0, 's', 0, 'e', 0, 'v', 0, 'e', 0, 'n', 0, ' ', 0, 'A', 0,
'S', 0, 'C', 0, 'I', 0, 'I', 0, ' ', 0, 'c', 0, 'h', 0, 'a', 0, 'r', 0, 'a', 0, 'c', 0, 't', 0, 'e', 0, 'r', 0, 's', 0,
'¡',
0,
}, mem.sliceAsBytes(utf16));
}
}
test "ArrayList functions on a re-used list" {
// utf8ToUtf16LeArrayList
{
var list = std.array_list.Managed(u16).init(testing.allocator);
defer list.deinit();
const init_slice = utf8ToUtf16LeStringLiteral("abcdefg");
try list.ensureTotalCapacityPrecise(init_slice.len);
list.appendSliceAssumeCapacity(init_slice);
try utf8ToUtf16LeArrayList(&list, "hijklmnopqrstuvwyxz");
try testing.expectEqualSlices(u16, utf8ToUtf16LeStringLiteral("abcdefghijklmnopqrstuvwyxz"), list.items);
}
// utf16LeToUtf8ArrayList
{
var list = std.array_list.Managed(u8).init(testing.allocator);
defer list.deinit();
const init_slice = "abcdefg";
try list.ensureTotalCapacityPrecise(init_slice.len);
list.appendSliceAssumeCapacity(init_slice);
try utf16LeToUtf8ArrayList(&list, utf8ToUtf16LeStringLiteral("hijklmnopqrstuvwyxz"));
try testing.expectEqualStrings("abcdefghijklmnopqrstuvwyxz", list.items);
}
// wtf8ToWtf16LeArrayList
{
var list = std.array_list.Managed(u16).init(testing.allocator);
defer list.deinit();
const init_slice = utf8ToUtf16LeStringLiteral("abcdefg");
try list.ensureTotalCapacityPrecise(init_slice.len);
list.appendSliceAssumeCapacity(init_slice);
try wtf8ToWtf16LeArrayList(&list, "hijklmnopqrstuvwyxz");
try testing.expectEqualSlices(u16, utf8ToUtf16LeStringLiteral("abcdefghijklmnopqrstuvwyxz"), list.items);
}
// wtf16LeToWtf8ArrayList
{
var list = std.array_list.Managed(u8).init(testing.allocator);
defer list.deinit();
const init_slice = "abcdefg";
try list.ensureTotalCapacityPrecise(init_slice.len);
list.appendSliceAssumeCapacity(init_slice);
try wtf16LeToWtf8ArrayList(&list, utf8ToUtf16LeStringLiteral("hijklmnopqrstuvwyxz"));
try testing.expectEqualStrings("abcdefghijklmnopqrstuvwyxz", list.items);
}
}
fn utf8ToUtf16LeStringLiteralImpl(comptime utf8: []const u8, comptime surrogates: Surrogates) *const [calcUtf16LeLenImpl(utf8, surrogates) catch |err| @compileError(err):0]u16 {
return comptime blk: {
const len: usize = calcUtf16LeLenImpl(utf8, surrogates) catch unreachable;
var utf16le: [len:0]u16 = @splat(0);
const utf16le_len = utf8ToUtf16LeImpl(&utf16le, utf8[0..], surrogates) catch |err| @compileError(err);
assert(len == utf16le_len);
const final = utf16le;
break :blk &final;
};
}
/// 将 UTF-8 字符串字面量转换为 UTF-16LE 字符串字面量。
pub fn utf8ToUtf16LeStringLiteral(comptime utf8: []const u8) *const [calcUtf16LeLen(utf8) catch |err| @compileError(err):0]u16 {
return utf8ToUtf16LeStringLiteralImpl(utf8, .cannot_encode_surrogate_half);
}
/// 将 WTF-8 字符串字面量转换为 WTF-16LE 字符串字面量。
pub fn wtf8ToWtf16LeStringLiteral(comptime wtf8: []const u8) *const [calcWtf16LeLen(wtf8) catch |err| @compileError(err):0]u16 {
return utf8ToUtf16LeStringLiteralImpl(wtf8, .can_encode_surrogate_half);
}
pub fn calcUtf16LeLenImpl(utf8: []const u8, comptime surrogates: Surrogates) !usize {
const utf8DecodeImpl = switch (surrogates) {
.cannot_encode_surrogate_half => utf8Decode,
.can_encode_surrogate_half => wtf8Decode,
};
var src_i: usize = 0;
var dest_len: usize = 0;
while (src_i < utf8.len) {
const n = try utf8ByteSequenceLength(utf8[src_i]);
const next_src_i = src_i + n;
const codepoint = try utf8DecodeImpl(utf8[src_i..next_src_i]);
if (codepoint < 0x10000) {
dest_len += 1;
} else {
dest_len += 2;
}
src_i = next_src_i;
}
return dest_len;
}
const CalcUtf16LeLenError = Utf8DecodeError || error{Utf8InvalidStartByte};
/// 以 []u16 的长度形式返回 UTF-8 切片的 UTF-16LE 长度。
/// []u8 中的长度为 2*len16。
pub fn calcUtf16LeLen(utf8: []const u8) CalcUtf16LeLenError!usize {
return calcUtf16LeLenImpl(utf8, .cannot_encode_surrogate_half);
}
const CalcWtf16LeLenError = Wtf8DecodeError || error{Utf8InvalidStartByte};
/// 返回 WTF-8 切片转换为 WTF-16LE 后的长度,按 []u16 元素数计。
/// 按 []u8 计的长度为 2*len16。
pub fn calcWtf16LeLen(wtf8: []const u8) CalcWtf16LeLenError!usize {
return calcUtf16LeLenImpl(wtf8, .can_encode_surrogate_half);
}
fn testCalcUtf16LeLenImpl(calcUtf16LeLenImpl_: anytype) !void {
try testing.expectEqual(@as(usize, 1), try calcUtf16LeLenImpl_("a"));
try testing.expectEqual(@as(usize, 10), try calcUtf16LeLenImpl_("abcdefghij"));
try testing.expectEqual(@as(usize, 10), try calcUtf16LeLenImpl_("äåéëþüúíóö"));
try testing.expectEqual(@as(usize, 5), try calcUtf16LeLenImpl_("こんにちは"));
}
test calcUtf16LeLen {
try testCalcUtf16LeLenImpl(calcUtf16LeLen);
try comptime testCalcUtf16LeLenImpl(calcUtf16LeLen);
}
test calcWtf16LeLen {
try testCalcUtf16LeLenImpl(calcWtf16LeLen);
try comptime testCalcUtf16LeLenImpl(calcWtf16LeLen);
}
/// 将给定 `utf16le` 字符串编码为 UTF-8 字节并打印。
/// 未配对代理项会被替换字符 (U+FFFD) 替换。
fn formatUtf16Le(utf16le: []const u16, writer: *std.Io.Writer) std.Io.Writer.Error!void {
var buf: [300]u8 = undefined; // 只是一个任意大小
var it = Utf16LeIterator.init(utf16le);
var u8len: usize = 0;
while (it.nextCodepoint() catch replacement_character) |codepoint| {
u8len += utf8Encode(codepoint, buf[u8len..]) catch
utf8Encode(replacement_character, buf[u8len..]) catch unreachable;
// 确保始终有足够空间容纳另一个最大长度的 UTF-8 码点
if (u8len + 4 > buf.len) {
try writer.writeAll(buf[0..u8len]);
u8len = 0;
}
}
try writer.writeAll(buf[0..u8len]);
}
/// 返回用于 UTF-16 LE 字符串(可能格式不正确)的 Formatter,
/// 格式化时将其转换为 UTF-8。
/// 未配对代理码元会替换为替换字符(U+FFFD)。
pub fn fmtUtf16Le(utf16le: []const u16) std.fmt.Alt([]const u16, formatUtf16Le) {
return .{ .data = utf16le };
}
test fmtUtf16Le {
const expectFmt = testing.expectFmt;
try expectFmt("", "{f}", .{fmtUtf16Le(utf8ToUtf16LeStringLiteral(""))});
try expectFmt("", "{f}", .{fmtUtf16Le(wtf8ToWtf16LeStringLiteral(""))});
try expectFmt("foo", "{f}", .{fmtUtf16Le(utf8ToUtf16LeStringLiteral("foo"))});
try expectFmt("foo", "{f}", .{fmtUtf16Le(wtf8ToWtf16LeStringLiteral("foo"))});
try expectFmt("𐐷", "{f}", .{fmtUtf16Le(wtf8ToWtf16LeStringLiteral("𐐷"))});
try expectFmt("", "{f}", .{fmtUtf16Le(&[_]u16{mem.readInt(u16, "\xff\xd7", native_endian)})});
try expectFmt("�", "{f}", .{fmtUtf16Le(&[_]u16{mem.readInt(u16, "\x00\xd8", native_endian)})});
try expectFmt("�", "{f}", .{fmtUtf16Le(&[_]u16{mem.readInt(u16, "\xff\xdb", native_endian)})});
try expectFmt("�", "{f}", .{fmtUtf16Le(&[_]u16{mem.readInt(u16, "\x00\xdc", native_endian)})});
try expectFmt("�", "{f}", .{fmtUtf16Le(&[_]u16{mem.readInt(u16, "\xff\xdf", native_endian)})});
try expectFmt("", "{f}", .{fmtUtf16Le(&[_]u16{mem.readInt(u16, "\x00\xe0", native_endian)})});
}
fn testUtf8ToUtf16LeStringLiteral(utf8ToUtf16LeStringLiteral_: anytype) !void {
{
const bytes = [_:0]u16{
mem.nativeToLittle(u16, 0x41),
};
const utf16 = utf8ToUtf16LeStringLiteral_("A");
try testing.expectEqualSlices(u16, &bytes, utf16);
try testing.expect(utf16[1] == 0);
}
{
const bytes = [_:0]u16{
mem.nativeToLittle(u16, 0xD801),
mem.nativeToLittle(u16, 0xDC37),
};
const utf16 = utf8ToUtf16LeStringLiteral_("𐐷");
try testing.expectEqualSlices(u16, &bytes, utf16);
try testing.expect(utf16[2] == 0);
}
{
const bytes = [_:0]u16{
mem.nativeToLittle(u16, 0x02FF),
};
const utf16 = utf8ToUtf16LeStringLiteral_("\u{02FF}");
try testing.expectEqualSlices(u16, &bytes, utf16);
try testing.expect(utf16[1] == 0);
}
{
const bytes = [_:0]u16{
mem.nativeToLittle(u16, 0x7FF),
};
const utf16 = utf8ToUtf16LeStringLiteral_("\u{7FF}");
try testing.expectEqualSlices(u16, &bytes, utf16);
try testing.expect(utf16[1] == 0);
}
{
const bytes = [_:0]u16{
mem.nativeToLittle(u16, 0x801),
};
const utf16 = utf8ToUtf16LeStringLiteral_("\u{801}");
try testing.expectEqualSlices(u16, &bytes, utf16);
try testing.expect(utf16[1] == 0);
}
{
const bytes = [_:0]u16{
mem.nativeToLittle(u16, 0xDBFF),
mem.nativeToLittle(u16, 0xDFFF),
};
const utf16 = utf8ToUtf16LeStringLiteral_("\u{10FFFF}");
try testing.expectEqualSlices(u16, &bytes, utf16);
try testing.expect(utf16[2] == 0);
}
}
test utf8ToUtf16LeStringLiteral {
try testUtf8ToUtf16LeStringLiteral(utf8ToUtf16LeStringLiteral);
}
test wtf8ToWtf16LeStringLiteral {
try testUtf8ToUtf16LeStringLiteral(wtf8ToWtf16LeStringLiteral);
}
fn testUtf8CountCodepoints() !void {
try testing.expectEqual(@as(usize, 10), try utf8CountCodepoints("abcdefghij"));
try testing.expectEqual(@as(usize, 10), try utf8CountCodepoints("äåéëþüúíóö"));
try testing.expectEqual(@as(usize, 5), try utf8CountCodepoints("こんにちは"));
// testing.expectError(error.Utf8EncodesSurrogateHalf, utf8CountCodepoints("\xED\xA0\x80"));
}
test "utf8 count codepoints" {
try testUtf8CountCodepoints();
try comptime testUtf8CountCodepoints();
}
fn testUtf8ValidCodepoint() !void {
try testing.expect(utf8ValidCodepoint('e'));
try testing.expect(utf8ValidCodepoint('ë'));
try testing.expect(utf8ValidCodepoint('は'));
try testing.expect(utf8ValidCodepoint(0xe000));
try testing.expect(utf8ValidCodepoint(0x10ffff));
try testing.expect(!utf8ValidCodepoint(0xd800));
try testing.expect(!utf8ValidCodepoint(0xdfff));
try testing.expect(!utf8ValidCodepoint(0x110000));
}
test "utf8 valid codepoint" {
try testUtf8ValidCodepoint();
try comptime testUtf8ValidCodepoint();
}
/// 若码点为代理码点(U+DC00 到 U+DFFF),则返回 true
pub fn isSurrogateCodepoint(c: u21) bool {
return switch (c) {
0xD800...0xDFFF => true,
else => false,
};
}
/// 将给定码点编码为 WTF-8 字节序列。
/// c:码点。
/// out:写入的输出缓冲区,长度必须 >= utf8CodepointSequenceLength(c)。
/// 错误:c 无法编码为 WTF-8 时。
/// 返回:写入 out 的字节数。
pub fn wtf8Encode(c: u21, out: []u8) error{CodepointTooLarge}!u3 {
return utf8EncodeImpl(c, out, .can_encode_surrogate_half);
}
const Wtf8DecodeError = Utf8Decode2Error || Utf8Decode3AllowSurrogateHalfError || Utf8Decode4Error;
/// 已弃用。此函数的 API 不够直观,容易误用。
pub fn wtf8Decode(bytes: []const u8) Wtf8DecodeError!u21 {
return switch (bytes.len) {
1 => bytes[0],
2 => utf8Decode2(bytes[0..2].*),
3 => utf8Decode3AllowSurrogateHalf(bytes[0..3].*),
4 => utf8Decode4(bytes[0..4].*),
else => unreachable,
};
}
/// 如果输入完全由 WTF-8 码点组成,则返回 true
/// (与 UTF-8 限制相同,但允许代理码点
/// U+D800 到 U+DFFF)。
/// 不检查 WTF-8 是否格式良好,也就是说,
/// 此函数不检查所有代理码元是否均未配对。
pub fn wtf8ValidateSlice(input: []const u8) bool {
return utf8ValidateSliceImpl(input, .can_encode_surrogate_half);
}
test "validate WTF-8 slice" {
try testValidateWtf8Slice();
try comptime testValidateWtf8Slice();
// 按可变大小的块(基于建议的向量大小)跳过
// ASCII 字符。确保分块方式正确。
const str = @as([550]u8, @splat('a')) ++ "\xc0";
for (0..str.len - 3) |i| {
try testing.expect(!wtf8ValidateSlice(str[i..]));
}
}
fn testValidateWtf8Slice() !void {
// 这些内容在 UTF-8 和 WTF-8 规则下均有效/无效。
try testing.expect(wtf8ValidateSlice("abc"));
try testing.expect(wtf8ValidateSlice("abc\xdf\xbf"));
try testing.expect(wtf8ValidateSlice(""));
try testing.expect(wtf8ValidateSlice("a"));
try testing.expect(wtf8ValidateSlice("abc"));
try testing.expect(wtf8ValidateSlice("Ж"));
try testing.expect(wtf8ValidateSlice("ЖЖ"));
try testing.expect(wtf8ValidateSlice("брэд-ЛГТМ"));
try testing.expect(wtf8ValidateSlice("☺☻☹"));
try testing.expect(wtf8ValidateSlice("a\u{fffdb}"));
try testing.expect(wtf8ValidateSlice("\xf4\x8f\xbf\xbf"));
try testing.expect(wtf8ValidateSlice("abc\xdf\xbf"));
try testing.expect(!wtf8ValidateSlice("abc\xc0"));
try testing.expect(!wtf8ValidateSlice("abc\xc0abc"));
try testing.expect(!wtf8ValidateSlice("aa\xe2"));
try testing.expect(!wtf8ValidateSlice("\x42\xfa"));
try testing.expect(!wtf8ValidateSlice("\x42\xfa\x43"));
try testing.expect(!wtf8ValidateSlice("abc\xc0"));
try testing.expect(!wtf8ValidateSlice("abc\xc0abc"));
try testing.expect(!wtf8ValidateSlice("\xf4\x90\x80\x80"));
try testing.expect(!wtf8ValidateSlice("\xf7\xbf\xbf\xbf"));
try testing.expect(!wtf8ValidateSlice("\xfb\xbf\xbf\xbf\xbf"));
try testing.expect(!wtf8ValidateSlice("\xc0\x80"));
// 但代理码点仅在 WTF-8 中有效。
try testing.expect(wtf8ValidateSlice("\xed\xa0\x80"));
try testing.expect(wtf8ValidateSlice("\xed\xbf\xbf"));
}
/// Wtf8View 遍历 WTF-8 编码字符串中的码点,包括未配对代理项。
///
/// ```
/// var wtf8 = (try std.unicode.Wtf8View.init("hi there")).iterator();
/// while (wtf8.nextCodepointSlice()) |codepoint| {
/// // 注意:码点可能是代理项,不属于有效 UTF-8;
/// // 避免直接打印、发送或输出它
/// }
/// ```
///
pub const Wtf8View = struct {
bytes: []const u8,
pub fn init(s: []const u8) error{InvalidWtf8}!Wtf8View {
if (!wtf8ValidateSlice(s)) {
return error.InvalidWtf8;
}
return initUnchecked(s);
}
pub fn initUnchecked(s: []const u8) Wtf8View {
return Wtf8View{ .bytes = s };
}
pub inline fn initComptime(comptime s: []const u8) Wtf8View {
return comptime if (init(s)) |r| r else |err| switch (err) {
error.InvalidWtf8 => {
@compileError("invalid wtf8");
},
};
}
pub fn iterator(s: Wtf8View) Wtf8Iterator {
return Wtf8Iterator{
.bytes = s.bytes,
.i = 0,
};
}
};
/// 断言 `bytes` 是有效的 WTF-8
pub const Wtf8Iterator = struct {
bytes: []const u8,
i: usize,
pub fn nextCodepointSlice(it: *Wtf8Iterator) ?[]const u8 {
if (it.i >= it.bytes.len) {
return null;
}
const cp_len = utf8ByteSequenceLength(it.bytes[it.i]) catch unreachable;
it.i += cp_len;
return it.bytes[it.i - cp_len .. it.i];
}
pub fn nextCodepoint(it: *Wtf8Iterator) ?u21 {
const slice = it.nextCodepointSlice() orelse return null;
return wtf8Decode(slice) catch unreachable;
}
/// 向前查看接下来的 n 个码点,而不推进迭代器。
/// 如果可用码点不足 n 个,则返回字符串的剩余部分。
pub fn peek(it: *Wtf8Iterator, n: usize) []const u8 {
const original_i = it.i;
defer it.i = original_i;
var end_ix = original_i;
var found: usize = 0;
while (found < n) : (found += 1) {
const next_codepoint = it.nextCodepointSlice() orelse return it.bytes[original_i..];
end_ix += next_codepoint.len;
}
return it.bytes[original_i..end_ix];
}
/// 向前查看下一个码点,而不推进迭代器。
/// 若不存在码点,则返回 null。
pub fn peekCodepoint(it: *Wtf8Iterator) ?u21 {
const original_i = it.i;
defer it.i = original_i;
return it.nextCodepoint();
}
};
pub fn wtf16LeToWtf8ArrayList(result: *std.array_list.Managed(u8), utf16le: []const u16) Allocator.Error!void {
try result.ensureUnusedCapacity(utf16le.len);
return utf16LeToUtf8ArrayListImpl(result, utf16le, .can_encode_surrogate_half);
}
/// 调用者必须释放返回的内存。
pub fn wtf16LeToWtf8Alloc(allocator: Allocator, wtf16le: []const u16) Allocator.Error![]u8 {
// 乐观地假设它们全都是 ASCII 字符。
var result = try std.array_list.Managed(u8).initCapacity(allocator, wtf16le.len);
errdefer result.deinit();
try utf16LeToUtf8ArrayListImpl(&result, wtf16le, .can_encode_surrogate_half);
return result.toOwnedSlice();
}
/// 调用者必须释放返回的内存。
pub fn wtf16LeToWtf8AllocZ(allocator: Allocator, wtf16le: []const u16) Allocator.Error![:0]u8 {
// 乐观地假定全部为 ascii(并为空终止符分配空间)
var result = try std.array_list.Managed(u8).initCapacity(allocator, wtf16le.len + 1);
errdefer result.deinit();
try utf16LeToUtf8ArrayListImpl(&result, wtf16le, .can_encode_surrogate_half);
return result.toOwnedSliceSentinel(0);
}
pub fn wtf16LeToWtf8(wtf8: []u8, wtf16le: []const u16) usize {
return utf16LeToUtf8Impl(wtf8, wtf16le, .can_encode_surrogate_half) catch |err| switch (err) {};
}
pub fn wtf8ToWtf16LeArrayList(result: *std.array_list.Managed(u16), wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }!void {
try result.ensureUnusedCapacity(wtf8.len);
return utf8ToUtf16LeArrayListImpl(result, wtf8, .can_encode_surrogate_half);
}
pub fn wtf8ToWtf16LeAlloc(allocator: Allocator, wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }![]u16 {
// 乐观地假定不需要代理对。
var result = try std.array_list.Managed(u16).initCapacity(allocator, wtf8.len);
errdefer result.deinit();
try utf8ToUtf16LeArrayListImpl(&result, wtf8, .can_encode_surrogate_half);
return result.toOwnedSlice();
}
pub fn wtf8ToWtf16LeAllocZ(allocator: Allocator, wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }![:0]u16 {
// 乐观地假定不需要代理对。
var result = try std.array_list.Managed(u16).initCapacity(allocator, wtf8.len + 1);
errdefer result.deinit();
try utf8ToUtf16LeArrayListImpl(&result, wtf8, .can_encode_surrogate_half);
return result.toOwnedSliceSentinel(0);
}
/// 返回下一个字符的索引。若恰好填满,返回索引等于输出切片长度。
/// 假定输出空间足够。
pub fn wtf8ToWtf16Le(wtf16le: []u16, wtf8: []const u8) error{InvalidWtf8}!usize {
return utf8ToUtf16LeImpl(wtf16le, wtf8, .can_encode_surrogate_half);
}
/// 将代理码点(U+D800 至 U+DFFF)替换为 Unicode 替换字符(U+FFFD)。
/// 所有代理码点与替换字符均编码为三个字节,因此输入与输出切片的长度始终相同。
/// 当 `utf8` 和 `wtf8` 指向同一切片时,支持原地转换。
/// 注意:若 `wtf8` 完全由结构良好的 UTF-8 组成,则无需转换。
/// 可使用 `utf8ValidateSlice` 检查是否值得执行有损转换。
/// 若 `wtf8` 不是有效 WTF-8,则返回 `error.InvalidWtf8`。
///
///
pub fn wtf8ToUtf8Lossy(utf8: []u8, wtf8: []const u8) error{InvalidWtf8}!void {
assert(utf8.len >= wtf8.len);
const in_place = utf8.ptr == wtf8.ptr;
const replacement_char_bytes = comptime blk: {
var buf: [3]u8 = undefined;
assert((utf8Encode(replacement_character, &buf) catch unreachable) == 3);
break :blk buf;
};
var dest_i: usize = 0;
const view = try Wtf8View.init(wtf8);
var it = view.iterator();
while (it.nextCodepointSlice()) |codepoint_slice| {
// 所有代理码点均编码为 3 字节。
if (codepoint_slice.len == 3) {
const codepoint = wtf8Decode(codepoint_slice) catch unreachable;
if (isSurrogateCodepoint(codepoint)) {
@memcpy(utf8[dest_i..][0..replacement_char_bytes.len], &replacement_char_bytes);
dest_i += replacement_char_bytes.len;
continue;
}
}
if (!in_place) {
@memcpy(utf8[dest_i..][0..codepoint_slice.len], codepoint_slice);
}
dest_i += codepoint_slice.len;
}
}
pub fn wtf8ToUtf8LossyAlloc(allocator: Allocator, wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }![]u8 {
const utf8 = try allocator.alloc(u8, wtf8.len);
errdefer allocator.free(utf8);
try wtf8ToUtf8Lossy(utf8, wtf8);
return utf8;
}
pub fn wtf8ToUtf8LossyAllocZ(allocator: Allocator, wtf8: []const u8) error{ InvalidWtf8, OutOfMemory }![:0]u8 {
const utf8 = try allocator.allocSentinel(u8, wtf8.len, 0);
errdefer allocator.free(utf8);
try wtf8ToUtf8Lossy(utf8, wtf8);
return utf8;
}
test wtf8ToUtf8Lossy {
var buf: [32]u8 = undefined;
const invalid_utf8 = "\xff";
try testing.expectError(error.InvalidWtf8, wtf8ToUtf8Lossy(&buf, invalid_utf8));
const ascii = "abcd";
try wtf8ToUtf8Lossy(&buf, ascii);
try testing.expectEqualStrings("abcd", buf[0..ascii.len]);
const high_surrogate_half = "ab\xed\xa0\xbdcd";
try wtf8ToUtf8Lossy(&buf, high_surrogate_half);
try testing.expectEqualStrings("ab\u{FFFD}cd", buf[0..high_surrogate_half.len]);
const low_surrogate_half = "ab\xed\xb2\xa9cd";
try wtf8ToUtf8Lossy(&buf, low_surrogate_half);
try testing.expectEqualStrings("ab\u{FFFD}cd", buf[0..low_surrogate_half.len]);
// 若 WTF-8 格式不良,则每个代理码元分别转换为一个
// 替换字符,而不是解释为代理项对。
const encoded_surrogate_pair = "ab\xed\xa0\xbd\xed\xb2\xa9cd";
try wtf8ToUtf8Lossy(&buf, encoded_surrogate_pair);
try testing.expectEqualStrings("ab\u{FFFD}\u{FFFD}cd", buf[0..encoded_surrogate_pair.len]);
// 原地
@memcpy(buf[0..low_surrogate_half.len], low_surrogate_half);
const slice = buf[0..low_surrogate_half.len];
try wtf8ToUtf8Lossy(slice, slice);
try testing.expectEqualStrings("ab\u{FFFD}cd", slice);
}
test wtf8ToUtf8LossyAlloc {
const invalid_utf8 = "\xff";
try testing.expectError(error.InvalidWtf8, wtf8ToUtf8LossyAlloc(testing.allocator, invalid_utf8));
{
const ascii = "abcd";
const utf8 = try wtf8ToUtf8LossyAlloc(testing.allocator, ascii);
defer testing.allocator.free(utf8);
try testing.expectEqualStrings("abcd", utf8);
}
{
const surrogate_half = "ab\xed\xa0\xbdcd";
const utf8 = try wtf8ToUtf8LossyAlloc(testing.allocator, surrogate_half);
defer testing.allocator.free(utf8);
try testing.expectEqualStrings("ab\u{FFFD}cd", utf8);
}
{
// 若 WTF-8 格式不良,则每个代理码元分别转换为一个
// 替换字符,而不是解释为代理项对。
const encoded_surrogate_pair = "ab\xed\xa0\xbd\xed\xb2\xa9cd";
const utf8 = try wtf8ToUtf8LossyAlloc(testing.allocator, encoded_surrogate_pair);
defer testing.allocator.free(utf8);
try testing.expectEqualStrings("ab\u{FFFD}\u{FFFD}cd", utf8);
}
}
test wtf8ToUtf8LossyAllocZ {
const invalid_utf8 = "\xff";
try testing.expectError(error.InvalidWtf8, wtf8ToUtf8LossyAllocZ(testing.allocator, invalid_utf8));
{
const ascii = "abcd";
const utf8 = try wtf8ToUtf8LossyAllocZ(testing.allocator, ascii);
defer testing.allocator.free(utf8);
try testing.expectEqualStrings("abcd", utf8);
}
{
const surrogate_half = "ab\xed\xa0\xbdcd";
const utf8 = try wtf8ToUtf8LossyAllocZ(testing.allocator, surrogate_half);
defer testing.allocator.free(utf8);
try testing.expectEqualStrings("ab\u{FFFD}cd", utf8);
}
{
// 若 WTF-8 格式不良,则每个代理码元分别转换为一个
// 替换字符,而不是解释为代理项对。
const encoded_surrogate_pair = "ab\xed\xa0\xbd\xed\xb2\xa9cd";
const utf8 = try wtf8ToUtf8LossyAllocZ(testing.allocator, encoded_surrogate_pair);
defer testing.allocator.free(utf8);
try testing.expectEqualStrings("ab\u{FFFD}\u{FFFD}cd", utf8);
}
}
pub const Wtf16LeIterator = struct {
bytes: []const u8,
i: usize,
pub fn init(s: []const u16) Wtf16LeIterator {
return Wtf16LeIterator{
.bytes = mem.sliceAsBytes(s),
.i = 0,
};
}
/// 若下一个码点由代理对编码,
/// 则返回代理对所表示的码点。
/// 若下一个码点是未配对代理码点,
/// 则返回该未配对代理码点本身。
pub fn nextCodepoint(it: *Wtf16LeIterator) ?u21 {
assert(it.i <= it.bytes.len);
if (it.i == it.bytes.len) return null;
var code_units: [2]u16 = undefined;
code_units[0] = mem.readInt(u16, it.bytes[it.i..][0..2], .little);
it.i += 2;
surrogate_pair: {
if (utf16IsHighSurrogate(code_units[0])) {
if (it.i >= it.bytes.len) break :surrogate_pair;
code_units[1] = mem.readInt(u16, it.bytes[it.i..][0..2], .little);
const codepoint = utf16DecodeSurrogatePair(&code_units) catch break :surrogate_pair;
it.i += 2;
return codepoint;
}
}
return code_units[0];
}
};
test "non-well-formed WTF-8 does not roundtrip" {
// 此处编码代理对 U+D83D U+DCA9。
// 其良构形式应为 U+1F4A9,即 \xF0\x9F\x92\xA9。
const non_well_formed_wtf8 = "\xed\xa0\xbd\xed\xb2\xa9";
var wtf16_buf: [2]u16 = undefined;
const wtf16_len = try wtf8ToWtf16Le(&wtf16_buf, non_well_formed_wtf8);
const wtf16 = wtf16_buf[0..wtf16_len];
try testing.expectEqualSlices(u16, &[_]u16{
mem.nativeToLittle(u16, 0xD83D), // 高位代理项
mem.nativeToLittle(u16, 0xDCA9), // 低代理项
}, wtf16);
var wtf8_buf: [4]u8 = undefined;
const wtf8_len = wtf16LeToWtf8(&wtf8_buf, wtf16);
const wtf8 = wtf8_buf[0..wtf8_len];
// 转换为 WTF-16 再转回后会得到格式良好的 WTF-8,
// 但它与输入的 WTF-8 不匹配。
try testing.expectEqualSlices(u8, "\xf0\x9f\x92\xa9", wtf8);
}
fn testRoundtripWtf8(wtf8: []const u8) !void {
// 缓冲区
{
var wtf16_buf: [32]u16 = undefined;
const wtf16_len = try wtf8ToWtf16Le(&wtf16_buf, wtf8);
try testing.expectEqual(wtf16_len, calcWtf16LeLen(wtf8));
const wtf16 = wtf16_buf[0..wtf16_len];
var roundtripped_buf: [32]u8 = undefined;
const roundtripped_len = wtf16LeToWtf8(&roundtripped_buf, wtf16);
const roundtripped = roundtripped_buf[0..roundtripped_len];
try testing.expectEqualSlices(u8, wtf8, roundtripped);
}
// 分配
{
const wtf16 = try wtf8ToWtf16LeAlloc(testing.allocator, wtf8);
defer testing.allocator.free(wtf16);
const roundtripped = try wtf16LeToWtf8Alloc(testing.allocator, wtf16);
defer testing.allocator.free(roundtripped);
try testing.expectEqualSlices(u8, wtf8, roundtripped);
}
// AllocZ
{
const wtf16 = try wtf8ToWtf16LeAllocZ(testing.allocator, wtf8);
defer testing.allocator.free(wtf16);
const roundtripped = try wtf16LeToWtf8AllocZ(testing.allocator, wtf16);
defer testing.allocator.free(roundtripped);
try testing.expectEqualSlices(u8, wtf8, roundtripped);
}
}
test "well-formed WTF-8 roundtrips" {
try testRoundtripWtf8("\xed\x9f\xbf"); // 不是代理项
try testRoundtripWtf8("\xed\xa0\xbd"); // 高位代理项
try testRoundtripWtf8("\xed\xb2\xa9"); // 低代理项
try testRoundtripWtf8("\xed\xa0\xbd \xed\xb2\xa9"); // <高代理项><空格><低代理项>
try testRoundtripWtf8("\xed\xa0\x80\xed\xaf\xbf"); // <高代理项><高代理项>
try testRoundtripWtf8("\xed\xa0\x80\xee\x80\x80"); // <高位代理项><非代理项>
try testRoundtripWtf8("\xed\x9f\xbf\xed\xb0\x80"); // <非代理项><低代理项>
try testRoundtripWtf8("a\xed\xb0\x80"); // <非代理项><低代理项>
try testRoundtripWtf8("\xf0\x9f\x92\xa9"); // U+1F4A9,在 WTF-16 中编码为代理对。
}
fn testRoundtripWtf16(wtf16le: []const u16) !void {
// 缓冲区
{
var wtf8_buf: [32]u8 = undefined;
const wtf8_len = wtf16LeToWtf8(&wtf8_buf, wtf16le);
const wtf8 = wtf8_buf[0..wtf8_len];
var roundtripped_buf: [32]u16 = undefined;
const roundtripped_len = try wtf8ToWtf16Le(&roundtripped_buf, wtf8);
const roundtripped = roundtripped_buf[0..roundtripped_len];
try testing.expectEqualSlices(u16, wtf16le, roundtripped);
}
// 分配
{
const wtf8 = try wtf16LeToWtf8Alloc(testing.allocator, wtf16le);
defer testing.allocator.free(wtf8);
const roundtripped = try wtf8ToWtf16LeAlloc(testing.allocator, wtf8);
defer testing.allocator.free(roundtripped);
try testing.expectEqualSlices(u16, wtf16le, roundtripped);
}
// AllocZ
{
const wtf8 = try wtf16LeToWtf8AllocZ(testing.allocator, wtf16le);
defer testing.allocator.free(wtf8);
const roundtripped = try wtf8ToWtf16LeAllocZ(testing.allocator, wtf8);
defer testing.allocator.free(roundtripped);
try testing.expectEqualSlices(u16, wtf16le, roundtripped);
}
}
test "well-formed WTF-16 roundtrips" {
try testRoundtripWtf16(&[_]u16{
mem.nativeToLittle(u16, 0xD83D), // 高位代理项
mem.nativeToLittle(u16, 0xDCA9), // 低代理项
});
try testRoundtripWtf16(&[_]u16{
mem.nativeToLittle(u16, 0xD83D), // 高位代理项
mem.nativeToLittle(u16, ' '), // 不是代理项
mem.nativeToLittle(u16, 0xDCA9), // 低代理项
});
try testRoundtripWtf16(&[_]u16{
mem.nativeToLittle(u16, 0xD800), // 高位代理项
mem.nativeToLittle(u16, 0xDBFF), // 高位代理项
});
try testRoundtripWtf16(&[_]u16{
mem.nativeToLittle(u16, 0xD800), // 高位代理项
mem.nativeToLittle(u16, 0xE000), // 不是代理项
});
try testRoundtripWtf16(&[_]u16{
mem.nativeToLittle(u16, 0xD7FF), // 不是代理项
mem.nativeToLittle(u16, 0xDC00), // 低代理项
});
try testRoundtripWtf16(&[_]u16{
mem.nativeToLittle(u16, 0x61), // 不是代理项
mem.nativeToLittle(u16, 0xDC00), // 低代理项
});
try testRoundtripWtf16(&[_]u16{
mem.nativeToLittle(u16, 0xDC00), // 低代理项
});
}
/// 返回将给定 WTF-16 LE 切片编码为 WTF-8
/// 所需的字节数。
pub fn calcWtf8Len(wtf16le: []const u16) usize {
var it = Wtf16LeIterator.init(wtf16le);
var num_wtf8_bytes: usize = 0;
while (it.nextCodepoint()) |codepoint| {
// 注意:若将 utf8CodepointSequenceLength 修改为对代理码点报错,则它将不再适用于此处。
//
num_wtf8_bytes += utf8CodepointSequenceLength(codepoint) catch |err| switch (err) {
error.CodepointTooLarge => unreachable,
};
}
return num_wtf8_bytes;
}
fn testCalcWtf8Len() !void {
const L = utf8ToUtf16LeStringLiteral;
try testing.expectEqual(@as(usize, 1), calcWtf8Len(L("a")));
try testing.expectEqual(@as(usize, 10), calcWtf8Len(L("abcdefghij")));
// 未配对的代理项
try testing.expectEqual(@as(usize, 3), calcWtf8Len(&[_]u16{
mem.nativeToLittle(u16, 0xD800),
}));
try testing.expectEqual(@as(usize, 15), calcWtf8Len(L("こんにちは")));
// 分别编码为 1、2、3、4 字节的首个码点
try testing.expectEqual(@as(usize, 1 + 2 + 3 + 4), calcWtf8Len(L("\u{0}\u{80}\u{800}\u{10000}")));
}
test "calculate wtf8 string length of given wtf16 string" {
try testCalcWtf8Len();
try comptime testCalcWtf8Len();
}