Data commit

This commit is contained in:
Ingy döt Net 2023-07-01 11:58:00 -04:00
parent 7387c8f97b
commit cb5bb5e222
199093 changed files with 3378972 additions and 0 deletions

View file

@ -0,0 +1,5 @@
Module ByteLength
Function GetByteLength(s As String, encoding As Text.Encoding) As Integer
Return encoding.GetByteCount(s)
End Function
End Module

View file

@ -0,0 +1,20 @@
Module CharacterLength
Function GetUTF16CodeUnitsLength(s As String) As Integer
Return s.Length
End Function
Private Function GetUTF16SurrogatePairCount(s As String) As Integer
GetUTF16SurrogatePairCount = 0
For i = 1 To s.Length - 1
If Char.IsSurrogatePair(s(i - 1), s(i)) Then GetUTF16SurrogatePairCount += 1
Next
End Function
Function GetCharacterLength_FromUTF16(s As String) As Integer
Return GetUTF16CodeUnitsLength(s) - GetUTF16SurrogatePairCount(s)
End Function
Function GetCharacterLength_FromUTF32(s As String) As Integer
Return GetByteLength(s, Text.Encoding.UTF32) \ 4
End Function
End Module

View file

@ -0,0 +1,13 @@
Module GraphemeLength
' Wraps an IEnumerator, allowing it to be used as an IEnumerable.
Private Iterator Function AsEnumerable(enumerator As IEnumerator) As IEnumerable
Do While enumerator.MoveNext()
Yield enumerator.Current
Loop
End Function
Function GraphemeCount(s As String) As Integer
Dim elements = Globalization.StringInfo.GetTextElementEnumerator(s)
Return AsEnumerable(elements).OfType(Of String).Count()
End Function
End Module

View file

@ -0,0 +1,39 @@
#Const PRINT_TESTCASE = True
Module Program
ReadOnly TestCases As String() =
{
"Hello, world!",
"møøse",
"𝔘𝔫𝔦𝔠𝔬𝔡𝔢", ' String normalization of the file makes the e and diacritic in é̲ one character, so use VB's char "escapes"
$"J{ChrW(&H332)}o{ChrW(&H332)}s{ChrW(&H332)}e{ChrW(&H301)}{ChrW(&H332)}"
}
Sub Main()
Const INDENT = " "
Console.OutputEncoding = Text.Encoding.Unicode
Dim writeResult = Sub(s As String, result As Integer) Console.WriteLine("{0}{1,-20}{2}", INDENT, s, result)
For i = 0 To TestCases.Length - 1
Dim c = TestCases(i)
Console.Write("Test case " & i)
#If PRINT_TESTCASE Then
Console.WriteLine(": " & c)
#Else
Console.WriteLine()
#End If
writeResult("graphemes", GraphemeCount(c))
writeResult("UTF-16 units", GetUTF16CodeUnitsLength(c))
writeResult("Cd pts from UTF-16", GetCharacterLength_FromUTF16(c))
writeResult("Cd pts from UTF-32", GetCharacterLength_FromUTF32(c))
Console.WriteLine()
writeResult("bytes (UTF-8)", GetByteLength(c, Text.Encoding.UTF8))
writeResult("bytes (UTF-16)", GetByteLength(c, Text.Encoding.Unicode))
writeResult("bytes (UTF-32)", GetByteLength(c, Text.Encoding.UTF32))
Console.WriteLine()
Next
End Sub
End Module