1# utf8 - text as code points.2#3# A tau string is a run of bytes, and indexing or slicing one counts bytes,4# the way Go does it. The text inside is UTF-8, so a letter outside ASCII5# takes more than one byte and len counts them all:6#7# utf8 = import("unicode/utf8")8# len("città ") # 6, the bytes9# utf8.RuneCount("città ") # 5, the letters10#11# A code point is a number here, what Go calls a rune. Decoding one at a time12# over a string is faster on bytes than on the string, because a string has to13# be converted first:14#15# b = bytes(s)16# i = 017# for i < len(b) {18# r = utf8.DecodeRune(b, i)19# println(r[0])20# i = i + r[1]21# }2223# RuneError is what a byte that isn't valid UTF-8 decodes to, the replacement24# character U+FFFD.25RuneError = 0xfffd2627# RuneSelf is the first code point that doesn't fit in a single byte.28RuneSelf = 0x802930# MaxRune is the largest code point there is.31MaxRune = 0x10ffff3233# UTFMax is the most bytes one code point takes.34UTFMax = 43536# The surrogate range belongs to UTF-16 and is not valid on its own.37surrogateMin = 0xd80038surrogateMax = 0xdfff3940asBytes = fn(s) {41 if type(s) == "bytes" {42 return s43 }44 return bytes(string(s))45}4647# ValidRune reports whether r is a code point that can be encoded.48ValidRune = fn(r) {49 if r < 0 || r > MaxRune {50 return false51 }52 return r < surrogateMin || r > surrogateMax53}5455# RuneLen returns how many bytes the code point r takes, or -1 when it cannot56# be encoded.57RuneLen = fn(r) {58 if !ValidRune(r) {59 return -160 }61 if r < 0x80 {62 return 163 }64 if r < 0x800 {65 return 266 }67 if r < 0x10000 {68 return 369 }70 return 471}7273# EncodeRune returns the string of the code point r. An r that cannot be74# encoded gives RuneError, as it does in Go.75EncodeRune = fn(r) {76 if !ValidRune(r) {77 r = RuneError78 }7980 if r < 0x80 {81 return string(bytes([r]))82 }83 if r < 0x800 {84 return string(bytes([0xc0 | (r >> 6), 0x80 | (r & 0x3f)]))85 }86 if r < 0x10000 {87 return string(bytes([88 0xe0 | (r >> 12),89 0x80 | ((r >> 6) & 0x3f),90 0x80 | (r & 0x3f)91 ]))92 }93 return string(bytes([94 0xf0 | (r >> 18),95 0x80 | ((r >> 12) & 0x3f),96 0x80 | ((r >> 6) & 0x3f),97 0x80 | (r & 0x3f)98 ]))99}100101# cont returns the six bits a continuation byte carries, or -1 when b isn't102# one.103cont = fn(b) {104 if (b & 0xc0) != 0x80 {105 return -1106 }107 return b & 0x3f108}109110# bad is what a decode returns when the bytes are not UTF-8: one byte eaten,111# so a loop over them always moves on.112bad = [RuneError, 1]113114# DecodeRune returns [code point, bytes read] for the character starting at115# offset i of s, which is a string or bytes. Bytes that are not valid UTF-8116# decode to [RuneError, 1].117DecodeRune = fn(s, i) {118 b = asBytes(s)119 if i < 0 || i >= len(b) {120 return [RuneError, 0]121 }122123 c = b[i]124 if c < 0x80 {125 return [c, 1]126 }127128 # 110xxxxx: two bytes, down to 0x80 or it is a shorter one written long.129 if (c & 0xe0) == 0xc0 {130 if i + 1 >= len(b) {131 return bad132 }133 c1 = cont(b[i + 1])134 if c1 < 0 {135 return bad136 }137 r = ((c & 0x1f) << 6) | c1138 if r < 0x80 {139 return bad140 }141 return [r, 2]142 }143144 # 1110xxxx: three bytes, and not a surrogate.145 if (c & 0xf0) == 0xe0 {146 if i + 2 >= len(b) {147 return bad148 }149 c1 = cont(b[i + 1])150 c2 = cont(b[i + 2])151 if c1 < 0 || c2 < 0 {152 return bad153 }154 r = ((c & 0x0f) << 12) | (c1 << 6) | c2155 if r < 0x800 || (r >= surrogateMin && r <= surrogateMax) {156 return bad157 }158 return [r, 3]159 }160161 # 11110xxx: four bytes, up to the last code point there is.162 if (c & 0xf8) == 0xf0 {163 if i + 3 >= len(b) {164 return bad165 }166 c1 = cont(b[i + 1])167 c2 = cont(b[i + 2])168 c3 = cont(b[i + 3])169 if c1 < 0 || c2 < 0 || c3 < 0 {170 return bad171 }172 r = ((c & 0x07) << 18) | (c1 << 12) | (c2 << 6) | c3173 if r < 0x10000 || r > MaxRune {174 return bad175 }176 return [r, 4]177 }178179 return bad180}181182# Runes returns the code points of s, one number per character.183Runes = fn(s) {184 b = asBytes(s)185 out = []186 i = 0187 for i < len(b) {188 r = DecodeRune(b, i)189 out = append(out, r[0])190 i = i + r[1]191 }192 return out193}194195# FromRunes returns the string of the code points in rs.196FromRunes = fn(rs) {197 out = ""198 for i = 0; i < len(rs); ++i {199 out = out + EncodeRune(rs[i])200 }201 return out202}203204# RuneCount returns how many characters s holds, which is len(s) only while205# the text stays inside ASCII.206RuneCount = fn(s) {207 b = asBytes(s)208 n = 0209 i = 0210 for i < len(b) {211 i = i + DecodeRune(b, i)[1]212 ++n213 }214 return n215}216217# Valid reports whether s is UTF-8 from end to end.218Valid = fn(s) {219 b = asBytes(s)220 i = 0221 for i < len(b) {222 r = DecodeRune(b, i)223 if r[0] == RuneError && r[1] == 1 {224 # A real U+FFFD is three bytes long, one byte means a broken one.225 return false226 }227 i = i + r[1]228 }229 return true230}231232# RuneIndex returns the byte offset where character n starts, or -1 when the233# string is shorter than that. It is what to feed slice when a position is234# counted in characters rather than bytes.235RuneIndex = fn(s, n) {236 b = asBytes(s)237 i = 0238 for k = 0; k < n; ++k {239 if i >= len(b) {240 return -1241 }242 i = i + DecodeRune(b, i)[1]243 }244 if i > len(b) {245 return -1246 }247 return i248}249250# Slice returns the characters of s from start to end, counted in characters251# rather than bytes.252Slice = fn(s, start, end) {253 str = string(s)254 from = RuneIndex(str, start)255 if from < 0 {256 return error("utf8: Slice start {start} is past the end")257 }258259 to = RuneIndex(str, end)260 if to < 0 {261 return error("utf8: Slice end {end} is past the end")262 }263 if to < from {264 return error("utf8: Slice end {end} is before start {start}")265 }266 return slice(str, from, to)267}