/
string.ex
3093 lines (2327 loc) · 90.9 KB
/
string.ex
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
import Kernel, except: [length: 1]
defmodule String do
@moduledoc ~S"""
Strings in Elixir are UTF-8 encoded binaries.
Strings in Elixir are a sequence of Unicode characters,
typically written between double quoted strings, such
as `"hello"` and `"héllò"`.
In case a string must have a double-quote in itself,
the double quotes must be escaped with a backslash,
for example: `"this is a string with \"double quotes\""`.
You can concatenate two strings with the `<>/2` operator:
iex> "hello" <> " " <> "world"
"hello world"
The functions in this module act according to
[The Unicode Standard, Version 15.1.0](http://www.unicode.org/versions/Unicode15.1.0/).
## Interpolation
Strings in Elixir also support interpolation. This allows
you to place some value in the middle of a string by using
the `#{}` syntax:
iex> name = "joe"
iex> "hello #{name}"
"hello joe"
Any Elixir expression is valid inside the interpolation.
If a string is given, the string is interpolated as is.
If any other value is given, Elixir will attempt to convert
it to a string using the `String.Chars` protocol. This
allows, for example, to output an integer from the interpolation:
iex> "2 + 2 = #{2 + 2}"
"2 + 2 = 4"
In case the value you want to interpolate cannot be
converted to a string, because it doesn't have a human
textual representation, a protocol error will be raised.
## Escape characters
Besides allowing double-quotes to be escaped with a backslash,
strings also support the following escape characters:
* `\0` - Null byte
* `\a` - Bell
* `\b` - Backspace
* `\t` - Horizontal tab
* `\n` - Line feed (New lines)
* `\v` - Vertical tab
* `\f` - Form feed
* `\r` - Carriage return
* `\e` - Command Escape
* `\s` - Space
* `\#` - Returns the `#` character itself, skipping interpolation
* `\\` - Single backslash
* `\xNN` - A byte represented by the hexadecimal `NN`
* `\uNNNN` - A Unicode code point represented by `NNNN`
* `\u{NNNNNN}` - A Unicode code point represented by `NNNNNN`
Note it is generally not advised to use `\xNN` in Elixir
strings, as introducing an invalid byte sequence would
make the string invalid. If you have to introduce a
character by its hexadecimal representation, it is best
to work with Unicode code points, such as `\uNNNN`. In fact,
understanding Unicode code points can be essential when doing
low-level manipulations of string, so let's explore them in
detail next.
## Unicode and code points
In order to facilitate meaningful communication between computers
across multiple languages, a standard is required so that the ones
and zeros on one machine mean the same thing when they are transmitted
to another. The Unicode Standard acts as an official registry of
virtually all the characters we know: this includes characters from
classical and historical texts, emoji, and formatting and control
characters as well.
Unicode organizes all of the characters in its repertoire into code
charts, and each character is given a unique numerical index. This
numerical index is known as a Code Point.
In Elixir you can use a `?` in front of a character literal to reveal
its code point:
iex> ?a
97
iex> ?ł
322
Note that most Unicode code charts will refer to a code point by its
hexadecimal (hex) representation, e.g. `97` translates to `0061` in hex,
and we can represent any Unicode character in an Elixir string by
using the `\u` escape character followed by its code point number:
iex> "\u0061" === "a"
true
iex> 0x0061 = 97 = ?a
97
The hex representation will also help you look up information about a
code point, e.g. [https://codepoints.net/U+0061](https://codepoints.net/U+0061)
has a data sheet all about the lower case `a`, a.k.a. code point 97.
Remember you can get the hex presentation of a number by calling
`Integer.to_string/2`:
iex> Integer.to_string(?a, 16)
"61"
## UTF-8 encoded and encodings
Now that we understand what the Unicode standard is and what code points
are, we can finally talk about encodings. Whereas the code point is **what**
we store, an encoding deals with **how** we store it: encoding is an
implementation. In other words, we need a mechanism to convert the code
point numbers into bytes so they can be stored in memory, written to disk, and such.
Elixir uses UTF-8 to encode its strings, which means that code points are
encoded as a series of 8-bit bytes. UTF-8 is a **variable width** character
encoding that uses one to four bytes to store each code point. It is capable
of encoding all valid Unicode code points. Let's see an example:
iex> string = "héllo"
"héllo"
iex> String.length(string)
5
iex> byte_size(string)
6
Although the string above has 5 characters, it uses 6 bytes, as two bytes
are used to represent the character `é`.
## Grapheme clusters
This module also works with the concept of grapheme cluster
(from now on referenced as graphemes). Graphemes can consist
of multiple code points that may be perceived as a single character
by readers. For example, "é" can be represented either as a single
"e with acute" code point, as seen above in the string `"héllo"`,
or as the letter "e" followed by a "combining acute accent"
(two code points):
iex> string = "\u0065\u0301"
"é"
iex> byte_size(string)
3
iex> String.length(string)
1
iex> String.codepoints(string)
["e", "́"]
iex> String.graphemes(string)
["é"]
Although it looks visually the same as before, the example above
is made of two characters, it is perceived by users as one.
Graphemes can also be two characters that are interpreted as one
by some languages. For example, some languages may consider "ch"
as a single character. However, since this information depends on
the locale, it is not taken into account by this module.
In general, the functions in this module rely on the Unicode
Standard, but do not contain any of the locale specific behaviour.
More information about graphemes can be found in the [Unicode
Standard Annex #29](https://www.unicode.org/reports/tr29/).
For converting a binary to a different encoding and for Unicode
normalization mechanisms, see Erlang's `:unicode` module.
## String and binary operations
To act according to the Unicode Standard, many functions
in this module run in linear time, as they need to traverse
the whole string considering the proper Unicode code points.
For example, `String.length/1` will take longer as
the input grows. On the other hand, `Kernel.byte_size/1` always runs
in constant time (i.e. regardless of the input size).
This means often there are performance costs in using the
functions in this module, compared to the more low-level
operations that work directly with binaries:
* `Kernel.binary_part/3` - retrieves part of the binary
* `Kernel.bit_size/1` and `Kernel.byte_size/1` - size related functions
* `Kernel.is_bitstring/1` and `Kernel.is_binary/1` - type-check function
* Plus a number of functions for working with binaries (bytes)
in the [`:binary` module](`:binary`)
A `utf8` modifier is also available inside the binary syntax `<<>>`.
It can be used to match code points out of a binary/string:
iex> <<eacute::utf8>> = "é"
iex> eacute
233
You can also fully convert a string into a list of integer code points,
known as "charlists" in Elixir, by calling `String.to_charlist/1`:
iex> String.to_charlist("héllo")
[104, 233, 108, 108, 111]
If you would rather see the underlying bytes of a string, instead of
its codepoints, a common trick is to concatenate the null byte `<<0>>`
to it:
iex> "héllo" <> <<0>>
<<104, 195, 169, 108, 108, 111, 0>>
Alternatively, you can view a string's binary representation by
passing an option to `IO.inspect/2`:
IO.inspect("héllo", binaries: :as_binaries)
#=> <<104, 195, 169, 108, 108, 111>>
## Self-synchronization
The UTF-8 encoding is self-synchronizing. This means that
if malformed data (i.e., data that is not possible according
to the definition of the encoding) is encountered, only one
code point needs to be rejected.
This module relies on this behaviour to ignore such invalid
characters. For example, `length/1` will return
a correct result even if an invalid code point is fed into it.
In other words, this module expects invalid data to be detected
elsewhere, usually when retrieving data from the external source.
For example, a driver that reads strings from a database will be
responsible to check the validity of the encoding. `String.chunk/2`
can be used for breaking a string into valid and invalid parts.
## Compile binary patterns
Many functions in this module work with patterns. For example,
`String.split/3` can split a string into multiple strings given
a pattern. This pattern can be a string, a list of strings or
a compiled pattern:
iex> String.split("foo bar", " ")
["foo", "bar"]
iex> String.split("foo bar!", [" ", "!"])
["foo", "bar", ""]
iex> pattern = :binary.compile_pattern([" ", "!"])
iex> String.split("foo bar!", pattern)
["foo", "bar", ""]
The compiled pattern is useful when the same match will
be done over and over again. Note though that the compiled
pattern cannot be stored in a module attribute as the pattern
is generated at runtime and does not survive compile time.
"""
@typedoc """
A UTF-8 encoded binary.
The types `String.t()` and `binary()` are equivalent to analysis tools.
Although, for those reading the documentation, `String.t()` implies
it is a UTF-8 encoded binary.
"""
@type t :: binary
@typedoc "A single Unicode code point encoded in UTF-8. It may be one or more bytes."
@type codepoint :: t
@typedoc "Multiple code points that may be perceived as a single character by readers"
@type grapheme :: t
@typedoc """
Pattern used in functions like `replace/4` and `split/3`.
It must be one of:
* a string
* an empty list
* a list containing non-empty strings
* a compiled search pattern created by `:binary.compile_pattern/1`
"""
@type pattern ::
t()
| [nonempty_binary]
| (compiled_search_pattern :: :binary.cp())
@conditional_mappings [:greek, :turkic]
@doc """
Checks if a string contains only printable characters up to `character_limit`.
Takes an optional `character_limit` as a second argument. If `character_limit` is `0`, this
function will return `true`.
## Examples
iex> String.printable?("abc")
true
iex> String.printable?("abc" <> <<0>>)
false
iex> String.printable?("abc" <> <<0>>, 2)
true
iex> String.printable?("abc" <> <<0>>, 0)
true
"""
@spec printable?(t, 0) :: true
@spec printable?(t, pos_integer | :infinity) :: boolean
def printable?(string, character_limit \\ :infinity)
when is_binary(string) and
(character_limit == :infinity or
(is_integer(character_limit) and character_limit >= 0)) do
recur_printable?(string, character_limit)
end
defp recur_printable?(_string, 0), do: true
defp recur_printable?(<<>>, _character_limit), do: true
for char <- 0x20..0x7E do
defp recur_printable?(<<unquote(char), rest::binary>>, character_limit) do
recur_printable?(rest, decrement(character_limit))
end
end
for char <- [?\n, ?\r, ?\t, ?\v, ?\b, ?\f, ?\e, ?\d, ?\a] do
defp recur_printable?(<<unquote(char), rest::binary>>, character_limit) do
recur_printable?(rest, decrement(character_limit))
end
end
defp recur_printable?(<<char::utf8, rest::binary>>, character_limit)
when char in 0xA0..0xD7FF
when char in 0xE000..0xFFFD
when char in 0x10000..0x10FFFF do
recur_printable?(rest, decrement(character_limit))
end
defp recur_printable?(_string, _character_limit) do
false
end
defp decrement(:infinity), do: :infinity
defp decrement(character_limit), do: character_limit - 1
@doc ~S"""
Divides a string into substrings at each Unicode whitespace
occurrence with leading and trailing whitespace ignored. Groups
of whitespace are treated as a single occurrence. Divisions do
not occur on non-breaking whitespace.
## Examples
iex> String.split("foo bar")
["foo", "bar"]
iex> String.split("foo" <> <<194, 133>> <> "bar")
["foo", "bar"]
iex> String.split(" foo bar ")
["foo", "bar"]
iex> String.split("no\u00a0break")
["no\u00a0break"]
"""
@spec split(t) :: [t]
defdelegate split(binary), to: String.Break
@doc ~S"""
Divides a string into parts based on a pattern.
Returns a list of these parts.
The `pattern` may be a string, a list of strings, a regular expression, or a
compiled pattern.
The string is split into as many parts as possible by
default, but can be controlled via the `:parts` option.
Empty strings are only removed from the result if the
`:trim` option is set to `true`.
When the pattern used is a regular expression, the string is
split using `Regex.split/3`.
## Options
* `:parts` (positive integer or `:infinity`) - the string
is split into at most as many parts as this option specifies.
If `:infinity`, the string will be split into all possible
parts. Defaults to `:infinity`.
* `:trim` (boolean) - if `true`, empty strings are removed from
the resulting list.
This function also accepts all options accepted by `Regex.split/3`
if `pattern` is a regular expression.
## Examples
Splitting with a string pattern:
iex> String.split("a,b,c", ",")
["a", "b", "c"]
iex> String.split("a,b,c", ",", parts: 2)
["a", "b,c"]
iex> String.split(" a b c ", " ", trim: true)
["a", "b", "c"]
A list of patterns:
iex> String.split("1,2 3,4", [" ", ","])
["1", "2", "3", "4"]
A regular expression:
iex> String.split("a,b,c", ~r{,})
["a", "b", "c"]
iex> String.split("a,b,c", ~r{,}, parts: 2)
["a", "b,c"]
iex> String.split(" a b c ", ~r{\s}, trim: true)
["a", "b", "c"]
iex> String.split("abc", ~r{b}, include_captures: true)
["a", "b", "c"]
A compiled pattern:
iex> pattern = :binary.compile_pattern([" ", ","])
iex> String.split("1,2 3,4", pattern)
["1", "2", "3", "4"]
Splitting on empty string returns graphemes:
iex> String.split("abc", "")
["", "a", "b", "c", ""]
iex> String.split("abc", "", trim: true)
["a", "b", "c"]
iex> String.split("abc", "", parts: 1)
["abc"]
iex> String.split("abc", "", parts: 3)
["", "a", "bc"]
Be aware that this function can split within or across grapheme boundaries.
For example, take the grapheme "é" which is made of the characters
"e" and the acute accent. The following will split the string into two parts:
iex> String.split(String.normalize("é", :nfd), "e")
["", "́"]
However, if "é" is represented by the single character "e with acute"
accent, then it will split the string into just one part:
iex> String.split(String.normalize("é", :nfc), "e")
["é"]
"""
@spec split(t, pattern | Regex.t(), keyword) :: [t]
def split(string, pattern, options \\ [])
def split(string, %Regex{} = pattern, options) when is_binary(string) and is_list(options) do
Regex.split(pattern, string, options)
end
def split(string, "", options) when is_binary(string) and is_list(options) do
parts = Keyword.get(options, :parts, :infinity)
index = parts_to_index(parts)
trim = Keyword.get(options, :trim, false)
if trim == false and index != 1 do
["" | split_empty(string, trim, index - 1)]
else
split_empty(string, trim, index)
end
end
def split(string, [], options) when is_binary(string) and is_list(options) do
if string == "" and Keyword.get(options, :trim, false) do
[]
else
[string]
end
end
def split(string, pattern, options) when is_binary(string) and is_list(options) do
parts = Keyword.get(options, :parts, :infinity)
trim = Keyword.get(options, :trim, false)
case {parts, trim} do
{:infinity, false} ->
:binary.split(string, pattern, [:global])
{:infinity, true} ->
:binary.split(string, pattern, [:global, :trim_all])
{2, false} ->
:binary.split(string, pattern)
_ ->
pattern = maybe_compile_pattern(pattern)
split_each(string, pattern, trim, parts_to_index(parts))
end
end
defp parts_to_index(:infinity), do: 0
defp parts_to_index(n) when is_integer(n) and n > 0, do: n
defp split_empty("", true, 1), do: []
defp split_empty(string, _, 1), do: [IO.iodata_to_binary(string)]
defp split_empty(string, trim, count) do
case :unicode_util.gc(string) do
[gc] -> [grapheme_to_binary(gc) | split_empty("", trim, 1)]
[gc | rest] -> [grapheme_to_binary(gc) | split_empty(rest, trim, count - 1)]
[] -> split_empty("", trim, 1)
{:error, <<byte, rest::bits>>} -> [<<byte>> | split_empty(rest, trim, count - 1)]
end
end
defp split_each("", _pattern, true, 1), do: []
defp split_each(string, _pattern, _trim, 1) when is_binary(string), do: [string]
defp split_each(string, pattern, trim, count) do
case do_splitter(string, pattern, trim) do
{h, t} -> [h | split_each(t, pattern, trim, count - 1)]
nil -> []
end
end
@doc """
Returns an enumerable that splits a string on demand.
This is in contrast to `split/3` which splits the
entire string upfront.
This function does not support regular expressions
by design. When using regular expressions, it is often
more efficient to have the regular expressions traverse
the string at once than in parts, like this function does.
## Options
* :trim - when `true`, does not emit empty patterns
## Examples
iex> String.splitter("1,2 3,4 5,6 7,8,...,99999", [" ", ","]) |> Enum.take(4)
["1", "2", "3", "4"]
iex> String.splitter("abcd", "") |> Enum.take(10)
["", "a", "b", "c", "d", ""]
iex> String.splitter("abcd", "", trim: true) |> Enum.take(10)
["a", "b", "c", "d"]
A compiled pattern can also be given:
iex> pattern = :binary.compile_pattern([" ", ","])
iex> String.splitter("1,2 3,4 5,6 7,8,...,99999", pattern) |> Enum.take(4)
["1", "2", "3", "4"]
"""
@spec splitter(t, pattern, keyword) :: Enumerable.t()
def splitter(string, pattern, options \\ [])
def splitter(string, "", options) when is_binary(string) and is_list(options) do
if Keyword.get(options, :trim, false) do
Stream.unfold(string, &next_grapheme/1)
else
Stream.unfold(:match, &do_empty_splitter(&1, string))
end
end
def splitter(string, [], options) when is_binary(string) and is_list(options) do
if string == "" and Keyword.get(options, :trim, false) do
Stream.duplicate(string, 0)
else
Stream.duplicate(string, 1)
end
end
def splitter(string, pattern, options) when is_binary(string) and is_list(options) do
pattern = maybe_compile_pattern(pattern)
trim = Keyword.get(options, :trim, false)
Stream.unfold(string, &do_splitter(&1, pattern, trim))
end
defp do_empty_splitter(:match, string), do: {"", string}
defp do_empty_splitter(:nomatch, _string), do: nil
defp do_empty_splitter("", _), do: {"", :nomatch}
defp do_empty_splitter(string, _), do: next_grapheme(string)
defp do_splitter(:nomatch, _pattern, _), do: nil
defp do_splitter("", _pattern, false), do: {"", :nomatch}
defp do_splitter("", _pattern, true), do: nil
defp do_splitter(bin, pattern, trim) do
case :binary.split(bin, pattern) do
["", second] when trim -> do_splitter(second, pattern, trim)
[first, second] -> {first, second}
[first] -> {first, :nomatch}
end
end
defp maybe_compile_pattern(pattern) when is_tuple(pattern), do: pattern
defp maybe_compile_pattern(pattern), do: :binary.compile_pattern(pattern)
@doc """
Splits a string into two at the specified offset. When the offset given is
negative, location is counted from the end of the string.
The offset is capped to the length of the string. Returns a tuple with
two elements.
Note: keep in mind this function splits on graphemes and for such it
has to linearly traverse the string. If you want to split a string or
a binary based on the number of bytes, use `Kernel.binary_part/3`
instead.
## Examples
iex> String.split_at("sweetelixir", 5)
{"sweet", "elixir"}
iex> String.split_at("sweetelixir", -6)
{"sweet", "elixir"}
iex> String.split_at("abc", 0)
{"", "abc"}
iex> String.split_at("abc", 1000)
{"abc", ""}
iex> String.split_at("abc", -1000)
{"", "abc"}
"""
@spec split_at(t, integer) :: {t, t}
def split_at(string, position)
def split_at(string, position)
when is_binary(string) and is_integer(position) and position >= 0 do
do_split_at(string, position)
end
def split_at(string, position)
when is_binary(string) and is_integer(position) and position < 0 do
position = length(string) + position
case position >= 0 do
true -> do_split_at(string, position)
false -> {"", string}
end
end
defp do_split_at(string, position) do
remaining = byte_size_remaining_at(string, position)
start = byte_size(string) - remaining
<<left::size(^start)-binary, right::size(^remaining)-binary>> = string
{left, right}
end
@doc ~S"""
Returns `true` if `string1` is canonically equivalent to `string2`.
It performs Normalization Form Canonical Decomposition (NFD) on the
strings before comparing them. This function is equivalent to:
String.normalize(string1, :nfd) == String.normalize(string2, :nfd)
If you plan to compare multiple strings, multiple times in a row, you
may normalize them upfront and compare them directly to avoid multiple
normalization passes.
## Examples
iex> String.equivalent?("abc", "abc")
true
iex> String.equivalent?("man\u0303ana", "mañana")
true
iex> String.equivalent?("abc", "ABC")
false
iex> String.equivalent?("nø", "nó")
false
"""
@spec equivalent?(t, t) :: boolean
def equivalent?(string1, string2) when is_binary(string1) and is_binary(string2) do
normalize(string1, :nfd) == normalize(string2, :nfd)
end
@doc """
Converts all characters in `string` to Unicode normalization
form identified by `form`.
Invalid Unicode codepoints are skipped and the remaining of
the string is converted. If you want the algorithm to stop
and return on invalid codepoint, use `:unicode.characters_to_nfd_binary/1`,
`:unicode.characters_to_nfc_binary/1`, `:unicode.characters_to_nfkd_binary/1`,
and `:unicode.characters_to_nfkc_binary/1` instead.
Normalization forms `:nfkc` and `:nfkd` should not be blindly applied
to arbitrary text. Because they erase many formatting distinctions,
they will prevent round-trip conversion to and from many legacy
character sets.
## Forms
The supported forms are:
* `:nfd` - Normalization Form Canonical Decomposition.
Characters are decomposed by canonical equivalence, and
multiple combining characters are arranged in a specific
order.
* `:nfc` - Normalization Form Canonical Composition.
Characters are decomposed and then recomposed by canonical equivalence.
* `:nfkd` - Normalization Form Compatibility Decomposition.
Characters are decomposed by compatibility equivalence, and
multiple combining characters are arranged in a specific
order.
* `:nfkc` - Normalization Form Compatibility Composition.
Characters are decomposed and then recomposed by compatibility equivalence.
## Examples
iex> String.normalize("yêṩ", :nfd)
"yêṩ"
iex> String.normalize("leña", :nfc)
"leña"
iex> String.normalize("fi", :nfkd)
"fi"
iex> String.normalize("fi", :nfkc)
"fi"
"""
@spec normalize(t, :nfd | :nfc | :nfkd | :nfkc) :: t
def normalize(string, form)
def normalize(string, :nfd) when is_binary(string) do
case :unicode.characters_to_nfd_binary(string) do
string when is_binary(string) -> string
{:error, good, <<head, rest::binary>>} -> good <> <<head>> <> normalize(rest, :nfd)
end
end
def normalize(string, :nfc) when is_binary(string) do
case :unicode.characters_to_nfc_binary(string) do
string when is_binary(string) -> string
{:error, good, <<head, rest::binary>>} -> good <> <<head>> <> normalize(rest, :nfc)
end
end
def normalize(string, :nfkd) when is_binary(string) do
case :unicode.characters_to_nfkd_binary(string) do
string when is_binary(string) -> string
{:error, good, <<head, rest::binary>>} -> good <> <<head>> <> normalize(rest, :nfkd)
end
end
def normalize(string, :nfkc) when is_binary(string) do
case :unicode.characters_to_nfkc_binary(string) do
string when is_binary(string) -> string
{:error, good, <<head, rest::binary>>} -> good <> <<head>> <> normalize(rest, :nfkc)
end
end
@doc """
Converts all characters in the given string to uppercase according to `mode`.
`mode` may be `:default`, `:ascii`, `:greek` or `:turkic`. The `:default` mode considers
all non-conditional transformations outlined in the Unicode standard. `:ascii`
uppercases only the letters a to z. `:greek` includes the context sensitive
mappings found in Greek. `:turkic` properly handles the letter i with the dotless variant.
## Examples
iex> String.upcase("abcd")
"ABCD"
iex> String.upcase("ab 123 xpto")
"AB 123 XPTO"
iex> String.upcase("olá")
"OLÁ"
The `:ascii` mode ignores Unicode characters and provides a more
performant implementation when you know the string contains only
ASCII characters:
iex> String.upcase("olá", :ascii)
"OLá"
And `:turkic` properly handles the letter i with the dotless variant:
iex> String.upcase("ıi")
"II"
iex> String.upcase("ıi", :turkic)
"Iİ"
Also see `downcase/2` and `capitalize/2` for other conversions.
"""
@spec upcase(t, :default | :ascii | :greek | :turkic) :: t
def upcase(string, mode \\ :default)
def upcase("", _mode) do
""
end
def upcase(string, :default) when is_binary(string) do
String.Unicode.upcase(string, [], :default)
end
def upcase(string, :ascii) when is_binary(string) do
IO.iodata_to_binary(upcase_ascii(string))
end
def upcase(string, mode) when is_binary(string) and mode in @conditional_mappings do
String.Unicode.upcase(string, [], mode)
end
defp upcase_ascii(<<char, rest::bits>>) when char >= ?a and char <= ?z,
do: [char - 32 | upcase_ascii(rest)]
defp upcase_ascii(<<char, rest::bits>>), do: [char | upcase_ascii(rest)]
defp upcase_ascii(<<>>), do: []
@doc """
Converts all characters in the given string to lowercase according to `mode`.
`mode` may be `:default`, `:ascii`, `:greek` or `:turkic`. The `:default` mode considers
all non-conditional transformations outlined in the Unicode standard. `:ascii`
lowercases only the letters A to Z. `:greek` includes the context sensitive
mappings found in Greek. `:turkic` properly handles the letter i with the dotless variant.
Also see `upcase/2` and `capitalize/2` for other conversions.
## Examples
iex> String.downcase("ABCD")
"abcd"
iex> String.downcase("AB 123 XPTO")
"ab 123 xpto"
iex> String.downcase("OLÁ")
"olá"
The `:ascii` mode ignores Unicode characters and provides a more
performant implementation when you know the string contains only
ASCII characters:
iex> String.downcase("OLÁ", :ascii)
"olÁ"
The `:greek` mode properly handles the context sensitive sigma in Greek:
iex> String.downcase("ΣΣ")
"σσ"
iex> String.downcase("ΣΣ", :greek)
"σς"
And `:turkic` properly handles the letter i with the dotless variant:
iex> String.downcase("Iİ")
"ii̇"
iex> String.downcase("Iİ", :turkic)
"ıi"
"""
@spec downcase(t, :default | :ascii | :greek | :turkic) :: t
def downcase(string, mode \\ :default)
def downcase("", _mode) do
""
end
def downcase(string, :default) when is_binary(string) do
String.Unicode.downcase(string, [], :default)
end
def downcase(string, :ascii) when is_binary(string) do
IO.iodata_to_binary(downcase_ascii(string))
end
def downcase(string, mode) when is_binary(string) and mode in @conditional_mappings do
String.Unicode.downcase(string, [], mode)
end
defp downcase_ascii(<<char, rest::bits>>) when char >= ?A and char <= ?Z,
do: [char + 32 | downcase_ascii(rest)]
defp downcase_ascii(<<char, rest::bits>>), do: [char | downcase_ascii(rest)]
defp downcase_ascii(<<>>), do: []
@doc """
Converts the first character in the given string to
uppercase and the remainder to lowercase according to `mode`.
`mode` may be `:default`, `:ascii`, `:greek` or `:turkic`. The `:default` mode
considers all non-conditional transformations outlined in the Unicode standard.
`:ascii` capitalizes only the letters A to Z. `:greek` includes the context
sensitive mappings found in Greek. `:turkic` properly handles the letter `i`
with the dotless variant.
Also see `upcase/2` and `capitalize/2` for other conversions. If you want
a variation of this function that does not lowercase the rest of string,
see Erlang's `:string.titlecase/1`.
## Examples
iex> String.capitalize("abcd")
"Abcd"
iex> String.capitalize("ABCD")
"Abcd"
iex> String.capitalize("fin")
"Fin"
iex> String.capitalize("olá")
"Olá"
"""
@spec capitalize(t, :default | :ascii | :greek | :turkic) :: t
def capitalize(string, mode \\ :default)
def capitalize(<<char, rest::binary>>, :ascii) do
char = if char >= ?a and char <= ?z, do: char - 32, else: char
<<char>> <> downcase(rest, :ascii)
end
@letter_I <<0x0049::utf8>>
@letter_i <<0x0069::utf8>>
@letter_I_dot_above <<0x0130::utf8>>
def capitalize(<<@letter_i, right::binary>>, mode) do
if(mode == :turkic, do: @letter_I_dot_above, else: @letter_I) <> downcase(right, mode)
end
def capitalize(string, mode) when is_binary(string) do
case :unicode_util.gc(string) do
[gc] -> grapheme_to_binary(:string.titlecase([gc]))
[gc, rest] -> grapheme_to_binary(:string.titlecase([gc])) <> downcase(rest, mode)
[gc | rest] -> grapheme_to_binary(:string.titlecase([gc])) <> downcase(rest, mode)
[] -> ""
{:error, <<byte, rest::bits>>} -> <<byte>> <> downcase(rest, mode)
end
end
@doc false
@deprecated "Use String.trim_trailing/1 instead"
defdelegate rstrip(binary), to: String.Break, as: :trim_trailing
@doc false
@deprecated "Use String.trim_trailing/2 with a binary as second argument instead"
def rstrip(string, char) when is_integer(char) do
replace_trailing(string, <<char::utf8>>, "")
end
@doc """
Replaces all leading occurrences of `match` by `replacement` of `match` in `string`.
Returns the string untouched if there are no occurrences.
If `match` is `""`, this function raises an `ArgumentError` exception: this
happens because this function replaces **all** the occurrences of `match` at
the beginning of `string`, and it's impossible to replace "multiple"
occurrences of `""`.
## Examples
iex> String.replace_leading("hello world", "hello ", "")
"world"