uutf.mli

(*---------------------------------------------------------------------------
   Copyright (c) 2012 The uutf programmers. All rights reserved.
   Distributed under the ISC license, see terms at the end of the file.
  ---------------------------------------------------------------------------*)

(** Non-blocking streaming Unicode codec.

  [Uutf] is a non-blocking streaming codec to {{:#decode}decode} and
  {{:#encode}encode} the {{:http://www.ietf.org/rfc/rfc3629.txt}
  UTF-8}, {{:http://www.ietf.org/rfc/rfc2781.txt} UTF-16}, UTF-16LE
  and UTF-16BE encoding schemes. It can efficiently work character by
  character without blocking on IO. Decoders perform
  character position tracking and support {{!nln}newline normalization}.

  Functions are also provided to {{!String} fold over} the characters
  of UTF encoded OCaml string values and to {{!Buffer}directly encode}
  characters in OCaml {!Stdlib.Buffer.t} values. {b Note} that since OCaml
  4.14, that functionality can be found in {!Stdlib.String} and
  {!Stdlib.Buffer} and you are encouraged to migrate to it.

  See {{:#examples}examples} of use.

  {b References}
  {ul
  {- The Unicode Consortium.
  {e {{:http://www.unicode.org/versions/latest}The Unicode Standard}}.
  (latest version)}}
*)

  (** {1:ucharcsts Special Unicode characters} *)

val u_bom : Uchar.t
(** [u_bom] is the {{:http://unicode.org/glossary/#byte_order_mark}byte
    order mark} (BOM) character ([U+FEFF]). From OCaml 4.06 on, use
    {!Uchar.bom}. *)

val u_rep : Uchar.t
(** [u_rep] is the
    {{:http://unicode.org/glossary/#replacement_character}replacement}
    character ([U+FFFD]). From OCaml 4.06 on, use
    {!Uchar.rep}. *)


(** {1:schemes Unicode encoding schemes} *)

type encoding = [ `UTF_16 | `UTF_16BE | `UTF_16LE | `UTF_8 ]
(** The type for Unicode
    {{:http://unicode.org/glossary/#character_encoding_scheme}encoding
    schemes}. *)

type decoder_encoding = [ encoding | `US_ASCII | `ISO_8859_1 ]
(** The type for encoding schemes {e decoded} by [Uutf]. Unicode encoding
    schemes plus {{:http://tools.ietf.org/html/rfc20}US-ASCII} and
    {{:http://www.ecma-international.org/publications/standards/Ecma-094.htm}
    ISO/IEC 8859-1} (latin-1). *)

val encoding_of_string : string -> decoder_encoding option
(** [encoding_of_string s] converts a (case insensitive)
    {{:http://www.iana.org/assignments/character-sets}IANA character set name}
    to an encoding. *)

val encoding_to_string : [< decoder_encoding] -> string
(** [encoding_to_string e] is a
    {{:http://www.iana.org/assignments/character-sets}IANA character set name}
    for [e]. *)

(** {1:decode Decode} *)

type src = [ `Channel of in_channel | `String of string | `Manual ]
(** The type for input sources. With a [`Manual] source the client
    must provide input with {!Manual.src}. *)

type nln = [ `ASCII of Uchar.t | `NLF of Uchar.t | `Readline of Uchar.t ]
(** The type for newline normalizations. The variant argument is the
    normalization character.
    {ul
    {- [`ASCII], normalizes CR ([U+000D]), LF ([U+000A]) and CRLF
       (<[U+000D], [U+000A]>).}
    {- [`NLF], normalizes the Unicode newline function (NLF). This is
       NEL ([U+0085]) and the normalizations of [`ASCII].}
    {- [`Readline], normalizes for a Unicode readline function. This is FF
       ([U+000C]), LS ([U+2028]), PS ([U+2029]), and the normalizations
       of [`NLF].}}
    Used with an appropriate normalization character the [`NLF] and
    [`Readline] normalizations allow to implement all the different
    recommendations of Unicode's newline guidelines (section 5.8 in
    Unicode 9.0.0). *)

type decoder
(** The type for decoders. *)

val decoder : ?nln:[< nln] -> ?encoding:[< decoder_encoding] -> [< src] ->
  decoder
(** [decoder nln encoding src] is a decoder that inputs from [src].

    {b Byte order mark.}
    {{:http://unicode.org/glossary/#byte_order_mark}Byte order mark}
    (BOM) constraints are application dependent and prone to
    misunderstandings (see the
    {{:http://www.unicode.org/faq/utf_bom.html#BOM}FAQ}). Hence,
    [Uutf] decoders have a simple rule: an {e initial BOM is always
    removed from the input and not counted in character position
    tracking}. The function {!decoder_removed_bom} does however return
    [true] if a BOM was removed so that all the information can be
    recovered if needed.

    For UTF-16BE and UTF-16LE the above rule is a violation of
    conformance D96 and D97 of the standard. [Uutf] favors the idea
    that if there's a BOM, decoding with [`UTF_16] or the [`UTF_16XX]
    corresponding to the BOM should decode the same character sequence
    (this is not the case if you stick to the standard). The client
    can however regain conformance by consulting the result of
    {!decoder_removed_bom} and take appropriate action.

    {b Encoding.} [encoding] specifies the decoded encoding
    scheme. If [`UTF_16] is used the endianness is determined
    according to the standard: from a
    {{:http://unicode.org/glossary/#byte_order_mark}BOM}
    if there is one, [`UTF_16BE] otherwise.

    If [encoding] is unspecified it is guessed. The result of a guess
    can only be [`UTF_8], [`UTF_16BE] or [`UTF_16LE]. The heuristic
    looks at the first three bytes of input (or less if impossible)
    and takes the {e first} matching byte pattern in the table below.
{v
xx = any byte
.. = any byte or no byte (input too small)
pp = positive byte
uu = valid UTF-8 first byte

Bytes    | Guess     | Rationale
---------+-----------+-----------------------------------------------
EF BB BF | `UTF_8    | UTF-8 BOM
FE FF .. | `UTF_16BE | UTF-16BE BOM
FF FE .. | `UTF_16LE | UTF-16LE BOM
00 pp .. | `UTF_16BE | ASCII UTF-16BE and U+0000 is often forbidden
pp 00 .. | `UTF_16LE | ASCII UTF-16LE and U+0000 is often forbidden
uu .. .. | `UTF_8    | ASCII UTF-8 or valid UTF-8 first byte.
xx xx .. | `UTF_16BE | Not UTF-8 => UTF-16, no BOM => UTF-16BE
.. .. .. | `UTF_8    | Single malformed UTF-8 byte or no input.
v}
    This heuristic is compatible both with BOM based
    recognitition and
    {{:http://tools.ietf.org/html/rfc4627#section-3}JSON-like encoding
    recognition} that relies on ASCII being present at the beginning
    of the stream. Also, {!decoder_removed_bom} will tell the client
    if the guess was BOM based.

    {b Newline normalization.} If [nln] is specified, the given
    newline normalization is performed, see {!nln}. Otherwise
    all newlines are returned as found in the input.

    {b Character position.} The line number, column number, byte count
    and character count of the last decoded character (including
    [`Malformed] ones) are respectively returned by {!decoder_line},
    {!decoder_col}, {!decoder_byte_count} and {!decoder_count}. Before
    the first call to {!val-decode} the line number is [1] and the column
    is [0].  Each {!val-decode} returning [`Uchar] or [`Malformed]
    increments the column until a newline.  On a newline, the line
    number is incremented and the column set to zero. For example the
    line is [2] and column [0] after the first newline was
    decoded. This can be understood as if {!val-decode} was moving an
    insertion point to the right in the data.  A {e newline} is
    anything normalized by [`Readline], see {!nln}.

    [Uutf] assumes that each Unicode scalar value has a column width
    of 1. The same assumption may not be made by the display program
    (e.g. for [emacs]' compilation mode you need to set
    [compilation-error-screen-columns] to [nil]). The problem is in
    general difficult to solve without interaction or convention with the
    display program's rendering engine. Depending on the context better column
    increments can be implemented by using {!Uucp.Break.tty_width_hint} or
    {{:http://unicode.org/reports/tr29/#Grapheme_Cluster_Boundaries}
    grapheme cluster boundaries} (see {!Uuseg}). *)

val decode : decoder ->
  [ `Await | `Uchar of Uchar.t | `End | `Malformed of string]
(** [decode d] is:
    {ul
    {- [`Await] if [d] has a [`Manual] input source and awaits
       for more input. The client must use {!Manual.src} to provide it.}
    {- [`Uchar u] if a Unicode scalar value [u] was decoded.}
    {- [`End] if the end of input was reached.}
    {- [`Malformed bytes] if the [bytes] sequence is malformed according to
       the decoded encoding scheme. If you are interested in a best-effort
       decoding you can still continue to decode after an error until the
       decoder synchronizes again on valid bytes. It may however be a good
       idea to signal the malformed characters by adding an {!u_rep}
       character to the parsed data, see the {{:#examples}examples}.}}

    {b Note.} Repeated invocation always eventually returns [`End], even
    in case of errors.  *)

val decoder_encoding : decoder -> decoder_encoding
(** [decoder_encoding d] is [d]'s the decoded encoding scheme of [d].

    {b Warning.} If the decoder guesses the encoding or uses [`UTF_16],
    rely on this value only after the first [`Uchar] was decoded. *)

(**/**)

(* This function is dangerous, it may destroy the current continuation.
   But it's needed for things like XML parsers. *)

val set_decoder_encoding : decoder -> [< decoder_encoding] -> unit
(** [set_decoder_encoding d enc] changes the decoded encoding
    to [enc] after decoding started.

    {b Warning.} Call only after {!val-decode} was called on [d] and that the
    last call to it returned something different from [`Await] or data may
    be lost. After encoding guess wait for at least three [`Uchar]s. *)

(**/**)

val decoder_line : decoder -> int
(** [decoder_line d] is the line number of the last
    decoded (or malformed) character. See {!val-decoder} for details. *)

val decoder_col : decoder -> int
(** [decoder_col d] is the column number of the last decoded
    (or malformed) character. See {!val-decoder} for details. *)

val decoder_byte_count : decoder -> int
(** [decoder_byte_count d] is the number of bytes already decoded on
    [d] (including malformed ones). This is the last {!val-decode}'s
    end byte offset counting from the beginning of the stream. *)

val decoder_count : decoder -> int
(** [decoder_count d] is the number of characters already decoded on [d]
    (including malformed ones). See {!val-decoder} for details. *)

val decoder_removed_bom : decoder -> bool
(** [decoder_removed_bom d] is [true] iff an {e initial}
    {{:http://unicode.org/glossary/#byte_order_mark}BOM} was
    removed from the input stream. See {!val-decoder} for details. *)

val decoder_src : decoder -> src
(** [decoder_src d] is [d]'s input source. *)

val decoder_nln : decoder -> nln option
(** [decoder_nln d] returns [d]'s newline normalization (if any). *)

val pp_decode : Format.formatter ->
  [< `Await | `Uchar of Uchar.t | `End | `Malformed of string] -> unit
(** [pp_decode ppf v] prints an unspecified representation of [v] on
    [ppf]. *)

(** {1:encode Encode} *)

type dst = [ `Channel of out_channel | `Buffer of Buffer.t | `Manual ]
(** The type for output destinations. With a [`Manual] destination the client
    must provide output storage with {!Manual.dst}. *)

type encoder
(** The type for Unicode encoders. *)

val encoder : [< encoding] -> [< dst] -> encoder
(** [encoder encoding dst] is an encoder for [encoding] that outputs
    to [dst].

    {b Note.} No initial
    {{:http://unicode.org/glossary/#byte_order_mark}BOM}
    is encoded. If needed, this duty is left to the client. *)

val encode :
  encoder -> [<`Await | `End | `Uchar of Uchar.t ] -> [`Ok | `Partial ]
(** [encode e v] is :
    {ul
    {- [`Partial] iff [e] has a [`Manual] destination and needs more output
       storage. The client must use {!Manual.dst} to provide a new buffer
       and then call {!val-encode} with [`Await] until [`Ok] is returned.}
    {- [`Ok] when the encoder is ready to encode a new [`Uchar] or [`End]}}

    For [`Manual] destination, encoding [`End] always returns
    [`Partial], the client should continue as usual with [`Await]
    until [`Ok] is returned at which point {!Manual.dst_rem} [e] is
    guaranteed to be the size of the last provided buffer (i.e. nothing
    was written).

    {b Raises.} [Invalid_argument] if an [`Uchar] or [`End] is encoded
    after a [`Partial] encode. *)

val encoder_encoding : encoder -> encoding
(** [encoder_encoding e] is [e]'s encoding. *)

val encoder_dst : encoder -> dst
(** [encoder_dst e] is [e]'s output destination. *)

(** {1:manual Manual sources and destinations.} *)

(** Manual sources and destinations.

    {b Warning.} Use only with [`Manual] decoder and encoders. *)
module Manual : sig
  val src : decoder -> Bytes.t -> int -> int -> unit
  (** [src d s j l] provides [d] with [l] bytes to read, starting at
      [j] in [s]. This byte range is read by calls to {!val-decode} with [d]
      until [`Await] is returned. To signal the end of input call the function
      with [l = 0]. *)

  val dst : encoder -> Bytes.t -> int -> int -> unit
  (** [dst e s j l] provides [e] with [l] bytes to write, starting
      at [j] in [s]. This byte range is written by calls to
      {!val-encode} with [e] until [`Partial] is returned. Use {!dst_rem} to
      know the remaining number of non-written free bytes in [s]. *)

  val dst_rem : encoder -> int
  (** [dst_rem e] is the remaining number of non-written, free bytes
      in the last buffer provided with {!Manual.dst}. *)
end

(** {1:strbuf String folders and Buffer encoders} *)

(** Fold over the characters of UTF encoded OCaml [string] values.

    {b Note.} Since OCaml 4.14, UTF decoders are available in
    {!Stdlib.String}. You are encouraged to migrate to them. *)
module String : sig

(** {1 Encoding guess} *)

  val encoding_guess : string -> [ `UTF_8 | `UTF_16BE | `UTF_16LE ] * bool
  (** [encoding_guess s] is the encoding guessed for [s] coupled with
      [true] iff there's an initial
      {{:http://unicode.org/glossary/#byte_order_mark}BOM}. *)

(** {1 String folders}

    {b Note.} Initial {{:http://unicode.org/glossary/#byte_order_mark}BOM}s
    are also folded over. *)

  type 'a folder = 'a -> int -> [ `Uchar of Uchar.t | `Malformed of string ] ->
    'a
  (** The type for character folders. The integer is the index in the
      string where the [`Uchar] or [`Malformed] starts. *)

  val fold_utf_8 : ?pos:int -> ?len:int -> 'a folder -> 'a -> string -> 'a
  (** [fold_utf_8 f a s ?pos ?len ()] is
      [f (] ... [(f (f a pos u]{_0}[) j]{_1}[ u]{_1}[)] ... [)] ... [)
      j]{_n}[ u]{_n}
      where [u]{_i}, [j]{_i} are characters and their start position
      in the UTF-8 encoded substring [s] starting at [pos] and [len]
      long. The default value for [pos] is [0] and [len] is
      [String.length s - pos]. *)

  val fold_utf_16be : ?pos:int -> ?len:int -> 'a folder -> 'a -> string -> 'a
  (** [fold_utf_16be f a s ?pos ?len ()] is
      [f (] ... [(f (f a pos u]{_0}[) j]{_1}[ u]{_1}[)] ... [)] ... [)
      j]{_n}[ u]{_n}
      where [u]{_i}, [j]{_i} are characters and their start position
      in the UTF-8 encoded substring [s] starting at [pos] and [len]
      long. The default value for [pos] is [0] and [len] is
      [String.length s - pos]. *)

  val fold_utf_16le : ?pos:int -> ?len:int -> 'a folder -> 'a -> string -> 'a
  (** [fold_utf_16le f a s ?pos ?len ()] is
      [f (] ... [(f (f a pos u]{_0}[) j]{_1}[ u]{_1}[)] ... [)] ... [)
      j]{_n}[ u]{_n}
      where [u]{_i}, [j]{_i} are characters and their start position
      in the UTF-8 encoded substring [s] starting at [pos] and [len]
      long. The default value for [pos] is [0] and [len] is
      [String.length s - pos]. *)
end

(**  UTF encode characters in OCaml {!Buffer.t} values.

     {b Note.} Since OCaml 4.06, these encoders are available in
     {!Stdlib.Buffer}. You are encouraged to migrate to them. *)
module Buffer : sig

  (** {1 Buffer encoders} *)

  val add_utf_8 : Buffer.t -> Uchar.t -> unit
  (** [add_utf_8 b u] adds the UTF-8 encoding of [u] to [b]. *)

  val add_utf_16be : Buffer.t -> Uchar.t -> unit
  (** [add_utf_16be b u] adds the UTF-16BE encoding of [u] to [b]. *)

  val add_utf_16le : Buffer.t -> Uchar.t -> unit
  (** [add_utf_16le b u] adds the UTF-16LE encoding of [u] to [b]. *)
end

(** {1:examples Examples}

    {2:readlines Read lines}

    The value of [lines src] is the list of lines in [src] as UTF-8
    encoded OCaml strings. Line breaks are determined according to the
    recommendation R4 for a [readline] function in section 5.8 of
    Unicode 9.0.0. If a decoding error occurs we silently replace the
    malformed sequence by the replacement character {!u_rep} and continue.
{[let lines ?encoding (src : [`Channel of in_channel | `String of string]) =
  let rec loop d buf acc = match Uutf.decode d with
  | `Uchar u ->
      begin match Uchar.to_int u with
      | 0x000A ->
          let line = Buffer.contents buf in
          Buffer.clear buf; loop d buf (line :: acc)
      | _ ->
          Uutf.Buffer.add_utf_8 buf u; loop d buf acc
      end
  | `End -> List.rev (Buffer.contents buf :: acc)
  | `Malformed _ -> Uutf.Buffer.add_utf_8 buf Uutf.u_rep; loop d buf acc
  | `Await -> assert false
  in
  let nln = `Readline (Uchar.of_int 0x000A) in
  loop (Uutf.decoder ~nln ?encoding src) (Buffer.create 512) []
]}
  Using the [`Manual] interface, [lines_fd] does the same but on a Unix file
  descriptor.
{[let lines_fd ?encoding (fd : Unix.file_descr) =
  let rec loop fd s d buf acc = match Uutf.decode d with
  | `Uchar u ->
      begin match Uchar.to_int u with
      | 0x000A ->
          let line = Buffer.contents buf in
          Buffer.clear buf; loop fd s d buf (line :: acc)
      | _ ->
          Uutf.Buffer.add_utf_8 buf u; loop fd s d buf acc
      end
  | `End -> List.rev (Buffer.contents buf :: acc)
  | `Malformed _ -> Uutf.Buffer.add_utf_8 buf Uutf.u_rep; loop fd s d buf acc
  | `Await ->
      let rec unix_read fd s j l = try Unix.read fd s j l with
      | Unix.Unix_error (Unix.EINTR, _, _) -> unix_read fd s j l
      in
      let rc = unix_read fd s 0 (Bytes.length s) in
      Uutf.Manual.src d s 0 rc; loop fd s d buf acc
  in
  let s = Bytes.create 65536 (* UNIX_BUFFER_SIZE in 4.0.0 *) in
  let nln = `Readline (Uchar.of_int 0x000A) in
  loop fd s (Uutf.decoder ~nln ?encoding `Manual) (Buffer.create 512) []
]}

    {2:recode Recode}

    The result of [recode src out_encoding dst] has the characters of
    [src] written on [dst] with encoding [out_encoding].  If a
    decoding error occurs we silently replace the malformed sequence
    by the replacement character {!u_rep} and continue.  Note that we
    don't add an initial
    {{:http://unicode.org/glossary/#byte_order_mark}BOM} to [dst],
    recoding will thus loose the initial BOM [src] may have. Whether
    this is a problem or not depends on the context.
{[let recode ?nln ?encoding out_encoding
    (src : [`Channel of in_channel | `String of string])
    (dst : [`Channel of out_channel | `Buffer of Buffer.t])
  =
  let rec loop d e = match Uutf.decode d with
  | `Uchar _ as u -> ignore (Uutf.encode e u); loop d e
  | `End -> ignore (Uutf.encode e `End)
  | `Malformed _ -> ignore (Uutf.encode e (`Uchar Uutf.u_rep)); loop d e
  | `Await -> assert false
  in
  let d = Uutf.decoder ?nln ?encoding src in
  let e = Uutf.encoder out_encoding dst in
  loop d e]}
  Using the [`Manual] interface, [recode_fd] does the same but between
  Unix file descriptors.
{[let recode_fd ?nln ?encoding out_encoding
    (fdi : Unix.file_descr)
    (fdo : Unix.file_descr)
  =
  let rec encode fd s e v = match Uutf.encode e v with `Ok -> ()
  | `Partial ->
      let rec unix_write fd s j l =
        let rec write fd s j l = try Unix.single_write fd s j l with
        | Unix.Unix_error (Unix.EINTR, _, _) -> write fd s j l
        in
        let wc = write fd s j l in
        if wc < l then unix_write fd s (j + wc) (l - wc) else ()
      in
      unix_write fd s 0 (Bytes.length s - Uutf.Manual.dst_rem e);
      Uutf.Manual.dst e s 0 (Bytes.length s);
      encode fd s e `Await
  in
  let rec loop fdi fdo ds es d e = match Uutf.decode d with
  | `Uchar _ as u -> encode fdo es e u; loop fdi fdo ds es d e
  | `End -> encode fdo es e `End
  | `Malformed _ -> encode fdo es e (`Uchar Uutf.u_rep); loop fdi fdo ds es d e
  | `Await ->
      let rec unix_read fd s j l = try Unix.read fd s j l with
      | Unix.Unix_error (Unix.EINTR, _, _) -> unix_read fd s j l
      in
      let rc = unix_read fdi ds 0 (Bytes.length ds) in
      Uutf.Manual.src d ds 0 rc; loop fdi fdo ds es d e
  in
  let ds = Bytes.create 65536 (* UNIX_BUFFER_SIZE in 4.0.0 *) in
  let es = Bytes.create 65536 (* UNIX_BUFFER_SIZE in 4.0.0 *) in
  let d = Uutf.decoder ?nln ?encoding `Manual in
  let e = Uutf.encoder out_encoding `Manual in
  Uutf.Manual.dst e es 0 (Bytes.length es);
  loop fdi fdo ds es d e]}
*)

(*---------------------------------------------------------------------------
   Copyright (c) 2012 The uutf programmers

   Permission to use, copy, modify, and/or distribute this software for any
   purpose with or without fee is hereby granted, provided that the above
   copyright notice and this permission notice appear in all copies.

   THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
   WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
   MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
   ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
   WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
   ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
   OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  ---------------------------------------------------------------------------*)