diff --git a/packages/kiri/lib/kiri/backend/nif.ex b/packages/kiri/lib/kiri/backend/nif.ex index 50527a4..4326d44 100644 --- a/packages/kiri/lib/kiri/backend/nif.ex +++ b/packages/kiri/lib/kiri/backend/nif.ex @@ -13,13 +13,12 @@ defmodule Kiri.Backend.Nif do @behaviour Kiri.Backend + alias Kiri.Backend alias Kiri.Dictionary alias Kiri.Lattice.Node - alias Kiri.Morpheme alias Kiri.Native alias Kiri.Native.WordInfo alias Kiri.PathRewrite - alias Kiri.Types @impl true def tokenize(%Dictionary{} = dict, text, opts \\ []) do @@ -126,90 +125,8 @@ defmodule Kiri.Backend.Nif do defp build_morphemes(nodes, nif_ref, result, original_text, mode, resolve_fn) do bto = List.to_tuple(result.byte_to_original) - - nodes - |> maybe_split(nif_ref, mode, resolve_fn) - |> Enum.map(fn node -> - word_info = resolve_fn.(node) - - pos_id = - if node.is_oov and node.oov_pos_id != nil and node.oov_pos_id >= 0, - do: node.oov_pos_id, - else: word_info.pos_id - - pos = Native.full_tokenizer_get_pos(nif_ref, pos_id) - - orig_begin = elem(bto, node.begin) - orig_end = elem(bto, node.end) - - surface = - case word_info.surface do - "" -> - original_text - |> String.to_charlist() - |> Enum.slice(orig_begin..(orig_end - 1)//1) - |> List.to_string() - - s -> - s - end - - %Morpheme{ - begin: orig_begin, - end: orig_end, - surface: surface, - part_of_speech: pos, - part_of_speech_id: word_info.pos_id, - dictionary_form: word_info.dictionary_form, - normalized_form: word_info.normalized_form, - reading_form: word_info.reading_form, - is_oov: node.is_oov, - word_id: node.word_id, - dictionary_id: Types.dictionary_id(node.word_id), - synonym_group_ids: Map.get(word_info, :synonym_gids, []) - } - end) - end - - defp maybe_split(nodes, _nif_ref, :c, _resolve_fn), do: nodes - - defp maybe_split(nodes, nif_ref, mode, resolve_fn) when mode in [:a, :b] do - Enum.flat_map(nodes, fn node -> - word_info = resolve_fn.(node) - - splits = - case mode do - :a -> word_info.a_unit_split - :b -> word_info.b_unit_split - end - - case splits do - [] -> [node] - [single_id] when single_id == node.word_id -> [node] - split_ids -> expand_splits(nif_ref, node, split_ids) - end - end) - end - - defp expand_splits(nif_ref, parent_node, split_ids) do - Enum.reduce(split_ids, {parent_node.begin, []}, fn wid, {offset, acc} -> - wi = Native.full_tokenizer_get_word_info(nif_ref, wid) - wp_end = offset + wi.headword_length - - split_node = %Node{ - begin: offset, - end: wp_end, - word_id: wid, - left_id: 0, - right_id: 0, - cost: 0, - is_oov: false - } - - {wp_end, [split_node | acc]} - end) - |> elem(1) - |> Enum.reverse() + get_pos_fn = fn pos_id -> Native.full_tokenizer_get_pos(nif_ref, pos_id) end + Backend.build_morphemes(nodes, original_text, bto, mode, resolve_fn, get_pos_fn) end # Get or create a NIF tokenizer handle for the dictionary.