/*
  Copyright (C) 2012-2018  Brazil
  Copyright (C) 2018-2025  Sutou Kouhei <kou@clear-code.com>

  This library is free software; you can redistribute it and/or
  modify it under the terms of the GNU Lesser General Public
  License as published by the Free Software Foundation; either
  version 2.1 of the License, or (at your option) any later version.

  This library is distributed in the hope that it will be useful,
  but WITHOUT ANY WARRANTY; without even the implied warranty of
  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  Lesser General Public License for more details.

  You should have received a copy of the GNU Lesser General Public
  License along with this library; if not, write to the Free Software
  Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
*/

#include "grn.h"
#include "grn_tokenizer.h"

#include <string.h>

#include "grn_ctx.h"
#include "grn_db.h"
#include "grn_str.h"
#include "grn_string.h"
#include "grn_token_cursor.h"

/*
  Just for backward compatibility. See grn_plugin_charlen() instead.
 */
int
grn_tokenizer_charlen(grn_ctx *ctx,
                      const char *str_ptr,
                      unsigned int str_length,
                      grn_encoding encoding)
{
  return grn_plugin_charlen(ctx, str_ptr, str_length, encoding);
}

/*
  Just for backward compatibility. See grn_plugin_isspace() instead.
 */
int
grn_tokenizer_isspace(grn_ctx *ctx,
                      const char *str_ptr,
                      unsigned int str_length,
                      grn_encoding encoding)
{
  return grn_plugin_isspace(ctx, str_ptr, str_length, encoding);
}

bool
grn_tokenizer_is_tokenized_delimiter(grn_ctx *ctx,
                                     const char *str_ptr,
                                     unsigned int str_length,
                                     grn_encoding encoding)
{
  if (encoding != GRN_ENC_UTF8) {
    return false;
  }

  if (str_length != GRN_TOKENIZER_TOKENIZED_DELIMITER_UTF8_LEN) {
    return false;
  }

  return memcmp(str_ptr,
                GRN_TOKENIZER_TOKENIZED_DELIMITER_UTF8,
                GRN_TOKENIZER_TOKENIZED_DELIMITER_UTF8_LEN) == 0;
}

bool
grn_tokenizer_have_tokenized_delimiter(grn_ctx *ctx,
                                       const char *str_ptr,
                                       unsigned int str_length,
                                       grn_encoding encoding)
{
  int char_length;
  const char *current = str_ptr;
  const char *end = str_ptr + str_length;

  if (encoding != GRN_ENC_UTF8) {
    return false;
  }

  if (str_length == 0) {
    return false;
  }

  while ((char_length = grn_charlen_(ctx, current, end, encoding)) > 0) {
    if (grn_tokenizer_is_tokenized_delimiter(ctx,
                                             current,
                                             (unsigned int)char_length,
                                             encoding)) {
      return true;
    }
    current += char_length;
  }
  return false;
}

static void
grn_tokenizer_query_ensure_normalized(grn_ctx *ctx, grn_tokenizer_query *query)
{
  if (!query->need_normalize) {
    return;
  }

  query->need_normalize = false;

  if (query->normalized_query) {
    grn_obj_close(ctx, query->normalized_query);
  }
  query->normalized_query = grn_string_open_(ctx,
                                             query->data,
                                             query->size,
                                             query->lexicon,
                                             (int)(query->normalize_flags),
                                             query->encoding);
  if (!query->normalized_query) {
    query->have_tokenized_delimiter = false;
    GRN_PLUGIN_ERROR(ctx,
                     GRN_TOKENIZER_ERROR,
                     "[tokenizer][normalize] failed to open normalized string: "
                     "<%.*s>(%s)",
                     (int)(query->size),
                     query->data,
                     grn_encoding_to_string(query->encoding));
    return;
  }

  query->need_delimiter_check = true;
}

static void
grn_tokenizer_query_ensure_have_tokenized_delimiter(grn_ctx *ctx,
                                                    grn_tokenizer_query *query)
{
  grn_tokenizer_query_ensure_normalized(ctx, query);

  if (!query->need_delimiter_check) {
    return;
  }

  query->need_delimiter_check = false;

  if (query->flags & GRN_TOKEN_CURSOR_ENABLE_TOKENIZED_DELIMITER) {
    const char *normalized_string;
    unsigned int normalized_string_length;

    grn_string_get_normalized(ctx,
                              query->normalized_query,
                              &normalized_string,
                              &normalized_string_length,
                              NULL);
    query->have_tokenized_delimiter =
      grn_tokenizer_have_tokenized_delimiter(ctx,
                                             normalized_string,
                                             normalized_string_length,
                                             query->encoding);
  } else {
    query->have_tokenized_delimiter = false;
  }
}

grn_rc
grn_tokenizer_query_init(grn_ctx *ctx, grn_tokenizer_query *query)
{
  query->normalize_flags = 0;
  query->normalized_query = NULL;
  query->data = NULL;
  query->ptr = NULL;
  query->size = 0;
  query->flags = 0;
  query->tokenize_mode = GRN_TOKENIZE_ADD;
  query->token_mode = query->tokenize_mode;
  query->lexicon = NULL;
  query->encoding = ctx->encoding;
  query->token_filter_index = 0;
  query->source_column = NULL;
  query->source_id = GRN_ID_NIL;
  query->index_column = NULL;

  query->need_normalize = true;
  query->need_delimiter_check = true;

  query->options = NULL;

  query->domain = GRN_ID_NIL;

  return ctx->rc;
}

grn_tokenizer_query *
grn_tokenizer_query_open(grn_ctx *ctx,
                         int num_args,
                         grn_obj **args,
                         uint32_t normalize_flags)
{
  grn_obj *flags;
  grn_obj *query_str;
  grn_obj *tokenize_mode;

  GRN_API_ENTER;

  flags = grn_ctx_pop(ctx);
  query_str = grn_ctx_pop(ctx);
  tokenize_mode = grn_ctx_pop(ctx);

  if (query_str == NULL) {
    GRN_PLUGIN_ERROR(ctx, GRN_INVALID_ARGUMENT, "missing argument");
    GRN_API_RETURN(NULL);
  }

  if ((num_args < 1) || (args == NULL) || (args[0] == NULL)) {
    GRN_PLUGIN_ERROR(ctx, GRN_INVALID_ARGUMENT, "invalid NULL pointer");
    GRN_API_RETURN(NULL);
  }

  {
    grn_tokenizer_query *const query =
      GRN_PLUGIN_MALLOC(ctx, sizeof(grn_tokenizer_query));
    if (!query) {
      GRN_API_RETURN(NULL);
    }
    grn_tokenizer_query_init(ctx, query);
    grn_tokenizer_query_set_raw_string(ctx,
                                       query,
                                       GRN_TEXT_VALUE(query_str),
                                       GRN_TEXT_LEN(query_str));
    if (ctx->rc != GRN_SUCCESS) {
      GRN_PLUGIN_FREE(ctx, query);
      GRN_API_RETURN(NULL);
    }
    if (flags) {
      grn_tokenizer_query_set_flags(ctx, query, GRN_UINT32_VALUE(flags));
    }
    if (tokenize_mode) {
      grn_tokenizer_query_set_mode(ctx, query, GRN_UINT32_VALUE(tokenize_mode));
    }
    grn_tokenizer_query_set_normalize_flags(ctx, query, normalize_flags);
    grn_tokenizer_query_set_lexicon(ctx, query, args[0]);

    grn_tokenizer_query_ensure_have_tokenized_delimiter(ctx, query);

    GRN_API_RETURN(query);
  }
}

grn_tokenizer_query *
grn_tokenizer_query_create(grn_ctx *ctx, int num_args, grn_obj **args)
{
  return grn_tokenizer_query_open(ctx, num_args, args, 0);
}

void
grn_tokenizer_query_fin(grn_ctx *ctx, grn_tokenizer_query *query)
{
  if (query->normalized_query) {
    grn_obj_unlink(ctx, query->normalized_query);
  }
  if (query->data) {
    GRN_PLUGIN_FREE(ctx, query->data);
  }
}

void
grn_tokenizer_query_close(grn_ctx *ctx, grn_tokenizer_query *query)
{
  GRN_API_ENTER;
  if (query) {
    grn_tokenizer_query_fin(ctx, query);
    GRN_PLUGIN_FREE(ctx, query);
  }
  GRN_API_RETURN();
}

void
grn_tokenizer_query_destroy(grn_ctx *ctx, grn_tokenizer_query *query)
{
  grn_tokenizer_query_close(ctx, query);
}

grn_rc
grn_tokenizer_query_set_normalize_flags(grn_ctx *ctx,
                                        grn_tokenizer_query *query,
                                        uint32_t normalize_flags)
{
  GRN_API_ENTER;
  if (query->normalize_flags != normalize_flags) {
    query->normalize_flags = normalize_flags;
    query->need_normalize = true;
  }
  GRN_API_RETURN(ctx->rc);
}

uint32_t
grn_tokenizer_query_get_normalize_flags(grn_ctx *ctx,
                                        grn_tokenizer_query *query)
{
  GRN_API_ENTER;
  GRN_API_RETURN(query->normalize_flags);
}

grn_obj *
grn_tokenizer_query_get_normalized_string(grn_ctx *ctx,
                                          grn_tokenizer_query *query)
{
  GRN_API_ENTER;
  grn_tokenizer_query_ensure_normalized(ctx, query);
  GRN_API_RETURN(query->normalized_query);
}

grn_rc
grn_tokenizer_query_set_raw_string(grn_ctx *ctx,
                                   grn_tokenizer_query *query,
                                   const char *string,
                                   size_t string_length)
{
  return grn_tokenizer_query_set_data(ctx,
                                      query,
                                      string,
                                      string_length,
                                      GRN_DB_TEXT);
}

const char *
grn_tokenizer_query_get_raw_string(grn_ctx *ctx,
                                   grn_tokenizer_query *query,
                                   size_t *length)
{
  GRN_API_ENTER;
  if (length) {
    *length = query->size;
  }
  GRN_API_RETURN(query->data);
}

grn_rc
grn_tokenizer_query_set_data(grn_ctx *ctx,
                             grn_tokenizer_query *query,
                             const char *data,
                             size_t size,
                             grn_id domain)
{
  GRN_API_ENTER;

  if (query->data) {
    GRN_PLUGIN_FREE(ctx, query->data);
  }

  if (size == 0) {
    query->data = NULL;
  } else {
    if (grn_type_id_is_text_family(ctx, domain)) {
      query->data = (char *)GRN_PLUGIN_MALLOC(ctx, size + 1);
      if (!query->data) {
        GRN_PLUGIN_ERROR(ctx,
                         GRN_TOKENIZER_ERROR,
                         "[tokenizer][query] failed to duplicate query");
        GRN_API_RETURN(ctx->rc);
      }
      grn_memcpy(query->data, data, size);
      query->data[size] = '\0';
    } else {
      query->data = (char *)GRN_PLUGIN_MALLOC(ctx, size);
      if (!query->data) {
        GRN_PLUGIN_ERROR(ctx,
                         GRN_TOKENIZER_ERROR,
                         "[tokenizer][query] failed to duplicate query");
        GRN_API_RETURN(ctx->rc);
      }
      grn_memcpy(query->data, data, size);
    }
  }
  query->size = size;
  query->ptr = query->data;
  query->domain = domain;
  query->need_normalize = true;

  GRN_API_RETURN(ctx->rc);
}

const char *
grn_tokenizer_query_get_data(grn_ctx *ctx,
                             grn_tokenizer_query *query,
                             size_t *size,
                             grn_id *domain)
{
  GRN_API_ENTER;
  if (size) {
    *size = query->size;
  }
  if (domain) {
    *domain = query->domain;
  }
  GRN_API_RETURN(query->data);
}

grn_encoding
grn_tokenizer_query_get_encoding(grn_ctx *ctx, grn_tokenizer_query *query)
{
  GRN_API_ENTER;
  GRN_API_RETURN(query->encoding);
}

grn_rc
grn_tokenizer_query_set_flags(grn_ctx *ctx,
                              grn_tokenizer_query *query,
                              uint32_t flags)
{
  GRN_API_ENTER;
  if (query->flags != flags) {
    query->flags = flags;
    query->need_normalize = true;
  }
  GRN_API_RETURN(ctx->rc);
}

uint32_t
grn_tokenizer_query_get_flags(grn_ctx *ctx, grn_tokenizer_query *query)
{
  GRN_API_ENTER;
  GRN_API_RETURN(query->flags);
}

bool
grn_tokenizer_query_have_tokenized_delimiter(grn_ctx *ctx,
                                             grn_tokenizer_query *query)
{
  GRN_API_ENTER;
  grn_tokenizer_query_ensure_have_tokenized_delimiter(ctx, query);
  GRN_API_RETURN(query->have_tokenized_delimiter);
}

grn_rc
grn_tokenizer_query_set_mode(grn_ctx *ctx,
                             grn_tokenizer_query *query,
                             grn_tokenize_mode mode)
{
  GRN_API_ENTER;
  if (query->tokenize_mode != mode) {
    query->tokenize_mode = mode;
    query->token_mode = query->tokenize_mode;
    query->need_normalize = true;
  }
  GRN_API_RETURN(ctx->rc);
}

grn_tokenize_mode
grn_tokenizer_query_get_mode(grn_ctx *ctx, grn_tokenizer_query *query)
{
  GRN_API_ENTER;
  GRN_API_RETURN(query->tokenize_mode);
}

grn_rc
grn_tokenizer_query_set_lexicon(grn_ctx *ctx,
                                grn_tokenizer_query *query,
                                grn_obj *lexicon)
{
  GRN_API_ENTER;

  if (query->lexicon != lexicon) {
    query->lexicon = lexicon;
    if (query->lexicon) {
      grn_table_get_info(ctx,
                         query->lexicon,
                         NULL,
                         &(query->encoding),
                         NULL,
                         NULL,
                         NULL);
    } else {
      query->encoding = ctx->encoding;
    }
    query->need_normalize = true;
  }

  GRN_API_RETURN(ctx->rc);
}

grn_obj *
grn_tokenizer_query_get_lexicon(grn_ctx *ctx, grn_tokenizer_query *query)
{
  GRN_API_ENTER;
  GRN_API_RETURN(query->lexicon);
}

grn_rc
grn_tokenizer_query_set_token_filter_index(grn_ctx *ctx,
                                           grn_tokenizer_query *query,
                                           unsigned int index)
{
  GRN_API_ENTER;
  query->token_filter_index = index;
  GRN_API_RETURN(ctx->rc);
}

unsigned int
grn_tokenizer_query_get_token_filter_index(grn_ctx *ctx,
                                           grn_tokenizer_query *query)
{
  GRN_API_ENTER;
  GRN_API_RETURN(query->token_filter_index);
}

grn_rc
grn_tokenizer_query_set_source_column(grn_ctx *ctx,
                                      grn_tokenizer_query *query,
                                      grn_obj *column)
{
  GRN_API_ENTER;
  query->source_column = column;
  GRN_API_RETURN(ctx->rc);
}

grn_obj *
grn_tokenizer_query_get_source_column(grn_ctx *ctx, grn_tokenizer_query *query)
{
  GRN_API_ENTER;
  GRN_API_RETURN(query->source_column);
}

grn_rc
grn_tokenizer_query_set_source_id(grn_ctx *ctx,
                                  grn_tokenizer_query *query,
                                  grn_id id)
{
  GRN_API_ENTER;
  query->source_id = id;
  GRN_API_RETURN(ctx->rc);
}

grn_id
grn_tokenizer_query_get_source_id(grn_ctx *ctx, grn_tokenizer_query *query)
{
  GRN_API_ENTER;
  GRN_API_RETURN(query->source_id);
}

grn_obj *
grn_tokenizer_query_get_index_column(grn_ctx *ctx, grn_tokenizer_query *query)
{
  GRN_API_ENTER;
  GRN_API_RETURN(query->index_column);
}

grn_rc
grn_tokenizer_query_set_index_column(grn_ctx *ctx,
                                     grn_tokenizer_query *query,
                                     grn_obj *column)
{
  GRN_API_ENTER;
  query->index_column = column;
  GRN_API_RETURN(ctx->rc);
}

grn_obj *
grn_tokenizer_query_get_options(grn_ctx *ctx, grn_tokenizer_query *query)
{
  GRN_API_ENTER;
  GRN_API_RETURN(query->options);
}

grn_rc
grn_tokenizer_query_set_options(grn_ctx *ctx,
                                grn_tokenizer_query *query,
                                grn_obj *options)
{
  GRN_API_ENTER;
  query->options = options;
  GRN_API_RETURN(ctx->rc);
}

void
grn_tokenizer_token_init(grn_ctx *ctx, grn_tokenizer_token *token)
{
  GRN_TEXT_INIT(&token->str, GRN_OBJ_DO_SHALLOW_COPY);
  GRN_UINT32_INIT(&token->status, 0);
}

void
grn_tokenizer_token_fin(grn_ctx *ctx, grn_tokenizer_token *token)
{
  GRN_OBJ_FIN(ctx, &(token->str));
  GRN_OBJ_FIN(ctx, &(token->status));
}

void
grn_tokenizer_token_push(grn_ctx *ctx,
                         grn_tokenizer_token *token,
                         const char *str_ptr,
                         unsigned int str_length,
                         grn_token_status status)
{
  GRN_TEXT_SET_REF(&token->str, str_ptr, str_length);
  GRN_UINT32_SET(ctx, &token->status, status);
  grn_ctx_push(ctx, &token->str);
  grn_ctx_push(ctx, &token->status);
}

const char *
grn_tokenizer_tokenized_delimiter_next(grn_ctx *ctx,
                                       grn_tokenizer_token *token,
                                       const char *str_ptr,
                                       unsigned int str_length,
                                       grn_encoding encoding)
{
  const char *start = str_ptr;
  const char *current;
  const char *end = str_ptr + str_length;
  const char *next_start = NULL;

  int char_length = 0;
  for (current = start; current < end; current += char_length) {
    char_length = grn_charlen_(ctx, current, end, encoding);
    if (char_length == 0) {
      break;
    }
    if (grn_tokenizer_is_tokenized_delimiter(ctx,
                                             current,
                                             (unsigned int)char_length,
                                             encoding)) {
      next_start = str_ptr + (current - start + char_length);
      break;
    }
  }

  ptrdiff_t token_length = current - start;
  grn_token_status status;
  if (current == end) {
    status = GRN_TOKENIZER_LAST;
  } else {
    status = GRN_TOKENIZER_CONTINUE;
  }
  grn_tokenizer_token_push(ctx,
                           token,
                           start,
                           (unsigned int)token_length,
                           status);

  return next_start;
}

const char *
grn_tokenizer_next_by_tokenized_delimiter(grn_ctx *ctx,
                                          grn_token *token,
                                          const char *str_ptr,
                                          unsigned int str_length,
                                          grn_encoding encoding)
{
  const char *start = str_ptr;
  const char *current;
  const char *end = str_ptr + str_length;
  const char *next_start = NULL;

  int char_length = 0;
  for (current = start; current < end; current += char_length) {
    char_length = grn_charlen_(ctx, current, end, encoding);
    if (char_length == 0) {
      break;
    }
    if (grn_tokenizer_is_tokenized_delimiter(ctx,
                                             current,
                                             (unsigned int)char_length,
                                             encoding)) {
      next_start = str_ptr + (current - start + char_length);
      break;
    }
  }

  grn_token_set_data(ctx, token, start, (int)(current - start));
  if (current == end) {
    grn_token_set_status(ctx, token, GRN_TOKEN_LAST);
  } else {
    grn_token_set_status(ctx, token, GRN_TOKEN_CONTINUE);
  }

  return next_start;
}

grn_rc
grn_tokenizer_register(grn_ctx *ctx,
                       const char *plugin_name_ptr,
                       unsigned int plugin_name_length,
                       grn_proc_func *init,
                       grn_proc_func *next,
                       grn_proc_func *fin)
{
  grn_expr_var vars[3];
  memset(vars, 0, sizeof(vars));
  GRN_TEXT_INIT(&vars[0].value, 0);
  GRN_TEXT_INIT(&vars[1].value, 0);
  GRN_UINT32_INIT(&vars[2].value, 0);

  {
    /*
      grn_proc_create() registers a plugin to the database which is associated
      with `ctx'. A returned object must not be finalized here.
     */
    grn_obj *const obj = grn_proc_create(ctx,
                                         plugin_name_ptr,
                                         (int)plugin_name_length,
                                         GRN_PROC_TOKENIZER,
                                         init,
                                         next,
                                         fin,
                                         3,
                                         vars);
    if (obj == NULL) {
      GRN_PLUGIN_ERROR(ctx, GRN_TOKENIZER_ERROR, "grn_proc_create() failed");
      return ctx->rc;
    }
  }
  return GRN_SUCCESS;
}

grn_obj *
grn_tokenizer_create(grn_ctx *ctx, const char *name, int name_length)
{
  grn_obj *tokenizer;

  GRN_API_ENTER;
  tokenizer = grn_proc_create(ctx,
                              name,
                              name_length,
                              GRN_PROC_TOKENIZER,
                              NULL,
                              NULL,
                              NULL,
                              0,
                              NULL);
  if (!tokenizer) {
    if (name_length < 0) {
      name_length = (int)strlen(name);
    }
    GRN_PLUGIN_ERROR(ctx,
                     GRN_TOKENIZER_ERROR,
                     "[tokenizer][create] failed to create: <%.*s>",
                     name_length,
                     name);
  }

  GRN_API_RETURN(tokenizer);
}

grn_rc
grn_tokenizer_set_build_func(grn_ctx *ctx,
                             grn_obj *tokenizer,
                             grn_tokenizer_build_func *build)
{
  GRN_API_ENTER;
  if (tokenizer) {
    ((grn_proc *)tokenizer)->callbacks.tokenizer.build = build;
  } else {
    GRN_PLUGIN_ERROR(ctx,
                     GRN_INVALID_ARGUMENT,
                     "[tokenizer][build][set] tokenizer is NULL");
  }
  GRN_API_RETURN(ctx->rc);
}

grn_rc
grn_tokenizer_set_init_func(grn_ctx *ctx,
                            grn_obj *tokenizer,
                            grn_tokenizer_init_func *init)
{
  GRN_API_ENTER;
  if (tokenizer) {
    ((grn_proc *)tokenizer)->callbacks.tokenizer.init = init;
  } else {
    GRN_PLUGIN_ERROR(ctx,
                     GRN_INVALID_ARGUMENT,
                     "[tokenizer][init][set] tokenizer is NULL");
  }
  GRN_API_RETURN(ctx->rc);
}

grn_rc
grn_tokenizer_set_next_func(grn_ctx *ctx,
                            grn_obj *tokenizer,
                            grn_tokenizer_next_func *next)
{
  GRN_API_ENTER;
  if (tokenizer) {
    ((grn_proc *)tokenizer)->callbacks.tokenizer.next = next;
  } else {
    GRN_PLUGIN_ERROR(ctx,
                     GRN_INVALID_ARGUMENT,
                     "[tokenizer][next][set] tokenizer is NULL");
  }
  GRN_API_RETURN(ctx->rc);
}

grn_rc
grn_tokenizer_set_fin_func(grn_ctx *ctx,
                           grn_obj *tokenizer,
                           grn_tokenizer_fin_func *fin)
{
  GRN_API_ENTER;
  if (tokenizer) {
    ((grn_proc *)tokenizer)->callbacks.tokenizer.fin = fin;
  } else {
    GRN_PLUGIN_ERROR(ctx,
                     GRN_INVALID_ARGUMENT,
                     "[tokenizer][fin][set] tokenizer is NULL");
  }
  GRN_API_RETURN(ctx->rc);
}

bool
grn_tokenizer_have_build_func(grn_ctx *ctx, grn_obj *tokenizer)
{
  return ((grn_proc *)tokenizer)->callbacks.tokenizer.build;
}

grn_rc
grn_tokenizer_build(grn_ctx *ctx,
                    grn_obj *tokenizer,
                    grn_tokenizer_build_data *data)
{
  return ((grn_proc *)tokenizer)->callbacks.tokenizer.build(ctx, data);
}

grn_obj *
grn_tokenizer_build_data_get_source_table(grn_ctx *ctx,
                                          grn_tokenizer_build_data *data)
{
  return data->source_table;
}

grn_obj *
grn_tokenizer_build_data_get_source_columns(grn_ctx *ctx,
                                            grn_tokenizer_build_data *data)
{
  return data->source_columns;
}

grn_obj *
grn_tokenizer_build_data_get_lexicon(grn_ctx *ctx,
                                     grn_tokenizer_build_data *data)
{
  return data->lexicon;
}

grn_obj *
grn_tokenizer_build_data_get_index_column(grn_ctx *ctx,
                                          grn_tokenizer_build_data *data)
{
  return data->index_column;
}

grn_rc
grn_tokenizer_build_data_processed_n_records(grn_ctx *ctx,
                                             grn_tokenizer_build_data *data,
                                             uint32_t n_records)
{
  return data->processed_n_records_func(ctx, data, n_records, data->user_data);
}

grn_rc
grn_tokenizer_build_data_start_vectorize(grn_ctx *ctx,
                                         grn_tokenizer_build_data *data,
                                         uint32_t n_total_records)
{
  return data->start_vectorize_func(ctx,
                                    data,
                                    n_total_records,
                                    data->user_data);
}

grn_rc
grn_tokenizer_build_data_start_cluster(grn_ctx *ctx,
                                       grn_tokenizer_build_data *data,
                                       uint32_t n_total_records)
{
  return data->start_cluster_func(ctx, data, n_total_records, data->user_data);
}

grn_rc
grn_tokenizer_build_data_start_load(grn_ctx *ctx,
                                    grn_tokenizer_build_data *data,
                                    uint32_t n_total_records)
{
  return data->start_load_func(ctx, data, n_total_records, data->user_data);
}

grn_rc
grn_tokenizer_build_data_start_record(grn_ctx *ctx,
                                      grn_tokenizer_build_data *data,
                                      grn_id rid)
{
  return data->start_record_func(ctx, data, rid, data->user_data);
}

grn_rc
grn_tokenizer_build_data_start_section(grn_ctx *ctx,
                                       grn_tokenizer_build_data *data,
                                       uint32_t sid)
{
  return data->start_section_func(ctx, data, sid, data->user_data);
}

grn_rc
grn_tokenizer_build_data_append_tokens(grn_ctx *ctx,
                                       grn_tokenizer_build_data *data,
                                       grn_obj *tokens)
{
  return data->append_tokens_func(ctx, data, tokens, data->user_data);
}

grn_rc
grn_tokenizer_build_data_finish_section(grn_ctx *ctx,
                                        grn_tokenizer_build_data *data)
{
  return data->finish_section_func(ctx, data, data->user_data);
}

grn_rc
grn_tokenizer_build_data_finish_record(grn_ctx *ctx,
                                       grn_tokenizer_build_data *data)
{
  return data->finish_record_func(ctx, data, data->user_data);
}
