diff -Nurd mysql-4.1.22/config.h.in mysql-4.1.22.senna/config.h.in --- mysql-4.1.22/config.h.in 2006-11-03 11:18:14.000000000 +0900 +++ mysql-4.1.22.senna/config.h.in 2007-03-24 12:53:40.343801000 +0900 @@ -26,6 +26,9 @@ /* READLINE: your system defines TIOCGWINSZ in sys/ioctl.h. */ #undef GWINSZ_IN_SYS_IOCTL +/* Define to 1 if Senna is enabled */ +#undef ENABLE_SENNA + /* Define to 1 if you have the `alarm' function. */ #undef HAVE_ALARM @@ -437,6 +440,9 @@ /* Define to 1 if you have the header file. */ #undef HAVE_MALLOC_H +/* Define to 1 if libmecab is available */ +#undef HAVE_MECAB + /* Define to 1 if you have the `memcpy' function. */ #undef HAVE_MEMCPY diff -Nurd mysql-4.1.22/configure.in mysql-4.1.22.senna/configure.in --- mysql-4.1.22/configure.in 2006-11-03 11:15:41.000000000 +0900 +++ mysql-4.1.22.senna/configure.in 2007-03-24 12:53:40.353907000 +0900 @@ -795,6 +795,110 @@ # Types that must be checked AFTER large file support is checked AC_TYPE_SIZE_T +# For senna +AC_ARG_WITH(mecab, +[ --with-mecab[=DIR] Specify install prefix of mecab], [ + if test "$withval" = "yes"; then + MECAB_PREFIX="" + else + MECAB_PREFIX="$withval" + fi +], [ + MECAB_PREFIX="" +]) + +MECAB_INCLUDES= +MECAB_LIBS= + +if test -z "$MECAB_PREFIX"; then + if eval 'mecab-config --prefix 2>&1 >/dev/null'; then + MECAB_CONFIG='mecab-config' + MECAB_PREFIX='mecab-config --prefix' + MECAB_INCLUDES="`\"$MECAB_CONFIG\" --cflags`" + MECAB_LIBS="${LDFLAGS} `\"$MECAB_CONFIG\" --libs`" + fi +else + MECAB_CONFIG="$MECAB_PREFIX/bin/mecab-config" + + if test -x "$MECAB_CONFIG"; then + MECAB_INCLUDES="`\"$MECAB_CONFIG\" --cflags`" + MECAB_LIBS="`\"$MECAB_CONFIG\" --libs`" + else + MECAB_INCLUDES="-I$MECAB_PREFIX/include" + MECAB_LIBS="-L$MECAB_PREFIX/lib" + fi +fi + +_CPPFLAGS="${CPPFLAGS}" +_LIBS="${LIBS}" +CPPFLAGS="${CPPFLAGS} ${MECAB_INCLUDES}" +LIBS="${LIBS} ${MECAB_LIBS}" + +AC_CHECK_FUNC(mecab_new, [ + AC_DEFINE([HAVE_MECAB], [1], [Define to 1 if libmecab is available]) + HAVE_MECAB=1 +], [ + HAVE_MECAB= + MECAB_INCLUDES= + MECAB_LIBS= +]) + +CPPFLAGS="${_CPPFLAGS}" +LIBS="${_LIBS}" + +AC_SUBST(MECAB_INCLUDES) +AC_SUBST(MECAB_LIBS) + +SENNA_INCLUDES= +SENNA_LIBS= + +AC_ARG_WITH(senna, +[ --with-senna[=DIR] Enable Senna fulltext search support], [ + case "$withval" in + no) : ;; + yes|*) + if test "$withval" = "yes"; then + if eval 'senna-cfg --cflags 2>&1 >/dev/null'; then + SENNA_CFG="senna-cfg" + SENNA_PREFIX="senna-cfg --prefix" + SENNA_INCLUDES="`\"$SENNA_CFG\" --cflags` $MECAB_INCLUDES" + SENNA_LIBS="`\"$SENNA_CFG\" --libs` -lsenna $MECAB_LIBS" + else + SENNA_INCLUDES="$MECAB_INCLUDES" + SENNA_LIBS="-lsenna $MECAB_LIBS" + fi + else + SENNA_PREFIX="$withval" + SENNA_CFG="$SENNA_PREFIX/bin/senna-cfg" + if test -x "$SENNA_CFG"; then + SENNA_INCLUDES="`\"$SENNA_CFG\" --cflags` $MECAB_INCLUDES" + SENNA_LIBS="`\"$SENNA_CFG\" --libs` -lsenna $MECAB_LIBS" + else + SENNA_INCLUDES="-I$SENNA_PREFIX/include $MECAB_INCLUDES" + SENNA_LIBS="-L$SENNA_PREFIX/lib -lsenna $MECAB_LIBS" + fi + fi + + _CPPFLAGS="${CPPFLAGS}" + _LIBS="${LIBS}" + CPPFLAGS="${CPPFLAGS} ${SENNA_INCLUDES}" + LIBS="${LIBS} ${SENNA_LIBS}" + + AC_CHECK_FUNC(sen_init, [ + AC_DEFINE([ENABLE_SENNA], [1], [Define to 1 if Senna is enabled]) + ], [ + AC_MSG_ERROR([Could not find libsenna. Check your Senna installation.]) + ]) + + CPPFLAGS="${_CPPFLAGS}" + LIBS="${_LIBS}" + ;; + esac +]) + +AC_SUBST([SENNA_INCLUDES]) +AC_SUBST([SENNA_LIBS]) + #-------------------------------------------------------------------- # Check for system header files #-------------------------------------------------------------------- diff -Nurd mysql-4.1.22/include/myisam.h mysql-4.1.22.senna/include/myisam.h --- mysql-4.1.22/include/myisam.h 2006-11-03 11:15:12.000000000 +0900 +++ mysql-4.1.22.senna/include/myisam.h 2007-03-24 12:53:40.366155000 +0900 @@ -33,6 +33,11 @@ #endif #include "my_handler.h" +#ifdef ENABLE_SENNA +#include +#define SEN_DISABLE_SENNA 0x80000000 /* Don't use Senna fulltext search engine */ +#endif /* ENABLE_SENNA */ + /* defines used by myisam-funktions */ /* The following defines can be increased if necessary */ @@ -88,6 +93,14 @@ ulong *rec_per_key; /* for sql optimizing */ uint raid_type,raid_chunks; ulong raid_chunksize; +#ifdef ENABLE_SENNA + unsigned int senna_keys_file_size; + unsigned int senna_keys_size; + unsigned int senna_lexicon_file_size; + unsigned int senna_lexicon_size; + unsigned int senna_inv_seg_size; + unsigned int senna_inv_chunk_size; +#endif /* ENABLE_SENNA */ } MI_ISAMINFO; @@ -126,6 +139,11 @@ uint32 version; /* For concurrent read/write */ HA_KEYSEG *seg,*end; +#ifdef ENABLE_SENNA + int senna_flags; + int senna_initial_n_segments; + sen_index *senna; +#endif /* ENABLE_SENNA */ int (*bin_search)(struct st_myisam_info *info,struct st_mi_keydef *keyinfo, uchar *page,uchar *key, uint key_len,uint comp_flag,uchar * *ret_pos, diff -Nurd mysql-4.1.22/libmysqld/ha_myisam.cc mysql-4.1.22.senna/libmysqld/ha_myisam.cc --- mysql-4.1.22/libmysqld/ha_myisam.cc 2006-11-03 11:15:51.000000000 +0900 +++ mysql-4.1.22.senna/libmysqld/ha_myisam.cc 2007-03-24 12:53:40.404359000 +0900 @@ -134,9 +134,59 @@ const char *ha_myisam::index_type(uint key_number) { - return ((table->key_info[key_number].flags & HA_FULLTEXT) ? - "FULLTEXT" : - (table->key_info[key_number].flags & HA_SPATIAL) ? + if (table->key_info[key_number].flags & HA_FULLTEXT) + { +#ifdef ENABLE_SENNA + MYISAM_SHARE *share = file->s; + if (share->keyinfo[key_number].senna_flags & SEN_DISABLE_SENNA) + { + return "FULLTEXT"; + } + else + { + if (share->keyinfo[key_number].senna_flags & SEN_INDEX_NORMALIZE) + { + if (share->keyinfo[key_number].senna_flags & SEN_INDEX_NGRAM) + { + return "FULLTEXT,SENNA,NORMALIZE,NGRAM"; + } + else + { + if (share->keyinfo[key_number].senna_flags & SEN_INDEX_DELIMITED) + { + return "FULLTEXT,SENNA,NORMALIZE,DELIMITED"; + } + else + { + return "FULLTEXT,SENNA,NORMALIZE"; + } + } + } + else + { + if (share->keyinfo[key_number].senna_flags & SEN_INDEX_NGRAM) + { + return "FULLTEXT,SENNA,NGRAM"; + } + else + { + if (share->keyinfo[key_number].senna_flags & SEN_INDEX_DELIMITED) + { + return "FULLTEXT,SENNA,DELIMITED"; + } + else + { + return "FULLTEXT,SENNA"; + } + } + } + } +#else /* ENABLE_SENNA */ + return "FULLTEXT"; +#endif /* ENABLE_SENNA */ + } + else + return ((table->key_info[key_number].flags & HA_SPATIAL) ? "SPATIAL" : (table->key_info[key_number].algorithm == HA_KEY_ALG_RTREE) ? "RTREE" : @@ -1229,6 +1279,14 @@ delete_length = info.delete_length; check_time = info.check_time; mean_rec_length=info.mean_reclength; +#ifdef ENABLE_SENNA + senna_keys_size = info.senna_keys_size; + senna_keys_file_size = info.senna_keys_file_size; + senna_lexicon_size = info.senna_lexicon_size; + senna_lexicon_file_size = info.senna_lexicon_file_size; + senna_inv_seg_size = info.senna_inv_seg_size; + senna_inv_chunk_size = info.senna_inv_chunk_size; +#endif /* ENABLE_SENNA */ } if (flag & HA_STATUS_CONST) { @@ -1370,6 +1428,13 @@ for (i=0; i < table_arg->keys ; i++, pos++) { keydef[i].flag= (pos->flags & (HA_NOSAME | HA_FULLTEXT | HA_SPATIAL)); +#ifdef ENABLE_SENNA + if ((keydef[i].flag & HA_FULLTEXT) && info->key_info) { + keydef[i].senna_flags=info->key_info[i].senna_flags; + keydef[i].senna_initial_n_segments=info->key_info[i].senna_initial_n_segments; + sen_log("creating index (%s, flags=%x initial_n_segments=%d)",name,keydef[i].senna_flags,keydef[i].senna_initial_n_segments); + } +#endif /* ENABLE_SENNA */ keydef[i].key_alg= pos->algorithm == HA_KEY_ALG_UNDEF ? (pos->flags & HA_SPATIAL ? HA_KEY_ALG_RTREE : HA_KEY_ALG_BTREE) : pos->algorithm; diff -Nurd mysql-4.1.22/libmysqld/sql_db.cc mysql-4.1.22.senna/libmysqld/sql_db.cc --- mysql-4.1.22/libmysqld/sql_db.cc 2006-11-03 11:16:02.000000000 +0900 +++ mysql-4.1.22.senna/libmysqld/sql_db.cc 2007-03-24 12:53:40.425503000 +0900 @@ -809,6 +809,11 @@ (file->name[1] == '.' && !file->name[2]))) continue; +#ifdef ENABLE_SENNA + /* senna files is skip */ + /* ".SEN",".SEN.i",".SEN.i.c",".SEN.l", ".SEN.i.c.001",.. and so on */ + if (strstr(file->name, ".SEN")) { continue; } +#endif /* ENABLE_SENNA */ /* Check if file is a raid directory */ if ((my_isdigit(&my_charset_latin1, file->name[0]) || (file->name[0] >= 'a' && file->name[0] <= 'f')) && diff -Nurd mysql-4.1.22/libmysqld/sql_delete.cc mysql-4.1.22.senna/libmysqld/sql_delete.cc --- mysql-4.1.22/libmysqld/sql_delete.cc 2006-11-03 11:17:34.000000000 +0900 +++ mysql-4.1.22.senna/libmysqld/sql_delete.cc 2007-03-24 12:53:40.427024000 +0900 @@ -650,6 +650,9 @@ if (thd->slave_thread) --slave_open_temp_tables; *fn_ext(path)=0; // Remove the .frm extension +#ifdef ENABLE_SENNA + create_info.key_info=table->key_info; +#endif /* ENABLE_SENNA */ ha_create_table(path, &create_info,1); // We don't need to call invalidate() because this table is not in cache if ((error= (int) !(open_temporary_table(thd, path, table_list->db, diff -Nurd mysql-4.1.22/libmysqld/sql_show.cc mysql-4.1.22.senna/libmysqld/sql_show.cc --- mysql-4.1.22/libmysqld/sql_show.cc 2006-11-03 11:15:56.000000000 +0900 +++ mysql-4.1.22.senna/libmysqld/sql_show.cc 2007-03-24 12:53:40.440020000 +0900 @@ -26,6 +26,12 @@ #include "ha_berkeley.h" // For berkeley_show_logs #endif +#ifdef ENABLE_SENNA +#ifdef HAVE_ISAM +#include "ha_myisam.h" // For isam +#endif +#endif /* ENABLE_SENNA */ + static const char *grant_names[]={ "select","insert","update","delete","create","drop","reload","shutdown", "process","file","grant","references","index","alter"}; @@ -500,6 +506,20 @@ item->maybe_null=1; field_list.push_back(item=new Item_empty_string("Create_options",255)); item->maybe_null=1; +#ifdef ENABLE_SENNA + field_list.push_back(item=new Item_int("Senna_key_size",(longlong) 1,21)); + item->maybe_null=1; + field_list.push_back(item=new Item_int("Senna_key_file_size",(longlong) 1,21)); + item->maybe_null=1; + field_list.push_back(item=new Item_int("Senna_lexicon_size",(longlong) 1,21)); + item->maybe_null=1; + field_list.push_back(item=new Item_int("Senna_lexicon_file_size",(longlong) 1,21)); + item->maybe_null=1; + field_list.push_back(item=new Item_int("Senna_inv_seg_size",(longlong) 1,21)); + item->maybe_null=1; + field_list.push_back(item=new Item_int("Senna_inv_chunk_size",(longlong) 1,21)); + item->maybe_null=1; +#endif /* ENABLE_SENNA */ field_list.push_back(item=new Item_empty_string("Comment",80)); item->maybe_null=1; if (protocol->send_fields(&field_list,1)) @@ -624,6 +644,38 @@ (ptr == option_buff ? 0 : (uint) (ptr-option_buff)-1) , system_charset_info); } +#ifdef ENABLE_SENNA + if (file->senna_keys_size >= 0) { + protocol->store((ulonglong) file->senna_keys_size); + } else { + protocol->store_null(); + } + if (file->senna_keys_file_size >= 0) { + protocol->store((ulonglong) file->senna_keys_file_size); + } else { + protocol->store_null(); + } + if (file->senna_lexicon_size >= 0) { + protocol->store((ulonglong) file->senna_lexicon_size); + } else { + protocol->store_null(); + } + if (file->senna_lexicon_file_size >= 0) { + protocol->store((ulonglong) file->senna_lexicon_file_size); + } else { + protocol->store_null(); + } + if (file->senna_inv_seg_size >= 0) { + protocol->store((ulonglong) file->senna_inv_seg_size); + } else { + protocol->store_null(); + } + if (file->senna_inv_chunk_size >= 0) { + protocol->store((ulonglong) file->senna_inv_chunk_size); + } else { + protocol->store_null(); + } +#endif /* ENABLE_SENNA */ { char *comment=table->file->update_table_comment(table->comment); protocol->store(comment, system_charset_info); diff -Nurd mysql-4.1.22/libmysqld/sql_table.cc mysql-4.1.22.senna/libmysqld/sql_table.cc --- mysql-4.1.22/libmysqld/sql_table.cc 2006-11-03 11:16:00.000000000 +0900 +++ mysql-4.1.22.senna/libmysqld/sql_table.cc 2007-03-24 12:53:40.447615000 +0900 @@ -974,6 +974,9 @@ if (!key_info_buffer || ! key_part_info) DBUG_RETURN(-1); // Out of memory +#ifdef ENABLE_SENNA + create_info->key_info=key_info_buffer; +#endif /* ENABLE_SENNA */ key_iterator.rewind(); key_number=0; for (; (key=key_iterator++) ; key_number++) @@ -1017,6 +1020,10 @@ if (key->generated) key_info->flags|= HA_GENERATED_KEY; +#ifdef ENABLE_SENNA + key_info->senna_flags=key->senna_flags; + key_info->senna_initial_n_segments=key->senna_initial_n_segments; +#endif /* ENABLE_SENNA */ key_info->key_parts=(uint8) key->columns.elements; key_info->key_part=key_part_info; key_info->usable_key_parts= key_number; @@ -3216,6 +3223,21 @@ key_part_length)); } if (key_parts.elements) +#ifdef ENABLE_SENNA + key_list.push_back(new Key(key_info->flags & HA_SPATIAL ? Key::SPATIAL : + (key_info->flags & HA_NOSAME ? + (!my_strcasecmp(system_charset_info, + key_name, primary_key_name) ? + Key::PRIMARY : Key::UNIQUE) : + (key_info->flags & HA_FULLTEXT ? + Key::FULLTEXT : Key::MULTIPLE)), + key_name, + key_info->algorithm, + test(key_info->flags & HA_GENERATED_KEY), + key_parts, + key_info->senna_flags, + key_info->senna_initial_n_segments)); +#else /* ENABLE_SENNA */ key_list.push_back(new Key(key_info->flags & HA_SPATIAL ? Key::SPATIAL : (key_info->flags & HA_NOSAME ? (!my_strcasecmp(system_charset_info, @@ -3227,6 +3249,7 @@ key_info->algorithm, test(key_info->flags & HA_GENERATED_KEY), key_parts)); +#endif /* ENABLE_SENNA */ } { Key *key; diff -Nurd mysql-4.1.22/myisam/Makefile.am mysql-4.1.22.senna/myisam/Makefile.am --- mysql-4.1.22/myisam/Makefile.am 2006-11-03 11:15:58.000000000 +0900 +++ mysql-4.1.22.senna/myisam/Makefile.am 2007-03-24 12:53:40.463135000 +0900 @@ -18,11 +18,13 @@ pkgdata_DATA = mi_test_all mi_test_all.res INCLUDES = @MT_INCLUDES@ \ - -I$(top_builddir)/include -I$(top_srcdir)/include + -I$(top_builddir)/include -I$(top_srcdir)/include \ + @SENNA_INCLUDES@ @MECAB_INCLUDES@ LDADD = @CLIENT_EXTRA_LDFLAGS@ libmyisam.a \ $(top_builddir)/mysys/libmysys.a \ $(top_builddir)/dbug/libdbug.a \ - $(top_builddir)/strings/libmystrings.a @ZLIB_LIBS@ + $(top_builddir)/strings/libmystrings.a @ZLIB_LIBS@ \ + @SENNA_LIBS@ @MECAB_LIBS@ pkglib_LIBRARIES = libmyisam.a bin_PROGRAMS = myisamchk myisamlog myisampack myisam_ftdump myisamchk_DEPENDENCIES= $(LIBRARIES) diff -Nurd mysql-4.1.22/myisam/ft_boolean_search.c mysql-4.1.22.senna/myisam/ft_boolean_search.c --- mysql-4.1.22/myisam/ft_boolean_search.c 2006-11-03 11:14:53.000000000 +0900 +++ mysql-4.1.22.senna/myisam/ft_boolean_search.c 2007-03-24 12:53:40.474268000 +0900 @@ -21,6 +21,11 @@ #define FT_CORE #include "ftdefs.h" +#ifdef ENABLE_SENNA +#include +#define SENNA_MAX_N_EXPR 32 +#endif /* ENABLE_SENNA */ + /* search with boolean queries */ static double _wghts[11]= @@ -104,6 +109,9 @@ uint keynr; uchar with_scan; enum { UNINITIALIZED, READY, INDEX_SEARCH, INDEX_DONE } state; +#ifdef ENABLE_SENNA + sen_records *sir; +#endif /* ENABLE_SENNA */ } FTB; static int FTB_WORD_cmp(my_off_t *v, FTB_WORD *a, FTB_WORD *b) @@ -391,6 +399,33 @@ DBUG_ASSERT(keynr==NO_SUCH_KEY || cs == info->s->keyinfo[keynr].seg->charset); ftb->with_scan=0; ftb->lastpos=HA_OFFSET_ERROR; +#ifdef ENABLE_SENNA + if (ftb->info->s->keyinfo[ftb->keynr].senna) + { + sen_index *i; + sen_query *q; + sen_encoding e; + const char *rest; + unsigned int rest_len; + if (keynr==NO_SUCH_KEY || + !(i = info->s->keyinfo[keynr].senna)) { + my_free((gptr)ftb,MYF(0)); + return 0; + } + sen_index_info(i, NULL, NULL, NULL, &e, NULL, NULL, NULL, NULL, NULL, NULL); + if (!(q = sen_query_open(query, query_len, sen_sel_or, SENNA_MAX_N_EXPR, e))) { + my_free((gptr)ftb,MYF(0)); + return 0; + } + if (rest_len = sen_query_rest(q, &rest)) { + sen_log("too long query. rest(%.*s) are ignored", rest_len, rest); + } + ftb->sir = sen_records_open(sen_rec_document, sen_rec_none, 0); + sen_query_exec(i, q, ftb->sir, sen_sel_or); + sen_query_close(q); + return ftb; + } +#endif /* ENABLE_SENNA */ bzero(& ftb->no_dupes, sizeof(TREE)); init_alloc_root(&ftb->mem_root, 1024, 1024); @@ -533,6 +568,27 @@ int ft_boolean_read_next(FT_INFO *ftb, char *record) { +#ifdef ENABLE_SENNA + if (ftb->info->s->keyinfo[ftb->keynr].senna) + { + my_off_t pos; + MI_INFO *info=ftb->info; + while (ftb->sir && sen_records_next(ftb->sir, &pos, sizeof(my_off_t), NULL)) { + info->update&= (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED); + info->lastpos=pos; + if (!(*info->read_record)(info,info->lastpos,record)) { + info->update|= HA_STATE_AKTIV; /* Record is read */ + return 0; + } + sen_log("my_errno=%d pos=%lld in ft_boolean_read_next()", my_errno, pos); + if (my_errno == 127) { continue; } + return my_errno; + } + return HA_ERR_END_OF_FILE; + } + else +#endif /* ENABLE_SENNA */ + { FTB_EXPR *ftbe; FTB_WORD *ftbw; MI_INFO *info=ftb->info; @@ -598,11 +654,24 @@ err: ftb->queue.first_cmp_arg=(void *)0; return my_errno; + } } float ft_boolean_find_relevance(FT_INFO *ftb, byte *record, uint length) { +#ifdef ENABLE_SENNA + if (ftb->info->s->keyinfo[ftb->keynr].senna) + { + my_off_t docid=ftb->info->lastpos; + if (!ftb->sir) { return 0.0; } + if (docid == HA_OFFSET_ERROR) + return -2.0; + return 1.0 * sen_records_find(ftb->sir, &docid); + } + else +#endif /* ENABLE_SENNA */ + { FT_WORD word; FTB_WORD *ftbw; FTB_EXPR *ftbe; @@ -681,11 +750,20 @@ { /* match failed ! */ return 0.0; } + } } void ft_boolean_close_search(FT_INFO *ftb) { +#ifdef ENABLE_SENNA + if (ftb->info->s->keyinfo[ftb->keynr].senna) + { + sen_records_close(ftb->sir); + my_free((gptr)ftb,MYF(0)); + return; + } +#endif /* ENABLE_SENNA */ if (is_tree_inited(& ftb->no_dupes)) { delete_tree(& ftb->no_dupes); @@ -697,12 +775,26 @@ float ft_boolean_get_relevance(FT_INFO *ftb) { +#ifdef ENABLE_SENNA + if (ftb->info->s->keyinfo[ftb->keynr].senna) + { + if (!ftb->sir) { return 0.0; } + return 1.0 * sen_records_curr_score(ftb->sir); + } +#endif /* ENABLE_SENNA */ return ftb->root->cur_weight; } void ft_boolean_reinit_search(FT_INFO *ftb) { +#ifdef ENABLE_SENNA + if (ftb->info->s->keyinfo[ftb->keynr].senna) + { + sen_records_rewind(ftb->sir); + return; + } +#endif /* ENABLE_SENNA */ _ftb_init_index_search(ftb); } diff -Nurd mysql-4.1.22/myisam/ft_nlq_search.c mysql-4.1.22.senna/myisam/ft_nlq_search.c --- mysql-4.1.22/myisam/ft_nlq_search.c 2006-11-03 11:15:58.000000000 +0900 +++ mysql-4.1.22.senna/myisam/ft_nlq_search.c 2007-03-24 12:53:40.480132000 +0900 @@ -33,6 +33,9 @@ MI_INFO *info; int ndocs; int curdoc; +#ifdef ENABLE_SENNA + sen_records *sir; +#endif /* ENABLE_SENNA */ FT_DOC doc[1]; }; @@ -210,8 +213,23 @@ FT_DOC *dptr; FT_INFO *dlist=NULL; my_off_t saved_lastpos=info->lastpos; +#ifdef ENABLE_SENNA + sen_records *sir; +#endif /* ENABLE_SENNA */ DBUG_ENTER("ft_init_nlq_search"); +#ifdef ENABLE_SENNA + if (info->s->keyinfo[keynr].senna) + { + // sen_log("ft_init_nlq_search(%p,%d,%p,%d,%d)", info, keynr, query, query_len, presort); + sir = sen_index_sel(info->s->keyinfo[keynr].senna, query, query_len); + // sen_log("sen_index_search done"); + } + else + { + sir = NULL; + } +#endif /* ENABLE_SENNA */ /* black magic ON */ if ((int) (keynr = _mi_check_index(info,keynr)) < 0) DBUG_RETURN(NULL); @@ -278,6 +296,9 @@ dlist->info=aio.info; dptr=dlist->doc; +#ifdef ENABLE_SENNA + dlist->sir = sir; +#endif /* ENABLE_SENNA */ tree_walk(&aio.dtree, (tree_walk_action) &walk_and_copy, &dptr, left_root_right); @@ -295,6 +316,27 @@ int ft_nlq_read_next(FT_INFO *handler, char *record) { MI_INFO *info= (MI_INFO *) handler->info; +#ifdef ENABLE_SENNA + // sen_log("ft_nlq_read_next(%p,%p)", handler, record); + if (handler->sir) + { + my_off_t pos; + info->update&= (HA_STATE_CHANGED | HA_STATE_ROW_CHANGED); + while (sen_records_next(handler->sir, &pos, sizeof(my_off_t), NULL)) + { + info->lastpos=pos; + if (!(*info->read_record)(info,info->lastpos,record)) + { + info->update|= HA_STATE_AKTIV; /* Record is read */ + return 0; + } + sen_log("my_errno=%d pos=%lld in ft_nlq_read_next()", my_errno, pos); + if (my_errno == 127) { continue; } + return my_errno; + } + return HA_ERR_END_OF_FILE; + } +#endif /* ENABLE_SENNA */ if (++handler->curdoc >= handler->ndocs) { @@ -322,9 +364,19 @@ FT_DOC *docs=handler->doc; my_off_t docid=handler->info->lastpos; +#ifdef ENABLE_SENNA + // sen_log("ft_nlq_find_relevance(docid=%d)", docid); +#endif /* ENABLE_SENNA */ if (docid == HA_POS_ERROR) return -5.0; +#ifdef ENABLE_SENNA + if (handler->sir) { + // sen_log("score = %d", sen_records_find(handler->sir, &docid)); + return 1.0 * sen_records_find(handler->sir, &docid); + } +#endif /* ENABLE_SENNA */ + /* Assuming docs[] is sorted by dpos... */ for (a=0, b=handler->ndocs, c=(a+b)/2; b-a>1; c=(a+b)/2) @@ -344,18 +396,45 @@ void ft_nlq_close_search(FT_INFO *handler) { +#ifdef ENABLE_SENNA + if (handler->sir) + { + sen_log("ft_nlq_close_search(%p)", handler); + sen_records_close(handler->sir); + } +#endif /* ENABLE_SENNA */ my_free((gptr)handler,MYF(0)); } float ft_nlq_get_relevance(FT_INFO *handler) { +#ifdef ENABLE_SENNA + // sen_log("ft_nlq_get_relevance(%p)", handler); + + if (!handler->sir) { + if (handler->doc) { + return (float) handler->doc[handler->curdoc].weight; + } + return 0.0; + } + + return 1.0 * sen_records_curr_score(handler->sir); +#else /* ENABLE_SENNA */ return (float) handler->doc[handler->curdoc].weight; +#endif /* ENABLE_SENNA */ } void ft_nlq_reinit_search(FT_INFO *handler) { +#ifdef ENABLE_SENNA + if (handler->sir) + { + sen_log("ft_nlq_reinit_search(%p)", handler); + sen_records_rewind(handler->sir); + } +#endif /* ENABLE_SENNA */ handler->curdoc=-1; } diff -Nurd mysql-4.1.22/myisam/ft_update.c mysql-4.1.22.senna/myisam/ft_update.c --- mysql-4.1.22/myisam/ft_update.c 2006-11-03 11:17:23.000000000 +0900 +++ mysql-4.1.22.senna/myisam/ft_update.c 2007-03-24 12:53:40.482261000 +0900 @@ -117,6 +117,19 @@ FT_WORD * _mi_ft_parserecord(MI_INFO *info, uint keynr, const byte *record) { +#ifdef ENABLE_SENNA + if (info->s->keyinfo[keynr].senna) + { + FT_WORD *wlist; + if (!(wlist = (FT_WORD *) my_malloc(sizeof(FT_WORD), MYF(0)))) { + return NULL; + } + wlist->pos = 0; + return wlist; + } + else +#endif /* ENABLE_SENNA */ + { TREE ptree; DBUG_ENTER("_mi_ft_parserecord"); @@ -125,6 +138,7 @@ DBUG_RETURN(NULL); DBUG_RETURN(ft_linearize(&ptree)); + } } static int _mi_ft_store(MI_INFO *info, uint keynr, byte *keybuf, @@ -189,6 +203,116 @@ DBUG_RETURN(GEE_THEY_ARE_ABSOLUTELY_IDENTICAL); } +#ifdef ENABLE_SENNA +#define SECTIONALIZE 0x00080000 +int ft_sen_index_add(MI_INFO *info, uint keynr, const byte *record, my_off_t pos) +{ + if (info->s->keyinfo[keynr].senna_flags & SECTIONALIZE) { + FT_SEG_ITERATOR ftsi; + uint len = 0; + unsigned int section; + sen_values *values; + _mi_ft_segiterator_init(info, keynr, record, &ftsi); + while (_mi_ft_segiterator(&ftsi)) { + if (ftsi.pos) { + if (ftsi.len > 1048576) { sen_log("ft_sen_index_add: ftsi.len=%d", ftsi.len); } + if (ftsi.len > len) { len = ftsi.len; } + } + } + if (!len) { return -1; } + _mi_ft_segiterator_init(info, keynr, record, &ftsi); + section = 1; + while (_mi_ft_segiterator(&ftsi)) { + if (ftsi.pos) { + values = sen_values_open(); + sen_values_add(values, ftsi.pos, ftsi.len, 0); + sen_index_update(info->s->keyinfo[keynr].senna, &pos, section, NULL, values); + sen_values_close(values); + } + section++; + } + return 0; + } else { + FT_SEG_ITERATOR ftsi; + char *buf, *p; + uint len = 0; + _mi_ft_segiterator_init(info, keynr, record, &ftsi); + while (_mi_ft_segiterator(&ftsi)) { + if (ftsi.pos) { + if (ftsi.len > 1048576) { sen_log("ft_sen_index_add: ftsi.len=%d", ftsi.len); } + len += ftsi.len + 1; + } + } + if (!len) { return -1; } + if (!(p = buf = malloc(len))) { return -1; } + _mi_ft_segiterator_init(info, keynr, record, &ftsi); + while (_mi_ft_segiterator(&ftsi)) { + if (ftsi.pos) { + if (p != buf) { *p++ = ' '; } + memcpy(p, ftsi.pos, ftsi.len); + p += ftsi.len; + } + } + sen_index_upd(info->s->keyinfo[keynr].senna, &pos, NULL, 0, buf, (p - buf)); + free(buf); + return 0; + } +} + +int ft_sen_index_del(MI_INFO *info, uint keynr, const byte *record, my_off_t pos) +{ + if (info->s->keyinfo[keynr].senna_flags & SECTIONALIZE) { + FT_SEG_ITERATOR ftsi; + uint len = 0; + unsigned int section; + sen_values *values; + _mi_ft_segiterator_init(info, keynr, record, &ftsi); + while (_mi_ft_segiterator(&ftsi)) { + if (ftsi.pos) { + if (ftsi.len > 1048576) { sen_log("ft_sen_index_del: ftsi.len=%d", ftsi.len); } + if (ftsi.len > len) { len = ftsi.len; } + } + } + if (!len) { return -1; } + _mi_ft_segiterator_init(info, keynr, record, &ftsi); + section = 1; + while (_mi_ft_segiterator(&ftsi)) { + if (ftsi.pos) { + values = sen_values_open(); + sen_values_add(values, ftsi.pos, ftsi.len, 0); + sen_index_update(info->s->keyinfo[keynr].senna, &pos, section, values, NULL); + sen_values_close(values); + } + section++; + } + return 0; + } else { + FT_SEG_ITERATOR ftsi; + char *buf, *p; + uint len = 0; + _mi_ft_segiterator_init(info, keynr, record, &ftsi); + while (_mi_ft_segiterator(&ftsi)) { + if (ftsi.pos) { + if (ftsi.len > 1048576) { sen_log("ft_sen_index_del: ftsi.len=%d", ftsi.len); } + len += ftsi.len + 1; + } + } + if (!len) { return -1; } + if (!(p = buf = malloc(len))) { return -1; } + _mi_ft_segiterator_init(info, keynr, record, &ftsi); + while (_mi_ft_segiterator(&ftsi)) { + if (ftsi.pos) { + if (p != buf) { *p++ = ' '; } + memcpy(p, ftsi.pos, ftsi.len); + p += ftsi.len; + } + } + sen_index_upd(info->s->keyinfo[keynr].senna, &pos, buf, (p - buf), NULL, 0); + free(buf); + return 0; + } +} +#endif /* ENABLE_SENNA */ /* update a document entry */ @@ -202,6 +326,14 @@ int cmp, cmp2; DBUG_ENTER("_mi_ft_update"); +#ifdef ENABLE_SENNA + if (info->s->keyinfo[keynr].senna) + { + // sen_log("_mi_ft_update(%p,%d,%p,%p,%p,%d)", info, keynr, keybuf, oldrec, newrec, pos); + ft_sen_index_del(info, keynr, oldrec, pos); + ft_sen_index_add(info, keynr, newrec, pos); + } +#endif /* ENABLE_SENNA */ if (!(old_word=oldlist=_mi_ft_parserecord(info, keynr, oldrec))) goto err0; if (!(new_word=newlist=_mi_ft_parserecord(info, keynr, newrec))) @@ -252,6 +384,11 @@ FT_WORD *wlist; DBUG_ENTER("_mi_ft_add"); +#ifdef ENABLE_SENNA + // sen_log("_mi_ft_add(%p,%d,%p,%p,%d)", info, keynr, keybuf, record, pos); + if (info->s->keyinfo[keynr].senna) + ft_sen_index_add(info, keynr, record, pos); +#endif /* ENABLE_SENNA */ if ((wlist=_mi_ft_parserecord(info, keynr, record))) { error=_mi_ft_store(info,keynr,keybuf,wlist,pos); @@ -271,6 +408,11 @@ DBUG_ENTER("_mi_ft_del"); DBUG_PRINT("enter",("keynr: %d",keynr)); +#ifdef ENABLE_SENNA + // sen_log("_mi_ft_del(%p,%d,%p,%p,%d)", info, keynr, keybuf, record, pos); + if (info->s->keyinfo[keynr].senna) + ft_sen_index_del(info, keynr, record, pos); +#endif /* ENABLE_SENNA */ if ((wlist=_mi_ft_parserecord(info, keynr, record))) { error=_mi_ft_erase(info,keynr,keybuf,wlist,pos); @@ -349,3 +491,31 @@ SEARCH_SAME)); } +#ifdef ENABLE_SENNA +void ft_index_truncate(MI_INFO *info) +{ + char buf[FN_REFLEN]; + MYISAM_SHARE *share= info->s; + uint i, keys= (uint) share->state.header.keys; + for (i=0 ; i < keys ; i++) + { + if (share->keyinfo[i].flag & HA_FULLTEXT) + { + if (share->keyinfo[i].senna) + { + sen_index_close(share->keyinfo[i].senna); + } + strcpy(buf, share->unique_file_name); + sprintf(buf + strlen(buf) - 3, "%03d", i); + sen_log("create index (%s)", buf); + share->keyinfo[i].senna = + (share->keyinfo[i].senna_flags & SEN_DISABLE_SENNA) + ? NULL + : sen_index_create(buf, sizeof(my_off_t), + share->keyinfo[i].senna_flags, + share->keyinfo[i].senna_initial_n_segments, + sen_enc_default); + } + } +} +#endif /* ENABLE_SENNA */ diff -Nurd mysql-4.1.22/myisam/fulltext.h mysql-4.1.22.senna/myisam/fulltext.h --- mysql-4.1.22/myisam/fulltext.h 2006-11-03 11:16:12.000000000 +0900 +++ mysql-4.1.22.senna/myisam/fulltext.h 2007-03-24 12:53:40.487488000 +0900 @@ -35,4 +35,5 @@ int _mi_ft_del(MI_INFO *, uint, byte *, const byte *, my_off_t); uint _mi_ft_convert_to_ft2(MI_INFO *, uint, uchar *); - +int ft_sen_index_add(MI_INFO *info, uint keynr, const byte *record, my_off_t pos); +void ft_index_truncate(MI_INFO *info); diff -Nurd mysql-4.1.22/myisam/mi_check.c mysql-4.1.22.senna/myisam/mi_check.c --- mysql-4.1.22/myisam/mi_check.c 2006-11-03 11:17:03.000000000 +0900 +++ mysql-4.1.22.senna/myisam/mi_check.c 2007-03-24 12:53:40.512917000 +0900 @@ -2038,6 +2038,9 @@ ulonglong key_map=share->state.key_map; DBUG_ENTER("mi_repair_by_sort"); +#ifdef ENABLE_SENNA + sen_log("mi_repair_by_sort"); +#endif /* ENABLE_SENNA */ start_records=info->state->records; got_error=1; new_file= -1; @@ -2950,6 +2953,10 @@ my_free((char*) wptr, MYF(MY_ALLOW_ZERO_PTR)); if ((error=sort_get_next_record(sort_param))) DBUG_RETURN(error); +#ifdef ENABLE_SENNA + if (info->s->keyinfo[sort_param->key].senna) + ft_sen_index_add(info, sort_param->key, sort_param->record, sort_param->filepos); +#endif /* ENABLE_SENNA */ if (!(wptr=_mi_ft_parserecord(info,sort_param->key,sort_param->record))) DBUG_RETURN(1); if (wptr->pos) diff -Nurd mysql-4.1.22/myisam/mi_close.c mysql-4.1.22.senna/myisam/mi_close.c --- mysql-4.1.22/myisam/mi_close.c 2006-11-03 11:16:23.000000000 +0900 +++ mysql-4.1.22.senna/myisam/mi_close.c 2007-03-24 12:53:40.515005000 +0900 @@ -31,6 +31,9 @@ DBUG_PRINT("enter",("base: %lx reopen: %u locks: %u", info,(uint) share->reopen, (uint) share->tot_locks)); +#ifdef ENABLE_SENNA + sen_log("closing index_file_name %s", share->index_file_name); +#endif /* ENABLE_SENNA */ pthread_mutex_lock(&THR_LOCK_myisam); if (info->lock_type == F_EXTRA_LCK) info->lock_type=F_UNLCK; /* HA_EXTRA_NO_USER_CHANGE */ @@ -98,6 +101,29 @@ keys = share->state.header.keys; for(i=0; ikey_root_lock[i])); + +#ifdef ENABLE_SENNA + if (share->keyinfo[i].flag & HA_FULLTEXT) + { + sen_log("share->delay_key_write=%d", share->delay_key_write); + sen_log("unique_file_name %s", share->unique_file_name); + sen_log("data_file_name %s", share->data_file_name); + sen_log("index_file_name %s", share->index_file_name); + + { + char buf[FN_REFLEN]; + strncpy(buf, share->unique_file_name, FN_REFLEN - 1); + buf[FN_REFLEN - 1] = '\0'; + sprintf(buf + strlen(buf) - 3, "%03d", i); + sen_log("closing (%s)", buf); + if (share->keyinfo[i].senna) { + sen_index_close(share->keyinfo[i].senna); + } + } + } +#endif /* ENABLE_SENNA */ + + } } #endif diff -Nurd mysql-4.1.22/myisam/mi_create.c mysql-4.1.22.senna/myisam/mi_create.c --- mysql-4.1.22/myisam/mi_create.c 2006-11-03 11:15:10.000000000 +0900 +++ mysql-4.1.22.senna/myisam/mi_create.c 2007-03-24 12:53:40.516489000 +0900 @@ -285,6 +285,42 @@ else if (keydef->flag & HA_FULLTEXT) { +#ifdef ENABLE_SENNA + sen_index *senna; + char buf[FN_REFLEN]; + strncpy(buf, name, FN_REFLEN - 1); + buf[FN_REFLEN - 1] = '\0'; + sprintf(buf + strlen(buf), ".%03d", i); + sen_log("keydef->senna_flags=%x", keydef->senna_flags); + if (!(keydef->senna_flags & SEN_DISABLE_SENNA)) + { + /* make index files */ + sen_log("create index (%s, flags=%x initial_n_segments=%d)", buf, + keydef->senna_flags, + keydef->senna_initial_n_segments); + senna = sen_index_create(buf, sizeof(my_off_t), + keydef->senna_flags, + keydef->senna_initial_n_segments, + sen_enc_default); + sen_index_close(senna); + } else { + senna = sen_index_open(buf); + if (senna) { + int senna_flags, senna_initial_n_segments; + sen_index_info(senna, NULL, &senna_flags, &senna_initial_n_segments, + NULL, NULL, NULL, NULL, NULL, NULL, NULL); + sen_index_close(senna); + sen_index_remove(buf); + sen_log("create index (%s, flags=%x initial_n_segments=%d)", buf, + senna_flags, senna_initial_n_segments); + senna = sen_index_create(buf, sizeof(my_off_t), + senna_flags, + senna_initial_n_segments, + sen_enc_default); + sen_index_close(senna); + } + } +#endif /* ENABLE_SENNA */ keydef->flag=HA_FULLTEXT | HA_PACK_KEY | HA_VAR_LENGTH_KEY; options|=HA_OPTION_PACK_KEYS; /* Using packed keys */ diff -Nurd mysql-4.1.22/myisam/mi_delete_all.c mysql-4.1.22.senna/myisam/mi_delete_all.c --- mysql-4.1.22/myisam/mi_delete_all.c 2006-11-03 11:15:32.000000000 +0900 +++ mysql-4.1.22.senna/myisam/mi_delete_all.c 2007-03-24 12:53:40.517160000 +0900 @@ -35,6 +35,24 @@ if (_mi_mark_file_changed(info)) goto err; +#ifdef ENABLE_SENNA + for (i = 0; i < share->base.keys; i++) { + sen_index *senna = share->keyinfo[i].senna; + if (senna) + { + char buf[FN_REFLEN]; + sen_index_path(senna, buf, FN_REFLEN); + sen_index_close(senna); + sen_index_remove(buf); + senna = sen_index_create(buf, sizeof(my_off_t), + share->keyinfo[i].senna_flags, + share->keyinfo[i].senna_initial_n_segments, + sen_enc_default); + share->keyinfo[i].senna = senna; + } + } +#endif /* ENABLE_SENNA */ + info->state->records=info->state->del=state->split=0; state->dellink = HA_OFFSET_ERROR; state->sortkey= (ushort) ~0; diff -Nurd mysql-4.1.22/myisam/mi_delete_table.c mysql-4.1.22.senna/myisam/mi_delete_table.c --- mysql-4.1.22/myisam/mi_delete_table.c 2006-11-03 11:14:55.000000000 +0900 +++ mysql-4.1.22.senna/myisam/mi_delete_table.c 2007-03-24 12:53:40.517942000 +0900 @@ -28,6 +28,9 @@ #endif DBUG_ENTER("mi_delete_table"); +#ifdef ENABLE_SENNA + sen_log("mi_delete_table(%s)", name); +#endif /* ENABLE_SENNA */ #ifdef EXTRA_DEBUG check_table_is_closed(name,"delete"); #endif @@ -58,6 +61,15 @@ #endif #endif /* USE_RAID */ +#ifdef ENABLE_SENNA + { + int i; + for (i = 0; i < 100; i++) { + my_snprintf(from, FN_REFLEN, "%s.%03d", name, i); + sen_index_remove(from); + } + } +#endif /* ENABLE_SENNA */ fn_format(from,name,"",MI_NAME_IEXT,4); if (my_delete_with_symlink(from, MYF(MY_WME))) DBUG_RETURN(my_errno); diff -Nurd mysql-4.1.22/myisam/mi_info.c mysql-4.1.22.senna/myisam/mi_info.c --- mysql-4.1.22/myisam/mi_info.c 2006-11-03 11:16:01.000000000 +0900 +++ mysql-4.1.22.senna/myisam/mi_info.c 2007-03-24 12:53:40.519021000 +0900 @@ -61,6 +61,51 @@ x->mean_reclength = info->state->records ? (ulong) ((info->state->data_file_length-info->state->empty)/ info->state->records) : (ulong) share->min_pack_length; +#ifdef ENABLE_SENNA + if (share->keyinfo) + { + int i; + + x->senna_keys_size = 0; + x->senna_keys_file_size = 0; + x->senna_lexicon_size = 0; + x->senna_lexicon_file_size = 0; + x->senna_inv_seg_size = 0; + x->senna_inv_chunk_size = 0; + + for (i = 0; i < share->base.keys; i++) + { + sen_index *senna = share->keyinfo[i].senna; + + if (senna) + { + unsigned nrecords_keys, file_size_keys; + unsigned nrecords_lexicon, file_size_lexicon; + unsigned inv_seg_size, inv_chunk_size; + + sen_index_info(senna, NULL, &share->keyinfo[i].senna_flags, + &share->keyinfo[i].senna_initial_n_segments, NULL, + &nrecords_keys, &file_size_keys, &nrecords_lexicon, + &file_size_lexicon, &inv_seg_size, &inv_chunk_size); + x->senna_keys_size += nrecords_keys; + x->senna_keys_file_size += file_size_keys; + x->senna_lexicon_size += nrecords_lexicon; + x->senna_lexicon_file_size += file_size_lexicon; + x->senna_inv_seg_size += inv_seg_size; + x->senna_inv_chunk_size += inv_chunk_size; + } + } + } + else + { + x->senna_keys_file_size = -1; + x->senna_keys_size = -1; + x->senna_lexicon_file_size = -1; + x->senna_lexicon_size = -1; + x->senna_inv_seg_size = -1; + x->senna_inv_chunk_size = -1; + } +#endif /* ENABLE_SENNA */ } if (flag & HA_STATUS_ERRKEY) { diff -Nurd mysql-4.1.22/myisam/mi_open.c mysql-4.1.22.senna/myisam/mi_open.c --- mysql-4.1.22/myisam/mi_open.c 2006-11-03 11:15:42.000000000 +0900 +++ mysql-4.1.22.senna/myisam/mi_open.c 2007-03-24 12:53:40.520823000 +0900 @@ -327,6 +327,11 @@ } } } +#ifdef ENABLE_SENNA + share->keyinfo[i].senna = NULL; + share->keyinfo[i].senna_flags = 0; + share->keyinfo[i].senna_initial_n_segments = 0; +#endif /* ENABLE_SENNA */ if (share->keyinfo[i].flag & HA_SPATIAL) { #ifdef HAVE_SPATIAL @@ -340,6 +345,28 @@ } else if (share->keyinfo[i].flag & HA_FULLTEXT) { +#ifdef ENABLE_SENNA + if (!(share->keyinfo[i].senna_flags & SEN_DISABLE_SENNA)) + { + sen_log("share->delay_key_write=%d", share->delay_key_write); + sen_log("unique_file_name %s", share->unique_file_name); + sen_log("data_file_name %s", share->data_file_name); + sen_log("index_file_name %s", share->index_file_name); + sen_log("share->keyinfo[%d].seg=%d", i, pos-FT_SEGS); + { + char buf[FN_REFLEN]; + strncpy(buf, share->unique_file_name, FN_REFLEN - 1); + buf[FN_REFLEN - 1] = '\0'; + sprintf(buf + strlen(buf) - 3, "%03d", i); + sen_log("open (%s)", buf); + share->keyinfo[i].senna = sen_index_open(buf); + sen_index_info(share->keyinfo[i].senna, NULL, + &share->keyinfo[i].senna_flags, + &share->keyinfo[i].senna_initial_n_segments, + NULL, NULL, NULL, NULL, NULL, NULL, NULL); + } + } +#endif /* ENABLE_SENNA */ if (!fulltext_keys) { /* 4.0 compatibility code, to be removed in 5.0 */ share->keyinfo[i].seg=pos-FT_SEGS; diff -Nurd mysql-4.1.22/myisam/mi_rename.c mysql-4.1.22.senna/myisam/mi_rename.c --- mysql-4.1.22/myisam/mi_rename.c 2006-11-03 11:17:13.000000000 +0900 +++ mysql-4.1.22.senna/myisam/mi_rename.c 2007-03-24 12:53:40.521539000 +0900 @@ -46,6 +46,17 @@ #endif #endif /* USE_RAID */ +#ifdef ENABLE_SENNA + sen_log("mi_rename(%s,%s)", old_name, new_name); + { + int i; + for (i = 0; i < 100; i++) { + my_snprintf(from, FN_REFLEN, "%s.%03d", old_name, i); + my_snprintf(to, FN_REFLEN, "%s.%03d", new_name, i); + sen_index_rename(from, to); + } + } +#endif /* ENABLE_SENNA */ fn_format(from,old_name,"",MI_NAME_IEXT,4); fn_format(to,new_name,"",MI_NAME_IEXT,4); if (my_rename_with_symlink(from, to, MYF(MY_WME))) diff -Nurd mysql-4.1.22/myisam/myisam_ftdump.c mysql-4.1.22.senna/myisam/myisam_ftdump.c --- mysql-4.1.22/myisam/myisam_ftdump.c 2006-11-03 11:15:46.000000000 +0900 +++ mysql-4.1.22.senna/myisam/myisam_ftdump.c 2007-03-24 12:53:40.522478000 +0900 @@ -64,6 +64,10 @@ struct { MI_INFO *info; } aio0, *aio=&aio0; /* for GWS_IN_USE */ MY_INIT(argv[0]); +#ifdef ENABLE_SENNA + sen_init(); +#endif /* ENABLE_SENNA */ + if ((error= handle_options(&argc, &argv, my_long_options, get_one_option))) exit(error); if (count || dump) diff -Nurd mysql-4.1.22/myisam/myisamchk.c mysql-4.1.22.senna/myisam/myisamchk.c --- mysql-4.1.22/myisam/myisamchk.c 2006-11-03 11:16:32.000000000 +0900 +++ mysql-4.1.22.senna/myisam/myisamchk.c 2007-03-24 12:53:40.536912000 +0900 @@ -102,6 +102,9 @@ get_options(&argc,(char***) &argv); myisam_quick_table_bits=decode_bits; error=0; +#ifdef ENABLE_SENNA + sen_init(); +#endif /* ENABLE_SENNA */ while (--argc >= 0) { int new_error=myisamchk(&check_param, *(argv++)); @@ -1010,6 +1013,9 @@ } if (!error) { +#ifdef ENABLE_SENNA + ft_index_truncate(info); +#endif /* ENABLE_SENNA */ if ((param->testflag & (T_REP_BY_SORT | T_REP_PARALLEL)) && (share->state.key_map || (rep_quick && !param->keys_in_use && !recreate)) && @@ -1066,6 +1072,9 @@ { if (param->verbose) puts("Table had a compressed index; We must now recreate the index"); +#ifdef ENABLE_SENNA + ft_index_truncate(info); +#endif /* ENABLE_SENNA */ error=mi_repair_by_sort(param,info,filename,1); } } diff -Nurd mysql-4.1.22/myisam/myisamlog.c mysql-4.1.22.senna/myisam/myisamlog.c --- mysql-4.1.22/myisam/myisamlog.c 2006-11-03 11:15:42.000000000 +0900 +++ mysql-4.1.22.senna/myisam/myisamlog.c 2007-03-24 12:53:40.544900000 +0900 @@ -85,6 +85,9 @@ int error,i,first; ulong total_count,total_error,total_recover; MY_INIT(argv[0]); +#ifdef ENABLE_SENNA + sen_init(); +#endif /* ENABLE_SENNA */ log_filename=myisam_log_filename; get_options(&argc,&argv); diff -Nurd mysql-4.1.22/myisam/myisampack.c mysql-4.1.22.senna/myisam/myisampack.c --- mysql-4.1.22/myisam/myisampack.c 2006-11-03 11:15:26.000000000 +0900 +++ mysql-4.1.22.senna/myisam/myisampack.c 2007-03-24 12:53:40.547822000 +0900 @@ -194,6 +194,9 @@ PACK_MRG_INFO merge; char **default_argv; MY_INIT(argv[0]); +#ifdef ENABLE_SENNA + sen_init(); +#endif /* ENABLE_SENNA */ load_defaults("my",load_default_groups,&argc,&argv); default_argv= argv; diff -Nurd mysql-4.1.22/myisammrg/Makefile.am mysql-4.1.22.senna/myisammrg/Makefile.am --- mysql-4.1.22/myisammrg/Makefile.am 2006-11-03 11:15:37.000000000 +0900 +++ mysql-4.1.22.senna/myisammrg/Makefile.am 2007-03-24 12:53:40.551101000 +0900 @@ -15,7 +15,8 @@ # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA INCLUDES = @MT_INCLUDES@ \ - -I$(top_builddir)/include -I$(top_srcdir)/include + -I$(top_builddir)/include -I$(top_srcdir)/include \ + @SENNA_INCLUDES@ @MECAB_INCLUDES@ pkglib_LIBRARIES = libmyisammrg.a noinst_HEADERS = myrg_def.h libmyisammrg_a_SOURCES = myrg_open.c myrg_extra.c myrg_info.c myrg_locking.c \ diff -Nurd mysql-4.1.22/sql/Makefile.am mysql-4.1.22.senna/sql/Makefile.am --- mysql-4.1.22/sql/Makefile.am 2006-11-03 11:16:50.000000000 +0900 +++ mysql-4.1.22.senna/sql/Makefile.am 2007-03-24 12:53:40.567562000 +0900 @@ -22,7 +22,8 @@ INCLUDES = @MT_INCLUDES@ @ZLIB_INCLUDES@ \ @bdb_includes@ @innodb_includes@ @ndbcluster_includes@ \ -I$(top_builddir)/include -I$(top_srcdir)/include \ - -I$(top_srcdir)/regex -I$(srcdir) $(openssl_includes) + -I$(top_srcdir)/regex -I$(srcdir) $(openssl_includes) \ + @SENNA_INCLUDES@ @MECAB_INCLUDES@ WRAPLIBS= @WRAPLIBS@ SUBDIRS = share libexec_PROGRAMS = mysqld @@ -43,7 +44,8 @@ @bdb_libs@ @innodb_libs@ @pstack_libs@ \ @innodb_system_libs@ \ @ndbcluster_libs@ @ndbcluster_system_libs@ \ - $(LDADD) $(CXXLDFLAGS) $(WRAPLIBS) @LIBDL@ @openssl_libs@ + $(LDADD) $(CXXLDFLAGS) $(WRAPLIBS) @LIBDL@ @openssl_libs@ \ + @SENNA_LIBS@ @MECAB_LIBS@ noinst_HEADERS = item.h item_func.h item_sum.h item_cmpfunc.h \ item_strfunc.h item_timefunc.h item_uniq.h \ item_create.h item_subselect.h item_row.h \ diff -Nurd mysql-4.1.22/sql/ha_myisam.cc mysql-4.1.22.senna/sql/ha_myisam.cc --- mysql-4.1.22/sql/ha_myisam.cc 2006-11-03 11:15:51.000000000 +0900 +++ mysql-4.1.22.senna/sql/ha_myisam.cc 2007-03-24 12:53:40.576714000 +0900 @@ -134,9 +134,59 @@ const char *ha_myisam::index_type(uint key_number) { - return ((table->key_info[key_number].flags & HA_FULLTEXT) ? - "FULLTEXT" : - (table->key_info[key_number].flags & HA_SPATIAL) ? + if (table->key_info[key_number].flags & HA_FULLTEXT) + { +#ifdef ENABLE_SENNA + MYISAM_SHARE *share = file->s; + if (share->keyinfo[key_number].senna_flags & SEN_DISABLE_SENNA) + { + return "FULLTEXT"; + } + else + { + if (share->keyinfo[key_number].senna_flags & SEN_INDEX_NORMALIZE) + { + if (share->keyinfo[key_number].senna_flags & SEN_INDEX_NGRAM) + { + return "FULLTEXT,SENNA,NORMALIZE,NGRAM"; + } + else + { + if (share->keyinfo[key_number].senna_flags & SEN_INDEX_DELIMITED) + { + return "FULLTEXT,SENNA,NORMALIZE,DELIMITED"; + } + else + { + return "FULLTEXT,SENNA,NORMALIZE"; + } + } + } + else + { + if (share->keyinfo[key_number].senna_flags & SEN_INDEX_NGRAM) + { + return "FULLTEXT,SENNA,NGRAM"; + } + else + { + if (share->keyinfo[key_number].senna_flags & SEN_INDEX_DELIMITED) + { + return "FULLTEXT,SENNA,DELIMITED"; + } + else + { + return "FULLTEXT,SENNA"; + } + } + } + } +#else /* ENABLE_SENNA */ + return "FULLTEXT"; +#endif /* ENABLE_SENNA */ + } + else + return ((table->key_info[key_number].flags & HA_SPATIAL) ? "SPATIAL" : (table->key_info[key_number].algorithm == HA_KEY_ALG_RTREE) ? "RTREE" : @@ -1229,6 +1279,14 @@ delete_length = info.delete_length; check_time = info.check_time; mean_rec_length=info.mean_reclength; +#ifdef ENABLE_SENNA + senna_keys_size = info.senna_keys_size; + senna_keys_file_size = info.senna_keys_file_size; + senna_lexicon_size = info.senna_lexicon_size; + senna_lexicon_file_size = info.senna_lexicon_file_size; + senna_inv_seg_size = info.senna_inv_seg_size; + senna_inv_chunk_size = info.senna_inv_chunk_size; +#endif /* ENABLE_SENNA */ } if (flag & HA_STATUS_CONST) { @@ -1370,6 +1428,18 @@ for (i=0; i < table_arg->keys ; i++, pos++) { keydef[i].flag= (pos->flags & (HA_NOSAME | HA_FULLTEXT | HA_SPATIAL)); +#ifdef ENABLE_SENNA + if ((keydef[i].flag & HA_FULLTEXT)) { + if (info->key_info) { + keydef[i].senna_flags=info->key_info[i].senna_flags; + keydef[i].senna_initial_n_segments=info->key_info[i].senna_initial_n_segments; + sen_log("creating index (%s, flags=%x initial_n_segments=%d)",name,keydef[i].senna_flags,keydef[i].senna_initial_n_segments); + } else { + keydef[i].senna_flags=SEN_DISABLE_SENNA; + keydef[i].senna_initial_n_segments=0; + } + } +#endif /* ENABLE_SENNA */ keydef[i].key_alg= pos->algorithm == HA_KEY_ALG_UNDEF ? (pos->flags & HA_SPATIAL ? HA_KEY_ALG_RTREE : HA_KEY_ALG_BTREE) : pos->algorithm; diff -Nurd mysql-4.1.22/sql/handler.h mysql-4.1.22.senna/sql/handler.h --- mysql-4.1.22/sql/handler.h 2006-11-03 11:16:03.000000000 +0900 +++ mysql-4.1.22.senna/sql/handler.h 2007-03-24 12:53:40.587367000 +0900 @@ -210,6 +210,9 @@ uint raid_type,raid_chunks; uint merge_insert_method; bool table_existed; /* 1 in create if table existed */ +#ifdef ENABLE_SENNA + KEY *key_info; +#endif /* ENABLE_SENNA */ } HA_CREATE_INFO; @@ -279,6 +282,14 @@ enum {NONE=0, INDEX, RND} inited; bool auto_increment_column_changed; bool implicit_emptied; /* Can be !=0 only if HEAP */ +#ifdef ENABLE_SENNA + longlong senna_keys_size; + longlong senna_keys_file_size; + longlong senna_lexicon_size; + longlong senna_lexicon_file_size; + longlong senna_inv_seg_size; + longlong senna_inv_chunk_size; +#endif /* ENABLE_SENNA */ handler(TABLE *table_arg) :table(table_arg), @@ -289,6 +300,11 @@ key_used_on_scan(MAX_KEY), active_index(MAX_KEY), ref_length(sizeof(my_off_t)), block_size(0), raid_type(0), ft_handler(0), inited(NONE), implicit_emptied(0) +#ifdef ENABLE_SENNA + , senna_keys_size((longlong)-1), senna_keys_file_size((longlong)-1) + , senna_lexicon_size((longlong)-1), senna_lexicon_file_size((longlong)-1) + , senna_inv_seg_size((longlong)-1), senna_inv_chunk_size((longlong)-1) +#endif /* ENABLE_SENNA */ {} virtual ~handler(void) { /* TODO: DBUG_ASSERT(inited == NONE); */ } int ha_open(const char *name, int mode, int test_if_locked); diff -Nurd mysql-4.1.22/sql/lex.h mysql-4.1.22.senna/sql/lex.h --- mysql-4.1.22/sql/lex.h 2006-11-03 11:17:09.000000000 +0900 +++ mysql-4.1.22.senna/sql/lex.h 2007-03-24 12:53:40.590117000 +0900 @@ -372,6 +372,10 @@ { "SECOND", SYM(SECOND_SYM)}, { "SECOND_MICROSECOND", SYM(SECOND_MICROSECOND_SYM)}, { "SELECT", SYM(SELECT_SYM)}, + { "SENNA", SYM(SENNA_SYM)}, + { "NORMALIZE", SYM(SENNA_NORMALIZE_SYM)}, + { "NGRAM", SYM(SENNA_NGRAM_SYM)}, + { "DELIMITED", SYM(SENNA_DELIMITED_SYM)}, { "SEPARATOR", SYM(SEPARATOR_SYM)}, { "SERIAL", SYM(SERIAL_SYM)}, { "SERIALIZABLE", SYM(SERIALIZABLE_SYM)}, diff -Nurd mysql-4.1.22/sql/mysqld.cc mysql-4.1.22.senna/sql/mysqld.cc --- mysql-4.1.22/sql/mysqld.cc 2006-11-03 11:15:29.000000000 +0900 +++ mysql-4.1.22.senna/sql/mysqld.cc 2007-03-24 12:53:40.683417000 +0900 @@ -3170,6 +3170,9 @@ (void) thr_setconcurrency(concurrency); // 10 by default +#ifdef ENABLE_SENNA + sen_init(); +#endif /* ENABLE_SENNA */ select_thread=pthread_self(); select_thread_in_use=1; init_ssl(); diff -Nurd mysql-4.1.22/sql/sql_class.h mysql-4.1.22.senna/sql/sql_class.h --- mysql-4.1.22/sql/sql_class.h 2006-11-03 11:17:01.000000000 +0900 +++ mysql-4.1.22.senna/sql/sql_class.h 2007-03-24 12:53:40.702621000 +0900 @@ -245,11 +245,22 @@ List columns; const char *name; bool generated; +#ifdef ENABLE_SENNA + int senna_flags; + int senna_initial_n_segments; +#endif /* ENABLE_SENNA */ +#ifdef ENABLE_SENNA + Key(enum Keytype type_par, const char *name_arg, enum ha_key_alg alg_par, + bool generated_arg, List &cols, int sen_flags=1, int sen_nsegs=0) + :type(type_par), algorithm(alg_par), columns(cols), name(name_arg), + generated(generated_arg), senna_flags(sen_flags), senna_initial_n_segments(sen_nsegs) +#else /* ENABLE_SENNA */ Key(enum Keytype type_par, const char *name_arg, enum ha_key_alg alg_par, bool generated_arg, List &cols) :type(type_par), algorithm(alg_par), columns(cols), name(name_arg), generated(generated_arg) +#endif /* ENABLE_SENNA */ {} ~Key() {} /* Equality comparison of keys (ignoring name) */ diff -Nurd mysql-4.1.22/sql/sql_db.cc mysql-4.1.22.senna/sql/sql_db.cc --- mysql-4.1.22/sql/sql_db.cc 2006-11-03 11:16:02.000000000 +0900 +++ mysql-4.1.22.senna/sql/sql_db.cc 2007-03-24 12:53:40.730938000 +0900 @@ -809,6 +809,11 @@ (file->name[1] == '.' && !file->name[2]))) continue; +#ifdef ENABLE_SENNA + /* senna files is skip */ + /* ".SEN",".SEN.i",".SEN.i.c",".SEN.l", ".SEN.i.c.001",.. and so on */ + if (strstr(file->name, ".SEN")) { continue; } +#endif /* ENABLE_SENNA */ /* Check if file is a raid directory */ if ((my_isdigit(&my_charset_latin1, file->name[0]) || (file->name[0] >= 'a' && file->name[0] <= 'f')) && diff -Nurd mysql-4.1.22/sql/sql_delete.cc mysql-4.1.22.senna/sql/sql_delete.cc --- mysql-4.1.22/sql/sql_delete.cc 2006-11-03 11:17:34.000000000 +0900 +++ mysql-4.1.22.senna/sql/sql_delete.cc 2007-03-24 12:53:40.743320000 +0900 @@ -650,6 +650,9 @@ if (thd->slave_thread) --slave_open_temp_tables; *fn_ext(path)=0; // Remove the .frm extension +#ifdef ENABLE_SENNA + create_info.key_info=table->key_info; +#endif /* ENABLE_SENNA */ ha_create_table(path, &create_info,1); // We don't need to call invalidate() because this table is not in cache if ((error= (int) !(open_temporary_table(thd, path, table_list->db, diff -Nurd mysql-4.1.22/sql/sql_lex.h mysql-4.1.22.senna/sql/sql_lex.h --- mysql-4.1.22/sql/sql_lex.h 2006-11-03 11:15:11.000000000 +0900 +++ mysql-4.1.22.senna/sql/sql_lex.h 2007-03-24 12:53:40.744963000 +0900 @@ -672,6 +672,11 @@ list of those tables after they are opened. */ TABLE_LIST *time_zone_tables_used; +#ifdef ENABLE_SENNA + int senna_flags; + int senna_initial_n_segments; + inline void senna_clear() { senna_flags=1; senna_initial_n_segments=0; } +#endif /* ENABLE_SENNA */ st_lex(); inline void uncacheable(uint8 cause) { diff -Nurd mysql-4.1.22/sql/sql_show.cc mysql-4.1.22.senna/sql/sql_show.cc --- mysql-4.1.22/sql/sql_show.cc 2006-11-03 11:15:56.000000000 +0900 +++ mysql-4.1.22.senna/sql/sql_show.cc 2007-03-24 12:53:40.758788000 +0900 @@ -26,6 +26,12 @@ #include "ha_berkeley.h" // For berkeley_show_logs #endif +#ifdef ENABLE_SENNA +#ifdef HAVE_ISAM +#include "ha_myisam.h" // For isam +#endif +#endif /* ENABLE_SENNA */ + static const char *grant_names[]={ "select","insert","update","delete","create","drop","reload","shutdown", "process","file","grant","references","index","alter"}; @@ -500,6 +506,20 @@ item->maybe_null=1; field_list.push_back(item=new Item_empty_string("Create_options",255)); item->maybe_null=1; +#ifdef ENABLE_SENNA + field_list.push_back(item=new Item_int("Senna_key_size",(longlong) 1,21)); + item->maybe_null=1; + field_list.push_back(item=new Item_int("Senna_key_file_size",(longlong) 1,21)); + item->maybe_null=1; + field_list.push_back(item=new Item_int("Senna_lexicon_size",(longlong) 1,21)); + item->maybe_null=1; + field_list.push_back(item=new Item_int("Senna_lexicon_file_size",(longlong) 1,21)); + item->maybe_null=1; + field_list.push_back(item=new Item_int("Senna_inv_seg_size",(longlong) 1,21)); + item->maybe_null=1; + field_list.push_back(item=new Item_int("Senna_inv_chunk_size",(longlong) 1,21)); + item->maybe_null=1; +#endif /* ENABLE_SENNA */ field_list.push_back(item=new Item_empty_string("Comment",80)); item->maybe_null=1; if (protocol->send_fields(&field_list,1)) @@ -624,6 +644,38 @@ (ptr == option_buff ? 0 : (uint) (ptr-option_buff)-1) , system_charset_info); } +#ifdef ENABLE_SENNA + if (file->senna_keys_size >= 0) { + protocol->store((ulonglong) file->senna_keys_size); + } else { + protocol->store_null(); + } + if (file->senna_keys_file_size >= 0) { + protocol->store((ulonglong) file->senna_keys_file_size); + } else { + protocol->store_null(); + } + if (file->senna_lexicon_size >= 0) { + protocol->store((ulonglong) file->senna_lexicon_size); + } else { + protocol->store_null(); + } + if (file->senna_lexicon_file_size >= 0) { + protocol->store((ulonglong) file->senna_lexicon_file_size); + } else { + protocol->store_null(); + } + if (file->senna_inv_seg_size >= 0) { + protocol->store((ulonglong) file->senna_inv_seg_size); + } else { + protocol->store_null(); + } + if (file->senna_inv_chunk_size >= 0) { + protocol->store((ulonglong) file->senna_inv_chunk_size); + } else { + protocol->store_null(); + } +#endif /* ENABLE_SENNA */ { char *comment=table->file->update_table_comment(table->comment); protocol->store(comment, system_charset_info); diff -Nurd mysql-4.1.22/sql/sql_table.cc mysql-4.1.22.senna/sql/sql_table.cc --- mysql-4.1.22/sql/sql_table.cc 2006-11-03 11:16:00.000000000 +0900 +++ mysql-4.1.22.senna/sql/sql_table.cc 2007-03-24 12:53:40.794999000 +0900 @@ -974,6 +974,9 @@ if (!key_info_buffer || ! key_part_info) DBUG_RETURN(-1); // Out of memory +#ifdef ENABLE_SENNA + create_info->key_info=key_info_buffer; +#endif /* ENABLE_SENNA */ key_iterator.rewind(); key_number=0; for (; (key=key_iterator++) ; key_number++) @@ -1017,6 +1020,10 @@ if (key->generated) key_info->flags|= HA_GENERATED_KEY; +#ifdef ENABLE_SENNA + key_info->senna_flags=key->senna_flags; + key_info->senna_initial_n_segments=key->senna_initial_n_segments; +#endif /* ENABLE_SENNA */ key_info->key_parts=(uint8) key->columns.elements; key_info->key_part=key_part_info; key_info->usable_key_parts= key_number; @@ -3216,6 +3223,21 @@ key_part_length)); } if (key_parts.elements) +#ifdef ENABLE_SENNA + key_list.push_back(new Key(key_info->flags & HA_SPATIAL ? Key::SPATIAL : + (key_info->flags & HA_NOSAME ? + (!my_strcasecmp(system_charset_info, + key_name, primary_key_name) ? + Key::PRIMARY : Key::UNIQUE) : + (key_info->flags & HA_FULLTEXT ? + Key::FULLTEXT : Key::MULTIPLE)), + key_name, + key_info->algorithm, + test(key_info->flags & HA_GENERATED_KEY), + key_parts, + key_info->senna_flags, + key_info->senna_initial_n_segments)); +#else /* ENABLE_SENNA */ key_list.push_back(new Key(key_info->flags & HA_SPATIAL ? Key::SPATIAL : (key_info->flags & HA_NOSAME ? (!my_strcasecmp(system_charset_info, @@ -3227,6 +3249,7 @@ key_info->algorithm, test(key_info->flags & HA_GENERATED_KEY), key_parts)); +#endif /* ENABLE_SENNA */ } { Key *key; diff -Nurd mysql-4.1.22/sql/sql_yacc.yy mysql-4.1.22.senna/sql/sql_yacc.yy --- mysql-4.1.22/sql/sql_yacc.yy 2006-11-03 11:16:13.000000000 +0900 +++ mysql-4.1.22.senna/sql/sql_yacc.yy 2007-03-24 12:53:40.853085000 +0900 @@ -355,6 +355,10 @@ %token ROW_FORMAT_SYM %token ROW_SYM %token RTREE_SYM +%token SENNA_DELIMITED_SYM +%token SENNA_NGRAM_SYM +%token SENNA_NORMALIZE_SYM +%token SENNA_SYM %token SET %token SEPARATOR_SYM %token SERIAL_SYM @@ -733,6 +737,7 @@ subselect_end select_var_list select_var_list_init help opt_len opt_extended_describe prepare prepare_src execute deallocate + opt_senna_list opt_senna_item END_OF_INPUT %type @@ -1023,6 +1028,9 @@ YYABORT; lex->create_list.empty(); lex->key_list.empty(); +#ifdef ENABLE_SENNA + lex->senna_clear(); +#endif /* ENABLE_SENNA */ lex->col_list.empty(); lex->change=NullS; bzero((char*) &lex->create_info,sizeof(lex->create_info)); @@ -1048,8 +1056,13 @@ '(' key_list ')' { LEX *lex=Lex; - +#ifdef ENABLE_SENNA + lex->key_list.push_back(new Key($2,$4.str, $5, 0, lex->col_list, + lex->senna_flags, lex->senna_initial_n_segments)); + lex->senna_clear(); +#else /* ENABLE_SENNA */ lex->key_list.push_back(new Key($2,$4.str, $5, 0, lex->col_list)); +#endif /* ENABLE_SENNA */ lex->col_list.empty(); } | CREATE DATABASE opt_if_not_exists ident @@ -1297,6 +1310,9 @@ field_spec opt_check_constraint | field_spec references { +#ifdef ENABLE_SENNA + Lex->senna_clear(); +#endif /* ENABLE_SENNA */ Lex->col_list.empty(); /* Alloced by sql_alloc */ } ; @@ -1305,15 +1321,28 @@ key_type opt_ident key_alg '(' key_list ')' { LEX *lex=Lex; +#ifdef ENABLE_SENNA + lex->key_list.push_back(new Key($1,$2, $3, 0, lex->col_list, + lex->senna_flags, lex->senna_initial_n_segments)); + lex->senna_clear(); +#else /* ENABLE_SENNA */ lex->key_list.push_back(new Key($1,$2, $3, 0, lex->col_list)); +#endif /* ENABLE_SENNA */ lex->col_list.empty(); /* Alloced by sql_alloc */ } | opt_constraint constraint_key_type opt_ident key_alg '(' key_list ')' { LEX *lex=Lex; const char *key_name= $3 ? $3:$1; +#ifdef ENABLE_SENNA + lex->key_list.push_back(new Key($2, key_name, $4, 0, + lex->col_list, + lex->senna_flags, lex->senna_initial_n_segments)); + lex->senna_clear(); +#else /* ENABLE_SENNA */ lex->key_list.push_back(new Key($2, key_name, $4, 0, lex->col_list)); +#endif /* ENABLE_SENNA */ lex->col_list.empty(); /* Alloced by sql_alloc */ } | opt_constraint FOREIGN KEY_SYM opt_ident '(' key_list ')' references @@ -1328,14 +1357,23 @@ lex->key_list.push_back(new Key(Key::MULTIPLE, $4 ? $4 : $1, HA_KEY_ALG_UNDEF, 1, lex->col_list)); +#ifdef ENABLE_SENNA + lex->senna_clear(); +#endif /* ENABLE_SENNA */ lex->col_list.empty(); /* Alloced by sql_alloc */ } | constraint opt_check_constraint { +#ifdef ENABLE_SENNA + Lex->senna_clear(); +#endif /* ENABLE_SENNA */ Lex->col_list.empty(); /* Alloced by sql_alloc */ } | opt_constraint check_constraint { +#ifdef ENABLE_SENNA + Lex->senna_clear(); +#endif /* ENABLE_SENNA */ Lex->col_list.empty(); /* Alloced by sql_alloc */ } ; @@ -1801,7 +1839,53 @@ { $$= HA_KEY_ALG_RTREE; } - | HASH_SYM { $$= HA_KEY_ALG_HASH; }; + | HASH_SYM { $$= HA_KEY_ALG_HASH; } + | opt_senna_list { $$= HA_KEY_ALG_UNDEF; }; + +opt_senna_list: + opt_senna_item + | opt_senna_item ',' opt_senna_list ; + +opt_senna_item: + SENNA_SYM { +#ifdef ENABLE_SENNA + Lex->senna_flags &= ~SEN_DISABLE_SENNA; +#endif /* ENABLE_SENNA */ + } + | NO_SYM SENNA_SYM { +#ifdef ENABLE_SENNA + Lex->senna_flags |= SEN_DISABLE_SENNA; +#endif /* ENABLE_SENNA */ + } + | SENNA_NORMALIZE_SYM { +#ifdef ENABLE_SENNA + Lex->senna_flags |= SEN_INDEX_NORMALIZE; +#endif /* ENABLE_SENNA */ + } + | NO_SYM SENNA_NORMALIZE_SYM { +#ifdef ENABLE_SENNA + Lex->senna_flags &= ~SEN_INDEX_NORMALIZE; +#endif /* ENABLE_SENNA */ + } + | SENNA_DELIMITED_SYM { +#ifdef ENABLE_SENNA + Lex->senna_flags |= SEN_INDEX_DELIMITED; +#endif /* ENABLE_SENNA */ + } + | SENNA_NGRAM_SYM { +#ifdef ENABLE_SENNA + Lex->senna_flags |= SEN_INDEX_NGRAM; +#endif /* ENABLE_SENNA */ + } + | ULONG_NUM { +#ifdef ENABLE_SENNA + if ($1 < 65536) { + Lex->senna_initial_n_segments=$1; + } else { + Lex->senna_flags |= $1; + } +#endif /* ENABLE_SENNA */ + }; key_list: key_list ',' key_part order_dir { Lex->col_list.push_back($3); } @@ -1852,6 +1936,9 @@ YYABORT; lex->create_list.empty(); lex->key_list.empty(); +#ifdef ENABLE_SENNA + lex->senna_clear(); +#endif /* ENABLE_SENNA */ lex->col_list.empty(); lex->select_lex.init_order(); lex->select_lex.db=lex->name=0; @@ -5380,6 +5467,10 @@ | RTREE_SYM {} | SAVEPOINT_SYM {} | SECOND_SYM {} + | SENNA_DELIMITED_SYM {} + | SENNA_NGRAM_SYM {} + | SENNA_NORMALIZE_SYM {} + | SENNA_SYM {} | SERIAL_SYM {} | SERIALIZABLE_SYM {} | SESSION_SYM {} diff -Nurd mysql-4.1.22/sql/structs.h mysql-4.1.22.senna/sql/structs.h --- mysql-4.1.22/sql/structs.h 2006-11-03 11:15:57.000000000 +0900 +++ mysql-4.1.22.senna/sql/structs.h 2007-03-24 12:53:40.854247000 +0900 @@ -104,6 +104,10 @@ union { int bdb_return_if_eq; } handler; +#ifdef ENABLE_SENNA + int senna_flags; + int senna_initial_n_segments; +#endif /* ENABLE_SENNA */ } KEY;