From 81f314d6769ca221d6fc71465d4c8d5b4f4ee6f9 Mon Sep 17 00:00:00 2001 From: hyung-hwan Date: Thu, 28 Oct 2010 06:54:37 +0000 Subject: [PATCH] improved qse_htb_t to use less memory. this resulted in backward-incompatible interface --- qse/cmd/awk/awk.c | 30 ++++- qse/include/qse/cmn/htb.h | 68 +++++++++- qse/lib/awk/Awk.cpp | 28 ++++- qse/lib/awk/awk.c | 132 ++++++++++++++----- qse/lib/awk/run.c | 26 +++- qse/lib/awk/val.c | 37 ++++-- qse/lib/cmn/htb.c | 259 ++++++++++++++++++++++---------------- qse/lib/utl/http.c | 4 +- qse/samples/cmn/htb.c | 8 +- 9 files changed, 411 insertions(+), 181 deletions(-) diff --git a/qse/cmd/awk/awk.c b/qse/cmd/awk/awk.c index b774c180..ef29ecab 100644 --- a/qse/cmd/awk/awk.c +++ b/qse/cmd/awk/awk.c @@ -1,5 +1,5 @@ /* - * $Id: awk.c 344 2010-08-17 13:15:14Z hyunghwan.chung $ + * $Id: awk.c 363 2010-10-27 12:54:37Z hyunghwan.chung $ * Copyright 2006-2009 Chung, Hyung-Hwan. This file is part of QSE. @@ -431,16 +431,34 @@ static int comparg (int argc, qse_char_t* argv[], struct arg_t* arg) goto oops; } - gvm = qse_htb_open (QSE_NULL, 0, 30, 70); + gvm = qse_htb_open ( + QSE_NULL, 0, 30, 70, + QSE_SIZEOF(qse_char_t), QSE_SIZEOF(struct gvmv_t) + ); if (gvm == QSE_NULL) { print_err (QSE_T("out of memory\n")); goto oops; } - /*qse_htb_setcopier (gvm, QSE_HTB_KEY, QSE_HTB_COPIER_INLINE);*/ - qse_htb_setscale (gvm, QSE_HTB_KEY, QSE_SIZEOF(qse_char_t)); - qse_htb_setcopier (gvm, QSE_HTB_VAL, QSE_HTB_COPIER_INLINE); - qse_htb_setscale (gvm, QSE_HTB_VAL, QSE_SIZEOF(struct gvmv_t)); + + { + static qse_htb_mancbs_t mancbs = + { + { + QSE_HTB_COPIER_DEFAULT, /* remember the key pointer */ + QSE_HTB_COPIER_INLINE /* copy a value inline */ + }, + { + QSE_HTB_FREEER_DEFAULT, + QSE_HTB_FREEER_DEFAULT + }, + QSE_HTB_HASHER_DEFAULT, + QSE_HTB_COMPER_DEFAULT, + QSE_HTB_KEEPER_DEFAULT, + QSE_HTB_SIZER_DEFAULT + }; + qse_htb_setmancbs (gvm, &mancbs); + } while ((c = qse_getopt (argc, argv, &opt)) != QSE_CHAR_EOF) { diff --git a/qse/include/qse/cmn/htb.h b/qse/include/qse/cmn/htb.h index 7990f6bd..db389006 100644 --- a/qse/include/qse/cmn/htb.h +++ b/qse/include/qse/cmn/htb.h @@ -1,5 +1,5 @@ /* - * $Id: htb.h 362 2010-10-26 13:01:16Z hyunghwan.chung $ + * $Id: htb.h 363 2010-10-27 12:54:37Z hyunghwan.chung $ * Copyright 2006-2009 Chung, Hyung-Hwan. This file is part of QSE. @@ -142,6 +142,18 @@ struct qse_htb_pair_t qse_htb_pair_t* next; }; +typedef struct qse_htb_mancbs_t qse_htb_mancbs_t; + +struct qse_htb_mancbs_t +{ + qse_htb_copier_t copier[2]; + qse_htb_freeer_t freeer[2]; + qse_htb_hasher_t hasher; /**< key hasher */ + qse_htb_comper_t comper; /**< key comparator */ + qse_htb_keeper_t keeper; /**< value keeper */ + qse_htb_sizer_t sizer; /**< bucket capacity recalculator */ +}; + /** * The qse_htb_t type defines a hash table. */ @@ -149,12 +161,16 @@ struct qse_htb_t { QSE_DEFINE_COMMON_FIELDS (htb) + const qse_htb_mancbs_t* mancbs; + +#if 0 qse_htb_copier_t copier[2]; qse_htb_freeer_t freeer[2]; qse_htb_hasher_t hasher; /**< key hasher */ qse_htb_comper_t comper; /**< key comparator */ qse_htb_keeper_t keeper; /**< value keeper */ qse_htb_sizer_t sizer; /**< bucket capacity recalculator */ +#endif qse_byte_t scale[2]; /**< length scale */ qse_byte_t factor; /**< load factor in percentage */ @@ -169,6 +185,13 @@ struct qse_htb_t #define QSE_HTB_COPIER_SIMPLE ((qse_htb_copier_t)1) #define QSE_HTB_COPIER_INLINE ((qse_htb_copier_t)2) +#define QSE_HTB_COPIER_DEFAULT (QSE_HTB_COPIER_SIMPLE) +#define QSE_HTB_FREEER_DEFAULT (QSE_NULL) +#define QSE_HTB_HASHER_DEFAULT (qse_htb_dflhash) +#define QSE_HTB_COMPER_DEFAULT (qse_htb_dflcomp) +#define QSE_HTB_KEEPER_DEFAULT (QSE_NULL) +#define QSE_HTB_SIZER_DEFAULT (QSE_NULL) + /** * The QSE_HTB_SIZE() macro returns the number of pairs in a hash table. */ @@ -210,7 +233,9 @@ qse_htb_t* qse_htb_open ( qse_mmgr_t* mmgr, /**< memory manager */ qse_size_t ext, /**< extension size in bytes */ qse_size_t capa, /**< initial capacity */ - int factor /**< load factor */ + int factor, /**< load factor */ + int kscale, /**< key scale */ + int vscale /**< value scale */ ); @@ -225,10 +250,12 @@ void qse_htb_close ( * The qse_htb_init() function initializes a hash table */ qse_htb_t* qse_htb_init ( - qse_htb_t* htb, - qse_mmgr_t* mmgr, - qse_size_t capa, - int factor + qse_htb_t* htb, /**< hash table */ + qse_mmgr_t* mmgr, /**< memory manager */ + qse_size_t capa, /**< initial capacity */ + int factor, /**< load factor */ + int kscale, /**< key scale */ + int vscale /**< value scale */ ); /** @@ -253,6 +280,7 @@ qse_size_t qse_htb_getcapa ( qse_htb_t* htb /**< hash table */ ); +#if 0 /** * The qse_htb_getscale() function returns the scale factor */ @@ -274,7 +302,18 @@ void qse_htb_setscale ( qse_htb_id_t id, /**< QSE_HTB_KEY or QSE_HTB_VAL */ int scale /**< scale factor in bytes */ ); +#endif +const qse_htb_mancbs_t* qse_htb_getmancbs ( + qse_htb_t* htb +); + +void qse_htb_setmancbs ( + qse_htb_t* htb, + const qse_htb_mancbs_t* mancbs +); + +#if 0 /** * The qse_htb_getcopier() function gets a data copier. */ @@ -353,6 +392,8 @@ void qse_htb_setsizer ( qse_htb_sizer_t sizer ); +#endif + /** * The qse_htb_search() function searches a hash table to find a pair with a * matching key. It returns the pointer to the pair found. If it fails @@ -468,6 +509,21 @@ qse_htb_pair_t* qse_htb_getnextpair ( qse_size_t* buckno /**< bucket number */ ); + +qse_size_t qse_htb_dflhash ( + qse_htb_t* htb, + const void* kptr, + qse_size_t klen +); + +int qse_htb_dflcomp ( + qse_htb_t* htb, + const void* kptr1, + qse_size_t klen1, + const void* kptr2, + qse_size_t klen2 +); + #ifdef __cplusplus } #endif diff --git a/qse/lib/awk/Awk.cpp b/qse/lib/awk/Awk.cpp index 2f6cad8a..3ab3a327 100644 --- a/qse/lib/awk/Awk.cpp +++ b/qse/lib/awk/Awk.cpp @@ -1,5 +1,5 @@ /* - * $Id: Awk.cpp 328 2010-07-08 06:58:44Z hyunghwan.chung $ + * $Id: Awk.cpp 363 2010-10-27 12:54:37Z hyunghwan.chung $ * Copyright 2006-2009 Chung, Hyung-Hwan. This file is part of QSE. @@ -1128,7 +1128,9 @@ int Awk::open () qse_awk_seterrstr (awk, xerrstr); functionMap = qse_htb_open ( - qse_awk_getmmgr(awk), QSE_SIZEOF(this), 512, 70); + qse_awk_getmmgr(awk), QSE_SIZEOF(this), 512, 70, + QSE_SIZEOF(qse_char_t), 1 + ); if (functionMap == QSE_NULL) { qse_awk_close (awk); @@ -1139,9 +1141,29 @@ int Awk::open () } *(Awk**)QSE_XTN(functionMap) = this; + + static qse_htb_mancbs_t mancbs = + { + { + QSE_HTB_COPIER_INLINE, + QSE_HTB_COPIER_DEFAULT + }, + { + QSE_HTB_FREEER_DEFAULT, + free_function_map_value + }, + QSE_HTB_HASHER_DEFAULT, + QSE_HTB_COMPER_DEFAULT, + QSE_HTB_KEEPER_DEFAULT, + QSE_HTB_SIZER_DEFAULT + }; + + qse_htb_setmancbs (functionMap, &mancbs); +#if 0 + qse_htb_setscale (functionMap, QSE_HTB_KEY, QSE_SIZEOF(qse_char_t)); qse_htb_setcopier (functionMap, QSE_HTB_KEY, QSE_HTB_COPIER_INLINE); qse_htb_setfreeer (functionMap, QSE_HTB_VAL, free_function_map_value); - qse_htb_setscale (functionMap, QSE_HTB_KEY, QSE_SIZEOF(qse_char_t)); +#endif return 0; } diff --git a/qse/lib/awk/awk.c b/qse/lib/awk/awk.c index 934c26a2..e1528256 100644 --- a/qse/lib/awk/awk.c +++ b/qse/lib/awk/awk.c @@ -1,5 +1,5 @@ /* - * $Id: awk.c 344 2010-08-17 13:15:14Z hyunghwan.chung $ + * $Id: awk.c 363 2010-10-27 12:54:37Z hyunghwan.chung $ * Copyright 2006-2009 Chung, Hyung-Hwan. This file is part of QSE. @@ -72,10 +72,75 @@ static void clear_token (qse_awk_tok_t* tok) tok->loc.colm = 0; } + qse_awk_t* qse_awk_open (qse_mmgr_t* mmgr, qse_size_t xtn, qse_awk_prm_t* prm) { qse_awk_t* awk; + static qse_htb_mancbs_t mancbs1 = + { + { + QSE_HTB_COPIER_INLINE, + QSE_HTB_COPIER_INLINE + }, + { + QSE_HTB_FREEER_DEFAULT, + QSE_HTB_FREEER_DEFAULT + }, + QSE_HTB_HASHER_DEFAULT, + QSE_HTB_COMPER_DEFAULT, + QSE_HTB_KEEPER_DEFAULT, + QSE_HTB_SIZER_DEFAULT + }; + + static qse_htb_mancbs_t mancbs2 = + { + { + QSE_HTB_COPIER_INLINE, + QSE_HTB_COPIER_DEFAULT + }, + { + QSE_HTB_FREEER_DEFAULT, + QSE_HTB_FREEER_DEFAULT + }, + QSE_HTB_HASHER_DEFAULT, + QSE_HTB_COMPER_DEFAULT, + QSE_HTB_KEEPER_DEFAULT, + QSE_HTB_SIZER_DEFAULT + }; + + static qse_htb_mancbs_t treefuncbs = + { + { + QSE_HTB_COPIER_INLINE, + QSE_HTB_COPIER_DEFAULT + }, + { + QSE_HTB_FREEER_DEFAULT, + free_fun + }, + QSE_HTB_HASHER_DEFAULT, + QSE_HTB_COMPER_DEFAULT, + QSE_HTB_KEEPER_DEFAULT, + QSE_HTB_SIZER_DEFAULT + }; + + static qse_htb_mancbs_t fncusercbs = + { + { + QSE_HTB_COPIER_INLINE, + QSE_HTB_COPIER_DEFAULT + }, + { + QSE_HTB_FREEER_DEFAULT, + free_fnc + }, + QSE_HTB_HASHER_DEFAULT, + QSE_HTB_COMPER_DEFAULT, + QSE_HTB_KEEPER_DEFAULT, + QSE_HTB_SIZER_DEFAULT + }; + if (mmgr == QSE_NULL) { mmgr = QSE_MMGR_GETDFL(); @@ -113,50 +178,51 @@ qse_awk_t* qse_awk_open (qse_mmgr_t* mmgr, qse_size_t xtn, qse_awk_prm_t* prm) if (init_token (mmgr, &awk->tok) == -1) goto oops; if (init_token (mmgr, &awk->ntok) == -1) goto oops; - awk->wtab = qse_htb_open (mmgr, QSE_SIZEOF(awk), 512, 70); + awk->wtab = qse_htb_open ( + mmgr, QSE_SIZEOF(awk), + 512, 70, QSE_SIZEOF(qse_char_t), QSE_SIZEOF(qse_char_t) + ); if (awk->wtab == QSE_NULL) goto oops; *(qse_awk_t**)QSE_XTN(awk->wtab) = awk; - qse_htb_setcopier (awk->wtab, QSE_HTB_KEY, QSE_HTB_COPIER_INLINE); - qse_htb_setcopier (awk->wtab, QSE_HTB_VAL, QSE_HTB_COPIER_INLINE); - qse_htb_setscale (awk->wtab, QSE_HTB_KEY, QSE_SIZEOF(qse_char_t)); - qse_htb_setscale (awk->wtab, QSE_HTB_VAL, QSE_SIZEOF(qse_char_t)); + qse_htb_setmancbs (awk->wtab, &mancbs1); - awk->rwtab = qse_htb_open (mmgr, QSE_SIZEOF(awk), 512, 70); + awk->rwtab = qse_htb_open ( + mmgr, QSE_SIZEOF(awk), + 512, 70, QSE_SIZEOF(qse_char_t), QSE_SIZEOF(qse_char_t) + ); if (awk->rwtab == QSE_NULL) goto oops; *(qse_awk_t**)QSE_XTN(awk->rwtab) = awk; - qse_htb_setcopier (awk->rwtab, QSE_HTB_KEY, QSE_HTB_COPIER_INLINE); - qse_htb_setcopier (awk->rwtab, QSE_HTB_VAL, QSE_HTB_COPIER_INLINE); - qse_htb_setscale (awk->rwtab, QSE_HTB_KEY, QSE_SIZEOF(qse_char_t)); - qse_htb_setscale (awk->rwtab, QSE_HTB_VAL, QSE_SIZEOF(qse_char_t)); + qse_htb_setmancbs (awk->rwtab, &mancbs1); - awk->sio.names = qse_htb_open (mmgr, QSE_SIZEOF(awk), 128, 70); + awk->sio.names = qse_htb_open ( + mmgr, QSE_SIZEOF(awk), 128, 70, QSE_SIZEOF(qse_char_t), 1 + ); if (awk->sio.names == QSE_NULL) goto oops; *(qse_awk_t**)QSE_XTN(awk->sio.names) = awk; - qse_htb_setcopier (awk->sio.names, QSE_HTB_KEY, QSE_HTB_COPIER_INLINE); - qse_htb_setscale (awk->sio.names, QSE_HTB_KEY, QSE_SIZEOF(qse_char_t)); + qse_htb_setmancbs (awk->sio.names, &mancbs2); awk->sio.inp = &awk->sio.arg; /* TODO: initial map size?? */ - awk->tree.funs = qse_htb_open (mmgr, QSE_SIZEOF(awk), 512, 70); + awk->tree.funs = qse_htb_open ( + mmgr, QSE_SIZEOF(awk), 512, 70, QSE_SIZEOF(qse_char_t), 1 + ); if (awk->tree.funs == QSE_NULL) goto oops; *(qse_awk_t**)QSE_XTN(awk->tree.funs) = awk; - qse_htb_setcopier (awk->tree.funs, QSE_HTB_KEY, QSE_HTB_COPIER_INLINE); - qse_htb_setfreeer (awk->tree.funs, QSE_HTB_VAL, free_fun); - qse_htb_setscale (awk->tree.funs, QSE_HTB_KEY, QSE_SIZEOF(qse_char_t)); + qse_htb_setmancbs (awk->tree.funs, &treefuncbs); - awk->parse.funs = qse_htb_open (mmgr, QSE_SIZEOF(awk), 256, 70); + awk->parse.funs = qse_htb_open ( + mmgr, QSE_SIZEOF(awk), 256, 70, QSE_SIZEOF(qse_char_t), 1 + ); if (awk->parse.funs == QSE_NULL) goto oops; *(qse_awk_t**)QSE_XTN(awk->parse.funs) = awk; - qse_htb_setcopier (awk->parse.funs, QSE_HTB_KEY, QSE_HTB_COPIER_INLINE); - qse_htb_setscale (awk->parse.funs, QSE_HTB_KEY, QSE_SIZEOF(qse_char_t)); + qse_htb_setmancbs (awk->parse.funs, &mancbs2); - awk->parse.named = qse_htb_open (mmgr, QSE_SIZEOF(awk), 256, 70); + awk->parse.named = qse_htb_open ( + mmgr, QSE_SIZEOF(awk), 256, 70, QSE_SIZEOF(qse_char_t), 1 + ); if (awk->parse.named == QSE_NULL) goto oops; *(qse_awk_t**)QSE_XTN(awk->parse.named) = awk; - qse_htb_setcopier ( - awk->parse.named, QSE_HTB_KEY, QSE_HTB_COPIER_INLINE); - qse_htb_setscale ( - awk->parse.named, QSE_HTB_KEY, QSE_SIZEOF(qse_char_t)); + qse_htb_setmancbs (awk->parse.named, &mancbs2); awk->parse.gbls = qse_lda_open (mmgr, QSE_SIZEOF(awk), 128); awk->parse.lcls = qse_lda_open (mmgr, QSE_SIZEOF(awk), 64); @@ -167,16 +233,16 @@ qse_awk_t* qse_awk_open (qse_mmgr_t* mmgr, qse_size_t xtn, qse_awk_prm_t* prm) awk->parse.params == QSE_NULL) goto oops; *(qse_awk_t**)QSE_XTN(awk->parse.gbls) = awk; - qse_lda_setcopier (awk->parse.gbls, QSE_LDA_COPIER_INLINE); qse_lda_setscale (awk->parse.gbls, QSE_SIZEOF(qse_char_t)); + qse_lda_setcopier (awk->parse.gbls, QSE_LDA_COPIER_INLINE); *(qse_awk_t**)QSE_XTN(awk->parse.lcls) = awk; - qse_lda_setcopier (awk->parse.lcls, QSE_LDA_COPIER_INLINE); qse_lda_setscale (awk->parse.lcls, QSE_SIZEOF(qse_char_t)); + qse_lda_setcopier (awk->parse.lcls, QSE_LDA_COPIER_INLINE); *(qse_awk_t**)QSE_XTN(awk->parse.params) = awk; - qse_lda_setcopier (awk->parse.params, QSE_LDA_COPIER_INLINE); qse_lda_setscale (awk->parse.params, QSE_SIZEOF(qse_char_t)); + qse_lda_setcopier (awk->parse.params, QSE_LDA_COPIER_INLINE); awk->option = QSE_AWK_CLASSIC; awk->errinf.num = QSE_AWK_ENOERR; @@ -197,12 +263,12 @@ qse_awk_t* qse_awk_open (qse_mmgr_t* mmgr, qse_size_t xtn, qse_awk_prm_t* prm) awk->tree.chain_size = 0; awk->fnc.sys = QSE_NULL; - awk->fnc.user = qse_htb_open (mmgr, QSE_SIZEOF(awk), 512, 70); + awk->fnc.user = qse_htb_open ( + mmgr, QSE_SIZEOF(awk), 512, 70, QSE_SIZEOF(qse_char_t), 1 + ); if (awk->fnc.user == QSE_NULL) goto oops; *(qse_awk_t**)QSE_XTN(awk->fnc.user) = awk; - qse_htb_setcopier (awk->fnc.user, QSE_HTB_KEY, QSE_HTB_COPIER_INLINE); - qse_htb_setfreeer (awk->fnc.user, QSE_HTB_VAL, free_fnc); - qse_htb_setscale (awk->fnc.user, QSE_HTB_KEY, QSE_SIZEOF(qse_char_t)); + qse_htb_setmancbs (awk->fnc.user, &fncusercbs); if (qse_awk_initgbls (awk) <= -1) goto oops; diff --git a/qse/lib/awk/run.c b/qse/lib/awk/run.c index b9ca49dc..cc173dc7 100644 --- a/qse/lib/awk/run.c +++ b/qse/lib/awk/run.c @@ -1,5 +1,5 @@ /* - * $Id: run.c 343 2010-08-05 07:31:17Z hyunghwan.chung $ + * $Id: run.c 363 2010-10-27 12:54:37Z hyunghwan.chung $ * Copyright 2006-2009 Chung, Hyung-Hwan. This file is part of QSE. @@ -787,6 +787,22 @@ static void same_namedval (qse_htb_t* map, void* dptr, qse_size_t dlen) static int init_rtx (qse_awk_rtx_t* rtx, qse_awk_t* awk, qse_awk_rio_t* rio) { + static qse_htb_mancbs_t mancbs_for_named = + { + { + QSE_HTB_COPIER_INLINE, + QSE_HTB_COPIER_DEFAULT + }, + { + QSE_HTB_FREEER_DEFAULT, + free_namedval + }, + QSE_HTB_HASHER_DEFAULT, + QSE_HTB_COMPER_DEFAULT, + same_namedval, + QSE_HTB_SIZER_DEFAULT + }; + /* zero out the runtime context excluding the extension */ QSE_MEMSET (rtx, 0, QSE_SIZEOF(qse_awk_rtx_t)); @@ -845,7 +861,8 @@ static int init_rtx (qse_awk_rtx_t* rtx, qse_awk_t* awk, qse_awk_rio_t* rio) } rtx->named = qse_htb_open ( - MMGR(rtx), QSE_SIZEOF(rtx), 1024, 70); + MMGR(rtx), QSE_SIZEOF(rtx), 1024, 70, QSE_SIZEOF(qse_char_t), 1 + ); if (rtx->named == QSE_NULL) { qse_str_fini (&rtx->format.fmt); @@ -856,10 +873,7 @@ static int init_rtx (qse_awk_rtx_t* rtx, qse_awk_t* awk, qse_awk_rio_t* rio) return -1; } *(qse_awk_rtx_t**)QSE_XTN(rtx->named) = rtx; - qse_htb_setcopier (rtx->named, QSE_HTB_KEY, QSE_HTB_COPIER_INLINE); - qse_htb_setfreeer (rtx->named, QSE_HTB_VAL, free_namedval); - qse_htb_setkeeper (rtx->named, same_namedval); - qse_htb_setscale (rtx->named, QSE_HTB_KEY, QSE_SIZEOF(qse_char_t)); + qse_htb_setmancbs (rtx->named, &mancbs_for_named); rtx->format.tmp.ptr = (qse_char_t*) QSE_AWK_ALLOC (rtx->awk, 4096*QSE_SIZEOF(qse_char_t*)); diff --git a/qse/lib/awk/val.c b/qse/lib/awk/val.c index bd2cec19..31e25ffa 100644 --- a/qse/lib/awk/val.c +++ b/qse/lib/awk/val.c @@ -1,5 +1,5 @@ /* - * $Id: val.c 328 2010-07-08 06:58:44Z hyunghwan.chung $ + * $Id: val.c 363 2010-10-27 12:54:37Z hyunghwan.chung $ * Copyright 2006-2009 Chung, Hyung-Hwan. This file is part of QSE. @@ -389,6 +389,25 @@ static void same_mapval (qse_htb_t* map, void* dptr, qse_size_t dlen) qse_awk_val_t* qse_awk_rtx_makemapval (qse_awk_rtx_t* rtx) { qse_awk_val_map_t* val; + static qse_htb_mancbs_t mancbs = + { + /* the key is copied inline into a pair and is freed when the pair + * is destroyed. not setting copier for a value means that the pointer + * to the data allocated somewhere else is remembered in a pair. but + * freeing the actual value is handled by free_mapval and same_mapval */ + { + QSE_HTB_COPIER_INLINE, + QSE_HTB_COPIER_DEFAULT + }, + { + QSE_HTB_FREEER_DEFAULT, + free_mapval + }, + QSE_HTB_HASHER_DEFAULT, + QSE_HTB_COMPER_DEFAULT, + same_mapval, + QSE_HTB_SIZER_DEFAULT + }; /* CHECK */ /* @@ -429,7 +448,9 @@ qse_awk_val_t* qse_awk_rtx_makemapval (qse_awk_rtx_t* rtx) val->nstr = 0; val->map = (qse_htb_t*)(val + 1); - val->map = qse_htb_init (val->map, rtx->awk->mmgr, 256, 70); + val->map = qse_htb_init ( + val->map, rtx->awk->mmgr, 256, 70, QSE_SIZEOF(qse_char_t), 1 + ); if (val->map == QSE_NULL) { QSE_AWK_FREE (rtx->awk, val); @@ -437,17 +458,7 @@ qse_awk_val_t* qse_awk_rtx_makemapval (qse_awk_rtx_t* rtx) return QSE_NULL; } *(qse_awk_rtx_t**)QSE_XTN(val->map) = rtx; - - /* the key is copied inline into a pair and is freed when the pair - * is destroyed */ - qse_htb_setcopier (val->map, QSE_HTB_KEY, QSE_HTB_COPIER_INLINE); - qse_htb_setscale (val->map, QSE_HTB_KEY, QSE_SIZEOF(qse_char_t)); - - /* not setting copier for a value means that the pointer to the data - * allocated somewhere else is remembered in a pair. but the freeing - * the actual value is handled by free_mapval and same_mapval */ - qse_htb_setfreeer (val->map, QSE_HTB_VAL, free_mapval); - qse_htb_setkeeper (val->map, same_mapval); + qse_htb_setmancbs (val->map, &mancbs); /* END CHECK */ return (qse_awk_val_t*)val; diff --git a/qse/lib/cmn/htb.c b/qse/lib/cmn/htb.c index 9ba64617..aa39d991 100644 --- a/qse/lib/cmn/htb.c +++ b/qse/lib/cmn/htb.c @@ -1,5 +1,5 @@ /* - * $Id: htb.c 348 2010-08-26 06:26:28Z hyunghwan.chung $ + * $Id: htb.c 363 2010-10-27 12:54:37Z hyunghwan.chung $ * Copyright 2006-2009 Chung, Hyung-Hwan. This file is part of QSE. @@ -52,41 +52,13 @@ QSE_IMPLEMENT_COMMON_FUNCTIONS (htb) #define ENSERT 3 #define INSERT 4 -static int reorganize (htb_t* htb); - -static size_t hash_key (htb_t* htb, const void* kptr, size_t klen) -{ - /*size_t h = 2166136261;*/ - /*size_t h = 0;*/ - size_t h = 5381; - const byte_t* p = (const byte_t*)kptr; - const byte_t* bound = p + klen; - - while (p < bound) - { - /*h = (h * 16777619) ^ *p++;*/ - /*h = h * 31 + *p++;*/ - h = ((h << 5) + h) + *p++; - } - - return h ; -} - -static QSE_INLINE int comp_key (htb_t* htb, - const void* kptr1, size_t klen1, - const void* kptr2, size_t klen2) -{ - if (klen1 == klen2) return QSE_MEMCMP (kptr1, kptr2, KTOB(htb,klen1)); - /* it just returns 1 to indicate that they are different. */ - return 1; -} static pair_t* alloc_pair (htb_t* htb, void* kptr, size_t klen, void* vptr, size_t vlen) { pair_t* n; - copier_t kcop = htb->copier[QSE_HTB_KEY]; - copier_t vcop = htb->copier[QSE_HTB_VAL]; + copier_t kcop = htb->mancbs->copier[QSE_HTB_KEY]; + copier_t vcop = htb->mancbs->copier[QSE_HTB_VAL]; size_t as = SIZEOF(pair_t); if (kcop == QSE_HTB_COPIER_INLINE) as += KTOB(htb,klen); @@ -134,8 +106,8 @@ static pair_t* alloc_pair (htb_t* htb, VPTR(n) = vcop (htb, vptr, vlen); if (VPTR(n) != QSE_NULL) { - if (htb->freeer[QSE_HTB_KEY] != QSE_NULL) - htb->freeer[QSE_HTB_KEY] (htb, KPTR(n), KLEN(n)); + if (htb->mancbs->freeer[QSE_HTB_KEY] != QSE_NULL) + htb->mancbs->freeer[QSE_HTB_KEY] (htb, KPTR(n), KLEN(n)); QSE_MMGR_FREE (htb->mmgr, n); return QSE_NULL; } @@ -146,10 +118,10 @@ static pair_t* alloc_pair (htb_t* htb, static void free_pair (htb_t* htb, pair_t* pair) { - if (htb->freeer[QSE_HTB_KEY] != QSE_NULL) - htb->freeer[QSE_HTB_KEY] (htb, KPTR(pair), KLEN(pair)); - if (htb->freeer[QSE_HTB_VAL] != QSE_NULL) - htb->freeer[QSE_HTB_VAL] (htb, VPTR(pair), VLEN(pair)); + if (htb->mancbs->freeer[QSE_HTB_KEY] != QSE_NULL) + htb->mancbs->freeer[QSE_HTB_KEY] (htb, KPTR(pair), KLEN(pair)); + if (htb->mancbs->freeer[QSE_HTB_VAL] != QSE_NULL) + htb->mancbs->freeer[QSE_HTB_VAL] (htb, VPTR(pair), VLEN(pair)); QSE_MMGR_FREE (htb->mmgr, pair); } @@ -161,14 +133,14 @@ static pair_t* change_pair_val ( /* if the old value and the new value are the same, * it just calls the handler for this condition. * No value replacement occurs. */ - if (htb->keeper != QSE_NULL) + if (htb->mancbs->keeper != QSE_NULL) { - htb->keeper (htb, vptr, vlen); + htb->mancbs->keeper (htb, vptr, vlen); } } else { - copier_t vcop = htb->copier[QSE_HTB_VAL]; + copier_t vcop = htb->mancbs->copier[QSE_HTB_VAL]; void* ovptr = VPTR(pair); size_t ovlen = VLEN(pair); @@ -204,9 +176,9 @@ static pair_t* change_pair_val ( } /* free up the old value */ - if (htb->freeer[QSE_HTB_VAL] != QSE_NULL) + if (htb->mancbs->freeer[QSE_HTB_VAL] != QSE_NULL) { - htb->freeer[QSE_HTB_VAL] (htb, ovptr, ovlen); + htb->mancbs->freeer[QSE_HTB_VAL] (htb, ovptr, ovlen); } } @@ -214,7 +186,24 @@ static pair_t* change_pair_val ( return pair; } -htb_t* qse_htb_open (mmgr_t* mmgr, size_t ext, size_t capa, int factor) +static qse_htb_mancbs_t mancbs = +{ + { + QSE_HTB_COPIER_DEFAULT, + QSE_HTB_COPIER_DEFAULT + }, + { + QSE_HTB_FREEER_DEFAULT, + QSE_HTB_FREEER_DEFAULT + }, + QSE_HTB_HASHER_DEFAULT, + QSE_HTB_COMPER_DEFAULT, + QSE_HTB_KEEPER_DEFAULT, + QSE_HTB_SIZER_DEFAULT +}; + +htb_t* qse_htb_open ( + mmgr_t* mmgr, size_t ext, size_t capa, int factor, int kscale, int vscale) { htb_t* htb; @@ -231,7 +220,7 @@ htb_t* qse_htb_open (mmgr_t* mmgr, size_t ext, size_t capa, int factor) htb = (htb_t*) QSE_MMGR_ALLOC (mmgr, QSE_SIZEOF(htb_t) + ext); if (htb == QSE_NULL) return QSE_NULL; - if (qse_htb_init (htb, mmgr, capa, factor) == QSE_NULL) + if (qse_htb_init (htb, mmgr, capa, factor, kscale, vscale) == QSE_NULL) { QSE_MMGR_FREE (mmgr, htb); return QSE_NULL; @@ -246,7 +235,8 @@ void qse_htb_close (htb_t* htb) QSE_MMGR_FREE (htb->mmgr, htb); } -htb_t* qse_htb_init (htb_t* htb, mmgr_t* mmgr, size_t capa, int factor) +htb_t* qse_htb_init ( + htb_t* htb, mmgr_t* mmgr, size_t capa, int factor, int kscale, int vscale) { if (mmgr == QSE_NULL) mmgr = QSE_MMGR_GETDFL(); @@ -270,15 +260,22 @@ htb_t* qse_htb_init (htb_t* htb, mmgr_t* mmgr, size_t capa, int factor) /*for (i = 0; i < capa; i++) htb->bucket[i] = QSE_NULL;*/ QSE_MEMSET (htb->bucket, 0, capa*SIZEOF(pair_t*)); + htb->factor = factor; +#if 0 htb->scale[QSE_HTB_KEY] = 1; htb->scale[QSE_HTB_VAL] = 1; - htb->factor = factor; +#endif + htb->scale[QSE_HTB_KEY] = (kscale < 1)? 1: kscale; + htb->scale[QSE_HTB_VAL] = (vscale < 1)? 1: vscale; htb->size = 0; htb->capa = capa; htb->threshold = htb->capa * htb->factor / 100; if (htb->capa > 0 && htb->threshold <= 0) htb->threshold = 1; + htb->mancbs = &mancbs; + +#if 0 htb->hasher = hash_key; htb->comper = comp_key; htb->copier[QSE_HTB_KEY] = QSE_HTB_COPIER_SIMPLE; @@ -290,6 +287,7 @@ htb_t* qse_htb_init (htb_t* htb, mmgr_t* mmgr, size_t capa, int factor) htb->keeper = QSE_NULL; htb->sizer = QSE_NULL; */ +#endif return htb; } @@ -300,6 +298,7 @@ void qse_htb_fini (htb_t* htb) QSE_MMGR_FREE (htb->mmgr, htb->bucket); } +#if 0 int qse_htb_getscale (htb_t* htb, qse_htb_id_t id) { QSE_ASSERTX (id == QSE_HTB_KEY || id == QSE_HTB_VAL, @@ -320,7 +319,20 @@ void qse_htb_setscale (htb_t* htb, qse_htb_id_t id, int scale) htb->scale[id] = scale; } +#endif +const qse_htb_mancbs_t* qse_htb_getmancbs (htb_t* htb) +{ + return htb->mancbs; +} + +void qse_htb_setmancbs (htb_t* htb, const qse_htb_mancbs_t* mancbs) +{ + QSE_ASSERT (mancbs != QSE_NULL); + htb->mancbs = mancbs; +} + +#if 0 copier_t qse_htb_getcopier (htb_t* htb, qse_htb_id_t id) { QSE_ASSERTX (id == QSE_HTB_KEY || id == QSE_HTB_VAL, @@ -391,6 +403,7 @@ void qse_htb_setsizer (htb_t* htb, sizer_t sizer) { htb->sizer = sizer; } +#endif size_t qse_htb_getsize (htb_t* htb) { @@ -407,12 +420,12 @@ pair_t* qse_htb_search (htb_t* htb, const void* kptr, size_t klen) pair_t* pair; size_t hc; - hc = htb->hasher(htb,kptr,klen) % htb->capa; + hc = htb->mancbs->hasher(htb,kptr,klen) % htb->capa; pair = htb->bucket[hc]; while (pair != QSE_NULL) { - if (htb->comper (htb, KPTR(pair), KLEN(pair), kptr, klen) == 0) + if (htb->mancbs->comper (htb, KPTR(pair), KLEN(pair), kptr, klen) == 0) { return pair; } @@ -423,13 +436,75 @@ pair_t* qse_htb_search (htb_t* htb, const void* kptr, size_t klen) return QSE_NULL; } +static QSE_INLINE_ALWAYS int reorganize (htb_t* htb) +{ + size_t i, hc, new_capa; + pair_t** new_buck; + + if (htb->mancbs->sizer) + { + new_capa = htb->mancbs->sizer (htb, htb->capa + 1); + + /* if no change in capacity, return success + * without reorganization */ + if (new_capa == htb->capa) return 0; + + /* adjust to 1 if the new capacity is not reasonable */ + if (new_capa <= 0) new_capa = 1; + } + else + { + /* the bucket is doubled until it grows up to 65536 slots. + * once it has reached it, it grows by 65536 slots */ + new_capa = (htb->capa >= 65536)? (htb->capa + 65536): (htb->capa << 1); + } + + new_buck = (pair_t**) QSE_MMGR_ALLOC ( + htb->mmgr, new_capa*SIZEOF(pair_t*)); + if (new_buck == QSE_NULL) + { + /* reorganization is disabled once it fails */ + htb->threshold = 0; + return -1; + } + + /*for (i = 0; i < new_capa; i++) new_buck[i] = QSE_NULL;*/ + QSE_MEMSET (new_buck, 0, new_capa*SIZEOF(pair_t*)); + + for (i = 0; i < htb->capa; i++) + { + pair_t* pair = htb->bucket[i]; + + while (pair != QSE_NULL) + { + pair_t* next = NEXT(pair); + + hc = htb->mancbs->hasher (htb, + KPTR(pair), + KLEN(pair)) % new_capa; + + NEXT(pair) = new_buck[hc]; + new_buck[hc] = pair; + + pair = next; + } + } + + QSE_MMGR_FREE (htb->mmgr, htb->bucket); + htb->bucket = new_buck; + htb->capa = new_capa; + htb->threshold = htb->capa * htb->factor / 100; + + return 0; +} + static pair_t* insert ( htb_t* htb, void* kptr, size_t klen, void* vptr, size_t vlen, int opt) { pair_t* pair, * p, * prev, * next; size_t hc; - hc = htb->hasher(htb,kptr,klen) % htb->capa; + hc = htb->mancbs->hasher(htb,kptr,klen) % htb->capa; pair = htb->bucket[hc]; prev = QSE_NULL; @@ -437,7 +512,7 @@ static pair_t* insert ( { next = NEXT(pair); - if (htb->comper (htb, KPTR(pair), KLEN(pair), kptr, klen) == 0) + if (htb->mancbs->comper (htb, KPTR(pair), KLEN(pair), kptr, klen) == 0) { /* found a pair with a matching key */ switch (opt) @@ -481,7 +556,7 @@ static pair_t* insert ( { if (reorganize(htb) == 0) /* ignore the error */ { - hc = htb->hasher(htb,kptr,klen) % htb->capa; + hc = htb->mancbs->hasher(htb,kptr,klen) % htb->capa; } } @@ -527,13 +602,13 @@ int qse_htb_delete (htb_t* htb, const void* kptr, size_t klen) pair_t* pair, * prev; size_t hc; - hc = htb->hasher(htb,kptr,klen) % htb->capa; + hc = htb->mancbs->hasher(htb,kptr,klen) % htb->capa; pair = htb->bucket[hc]; prev = QSE_NULL; while (pair != QSE_NULL) { - if (htb->comper (htb, KPTR(pair), KLEN(pair), kptr, klen) == 0) + if (htb->mancbs->comper (htb, KPTR(pair), KLEN(pair), kptr, klen) == 0) { if (prev == QSE_NULL) htb->bucket[hc] = NEXT(pair); @@ -635,64 +710,30 @@ pair_t* qse_htb_getnextpair (htb_t* htb, pair_t* pair, size_t* buckno) return QSE_NULL; } -static int reorganize (htb_t* htb) +size_t qse_htb_dflhash (htb_t* htb, const void* kptr, size_t klen) { - size_t i, hc, new_capa; - pair_t** new_buck; + /*size_t h = 2166136261;*/ + /*size_t h = 0;*/ + size_t h = 5381; + const byte_t* p = (const byte_t*)kptr; + const byte_t* bound = p + klen; - if (htb->sizer) + while (p < bound) { - new_capa = htb->sizer (htb, htb->capa + 1); + /*h = (h * 16777619) ^ *p++;*/ + /*h = h * 31 + *p++;*/ + h = ((h << 5) + h) + *p++; + } - /* if no change in capacity, return success - * without reorganization */ - if (new_capa == htb->capa) return 0; - - /* adjust to 1 if the new capacity is not reasonable */ - if (new_capa <= 0) new_capa = 1; - } - else - { - /* the bucket is doubled until it grows up to 65536 slots. - * once it has reached it, it grows by 65536 slots */ - new_capa = (htb->capa >= 65536)? (htb->capa + 65536): (htb->capa << 1); - } - - new_buck = (pair_t**) QSE_MMGR_ALLOC ( - htb->mmgr, new_capa*SIZEOF(pair_t*)); - if (new_buck == QSE_NULL) - { - /* reorganization is disabled once it fails */ - htb->threshold = 0; - return -1; - } - - /*for (i = 0; i < new_capa; i++) new_buck[i] = QSE_NULL;*/ - QSE_MEMSET (new_buck, 0, new_capa*SIZEOF(pair_t*)); - - for (i = 0; i < htb->capa; i++) - { - pair_t* pair = htb->bucket[i]; - - while (pair != QSE_NULL) - { - pair_t* next = NEXT(pair); - - hc = htb->hasher (htb, - KPTR(pair), - KLEN(pair)) % new_capa; - - NEXT(pair) = new_buck[hc]; - new_buck[hc] = pair; - - pair = next; - } - } - - QSE_MMGR_FREE (htb->mmgr, htb->bucket); - htb->bucket = new_buck; - htb->capa = new_capa; - htb->threshold = htb->capa * htb->factor / 100; - - return 0; + return h ; } + +int qse_htb_dflcomp (htb_t* htb, + const void* kptr1, size_t klen1, + const void* kptr2, size_t klen2) +{ + if (klen1 == klen2) return QSE_MEMCMP (kptr1, kptr2, KTOB(htb,klen1)); + /* it just returns 1 to indicate that they are different. */ + return 1; +} + diff --git a/qse/lib/utl/http.c b/qse/lib/utl/http.c index db34dfd5..1d69f16c 100644 --- a/qse/lib/utl/http.c +++ b/qse/lib/utl/http.c @@ -376,7 +376,7 @@ qse_http_t* qse_http_init (qse_http_t* http, qse_mmgr_t* mmgr) http->state.plen = 0; init_buffer (http, &http->req.raw); - if (qse_htb_init (&http->req.hdr, mmgr, 60, 70) == QSE_NULL) + if (qse_htb_init (&http->req.hdr, mmgr, 60, 70, 1, 1) == QSE_NULL) { fini_buffer (http, &http->req.raw); return QSE_NULL; @@ -587,6 +587,7 @@ qse_byte_t* parse_http_header (qse_http_t* http, qse_byte_t* line) } *last = '\0'; + /* add the field name and value into a hash table */ if (qse_htb_insert (&http->req.hdr, name.ptr, name.len, value.ptr, value.len) == QSE_NULL) { @@ -605,6 +606,7 @@ qse_printf (QSE_T("BADHDR\n"), name.ptr, value.ptr); static qse_htb_walk_t walk (qse_htb_t* htb, qse_htb_pair_t* pair, void* ctx) { qse_printf (QSE_T("HEADER OK %d[%S] %d[%S]\n"), (int)pair->klen, pair->kptr, (int)pair->vlen, pair->vptr); + return QSE_HTB_WALK_FORWARD; } /* feed the percent encoded string */ diff --git a/qse/samples/cmn/htb.c b/qse/samples/cmn/htb.c index 40866dae..603f99bb 100644 --- a/qse/samples/cmn/htb.c +++ b/qse/samples/cmn/htb.c @@ -161,10 +161,10 @@ static int test2 () return -1; } - qse_htb_setcopier (s1, QSE_HTB_KEY, QSE_HTB_COPIER_INLINE); - qse_htb_setcopier (s1, QSE_HTB_VAL, QSE_HTB_COPIER_INLINE); qse_htb_setscale (s1, QSE_HTB_KEY, QSE_SIZEOF(qse_char_t)); qse_htb_setscale (s1, QSE_HTB_VAL, QSE_SIZEOF(qse_char_t)); + qse_htb_setcopier (s1, QSE_HTB_KEY, QSE_HTB_COPIER_INLINE); + qse_htb_setcopier (s1, QSE_HTB_VAL, QSE_HTB_COPIER_INLINE); for (i = 0; i < QSE_COUNTOF(keys); i++) { @@ -227,10 +227,10 @@ static int test3 () return -1; } + qse_htb_setscale (s1, QSE_HTB_KEY, QSE_SIZEOF(qse_char_t)); qse_htb_setcopier (s1, QSE_HTB_KEY, QSE_HTB_COPIER_INLINE); qse_htb_setcopier (s1, QSE_HTB_VAL, QSE_HTB_COPIER_INLINE); qse_htb_setcomper (s1, comp_key); - qse_htb_setscale (s1, QSE_HTB_KEY, QSE_SIZEOF(qse_char_t)); for (i = 0; i < QSE_COUNTOF(keys); i++) { @@ -273,10 +273,10 @@ static int test4 () return -1; } + qse_htb_setscale (s1, QSE_HTB_KEY, QSE_SIZEOF(qse_char_t)); qse_htb_setcopier (s1, QSE_HTB_KEY, QSE_HTB_COPIER_INLINE); qse_htb_setcopier (s1, QSE_HTB_VAL, QSE_HTB_COPIER_INLINE); qse_htb_setcomper (s1, comp_key); - qse_htb_setscale (s1, QSE_HTB_KEY, QSE_SIZEOF(qse_char_t)); for (i = 0; i < QSE_COUNTOF(keys); i++) {