enabled qse_http_feed() to get trailing headers after 0

qse/doc/Doxyfile.in

@@ -186,7 +186,7 @@ ALIASES                =
 # For instance, some of the names that are used will be different. The list 
 # of all members will be omitted, etc.
 
-OPTIMIZE_OUTPUT_FOR_C  = NO
+OPTIMIZE_OUTPUT_FOR_C  = YES
 
 # Set the OPTIMIZE_OUTPUT_JAVA tag to YES if your project consists of Java 
 # sources only. Doxygen will then generate output that is more tailored for 
@@ -1397,12 +1397,12 @@ CLASS_GRAPH            = YES
 # indirect implementation dependencies (inheritance, containment, and 
 # class references variables) of the class with other documented classes.
 
-COLLABORATION_GRAPH    = YES
+COLLABORATION_GRAPH    = NO
 
 # If the GROUP_GRAPHS and HAVE_DOT tags are set to YES then doxygen 
 # will generate a graph for groups, showing the direct groups dependencies
 
-GROUP_GRAPHS           = YES
+GROUP_GRAPHS           = NO
 
 # If the UML_LOOK tag is set to YES doxygen will generate inheritance and 
 # collaboration diagrams in a style similar to the OMG's Unified Modeling 

qse/doc/page/awk.doc

@@ -1,4 +1,4 @@
-/** @page awk AWK
+/** @defgroup awk AWK
 
 @section awk_intro INTRODUCTION
 

qse/doc/page/cut.doc

@@ -1,4 +1,4 @@
-/** @page cut TEXT CUTTER
+/** @defgroup cut TEXT CUTTER
 
 Text Cutting Utility
 */

qse/doc/page/sed.doc

@@ -1,4 +1,4 @@
-/** @page sed STREAM EDITOR
+/** @defgroup sed Stream Editor
 
 @section sed_contents CONTENTS
 - \ref sed_intro

qse/include/qse/cut/cut.h

@@ -37,7 +37,8 @@
  */
 
 /** @struct qse_cut_t
- * The qes_cut_t type defines a text cutter.
+ * The qse_cut_t type defines a text cutter. The details are hidden as it is
+ * a large complex structure vulnerable to unintended changes. 
  */
 typedef struct qse_cut_t qse_cut_t;
 
@@ -133,12 +134,12 @@ extern "C" {
 QSE_DEFINE_COMMON_FUNCTIONS (cut)
 
 /**
- * The qse_cut_open() function creates a text cutter object.
+ * The qse_cut_open() function creates a text cutter.
- * @return A pointer to a text cutter on success, QSE_NULL on failure
+ * @return A pointer to a text cutter on success, #QSE_NULL on failure
  */
 qse_cut_t* qse_cut_open (
-	qse_mmgr_t*    mmgr, /**< a memory manager */
+	qse_mmgr_t*    mmgr,   /**< memory manager */
-	qse_size_t     xtn   /**< the size of extension in bytes */
+	qse_size_t     xtnsize /**< extension size in bytes */
 );
 
 /**

qse/include/qse/sed/Sed.hpp

@@ -1,5 +1,5 @@
 /*
- * $Id: Sed.hpp 321 2009-12-21 12:44:33Z hyunghwan.chung $
+ * $Id: Sed.hpp 373 2010-11-26 15:00:57Z hyunghwan.chung $
  *
     Copyright 2006-2009 Chung, Hyung-Hwan.
     This file is part of QSE.
@@ -24,6 +24,18 @@
 #include <qse/cmn/Mmged.hpp>
 #include <qse/sed/sed.h>
 
+/**
+ * @defgroup sed_cxx C++ 
+ * @ingroup sed
+ * @{
+ */
+
+/**
+ * @defgroup sed_cxx_core Core interface
+ * @ingroup sed_cxx
+ * @{
+ */
+
 /** @file
  * Stream Editor
  */
@@ -310,4 +322,7 @@ private:
 QSE_END_NAMESPACE(QSE)
 /////////////////////////////////
 
+/* @} */
+/* @} */
+
 #endif

qse/include/qse/sed/sed.h

@@ -1,5 +1,5 @@
 /*
- * $Id: sed.h 344 2010-08-17 13:15:14Z hyunghwan.chung $
+ * $Id: sed.h 373 2010-11-26 15:00:57Z hyunghwan.chung $
  *
     Copyright 2006-2009 Chung, Hyung-Hwan.
     This file is part of QSE.
@@ -24,6 +24,25 @@
 #include <qse/types.h>
 #include <qse/macros.h>
 
+/**
+ * @defgroup sed_c C
+ * @ingroup sed
+ * @{
+ * hello
+ */
+
+/**
+ * @defgroup sed_c_core Core interface
+ * @ingroup sed_c
+ * @{
+ * The sed library provides data types and functions for creating a custom
+ * stream editor commonly available on platforms. It is a non-interactive 
+ * text editing tool that reads text from an input stream, stores it to 
+ * pattern space, manipulates the pattern space by applying a set of editing 
+ * commands, and writes the pattern space to an output stream. Typically, 
+ * the input and output streams are a console or a file.
+ */
+ 
 /** @file
  * A stream editor performs text transformation on a text stream. 
  *
@@ -423,4 +442,7 @@ void qse_sed_setlinnum (
 }
 #endif
 
+/* @} */
+/* @} */
+
 #endif

qse/include/qse/sed/std.h

@@ -21,10 +21,20 @@
 #ifndef _QSE_SED_STD_H_
 #define _QSE_SED_STD_H_
 
+/** 
+ * @defgroup sed_c_std Helper functions
+ * @ingroup sed_c
+ * @{
+ * If you don't care about the details of memory management and I/O,
+ * you can choose to use the helper functions provided here. It is  
+ * a higher-level interface that is easier to use as it implements 
+ * default handlers for I/O and memory management.
+ */
+
 #include <qse/sed/sed.h>
 
 /** @file
- * This file provides easier-to-use versions of selected API functions
+ * This file provides easier-to-use interface  
  * by implementing default handlers for I/O and memory management.
  *
  * @example sed01.c
@@ -38,8 +48,9 @@ extern "C" {
 
 /**
  * The qse_sed_openstd() function creates a stream editor with the default
- * memory manager and initialized it for other qse_sed_xxxxstd functions.
+ * memory manager and initializes it. Use qse_getxtnstd(), not qse_getxtn() 
- * @return pointer to a stream editor on success, QSE_NULL on failure.
+ * to get the pointer to the extension area.
+ * @return pointer to a stream editor on success, #QSE_NULL on failure.
  */
 qse_sed_t* qse_sed_openstd (
 	qse_size_t xtnsize  /**< extension size in bytes */
@@ -48,8 +59,9 @@ qse_sed_t* qse_sed_openstd (
 /**
  * The qse_sed_openstdwithmmgr() function creates a stream editor with a 
  * user-defined memory manager. It is equivalent to qse_sed_openstd(), 
- * except that you can specify your own memory manager.
+ * except that you can specify your own memory manager.Use qse_getxtnstd(), 
- * @return pointer to a stream editor on success, QSE_NULL on failure.
+ * not qse_getxtn() to get the pointer to the extension area.
+ * @return pointer to a stream editor on success, #QSE_NULL on failure.
  */
 qse_sed_t* qse_sed_openstdwithmmgr (
 	qse_mmgr_t* mmgr,    /**< memory manager */
@@ -62,7 +74,7 @@ qse_sed_t* qse_sed_openstdwithmmgr (
  * created with qse_sed_openstd() and qse_sed_openstdwithmmgr().
  */
 void* qse_sed_getxtnstd (
-	qse_sed_t* sed
+	qse_sed_t* sed /**< stream editor */
 );
 
 /**
@@ -71,26 +83,27 @@ void* qse_sed_getxtnstd (
  * @return 0 on success, -1 on failure
  */
 int qse_sed_compstd (
-	qse_sed_t*        sed,
+	qse_sed_t*        sed, /**< stream editor */
-	const qse_char_t* str
+	const qse_char_t* str  /**< null-terminated script */
 );
 
 /**
  * The qse_sed_execstd() function executes the compiled script
  * over an input file @a infile and an output file @a outfile.
- * If @a infile is QSE_NULL, the standard console input is used.
+ * If @a infile is #QSE_NULL, the standard console input is used.
- * If @a outfile is QSE_NULL, the standard console output is used.
+ * If @a outfile is #QSE_NULL, the standard console output is used.
  * @return 0 on success, -1 on failure
  */
 int qse_sed_execstd (
-	qse_sed_t*        sed,
+	qse_sed_t*        sed,    /**< stream editor */
-	const qse_char_t* infile,
+	const qse_char_t* infile, /**< input file */
-	const qse_char_t* outfile
+	const qse_char_t* outfile /**< output file */
 );
 
 #ifdef __cplusplus
 }
 #endif
 
+/* @} */
 
 #endif

qse/include/qse/utl/http.h

@@ -57,6 +57,7 @@ struct qse_http_t
 
 		qse_http_octb_t raw;
 		qse_http_octb_t con;
+		qse_http_octb_t tra;
 
 		enum
 		{

qse/lib/cut/cut.h

@@ -101,7 +101,6 @@ struct qse_cut_t
 			qse_cstr_t* flds;
 			int delimited;
 		} in;
-
 	} e;
 };
 

qse/lib/utl/http.c

@@ -24,6 +24,8 @@
 
 QSE_IMPLEMENT_COMMON_FUNCTIONS (http)
 
+static const qse_byte_t NUL = '\0';
+
 static QSE_INLINE int is_http_space (qse_char_t c)
 {
 	return QSE_ISSPACE(c) && c != QSE_T('\r') && c != QSE_T('\n');
@@ -354,8 +356,12 @@ static QSE_INLINE void clear_request (qse_http_t* http)
 	 * reading the next request */
 	QSE_MEMSET (&http->req.state, 0, QSE_SIZEOF(http->req.state));
 	QSE_MEMSET (&http->req.attr, 0, QSE_SIZEOF(http->req.attr));
+
 	qse_htb_clear (&http->req.hdr.tab);
+
 	clear_combined_headers (http);
+
+	clear_buffer (http, &http->req.tra);
 	clear_buffer (http, &http->req.con);
 	clear_buffer (http, &http->req.raw);
 }
@@ -407,9 +413,12 @@ qse_http_t* qse_http_init (qse_http_t* http, qse_mmgr_t* mmgr)
 
 	init_buffer (http, &http->req.raw);
 	init_buffer (http, &http->req.con);
+	init_buffer (http, &http->req.tra);
 
 	if (qse_htb_init (&http->req.hdr.tab, mmgr, 60, 70, 1, 1) == QSE_NULL) 
 	{
+		fini_buffer (http, &http->req.tra);
+		fini_buffer (http, &http->req.con);
 		fini_buffer (http, &http->req.raw);
 		return QSE_NULL;
 	}
@@ -421,6 +430,7 @@ void qse_http_fini (qse_http_t* http)
 {
 	qse_htb_fini (&http->req.hdr.tab);
 	clear_combined_headers (http);
+	fini_buffer (http, &http->req.tra);
 	fini_buffer (http, &http->req.con);
 	fini_buffer (http, &http->req.raw);
 }
@@ -951,14 +961,13 @@ qse_printf (QSE_T("HEADER OK %d[%S] %d[%S]\n"),  (int)pair->klen, pair->kptr, (i
 static QSE_INLINE int parse_request (
 	qse_http_t* http, const qse_byte_t* req, qse_size_t rlen)
 {
-	static const qse_byte_t nul = '\0';
 	qse_byte_t* p;
 
 	/* add the actual request */
 	if (push_to_buffer (http, &http->req.raw, req, rlen) <= -1) return -1;
 
 	/* add the terminating null for easier parsing */
-	if (push_to_buffer (http, &http->req.raw, &nul, 1) <= -1) return -1;
+	if (push_to_buffer (http, &http->req.raw, &NUL, 1) <= -1) return -1;
 
 	p = http->req.raw.data;
 
@@ -1037,8 +1046,8 @@ static const qse_byte_t* getchunklen (qse_http_t* http, const qse_byte_t* ptr, q
 			{
 				/* length explicity specified to 0
 				   get trailing headers .... */
-				/*TODO: => http->req.state.chunk.phase = GET_CHUNK_TRAILERS;*/
+				http->req.state.chunk.phase = GET_CHUNK_TRAILERS;
-				http->req.state.chunk.phase = GET_CHUNK_DATA;
+//qse_printf (QSE_T("SWITCH TO GET_CHUNK_TRAILERS....\n"));
 			}
 			else
 			{
@@ -1061,11 +1070,86 @@ static const qse_byte_t* getchunklen (qse_http_t* http, const qse_byte_t* ptr, q
 	return ptr;
 }
 
-/* feed the percent encoded string */
+static const qse_byte_t* get_trailing_headers (
-int qse_http_feed (qse_http_t* http, const qse_byte_t* ptr, qse_size_t len)
+	qse_http_t* http, const qse_byte_t* req, const qse_byte_t* end)
 {
-	const qse_byte_t* end = ptr + len;
+	const qse_byte_t* ptr = req;
-	const qse_byte_t* req = ptr;
+
+	while (ptr < end)
+	{
+		register qse_byte_t b = *ptr++;
+
+		switch (b)
+		{
+			case '\0':
+				/* guarantee that the request does not contain a null 
+				 * character */
+				http->errnum = QSE_HTTP_EBADREQ;
+				return -1;
+
+			case '\n':
+				if (http->req.state.crlf <= 1) 
+				{
+					http->req.state.crlf = 2;
+					break;
+				}
+				else
+				{
+					qse_byte_t* p;
+	
+					QSE_ASSERT (http->req.state.crlf <= 3);
+					http->req.state.crlf = 0;
+	
+					if (push_to_buffer (
+						http, &http->req.tra, req, ptr - req) <= -1)
+						return QSE_NULL;
+					if (push_to_buffer (
+						http, &http->req.tra, &NUL, 1) <= -1) 
+						return QSE_NULL;
+	
+					p = http->req.tra.data;
+	
+					do
+					{
+						while (is_whspace_octet(*p)) p++;
+						if (*p == '\0') break;
+	
+						/* TODO: return error if protocol is 0.9.
+						 * HTTP/0.9 must not get headers... */
+	
+						p = parse_header_fields (http, p);
+						if (p == QSE_NULL) return QSE_NULL;
+					}
+					while (1);
+
+					http->req.state.chunk.phase = GET_CHUNK_DONE;
+					goto done;
+				}
+
+			case '\r':
+				if (http->req.state.crlf == 0 || http->req.state.crlf == 2) 
+					http->req.state.crlf++;
+				else http->req.state.crlf = 1;
+				break;
+
+			default:
+				/* mark that neither CR nor LF was seen */
+				http->req.state.crlf = 0;
+		}
+	}
+
+	if (push_to_buffer (http, &http->req.tra, req, ptr - req) <= -1) return QSE_NULL;
+
+done:
+	return ptr;
+}
+
+
+/* feed the percent encoded string */
+int qse_http_feed (qse_http_t* http, const qse_byte_t* req, qse_size_t len)
+{
+	const qse_byte_t* end = req + len;
+	const qse_byte_t* ptr = req;
 
 	/* does this goto drop code maintainability? */
 	if (http->req.state.need > 0) goto content_resume;
@@ -1082,10 +1166,8 @@ int qse_http_feed (qse_http_t* http, const qse_byte_t* ptr, qse_size_t len)
 		case GET_CHUNK_CRLF:
 			goto dechunk_crlf;
 
-		/*
 		case GET_CHUNK_TRAILERS:
-			goto ....
+			goto dechunk_get_trailers;
-		*/
 	}
 
 	while (ptr < end)
@@ -1100,21 +1182,32 @@ int qse_http_feed (qse_http_t* http, const qse_byte_t* ptr, qse_size_t len)
 			continue;
 		}
 
-		if (b == '\n')
+		switch (b)
 		{
+			case '\0':
+				/* guarantee that the request does not contain a null 
+				 * character */
+				http->errnum = QSE_HTTP_EBADREQ;
+				return -1;
+
+			case '\n':
 				if (http->req.state.crlf <= 1) 
 				{
-				/* http->req.state.crlf == 0, CR was not seen
+					/* http->req.state.crlf == 0
-				 * http->req.state.crlf == 1, CR was seen 
+					 *   => CR was not seen
-				 * whatever the current case is, mark the 
+					 * http->req.state.crlf == 1
-				 * first LF is seen here.
+					 *   => CR was seen 
+					 * whatever the current case is, 
+					 * mark the first LF is seen here.
 					 */
 					http->req.state.crlf = 2;
 				}
 				else
 				{
-				/* http->req.state.crlf == 2, no 2nd CR before LF
+					/* http->req.state.crlf == 2
-				 * http->req.state.crlf == 3, 2nd CR before LF
+					 *   => no 2nd CR before LF
+					 * http->req.state.crlf == 3
+					 *   => 2nd CR before LF
 					 */
 	
 					/* we got a complete request. */
@@ -1148,6 +1241,19 @@ int qse_http_feed (qse_http_t* http, const qse_byte_t* ptr, qse_size_t len)
 							 * the length has been partially read. */
 							goto feedme_more;
 						}
+						else if (http->req.state.chunk.phase == GET_CHUNK_TRAILERS)
+						{
+						dechunk_get_trailers:
+							ptr = get_trailing_headers (http, ptr, end);
+							if (ptr == QSE_NULL) return -1;
+	
+							if (http->req.state.chunk.phase == GET_CHUNK_TRAILERS)
+							{
+								/* still in the same state.
+								 * the trailers have not been processed fully */
+								goto feedme_more;
+							}
+						}
 					}
 					else
 					{
@@ -1174,8 +1280,10 @@ int qse_http_feed (qse_http_t* http, const qse_byte_t* ptr, qse_size_t len)
 						}
 						else 
 						{
-						/* we are given all needed or more than needed */
+							/* we got all or more than needed */
-						if (push_to_buffer (http, &http->req.con, ptr, http->req.state.need) <= -1) return -1;
+							if (push_to_buffer (
+								http, &http->req.con, ptr, 
+								http->req.state.need) <= -1) return -1;
 							ptr += http->req.state.need;
 							http->req.state.need = 0;
 						}
@@ -1196,13 +1304,15 @@ int qse_http_feed (qse_http_t* http, const qse_byte_t* ptr, qse_size_t len)
 								ptr++;
 	
 								/* more octets still available. 
-							 * let it decode the next chunk */
+								 * let it decode the next chunk 
+								 */
 								if (ptr < end) goto dechunk_start; 
 							
-							/* no more octets available after chunk data.
+								/* no more octets available after 
-							 * the chunk state variables need to be
+								 * chunk data. the chunk state variables
-							 * reset when a jump is made to dechunk_resume
+								 * need to be reset when a jump is made
-							 * upon the next call */
+								 * to dechunk_resume upon the next call
+								 */
 								http->req.state.chunk.phase = GET_CHUNK_LEN;
 								http->req.state.chunk.len = 0;
 								http->req.state.chunk.count = 0;
@@ -1232,25 +1342,19 @@ if (http->req.con.size > 0)
 
 					clear_request (http);
 
-				/* let ptr point to the next character to LF or the optional contents */
+					/* let ptr point to the next character to LF or 
+					 * the optional contents */
 					req = ptr; 
 				}
-		}
+				break;
-		else if (b == '\r')
+
-		{
+			case '\r':
 				if (http->req.state.crlf == 0 || http->req.state.crlf == 2) 
 					http->req.state.crlf++;
 				else http->req.state.crlf = 1;
-		}
+				break;
-		else if (b == '\0')
+
-		{
+			default:
-			/* guarantee that the request does not contain a null 
-			 * character */
-			http->errnum = QSE_HTTP_EBADREQ;
-			return -1;
-		}
-		else
-		{
 				/* increment length of a request in raw 
 				 * excluding crlf */
 				http->req.state.plen++; 
@@ -1267,5 +1371,6 @@ if (http->req.con.size > 0)
 
 feedme_more:
 	return 0;
+
 }