renamed hcl_readchar() to chl_readbaseinchar().

changed its behavior to work on hcl->c->inarg only
fixed bugs in feed_begin_include() and feed_end_include()
fixed the bug for which the location information is not updated in reading an included file and feeding its content

bin/main-c.c

@@ -568,19 +568,96 @@ static int feed_data (hcl_client_t* client, const void* ptr, hcl_oow_t len)
 
 /* ========================================================================= */
 
+static int send_iov (int sck, struct iovec* iov, int count)
+{
+	int index = 0;
+
+	while (1)
+	{
+		ssize_t nwritten;
+		struct msghdr msg;
+
+		memset (&msg, 0, HCL_SIZEOF(msg));
+		msg.msg_iov = (struct iovec*)&iov[index];
+		msg.msg_iovlen = count - index;
+		nwritten = sendmsg(sck, &msg, 0);
+		/*nwritten = writev(proto->worker->sck, (const struct iovec*)&iov[index], count - index);*/
+		if (nwritten <= -1) 
+		{
+			/* error occurred inside the worker thread shouldn't affect the error information
+			 * in the server object. so here, i just log a message */
+			fprintf (stderr, "Unable to sendmsg on %d - %s\n", sck, strerror(errno));
+			return -1;
+		}
+
+		while (index < count && (size_t)nwritten >= iov[index].iov_len)
+			nwritten -= iov[index++].iov_len;
+
+		if (index == count) break;
+
+		iov[index].iov_base = (void*)((hcl_uint8_t*)iov[index].iov_base + nwritten);
+		iov[index].iov_len -= nwritten;
+	}
+
+	return 0;
+}
+
+static int send_script_line (int sck, const char* line, size_t len)
+{
+	struct iovec iov[3];
+	int count;
+
+	count = 0;
+	iov[count].iov_base = ".SCRIPT ";
+	iov[count++].iov_len = 8;
+	iov[count].iov_base = (char*)line;
+	iov[count++].iov_len = len;
+	iov[count].iov_base = "\n";
+	iov[count++].iov_len = 1;
+
+	return send_iov(sck, iov, count);
+}
+
+static int send_begin_line (int sck)
+{
+	struct iovec iov[2];
+	int count;
+
+	count = 0;
+	iov[count].iov_base = ".BEGIN";
+	iov[count++].iov_len = 6;
+	iov[count].iov_base = "\n";
+	iov[count++].iov_len = 1;
+
+	return send_iov(sck, iov, count);
+}
+
+static int send_end_line (int sck)
+{
+	struct iovec iov[2];
+	int count;
+
+	count = 0;
+	iov[count].iov_base = ".END";
+	iov[count++].iov_len = 4;
+	iov[count].iov_base = "\n";
+	iov[count++].iov_len = 1;
+
+	return send_iov(sck, iov, count);
+}
+
 static int handle_request (hcl_client_t* client, const char* ipaddr, const char* script, int reuse_addr, int shut_wr_after_req)
 {
 	hcl_sckaddr_t sckaddr;
 	hcl_scklen_t scklen;
 	int sckfam;
 	int sck = -1;
-	struct iovec iov[3];
-	int index, count;
 
 	hcl_oow_t used, avail;
 	int x;
 	hcl_bch_t buf[256];
 	ssize_t n;
+	const char* scptr;
 
 	client_xtn_t* client_xtn;
 
@@ -628,42 +705,19 @@ static int handle_request (hcl_client_t* client, const char* ipaddr, const char*
 		goto oops;
 	}
 
-	count = 0;
+	if (send_begin_line(sck) <= -1) goto oops;
-	iov[count].iov_base = ".SCRIPT (do ";
-	iov[count++].iov_len = 12;
-	iov[count].iov_base = (char*)script;
-	iov[count++].iov_len = strlen(script);
-	/* the script above must not include trailing newlines */
-	iov[count].iov_base = ")\n";
-	iov[count++].iov_len = 2;
 
-	index = 0;
+	scptr = script;
 	while (1)
 	{
-		ssize_t nwritten;
+		const char* nl;
-		struct msghdr msg;
+		nl = strchr(scptr, '\n');
-
+		if (send_script_line(sck, scptr, (nl? (nl - scptr): strlen(scptr))) <= -1) goto oops;
-		memset (&msg, 0, HCL_SIZEOF(msg));
+		if (!nl) break;
-		msg.msg_iov = (struct iovec*)&iov[index];
+		scptr = nl + 1;
-		msg.msg_iovlen = count - index;
-		nwritten = sendmsg(sck, &msg, 0);
-		/*nwritten = writev(proto->worker->sck, (const struct iovec*)&iov[index], count - index);*/
-		if (nwritten <= -1) 
-		{
-			/* error occurred inside the worker thread shouldn't affect the error information
-			 * in the server object. so here, i just log a message */
-			fprintf (stderr, "Unable to sendmsg on %d - %s\n", sck, strerror(errno));
-			goto oops;
 	}
 
-		while (index < count && (size_t)nwritten >= iov[index].iov_len)
+	if (send_end_line(sck) <= -1) goto oops;
-			nwritten -= iov[index++].iov_len;
-
-		if (index == count) break;
-
-		iov[index].iov_base = (void*)((hcl_uint8_t*)iov[index].iov_base + nwritten);
-		iov[index].iov_len -= nwritten;
-	}
 
 	if (shut_wr_after_req) shutdown (sck, SHUT_WR);
 

bin/main.c

@@ -258,8 +258,11 @@ static HCL_INLINE int read_input (hcl_t* hcl, hcl_ioinarg_t* arg)
 	ucslen = HCL_COUNTOF(arg->buf);
 	x = hcl_convbtooochars(hcl, bb->buf, &bcslen, arg->buf, &ucslen);
 	if (x <= -1 && ucslen <= 0) return -1;
-	/* if ucslen is greater than 0, i see that some characters have been
+	/* if ucslen is greater than 0, i assume that some characters have been
-	 * converted properly */
+	 * converted properly. as the loop above reads an entire line if not too 
+	 * large, the incomplete sequence error (x == -3) must happen after 
+	 * successful conversion of at least 1 ooch character. so no explicit
+	 * check for the incomplete sequence error is required */
 #else
 	bcslen = (bb->len < HCL_COUNTOF(arg->buf))? bb->len: HCL_COUNTOF(arg->buf);
 	ucslen = bcslen;

lib/hcl-prv.h

@@ -688,6 +688,7 @@ struct hcl_compiler_t
 		{
 			hcl_flx_state_t state;
 			hcl_ioloc_t loc;
+			hcl_ioloc_t _oloc;
 
 			union 
 			{

lib/hcl-s.c

@@ -151,6 +151,8 @@ struct hcl_server_proto_t
 
 	hcl_t* hcl;
 	hcl_iolxc_t* lxc;
+	hcl_oow_t unread_count;
+	hcl_iolxc_t unread_lxc;
 	hcl_server_proto_token_t tok;
 	hcl_tmr_index_t exec_runtime_event_index;
 
@@ -428,28 +430,29 @@ static HCL_INLINE int read_input (hcl_t* hcl, hcl_ioinarg_t* arg)
 	bb_t* bb;
 	hcl_oow_t bcslen, ucslen, remlen;
 	hcl_server_worker_t* worker;
-	int x;
+	ssize_t x;
+	int y;
 
 	bb = (bb_t*)arg->handle;
 	HCL_ASSERT (hcl, bb != HCL_NULL && bb->fd >= 0);
 
 	worker = xtn->proto->worker;
 
+start_over:
 	if (bb->fd == worker->sck)
 	{
-		ssize_t x;
 		hcl_server_t* server;
 
+		HCL_ASSERT (hcl, arg->includer == HCL_NULL);
 		server = worker->server;
 
-	start_over:
 		while (1)
 		{
 			int n;
 			struct pollfd pfd;
 			int tmout, actual_tmout;
 
-			if (server->stopreq)
+			if (HCL_UNLIKELY(server->stopreq))
 			{
 				hcl_seterrbfmt (hcl, HCL_EGENERIC, "stop requested");
 				return -1;
@@ -489,11 +492,18 @@ static HCL_INLINE int read_input (hcl_t* hcl, hcl_ioinarg_t* arg)
 	}
 	else
 	{
-		ssize_t x;
+		HCL_ASSERT (hcl, arg->includer != HCL_NULL);
+
+		if (HCL_UNLIKELY(worker->server->stopreq))
+		{
+			hcl_seterrbfmt (hcl, HCL_EGENERIC, "stop requested");
+			return -1;
+		}
+
 		x = read(bb->fd, &bb->buf[bb->len], HCL_COUNTOF(bb->buf) - bb->len);
 		if (x <= -1)
 		{
-			/* TODO: if (errno == EINTR) retry? */
+			if (errno == EINTR) goto start_over;
 			hcl_seterrwithsyserr (hcl, 0, errno);
 			return -1;
 		}
@@ -504,8 +514,12 @@ static HCL_INLINE int read_input (hcl_t* hcl, hcl_ioinarg_t* arg)
 #if defined(HCL_OOCH_IS_UCH)
 	bcslen = bb->len;
 	ucslen = HCL_COUNTOF(arg->buf);
-	x = hcl_convbtooochars(hcl, bb->buf, &bcslen, arg->buf, &ucslen);
+	y = hcl_convbtooochars(hcl, bb->buf, &bcslen, arg->buf, &ucslen);
-	if (x <= -1 && ucslen <= 0) return -1;
+	if (y <= -1 && ucslen <= 0) 
+	{
+		if (y == -3 && x != 0) goto start_over; /* incomplete sequence and not EOF yet */
+		return -1;
+	}
 	/* if ucslen is greater than 0, i see that some characters have been
 	 * converted properly */
 #else
@@ -919,26 +933,42 @@ static HCL_INLINE int is_digitchar (hcl_ooci_t c)
 
 static HCL_INLINE int read_char (hcl_server_proto_t* proto)
 {
-	proto->lxc = hcl_readchar(proto->hcl);
+	if (proto->unread_count > 0) 
+	{
+		proto->lxc = &proto->unread_lxc;
+		proto->unread_count--;
+		return 0;
+	}
+
+	proto->lxc = hcl_readbaseinchar(proto->hcl);
 	if (!proto->lxc) return -1;
 	return 0;
 }
 
 static HCL_INLINE int unread_last_char (hcl_server_proto_t* proto)
 {
-	return hcl_unreadchar(proto->hcl, proto->lxc);
+	if (proto->unread_count >= 1)
+	{
+		/* only 1 character can be unread */
+		hcl_seterrbfmt (proto->hcl, HCL_EFLOOD, "too many unread characters");
+		return -1;
 	}
 
-#define GET_CHAR_TO(proto,c) \
+	if (proto->lxc != &proto->unread_lxc) proto->unread_lxc = *proto->lxc;
+	proto->unread_count++;
+	return 0;
+}
+
+#define GET_CHAR_TO(proto,ch) \
 	do { \
 		if (read_char(proto) <= -1) return -1; \
-		c = (proto)->lxc->c; \
+		ch = (proto)->lxc->c; \
 	} while(0)
 
-#define GET_CHAR_TO_WITH_GOTO(proto,c,oops) \
+#define GET_CHAR_TO_WITH_GOTO(proto,ch,oops) \
 	do { \
 		if (read_char(proto) <= -1) goto oops; \
-		c = (proto)->lxc->c; \
+		ch = (proto)->lxc->c; \
 	} while(0)
 
 #define UNGET_LAST_CHAR(proto) \
@@ -947,8 +977,8 @@ static HCL_INLINE int unread_last_char (hcl_server_proto_t* proto)
 	} while (0)
 
 #define SET_TOKEN_TYPE(proto,tv) ((proto)->tok.type = (tv))
-#define ADD_TOKEN_CHAR(proto,c) \
+#define ADD_TOKEN_CHAR(proto,ch) \
-	do { if (add_token_char(proto, c) <= -1) return -1; } while (0)
+	do { if (add_token_char(proto, ch) <= -1) return -1; } while (0)
 
 
 static HCL_INLINE int add_token_char (hcl_server_proto_t* proto, hcl_ooch_t c)
@@ -1200,6 +1230,30 @@ static void send_error_message (hcl_server_proto_t* proto, const hcl_ooch_t* err
 	}
 }
 
+static void send_proto_hcl_error (hcl_server_proto_t* proto)
+{
+	hcl_errnum_t err;
+
+	err = hcl_geterrnum(proto->hcl);
+	if (err == HCL_ESYNERR)
+	{
+		const hcl_ooch_t* bem;
+		hcl_synerr_t synerr;
+		static hcl_ooch_t nullstr[] = { '\0' };
+
+		/* concatenate the error message with the error location */
+		hcl_getsynerr (proto->hcl, &synerr);
+		bem = hcl_backuperrmsg(proto->hcl);
+		hcl_seterrbfmt (proto->hcl, HCL_ESYNERR, "%js (%js%hs%zu,%zu)", 
+			bem, 
+			(synerr.loc.file? synerr.loc.file: nullstr),
+			(synerr.loc.file? ":": ""),
+			synerr.loc.line, synerr.loc.colm);
+	}
+	
+	send_error_message (proto, hcl_geterrmsg(proto->hcl));
+}
+
 static void show_server_workers (hcl_server_proto_t* proto)
 {
 	hcl_server_t* server;
@@ -1346,6 +1400,8 @@ int hcl_server_proto_handle_request (hcl_server_proto_t* proto)
 				goto fail_with_errmsg;
 			}
 
+			if (hcl_endfeed(proto->hcl) <= -1) goto fail_with_errmsg;
+
 			proto->worker->opstate = HCL_SERVER_WORKER_OPSTATE_EXECUTE;
 			/* i must not jump to fail_with_errmsg when execute_script() fails.
 			 * it may have produced some normal output already. so the function
@@ -1356,59 +1412,62 @@ int hcl_server_proto_handle_request (hcl_server_proto_t* proto)
 
 		case HCL_SERVER_PROTO_TOKEN_SCRIPT:
 		{
-			hcl_cnode_t* obj;
 			hcl_ooci_t c;
-			int n;
+			hcl_ooch_t ch;
+			hcl_oow_t feed_count = 0;
 
+			if (proto->req.state == HCL_SERVER_PROTO_REQ_IN_TOP_LEVEL) 
+			{
 				hcl_setbaseinloc (proto->hcl, 1, 1);
+				hcl_reset(proto->hcl);
+			}
 
-			/* do a special check bypassing get_token(). it checks if the script contents
+			/* check the first character after .SCRIPT */
-			 * come on the same line as .SCRIPT */
 			GET_CHAR_TO_WITH_GOTO (proto, c, fail_with_errmsg);
-			while (is_spacechar(c)) GET_CHAR_TO (proto, c);
+			if (c == HCL_OOCI_EOF)
-			if (c == HCL_OOCI_EOF || c == '\n')
 			{
 				hcl_seterrbfmt (proto->hcl, HCL_EINVAL, "No contents on the .SCRIPT line");
 				goto fail_with_errmsg;
 			}
-			UNGET_LAST_CHAR (proto);
 
-			if (proto->req.state == HCL_SERVER_PROTO_REQ_IN_TOP_LEVEL) hcl_reset(proto->hcl);
+			if (c == '\n') goto script_eol;
 
-			proto->worker->opstate = HCL_SERVER_WORKER_OPSTATE_READ;
+			if (!is_spacechar(c))
-			obj = hcl_read(proto->hcl);
-			if (!obj)
 			{
-				if (hcl_geterrnum(proto->hcl) == HCL_ESYNERR) reformat_synerr (proto->hcl);
+				hcl_seterrbfmt (proto->hcl, HCL_EINVAL, "No space after .SCRIPT");
 				goto fail_with_errmsg;
 			}
 
-			if (get_token(proto) <= -1) 
+			while (1)
 			{
-				hcl_freecnode (proto->hcl, obj);
+				GET_CHAR_TO_WITH_GOTO (proto, c, fail_with_errmsg);
-				goto fail_with_errmsg;
+				if (c == '\n' || c == HCL_OOCI_EOF)
+				{
+					if (c == '\n')
+					{
+					script_eol:
+						ch = c;
+						if (hcl_feed(proto->hcl, &ch, 1) <= -1) goto fail_with_errmsg;
+						feed_count++;
 					}
-			if (proto->tok.type != HCL_SERVER_PROTO_TOKEN_NL)
+					break;
-			{
-				hcl_seterrbfmt (proto->hcl, HCL_EINVAL, "No new line after .SCRIPT contents");
-				hcl_freecnode (proto->hcl, obj);
-				goto fail_with_errmsg;
 				}
 
-			proto->worker->opstate = HCL_SERVER_WORKER_OPSTATE_COMPILE;
+				ch = c;
-
+				if (hcl_feed(proto->hcl, &ch, 1) <= -1) goto fail_with_errmsg;
-			/*n = hcl_compile(proto->hcl, obj, HCL_COMPILE_CLEAR_CODE | HCL_COMPILE_CLEAR_FNBLK);*/
+				feed_count++;
-			n = hcl_compile(proto->hcl, obj, 0);
-
-			hcl_freecnode (proto->hcl, obj);
-			if (n <= -1)
-			{
-				if (hcl_geterrnum(proto->hcl) == HCL_ESYNERR) reformat_synerr (proto->hcl);
-				goto fail_with_errmsg;
 			}
 
+			/*
+			if (feed_count == 0)
+			{
+				hcl_seterrbfmt (proto->hcl, HCL_EINVAL, "No contents on the .SCRIPT line");
+				goto fail_with_errmsg;
+			}*/
+
 			if (proto->req.state == HCL_SERVER_PROTO_REQ_IN_TOP_LEVEL)
 			{
+				if (hcl_endfeed(proto->hcl) <= -1) goto fail_with_errmsg;
 				proto->worker->opstate = HCL_SERVER_WORKER_OPSTATE_EXECUTE;
 				if (execute_script(proto, ".SCRIPT") <= -1) return -1;
 			}
@@ -1488,7 +1547,7 @@ int hcl_server_proto_handle_request (hcl_server_proto_t* proto)
 	return 1;
 
 fail_with_errmsg:
-	send_error_message (proto, hcl_geterrmsg(proto->hcl));
+	send_proto_hcl_error (proto);
 	HCL_LOG1 (proto->hcl, SERVER_LOGMASK_ERROR, "Unable to compile .SCRIPT contents - %js\n", hcl_geterrmsg(proto->hcl));
 	return -1;
 }

lib/hcl.h

@@ -1257,8 +1257,6 @@ struct hcl_ioinarg_t
 	{
 		hcl_oow_t pos;
 		hcl_oow_t len;
-		/* <<B.STATE>>
-		int state;*/
 	} b;
 
 	hcl_oow_t line;
@@ -2154,6 +2152,12 @@ HCL_EXPORT void hcl_setbaseinloc (
 	hcl_oow_t colm
 );
 
+/* if you should read charcters from the input stream before hcl_read(),
+ * you can call hcl_readbaseinchar() */
+HCL_EXPORT hcl_iolxc_t* hcl_readbaseinchar (
+	hcl_t* hcl
+);
+
 HCL_EXPORT int hcl_attachio (
 	hcl_t*         hcl,
 	hcl_ioimpl_t   reader,
@@ -2240,20 +2244,6 @@ HCL_EXPORT int hcl_decode (
 #endif
 
 
-/* if you should read charcters from the input stream before hcl_read(),
- * you can call hcl_readchar() */
-HCL_EXPORT hcl_iolxc_t* hcl_readchar (
-	hcl_t* hcl
-);
-
-/* If you use hcl_readchar() to read the input stream, you may use
- * hcl_unreadchar() to put back characters read for hcl_readchar()
- * to return before reading the stream again. */
-HCL_EXPORT int hcl_unreadchar (
-	hcl_t*             hcl,
-	const hcl_iolxc_t* c
-);
-
 /* =========================================================================
  * SYNTAX ERROR HANDLING
  * ========================================================================= */

lib/read.c

@@ -451,64 +451,39 @@ static int get_directive_token_type (hcl_t* hcl, hcl_iotok_type_t* tok_type)
 	return -1;
 }
 
-static int get_char (hcl_t* hcl)
+static int _get_char (hcl_t* hcl, hcl_ioinarg_t* inp)
 {
 	hcl_ooci_t lc;
 
-	if (hcl->c->nungots > 0)
+	if (inp->b.pos >= inp->b.len)
 	{
-		/* something in the unget buffer */
+		if (hcl->c->reader(hcl, HCL_IO_READ, inp) <= -1) return -1;
-		hcl->c->lxc = hcl->c->ungot[--hcl->c->nungots];
-		return 0;
-	}
 
-/* <<B.STATE>> -> probably not needed any more?
+		if (inp->xlen <= 0)
-	if (hcl->c->curinp->b.state == -1)
 		{
-		hcl->c->curinp->b.state = 0;
+			inp->lxc.c = HCL_OOCI_EOF;
-		return -1;
+			inp->lxc.l.line = inp->line;
-	}
+			inp->lxc.l.colm = inp->colm;
-	else if (hcl->c->curinp->b.state == 1)
+			inp->lxc.l.file = inp->name;
-	{
-		hcl->c->curinp->b.state = 0;
-		goto return_eof;
-	}
-*/
-
-	if (hcl->c->curinp->b.pos >= hcl->c->curinp->b.len)
-	{
-		if (hcl->c->reader(hcl, HCL_IO_READ, hcl->c->curinp) <= -1) return -1;
-
-		if (hcl->c->curinp->xlen <= 0)
-		{
-/* <<B.STATE>>
-		return_eof:
- */
-			hcl->c->curinp->lxc.c = HCL_OOCI_EOF;
-			hcl->c->curinp->lxc.l.line = hcl->c->curinp->line;
-			hcl->c->curinp->lxc.l.colm = hcl->c->curinp->colm;
-			hcl->c->curinp->lxc.l.file = hcl->c->curinp->name;
-			hcl->c->lxc = hcl->c->curinp->lxc;
-
 			/* indicate that EOF has been read. lxc.c is also set to EOF. */
 			return 0;
 		}
 
-		hcl->c->curinp->b.pos = 0;
+		inp->b.pos = 0;
-		hcl->c->curinp->b.len = hcl->c->curinp->xlen;
+		inp->b.len = inp->xlen;
 	}
 
-	if (hcl->c->curinp->lxc.c == '\n' || hcl->c->curinp->lxc.c == '\r')
+	if (inp->lxc.c == '\n' || inp->lxc.c == '\r')
 	{
-		/* hcl->c->curinp->lxc.c is a previous character. the new character
+		/* inp->lxc.c is a previous character. the new character
-		 * to be read is still in the buffer (hcl->c->curinp->buf).
+		 * to be read is still in the buffer (inp->buf).
 		 * hcl->cu->curinp->colm has been incremented when the previous
 		 * character has been read. */
-		if (hcl->c->curinp->line > 1 && hcl->c->curinp->colm == 2 && hcl->c->curinp->nl != hcl->c->curinp->lxc.c)
+		if (inp->line > 1 && inp->colm == 2 && inp->nl != inp->lxc.c)
 		{
 			/* most likely, it's the second character in '\r\n' or '\n\r'
 			 * sequence. let's not update the line and column number. */
-			/*hcl->c->curinp->colm = 1;*/
+			/*inp->colm = 1;*/
 		}
 		else
 		{
@@ -517,23 +492,38 @@ static int get_char (hcl_t* hcl)
 			 * incrementing the line number here instead of
 			 * updating inp->lxc causes the line number for
 			 * TOK_EOF to be the same line as the lxc newline. */
-			hcl->c->curinp->line++;
+			inp->line++;
-			hcl->c->curinp->colm = 1;
+			inp->colm = 1;
-			hcl->c->curinp->nl = hcl->c->curinp->lxc.c;
+			inp->nl = inp->lxc.c;
 		}
 	}
 
-	lc = hcl->c->curinp->buf[hcl->c->curinp->b.pos++];
+	lc = inp->buf[inp->b.pos++];
 
-	hcl->c->curinp->lxc.c = lc;
+	inp->lxc.c = lc;
-	hcl->c->curinp->lxc.l.line = hcl->c->curinp->line;
+	inp->lxc.l.line = inp->line;
-	hcl->c->curinp->lxc.l.colm = hcl->c->curinp->colm++;
+	inp->lxc.l.colm = inp->colm++;
-	hcl->c->curinp->lxc.l.file = hcl->c->curinp->name;
+	inp->lxc.l.file = inp->name;
-	hcl->c->lxc = hcl->c->curinp->lxc;
 
 	return 1; /* indicate that a normal character has been read */
 }
 
+static int get_char (hcl_t* hcl)
+{
+	int n;
+
+	if (hcl->c->nungots > 0)
+	{
+		/* something in the unget buffer */
+		hcl->c->lxc = hcl->c->ungot[--hcl->c->nungots];
+		return 0;
+	}
+
+	n = _get_char(hcl, hcl->c->curinp);
+	if (n >= 0) hcl->c->lxc = hcl->c->curinp->lxc;
+	return n;
+}
+
 static int skip_comment (hcl_t* hcl)
 {
 	hcl_ooci_t c = hcl->c->lxc.c;
@@ -2211,25 +2201,6 @@ hcl_cnode_t* hcl_read (hcl_t* hcl)
 
 /* ------------------------------------------------------------------------ */
 
-hcl_iolxc_t* hcl_readchar (hcl_t* hcl)
-{
-	int n = get_char(hcl);
-	if (n <= -1) return HCL_NULL;
-	return &hcl->c->lxc;
-}
-
-int hcl_unreadchar (hcl_t* hcl, const hcl_iolxc_t* c)
-{
-	if (hcl->c->nungots >= HCL_COUNTOF(hcl->c->ungot))
-	{
-		hcl_seterrbfmt (hcl, HCL_EBUFFULL, "character unread buffer full");
-		return -1;
-	}
-
-	unget_char (hcl, c);
-	return 0;
-}
-
 /* TODO:
 hcl_cnodetoobj (hcl_t* hcl, hcl_cnode_t* x)
 {
@@ -2285,6 +2256,24 @@ static int feed_begin_include (hcl_t* hcl)
 		goto oops;
 	}
 
+	if (arg->includer == &hcl->c->inarg)
+	{
+		/* TODO: remove hcl_readbaseinchar() and clean up this part.
+		 * hcl_readbaseinchar(), if called in the middle of feeds,
+		 * updates hcl->c->inarg's line and colm. so use a separate
+		 * field to store the current feed location for now */
+		hcl->c->feed.lx._oloc = hcl->c->feed.lx.loc;
+	}
+	else
+	{
+		arg->includer->name = hcl->c->feed.lx.loc.file;
+		arg->includer->line = hcl->c->feed.lx.loc.line;
+		arg->includer->colm = hcl->c->feed.lx.loc.colm;
+	}
+	hcl->c->feed.lx.loc.file = arg->name;
+	hcl->c->feed.lx.loc.line = arg->line;
+	hcl->c->feed.lx.loc.colm = arg->colm;
+
 	/* switch to the includee's stream */
 	hcl->c->curinp = arg;
 	/* hcl->c->depth.incl++; */
@@ -2315,6 +2304,17 @@ static int feed_end_include (hcl_t* hcl)
 	cur = hcl->c->curinp;
 	hcl->c->curinp = hcl->c->curinp->includer;
 
+	if (hcl->c->curinp == &hcl->c->inarg)
+	{
+		hcl->c->feed.lx.loc = hcl->c->feed.lx._oloc;
+	}
+	else
+	{
+		hcl->c->feed.lx.loc.file = hcl->c->curinp->name;
+		hcl->c->feed.lx.loc.line = hcl->c->curinp->line;
+		hcl->c->feed.lx.loc.colm = hcl->c->curinp->colm;
+	}
+
 	HCL_ASSERT (hcl, cur->name != HCL_NULL);
 	hcl_freemem (hcl, cur);
 	/* hcl->parse.depth.incl--; */
@@ -3692,10 +3692,22 @@ static int feed_char (hcl_t* hcl, hcl_ooci_t c)
 	}
 
 	HCL_ASSERT (hcl, !"internal error - this must never happen");
-	hcl_seterrnum (hcl, HCL_EINTERN);
+	hcl_seterrbfmt (hcl, HCL_EINTERN, "internal error - unknown flx state - %d", FLX_STATE(hcl));
 	return -1;
 }
 
+static void feed_update_lx_loc (hcl_t* hcl, hcl_ooci_t ch)
+{
+	if (is_linebreak(ch))
+	{
+		hcl->c->feed.lx.loc.line++;
+		hcl->c->feed.lx.loc.colm = 1;
+	}
+	else
+	{
+		hcl->c->feed.lx.loc.colm++;
+	}
+}
 
 static int feed_from_included (hcl_t* hcl)
 {
@@ -3725,7 +3737,12 @@ static int feed_from_included (hcl_t* hcl)
 
 		x = feed_char(hcl, hcl->c->curinp->buf[hcl->c->curinp->b.pos]);
 		if (x <= -1) return -1;
+		if (x >= 1) 
+		{
+			/* consumed */
+			feed_update_lx_loc (hcl, hcl->c->curinp->buf[hcl->c->curinp->b.pos]);
 			hcl->c->curinp->b.pos += x;
+		}
 
 		if (hcl->c->feed.rd.do_include_file)
 		{
@@ -3758,7 +3775,6 @@ int hcl_feed (hcl_t* hcl, const hcl_ooch_t* data, hcl_oow_t len)
 	hcl_oow_t i;
 	int x;
 
-
 	if (data) 
 	{
 		for (i = 0; i < len; ) 
@@ -3769,15 +3785,7 @@ int hcl_feed (hcl_t* hcl, const hcl_ooch_t* data, hcl_oow_t len)
 			if (x > 0)
 			{
 				/* consumed */
-				if (is_linebreak(data[i]))
+				feed_update_lx_loc (hcl, data[i]);
-				{
-					hcl->c->feed.lx.loc.line++;
-					hcl->c->feed.lx.loc.colm = 1;
-				}
-				else
-				{
-					hcl->c->feed.lx.loc.colm++;
-				}
 				i += x; /* x is supposed to be 1. otherwise, some characters may get skipped. */
 			}
 
@@ -4061,3 +4069,13 @@ void hcl_setbaseinloc (hcl_t* hcl, hcl_oow_t line, hcl_oow_t colm)
 	hcl->c->inarg.line = line;
 	hcl->c->inarg.colm = colm;
 }
+
+hcl_iolxc_t* hcl_readbaseinchar (hcl_t* hcl)
+{
+	/* read a character using the base input stream. the caller must care extra 
+	 * care when using this function. this function reads the main stream regardless
+	 * of the inclusion status and ignores the ungot characters. */
+	int n = _get_char(hcl, &hcl->c->inarg);
+	if (n <= -1) return HCL_NULL;
+	return &hcl->c->inarg.lxc;
+}