added lexical analysis code

stix/lib/main.c

@@ -179,6 +179,16 @@ static void dump_symbol_table (stix_t* stix)
 	printf ("--------------------------------------------\n");
 }
 
+static char* syntax_error_msg[] = 
+{
+	"no error",
+	"illegal character",
+	"comment not closed",
+	"string not closed",
+	"no character after $",
+	"no valid character after #"
+};
+
 int main (int argc, char* argv[])
 {
 	stix_t* stix;
@@ -250,7 +260,31 @@ printf ("%p\n", a);
 	xtn->source_path = argv[1];
 	if (stix_compile (stix, input_handler) <= -1)
 	{
-		printf ("cannot compile code\n");
+		if (stix->errnum == STIX_ESYNTAX)
+		{
+			stix_synerr_t synerr;
+			stix_getsynerr (stix, &synerr);
+			printf ("ERROR: syntax error at line %lu column %lu - %s", 
+				(unsigned long int)synerr.loc.line, (unsigned long int)synerr.loc.colm,
+				syntax_error_msg[synerr.num]);
+			if (synerr.tgt.len > 0)
+			{
+				stix_bch_t bcs[1024]; /* TODO: right buffer size */
+				stix_size_t bcslen = STIX_COUNTOF(bcs);
+				stix_size_t ucslen = synerr.tgt.len;
+
+				if (stix_ucstoutf8 (synerr.tgt.ptr, &ucslen, bcs, &bcslen) >= 0)
+				{
+					printf (" [%.*s]", (int)bcslen, bcs);
+				}
+
+			}
+			printf ("\n");
+		}
+		else
+		{
+			printf ("ERROR: cannot compile code\n");
+		}
 		stix_close (stix);
 		return -1;
 	}

stix/lib/stix-prv.h

@@ -137,14 +137,20 @@ enum stix_iocmd_t
 };
 typedef enum stix_iocmd_t stix_iocmd_t;
 
-typedef struct stix_iolxc_t stix_iolxc_t;
+struct stix_ioloc_t
-struct stix_iolxc_t
 {
-	stix_uch_t        c;    /**< character */
 	unsigned long     line; /**< line */
 	unsigned long     colm; /**< column */
 	const stix_uch_t* file; /**< file specified in #include */
 };
+typedef struct stix_ioloc_t stix_ioloc_t;
+
+struct stix_iolxc_t
+{
+	stix_uci_t    c; /**< character */
+	stix_ioloc_t  l; /**< location */
+};
+typedef struct stix_iolxc_t stix_iolxc_t;
 
 enum stix_ioarg_flag_t
 {
@@ -188,8 +194,8 @@ struct stix_ioarg_t
 		int pos, len;
 	} b;
 
-	stix_oow_t line;
+	unsigned long line;
-	stix_oow_t colm;
+	unsigned long colm;
 
 	stix_iolxc_t lxc;
 	/*-----------------------------------------------------------------*/
@@ -201,12 +207,101 @@ typedef stix_ssize_t (*stix_ioimpl_t) (
 	stix_ioarg_t* arg
 );
 
+
+
+
+struct stix_iotok_t
+{
+	enum
+	{
+		 STIX_IOTOK_EOF,
+		 STIX_IOTOK_CHRLIT,
+		 STIX_IOTOK_STRLIT,
+		 STIX_IOTOK_SYMLIT,
+		 STIX_IOTOK_NUMLIT,
+		 STIX_IOTOK_IDENT,
+		 STIX_IOTOK_BINSEL,
+		 STIX_IOTOK_KEYWORD,
+		 STIX_IOTOK_PRIMITIVE,
+		 STIX_IOTOK_ASSIGN,
+		 STIX_IOTOK_COLON,
+		 STIX_IOTOK_RETURN,
+		 STIX_IOTOK_LBRACE,
+		 STIX_IOTOK_RBRACE,
+		 STIX_IOTOK_LBRACK,
+		 STIX_IOTOK_RBRACK,
+		 STIX_IOTOK_LPAREN,
+		 STIX_IOTOK_RPAREN,
+		 STIX_IOTOK_APAREN, /* #( */
+		 STIX_IOTOK_BPAREN, /* #[ */
+		 STIX_IOTOK_PERIOD,
+		 STIX_IOTOK_SEMICOLON
+	} type;
+
+	stix_ucs_t name;
+	stix_size_t name_capa;
+
+	stix_ioloc_t loc;
+};
+typedef struct stix_iotok_t stix_iotok_t;
+
+enum stix_synerrnum_t
+{
+	STIX_SYNERR_NOERR,
+	STIX_SYNERR_ILCHR, /* illegal character */
+	STIX_SYNERR_CMTNC, /* comment not closed */
+	STIX_SYNERR_STRNC, /* string not closed */
+	STIX_SYNERR_CLTNT, /* character literal not terminated */
+	STIX_SYNERR_HLTNT, /* hased literal not terminated */
+};
+typedef enum stix_synerrnum_t stix_synerrnum_t;
+
+typedef struct stix_iolink_t stix_iolink_t;
+struct stix_iolink_t
+{
+	stix_iolink_t* link;
+};
+
+struct stix_synerr_t
+{
+	stix_synerrnum_t num;
+	stix_ioloc_t     loc;
+	stix_ucs_t       tgt;
+};
+typedef struct stix_synerr_t stix_synerr_t;
+
 struct stix_compiler_t
 {
-	stix_ioimpl_t impl;   /* input handler */
+	/* input handler */
+	stix_ioimpl_t impl;
+
+	/* information about the last meaningful character read.
+	 * this is a copy of curinp->lxc if no ungetting is performed.
+	 * if there is something in the unget buffer, this is overwritten
+	 * by a value from the buffer when the request to read a character
+	 * is served */
 	stix_iolxc_t  lxc;
-	stix_ioarg_t  arg;    /* static top-level data */
+
-	stix_ioarg_t* curinp;  /* pointer to the current data */
+	/* unget buffer */
+	stix_iolxc_t  ungot[10];
+	int           nungots;
+
+	/* static input data buffer */
+	stix_ioarg_t  arg;    
+
+	/* pointer to the current input data. initially, it points to &arg */
+	stix_ioarg_t* curinp;
+
+	/* the last token read */
+	stix_iotok_t  tok;
+
+	stix_iolink_t* io_names;
+
+	stix_synerr_t synerr;
+
+	/* temporary space to handle an illegal character */
+	stix_uch_t ilchr;
+	stix_ucs_t ilchr_ucs;
 };
 
 #endif
@@ -410,6 +505,11 @@ int stix_compile (
 	stix_ioimpl_t io
 );
 
+void stix_getsynerr (
+	stix_t* stix,
+	stix_synerr_t* synerr
+);
+
 #if defined(__cplusplus)
 }
 #endif

stix/lib/stix.c

@@ -79,9 +79,20 @@ oops:
 
 void stix_fini (stix_t* stix)
 {
+	stix_cb_t* cb;
+
+	for (cb = stix->cblist; cb; cb = cb->next)
+	{
+		if (cb->fini) cb->fini (stix);
+	}
+
 	stix_killheap (stix, stix->newheap);
 	stix_killheap (stix, stix->curheap);
 	stix_killheap (stix, stix->permheap);
+
+
+	/* deregister all callbacks */
+	while (stix->cblist) stix_deregcb (stix, stix->cblist);
 }
 
 stix_mmgr_t* stix_getmmgr (stix_t* stix)
@@ -144,6 +155,37 @@ int stix_getoption (stix_t* stix, stix_option_t id, void* value)
 }
 
 
+stix_cb_t* stix_regcb (stix_t* stix, stix_cb_t* tmpl)
+{
+	stix_cb_t* actual;
+
+	actual = stix_allocmem (stix, STIX_SIZEOF(*actual));
+	if (!actual) return STIX_NULL;
+
+	*actual = *tmpl;
+	actual->next = stix->cblist;
+	actual->prev = STIX_NULL;
+	stix->cblist = actual;
+
+	return actual;
+}
+
+void stix_deregcb (stix_t* stix, stix_cb_t* cb)
+{
+	if (cb == stix->cblist)
+	{
+		stix->cblist = stix->cblist->next;
+		if (stix->cblist) stix->cblist->prev = STIX_NULL;
+	}
+	else
+	{
+		if (cb->next) cb->next->prev = cb->prev;
+		if (cb->prev) cb->prev->next = cb->next;
+	}
+
+	stix_freemem (stix, cb);
+}
+
 stix_oow_t stix_hashbytes (const stix_uint8_t* ptr, stix_oow_t len)
 {
 	stix_oow_t h = 0;
@@ -178,7 +220,7 @@ void* stix_allocmem (stix_t* stix, stix_size_t size)
 	void* ptr;
 
 	ptr = STIX_MMGR_ALLOC (stix->mmgr, size);
-	if (ptr == STIX_NULL) stix->errnum = STIX_ENOMEM;
+	if (!ptr) stix->errnum = STIX_ENOMEM;
 	return ptr;
 }
 
@@ -187,11 +229,18 @@ void* stix_callocmem (stix_t* stix, stix_size_t size)
 	void* ptr;
 
 	ptr = STIX_MMGR_ALLOC (stix->mmgr, size);
-	if (ptr == STIX_NULL) stix->errnum = STIX_ENOMEM;
+	if (!ptr) stix->errnum = STIX_ENOMEM;
 	else STIX_MEMSET (ptr, 0, size);
 	return ptr;
 }
 
+void* stix_reallocmem (stix_t* stix, void* ptr, stix_size_t size)
+{
+	ptr = STIX_MMGR_REALLOC (stix->mmgr, ptr, size);
+	if (!ptr) stix->errnum = STIX_ENOMEM;
+	return ptr;
+}
+
 void stix_freemem (stix_t* stix, void* ptr)
 {
 	STIX_MMGR_FREE (stix->mmgr, ptr);

stix/lib/stix.h

@@ -36,17 +36,26 @@
  * ========================================================================= */
 /* TODO: define these types and macros using autoconf */
 typedef unsigned char      stix_uint8_t;
+typedef signed char        stix_int8_t;
+
 typedef unsigned short int stix_uint16_t;
+typedef signed short int stix_int16_t;
+
 #if defined(__MSDOS__)
 	typedef unsigned long int stix_uint32_t;
+	typedef signed long int stix_int32_t;
 #else
 	typedef unsigned int stix_uint32_t;
+	typedef signed int stix_int32_t;
 #endif
+
 typedef unsigned long int stix_uintptr_t;
 typedef unsigned long int stix_size_t;
-typedef long int          stix_ssize_t;
+typedef signed long int   stix_ssize_t;
+
+typedef stix_uint16_t stix_uch_t; /* TODO ... wchar_t??? */
+typedef stix_int32_t  stix_uci_t;
 
-typedef unsigned short int stix_uch_t; /* TODO ... wchar_t??? */
 typedef char          stix_bch_t;
 
 
@@ -55,19 +64,15 @@ struct stix_ucs_t
 	stix_uch_t* ptr;
 	stix_size_t len;
 };
-
-struct stix_bcs_t
-{
-	stix_bch_t* ptr;
-	stix_size_t len;
-};
-
 typedef struct stix_ucs_t stix_ucs_t;
-typedef struct stix_bcs_t stix_bcs_t;
 
 /* =========================================================================
  * PRIMITIVE MACROS
  * ========================================================================= */
+#define STIX_UCI_EOF ((stix_uci_t)-1)
+#define STIX_UCI_NL  ((stix_uci_t)'\n')
+
+
 #define STIX_SIZEOF(x) (sizeof(x))
 #define STIX_COUNTOF(x) (sizeof(x) / sizeof(x[0]))
 
@@ -105,6 +110,7 @@ typedef struct stix_bcs_t stix_bcs_t;
 	(value = (((type)(value)) | (((bits) & STIX_LBMASK(type,length)) << (offset))))
 
 
+#define STIX
 /** 
  * The STIX_BITS_MAX() macros calculates the maximum value that the 'nbits'
  * bits of an unsigned integer of the given 'type' can hold.
@@ -276,7 +282,11 @@ enum stix_errnum_t
 	STIX_EINVAL,  /**< invalid parameter or data */
 	STIX_ENOENT,  /**< no matching entry */
 	STIX_EIOERR,  /**< I/O error */
-	STIX_EECERR   /**< encoding conversion error */
+	STIX_EECERR,  /**< encoding conversion error */
+
+#if defined(STIX_INCLUDE_COMPILER)
+	STIX_ESYNTAX /** < syntax error */
+#endif
 };
 typedef enum stix_errnum_t stix_errnum_t;
 
@@ -707,12 +717,25 @@ struct stix_heap_t
 	stix_uint8_t* ptr;   /* next allocation pointer */
 };
 
+
+
+
+typedef struct stix_t stix_t;
+
+typedef void (*stix_cbimpl_t) (stix_t* stix);
+
+typedef struct stix_cb_t stix_cb_t;
+struct stix_cb_t
+{
+	stix_cbimpl_t fini;
+	stix_cb_t*    prev;
+	stix_cb_t*    next;
+};
+
 #if defined(STIX_INCLUDE_COMPILER)
 typedef struct stix_compiler_t stix_compiler_t;
 #endif
 
-typedef struct stix_t stix_t;
-
 struct stix_t
 {
 	stix_mmgr_t*  mmgr;
@@ -725,6 +748,8 @@ struct stix_t
 		stix_oow_t dfl_sysdic_size;
 	} option;
 
+	stix_cb_t* cblist;
+
 	/* ========================= */
 
 	stix_heap_t* permheap; /* TODO: put kernel objects to here */
@@ -816,7 +841,7 @@ STIX_EXPORT void stix_seterrnum (
  * \return 0 on success, -1 on failure
  */
 STIX_EXPORT int stix_getoption (
-	stix_t*        vm,
+	stix_t*        stix,
 	stix_option_t  id,
 	void*          value
 );
@@ -828,17 +853,28 @@ STIX_EXPORT int stix_getoption (
  * \return 0 on success, -1 on failure
  */
 STIX_EXPORT int stix_setoption (
-	stix_t*       vm,
+	stix_t*       stix,
 	stix_option_t id,
 	const void*   value
 );
 
+
+STIX_EXPORT stix_cb_t* stix_regcb (
+	stix_t*    stix,
+	stix_cb_t* tmpl
+);
+
+STIX_EXPORT void stix_deregcb (
+	stix_t*    stix,
+	stix_cb_t* cb
+);
+
 /**
  * The stix_gc() function performs garbage collection.
  * It is not affected by #STIX_NOGC.
  */
 STIX_EXPORT void stix_gc (
-	stix_t* vm
+	stix_t* stix
 );
 
 /**
@@ -907,6 +943,12 @@ STIX_EXPORT void* stix_callocmem (
 	stix_size_t size
 );
 
+STIX_EXPORT void* stix_reallocmem (
+	stix_t*     stix,
+	void*       ptr,
+	stix_size_t size
+);
+
 STIX_EXPORT void stix_freemem (
 	stix_t* stix,
 	void*   ptr