lexer_lex statemachine lexes correctly

[henge/webcc.git] / src / apc / lexer_lex.rl

diff --git a/src/apc/lexer_lex.rl b/src/apc/lexer_lex.rl
index f7eb429..edb9a6a 100755 (executable)
--- a/src/apc/lexer_lex.rl
+++ b/src/apc/lexer_lex.rl
@@ -20,7 +20,7 @@ char* ttos(const char* str, int);
   do {                                                 \\r
     printf("PUSHTOK(" $(TOK) $(LFUNC) $(UTYPE) ")\n"); \\r
     tok_t = TOK;                                       \\r
-    yylval.UTYPE = LFUNC(ts, ts-p-1);                  \\r
+    yylval.UTYPE = LFUNC(ts, p-ts+1);                  \\r
     lexer_pushtok(tok_t, yylval);                      \\r
     ++ntok;                                            \\r
   } while (0)\r
@@ -36,12 +36,14 @@ char* ttos(const char* str, int);
   action lex_error {printf("input error: character %c in filename %s is invalid\n", fc, str);}\r
 \r
   # instantiate machines for each possible token\r
-  ref = '0x'. xdigit+ . '_'  %set_ref;\r
-  val = digit+ . '_' %set_val;\r
-  name = alpha+ . '_' %set_name;\r
-  tok = (ref | val | name) %set_ts;\r
+  ref = '0x'. xdigit+  %set_ref;\r
+  val = digit+  %set_val;\r
+  name = alpha+  %set_name;\r
+  tok = (ref | val | name);\r
+  segment = tok . [_\0] %set_ts;\r
+\r
+  main := segment+ ;\r
 \r
- main := alpha+ @set_name;\r
 }%%\r
 \r
 \r
@@ -54,8 +56,7 @@ int lexer_lex (const char* str)
   int  cs, tok_t, ntok = 0;\r
   printf ("Lexing: %s\n",str);\r
   p = ts = str;\r
-  pe = p + strlen(str);\r
-  printf("p = %s \n", p);\r
+  pe = p + strlen(str) + 1;\r
   %%write init;\r
   %%write exec;\r
   printf ("Lexed %i tokens\n",ntok);\r