#define MAX_TOK_LEN 64
#define MAX_TOKENS 16
#define MAX_STR_SIZE (MAX_TOK_LEN * MAX_TOKENS)
-
+#define $($)#$
+#define PUSHTOK(TOK,LFUNC,UTYPE) \
+ do { \
+ printf("PUSHTOK(" $(TOK) $(LFUNC) $(UTYPE) ")\n"); \
+ tok_t = TOK; \
+ yylval.UTYPE = LFUNC(ts, p-ts); \
+ lexer_pushtok(tok_t, yylval); \
+ ++ntok; \
+ } while (0)
%%{
machine token_matcher;
# set up yylval and tok_t to be pushed to stack
- action set_ref {
- tok_t = REF; \
- yylval.ref = ttor(ts, p-ts); \
- lexer_pushtok(tok_t, yylval); \
- ts = p; }
-
- action set_val { tok_t = NUM; \
- yylval.val = ttov(ts, p-ts); \
- lexer_pushtok(tok_t, yylval); \
- ts = p; }
-
- action set_name { tok_t = NAME; \
- yylval.str = ttos(ts, p-ts); \
- lexer_pushtok(tok_t, yylval); \
- ts = p; }
+ action set_ref { PUSHTOK(REF, ttor, ref); }
+ action set_val { PUSHTOK(NUM, ttov, val); }
+ action set_name { PUSHTOK(NAME, ttos, str); }
+ action set_ts { ts = p; }
+ action lex_error {printf("input error: character %c in filename %s is invalid\n", fc, str);}
# instantiate machines for each possible token
ref = '0x' xdigit+ %set_ref;
val = digit+ %set_val;
name = alpha+ %set_name;
tok = ref | val | name;
+ segment = (tok . '_') %set_ts;
- main := (tok . '_')* . tok;
+ main := segment* . tok $lerr(lex_error);
}%%
%%write data;
-/* 0xxdigit+ => tok_t REF, yylval.ref = uint64_t
- [0-9]+ => tok_t NUM, yylval.val = int
- [a-zA-Z]+ => tok_t NAME, yylval.str = char* */
-
/* Scan filename and push the its tokens
onto the stack */
int lexer_lex (const char* str)
{
const char *p, *pe, *ts, *eof;
- int cs, tok_t ; //tok_t == token type
-
+ int cs, tok_t, ntok = 0;
+ printf ("Lexing: %s\n",str);
p = ts = str;
- pe = p + strlen(str) + 1;
+ pe = p + strlen(str);
%%write init;
%%write exec;
-
- lexer_pushtok(tok_t, yylval);
-
- printf (str);
- return 1;
+ printf ("Lexed %i tokens\n",ntok);
+ return ntok;
}
int ipow(int base, int exp)