1 /* Ragel State Machine for tokenizing text */
4 #include <apc/parser.tab.h>
6 extern void lexer_pushtok(int, YYSTYPE);
8 int lexer_lex(const char*);
10 int ttov(const char* str, int);
11 uint64_t ttor(const char* str, int);
12 char* ttos(const char* str, int);
15 #define MAX_TOK_LEN 64
17 #define MAX_STR_SIZE (MAX_TOK_LEN * MAX_TOKENS)
19 #define PUSHTOK(TOK,LFUNC,UTYPE) \
21 printf("PUSHTOK(" $(TOK) $(LFUNC) $(UTYPE) ")\n"); \
23 yylval.UTYPE = LFUNC(ts, ts-p-1); \
24 lexer_pushtok(tok_t, yylval); \
29 machine token_matcher;
31 # set up yylval and tok_t to be pushed to stack
32 action set_ref { PUSHTOK(REF, ttor, ref); }
33 action set_val { PUSHTOK(NUM, ttov, val); }
34 action set_name { PUSHTOK(NAME, ttos, str); }
35 action set_ts { ts = p; }
36 action lex_error {printf("input error: character %c in filename %s is invalid\n", fc, str);}
38 # instantiate machines for each possible token
39 ref = '0x'. xdigit+ . '_' %set_ref;
40 val = digit+ . '_' %set_val;
41 name = alpha+ . '_' %set_name;
42 tok = (ref | val | name) %set_ts;
44 main := alpha+ @set_name;
49 /* Scan filename and push the its tokens
51 int lexer_lex (const char* str)
53 const char *p, *pe, *ts, *eof;
54 int cs, tok_t, ntok = 0;
55 printf ("Lexing: %s\n",str);
58 printf("p = %s \n", p);
61 printf ("Lexed %i tokens\n",ntok);
65 int ipow(int base, int exp)
71 result = result * base;
80 int ttov(const char* str, int len)
84 for (i = 0; i < len; i++)
86 val += ((str[len - (i + 1)] - '0') * ipow(10,i));
92 uint64_t ttor(const char* str, int len)
97 for (i = 0; i < len; i++)
99 num += ((str[len - (i + 1)] - '0') * ipow(10,i));
105 char* ttos(const char* str, int len)
108 char token_buf[MAX_TOK_LEN];
110 memmove(token_buf, str, len);
111 token_buf[len+1] = '\0';
113 return strdup(token_buf);