1 /* Ragel State Machine for tokenizing text */
4 #include <apc/parser.tab.h>
6 extern void lexer_pushtok(int, YYSTYPE);
8 int lexer_lex(const char*);
10 int ttov(const char* str, int);
11 uint64_t ttor(const char* str, int);
12 char* ttos(const char* str, int);
15 #define MAX_TOK_LEN 64
17 #define MAX_STR_SIZE (MAX_TOK_LEN * MAX_TOKENS)
21 machine token_matcher;
23 # set up yylval and tok_t to be pushed to stack
26 yylval.ref = ttor(ts, p-ts); \
27 lexer_pushtok(tok_t, yylval); }
29 action set_val { tok_t = NUM; \
30 yylval.val = ttov(ts, p-ts); \
31 lexer_pushtok(tok_t, yylval); }
33 action set_name { tok_t = NAME; \
34 yylval.str = ttos(ts, p-ts); \
35 lexer_pushtok(tok_t, yylval); }
37 action set_ts { ts = p; }
39 # instantiate machines for each possible token
40 ref = '0x' xdigit+ %set_ref;
41 val = digit+ %set_val;
42 name = alpha+ %set_name;
43 tok = ref | val | name;
44 segment = (tok . '_') %set_ts;
46 main := segment* . tok;
52 /* 0xxdigit+ => tok_t REF, yylval.ref = uint64_t
53 [0-9]+ => tok_t NUM, yylval.val = int
54 [a-zA-Z]+ => tok_t NAME, yylval.str = char* */
56 /* Scan filename and push the its tokens
58 int lexer_lex (const char* str)
60 const char *p, *pe, *ts, *eof;
61 int cs, tok_t ; //tok_t == token type
64 pe = p + strlen(str) + 1;
68 lexer_pushtok(tok_t, yylval);
74 int ipow(int base, int exp)
80 result = result * base;
89 int ttov(const char* str, int len)
93 for (i = 0; i < len; i++)
95 val += ((str[len - (i + 1)] - '0') * ipow(10,i));
101 uint64_t ttor(const char* str, int len)
106 for (i = 0; i < len; i++)
108 num += ((str[len - (i + 1)] - '0') * ipow(10,i));
114 char* ttos(const char* str, int len)
117 char token_buf[MAX_TOK_LEN];
119 memmove(token_buf, str, len);
120 token_buf[len+1] = '\0';
122 return strdup(token_buf);