no longer hidden

[henge/webcc.git] / org / data / fa / f3ff67-f8e9-41fe-9a32-2ced4bbf5b99 / lexer_lex.rl

diff --git a/org/data/fa/f3ff67-f8e9-41fe-9a32-2ced4bbf5b99/lexer_lex.rl b/org/data/fa/f3ff67-f8e9-41fe-9a32-2ced4bbf5b99/lexer_lex.rl
new file mode 100755 (executable)
index 0000000..0224bb8
--- /dev/null
+++ b/org/data/fa/f3ff67-f8e9-41fe-9a32-2ced4bbf5b99/lexer_lex.rl
@@ -0,0 +1,115 @@
+/* Ragel State Machine for tokenizing text */\r
+#include <stdio.h>\r
+#include <string.h>\r
+#include <apc/parser.tab.h>\r
+\r
+extern void lexer_pushtok(int, YYSTYPE);\r
+\r
+int lexer_lex(const char*);\r
+int ipow(int, int);\r
+int ttov(const char* str, int);\r
+uint64_t ttor(const char* str, int);\r
+char* ttos(const char* str, int);\r
+\r
+\r
+#define MAX_TOK_LEN 64\r
+#define MAX_TOKENS 16\r
+#define MAX_STR_SIZE (MAX_TOK_LEN * MAX_TOKENS)\r
+#define $($)#$\r
+#define PUSHTOK(TOK,LFUNC,UTYPE)                       \\r
+  do {                                                 \\r
+    printf("PUSHTOK(" $(TOK) $(LFUNC) $(UTYPE) ")\n"); \\r
+    tok_t = TOK;                                       \\r
+    yylval.UTYPE = LFUNC(ts, p-ts);                    \\r
+    lexer_pushtok(tok_t, yylval);                      \\r
+    ++ntok;                                            \\r
+  } while (0)\r
+\r
+%%{\r
+  machine token_matcher;\r
+\r
+  # set up yylval and tok_t to be pushed to stack\r
+  action set_ref  { PUSHTOK(REF, ttor, ref); }\r
+  action set_val  { PUSHTOK(NUM, ttov, val); }\r
+  action set_name { PUSHTOK(NAME, ttos, str); }\r
+  action set_ts   { ts = p; }\r
+  action lex_error {printf("input error: character %c in filename %s is invalid\n", fc, str);}\r
+\r
+  # instantiate machines for each possible token\r
+  ref = '0x' xdigit+ %set_ref;\r
+  val = digit+ %set_val;\r
+  name = alpha+ %set_name;\r
+  tok = ref | val | name;\r
+  segment = (tok . '_') %set_ts;\r
+\r
+ main := segment* . tok $lerr(lex_error);\r
+}%%\r
+\r
+\r
+%%write data;\r
+\r
+/* Scan filename and push the its tokens\r
+   onto the stack */\r
+int lexer_lex (const char* str)\r
+{\r
+  const char *p, *pe, *ts, *eof;\r
+  int  cs, tok_t, ntok = 0;\r
+  printf ("Lexing: %s\n",str);\r
+  p = ts = str;\r
+  pe = p + strlen(str);\r
+  %%write init;\r
+  %%write exec;\r
+  printf ("Lexed %i tokens\n",ntok);\r
+  return ntok;\r
+}\r
+\r
+int ipow(int base, int exp)\r
+{\r
+  int result = 1;\r
+  while (exp)\r
+    {\r
+      if (exp & 1)\r
+        result = result * base;\r
+      exp = exp >> 1;\r
+      base *= base;\r
+    }\r
+\r
+  return result;\r
+}\r
+\r
+/*  Token to Value */\r
+int ttov(const char* str, int len)\r
+{\r
+  int i, val = 0;\r
+\r
+  for (i = 0; i < len; i++)\r
+    {\r
+      val += ((str[len - (i + 1)] - '0') * ipow(10,i));\r
+    }\r
+\r
+  return val;\r
+}\r
+\r
+uint64_t ttor(const char* str, int len)\r
+{\r
+  int i;\r
+  uint64_t num = 0;\r
+\r
+  for (i = 0; i < len; i++)\r
+    {\r
+      num += ((str[len - (i + 1)] - '0') * ipow(10,i));\r
+    }\r
+\r
+  return num;\r
+}\r
+\r
+char* ttos(const char* str, int len)\r
+{\r
+  int i;\r
+  char token_buf[MAX_TOK_LEN];\r
+\r
+  memmove(token_buf, str, len);\r
+  token_buf[len+1] = '\0';\r
+\r
+  return strdup(token_buf);\r
+}\r