X-Git-Url: https://www.kengrimes.com/gitweb/?p=henge%2Fapc.git;a=blobdiff_plain;f=src%2Flexer.rl;h=ff2fa144030048677fc53b985315e3b48def596f;hp=5e2fb774e23764188d7556a29f7483739e78de91;hb=ee18200a9d3817728d6d09745cd29900649d4508;hpb=78ec1b2aecefbd7f6839f54c52b2dd6e3f80bf1a diff --git a/src/lexer.rl b/src/lexer.rl index 5e2fb77..ff2fa14 100644 --- a/src/lexer.rl +++ b/src/lexer.rl @@ -2,32 +2,45 @@ #include #include #include +#include #include "parser.tab.h" #include "apc.h" -#include -#include -extern //lexer.c -void lexer_pushtok(int, YYSTYPE); +#include "print.h" /* Public */ -int lexer_setdirection(uint8_t*, int); -int lexer_lexfile(uint8_t*); -int lexer_lexdir(uint8_t*); -int lexer_lexstring(uint8_t*, int); -int lexer_setstr(uint8_t*, int); +int lexer_init(void); +void lexer_quit(void); +int lexer_lexfile(uint8_t*); +int lexer_lexdir(uint8_t*); +int lexer_lexstring(uint8_t*, int); //apc.c -extern -yypstate* apc_pstate; -extern -yycstate* apc_cstate; static -YYSTYPE lval_stack[1024], * lvalsp; -#define PUSHTOK(T,L) yypush_parse(apc_pstate, T, (L), apc_cstate) -#define LEXTOK(T,Y,L) do { \ - lvalsp->Y = L; \ - PUSHTOK(T,lvalsp); \ - lvalsp++; \ - ntok++; \ - } while (0); +yypstate* pstate; +static +yycstate* cstate; +/* Ring buffer for keeping lexical tokens valid for up to 255 tokens */ +static +YYSTYPE lval_stack[0xFF + 1]; +static +uint8_t lval_offs; +#define $($)#$ +#define PUSHTOK(T,L) yypush_parse(pstate, T, (L), cstate) +#define LEXTOK(T,Y,L) do { \ + if (DEBUG) { \ + ulc_fprintf(stdout, "["$(T)); \ + switch (T) { \ + case NAME: case PATH: \ + ulc_fprintf(stdout, "->%U", L); break; \ + case REF: \ + ulc_fprintf(stdout, "->%X", L); break; \ + default: break; \ + } \ + ulc_fprintf(stdout, "]"); \ + } \ + lval_stack[lval_offs].Y = L; \ + PUSHTOK(T,lval_stack + lval_offs); \ + lval_offs++; \ + ntok++; \ + } while (0) #define PUSHFACE(F) LEXTOK(FACING, face, F) #define PUSHREF(R) LEXTOK(REF, ref, R) #define PUSHLINK() LEXTOK(LINK, val, 0) @@ -47,7 +60,7 @@ YYSTYPE lval_stack[1024], * lvalsp; action push_ref { errno = 0; lval.ref = strtoll((char*)ts,NULL,16); if (errno) - { fprintf(stderr, "Invalid hex number in file %s\n",(char*)str); + { ulc_fprintf(stderr, "Invalid hex number in file %U\n",str); exit(1); } PUSHREF(lval.ref); @@ -56,49 +69,27 @@ YYSTYPE lval_stack[1024], * lvalsp; action push_val { errno = 0; lval.val = strtoll((char*)ts,NULL,10); if (errno) - { fprintf(stderr, "strtoll could not parse %s\n", (char*)str); + { ulc_fprintf(stderr, "strtoll could not parse %U\n",str); exit(1); } PUSHNUM(lval.val); } - action push_name { printf("Lexer_lexstring:: action:push_name: from %s to %s\n", ts, p); - PUSHNAME(ts); - } - action push_map { printf("Lexer_lexstring:: action:push_map: pushing map token\n"); - PUSHOP(MAP); - } - action set_ts { printf("Lexer_lexstring:: action:set_ts. ts = %s\n", p); ts = p; } - action push_SS { printf("Lexer_lexstring:: action:push_SS. p = %s\n",p); - PUSHOP(SS); - } - action push_S { printf("Lexer_lexstring:: action:push_S. p = %s\n", p); - PUSHFACE(SFACE); - } - action push_SW { printf("Lexer_lexstring:: action:push_SW. p = %s\n", p); - PUSHFACE(SWFACE); - } - action push_W { printf("Lexer_lexstring:: action:push_W. p = %s\n", p); - PUSHFACE(WFACE); - } - action push_NW { printf("Lexer_lexstring:: action:push_NW. p = %s\n", p); - PUSHFACE(NWFACE); - } - action push_N { printf("Lexer_lexstring:: action:push_N. p = %s\n", p); - PUSHFACE(NFACE); - } - action push_NE { printf("Lexer_lexstring:: action:push_NE. p = %s\n", p); - PUSHFACE(NEFACE); - } - action push_E { printf("Lexer_lexstring:: action:push_N. p = %s\n", p); - PUSHFACE(EFACE); - } - action push_SE { printf("Lexer_lexstring:: action:push_N. p = %s\n", p); - PUSHFACE(SEFACE); - } - action ref_error { printf("ref from %s to %s has an inappropriate amount of hex digits, it must have eight.\n", ts, p); + action push_name { PUSHNAME(ts); } + action push_map { PUSHOP(MAP); } + action set_ts { ts = p; } + action push_SS { PUSHOP(SS); } + action push_S { PUSHFACE(SFACE); } + action push_SW { PUSHFACE(SWFACE); } + action push_W { PUSHFACE(WFACE); } + action push_NW { PUSHFACE(NWFACE); } + action push_N { PUSHFACE(NFACE); } + action push_NE { PUSHFACE(NEFACE); } + action push_E { PUSHFACE(EFACE); } + action push_SE { PUSHFACE(SEFACE); } + action ref_error { ulc_fprintf(stderr, "ref from %U to %U has an inappropriate amount of hex digits, it must have eight.\n", ts, p); exit(1); } - action p { printf("Lexer_lexstring:: p = %s\n", p);} + action p { dprintf("Lexer_lexstring:: p = %U\n", p); } N = 'N' %push_N; W = 'W' %push_W; @@ -109,7 +100,7 @@ YYSTYPE lval_stack[1024], * lvalsp; SW = 'SW' %push_SW; SE = 'SE' %push_SE; - tok_delimiter = [_]; + tok_delimiter = [_\0]; direction = (N | W | S | E | NW | NE | SW | SE) ; dimensions = (digit+ - '0') >set_ts %push_val 'x' (digit+ - '0') >set_ts %push_val; @@ -122,33 +113,39 @@ YYSTYPE lval_stack[1024], * lvalsp; tok = (name | val | ref | dimensions | map | link | SS | direction); - main := (tok tok_delimiter)* tok [\0]; + main := (tok tok_delimiter)* tok ; write data nofinal noerror noprefix; }%% +int lexer_init +( void ) +{ pstate = yypstate_new(); + cstate = yycstate_new(); + lval_offs = 0; + return !pstate || !cstate; + return en_main == 1; +} + +void lexer_quit +( void ) +{ if (pstate) yypstate_delete(pstate); + if (cstate) yycstate_delete(cstate); +} + int lexer_lexstring ( uint8_t* str, int size ) -{ uint8_t *p; - uint8_t *ts, *pe, *eof; +{ uint8_t* p, * ts, * pe, * eof; int cs, ntok; YYSTYPE lval; - - lvalsp = lval_stack; ntok = 0; p = ts = str; - pe = eof = p + size + 1; - - printf("|---Begin lexstring on p = %s, pe = %s.\n",p, pe); - + pe = eof = p + size; %%write init; %%write exec; - - printf("Ending lexstring of file %s, pushed %d tokens.\n",str, ntok); - return ntok; } @@ -160,7 +157,7 @@ int lexer_lexstring int lexer_lexfile ( uint8_t* filename ) { uint8_t* last_period,* iter,* filename_end; - int ntok; + int ntok; last_period = NULL; for (iter = filename; *iter; iter++) switch (*iter) @@ -173,265 +170,36 @@ int lexer_lexfile // Mark the end of the filename filename_end = iter; // Lex from either the last period, if present, or filename end + dprintf("%U\n\t",filename); ntok = (last_period) ? lexer_lexstring(filename, (int)(last_period - filename)) : lexer_lexstring(filename, (int)(iter - filename)); + // Replace nulls with their original '_' for (iter = filename; iter < filename_end; iter++) if (*iter == '\0') *iter = '_'; PUSHPATH(filename); + dprintf("\n\t[%i Token%s]\n", ntok, (ntok > 1) ? "s" : ""); return ntok + 1; - return en_main == 1; } int lexer_lexdir ( uint8_t* dirname ) -{ uint8_t* de = dirname; - int ntok; +{ int ntok; ntok = 0; - de = dirname; - if (*de) while (*++de); - ntok = lexer_lexstring(dirname, (int)(de - dirname)); + if (DEBUG) putchar('\t'); + PUSHNAME(dirname); PUSHOP(CLOPEN); + if (DEBUG) putchar('\n'); return ntok; } int lexer_closedir ( void ) { int ntok = 0; + if (DEBUG) putchar('\t'); PUSHOP(CLCLOSE); + if (DEBUG) putchar('\n'); return ntok; } - -/**************************/ -/****Abandon All Hope******/ -/**************************/ -/*** ***/ -/*** ***/ -/*** ***/ -/*** ***/ - - -#if 0 - -%%{ - machine setdirection; - - action ret_north {printf("Lexer_setdirection:: direction is north, returning 4\n"); return 4;; } - action ret_west { printf("Lexer_setdirection:: direction is west, returning 2\n");return 2;} - action ret_east { printf("Lexer_setdirection:: direction is east, returning 6\n");return 6;} - action ret_south { printf("Lexer_setdirection:: direction is south, returning 0\n");return 0;} - action ret_northeast { printf("Lexer_setdirection:: direction is northeast, returning 5\n");return 5 ;} - action ret_northwest { printf("Lexer_setdirection:: direction is northwest, returning 3\n");return 3;} - action ret_southeast { printf("Lexer_setdirection:: direction is southeast, returning 7\n");return 7;} - action ret_southwest { printf("Lexer_setdirection:: direction is southwest, returning 1\n");return 1;} - - def = [_\0] %to(ret_south); - N = 'N'[_\0] %to(ret_north); - W = 'W' [_\0] %to(ret_west); - S = 'S' [_\0] %to(ret_south); - E = 'E' [_\0] %to(ret_east); - NW = 'NW' [_\0] %to(ret_northwest); - NE = 'NE' [_\0] %to(ret_northeast); - SW = 'SW' [_\0] %to(ret_southwest); - SE = 'SE' [_\0] %to(ret_southeast); - - direction = (N | W | S | E | NW | NE | SW | SE | def); - - main := direction; - - write data nofinal noprefix noerror; - - -}%% - - -int -lexer_setdirection -(uint8_t* str, int size) -{ uint8_t *p, *pe, *eof; - int cs; - - - p = str; - pe = str + size + 1; - - printf("|--- Begin lexer_setdirection str = %s, p = %s, pe = %s ---|\n", str,p, pe); - - %%write init; - %%write exec noend; - - printf("|--- Error in: lexer_setdirection ---|\n"); - - return -1; -} - - - -%%{ - machine setstr; - - - action lex_setvlink {printf("Lexer_setstr:: Returning setvlink filetype for %s\n", str); type = 5; newstrt = lexer_lexsetvlink(str); fbreak;} - action lex_elevlink {printf("Lexer_setstr:: Returning elevlink filetype for %s\n", str); type = 6; newstrt = lexer_lexelevlink(str); fbreak;} - action lex_setmodel {printf("Lexer_setstr:: Returning setmodel filetype\n"); newstrt = lexer_lexsetmodel(str); type = 1; fbreak;} - action lex_setmap {printf("Lexer_setstr:: Returning setmap filetype\n"); newstrt = lexer_lexsetmap(str); type = 2; fbreak;} - action lex_elemodel {printf("Lexer_setstr:: Returning elemodel filetype for %s\n", str); newstrt = lexer_lexelemodel(str); type = 3; fbreak;} - action lex_elemap {printf("Lexer_setstr:: Returning elemap filetype for %s\n", str); newstrt = lexer_lexelemap(str); type = 4; fbreak;} - action lex_setolink { printf("Lexer_setstr:: Returning setolink filetype\n"); type = 8; newstrt = lexer_lexsetolink(str); fbreak;} - action lex_eleolink { printf("Lexer_setstr:: Returning eleolink filetype\n"); type = 7; newstrt = lexer_lexeleolink(str); fbreak;} - action p {printf("p = %s \n",p);} - action name_error {printf("In %s, there is a syntactic error. Make sure your set/element names dont conflict with the reserved keywords.\n", str);} - - - N = 'N'; - W = 'W'; - S = 'S'; - E = 'E'; - NW = 'NW'; - NE = 'NE'; - SW = 'SW'; - SE = 'SE'; - - SS = 'SS'; - direction = (N | W | S | E | NW | NE | SW | SE) $p; - - SSD = SS direction; - - - - name = alpha+ $p - SSD $p; - num = digit+ $p; - ref = '0x' $p alnum+ $p; - - - set_label = name | (name '_' ref); - ele_label = name | (name '_' ref); - - model_types = (name) | (name '_' num '_' num) | (name '_' num); - - - set_model = set_label '_' SS %to(lex_setmodel); - set_map = set_label '_' '~' %to(lex_setmap); - ele_model = set_label '_' ele_label '_' SS %to(lex_elemodel); - ele_map = set_label '_' ele_label '_' '~' %to(lex_elemap); - set_olink = ref %to(lex_setolink) [\0] ; - ele_olink = set_label '_' '~' '_' ref [\0] %to(lex_eleolink); - set_vlink = set_label '_' '#' '_' (ref | ref '_' name) [\0] %to(lex_setvlink); - ele_vlink = set_label '_' ele_label '_' '#' '_' (ref | ref '_' name) [\0] %to(lex_elevlink); - - main := (ele_map | set_model | set_map |ele_model | ele_vlink | set_vlink | set_olink | ele_olink); - - write data; - - -}%% - -int -lexer_setstr -(uint8_t* str, int size) -{ uint8_t *p, *pe, *eof; - int cs, type, newstrt; - - type = newstrt = 0; - - p = str; - pe = str + size + 1; - - printf("|--- Begin lexer_setstr with str = %s, p = %s, pe = %s ---|\n", str,p, pe); - - %%write init; - %%write exec noend; - - printf("|--- End lexer_setstr. Incrementing str by %d, type is %d ---|\n", newstrt, type); - - return newstrt; -} - -#endif - - -/* %%{ */ -/* machine file_matcher; */ - -/* action call_ml { ts = p; fgoto set_hw ;} */ -/* action call_tl { return 0;} */ -/* action set_height {height = ttov(p, p-ts+1); ts = p;} */ -/* action set_width { width = ttov(p, p-ts+1);} */ -/* action call_lmf {lexer_lexmapfile(height, width); } */ -/* action lex_error {printf("input error: character %c in filename %s is invalid\n = %s\n", fc, str, p);} */ - -/* #This machine determines the type of file we are lexing */ -/* #and calls the appropriate machine to handle it. */ - -/* #TODO add mapping name */ -/* width = digit+ %set_width; */ -/* height = digit+ %set_height; */ - -/* set_hw := height . '_' . width [\0] %to(call_lmf); */ - -/* tok_segment = alnum; */ -/* map_end = 'm' . '_' %to(call_ml); */ -/* tok_end = alnum+ . [\0] %to(call_tl); */ - -/* file_matcher := (tok_segment+ . '_' )+ ( map_end | tok_end ); */ - -/* write data; */ -/* }%% */ - -/* int */ -/* lexer_matchfile */ -/* (char* str, int size) */ -/* { *p, *pe; */ -/* char* ts; */ -/* int cs, ntok, height, width; */ - -/* p = str; */ -/* pe = p + size; */ -/* height = width = 0; */ - -/* printf("Checking if filename is a map file:: filename = %s, p = %c, pe = %c\n", str, *p, *pe); */ - -/* %%write init; */ -/* %%write exec noend; */ - -/* printf("Ending lexer_ismapfile on %s\n", str); */ - -/* return ntok; */ -/* } */ - -/* %%{ */ -/* machine vartype; */ - -/* action isele {return 0;} */ -/* action ismodel {return 1;} */ - -/* set_name = alpha+; */ -/* ele_name = alpha+; */ -/* model_name = alpha+; */ - -/* ele = set_name '_' model_name '_' ele_name %isele; */ -/* model = set_name '_' model_name [\0] %ismodel; */ - - -/* ismodel := (ele | model); */ - -/* write data; */ - -/* }%% */ - -/* int */ -/* lexer_ismodel */ -/* (uint8_t* str, int size) */ -/* { uint8_t *p, *pe, *eof; */ -/* int cs; */ - -/* p = str; */ -/* pe = p + size + 1; */ - -/* %%write init; */ -/* %%write exec; */ - - -/* } */