X-Git-Url: https://www.kengrimes.com/gitweb/?p=henge%2Fapc.git;a=blobdiff_plain;f=src%2Flexer.rl;h=6bb1d1bb7460781ee2629d62d160c4a90dec5f7c;hp=36a72abf747f0a47de3cc5e999e26734ae824d96;hb=25a23e5134b1d9649f0279a4028b887c9deb9987;hpb=6a37e5819abe57422b43d1b356f97e3191ab8850 diff --git a/src/lexer.rl b/src/lexer.rl index 36a72ab..6bb1d1b 100644 --- a/src/lexer.rl +++ b/src/lexer.rl @@ -6,87 +6,100 @@ #include "apc.h" #include #include -extern //lexer.c -void lexer_pushtok(int, YYSTYPE); /* Public */ -int lexer_setdirection(uint8_t*, int); -int lexer_lexstring(uint8_t*, int); -int lexer_setstr(uint8_t*, int); +int lexer_init(void); +void lexer_quit(void); +int lexer_lexfile(uint8_t*); +int lexer_lexdir(uint8_t*); +int lexer_lexstring(uint8_t*, int); +//apc.c +static +yypstate* pstate; +static +yycstate* cstate; +/* Ring buffer for keeping lexical tokens valid for up to 255 tokens */ +static +YYSTYPE lval_stack[0xFF + 1]; +static +uint8_t lval_offs; +#define PUSHTOK(T,L) yypush_parse(pstate, T, (L), cstate) +#define LEXTOK(T,Y,L) do { \ + printf("lval[%i]\n",lval_offs);\ + lval_stack[lval_offs].Y = L; \ + PUSHTOK(T,lval_stack + lval_offs); \ + lval_offs = (lval_offs + 1); \ + ntok++; \ + } while (0); +#define PUSHFACE(F) LEXTOK(FACING, face, F) +#define PUSHREF(R) LEXTOK(REF, ref, R) +#define PUSHLINK() LEXTOK(LINK, val, 0) +#define PUSHNUM(N) LEXTOK(NUM, val, N) +#define PUSHNAME(N) LEXTOK(NAME, str, N) +#define PUSHOP(O) LEXTOK(O, val, 0) +#define PUSHPATH(P) LEXTOK(PATH, str, P) + +/* Lexstring is the main lexer for APC and is generated by ragel. It lexes file names of files + that have been scanned and pushes their types and values into the tok_stack, which yyparse + eventually calls during parsing. */ %%{ machine lexstring; # set up yylval and tok_t to be pushed to stack - action push_ref { te = NULL; errno = 0; - yylval.ref = strtoll((char*)ts,(char**)&te,16); - if (errno | (te != NULL)) + action push_ref { errno = 0; + lval.ref = strtoll((char*)ts,NULL,16); + if (errno) { fprintf(stderr, "Invalid hex number in file %s\n",(char*)str); - if (te != NULL) - { while (str++ < te) - fputc(' ', stderr); - fputc('^', stderr); - } exit(1); } - lexer_pushtok(REF, yylval); ntok++; + PUSHREF(lval.ref); } - action push_link { lexer_pushtok(LINK,(YYSTYPE)0); ntok++; } - action push_val { te = NULL; errno = 0; - yylval.val = strtoll((char*)ts,(char**)&te,10); + action push_link { PUSHLINK(); } + action push_val { errno = 0; + lval.val = strtoll((char*)ts,NULL,10); if (errno) { fprintf(stderr, "strtoll could not parse %s\n", (char*)str); exit(1); } - lexer_pushtok(NUM, yylval); + PUSHNUM(lval.val); } action push_name { printf("Lexer_lexstring:: action:push_name: from %s to %s\n", ts, p); - lexer_pushtok(NAME, yylval); - ntok++; + PUSHNAME(ts); } action push_map { printf("Lexer_lexstring:: action:push_map: pushing map token\n"); - yylval.str = (uint8_t*) '~'; - lexer_pushtok(MOPEN, yylval); - ntok++; + PUSHOP(MAP); } action set_ts { printf("Lexer_lexstring:: action:set_ts. ts = %s\n", p); ts = p; } action push_SS { printf("Lexer_lexstring:: action:push_SS. p = %s\n",p); - yylval.str = (uint8_t*) "SS"; - lexer_pushtok(SS, yylval); - ntok++; + PUSHOP(SS); } action push_S { printf("Lexer_lexstring:: action:push_S. p = %s\n", p); - yylval.val = 0; - lexer_pushtok(D, yylval); + PUSHFACE(SFACE); } action push_SW { printf("Lexer_lexstring:: action:push_SW. p = %s\n", p); - yylval.val = 1; - lexer_pushtok(D, yylval); - } + PUSHFACE(SWFACE); + } action push_W { printf("Lexer_lexstring:: action:push_W. p = %s\n", p); - yylval.val = 2; - lexer_pushtok(D, yylval); + PUSHFACE(WFACE); } action push_NW { printf("Lexer_lexstring:: action:push_NW. p = %s\n", p); - yylval.val = 3; - lexer_pushtok(D, yylval); + PUSHFACE(NWFACE); } action push_N { printf("Lexer_lexstring:: action:push_N. p = %s\n", p); - yylval.val = 4; - lexer_pushtok(D, yylval); + PUSHFACE(NFACE); } action push_NE { printf("Lexer_lexstring:: action:push_NE. p = %s\n", p); - yylval.val = 5; - lexer_pushtok(D, yylval); + PUSHFACE(NEFACE); } action push_E { printf("Lexer_lexstring:: action:push_N. p = %s\n", p); - yylval.val = 6; - lexer_pushtok(D, yylval); + PUSHFACE(EFACE); } action push_SE { printf("Lexer_lexstring:: action:push_N. p = %s\n", p); - yylval.val = 7; - lexer_pushtok(D, yylval); + PUSHFACE(SEFACE); + } + action ref_error { printf("ref from %s to %s has an inappropriate amount of hex digits, it must have eight.\n", ts, p); + exit(1); } - #action lex_error { printf("input error: character %c in filename %s is invalid\n p = %s\n", fc, str, p);} action p { printf("Lexer_lexstring:: p = %s\n", p);} N = 'N' %push_N; @@ -98,279 +111,110 @@ int lexer_setstr(uint8_t*, int); SW = 'SW' %push_SW; SE = 'SE' %push_SE; - #what goes in between tokens in a filename tok_delimiter = [_]; - #types of tokes a filename can contain direction = (N | W | S | E | NW | NE | SW | SE) ; - #make sure 0x123123 doesnt get mistaken for a ref dimensions = (digit+ - '0') >set_ts %push_val 'x' (digit+ - '0') >set_ts %push_val; link = '#' %push_link; SS = ('+SS' %to(push_SS)) | ('+SS' %to(push_SS) link ) ; - ref = '0x' >set_ts alnum+ %push_ref; + ref = '0x' >set_ts alnum{8} $err(ref_error) %push_ref ; val = digit+ >set_ts %push_val ; - name = (lower+ >set_ts) %push_name ; + name = lower >set_ts (lower | digit)* %push_name ; map = '+MAP' %to(push_map); tok = (name | val | ref | dimensions | map | link | SS | direction); - main := (tok tok_delimiter)+ tok [\0]; + main := (tok tok_delimiter)* tok [\0]; write data nofinal noerror noprefix; }%% +int lexer_init +( void ) +{ pstate = yypstate_new(); + cstate = yycstate_new(); + lval_offs = 0; + return !pstate || !cstate; +} + +void lexer_quit +( void ) +{ if (pstate) yypstate_delete(pstate); + if (cstate) yycstate_delete(cstate); +} + int lexer_lexstring ( uint8_t* str, int size ) { uint8_t *p; - uint8_t *ts, *pe, *te; - int cs, ntok;//, tok_t; + uint8_t *ts, *pe, *eof; + int cs, ntok; + YYSTYPE lval; ntok = 0; p = ts = str; - pe = p + size + 1; + pe = eof = p + size + 1; printf("|---Begin lexstring on p = %s, pe = %s.\n",p, pe); %%write init; %%write exec; - printf("Ending lexstring of file %s, pushed %d tokens.\n",str, ntok); + printf("Ending lexstring of file %s, pushed %d tokens.\n",str, ntok); return ntok; } - -/**************************/ -/****Abandon All Hope******/ -/**************************/ -/*** ***/ -/*** ***/ -/*** ***/ -/*** ***/ - - -#if 0 - -%%{ - machine setdirection; - - action ret_north {printf("Lexer_setdirection:: direction is north, returning 4\n"); return 4;; } - action ret_west { printf("Lexer_setdirection:: direction is west, returning 2\n");return 2;} - action ret_east { printf("Lexer_setdirection:: direction is east, returning 6\n");return 6;} - action ret_south { printf("Lexer_setdirection:: direction is south, returning 0\n");return 0;} - action ret_northeast { printf("Lexer_setdirection:: direction is northeast, returning 5\n");return 5 ;} - action ret_northwest { printf("Lexer_setdirection:: direction is northwest, returning 3\n");return 3;} - action ret_southeast { printf("Lexer_setdirection:: direction is southeast, returning 7\n");return 7;} - action ret_southwest { printf("Lexer_setdirection:: direction is southwest, returning 1\n");return 1;} - - def = [_\0] %to(ret_south); - N = 'N'[_\0] %to(ret_north); - W = 'W' [_\0] %to(ret_west); - S = 'S' [_\0] %to(ret_south); - E = 'E' [_\0] %to(ret_east); - NW = 'NW' [_\0] %to(ret_northwest); - NE = 'NE' [_\0] %to(ret_northeast); - SW = 'SW' [_\0] %to(ret_southwest); - SE = 'SE' [_\0] %to(ret_southeast); - - direction = (N | W | S | E | NW | NE | SW | SE | def); - - main := direction; - - write data nofinal noprefix noerror; - - -}%% - - -int -lexer_setdirection -(uint8_t* str, int size) -{ uint8_t *p, *pe, *eof; - int cs; - - - p = str; - pe = str + size + 1; - - printf("|--- Begin lexer_setdirection str = %s, p = %s, pe = %s ---|\n", str,p, pe); - - %%write init; - %%write exec noend; - - printf("|--- Error in: lexer_setdirection ---|\n"); - - return -1; +/* Lexical analysis of a file + Strips a filename to its base name, then sends it to lexer_lexstring before + pushing a PATH token with the filename + Returns the number of tokens pushed to the parser. +*/ +int lexer_lexfile +( uint8_t* filename ) +{ uint8_t* last_period,* iter,* filename_end; + int ntok; + last_period = NULL; + for (iter = filename; *iter; iter++) + switch (*iter) + { // Keep track of the last 'dot' in the name + case '.' : last_period = iter; continue; + // replace '_' with '\0' so bison can use strlen on them as tokens. + case '_' : *iter = '\0'; + default: continue; + } + // Mark the end of the filename + filename_end = iter; + // Lex from either the last period, if present, or filename end + ntok = (last_period) ? + lexer_lexstring(filename, (int)(last_period - filename)) + : lexer_lexstring(filename, (int)(iter - filename)); + // Replace nulls with their original '_' + for (iter = filename; iter < filename_end; iter++) + if (*iter == '\0') + *iter = '_'; + PUSHPATH(filename); + return ntok + 1; + return en_main == 1; } - - -%%{ - machine setstr; - - - action lex_setvlink {printf("Lexer_setstr:: Returning setvlink filetype for %s\n", str); type = 5; newstrt = lexer_lexsetvlink(str); fbreak;} - action lex_elevlink {printf("Lexer_setstr:: Returning elevlink filetype for %s\n", str); type = 6; newstrt = lexer_lexelevlink(str); fbreak;} - action lex_setmodel {printf("Lexer_setstr:: Returning setmodel filetype\n"); newstrt = lexer_lexsetmodel(str); type = 1; fbreak;} - action lex_setmap {printf("Lexer_setstr:: Returning setmap filetype\n"); newstrt = lexer_lexsetmap(str); type = 2; fbreak;} - action lex_elemodel {printf("Lexer_setstr:: Returning elemodel filetype for %s\n", str); newstrt = lexer_lexelemodel(str); type = 3; fbreak;} - action lex_elemap {printf("Lexer_setstr:: Returning elemap filetype for %s\n", str); newstrt = lexer_lexelemap(str); type = 4; fbreak;} - action lex_setolink { printf("Lexer_setstr:: Returning setolink filetype\n"); type = 8; newstrt = lexer_lexsetolink(str); fbreak;} - action lex_eleolink { printf("Lexer_setstr:: Returning eleolink filetype\n"); type = 7; newstrt = lexer_lexeleolink(str); fbreak;} - action p {printf("p = %s \n",p);} - action name_error {printf("In %s, there is a syntactic error. Make sure your set/element names dont conflict with the reserved keywords.\n", str);} - - - N = 'N'; - W = 'W'; - S = 'S'; - E = 'E'; - NW = 'NW'; - NE = 'NE'; - SW = 'SW'; - SE = 'SE'; - - SS = 'SS'; - direction = (N | W | S | E | NW | NE | SW | SE) $p; - - SSD = SS direction; - - - - name = alpha+ $p - SSD $p; - num = digit+ $p; - ref = '0x' $p alnum+ $p; - - - set_label = name | (name '_' ref); - ele_label = name | (name '_' ref); - - model_types = (name) | (name '_' num '_' num) | (name '_' num); - - - set_model = set_label '_' SS %to(lex_setmodel); - set_map = set_label '_' '~' %to(lex_setmap); - ele_model = set_label '_' ele_label '_' SS %to(lex_elemodel); - ele_map = set_label '_' ele_label '_' '~' %to(lex_elemap); - set_olink = ref %to(lex_setolink) [\0] ; - ele_olink = set_label '_' '~' '_' ref [\0] %to(lex_eleolink); - set_vlink = set_label '_' '#' '_' (ref | ref '_' name) [\0] %to(lex_setvlink); - ele_vlink = set_label '_' ele_label '_' '#' '_' (ref | ref '_' name) [\0] %to(lex_elevlink); - - main := (ele_map | set_model | set_map |ele_model | ele_vlink | set_vlink | set_olink | ele_olink); - - write data; - - -}%% - -int -lexer_setstr -(uint8_t* str, int size) -{ uint8_t *p, *pe, *eof; - int cs, type, newstrt; - - type = newstrt = 0; - - p = str; - pe = str + size + 1; - - printf("|--- Begin lexer_setstr with str = %s, p = %s, pe = %s ---|\n", str,p, pe); - - %%write init; - %%write exec noend; - - printf("|--- End lexer_setstr. Incrementing str by %d, type is %d ---|\n", newstrt, type); - - return newstrt; +int lexer_lexdir +( uint8_t* dirname ) +{ uint8_t* de = dirname; + int ntok; + ntok = 0; + de = dirname; + if (*de) while (*++de); + ntok = lexer_lexstring(dirname, (int)(de - dirname)); + PUSHOP(CLOPEN); + return ntok; } -#endif - - -/* %%{ */ -/* machine file_matcher; */ - -/* action call_ml { ts = p; fgoto set_hw ;} */ -/* action call_tl { return 0;} */ -/* action set_height {height = ttov(p, p-ts+1); ts = p;} */ -/* action set_width { width = ttov(p, p-ts+1);} */ -/* action call_lmf {lexer_lexmapfile(height, width); } */ -/* action lex_error {printf("input error: character %c in filename %s is invalid\n = %s\n", fc, str, p);} */ - -/* #This machine determines the type of file we are lexing */ -/* #and calls the appropriate machine to handle it. */ - -/* #TODO add mapping name */ -/* width = digit+ %set_width; */ -/* height = digit+ %set_height; */ - -/* set_hw := height . '_' . width [\0] %to(call_lmf); */ - -/* tok_segment = alnum; */ -/* map_end = 'm' . '_' %to(call_ml); */ -/* tok_end = alnum+ . [\0] %to(call_tl); */ - -/* file_matcher := (tok_segment+ . '_' )+ ( map_end | tok_end ); */ - -/* write data; */ -/* }%% */ - -/* int */ -/* lexer_matchfile */ -/* (char* str, int size) */ -/* { *p, *pe; */ -/* char* ts; */ -/* int cs, ntok, height, width; */ - -/* p = str; */ -/* pe = p + size; */ -/* height = width = 0; */ - -/* printf("Checking if filename is a map file:: filename = %s, p = %c, pe = %c\n", str, *p, *pe); */ - -/* %%write init; */ -/* %%write exec noend; */ - -/* printf("Ending lexer_ismapfile on %s\n", str); */ - -/* return ntok; */ -/* } */ - -/* %%{ */ -/* machine vartype; */ - -/* action isele {return 0;} */ -/* action ismodel {return 1;} */ - -/* set_name = alpha+; */ -/* ele_name = alpha+; */ -/* model_name = alpha+; */ - -/* ele = set_name '_' model_name '_' ele_name %isele; */ -/* model = set_name '_' model_name [\0] %ismodel; */ - - -/* ismodel := (ele | model); */ - -/* write data; */ - -/* }%% */ - -/* int */ -/* lexer_ismodel */ -/* (uint8_t* str, int size) */ -/* { uint8_t *p, *pe, *eof; */ -/* int cs; */ - -/* p = str; */ -/* pe = p + size + 1; */ - -/* %%write init; */ -/* %%write exec; */ - - -/* } */ +int lexer_closedir +( void ) +{ int ntok = 0; + PUSHOP(CLCLOSE); + return ntok; +}