词法分析器--DFA（c++实现）

语言名为TINY

实例程序：

begin
     var x,y:interger;
     x:=10;
     read(x);
     if y<0 then x:=x-y;
     x:=x+y;
     write(x);
end

TINY语言扫描程序的DFA：

代码

//ExplLexicalAnalyzer.h
#ifndef EXPLLEXICALANALYZER_H
#define EXPLLEXICALANALYZER_H


#define MAXTOKENLEN 40
#define MAXRESERVED 13


typedef enum {
    ENDFILE, ERROR,
    IF, THEN, ELSE, END, REPEAT, UNTIL, READ, WRITE, VAR, BEGIN, INTEGER, DOUBLE, STRING,
    ID, NUM,
    ASSIGN, EQ, LT, PLUS, MINUS, TIMES, OVER, LPAREN, RPAREN, SEMI, COMMA, DEFINE
} TokenType;

//typedef struct {
//    TokenType kind;
//    int row = -1;
//    int column = -1;
//    double value;
//    std::string ID;
//} Token;


TokenType getToken(void);

#endif //LEARN_2_EXPLLEXICALANALYZER_H

//ExplLexicalAnalyzer.cpp
#include <cstdio>
#include <iostream>
#include <fstream>
#include <cstring>
#include  "ExplLexicalAnalyzer.h"

using namespace std;

typedef enum {
    START, INASSIGN, INCOMMENT, INNUM, INID, DONE
} StateType;

char tokenString[MAXTOKENLEN + 1];

#define BUFLEN 256

static char lineBuf[BUFLEN];
static int linepos = 0;
static int bufsize = 0;
static int EOF_flag = false;
static string filename;
static fstream get;
static int lineno = 0;
static int columnpos = 0;
bool TraceScan = true;
StateType state;

static struct {
    const char *str;
    TokenType tok;
} reservedWords[MAXRESERVED]
        = {{"if",       IF},
           {"then",     THEN},
           {"else",     ELSE},
           {"end",      END},
           {"repeat",   REPEAT},
           {"until",    UNTIL},
           {"read",     READ},
           {"write",    WRITE},
           {"begin",    BEGIN},
           {"var",      VAR},
           {"interger", INTEGER},
           {"double",   DOUBLE},
           {"string",   STRING}};


static char
getNextChar() {
    if (linepos >= bufsize) {
        lineno = 0;
        if (state != START)
            return ' ';
        if (get.getline(lineBuf, BUFLEN - 1)) {
            printf("%d: %s
", columnpos++, lineBuf);
            bufsize = (int) strlen(lineBuf);
            linepos = 0;
            return lineBuf[linepos++];
        } else {
            return EOF;
        }
    } else return lineBuf[linepos++];
}


static TokenType reservedLookup(char *s) {
    int i;
    for (i = 0; i < MAXRESERVED; i++)
        if (!strcmp(s, reservedWords[i].str))
            return reservedWords[i].tok;
    return ID;
}

//退回一个字符
static void ungetNextChar(void) { if (!EOF_flag) linepos--; }

//打印分析结果
void printToken(TokenType token, const char *tokenString) {
    switch (token) {
        case IF:
        case THEN:
        case ELSE:
        case END:
        case REPEAT:
        case UNTIL:
        case READ:
        case WRITE:
        case BEGIN:
        case VAR:
        case INTEGER:
        case DOUBLE:
        case STRING:
            printf("reserved word: %s
", tokenString);
            break;
        case DEFINE:
            printf(":
");
            break;
        case COMMA:
            printf(",
");
            break;
        case ASSIGN:
            printf(":=
");
            break;
        case LT:
            printf("<
");
            break;
        case EQ:
            printf("=
");
            break;
        case LPAREN:
            printf("(
");
            break;
        case RPAREN:
            printf(")
");
            break;
        case SEMI:
            printf(";
");
            break;
        case PLUS:
            printf("+
");
            break;
        case MINUS:
            printf("-
");
            break;
        case TIMES:
            printf("*
");
            break;
        case OVER:
            printf("/
");
            break;
        case ENDFILE:
            break;
        case NUM:
            printf("NUM, val= %s
", tokenString);
            break;
        case ID:
            printf("ID, name= %s
", tokenString);
            break;
        case ERROR:
            printf("ERROR: %s
", tokenString);
            break;
        default:
            printf("Unknown token: %d
", token);
    }
}


//词法分析
TokenType getToken(void) {
    int tokenStringIndex = 0;
    TokenType currentToken;
    state = START;
    bool save;
    while (state != DONE) {
        char c = getNextChar();
        save = true;
        switch (state) {
            case START:
                if (isdigit(c))
                    state = INNUM;
                else if (isalpha(c))
                    state = INID;
                else if (c == ':')
                    state = INASSIGN;
                else if ((c == ' ') || (c == '	') || (c == '
'))
                    save = false;
                else if (c == '{') {
                    save = false;
                    state = INCOMMENT;
                } else {
                    state = DONE;
                    switch (c) {
                        case EOF:
                            return ENDFILE;
                        case ',':
                            currentToken = COMMA;
                            break;
                        case '=':
                            currentToken = EQ;
                            break;
                        case '<':
                            currentToken = LT;
                            break;
                        case '+':
                            currentToken = PLUS;
                            break;
                        case '-':
                            currentToken = MINUS;
                            break;
                        case '*':
                            currentToken = TIMES;
                            break;
                        case '/':
                            currentToken = OVER;
                            break;
                        case '(':
                            currentToken = LPAREN;
                            break;
                        case ')':
                            currentToken = RPAREN;
                            break;
                        case ';':
                            currentToken = SEMI;
                            break;
                        default:
                            currentToken = ERROR;
                            break;
                    }
                }
                break;
            case INCOMMENT:
                save = false;
                if (c == EOF) {
                    state = DONE;
                    currentToken = ENDFILE;
                } else if (c == '}') state = START;
                break;
            case INASSIGN:
                state = DONE;
                if (c == '=')
                    currentToken = ASSIGN;
                else {
                    currentToken = DEFINE;
                    ungetNextChar();
                }
                break;
            case INNUM:
                if (!isdigit(c)) {
                    ungetNextChar();
                    save = false;
                    state = DONE;
                    currentToken = NUM;
                }
                break;
            case INID:
                if (!isalpha(c)) {
                    tokenString[tokenStringIndex] = '';
                    if (!strcmp(tokenString, "begin") || !strcmp(tokenString, "end")) {
                        save = false;
                        state = DONE;
                        currentToken = ID;
                        break;
                    }
                    ungetNextChar();
                    save = false;
                    state = DONE;
                    currentToken = ID;
                }
                break;
            case DONE:
                break;
        }
        if ((save) && (tokenStringIndex <= MAXTOKENLEN) && (state != START && !isspace(c)))
            tokenString[tokenStringIndex++] = c;
        if (state == DONE) {
            tokenString[tokenStringIndex] = '';
            if (currentToken == ID)
                currentToken = reservedLookup(tokenString);
        }
    }
    if (TraceScan) {
        printf("	%d: ", lineno++);
        printToken(currentToken, tokenString);
    }
    return currentToken;
}


int
main() {
    if (cin >> filename && filename == "q") {
        filename = "......";
    }
    get.open(filename, ios::in);
    while (getToken() != ENDFILE);
}

 运行结果：