CMM词法分析

Invalid token at row 5, column 9:
int a=0..4545；
        ^

def _getch(self):
        """
        get char from buffer or input
        :return:
        """
        if len(self.buf) == 0:
            c = self.input.read(1)
            if len(c) == 0:
                return None
        else:
            c = self.buf.pop()

        if c == '\n':  # record current state and prepare for a new line
            self.line += 1
            self.pre_read = self.read
            self.read = []
        else:
            self.read.append(c)
        return c

def _ungetc(self, c):
    """
    put a char that was read to buffer
    :param c: the char to unget
    :return:
    """
    self.buf.append(c)
    if c == '\n':
        # if unget '\n', recover to the previous state which has been recorded
        self.line -= 1
        self.read = self.pre_read
        self.pre_read = []
    else:
        self.read.pop()

$ cat test/lexer/arithmetic.t
// This is a test for arithmetic
a+b;  a- b;  a*b;  a / b;

int b=0.00134446
int a=0..4545   // invalid

$ ./parser.py test/lexer/arithmetic.t
2: <ID: 'a'>
   2: <PLUS: '+'>
   2: <ID: 'b'>
   2: <SEMICOLON: ';'>
   2: <ID: 'a'>
   2: <MINUS: '-'>
   2: <ID: 'b'>
   2: <SEMICOLON: ';'>
   2: <ID: 'a'>
   2: <TIMES: '*'>
   2: <ID: 'b'>
   2: <SEMICOLON: ';'>
   2: <ID: 'a'>
   2: <DIVIDE: '/'>
   2: <ID: 'b'>
   2: <SEMICOLON: ';'>
4: <RESERVE: 'int'>
   4: <ID: 'b'>
   4: <ASSIGN: '='>
   4: <REAL_LITERAL: '0.00134446'>
5: <RESERVE: 'int'>
   5: <ID: 'a'>
   5: <ASSIGN: '='>

Invalid token at row 5, column 9:
int a=0..4545   // invalid
        ^

$ cat test/lexer/comment2.t
/**
 * This is a test for multiple line comments
 */
int i/*~ /*Ha*ha/* ~*/=0;

/*
Below is a nest comment, which is invalid.
But at lexical analysis state, it's valid,
which can be recognized as "<ID: 'ccc'> <TIMES: '*'> <DIVIDE: '/'>"
*/

/*aaa/*bbb*/ccc*/

$ ./parser.py test/lexer/comment2.t
4: <RESERVE: 'int'>
   4: <ID: 'i'>
   4: <ASSIGN: '='>
   4: <INT_LITERAL: '0'>
   4: <SEMICOLON: ';'>
12: <ID: 'ccc'>
   12: <TIMES: '*'>
   12: <DIVIDE: '/'>

$ cat test/lexer/comparison.t
// This is a test for comparison
a>b;a<b;a<>b;a==b;

$ ./parser.py test/lexer/comparison.t
2: <ID: 'a'>
   2: <GT: '>'>
   2: <ID: 'b'>
   2: <SEMICOLON: ';'>
   2: <ID: 'a'>
   2: <LT: '<'>
   2: <ID: 'b'>
   2: <SEMICOLON: ';'>
   2: <ID: 'a'>
   2: <NEQUAL: '<>'>
   2: <ID: 'b'>
   2: <SEMICOLON: ';'>
   2: <ID: 'a'>
   2: <EQUAL: '=='>
   2: <ID: 'b'>
   2: <SEMICOLON: ';'>

$ cat test/lexer/identifier.t
//This is a test for identifier
a2b_c   // valid
a2b_    // invalid

$ ./parser.py test/lexer/identifier.t
2: <ID: 'a2b_c'>

Invalid token at row 3, column 4:
a2b_    // invalid
   ^

$ cat test/lexer/reserved.t
// This is a test for reserved tokens
int arr[5];
bool p=1;
real a=0.22325;
if(p>0){ write(p)}else{read(p)}

$ ./parser.py test/lexer/reserved.t
2: <RESERVE: 'int'>
   2: <ID: 'arr'>
   2: <LBRACKET: '['>
   2: <INT_LITERAL: '5'>
   2: <RBRACKET: ']'>
   2: <SEMICOLON: ';'>
3: <RESERVE: 'bool'>
   3: <ID: 'p'>
   3: <ASSIGN: '='>
   3: <INT_LITERAL: '1'>
   3: <SEMICOLON: ';'>
4: <RESERVE: 'real'>
   4: <ID: 'a'>
   4: <ASSIGN: '='>
   4: <REAL_LITERAL: '0.22325'>
   4: <SEMICOLON: ';'>
5: <RESERVE: 'if'>
   5: <LPAREN: '('>
   5: <ID: 'p'>
   5: <GT: '>'>
   5: <INT_LITERAL: '0'>
   5: <RPAREN: ')'>
   5: <LBRACE: '{'>
   5: <RESERVE: 'write'>
   5: <LPAREN: '('>
   5: <ID: 'p'>
   5: <RPAREN: ')'>
   5: <RBRACE: '}'>
   5: <RESERVE: 'else'>
   5: <LBRACE: '{'>
   5: <RESERVE: 'read'>
   5: <LPAREN: '('>
   5: <ID: 'p'>
   5: <RPAREN: ')'>
   5: <RBRACE: '}'>