llgocppg ast/token

This commit is contained in:
xushiwei
2024-08-20 08:58:43 +08:00
parent 1a8b319ce2
commit 864b078610
2 changed files with 20 additions and 15 deletions

View File

@@ -3,25 +3,30 @@ package token
type Token uint
const (
ILLEGAL Token = iota
/**
* A token that contains some kind of punctuation.
*/
PunctuationToken Token = iota
PUNCT
/**
* A language keyword.
*/
KeywordToken
KEYWORD
/**
* An identifier (that is not a keyword).
*/
IdentifierToken
IDENT
/**
* A numeric, string, or character literal.
*/
LiteralToken
LITERAL
/**
* A comment.
*/
CommentToken
COMMENT
)