词法分析器部分完成。
支持：
1.支持单词分割
2.支持数字类型
3.支持字符串
4.支持换行
6.支持注释
不支持：
1.不支持关键字
2.不支持变量。
3.不支持关键字。
4.不支操作符。
偶没有被那些个编译原理课程所吓倒。。。。。真的勇士，只管前行！
Cpp代码
#ifndef _ISTREAMTOKENIZER_H_
#define _ISTREAMTOKENIZER_H_
#include
#include
#include
#include
#define _COUNT_OF(a) (sizeof(a)/sizeof(a[0]))
class IstreamTokenizer
{
private:
/**
* The next character to be considered by the nextToken method. May also
* be NEED_CHAR to indicate that a new character should be read, or SKIP_LF
* to indicate that a new character should be read and, if it is a '\n'
* character, it should be discarded and a second new character should be
* read.
*/
static const int SKIP_LF;
static const int NEED_CHAR;
//字符类型
static const unsigned char CT_WHITESPACE;
static const unsigned char CT_DIGIT;
static const unsigned char CT_ALPHA;
static const unsigned char CT_QUOTE;
static const unsigned char CT_COMMENT;
public:
//token类型
static const int TT_EOF;
static const int TT_EOL;
static const int TT_NUMBER;
static const int TT_WORD;
static const int TT_NOTHING;
private:
std::istream& input;
std::vector buf;
int peekc;
bool pushedBack;
bool forceLower;
int LINENO;
bool eolIsSignificantP;
bool slashSlashCommentsP;
bool slashStarCommentsP;
unsigned char ctype[256];
public:
std::string sval;
double nval;
int ttype;
private:
void init()
{
wordChars('a', 'z');
wordChars('A', 'Z');
wordChars(128 + 32, 255);
whitespaceChars(0, ' ');
commentChar('/');
quoteChar('"');
quoteChar('\'');
parseNumbers();
}
public:
IstreamTokenizer(std::istream& is): input(is), peekc(NEED_CHAR)
{
init();
}
void resetSyntax()
{
for (int i = _COUNT_OF(ctype); --i >= 0;)
ctype[i] = 0;
}
void wordChars(int low, int hi)
{
if (low < 0)
low = 0;
if (hi >= _COUNT_OF(ctype))
hi = _COUNT_OF(ctype) - 1;
while (low <= hi)
ctype[low++] |= CT_ALPHA;
}
void whitespaceChars(int low, int hi)
{
if (low < 0)
low = 0;
if (hi >= _COUNT_OF(ctype))
hi = _COUNT_OF(ctype) - 1;
while (low <= hi)
ctype[low++] = CT_WHITESPACE;
}
void ordinaryChars(int low, int hi)
{
if (low < 0)
low = 0;
if (hi >= _COUNT_OF(ctype))
hi = _COUNT_OF(ctype) - 1;
while (low <= hi)
ctype[low++] = 0;
}
void ordinaryChar(int ch)
{
if (ch >= 0 && ch < _COUNT_OF(ctype))