| 12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455 | var SyntaxError = require('./SyntaxError');var TAB = 9;var N = 10;var F = 12;var R = 13;var SPACE = 32;var Tokenizer = function(str) {    this.str = str;    this.pos = 0;};Tokenizer.prototype = {    charCodeAt: function(pos) {        return pos < this.str.length ? this.str.charCodeAt(pos) : 0;    },    charCode: function() {        return this.charCodeAt(this.pos);    },    nextCharCode: function() {        return this.charCodeAt(this.pos + 1);    },    nextNonWsCode: function(pos) {        return this.charCodeAt(this.findWsEnd(pos));    },    findWsEnd: function(pos) {        for (; pos < this.str.length; pos++) {            var code = this.str.charCodeAt(pos);            if (code !== R && code !== N && code !== F && code !== SPACE && code !== TAB) {                break;            }        }        return pos;    },    substringToPos: function(end) {        return this.str.substring(this.pos, this.pos = end);    },    eat: function(code) {        if (this.charCode() !== code) {            this.error('Expect `' + String.fromCharCode(code) + '`');        }        this.pos++;    },    peek: function() {        return this.pos < this.str.length ? this.str.charAt(this.pos++) : '';    },    error: function(message) {        throw new SyntaxError(message, this.str, this.pos);    }};module.exports = Tokenizer;
 |