class Liquid::Lexer
Constants
- COMPARISON_OPERATOR
- DOTDOT
- DOUBLE_STRING_LITERAL
- IDENTIFIER
- NUMBER_LITERAL
- SINGLE_STRING_LITERAL
- SPECIALS
Public Class Methods
new(input)
click to toggle source
# File lib/liquid/lexer.rb, line 21 def initialize(input) @ss = StringScanner.new(input.rstrip) end
Public Instance Methods
tokenize()
click to toggle source
# File lib/liquid/lexer.rb, line 25 def tokenize @output = [] while !@ss.eos? @ss.skip(/\s*/) tok = case when t = @ss.scan(COMPARISON_OPERATOR) then [:comparison, t] when t = @ss.scan(SINGLE_STRING_LITERAL) then [:string, t] when t = @ss.scan(DOUBLE_STRING_LITERAL) then [:string, t] when t = @ss.scan(NUMBER_LITERAL) then [:number, t] when t = @ss.scan(IDENTIFIER) then [:id, t] when t = @ss.scan(DOTDOT) then [:dotdot, t] else c = @ss.getch if s = SPECIALS[c] [s,c] else raise SyntaxError, "Unexpected character #{c}" end end @output << tok end @output << [:end_of_string] end