Racc::Parser
# File lib/regin/tokenizer.rb, line 54 def _next_token text = @ss.peek(1) @lineno += 1 if text == "\n" token = case @state when nil case when (text = @ss.scan(/\\[dDsSwW]/)) action { [:CCLASS, text] } when (text = @ss.scan(/\^|\\A/)) action { [:L_ANCHOR, text] } when (text = @ss.scan(/\$|\\Z/)) action { [:R_ANCHOR, text] } when (text = @ss.scan(/<(\w+)>/)) action { [:NAME, @ss[1]] } when (text = @ss.scan(/\(/)) action { @capture_index_stack << @capture_index @capture_index += 1 @state = :OPTIONS if @ss.peek(1) == '?'; [:LPAREN, text] } when (text = @ss.scan(/\)/)) action { [:RPAREN, text] } when (text = @ss.scan(/\[/)) action { @state = :CCLASS; [:LBRACK, text] } when (text = @ss.scan(/\{/)) action { [:LCURLY, text] } when (text = @ss.scan(/\}/)) action { [:RCURLY, text] } when (text = @ss.scan(/\|/)) action { [:BAR, text] } when (text = @ss.scan(/\./)) action { [:DOT, text] } when (text = @ss.scan(/\?/)) action { [:QMARK, text] } when (text = @ss.scan(/\+/)) action { [:PLUS, text] } when (text = @ss.scan(/\*/)) action { [:STAR, text] } when (text = @ss.scan(/\#/)) action { if @options_stack[-1][:extended] @state = :COMMENT; next_token else [:CHAR, text] end } when (text = @ss.scan(/\s|\n/)) action { if @options_stack[-1][:extended] next_token else [:CHAR, text] end } when (text = @ss.scan(/\\(.)/)) action { [:CHAR, @ss[1]] } when (text = @ss.scan(/./)) action { [:CHAR, text] } else text = @ss.string[@ss.pos .. -1] raise ScanError, "can not match: '" + text + "'" end # if when :CCLASS case when (text = @ss.scan(/\[/)) action { [:LBRACK, text] } when (text = @ss.scan(/\]/)) action { @state = nil; [:RBRACK, text] } when (text = @ss.scan(/\^/)) action { [@ss.string[@ss.pos-2, 1] == '[' ? :NEGATE : :CHAR, text] } when (text = @ss.scan(/:/)) action { if @ss.string[@ss.pos-2, 1] == '[' @state = :POSIX_CCLASS [:COLON, text] else [:CHAR, text] end } when (text = @ss.scan(/\\-/)) action { [:CHAR, text] } when (text = @ss.scan(/\\[dDsSwW]/)) action { [:CHAR, text] } when (text = @ss.scan(/\\(.)/)) action { [:CHAR, @ss[1]] } when (text = @ss.scan(/./)) action { [:CHAR, text] } else text = @ss.string[@ss.pos .. -1] raise ScanError, "can not match: '" + text + "'" end # if when :POSIX_CCLASS case when (text = @ss.scan(/\w+/)) action { [text, text] } when (text = @ss.scan(/:/)) action { [:COLON, text] } when (text = @ss.scan(/\]/)) action { @state = :CCLASS; [:RBRACK, text] } else text = @ss.string[@ss.pos .. -1] raise ScanError, "can not match: '" + text + "'" end # if when :OPTIONS case when (text = @ss.scan(/\?/)) action { @state = nil unless @ss.peek(1) =~ /-|m|i|x|:/ [:QMARK, text] } when (text = @ss.scan(/\-/)) action { [:MINUS, text] } when (text = @ss.scan(/m/)) action { [:MULTILINE, text] } when (text = @ss.scan(/i/)) action { [:IGNORECASE, text] } when (text = @ss.scan(/x/)) action { [:EXTENDED, text] } when (text = @ss.scan(/\:/)) action { @capture_index_stack.pop @capture_index -= 1 @state = nil; [:COLON, text] } else text = @ss.string[@ss.pos .. -1] raise ScanError, "can not match: '" + text + "'" end # if when :COMMENT case when (text = @ss.scan(/\n/)) action { @state = nil; next_token } when (text = @ss.scan(/./)) action { next_token } else text = @ss.string[@ss.pos .. -1] raise ScanError, "can not match: '" + text + "'" end # if else raise ScanError, "undefined state: '" + state.to_s + "'" end # case state token end
# File lib/regin/tokenizer.rb, line 33 def load_file( filename ) @filename = filename open(filename, "r") do |f| scan_setup(f.read) end end
# File lib/regin/tokenizer.rb, line 46 def next_token return if @ss.eos? # skips empty actions until token = _next_token or @ss.eos?; end token end
# File lib/regin/tokenizer.rb, line 40 def scan_file( filename ) load_file(filename) do_parse end
# File lib/regin/tokenizer.rb, line 17 def scan_setup(str) @ss = StringScanner.new(str) @lineno = 1 @state = nil end
# File lib/regin/tokenizer.rb, line 27 def scan_str(str) scan_setup(str) do_parse end
Generated with the Darkfish Rdoc Generator 2.