class Gem::RequestSet::Lockfile::Tokenizer

Public Class Methods

from_file(file) click to toggle source
# File lib/rubygems/request_set/lockfile/tokenizer.rb, line 5
def self.from_file file
  new File.read(file), file
end
new(input, filename = nil, line = 0, pos = 0) click to toggle source
# File lib/rubygems/request_set/lockfile/tokenizer.rb, line 9
def initialize input, filename = nil, line = 0, pos = 0
  @line     = line
  @line_pos = pos
  @tokens   = []
  @filename = filename
  tokenize input
end

Public Instance Methods

empty?() click to toggle source
# File lib/rubygems/request_set/lockfile/tokenizer.rb, line 37
def empty?
  @tokens.empty?
end
make_parser(set, platforms) click to toggle source
# File lib/rubygems/request_set/lockfile/tokenizer.rb, line 17
def make_parser set, platforms
  Gem::RequestSet::Lockfile::Parser.new self, set, platforms, @filename
end
next_token() click to toggle source
# File lib/rubygems/request_set/lockfile/tokenizer.rb, line 45
def next_token
  @tokens.shift
end
Also aliased as: shift
peek() click to toggle source
# File lib/rubygems/request_set/lockfile/tokenizer.rb, line 50
def peek
  @tokens.first || [:EOF]
end
shift()
Alias for: next_token
skip(type) click to toggle source
# File lib/rubygems/request_set/lockfile/tokenizer.rb, line 25
def skip type
  @tokens.shift while not @tokens.empty? and peek.first == type
end
to_a() click to toggle source
# File lib/rubygems/request_set/lockfile/tokenizer.rb, line 21
def to_a
  @tokens
end
unshift(token) click to toggle source
# File lib/rubygems/request_set/lockfile/tokenizer.rb, line 41
def unshift token
  @tokens.unshift token
end

Private Instance Methods

tokenize(input) click to toggle source
# File lib/rubygems/request_set/lockfile/tokenizer.rb, line 56
def tokenize input
  s = StringScanner.new input

  until s.eos? do
    pos = s.pos

    pos = s.pos if leading_whitespace = s.scan(/ +/)

    if s.scan(/[<|=>]{7}/) then
      message = "your #{@filename} contains merge conflict markers"
      column, line = token_pos pos

      raise Gem::RequestSet::Lockfile::ParseError.new message, column, line, @filename
    end

    @tokens <<
      case
      when s.scan(/\r?\n/) then
        token = [:newline, nil, *token_pos(pos)]
        @line_pos = s.pos
        @line += 1
        token
      when s.scan(/[A-Z]+/) then
        if leading_whitespace then
          text = s.matched
          text += s.scan(/[^\s)]*/).to_s # in case of no match
          [:text, text, *token_pos(pos)]
        else
          [:section, s.matched, *token_pos(pos)]
        end
      when s.scan(/([a-z]+):\s/) then
        s.pos -= 1 # rewind for possible newline
        [:entry, s[1], *token_pos(pos)]
      when s.scan(/\(/) then
        [:l_paren, nil, *token_pos(pos)]
      when s.scan(/\)/) then
        [:r_paren, nil, *token_pos(pos)]
      when s.scan(/<=|>=|=|~>|<|>|!=/) then
        [:requirement, s.matched, *token_pos(pos)]
      when s.scan(/,/) then
        [:comma, nil, *token_pos(pos)]
      when s.scan(/!/) then
        [:bang, nil, *token_pos(pos)]
      when s.scan(/[^\s),!]*/) then
        [:text, s.matched, *token_pos(pos)]
      else
        raise "BUG: can't create token for: #{s.string[s.pos..-1].inspect}"
      end
  end

  @tokens
end