class WebRobots::RobotsTxt::Record

Attributes

delay[R]
options[R]

Public Class Methods

new(agentlines, rulelines) click to toggle source
# File lib/webrobots/robotstxt.rb, line 617
def initialize(agentlines, rulelines)
  @patterns = agentlines.map { |agentline| agentline.pattern }
  @acls = []
  @delay = nil
  @options = {}
  rulelines.each { |ruleline|
    case ruleline
    when AccessControlLine
      @acls << ruleline
    when CrawlDelayLine
      @delay = ruleline.delay
    else
      @options[ruleline.token.downcase] = ruleline.value
    end
  } if rulelines
  @acls.replace @acls.sort_by { |x|
    [-x.value.length, x.is_a?(AllowLine) ? -1 : 0]
  }
end

Public Instance Methods

allow?(request_uri) click to toggle source
# File lib/webrobots/robotstxt.rb, line 649
def allow?(request_uri)
  @acls.each { |acl|
    if acl.match?(request_uri)
      return acl.allow?
    end
  }
  return true
end
default?() click to toggle source
# File lib/webrobots/robotstxt.rb, line 645
def default?
  @patterns.include?(//)
end
match?(user_agent) click to toggle source
# File lib/webrobots/robotstxt.rb, line 639
def match?(user_agent)
  @patterns.any? { |pattern|
    pattern.match(user_agent)
  }
end