Arie/serveme

# typed: true

# DO NOT EDIT MANUALLY
# This is an autogenerated file for types exported from the `webrobots` gem.
# Please instead update this file by running `bin/tapioca gem webrobots`.

# source://nokogiri/1.15.2/lib/nokogiri/html.rb#8
Nokogiri::HTML = Nokogiri::HTML4

# source://webrobots//lib/webrobots/robotstxt.rb#12
class WebRobots
  # Creates a WebRobots object for a robot named +user_agent+, with
  # optional +options+.
  #
  # * :http_get => a custom method, proc, or anything that responds to
  #   .call(uri), to be used for fetching robots.txt.  It must return
  #   the response body if successful, return an empty string if the
  #   resource is not found, and return nil or raise any error on
  #   failure.  Redirects should be handled within this proc.
  #
  # * :crawl_delay => determines how to react to Crawl-delay
  #   directives.  If +:sleep+ is given, WebRobots sleeps as demanded
  #   when allowed?(url)/disallowed?(url) is called.  This is the
  #   default behavior.  If +:ignore+ is given, WebRobots does
  #   nothing.  If a custom method, proc, or anything that responds to
  #   .call(delay, last_checked_at), it is called.
  #
  # @return [WebRobots] a new instance of WebRobots
  #
  # source://webrobots//lib/webrobots.rb#28
  def initialize(user_agent, options = T.unsafe(nil)); end

  # Tests if the robot is allowed to access a resource at +url+.  If a
  # malformed URI string is given, URI::InvalidURIError is raised.  If
  # a relative URI or a non-HTTP/HTTPS URI is given, ArgumentError is
  # raised.
  #
  # @return [Boolean]
  #
  # source://webrobots//lib/webrobots.rb#70
  def allowed?(url); end

  # Returns the number of seconds that the configured agent should wait
  # between successive requests to the site identified by +url+ according
  # to the site's robots.txt +Crawl-delay+ directive.
  #
  # source://webrobots//lib/webrobots.rb#85
  def crawl_delay(url); end

  # source://webrobots//lib/webrobots.rb#54
  def create_cache; end

  # Equivalent to !allowed?(url).
  #
  # @return [Boolean]
  #
  # source://webrobots//lib/webrobots.rb#78
  def disallowed?(url); end

  # Returns an error object if there is an error in fetching or
  # parsing robots.txt of the site +url+.
  #
  # source://webrobots//lib/webrobots.rb#109
  def error(url); end

  # Raises the error if there was an error in fetching or parsing
  # robots.txt of the site +url+.
  #
  # source://webrobots//lib/webrobots.rb#115
  def error!(url); end

  # Flushes robots.txt cache.
  #
  # source://webrobots//lib/webrobots.rb#59
  def flush_cache; end

  # Equivalent to option(url)[token.downcase].
  #
  # source://webrobots//lib/webrobots.rb#97
  def option(url, token); end

  # Returns extended option values for a resource at +url+ in a hash
  # with each field name lower-cased.  See allowed?() for a list of
  # errors that may be raised.
  #
  # source://webrobots//lib/webrobots.rb#92
  def options(url); end

  # Removes robots.txt cache for the site +url+.
  #
  # source://webrobots//lib/webrobots.rb#120
  def reset(url); end

  # Returns an array of Sitemap URLs.  See allowed?() for a list of
  # errors that may be raised.
  #
  # source://webrobots//lib/webrobots.rb#103
  def sitemaps(url); end

  # Returns the robot name initially given.
  #
  # source://webrobots//lib/webrobots.rb#64
  def user_agent; end

  private

  # source://webrobots//lib/webrobots.rb#207
  def crawl_delay_handler(delay, last_checked_at); end

  # source://webrobots//lib/webrobots.rb#162
  def fetch_robots_txt(site); end

  # source://webrobots//lib/webrobots.rb#158
  def get_robots_txt(site); end

  # source://webrobots//lib/webrobots.rb#173
  def http_get(uri); end

  # source://webrobots//lib/webrobots.rb#153
  def robots_txt_for(url); end

  # source://webrobots//lib/webrobots.rb#127
  def split_uri(url); end
end

# source://webrobots//lib/webrobots/robotstxt.rb#13
class WebRobots::Error < ::StandardError; end

# source://webrobots//lib/webrobots/robotstxt.rb#16
class WebRobots::ParseError < ::WebRobots::Error
  # @return [ParseError] a new instance of ParseError
  #
  # source://webrobots//lib/webrobots/robotstxt.rb#20
  def initialize(message, site); end

  # The site's root URI
  #
  # source://webrobots//lib/webrobots/robotstxt.rb#18
  def site; end

  # source://webrobots//lib/webrobots/robotstxt.rb#25
  def to_s; end
end

# source://webrobots//lib/webrobots/robotstxt.rb#30
class WebRobots::RobotsTxt
  # class Parser
  #
  # @return [RobotsTxt] a new instance of RobotsTxt
  #
  # source://webrobots//lib/webrobots/robotstxt.rb#533
  def initialize(site, records, options = T.unsafe(nil)); end

  # @return [Boolean]
  #
  # source://webrobots//lib/webrobots/robotstxt.rb#585
  def allow?(request_uri, user_agent = T.unsafe(nil)); end

  # source://webrobots//lib/webrobots/robotstxt.rb#595
  def crawl_delay(user_agent = T.unsafe(nil)); end

  # Returns the value of attribute error.
  #
  # source://webrobots//lib/webrobots/robotstxt.rb#560
  def error; end

  # @raise [@error]
  #
  # source://webrobots//lib/webrobots/robotstxt.rb#562
  def error!; end

  # Sets the attribute error
  #
  # @param value the value to set the attribute error to.
  #
  # source://webrobots//lib/webrobots/robotstxt.rb#560
  def error=(_arg0); end

  # source://webrobots//lib/webrobots/robotstxt.rb#600
  def options(user_agent = T.unsafe(nil)); end

  # Returns the value of attribute site.
  #
  # source://webrobots//lib/webrobots/robotstxt.rb#559
  def site; end

  # Returns the value of attribute sitemaps.
  #
  # source://webrobots//lib/webrobots/robotstxt.rb#559
  def sitemaps; end

  # Returns the value of attribute timestamp.
  #
  # source://webrobots//lib/webrobots/robotstxt.rb#559
  def timestamp; end

  private

  # source://webrobots//lib/webrobots/robotstxt.rb#577
  def find_record(user_agent = T.unsafe(nil)); end

  # source://webrobots//lib/webrobots/robotstxt.rb#566
  def target(user_agent = T.unsafe(nil)); end

  class << self
    # source://webrobots//lib/webrobots/robotstxt.rb#610
    def unfetchable(site, reason, target = T.unsafe(nil)); end
  end
end

# source://webrobots//lib/webrobots/robotstxt.rb#686
class WebRobots::RobotsTxt::AccessControlLine < ::WebRobots::RobotsTxt::Line
  # source://webrobots//lib/webrobots/robotstxt.rb#687
  def compile; end

  # @return [Boolean]
  #
  # source://webrobots//lib/webrobots/robotstxt.rb#714
  def match?(request_uri); end
end

# source://webrobots//lib/webrobots/robotstxt.rb#673
class WebRobots::RobotsTxt::AgentLine < ::WebRobots::RobotsTxt::Line
  # source://webrobots//lib/webrobots/robotstxt.rb#674
  def compile; end

  # Returns the value of attribute pattern.
  #
  # source://webrobots//lib/webrobots/robotstxt.rb#683
  def pattern; end
end

# source://webrobots//lib/webrobots/robotstxt.rb#721
class WebRobots::RobotsTxt::AllowLine < ::WebRobots::RobotsTxt::AccessControlLine
  # @return [Boolean]
  #
  # source://webrobots//lib/webrobots/robotstxt.rb#722
  def allow?; end
end

# source://webrobots//lib/webrobots/robotstxt.rb#733
class WebRobots::RobotsTxt::CrawlDelayLine < ::WebRobots::RobotsTxt::Line
  # source://webrobots//lib/webrobots/robotstxt.rb#734
  def compile; end

  # Returns the value of attribute delay.
  #
  # source://webrobots//lib/webrobots/robotstxt.rb#746
  def delay; end
end

# source://webrobots//lib/webrobots/robotstxt.rb#605
WebRobots::RobotsTxt::DISALLOW_ALL = T.let(T.unsafe(nil), String)

# source://webrobots//lib/webrobots/robotstxt.rb#727
class WebRobots::RobotsTxt::DisallowLine < ::WebRobots::RobotsTxt::AccessControlLine
  # @return [Boolean]
  #
  # source://webrobots//lib/webrobots/robotstxt.rb#728
  def allow?; end
end

# source://webrobots//lib/webrobots/robotstxt.rb#749
class WebRobots::RobotsTxt::ExtentionLine < ::WebRobots::RobotsTxt::Line; end

# source://webrobots//lib/webrobots/robotstxt.rb#659
class WebRobots::RobotsTxt::Line
  # @return [Line] a new instance of Line
  #
  # source://webrobots//lib/webrobots/robotstxt.rb#660
  def initialize(token, value); end

  # source://webrobots//lib/webrobots/robotstxt.rb#668
  def compile; end

  # Returns the value of attribute token.
  #
  # source://webrobots//lib/webrobots/robotstxt.rb#666
  def token; end

  # Returns the value of attribute value.
  #
  # source://webrobots//lib/webrobots/robotstxt.rb#666
  def value; end
end

# source://webrobots//lib/webrobots/robotstxt.rb#31
class WebRobots::RobotsTxt::Parser < ::Racc::Parser
  def initialize(target, crawl_delay_handler = T.unsafe(nil)); end

  def _reduce_1(val, _values, result); end
  def _reduce_17(val, _values, result); end
  def _reduce_18(val, _values, result); end
  def _reduce_19(val, _values, result); end
  def _reduce_2(val, _values, result); end
  def _reduce_20(val, _values, result); end
  def _reduce_21(val, _values, result); end
  def _reduce_24(val, _values, result); end
  def _reduce_25(val, _values, result); end
  def _reduce_26(val, _values, result); end
  def _reduce_28(val, _values, result); end
  def _reduce_31(val, _values, result); end
  def _reduce_32(val, _values, result); end
  def _reduce_38(val, _values, result); end
  def _reduce_39(val, _values, result); end
  def _reduce_40(val, _values, result); end
  def _reduce_41(val, _values, result); end

  # reduce 43 omitted
  #
  # source://webrobots//lib/webrobots/robotstxt.rb#527
  def _reduce_none(val, _values, result); end

  def next_token; end
  def on_error(token_id, value, stack); end
  def parse(input, site); end
  def parse!(input, site); end
  def parse_error(message); end
end

WebRobots::RobotsTxt::Parser::KNOWN_TOKENS = T.let(T.unsafe(nil), Array)
WebRobots::RobotsTxt::Parser::RE_KNOWN_TOKENS = T.let(T.unsafe(nil), Regexp)

# source://webrobots//lib/webrobots/robotstxt.rb#268
WebRobots::RobotsTxt::Parser::Racc_arg = T.let(T.unsafe(nil), Array)

# source://webrobots//lib/webrobots/robotstxt.rb#323
WebRobots::RobotsTxt::Parser::Racc_debug_parser = T.let(T.unsafe(nil), FalseClass)

# source://webrobots//lib/webrobots/robotstxt.rb#284
WebRobots::RobotsTxt::Parser::Racc_token_to_s_table = T.let(T.unsafe(nil), Array)

# source://webrobots//lib/webrobots/robotstxt.rb#616
class WebRobots::RobotsTxt::Record
  # @return [Record] a new instance of Record
  #
  # source://webrobots//lib/webrobots/robotstxt.rb#617
  def initialize(agentlines, rulelines); end

  # @return [Boolean]
  #
  # source://webrobots//lib/webrobots/robotstxt.rb#649
  def allow?(request_uri); end

  # @return [Boolean]
  #
  # source://webrobots//lib/webrobots/robotstxt.rb#645
  def default?; end

  # Returns the value of attribute delay.
  #
  # source://webrobots//lib/webrobots/robotstxt.rb#637
  def delay; end

  # @return [Boolean]
  #
  # source://webrobots//lib/webrobots/robotstxt.rb#639
  def match?(user_agent); end

  # Returns the value of attribute options.
  #
  # source://webrobots//lib/webrobots/robotstxt.rb#637
  def options; end
end

# source://webrobots//lib/webrobots/version.rb#1
module Webrobots; end

# source://webrobots//lib/webrobots/version.rb#2
Webrobots::VERSION = T.let(T.unsafe(nil), String)