gem 'makuri'

# quotes_spider.rb
require 'makuri'

class QuotesSpider
  include Makuri::Spider
  start_urls ['https://quotes.toscrape.com/tag/humor/']

  def parse
    response.css('div.quote').each { |quote| extract(quote) }

    next_page = response.at_css('li.next>a')
    request_to :parse, url: next_page[:href] unless next_page.nil?
  end

  def extract(quote)
    item = {
      author: quote.at_css('span>small').text,
      text: quote.at_css('span.text').text
    }

    puts item.to_json
  end
end

QuotesSpider.run

# infinite_scroll_spider.rb
require 'makuri'

class InfiniteScrollSpider
  include Makuri::Spider
  spider_options engine: :ferrum, headless: true

  start_urls ['https://infinite-scroll.com/demo/full-page/']

  def parse
    post_title_xpath = '//article/h2'
    count = response.xpath(post_title_xpath).count

    current_response = nil
    loop do
      browser.page.execute('window.scrollBy(0,10000)'); sleep 2

      current_response = browser.current_response
      new_count = current_response.xpath(post_title_xpath).count

      logger.info '> Pagination is done' and break if count == new_count

      count = new_count
      logger.info "> Continue scrolling, current count is #{count}..."
    end

    posts_headers = current_response.xpath(post_title_xpath).map(&:text)
    logger.info "> All post titles: #{posts_headers.join('; ')}"
  end
end