ynnsnr / google-parser Goto Github PK

class GoogleScraper
  def initialize
    @driver = selenium_driver
  end

  def scrape(keyword)
    search(keyword)

    {
      raw_html: driver.page_source,
      links_count: scraped_links_count,
      adwords_advertiser_count: scraped_ads_count,
      results_count: scraped_results_count
    }
  end

  def quit
    @driver.quit
  end

  private

  attr_reader :driver

  def selenium_driver
    Selenium::WebDriver.for :chrome,
                            options: selenium_options,
                            desired_capabilities: driver_capabilities
  end

  def driver_options
    options = Selenium::WebDriver::Chrome::Options.new
    options.add_argument('--ignore-certificate-errors')
    options.add_argument('--disable-popup-blocking')
    options.add_argument('--disable-translate')
    options.add_argument('--no-sandbox')
    options.add_argument('--disable-extensions')
    options.add_argument('--window-size=800,600')
    options.add_argument('--headless')
    options
  end

  def driver_capabilities
    Selenium::WebDriver::Remote::Capabilities.chrome
  end

  def search(keyword)
    driver.navigate.to('https://www.google.com/?hl=en')
    driver.find_element(name: 'q').send_keys(keyword)
    driver.find_element(name: 'q').send_keys(:return)
  end

  def scraped_links_count
    driver.find_elements(tag_name: 'a').count
  end

  def scraped_results_count
    driver.find_element(id: 'resultStats')
          .text
          .match(/(?<count>[\d,]+) results/)[:count]
          .delete(',')
          .to_i
  end

  def scraped_ads_count
    driver.find_elements(class_name: 'ads-ad').count + driver.find_elements(class_name: 'pla-unit').count
  end
end

RSpec.describe Address do
  it '' do
  end

  describe '#keywords' do
  end
end