teamhg-memex / autologin-middleware Goto Github PK

import scrapy
from twitter_scraper.items import TweetItem
from scrapy.shell import inspect_response
from scrapy_splash import SplashRequest
from twitter_scraper.settings import *
import os
import pdb
from scrapy.utils.response import open_in_browser
from scrapy.http import HtmlResponse, FormRequest, Request
from twitter_scraper.lua_scripts import infinite_scroll, twitter_login
from autologin import AutoLogin
import requests
import json
import autologin_middleware

class FollowersSpider(scrapy.Spider):
    name = "twitter_followers"    
    start_urls = ["https://twitter.com/NBA/followers"]
    login_page = "https://twitter.com/login"


    def start_requests(self):
        print("\nGetting cookies...\n")
        res = requests.post(
            url=AUTOLOGIN_URL + "/login-cookies",
            json={
                "url":self.login_page,
                "username":TWITTER_USER,
                "password":TWITTER_PASS
            }
        )
        cookies_json = json.loads(res.content.decode("utf-8"))["cookies"]
        print(f"\nGot cookies! Yeilding request...\n")
        # pdb.set_trace()

        # yield scrapy.Request(
        #     url="https://twitter.com/nyxl/followers", 
        #     callback=self.parse,
        #     cookies=cookies_json,
        #     meta={"dont_merge_cookies":True})

        yield SplashRequest(
            url=self.start_urls[0],
            callback=self.parse, 
            endpoint='execute',
            args={"lua_source":infinite_scroll,"cookies":cookies_json})


    def parse(self, response):
        pdb.set_trace()
        
        ht = HtmlResponse(
            url=response.url, body=response.body, 
            encoding="utf-8", request=response.request)
        open_in_browser(ht)
        inspect_response(response, self)

        
        pdb.set_trace()
        pass

# -*- coding: utf-8 -*-

BOT_NAME = 'twitter_scraper'

SPIDER_MODULES = ['twitter_scraper.spiders']
NEWSPIDER_MODULE = 'twitter_scraper.spiders'

ROBOTSTXT_OBEY = True
COOKIES_ENABLED = True

DOWNLOADER_MIDDLEWARES = {
    'autologin_middleware.AutologinMiddleware': 605,
    'scrapy_splash.SplashCookiesMiddleware': 723,
    'scrapy_splash.SplashMiddleware': 725,
    'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 810,
}


ITEM_PIPELINES = {
    'twitter_scraper.pipelines.TweetPipeline': 100
}

SPIDER_MIDDLEWARES = {
    'scrapy_splash.SplashDeduplicateArgsMiddleware': 100,
}

# Scrapy-splash 
SPLASH_URL = 'http://0.0.0.0:8050'
DUPEFILTER_CLASS = 'scrapy_splash.SplashAwareDupeFilter'
HTTPCACHE_STORAGE = 'scrapy_splash.SplashAwareFSCacheStorage'

# CUSTOM
DATA = "../data/"
USER_ID_CSV = lambda fn: DATA + fn

# logins
TWITTER_USER = "example_username"
TWITTER_PASS = "example_password"


# Autologin
AUTOLOGIN_URL = 'http://127.0.0.1:8089'
AUTOLOGIN_ENABLED = True
DOWNLOADER_MIDDLEWARES['autologin_middleware.AutologinMiddleware'] = 605

AUTOLOGIN_CHECK_LOGOUT = True

AssertionError: Middleware AutologinMiddleware.process_request must return None, Response or Request, got Deferred
Unhandled error in Deferred:
2016-05-25 11:01:18 [twisted] CRITICAL: Unhandled error in Deferred: