clericpy / uniparser Goto Github PK

{
    'parse_rule': 'This is article title',
    '__child__': {
        'rule1': 'This is hello world',
        '__child__': {
            'rule2': 'dlrow olleh si sihT',
            'rule3': 'dlrow olleh si sihT'
        }
    }
}

{'parse_rule': {'rule1': {'rule2': 'dlrow olleh si sihT', 'rule3': 'hello world'}}}

def find_rule(url, storage):
    host = urlparse(url).netloc
    rules = storage.get(host, [])
    for rule in rules:
        if re.match(rule.regex_str, url):
            return rule

from jmespath import compile as jc, search as js
from jsonpath_ng import parse
from objectpath import Tree
import timeit

JSON = {'a': {'a': 'a'}}

cc = jc('a.a')
pp = parse('$.a.a')
t = Tree(JSON)


def test1():
    "jmespath compiled"
    return cc.search(JSON)


def test2():
    "jmespath uncompiled"
    return js('a.a', JSON)


def test3():
    "jsonpath_ng compiled"
    return [i.value for i in pp.find(JSON)]


def test4():
    "jsonpath_ng uncompiled"
    return [i.value for i in parse('$.a.a').find(JSON)]


def test5():
    "objectpath compiled"
    return t.execute('$.a.a')


def test6():
    "objectpath uncompiled"
    t = Tree(JSON)
    return t.execute('$.a.a')


num = 100000

print(test1.__doc__, ':', round(
    timeit.timeit(test1, number=num) * 1000 / num, 3), 'ms')
print(test2.__doc__, ':', round(
    timeit.timeit(test2, number=num) * 1000 / num, 3), 'ms')
print(test3.__doc__, ':', round(
    timeit.timeit(test3, number=num) * 1000 / num, 3), 'ms')
print(test4.__doc__, ':',
      round(timeit.timeit(test4, number=1000) * 1000 / 1000, 3), 'ms')
print(test5.__doc__, ':', round(
    timeit.timeit(test5, number=num) * 1000 / num, 3), 'ms')
print(test6.__doc__, ':', round(
    timeit.timeit(test6, number=num) * 1000 / num, 3), 'ms')
# jmespath compiled : 0.007 ms
# jmespath uncompiled : 0.009 ms
# jsonpath_ng compiled : 0.011 ms
# jsonpath_ng uncompiled : 12.536 ms
# objectpath compiled : 0.022 ms
# objectpath uncompiled : 0.021 ms

result_dict = {
        'parse_rule': 'This is article title',
        '__child__': {
            'rule1': 'This is hello world',
            '__child__': {
                'rule2': 'dlrow olleh si sihT'
            }
        }
    }

from uniparser import HostRule
from uniparser.config import GlobalConfig
import orjson


def ordumps(*args, **kwargs):
    return orjson.dumps(*args, **kwargs).decode('utf-8')


GlobalConfig.json_dumps = ordumps

rule = HostRule(
    **{
        'host': 'httpbin.org',
        'crawler_rules': {
            'HelloWorld': {
                'name': 'HelloWorld',
                'parse_rules': [{
                    'name': 'result',
                    'chain_rules': [['css', 'p', '$text'],
                                    ['python', 'getitem', '[0]']],
                    'child_rules': [],
                    'childs': ''
                }],
                'request_args': {
                    'method': 'get',
                    'url': 'http://httpbin.org/forms/post',
                    'headers': {
                        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36'
                    }
                },
                'regex': '',
                'encoding': ''
            }
        }
    })
print(rule.dumps())
# orjson.JSONEncodeError: Type is not JSON serializable: CrawlerRule