huytuong010101 / crawlnewsfromtuoitre.vn Goto Github PK

for link in links:
        try:
            news = requests.get(baseUrl + link)
            soup = BeautifulSoup(news.content, "html.parser")
            title = soup.find("h1", class_="article-title").text
            abstract = soup.find("h2", class_="sapo").text
            body = soup.find("div", id="main-detail-body")
            content = ""
            try:
                content = body.findChildren("p", recursive=False)[0].text + body.findChildren("p", recursive=False)[1].text
            except:
                content = ""
            image = body.find("img").attrs["src"]
            data.append({
                "title": title,
                "abstract": abstract,
                "content": content,
                "image": image,
            })
            print("craw " + title)
        except:
            continue
    return data