автоматический просмотр веб-сайта

Еще один пример для lawbot.org :

import requests
from bs4 import BeautifulSoup

base_url = 'http://lawbot.org'
search_url = base_url + '/?q=유죄'

response = requests.get(search_url)

page = BeautifulSoup(response.text, "html.parser")
lastPageNumber = int(page.select_one("li.page-item:not(.next):nth-last-child(2)").text)

casesList = []

for i in range(1, lastPageNumber + 1):
    if i > 1:
        response = requests.get(search_url + "&page=" + str(i))
        page = BeautifulSoup(response.text, "html.parser")

    cases = page.select("div.panre_center > ul.media-list li.panre_lists")
    for case in cases:
        title = case.findChild("h6").text
        caseDocNumber = case.findChild(attrs={"class": "caseDocNumber"}).text
        caseCourt = case.findChild(attrs={"class": "caseCourt"}).text
        case_url = base_url + case.findChild("a")['href']

        casesList.append({"title": title, "caseDocNumber": caseDocNumber, "caseCourt": caseCourt, "case_url": case_url})
        # print("title:{}, caseDocNumber:{}, caseCourt:{}, caseUrl:{}".format(title, caseDocNumber, caseCourt, case_url))

for case in casesList:
    response = requests.get(case["case_url"])
    page = BeautifulSoup(response.text, "html.parser")
    body = page.find(attrs={"class": "panre_body"}).text
    print(body)