웹 크롤링 파이썬 6

[예제]

import requests
from bs4 import BeautifulSoup

#url = "http://jolse.com/category/tonermist/43/"url = "http://jolse.com/category/tonermist/43/"headers = {'User-Agent': 'Mozilla/5.0'}
result = requests.get(url, headers = headers)

bs_obj = BeautifulSoup(result.content, "html.parser")

#print(bs_obj)ul = bs_obj.find("ul",{"class":"prdList column5"})
#print(ul)
'''import requestsfrom bs4 import BeautifulSoup
URL = 'http://www.naver.com'res = requests.get(URL, headers=headers)soup = BeautifulSoup(res.text)print(soup)
'''

boxes = ul.findAll("div", {"class":"box"})

def get_product_info(box):
    ptag = box.find("p", {"class":"name"})
    spans_name = ptag.findAll("span")
    ul = box.find("ul")
    spans_price = ul.findAll("span")

    name = spans_name[1].text
    price = spans_price[1].text

    atag = box.find("a")
    link = atag['href']
    #print(atag)    #print(ul)    #print(spans_price[1].text)    #print(price)
    #return spans[1].text    #return spans_name[1].text    return {"name":name, "price":price, "link":link}
'''for box in boxes:    product_info = get_product_info(box)    print(product_info)'''
product_info_list = [get_product_info(box) for box in boxes]

print(product_info_list)

[결과]

[{'name': 'iUNIK Tea Tree Relief Toner 200ml', 'price': 'USD 21.99', 'link': '/product/detail.html?product_no=18491&cate_no=43&display_group=1&product_no=18491'}, {'name': 'iUNIK Vitamin Hyaluronic Acid Vitalizing Toner 200ml', 'price': 'USD 21.99', 'link': '/product/detail.html?product_no=18489&cate_no=43&display_group=1&product_no=18489'}, {'name': 'iUNIK Rose Galactomyces Essential Toner 200ml', 'price': 'USD 21.99', 'link': '/product/detail.html?product_no=18488&cate_no=43&display_group=1&product_no=18488'}, {'name': 'MISSHA Vitamin B12 Double Hydrop Booster 195ml', 'price': 'USD 19.66', 'link': '/product/detail.html?product_no=18390&cate_no=43&display_group=1&product_no=18390'}, {'name': 'MISSHA Bee Pollen Renew Treatment 150ml', 'price': 'USD 20.66', 'link': '/product/detail.html?product_no=18387&cate_no=43&display_group=1&product_no=18387'}, {'name': 'G9SKIN White In Milk Toner 300ml', 'price': 'USD 18.00', 'link': '/product/detail.html?product_no=18332&cate_no=43&display_group=1&product_no=18332'}, {'name': "A'PIEU Cicative Boosting Toner 200ml", 'price': 'USD 20.07', 'link': '/product/detail.html?product_no=18327&cate_no=43&display_group=1&product_no=18327'}, {'name': 'seaNtree Donkey Milk Waterising Toner 175ml', 'price': 'USD 20.50', 'link': '/product/detail.html?product_no=18252&cate_no=43&display_group=1&product_no=18252'}, {'name': 'seaNtree Green Tea Deep Deep Deep Toner 180ml', 'price': 'USD 18.00', 'link': '/product/detail.html?product_no=18250&cate_no=43&display_group=1&product_no=18250'}, {'name': 'O HUI Meister For Men Moisturizing All-in-One 110ml', 'price': 'USD 74.98', 'link': '/product/detail.html?product_no=18239&cate_no=43&display_group=1&product_no=18239'}, {'name': 'O HUI Prime Advancer Skin Softener 150ml', 'price': 'USD 55.98', 'link': '/product/detail.html?product_no=18224&cate_no=43&display_group=1&product_no=18224'}, {'name': 'O HUI The First Geniture Skin Softener 150ml', 'price': 'USD 74.98', 'link': '/product/detail.html?product_no=18212&cate_no=43&display_group=1&product_no=18212'}, {'name': 'ELRASTORY Rose Gold blossom Moist Up Toner 270ml', 'price': 'USD 23.50', 'link': '/product/detail.html?product_no=18196&cate_no=43&display_group=1&product_no=18196'}, {'name': 'INBELLO Hibiscus Hidratante Toner 140ml', 'price': 'USD 18.00', 'link': '/product/detail.html?product_no=17935&cate_no=43&display_group=1&product_no=17935'}, {'name': 'The FACE Shop Triple Acid Boosting Water 200ml', 'price': 'USD 19.66', 'link': '/product/detail.html?product_no=17923&cate_no=43&display_group=1&product_no=17923'}, {'name': 'VDL PANTONE Expert Radiance Fix Mist 120ml', 'price': 'USD 24.48', 'link': '/product/detail.html?product_no=17915&cate_no=43&display_group=1&product_no=17915'}, {'name': 'ETUDE HOUSE Modern Homme All In One Essence 100ml', 'price': 'USD 24.16', 'link': '/product/detail.html?product_no=17900&cate_no=43&display_group=1&product_no=17900'}, {'name': 'ETUDE HOUSE Glow On Mist 93ml', 'price': 'USD 16.66', 'link': '/product/detail.html?product_no=17891&cate_no=43&display_group=1&product_no=17891'}, {'name': 'innisfree Green Tea Balancing Skin EX (7days) 10ml', 'price': 'USD 5.48', 'link': '/product/detail.html?product_no=17877&cate_no=43&display_group=1&product_no=17877'}, {'name': 'innisfree Cauliflower Mushroom Vital Skin 200ml', 'price': 'USD 28.66', 'link': '/product/detail.html?product_no=17873&cate_no=43&display_group=1&product_no=17873'}, {'name': 'TONYMOLY Floria Nutra Energy Deep Cream Mist 120ml', 'price': 'USD 14.48', 'link': '/product/detail.html?product_no=17855&cate_no=43&display_group=1&product_no=17855'}, {'name': 'Aromatica Toner 375ml', 'price': 'USD 23.00', 'link': '/product/detail.html?product_no=17837&cate_no=43&display_group=1&product_no=17837'}, {'name': 'Aromatica Tea Tree Balancing Toner 130ml', 'price': 'USD 21.00', 'link': '/product/detail.html?product_no=17816&cate_no=43&display_group=1&product_no=17816'}, {'name': 'Sulwhasoo Timeterasure Renovating Water EX 125ml', 'price': 'USD 108.48', 'link': '/product/detail.html?product_no=17805&cate_no=43&display_group=1&product_no=17805'}, {'name': 'COSRX AC Collection Calming Liquid Mild 125ml', 'price': 'USD 26.00', 'link': '/product/detail.html?product_no=17796&cate_no=43&display_group=1&product_no=17796'}, {'name': 'Nature Republic Real Comforting Hyaluronic Water Toner 400ml', 'price': 'USD 25.66', 'link': '/product/detail.html?product_no=17781&cate_no=43&display_group=1&product_no=17781'}, {'name': 'TONYMOLY The Black Tea London Classic Toner 160ml', 'price': 'USD 26.57', 'link': '/product/detail.html?product_no=17630&cate_no=43&display_group=1&product_no=17630'}, {'name': 'SIORIS Feel So Fresh Toner 150ml', 'price': 'USD 22.00', 'link': '/product/detail.html?product_no=17539&cate_no=43&display_group=1&product_no=17539'}, {'name': 'SIORIS Time Is Running Out Mist 100ml', 'price': 'USD 22.00', 'link': '/product/detail.html?product_no=17538&cate_no=43&display_group=1&product_no=17538'}, {'name': 'CARYOPHY Portulaca Cleansing Foam 150ml + CARYOPHY Portulaca Toner 300ml', 'price': 'USD 30.45', 'link': '/product/detail.html?product_no=17522&cate_no=43&display_group=1&product_no=17522'}, {'name': 'Hanyul Mentha Trouble Toner 300ml', 'price': 'USD 30.57', 'link': '/product/detail.html?product_no=17496&cate_no=43&display_group=1&product_no=17496'}, {'name': 'Hanyul Pure Artemisia Fresh Calming Water 150ml', 'price': 'USD 30.57', 'link': '/product/detail.html?product_no=17494&cate_no=43&display_group=1&product_no=17494'}, {'name': 'Hanyul Rice Essential Skin Mist 150ml', 'price': 'USD 29.07', 'link': '/product/detail.html?product_no=17468&cate_no=43&display_group=1&product_no=17468'}, {'name': 'HanYul Rice Essential Skin Softner 150ml', 'price': 'USD 38.07', 'link': '/product/detail.html?product_no=17464&cate_no=43&display_group=1&product_no=17464'}, {'name': 'HanYul Geuk Jin Toner 150ml', 'price': 'USD 58.57', 'link': '/product/detail.html?product_no=17460&cate_no=43&display_group=1&product_no=17460'}, {'name': 'KEEP COOL Soothe Bamboo Toner 350ml', 'price': 'USD 30.00', 'link': '/product/detail.html?product_no=17454&cate_no=43&display_group=1&product_no=17454'}, {'name': 'IOPE Moistgen Softner 150ml', 'price': 'USD 34.16', 'link': '/product/detail.html?product_no=17450&cate_no=43&display_group=1&product_no=17450'}, {'name': 'TOAS Miracle Laser Skin Toner 150ml', 'price': 'USD 35.00', 'link': '/product/detail.html?product_no=17445&cate_no=43&display_group=1&product_no=17445'}, {'name': "A'PIEU Nonco Tea Tree Toner Tissue 20sheets", 'price': 'USD 8.57', 'link': '/product/detail.html?product_no=17418&cate_no=43&display_group=1&product_no=17418'}, {'name': 'NACIFIC Fresh Herb Origin Cotton Toner 70ea', 'price': 'USD 22.00', 'link': '/product/detail.html?product_no=16882&cate_no=43&display_group=1&product_no=16882'}, {'name': 'NACIFIC Glow Intensive Bubble Toner 145ml', 'price': 'USD 23.00', 'link': '/product/detail.html?product_no=16880&cate_no=43&display_group=1&product_no=16880'}, {'name': 'NACIFIC Phyto Niacin Whitening Toner 150ml', 'price': 'USD 23.00', 'link': '/product/detail.html?product_no=16879&cate_no=43&display_group=1&product_no=16879'}, {'name': 'NACIFIC Real Rose Floral Toner 180ml', 'price': 'USD 20.50', 'link': '/product/detail.html?product_no=16878&cate_no=43&display_group=1&product_no=16878'}, {'name': 'NACIFIC Real Calendula Floral Toner 180ml', 'price': 'USD 20.50', 'link': '/product/detail.html?product_no=16877&cate_no=43&display_group=1&product_no=16877'}, {'name': 'NACIFIC Fresh Herb Origin Toner 150ml', 'price': 'USD 20.50', 'link': '/product/detail.html?product_no=16876&cate_no=43&display_group=1&product_no=16876'}, {'name': 'Dr.Jart+ Ceramidin Liquid 150ml', 'price': 'USD 31.00', 'link': '/product/detail.html?product_no=16860&cate_no=43&display_group=1&product_no=16860'}, {'name': 'PURITO Centella Green Level Calming Toner 200ml', 'price': 'USD 22.12', 'link': '/product/detail.html?product_no=16854&cate_no=43&display_group=1&product_no=16854'}, {'name': 'CNP Laboratory Aqua Soothing Toner Special Set 200ml', 'price': 'USD 31.00', 'link': '/product/detail.html?product_no=16831&cate_no=43&display_group=1&product_no=16831'}, {'name': 'CNP Laboratory Hydro Intensive Toner 150ml', 'price': 'USD 26.40', 'link': '/product/detail.html?product_no=16819&cate_no=43&display_group=1&product_no=16819'}, {'name': 'CNP Laboratory Dual Balance Anti-Oily Toner 150ml', 'price': 'USD 34.00', 'link': '/product/detail.html?product_no=16812&cate_no=43&display_group=1&product_no=16812'}]