Selenium을 활용한 크림 판매정보 크롤링

오전부터 새벽 4시까지, 중간에 국취 상담도 받고 크로스핏도 하고 오며 만든 코드다. 🔥

 

먼저 제품 ID 검색하기

from selenium import webdriver
from selenium.webdriver.common.by import By

options = webdriver.ChromeOptions()
options.add_argument('--headless')
options.add_argument('user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36') # 크림에서 headless 브라우저 걸러냄, user-agent 추가해야 데이터 스크랩 가능
options.add_argument('--no-sandbox')
options.add_argument('--disable-dev-shm-usage')

driver = webdriver.Chrome(options=options)

a = input('아이템명/품번 입력 : ')
driver.get(f'https://kream.co.kr/search?keyword={a}&tab=products')

search_result_list = []

for b in driver.find_elements(By.CSS_SELECTOR,'.search_result_item.product'):
    name = b.find_element(By.CLASS_NAME,'name').text
    pid = b.get_attribute('data-product-id')
    search_result_list.append('[' + pid + '] ' + name)

for c in search_result_list:
    print(c)

 

제품ID 알았으면 로그인하고 판매 데이터 끌어오기

from selenium import webdriver
from selenium.webdriver.common.by import By
import time
import pandas as pd

options = webdriver.ChromeOptions()
options.add_argument('--headless')
options.add_argument('--window-size=1920,1080') # ViewPort를 PC 크기로 고정
options.add_argument('user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36')
options.add_argument('--no-sandbox')
options.add_argument('--disable-dev-shm-usage')

driver = webdriver.Chrome(options=options)

salesList = []

def login():
    driver.get('https://kream.co.kr/login')
    driver.find_element(By.XPATH,'🙃').send_keys('email@kream.co.kr')
    driver.find_element(By.XPATH,'🙃').send_keys('Password1!')
    driver.find_element(By.CLASS_NAME,'🙃').click()

def productPage():
    global pid, size, item
    pid = input('product ID : ')
    size = input('size : ')
    driver.get(f'https://kream.co.kr/products/{pid}/?size={size}')
    item = driver.title
    print(item)

def scrapeSales(salesList):
    driver.find_element(By.XPATH,'🙃').click()
    mpt = driver.find_element(By.CLASS_NAME,'🙃')
    pb = mpt.find_element(By.CLASS_NAME,'🙃')
    for i in range(0,50000,1000): # 자바스크립트로 페이지 스크롤해서 데이터 가져오기
        driver.execute_script('arguments[0].scrollTop = arguments[1];', pb, i)
        time.sleep(0.5)
    body_list = pb.find_elements(By.CLASS_NAME,'🙃')
    for sales in body_list:
        size_sold = sales.find_element(By.XPATH,'🙃').text
        price_sold = sales.find_element(By.XPATH,'🙃').text
        date_sold = sales.find_element(By.XPATH,'🙃').text
        try:
            express = ' (' + sales.find_element(By.XPATH,'🙃').text + ')'
        except:
            express = ''
        salesList.append({
            'date': date_sold,
            'size': size_sold,
            'price': price_sold + express
        })

def exportCSV(salesList):
    tables = pd.DataFrame(salesList)
    if size == '':
        tables.to_csv(f'[{pid}] {item}.csv')
    else:
        tables.to_csv(f'[{pid}-{size}] {item}.csv')

login()
productPage()
scrapeSales(salesList)
exportCSV(salesList)

 

로직은 다 공유했으니 🙃에 들어갈 부분은 알아서 찾아보시길. 😛