크롤링 ( 웹 스크랩핑 ) #python #BeautifulSoup #csv

import csv

import requests

from bs4 import BeautifulSoup

def extract_data(url):

response = requests.get(url)

soup = BeautifulSoup(response.text, 'html.parser')

# 각 필요한 태그 찾기

name_tags = soup.find("ul", attrs={"id":"listCompany"}).find_all("div", attrs={"class":"text"})

salary_tags = soup.find("ul", attrs={"id":"listCompany"}).find_all("div", attrs={"class":"salary"})

summary_tags = soup.find("ul", attrs={"id":"listCompany"}).find_all("div", attrs={"class":"summary"})

# photo_tags = soup.find("ul", attrs={"id":"listCompany"}).find_all("div", attrs={"class":"thumbnail"})

photo_tags = soup.find("ul", attrs={"id":"listCompany"}).find_all("img")

# name_tags = soup.select('ul id="listcompany" li a div div class="text"')

# sales_tags = soup.select('ul id="listcompany" li a div div class="item"')

# ppl_tags = soup.select('td[width="107"] p font span b')

# salary_tags = soup.select('td[width="107"] p font span b')

# 각 태그의 텍스트를 하나의 리스트에 저장

data = []

for name,salary,summary,photo in zip(name_tags,salary_tags,summary_tags,photo_tags):

data.append([name.text.strip(),salary.text.strip(),summary.text.strip(),photo["src"]])

return data

# 모든 페이지에서 데이터를 수집

all_data = []

for page in range(1, 11): # 1페이지부터 10페이지까지

url = f'https://www.jobkorea.co.kr/Salary/Index?coKeyword=&tabindex=0&indsCtgrCode=&indsCode=&jobTypeCode=&haveAGI=0&orderCode=2&coPage={page}#salarySearchCompany'

all_data.extend(extract_data(url))

# for row in all_data:

# try:

# parse=[row[0], row[1], row[2].split('\n')[1], row[2].split('\n')[2]]

# except IndexError:

# continue

# print(parse)

 

#찾은 태그들의 데이터를 CSV 파일로 저장

file_path = '/Users/hyunohsmacbook/DevsterCompanyinfo.csv'

with open(file_path, 'w', newline='', encoding='utf-8') as csvfile:

csv_writer = csv.writer(csvfile)

csv_writer.writerow(['회사명', '연봉', '매출액', '사원수','사진링크']) # 헤더 작성

for row in all_data:

try:

parse=[row[0], row[1], row[2].split('\n')[1], row[2].split('\n')[2],row[3]]

except IndexError:

continue

csv_writer.writerow(parse)