Scrapy框架爬取Boss直聘網Python職位信息的源碼

LinkExtractor(allow=(), # 使用正則定義提取規則
       deny=(), # 排除規則
       allow_domains=(), # 限定域名范圍
       deny_domains=(), # 排除域名范圍
       restrict_xpaths=(), # 使用xpath定義提取隊則
       tags=('a', 'area'), 
       attrs=('href',), 
       canonicalize=False,
       unique=True, 
       process_value=None,
       deny_extensions=None, 
       restrict_css=(), # 使用css選擇器定義提取規則
       strip=True):

Rule(link_extractor, # LinkExtractor對象,必選參數
   callback=None, # 回調方法,可選
   cb_kwargs=None, 
   follow=None, # 是否進行深度爬取,True、False
   process_links=None, # 用于處理鏈接(有些反爬策略是返回假的url)
   process_request=identity)

class BosszhipinItem(scrapy.Item):
  """Boss直聘Pytho職位爬蟲Item"""
  # 職位名稱
  position=scrapy.Field()
  # 公司名稱
  company=scrapy.Field()
  # 薪資
  salary=scrapy.Field()
  # 工作地點
  location=scrapy.Field()
  # 學歷要求
  education=scrapy.Field()
  # 工作時間
  year=scrapy.Field()

# !/usr/bin/env python
# -*- coding:utf-8 -*-
import scrapy
from scrapy.spider import CrawlSpider,Rule
from scrapy.linkextractors import LinkExtractor
from myscrapy.items import BosszhipinItem
class BosszhipinSpider(CrawlSpider):
  """
  Boss直聘Python職位爬蟲Spider
    使用CrawlSpider基類實現
  """
  name = 'bosszhipin'
  allowed_domains=['zhipin.com',]
  start_urls=['http://www.zhipin.com/c100010000/h_100010000/?query=Python&page=1',]
  # 鏈接提取器對象(規定鏈接提取規則)
  link_extractor=LinkExtractor(allow=(r'page=\d+'))
  # 鏈接提取規則對象列表
  # 自動調用callback指定的方法,去取爬取由link_extractor指定的鏈接提取規則匹配到的url
  # 原理:link_extractor.extract_links(response)返回匹配到的鏈接
  rules = [
    Rule(link_extractor=link_extractor,callback='parse_page',follow=True),
  ]
  def parse_page(self,response):
    """定義回調方法,用于解析每個response對象"""
    job_list=response.xpath('//div[@class="job-list"]//li')
    for job in job_list:
      position = job.xpath('.//div[@class="info-primary"]//h4[@class="name"]/a/text()')[0].extract()
      salary =job.xpath('.//div[@class="info-primary"]//h4[@class="name"]//span/text()')[0].extract()
      company =job.xpath('.//div[@class="company-text"]//a/text()')[0].extract()
      location =job.xpath('.//div[@class="info-primary"]/p/text()[1]')[0].extract()
      year =job.xpath('.//div[@class="info-primary"]/p/text()[2]')[0].extract()
      education =job.xpath('.//div[@class="info-primary"]/p/text()[3]')[0].extract()
      item=BosszhipinItem()
      item['position']=position
      item['salary']=salary
      item['company']=company
      item['location']=location
      item['year']=year
      item['education']=education
      yield item

class BosszhipinPipeline(object):
  """Boss直聘Python職位爬蟲Item Pipeline"""
  def __init__(self):
    self.f=open('data/bosszhipin.json',mode='wb')
    self.f.write(b'[')
  def process_item(self,item,spider):
    data=json.dumps(dict(item),ensure_ascii=False,indent=4)
    self.f.write(data.encode('utf-8'))
    self.f.write(b',')
    return item
  def close_spider(self,spider):
    self.f.write(b']')
    self.f.close()

ITEM_PIPELINES = {
  'myscrapy.pipelines.BosszhipinPipeline': 1,
}