Python爬虫采集商品评价信息--京东

https://club.jd.com/comment/productPageComments.action?callback=fetchJSON_comment98&productId=10056364088483&score=0&sortType=5&page=0&pageSize=10&isShadowSku=0&fold=1

import random
import requests
import json
import re
import csv
import time
import pymysql

header = {       'refer': 'https: // item.jd.com /',       'cookie': '',       'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/110.0.0.0 Safari/537.36 Edg/110.0.1587.50'   }

parm = {       'callback': 'fetchJSON_comment98',       'productId': '10056364088483',       'score': '0',       'sortType': '5',       'page': page,       'pageSize': '10',       'isShadowSku': '0',       'fold': '1'   }
url = 'https://club.jd.com/comment/productPageComments.action'
res = requests.get(url, params=parm, headers=header)

# 程序休眠
time.sleep(random.randint(40, 80) * 0.1)
print('第%d页正在爬取' % (page + 1)) 爬取完成后，需要对页面进行编码，不影响后期的数据提取和数据清洗工作。 使用正则对数据进行提取，返回字符串。 字符串转换为json格式数据。
res.encoding = 'gb18030'
html = res.text
data = re.findall('fetchJSON_comment98\((.*?)\);', html)
data = json.loads(data[0])  # 将处理的数据进行解析
comments = data['comments']
print(data['comments'])

# 写入csv文件
f = open("evalution_data.csv", "a", newline='', encoding='gb18030')
header = ["id", "content", "creationTime", "score", "productColor", "productSize"]
# 创建一个DictWriter对象，第二个参数就是上面创建的表头
writer = csv.DictWriter(f, header)
writer.writeheader()   for i in comments:       id = i['id']       content = i['content']       creationTime = i['creationTime']       score = i['score']       productColor = i['productColor']       productSize = i['productSize']       writer.writerow(           {"id": id, "content": content, "creationTime": creationTime, "score": score, "productColor": productColor,            "productSize": productSize})
f.close()

# 写入数据库
conn = pymysql.connect(host='', user='', password='', port=, db='')
cursor = conn.cursor()   for i in comments:       id = i['id']       content = i['content']       creationTime = i['creationTime']       score = i['score']       productColor = i['productColor']       productSize = i['productSize']       sql = "insert into evalution_data(id,content,creationTime,score,productColor,productSize) values('%d','%s','%s','%d','%s','%s')"       cursor.execute(sql)       conn.commit()
cursor.close()
conn.close()