百度文库,文字类型的资源(如txt, doc, pdf),是可以在前端源码分析获取到的,如果能按照规则合理的提取这些文字资源,就可以实现免下载券获取资源.
作者完成了百度文库的在线版爬虫,只需提供文库资源页面的url,就可以获取对应的文库资源(免下载券)
####在线测试地址:
http://zhaozhaoli.vicp.io/spider/bdwk复制代码
####效果展示
关于ppt的爬取,作者也写出来了,但是我的个人服务器带宽太小,导致前端等待时间过长,就把它注释了...有兴趣的可以看以前写过的这篇文章,文章底部提供了编译好的可执行文件(支持ppt,txt,pdf,word格式的下载),可以下载到本地玩一玩...
核心源码:
import osimport reimport jsonimport requestsfrom lxml import etreeimport sysfrom fangyuanxiaozhan.settings import BASE_DIR# import tkinter# 创建文库基类class BaiduWK(object): def __init__(self, url): self.title = None self.url = url self.docType = None self.headers = { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.75 Safari/537.36'} self.get_response_content(self.url) self.get_doc_type_and_title() def get_response_content(self, url): try: response = requests.get(url, headers=self.headers) return response.content except Exception as e: print(e) pass def get_doc_type_and_title(self): # 获取源码 source_html = self.get_response_content(self.url) # 解析源码 content = source_html.decode('gbk') # 获取文档类型 self.docType = re.findall(r"docType.*?\:.*?\'(.*?)\'\,", content)[0] # 获取文档标题 self.title = re.findall(r"title.*?\:.*?\'(.*?)\'\,", content)[0]# 创建获取txt的类class BDWKTXT(BaiduWK): def __init__(self, url): super().__init__(url) self.docId = None pass def get_txt(self, url): # 获取源码 source_html = self.get_response_content(url) content = source_html.decode("gbk") # 获取docId self.docId = re.findall(r"docId.*?(\w{24}?)\'\,", content)[0] # 拼接请求url token_url = "https://wenku.baidu.com/api/doc/getdocinfo?callback=cb&doc_id=" + self.docId # 再次请求 first_json = self.get_response_content(token_url).decode() str_first_json = re.match(r'.*?\((\{.*?\})\).*', first_json).group(1) # print(str_first_json) the_first_json = json.loads(str_first_json) md5sum = the_first_json["md5sum"] rn = the_first_json["docInfo"]["totalPageNum"] rsign = the_first_json["rsign"] # print(md5sum,"-->",rn) # 请求目标url target_url = "https://wkretype.bdimg.com/retype/text/" + self.docId + "?" + md5sum + "&callback=cb" + "&pn=1&rn=" + rn + "&type=txt" + "&rsign=" + rsign # https://wkretype.bdimg.com/retype/text/de53bafaf705cc1755270982?md5sum=b89f055e765e6f73db57525bcfc3c2d2&sign=7b385f9cf7&callback=cb&pn=1&rn=12&type=txt sec_json = self.get_response_content(target_url).decode() # print(type(sec_json),"-->") str_sec_json = re.match(r'.*?\(\[(.*)\]\)$', sec_json).group(1) str_sec_json += "," str_json_list = str_sec_json.split('},') result_txt = "" # 截取尾部空格 str_json_list = str_json_list[:-1] for str_json in str_json_list: str_json += "}" pure_dic = json.loads(str_json) pure_txt = pure_dic["parags"][0]["c"].strip() result_txt += pure_txt # 创建文件目录 try: # path = "." + os.sep + self.docType path = BASE_DIR + "/static/spider_file/bdwk" + os.sep + self.docType print(type(path),"path-->",path) os.makedirs(path) except Exception as e: # print("文件夹%s已存在"%(path)) pass # 创建文件,保存信息 try: print(os.path) file_name = BASE_DIR+"/static/spider_file/bdwk" + os.sep + self.docType + os.sep + self.title + ".txt" with open(file_name, 'w', encoding='utf-8') as f: f.write(result_txt) print("已经保存为:", self.title + '.txt') except Exception as e: # print(e) pass# 创建获取word的类class BDWKDOC(BaiduWK): def __init__(self, url): super().__init__(url) # 保存数据来源url self.pure_addr_list = list() # 获取数据来源url def get_pure_addr_list(self): # 获取页面源码 source_html = self.get_response_content(self.url).decode('gbk') # 从源码中批量提取数据url all_addr = re.findall(r'wkbos\.bdimg\.com.*?json.*?expire.*?\}', source_html) pure_addr_list = list() # 获取文档标题 self.title = etree.HTML(source_html).xpath("//title/text()")[0].strip() # 净化数据来源url列表 for addr in all_addr: addr = "https://" + addr.replace("\\\\\\/", "/") addr = addr[:-5] pure_addr_list.append(addr) # 将处理好的url列表保存为全局属性 self.pure_addr_list = pure_addr_list return pure_addr_list # 从数据来源的url列表中提取数据 def get_json_content(self, url_list): content = '' result = '' sum = len(url_list) i = 1 for pure_addr in url_list: print("正在下载第%d条数据, 剩余%d条" % (i, sum - i)) i += 1 try: # 获取json数据 content = self.get_response_content(pure_addr).decode() # 处理json数据 content = re.match(r'.*?\((.*)\)$', content).group(1) # 将json数据中需要的内容提取出来 all_body_info = json.loads(content)["body"] # 遍历获取所有信息,并将信息拼接到一起 for body_info in all_body_info: try: result = result + body_info["c"].strip() # print(">>",result) except Exception as e: print(e) pass except Exception as e: print(e) pass # 创建文件目录 try: path = BASE_DIR+"/static/spider_file/bdwk" + os.sep + self.docType # print(type(path),"path-->",path) os.makedirs(path) except Exception as e: # print("文件夹%s已存在"%(path)) pass # 创建文件,保存信息 try: file_name = BASE_DIR+"/static/spider_file/bdwk" + os.sep + self.docType + os.sep + self.title + ".txt" with open(file_name, 'w', encoding='utf-8') as f: f.write(result) print("已经保存为:", self.title + '.txt') except Exception as e: print(e)# 创建获取ppt的类class BDWKPPT(BaiduWK): def __init__(self, url): self.all_img_url = list() super().__init__(url) # 获取json数据保存文件 def get_ppt_json_info(self): # 获取源文件 ppt_source_html = self.get_response_content(self.url) # 解析源文件 content = ppt_source_html.decode('gbk') # 测试 with open("test.html", "w") as f: f.write(content) # 获取文档Id self.docId = re.findall(r"docId.*?(\w{24}?)\'\,", content)[0] # 拼接请求json的接口 source_json_url = 'https://wenku.baidu.com/browse/getbcsurl?doc_id=%s&type=ppt&callback=zhaozhao' % self.docId # 获取字符串类型的json数据 str_source_json = self.get_response_content(source_json_url).decode() # 处理字符串类型的json数据,使其成为标准格式 pure_str_source_json = re.match(r'.*?\((.*?)\)', str_source_json).group(1) # 将字符串json转为可处理的正式json source_json = json.loads(pure_str_source_json) # 遍历字典中的数据类型list for j in source_json['list']: # 创建临时列表 temp_num_url = list() # 将url和page拼接到列表中 temp_num_url.append(j["zoom"]) temp_num_url.append(j["page"]) # 将列表信息添加到全局变量中 self.all_img_url.append(temp_num_url) # 建立文件夹 try: os.makedirs(BASE_DIR+"/static/spider_file/bdwk/ppt/%s" % (self.title)) except Exception as e: pass for img_url in self.all_img_url: print(BASE_DIR+"正在获取第%d页资源(剩余%d页)" % (img_url[1], len(self.all_img_url) - img_url[1])) data = self.get_response_content(img_url[0]) path = BASE_DIR+"/static/spider_file/bdwk/ppt/%s/%s" % (self.title, str(img_url[1]) + '.jpg') with open(path, 'wb') as f: f.write(data) print("写入完毕")# 运行主程序def main(url): try: # url = input("请输入资源所在的网址:") docType = BaiduWK(url).docType except: print("您输入的url,有误请重新输入!") os.exit() print("类型为", "-->", docType) if docType == "ppt2": ppt = BDWKPPT(url) print("您将要获取的演示文稿(ppt)名称为:", ppt.title) ppt.get_ppt_json_info() data = dict() src = "/static/spider_file/bdwk/ppt/%s/" % (ppt.title) title_name = ppt.title data["src"] = src data["title_name"] = title_name return data elif docType == "doc": word = BDWKDOC(url) print("您将要获取的文档(word)名称为", word.title) pure_addr_list = word.get_pure_addr_list() word.get_json_content(pure_addr_list) data = dict() src = "/static/spider_file/bdwk/doc/%s.txt" % (word.title) title_name = word.title data["src"] = src data["title_name"] = title_name+".txt" return data elif docType == "pdf": pdf = BDWKDOC(url) print("您将要获取的PDF名称为:", pdf.title) pure_addr_list = pdf.get_pure_addr_list() pdf.get_json_content(pure_addr_list) data = dict() src = "/static/spider_file/bdwk/pdf/%s.txt" % (pdf.title) title_name = pdf.title data["src"] = src data["title_name"] = title_name+".txt" return data elif docType == "txt": txt = BDWKTXT(url) print("您将要下载的文本文档(txt)名称为:", txt.title) txt.get_txt(url) data = dict() src = "/static/spider_file/bdwk/txt/%s.txt" % (txt.title) title_name = txt.title data["src"] = src data["title_name"] = title_name+".txt" return data else: other = BDWKPPT(url) print("暂不支持下载%s类型" % (other.docType)) data = dict() data["src"] = "" data["title_name"] = "暂不支持%s类型"%(other.docType) return data复制代码