百度网盘搜索引擎（基于python）

#! /usr/bin/env python

#coding=utf-8 # import os from Queue import Queue import threading import time import urllib from urlparse import * from urlparse import urljoin import redis import pymongo import datetime import urllib2 import json import re import sys reload(sys) sys.setdefaultencoding('utf-8') def get_url(url): request = urllib2.Request(url) return urllib2.urlopen(request,timeout=20).read() # def get_count(url): # data = re.findall(r'totalCount:"(.+?)"', get_url(url)) # if len(data) == 0: # raise Exception('get_count err') # return data[0] # def get_count_all(uk): # follow_count = get_count('http://pan.baidu.com/wap/share/home/followers?uk='+str(uk)+'&start=0') # fan_count = get_count('http://pan.baidu.com/wap/share/home/fans?uk='+str(uk)+'&start=0') # return follow_count, fan_count def get_ren_info(url,neibie): data = get_url(url) matches = re.findall(r'parse("(.+?)"),totalCount', data) if len(matches) == 0: raise Exception('get_ren_info err') data_decode = matches[0].decode("unicode_escape") jsondata = json.loads(data_decode) rens = [] if len(jsondata) != 0: for i in jsondata: if neibie == 'follow': ren = {'fangwen':1,'uk':i['follow_uk'],'uname':i['follow_uname'],'avatar_url':i['avatar_url'],'intro':i['intro'],'fans_count':i['fans_count'],'follow_count':i['follow_count'],'pubshare_count':i['pubshare_count'],'album_count':i['album_count']} elif neibie == 'fans': ren = {'fangwen':1,'uk':i['fans_uk'],'uname':i['fans_uname'],'avatar_url':i['avatar_url'],'intro':i['intro'],'fans_count':i['fans_count'],'follow_count':i['follow_count'],'pubshare_count':i['pubshare_count'],'album_count':i['album_count']} rens.append(ren) return rens def get_uk_all(url): url.index('followers') return get_ren_info(url,'follow') url.index('fans') return get_ren_info(url,'fans') raise Exception('url str not found followers and fans') def ren_mongo(uk, number): db.ren.update({'uk':uk}, {'$set':{'fangwen':number}}) def url_mongo(url, number): db.url.update({'url':url}, {'$set':{'fangwen':number}}) def drop(): db.ren.drop() db.url.drop() def url_save(dics): db.url.save(dics) print pymongo.errors.DuplicateKeyError('url save suoying chongfu') def ren_save(dics): db.ren.save(dics) print pymongo.errors.DuplicateKeyError('ren save suoying chongfu') def init(): if not ren_one: url = 'http://pan.baidu.com/wap/share/home/followers?uk=657260084&start=0' url = 'http://pan.baidu.com/wap/share/home/followers?uk=657260084&start=24' print pymongo.errors.DuplicateKeyError('zai suoying chongfu') #queue for i in range(20): def check(): if ren_queue.qsize()<20: if url_queue.qsize() <20: def deal_ren(): # = '1' ren_one = ren_queue.get() uk = ren_one['uk'] print "["+datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")+"]["+name+"号] "+str(uk)+"doing " follow_count = ren_one['follow_count'] fans_count = ren_one['fans_count'] print follow_count print fans_count for i in range(0,(24 if follow_count > 24 else follow_count),24): url_save({'url':'http://pan.baidu.com/wap/share/home/followers?uk='+str(uk)+'&start='+str(i),'fangwen':1}) for j in range(0,(24 if fans_count > 24 else fans_count),24): url_save({'url':'http://pan.baidu.com/wap/share/home/fans?uk='+str(uk)+'&start='+str(j),'fangwen':1}) ren_mongo(uk, 3) #db.ren.update({'uk':uk}, {'$set':{'fangwen':1}}) def deal_url(): #name = '1' url_one = url_queue.get() url = url_one['url'] #db.url.update({'url':url}, {'$set':{'fangwen':2}})