from grab.spider import Spider, Task, Data
from grab.tools.logs import default_logging
from grab import Grab
import re
class KinoSpider(Spider):
def task_initial(self, grab, task):
for xpath in grab.tree.xpath('//*/div[@class="stat"]/div/a'):
name = xpath.text_content().encode('utf8')
name = re.sub("\(.*?\)",'',name)
self.out.write(name+"\n")
self.out.flush()
def main():
initial_urls = ["http://www.kinopoisk.ru/level/56/day/2012-02-22/page/{0}/".format(x) for x in xrange(1,51)]
threads = 50
default_logging(grab_log="log/log.txt")
fl = open("out.txt","w")
bot = KinoSpider(thread_number=threads,network_try_limit=2)
bot.initial_urls = initial_urls
bot.out = fl
try: bot.run()
except KeyboardInterrupt: pass
fl.close()
print bot.render_stats()
if __name__ == '__main__':
main()
Процесс парсинга занимает 3-5 секунд!