I use python Selenium and Scrapy for crawling a website.
but my script is so slow,
Crawled 1 pages (at 1 pages/min)
i use CSS SELECTOR instead of XPATH for optimise the time. i change the midllewares
'tutorial.middlewares.MyCustomDownloaderMiddleware': 543,
is Selenium is too slow or i should change something in Setting?
my Code:
def start_requests(self):
yield Request(self.start_urls, callback=self.parse)
def parse(self, response):
display = Display(visible=0, size=(800, 600))
display.start()
driver = webdriver.Firefox()
driver.get("http://www.example.com")
inputElement = driver.find_element_by_name("OneLineCustomerAddress")
inputElement.send_keys("75018")
inputElement.submit()
catNums = driver.find_elements_by_css_selector("html body div#page div#main.content div#sContener div#menuV div#mvNav nav div.mvNav.bcU div.mvNavLk form.jsExpSCCategories ul.mvSrcLk li")
#INIT
driver.find_element_by_css_selector(".mvSrcLk>li:nth-child(1)>label.mvNavSel.mvNavLvl1").click()
for catNumber in xrange(1,len(catNums)+1):
print "\n IN catnumber \n"
driver.find_element_by_css_selector("ul#catMenu.mvSrcLk> li:nth-child(%s)> label.mvNavLvl1" % catNumber).click()
time.sleep(5)
self.parse_articles(driver)
pages = driver.find_elements_by_xpath('//*[@class="pg"]/ul/li[last()]/a')
if(pages):
page = driver.find_element_by_xpath('//*[@class="pg"]/ul/li[last()]/a')
checkText = (page.text).strip()
if(len(checkText) > 0):
pageNums = int(page.text)
pageNums = pageNums - 1
for pageNumbers in range (pageNums):
WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.ID, "waitingOverlay")))
driver.find_element_by_css_selector('.jsNxtPage.pgNext').click()
self.parse_articles(driver)
time.sleep(5)
def parse_articles(self,driver) :
test = driver.find_elements_by_css_selector('html body div#page div#main.content div#sContener div#sContent div#lpContent.jsTab ul#lpBloc li div.prdtBloc p.prdtBDesc strong.prdtBCat')
def between(self, value, a, b):
pos_a = value.find(a)
if pos_a == -1: return ""
pos_b = value.rfind(b)
if pos_b == -1: return ""
adjusted_pos_a = pos_a + len(a)
if adjusted_pos_a >= pos_b: return ""
return value[adjusted_pos_a:pos_b]