PyQt5 GUI become unresponsive while looping

Question

After i click the button, the form become unresponsive until the parsing function finish its work.

I'd like to move searchAll function to thread. I did read several answers to similar questions, but i didn't understand how.

class MyForm(QDialog):


    def __init__(self):
        super().__init__()
        self.ui = Ui_Dialog()
        self.ui.setupUi(self)
        self.ui.buttonOK.clicked.connect(self.searchAll)
        self.show()

    def searchAll(self):

        sID = self.ui.txtSellerID.text()
        sUrl = "https://removed.com/" + sID + "/p/?section=2&page=1"
        sr = requests.get(sUrl)
        soup1 = BeautifulSoup(sr.text, "html.parser")

        NumberOfPagesBlock = soup1.find_all("li", class_="text-gray")

        if not NumberOfPagesBlock:
            QMessageBox.about(self, "Warning", "Nothing Here")
        else:
            items = re.compile(r'[^\d.]+')
            PagesCount = -(-items // 60)

            for i in range(1, int(PagesCount + 1)):
                itemsIdDs = soup1.find_all("div", class_="large-single-item")

                for itemsIdD in itemsIdDs:
                    iUrl = ("https://removed.com/" + itemsIdDs.get('data-ean') + "/s")
                    r = requests.get(iUrl)
                    soup = BeautifulSoup(r.text, "html.parser")
                    seller = soup.find("div", id="productTrackingParams")
                    title = (str(ctr) + '- ' + "Title " + str(seller.get('data-title')))
                    self.ui.txtDetails.appendPlainText(title)


if __name__ == "__main__":

    app = QApplication(sys.argv)
    w = MyForm()
    w.show()
    sys.exit(app.exec_())

Answer 1

You have to execute the heavy task (requests + BeautifulSoup) in another thread since they block the main thread where the GUI lives, preventing the GUI from working correctly and this is manifested, for example, by freezing the screen. In this case I will implement a worker-thread approach:

import re
import ssl
import sys
from functools import partial

import requests
from PyQt5.QtCore import QObject, QThread, QTimer, pyqtSignal, pyqtSlot
from PyQt5.QtWidgets import QApplication, QDialog, QMessageBox
from bs4 import BeautifulSoup

from foo_module import Ui_Dialog


class ScrapeWorker(QObject):
    started = pyqtSignal()
    finished = pyqtSignal()
    resultsChanged = pyqtSignal(str)
    errorSignal = pyqtSignal(str)

    @pyqtSlot(str)
    def run(self, text):
        self.started.emit()
        sUrl = "https://removed.com/{}/p/?section=2&page=1".format(text)
        try:
            sr = requests.get(sUrl)
        except Exception as e:
            self.errorSignal.emit("error: {}".format(e))
            self.finished.emit()
            return
        soup1 = BeautifulSoup(sr.text, "html.parser")
        NumberOfPagesBlock = soup1.find_all("li", class_="text-gray")
        if not NumberOfPagesBlock:
            self.errorSignal.emit("Nothing Here")
        else:
            items = re.compile(r"[^\d.]+")
            PagesCount = -(-items // 60)
            for i in range(1, int(PagesCount + 1)):
                itemsIdDs = soup1.find_all("div", class_="large-single-item")

                for itemsIdD in itemsIdDs:
                    iUrl = "https://removed.com/{}/s".format(itemsIdDs.get("data-ean"))
                    r = requests.get(iUrl)
                    soup = BeautifulSoup(r.text, "html.parser")
                    seller = soup.find("div", id="productTrackingParams")
                    title = "{}- Title {}".format(ctr, seller.get("data-title"))
                    self.resultsChanged.emit(title)
        self.finished.emit()


class MyForm(QDialog):
    def __init__(self):
        super().__init__()
        self.ui = Ui_Dialog()
        self.ui.setupUi(self)
        self.ui.buttonOK.clicked.connect(self.searchAll)

        thread = QThread(self)
        thread.start()

        self.m_worker = ScrapeWorker()
        self.m_worker.moveToThread(thread)
        self.m_worker.started.connect(self.onStarted)
        self.m_worker.finished.connect(self.onFinished)
        self.m_worker.resultsChanged.connect(self.onResultChanged)
        self.m_worker.errorSignal.connect(self.onErrorSignal)

    @pyqtSlot()
    def searchAll(self):
        sID = self.ui.txtSellerID.text()
        wrapper = partial(self.m_worker.run, sID)
        QTimer.singleShot(0, wrapper)

    @pyqtSlot(str)
    def onResultChanged(self, title):
        self.ui.txtDetails.appendPlainText(title)

    @pyqtSlot()
    def onStarted(self):
        self.ui.buttonOK.setEnabled(False)

    @pyqtSlot()
    def onFinished(self):
        self.ui.buttonOK.setEnabled(True)

    @pyqtSlot(str)
    def onErrorSignal(self, message):
        QMessageBox.about(self, "Warning", message)


if __name__ == "__main__":

    app = QApplication(sys.argv)
    w = MyForm()
    w.show()
    sys.exit(app.exec_())