Scrapy + pyqt5: signal only works in main thread Error

Question

I wrote a spider with scrapy which used CrawlerProcess and worked fine.

Next step, I wanted a GUI dialog for some input data made with pyqt5, and set a spinner while scrapy is working. This task was made using a dummy process and also worked fine.

I am aware that multithreading is not compatible with CrawlerProcess, so I changed it for CrawlerRunner. Anyway, it fails:

signal.signal(signal.SIGINT, self.sigInt)
  File "C:\Program Files\Python36\lib\signal.py", line 47, in signal
    handler = _signal.signal(_enum_to_int(signalnum), _enum_to_int(handler))
builtins.ValueError: signal only works in main thread

I have created a minimal example to reproduce the error:

from PyQt5.QtWidgets import QApplication, QDialog, QTabWidget, QWidget, QGroupBox, QPushButton, QVBoxLayout
from PyQt5.QtCore import Qt, QRunnable, QMetaObject, pyqtSlot, QThreadPool
from PyQt5.QtGui import QIcon
from scrapy.crawler import CrawlerRunner
from scrapy import Request, Spider, Item, Field
from twisted.internet import reactor
from waitingspinnerwidget import QtWaitingSpinner

class RequestRunnable(QRunnable):
    def __init__(self, dialog):
        super(RequestRunnable, self).__init__()
        self.w = dialog
        print('runnable __init__')

    def run(self):
        print('run')
        crawler = CrawlerRunner({
            'USER_AGENT': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)',
            'FEED_FORMAT': 'json',
            'FEED_URI': 'output.json',
            'DOWNLOAD_DELAY': 3,
            'LOG_STDOUT': True,
            'LOG_FILE': 'scrapy_output.txt',
            'ROBOTSTXT_OBEY': False,
            'RETRY_ENABLED': True,
            'RETRY_HTTP_CODES': [500, 503, 504, 400, 404, 408],
            'RETRY_TIMES': 5
        })

        # instantiate a spider
        spider = CustomSpider()
        d = crawler.crawl(spider)
        d.addBoth(lambda _: reactor.stop()) # I think this is for manually stopping the reactor when we are done
        reactor.run() #TODO gives error
        QMetaObject.invokeMethod(self.w, "FinishedDownload", Qt.QueuedConnection)


class CustomSpider(Spider):
    name = 'quotes_spider'
    allowed_domains = ['quotes.toscrape.com']
    start_urls = ['http://quotes.toscrape.com/']

    def parse(self, response):
        quotes = response.xpath("//div[@class='quote']//span[@class='text']/text()").extract()
        yield {'quotes': quotes}


class DownloadDataDialog(QDialog):
    def __init__(self, parent=None):
        super(DownloadDataDialog, self).__init__(parent)

        self.spinner = QtWaitingSpinner(self, True, True, Qt.ApplicationModal)

        tabWidget = QTabWidget(self)
        tabWidget.addTab(MyTab(tabWidget), "MyTab")

        mainLayout = QVBoxLayout()
        mainLayout.addWidget(tabWidget)
        self.setLayout(mainLayout)

        self.setWindowTitle("Download option chain data from web")


class MyTab(QWidget):
    def __init__(self, parent=None):
        super(MyTab, self).__init__(parent)

        dataGroup = QGroupBox('Data')

        getButton = QPushButton('Download')
        getButton.clicked.connect(self.download_data)

        dataLayout = QVBoxLayout()
        dataLayout.addWidget(getButton)
        dataGroup.setLayout(dataLayout)

        mainLayout = QVBoxLayout()
        mainLayout.addWidget(dataGroup)
        mainLayout.addStretch(1)
        self.setLayout(mainLayout)

    def download_data(self):
        self.parentWidget().window().spinner.start()
        runnable = RequestRunnable(self)
        QThreadPool.globalInstance().start(runnable)

    @pyqtSlot()
    def FinishedDownload(self):
        self.parentWidget().window().spinner.stop()

if __name__ == '__main__':

    import sys

    app = QApplication(sys.argv)

    tabdialog = DownloadDataDialog()
    tabdialog.show()
    sys.exit(app.exec_())

In order to execute it, you will need waitingspinnerwidget.py:

# -*- coding: utf-8 -*-
"""
The MIT License (MIT)

Copyright (c) 2012-2014 Alexander Turkin
Copyright (c) 2014 William Hallatt
Copyright (c) 2015 Jacob Dawid
Copyright (c) 2016 Luca Weiss

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""

import math

from PyQt5.QtCore import *
from PyQt5.QtGui import *
from PyQt5.QtWidgets import *


class QtWaitingSpinner(QWidget):
    def __init__(self, parent, centerOnParent=True, disableParentWhenSpinning=False, modality=Qt.NonModal):
        super().__init__(parent, Qt.Dialog | Qt.FramelessWindowHint)

        self._centerOnParent = centerOnParent
        self._disableParentWhenSpinning = disableParentWhenSpinning

        # WAS IN initialize()
        self._color = QColor(Qt.black)
        self._roundness = 100.0
        self._minimumTrailOpacity = 3.14159265358979323846
        self._trailFadePercentage = 80.0
        self._revolutionsPerSecond = 1.57079632679489661923
        self._numberOfLines = 20
        self._lineLength = 10
        self._lineWidth = 2
        self._innerRadius = 10
        self._currentCounter = 0
        self._isSpinning = False

        self._timer = QTimer(self)
        self._timer.timeout.connect(self.rotate)
        self.updateSize()
        self.updateTimer()
        self.hide()
        # END initialize()

        self.setWindowModality(modality)
        self.setAttribute(Qt.WA_TranslucentBackground)

    def paintEvent(self, QPaintEvent):
        self.updatePosition()
        painter = QPainter(self)
        painter.fillRect(self.rect(), Qt.transparent)
        painter.setRenderHint(QPainter.Antialiasing, True)

        if self._currentCounter >= self._numberOfLines:
            self._currentCounter = 0

        painter.setPen(Qt.NoPen)
        for i in range(0, self._numberOfLines):
            painter.save()
            painter.translate(self._innerRadius + self._lineLength, self._innerRadius + self._lineLength)
            rotateAngle = float(360 * i) / float(self._numberOfLines)
            painter.rotate(rotateAngle)
            painter.translate(self._innerRadius, 0)
            distance = self.lineCountDistanceFromPrimary(i, self._currentCounter, self._numberOfLines)
            color = self.currentLineColor(distance, self._numberOfLines, self._trailFadePercentage,
                                          self._minimumTrailOpacity, self._color)
            painter.setBrush(color)
            painter.drawRoundedRect(QRect(0, -self._lineWidth / 2, self._lineLength, self._lineWidth), self._roundness,
                                    self._roundness, Qt.RelativeSize)
            painter.restore()

    def start(self):
        self.updatePosition()
        self._isSpinning = True
        self.show()

        if self.parentWidget and self._disableParentWhenSpinning:
            self.parentWidget().setEnabled(False)

        if not self._timer.isActive():
            self._timer.start()
            self._currentCounter = 0

    def stop(self):
        self._isSpinning = False
        self.hide()

        if self.parentWidget() and self._disableParentWhenSpinning:
            self.parentWidget().setEnabled(True)

        if self._timer.isActive():
            self._timer.stop()
            self._currentCounter = 0

    def setNumberOfLines(self, lines):
        self._numberOfLines = lines
        self._currentCounter = 0
        self.updateTimer()

    def setLineLength(self, length):
        self._lineLength = length
        self.updateSize()

    def setLineWidth(self, width):
        self._lineWidth = width
        self.updateSize()

    def setInnerRadius(self, radius):
        self._innerRadius = radius
        self.updateSize()

    def color(self):
        return self._color

    def roundness(self):
        return self._roundness

    def minimumTrailOpacity(self):
        return self._minimumTrailOpacity

    def trailFadePercentage(self):
        return self._trailFadePercentage

    def revolutionsPersSecond(self):
        return self._revolutionsPerSecond

    def numberOfLines(self):
        return self._numberOfLines

    def lineLength(self):
        return self._lineLength

    def lineWidth(self):
        return self._lineWidth

    def innerRadius(self):
        return self._innerRadius

    def isSpinning(self):
        return self._isSpinning

    def setRoundness(self, roundness):
        self._roundness = max(0.0, min(100.0, roundness))

    def setColor(self, color=Qt.black):
        self._color = QColor(color)

    def setRevolutionsPerSecond(self, revolutionsPerSecond):
        self._revolutionsPerSecond = revolutionsPerSecond
        self.updateTimer()

    def setTrailFadePercentage(self, trail):
        self._trailFadePercentage = trail

    def setMinimumTrailOpacity(self, minimumTrailOpacity):
        self._minimumTrailOpacity = minimumTrailOpacity

    def rotate(self):
        self._currentCounter += 1
        if self._currentCounter >= self._numberOfLines:
            self._currentCounter = 0
        self.update()

    def updateSize(self):
        size = (self._innerRadius + self._lineLength) * 2
        self.setFixedSize(size, size)

    def updateTimer(self):
        self._timer.setInterval(1000 / (self._numberOfLines * self._revolutionsPerSecond))

    def updatePosition(self):
        if self.parentWidget() and self._centerOnParent:
            self.move(self.parentWidget().width() / 2 - self.width() / 2,
                      self.parentWidget().height() / 2 - self.height() / 2)
            dialogCenter = self.mapToGlobal(self.rect().center())
            parentWindowCenter = self.parentWidget().window().mapToGlobal(self.parentWidget().window().rect().center())
            self.move(parentWindowCenter - dialogCenter)
            parentRect = QRect(self.parentWidget().mapToGlobal(QPoint(0, 0)), self.parentWidget().size())
            self.move(QStyle.alignedRect(Qt.LeftToRight, Qt.AlignCenter, self.size(), parentRect).topLeft())

    def lineCountDistanceFromPrimary(self, current, primary, totalNrOfLines):
        distance = primary - current
        if distance < 0:
            distance += totalNrOfLines
        return distance

    def currentLineColor(self, countDistance, totalNrOfLines, trailFadePerc, minOpacity, colorinput):
        color = QColor(colorinput)
        if countDistance == 0:
            return color
        minAlphaF = minOpacity / 100.0
        distanceThreshold = int(math.ceil((totalNrOfLines - 1) * trailFadePerc / 100.0))
        if countDistance > distanceThreshold:
            color.setAlphaF(minAlphaF)
        else:
            alphaDiff = color.alphaF() - minAlphaF
            gradient = alphaDiff / float(distanceThreshold + 1)
            resultAlpha = color.alphaF() - gradient * countDistance
            # If alpha is out of bounds, clip it.
            resultAlpha = min(1.0, max(0.0, resultAlpha))
            color.setAlphaF(resultAlpha)
        return color

Full traceback:

Traceback (most recent call last):
  File "C:\Users\BonifacioFdez\AppData\Roaming\Python\Python36\site-packages\twisted\internet\base.py", line 428, in fireEvent
    DeferredList(beforeResults).addCallback(self._continueFiring)
  File "C:\Users\BonifacioFdez\AppData\Roaming\Python\Python36\site-packages\twisted\internet\defer.py", line 322, in addCallback
    callbackKeywords=kw)
  File "C:\Users\BonifacioFdez\AppData\Roaming\Python\Python36\site-packages\twisted\internet\defer.py", line 311, in addCallbacks
    self._runCallbacks()
  File "C:\Users\BonifacioFdez\AppData\Roaming\Python\Python36\site-packages\twisted\internet\defer.py", line 654, in _runCallbacks
    current.result = callback(current.result, *args, **kw)
--- <exception caught here> ---
  File "C:\Users\BonifacioFdez\AppData\Roaming\Python\Python36\site-packages\twisted\internet\base.py", line 441, in _continueFiring
    callable(*args, **kwargs)
  File "C:\Users\BonifacioFdez\AppData\Roaming\Python\Python36\site-packages\twisted\internet\base.py", line 1256, in _reallyStartRunning
    self._handleSignals()
  File "C:\Users\BonifacioFdez\AppData\Roaming\Python\Python36\site-packages\twisted\internet\posixbase.py", line 295, in _handleSignals
    _SignalReactorMixin._handleSignals(self)
  File "C:\Users\BonifacioFdez\AppData\Roaming\Python\Python36\site-packages\twisted\internet\base.py", line 1221, in _handleSignals
    signal.signal(signal.SIGINT, self.sigInt)
  File "C:\Program Files\Python36\lib\signal.py", line 47, in signal
    handler = _signal.signal(_enum_to_int(signalnum), _enum_to_int(handler))
builtins.ValueError: signal only works in main thread

Any idea of what is going on and how to solve it?

Answer 1

Don't use threads, just use a qt reactor for twisted, that way both your twisted application (scrapy) and your qt5 application can run in the same event loop:

Install qt5reactor:

pip install qt5reactor

Then add code to install the reactor and run it:

app = QApplication(sys.argv)

import qt5reactor
qt5reactor.install()

reactor.run()