Scrapy + pyqt5：信号仅在主线程错误中有效

Question

我用scrapy写了一个蜘蛛，它使用了CrawlerProcess并且运行良好。

下一步，我想要一个用于pyqt5的输入数据的GUI对话框，并在scrapy工作时设置一个微调器。这项任务是使用虚拟过程完成的，并且工作正常。

我知道多线程与CrawlerProcess不兼容，所以我为CrawlerRunner更改了它。无论如何，它失败了：

signal.signal(signal.SIGINT, self.sigInt)
  File "C:Program FilesPython36libsignal.py", line 47, in signal
    handler = _signal.signal(_enum_to_int(signalnum), _enum_to_int(handler))
builtins.ValueError: signal only works in main thread

我创建了一个重现错误的最小示例：

from PyQt5.QtWidgets import QApplication, QDialog, QTabWidget, QWidget, QGroupBox, QPushButton, QVBoxLayout
from PyQt5.QtCore import Qt, QRunnable, QMetaObject, pyqtSlot, QThreadPool
from PyQt5.QtGui import QIcon
from scrapy.crawler import CrawlerRunner
from scrapy import Request, Spider, Item, Field
from twisted.internet import reactor
from waitingspinnerwidget import QtWaitingSpinner

class RequestRunnable(QRunnable):
    def __init__(self, dialog):
        super(RequestRunnable, self).__init__()
        self.w = dialog
        print('runnable __init__')

    def run(self):
        print('run')
        crawler = CrawlerRunner({
            'USER_AGENT': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)',
            'FEED_FORMAT': 'json',
            'FEED_URI': 'output.json',
            'DOWNLOAD_DELAY': 3,
            'LOG_STDOUT': True,
            'LOG_FILE': 'scrapy_output.txt',
            'ROBOTSTXT_OBEY': False,
            'RETRY_ENABLED': True,
            'RETRY_HTTP_CODES': [500, 503, 504, 400, 404, 408],
            'RETRY_TIMES': 5
        })

        # instantiate a spider
        spider = CustomSpider()
        d = crawler.crawl(spider)
        d.addBoth(lambda _: reactor.stop()) # I think this is for manually stopping the reactor when we are done
        reactor.run() #TODO gives error
        QMetaObject.invokeMethod(self.w, "FinishedDownload", Qt.QueuedConnection)


class CustomSpider(Spider):
    name = 'quotes_spider'
    allowed_domains = ['quotes.toscrape.com']
    start_urls = ['http://quotes.toscrape.com/']

    def parse(self, response):
        quotes = response.xpath("//div[@class='quote']//span[@class='text']/text()").extract()
        yield {'quotes': quotes}


class DownloadDataDialog(QDialog):
    def __init__(self, parent=None):
        super(DownloadDataDialog, self).__init__(parent)

        self.spinner = QtWaitingSpinner(self, True, True, Qt.ApplicationModal)

        tabWidget = QTabWidget(self)
        tabWidget.addTab(MyTab(tabWidget), "MyTab")

        mainLayout = QVBoxLayout()
        mainLayout.addWidget(tabWidget)
        self.setLayout(mainLayout)

        self.setWindowTitle("Download option chain data from web")


class MyTab(QWidget):
    def __init__(self, parent=None):
        super(MyTab, self).__init__(parent)

        dataGroup = QGroupBox('Data')

        getButton = QPushButton('Download')
        getButton.clicked.connect(self.download_data)

        dataLayout = QVBoxLayout()
        dataLayout.addWidget(getButton)
        dataGroup.setLayout(dataLayout)

        mainLayout = QVBoxLayout()
        mainLayout.addWidget(dataGroup)
        mainLayout.addStretch(1)
        self.setLayout(mainLayout)

    def download_data(self):
        self.parentWidget().window().spinner.start()
        runnable = RequestRunnable(self)
        QThreadPool.globalInstance().start(runnable)

    @pyqtSlot()
    def FinishedDownload(self):
        self.parentWidget().window().spinner.stop()

if __name__ == '__main__':

    import sys

    app = QApplication(sys.argv)

    tabdialog = DownloadDataDialog()
    tabdialog.show()
    sys.exit(app.exec_())

要执行它，您需要waitingspinnerwidget.py：

# -*- coding: utf-8 -*-
"""
The MIT License (MIT)

Copyright (c) 2012-2014 Alexander Turkin
Copyright (c) 2014 William Hallatt
Copyright (c) 2015 Jacob Dawid
Copyright (c) 2016 Luca Weiss

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
"""

import math

from PyQt5.QtCore import *
from PyQt5.QtGui import *
from PyQt5.QtWidgets import *


class QtWaitingSpinner(QWidget):
    def __init__(self, parent, centerOnParent=True, disableParentWhenSpinning=False, modality=Qt.NonModal):
        super().__init__(parent, Qt.Dialog | Qt.FramelessWindowHint)

        self._centerOnParent = centerOnParent
        self._disableParentWhenSpinning = disableParentWhenSpinning

        # WAS IN initialize()
        self._color = QColor(Qt.black)
        self._roundness = 100.0
        self._minimumTrailOpacity = 3.14159265358979323846
        self._trailFadePercentage = 80.0
        self._revolutionsPerSecond = 1.57079632679489661923
        self._numberOfLines = 20
        self._lineLength = 10
        self._lineWidth = 2
        self._innerRadius = 10
        self._currentCounter = 0
        self._isSpinning = False

        self._timer = QTimer(self)
        self._timer.timeout.connect(self.rotate)
        self.updateSize()
        self.updateTimer()
        self.hide()
        # END initialize()

        self.setWindowModality(modality)
        self.setAttribute(Qt.WA_TranslucentBackground)

    def paintEvent(self, QPaintEvent):
        self.updatePosition()
        painter = QPainter(self)
        painter.fillRect(self.rect(), Qt.transparent)
        painter.setRenderHint(QPainter.Antialiasing, True)

        if self._currentCounter >= self._numberOfLines:
            self._currentCounter = 0

        painter.setPen(Qt.NoPen)
        for i in range(0, self._numberOfLines):
            painter.save()
            painter.translate(self._innerRadius + self._lineLength, self._innerRadius + self._lineLength)
            rotateAngle = float(360 * i) / float(self._numberOfLines)
            painter.rotate(rotateAngle)
            painter.translate(self._innerRadius, 0)
            distance = self.lineCountDistanceFromPrimary(i, self._currentCounter, self._numberOfLines)
            color = self.currentLineColor(distance, self._numberOfLines, self._trailFadePercentage,
                                          self._minimumTrailOpacity, self._color)
            painter.setBrush(color)
            painter.drawRoundedRect(QRect(0, -self._lineWidth / 2, self._lineLength, self._lineWidth), self._roundness,
                                    self._roundness, Qt.RelativeSize)
            painter.restore()

    def start(self):
        self.updatePosition()
        self._isSpinning = True
        self.show()

        if self.parentWidget and self._disableParentWhenSpinning:
            self.parentWidget().setEnabled(False)

        if not self._timer.isActive():
            self._timer.start()
            self._currentCounter = 0

    def stop(self):
        self._isSpinning = False
        self.hide()

        if self.parentWidget() and self._disableParentWhenSpinning:
            self.parentWidget().setEnabled(True)

        if self._timer.isActive():
            self._timer.stop()
            self._currentCounter = 0

    def setNumberOfLines(self, lines):
        self._numberOfLines = lines
        self._currentCounter = 0
        self.updateTimer()

    def setLineLength(self, length):
        self._lineLength = length
        self.updateSize()

    def setLineWidth(self, width):
        self._lineWidth = width
        self.updateSize()

    def setInnerRadius(self, radius):
        self._innerRadius = radius
        self.updateSize()

    def color(self):
        return self._color

    def roundness(self):
        return self._roundness

    def minimumTrailOpacity(self):
        return self._minimumTrailOpacity

    def trailFadePercentage(self):
        return self._trailFadePercentage

    def revolutionsPersSecond(self):
        return self._revolutionsPerSecond

    def numberOfLines(self):
        return self._numberOfLines

    def lineLength(self):
        return self._lineLength

    def lineWidth(self):
        return self._lineWidth

    def innerRadius(self):
        return self._innerRadius

    def isSpinning(self):
        return self._isSpinning

    def setRoundness(self, roundness):
        self._roundness = max(0.0, min(100.0, roundness))

    def setColor(self, color=Qt.black):
        self._color = QColor(color)

    def setRevolutionsPerSecond(self, revolutionsPerSecond):
        self._revolutionsPerSecond = revolutionsPerSecond
        self.updateTimer()

    def setTrailFadePercentage(self, trail):
        self._trailFadePercentage = trail

    def setMinimumTrailOpacity(self, minimumTrailOpacity):
        self._minimumTrailOpacity = minimumTrailOpacity

    def rotate(self):
        self._currentCounter += 1
        if self._currentCounter >= self._numberOfLines:
            self._currentCounter = 0
        self.update()

    def updateSize(self):
        size = (self._innerRadius + self._lineLength) * 2
        self.setFixedSize(size, size)

    def updateTimer(self):
        self._timer.setInterval(1000 / (self._numberOfLines * self._revolutionsPerSecond))

    def updatePosition(self):
        if self.parentWidget() and self._centerOnParent:
            self.move(self.parentWidget().width() / 2 - self.width() / 2,
                      self.parentWidget().height() / 2 - self.height() / 2)
            dialogCenter = self.mapToGlobal(self.rect().center())
            parentWindowCenter = self.parentWidget().window().mapToGlobal(self.parentWidget().window().rect().center())
            self.move(parentWindowCenter - dialogCenter)
            parentRect = QRect(self.parentWidget().mapToGlobal(QPoint(0, 0)), self.parentWidget().size())
            self.move(QStyle.alignedRect(Qt.LeftToRight, Qt.AlignCenter, self.size(), parentRect).topLeft())

    def lineCountDistanceFromPrimary(self, current, primary, totalNrOfLines):
        distance = primary - current
        if distance < 0:
            distance += totalNrOfLines
        return distance

    def currentLineColor(self, countDistance, totalNrOfLines, trailFadePerc, minOpacity, colorinput):
        color = QColor(colorinput)
        if countDistance == 0:
            return color
        minAlphaF = minOpacity / 100.0
        distanceThreshold = int(math.ceil((totalNrOfLines - 1) * trailFadePerc / 100.0))
        if countDistance > distanceThreshold:
            color.setAlphaF(minAlphaF)
        else:
            alphaDiff = color.alphaF() - minAlphaF
            gradient = alphaDiff / float(distanceThreshold + 1)
            resultAlpha = color.alphaF() - gradient * countDistance
            # If alpha is out of bounds, clip it.
            resultAlpha = min(1.0, max(0.0, resultAlpha))
            color.setAlphaF(resultAlpha)
        return color

完全追溯：

Traceback (most recent call last):
  File "C:UsersBonifacioFdezAppDataRoamingPythonPython36site-packages	wistedinternetase.py", line 428, in fireEvent
    DeferredList(beforeResults).addCallback(self._continueFiring)
  File "C:UsersBonifacioFdezAppDataRoamingPythonPython36site-packages	wistedinternetdefer.py", line 322, in addCallback
    callbackKeywords=kw)
  File "C:UsersBonifacioFdezAppDataRoamingPythonPython36site-packages	wistedinternetdefer.py", line 311, in addCallbacks
    self._runCallbacks()
  File "C:UsersBonifacioFdezAppDataRoamingPythonPython36site-packages	wistedinternetdefer.py", line 654, in _runCallbacks
    current.result = callback(current.result, *args, **kw)
--- <exception caught here> ---
  File "C:UsersBonifacioFdezAppDataRoamingPythonPython36site-packages	wistedinternetase.py", line 441, in _continueFiring
    callable(*args, **kwargs)
  File "C:UsersBonifacioFdezAppDataRoamingPythonPython36site-packages	wistedinternetase.py", line 1256, in _reallyStartRunning
    self._handleSignals()
  File "C:UsersBonifacioFdezAppDataRoamingPythonPython36site-packages	wistedinternetposixbase.py", line 295, in _handleSignals
    _SignalReactorMixin._handleSignals(self)
  File "C:UsersBonifacioFdezAppDataRoamingPythonPython36site-packages	wistedinternetase.py", line 1221, in _handleSignals
    signal.signal(signal.SIGINT, self.sigInt)
  File "C:Program FilesPython36libsignal.py", line 47, in signal
    handler = _signal.signal(_enum_to_int(signalnum), _enum_to_int(handler))
builtins.ValueError: signal only works in main thread

知道发生了什么以及如何解决它？