Pfthb

I'm making a simple WebScraping that download the image of the items of some champions of a site, I put a "for" with 5 characters and it only executes 2 of them and then closes without giving any error!

import bs4 as bs
import sys,os
import urllib.request
from PyQt5.QtWebEngineWidgets import QWebEnginePage
from PyQt5.QtWidgets import QApplication
from PyQt5.QtCore import QUrl

class Page(QWebEnginePage):
 def __init__(self, url):
 self.app = QApplication(sys.argv)
 QWebEnginePage.__init__(self)
 self.html = ''
 print("#1 __init__")
 self.loadFinished.connect(self._on_load_finished)
 self.load(QUrl(url))
 self.app.exec_()

 def _on_load_finished(self):
 self.html = self.toHtml(self.Callable)
 print('#2 On Load finished')

 def Callable(self, html_str):
 print("#3 Callablen")
 self.html = html_str
 self.app.quit()

def already_exist(image_name):
 for _, _, folder in os.walk('Images'):
 if image_name in folder:
 return False
 else:
 return True

def ImageDownload(url):
 image_name = url.split("/")
 try:
 if already_exist(image_name[-1]):
 full_path = "Images/" + image_name[-1]
 urllib.request.urlretrieve(url, full_path)
 print("Download %s" % image_name)
 else:
 print("Image already Downloaded >: %s" % image_name[-1])
 except:
 print("Error Download")

def main():
 champions = ['Amumu','Akali','Zed','Nunu'] #champions
 for champ in champions:
 try:
 print("nDownloading Images >: %s"% champ)
 data = Page('https://www.probuilds.net/champions/details/%s' % champ.strip())
 soup = bs.BeautifulSoup(data.html, 'html.parser')
 items = soup.find_all('div','class':'items')
 for photos in items:
 images = photos.find_all('img')
 for image in images:
 ImageDownload(image['src'])
 except:
 print("Shi...")

main()

i'm getting no error but the program only executes 2 times this is the problem, someone help me !!!

What it seems is that the QWebEnginePage does not close correctly, it is also advisable to reuse instead of creating another QWebEnginePage, so using an old answer as a basis I have implemented the following solution:

import os
import sys
import bs4 as bs
import urllib.request
from PyQt5 import QtCore, QtWidgets, QtWebEngineWidgets

class WebPage(QtWebEngineWidgets.QWebEnginePage):
 def __init__(self):
 super(WebPage, self).__init__()
 self.loadFinished.connect(self.handleLoadFinished)

 def start(self, urls):
 self._urls = iter(urls)
 self.fetchNext()

 def fetchNext(self):
 try:
 url = next(self._urls)
 except StopIteration:
 return False
 else:
 self.load(QtCore.QUrl(url))
 return True

 def processCurrentPage(self, html):
 self.process(self.url(), html)
 if not self.fetchNext():
 QtWidgets.qApp.quit()

 def handleLoadFinished(self):
 self.toHtml(self.processCurrentPage)

 def process(self, url, html):
 print('loaded: [%d chars] %s' % (len(html), url.toString()))

class ScrapePage(WebPage):
 def __init__(self):
 super(ScrapePage, self).__init__()
 self.results = set()

 def process(self, url, html):
 soup = bs.BeautifulSoup(html, 'html.parser')
 items = soup.find_all('div','class':'items')
 for photos in items:
 images = photos.find_all('img')
 for image in images:
 self.results.add(image['src'])

def already_exist(image_name):
 for _, _, folder in os.walk('Images'):
 if image_name in folder:
 return False
 else:
 return True

def ImageDownload(url):
 image_name = url.split("/")
 try:
 if already_exist(image_name[-1]):
 full_path = "Images/" + image_name[-1]
 urllib.request.urlretrieve(url, full_path)
 print("Download %s" % image_name)
 else:
 print("Image already Downloaded >: %s" % image_name[-1])
 except:
 print("Error Download")

if __name__ == '__main__':

 app = QtWidgets.QApplication(sys.argv)
 webpage = ScrapePage()

 champions = ['Amumu','Akali','Zed','Nunu']
 base_url = 'https://www.probuilds.net/champions/details/'

 urls = 
 for champ in champions:
 url = QtCore.QUrl(base_url).resolved(QtCore.QUrl(champ))
 urls.append(url)
 webpage.start(urls)
 app.exec_()
 for url in webpage.results:
 ImageDownload(url)