无法运行我的班级搜寻器

Question

用python编写类爬虫，我陷入了中途。 我不知道如何将[app_crawler类生成的]新生成的链接传递给“ App”类，以便在那里进行其余的操作。 如果有人通过展示如何运行它为我指明了正确的方向，那么我将非常有帮助。 提前致谢。 顺便说一句，它也正在运行，但仅用于单个链接。

from lxml import html
import requests

class app_crawler:

    starturl = "https://itunes.apple.com/us/app/candy-crush-saga/id553834731?mt=8"

    def crawler(self):
        self.get_app(self.starturl)


    def get_app(self, link):
        page = requests.get(link)
        tree = html.fromstring(page.text)
        links = tree.xpath('//div[@class="lockup-info"]//*/a[@class="name"]/@href')
        for link in links:
            return link # I wish to make this link penetrate through the App class but can't get any idea


class App(app_crawler):

    def __init__(self, link):
        self.links = [link]

    def process_links(self):
        for link in self.links:
            self.get_item(link)

    def get_item(self, url):
        page = requests.get(url)
        tree = html.fromstring(page.text)
        name = tree.xpath('//h1[@itemprop="name"]/text()')[0]
        developer = tree.xpath('//div[@class="left"]/h2/text()')[0]        
        price = tree.xpath('//div[@itemprop="price"]/text()')[0]
        print(name, developer, price)

if __name__ == '__main__':

    parse = App(app_crawler.starturl)
    parse.crawler()
    parse.process_links()

我创建了另一个工作正常的工具，但我想使上述爬虫具有不同的外观。 这是工作对象的链接：“ https://www.dropbox.com/s/galjorcdynueequ/Working%20one.txt?dl=0 ”

Answer 1

您的代码有几个问题：

• App继承自app_crawler但您向App.__init__提供了一个app_crawler实例。

• App.__init__调用app_crawler.__init__而不是super().__init__() 。

• app_crawler.get_app不仅不返回任何内容 ，还创建了一个全新的App对象。

这将导致您的代码传递一个app_crawler对象requests.get ，而不是一个URL字符串。

您的代码中封装太多 。

考虑下面的代码，这些代码比不工作的代码短，更干净，并且不需要不必要地传递对象：

from lxml import html
import requests

class App:
    def __init__(self, starturl):
        self.starturl = starturl
        self.links = []

    def get_links(self):
        page = requests.get(self.starturl)
        tree = html.fromstring(page.text)
        self.links = tree.xpath('//div[@class="lockup-info"]//*/a[@class="name"]/@href')

    def process_links(self):
        for link in self.links:
            self.get_docs(link)

    def get_docs(self, url):
        page = requests.get(url)
        tree = html.fromstring(page.text)
        name = tree.xpath('//h1[@itemprop="name"]/text()')[0]
        developper = tree.xpath('//div[@class="left"]/h2/text()')[0]
        price = tree.xpath('//div[@itemprop="price"]/text()')[0]
        print(name, developper, price)

if __name__ == '__main__':
    parse = App("https://itunes.apple.com/us/app/candy-crush-saga/id553834731?mt=8")
    parse.get_links()
    parse.process_links()

输出

Cookie Jam By Jam City, Inc. Free
Zombie Tsunami By Mobigame Free
Flow Free By Big Duck Games LLC Free
Bejeweled Blitz By PopCap Free
Juice Jam By Jam City, Inc. Free
Candy Crush Soda Saga By King Free
Bubble Witch 3 Saga By King Free
Candy Crush Jelly Saga By King Free
Farm Heroes Saga By King Free
Pet Rescue Saga By King Free

Answer 2

这是我期望代码的方式：

from lxml import html
import requests

class app_crawler:

    starturl = "https://itunes.apple.com/us/app/candy-crush-saga/id553834731?mt=8"

    def __init__(self):
        self.links = [self.starturl]


    def crawler(self):
        for link in self.links:
            self.get_app(link)


    def get_app(self, link):
        page = requests.get(link)
        tree = html.fromstring(page.text)
        links = tree.xpath('//div[@class="lockup-info"]//*/a[@class="name"]/@href')
        for link in links:
            if not len(self.links)>=5:
                self.links.append(link)


class App(app_crawler):

    def __init__(self):
        app_crawler.__init__(self)


    def process_links(self):
        for link in self.links:
            self.get_item(link)

    def get_item(self, url):
        page = requests.get(url)
        tree = html.fromstring(page.text)
        name = tree.xpath('//h1[@itemprop="name"]/text()')[0]
        developer = tree.xpath('//div[@class="left"]/h2/text()')[0]        
        price = tree.xpath('//div[@itemprop="price"]/text()')[0]
        print(name, developer, price)

if __name__ == '__main__':

    scrape = App()
    scrape.crawler()
    scrape.process_links()