Parsing Python Queue object

问题:

I am thinking where the problem is in my code

from queue import Queue
from threading import Thread
from html.parser import HTMLParser
import urllib.request

hosts = ["http://yahoo.com", "http://google.com", "http://ibm.com"]

queue = Queue()

class ThreadUrl(Thread):
   def __init__(self, queue):
       Thread.__init__(self)
       self.queue = queue

   def run(self):
      while True:
         host = self.queue.get()
         url=urllib.request.urlopen(host)
         url.read(4096)
         self.queue.task_done()


class MyHTMLParser(HTMLParser):
    def handle_starttag(self, tag, attrs):
        print("Start tag:", tag)
        for attr in attrs:
            print("     attr:", attr)



def consumer():
    for i in range(3):
        t = ThreadUrl(queue)
        t.setDaemon(True)
        t.start()

    for host in hosts:
        parser = MyHTMLParser()
        parser.feed(host)
        queue.put(host) 
    queue.join()

consumer()

My goal is to extract content of the URLS,read the queue and finally parse it.When I execute the code it does not print anything.Where should I place the parser?

回答1:

Here is an example:

from queue import Queue
from threading import Thread
from html.parser import HTMLParser
import urllib.request


NUMBER_OF_THREADS = 3


HOSTS = ["http://yahoo.com", "http://google.com", "http://ibm.com"]


class MyHTMLParser(HTMLParser):
    def handle_starttag(self, tag, attrs):
        print("Start tag:", tag)
        for attr in attrs:
            print("\tattr:", attr)


class ThreadUrl(Thread):
   def __init__(self, queue):
       Thread.__init__(self)
       self.queue = queue

   def run(self):
       while True:
           host = self.queue.get()
           url = urllib.request.urlopen(host)
           content = str(url.read(4096))
           parser = MyHTMLParser()
           parser.feed( content )
           self.queue.task_done()


def consumer():
    queue = Queue()
    for i in range(NUMBER_OF_THREADS):
        thread = ThreadUrl(queue)
        thread.setDaemon(True)
        thread.start()
    for host in HOSTS:
        queue.put(host) 
    queue.join()


if __name__ == '__main__':
    consumer()