当前位置 博文首页 > Python中运行并行任务技巧

    Python中运行并行任务技巧

    作者:admin 时间:2021-07-25 10:48

    示例

    标准线程多进程,生产者/消费者示例:
    Worker越多,问题越大

    复制代码 代码如下:

    # -*- coding: utf8 -*-

    import os
    import time
    import Queue
    import threading
    from PIL import Image

    def create_thumbnail(filename, size=(128, 128)):
        try:
            fp, fmt = filename.rsplit('.', 1)
            im = Image.open(filename)
            im.thumbnail(size, Image.ANTIALIAS)
            im.save((fp + '_'+'x'.join(str(i) for i in size) + '.'+fmt), im.format)
            return '%s thumbnail success!' % filename
        except Exception:
            return '%s thumbnail failed!' % filename


    def get_image_paths(folder):
        return [os.path.join(folder, f) for f in os.listdir(folder) if 'png' in f]


    class Consumer(threading.Thread):
        def __init__(self, queue):
            threading.Thread.__init__(self)
            self._queue = queue

        def run(self):
            while True:
                content = self._queue.get()
                if isinstance(content, str) and content == 'quit':
                    break
                respone = create_thumbnail(content)
            print 'Bye bye!'


    def Producer():
        filenames = get_image_paths('images')
        queue = Queue.Queue()
        worker_threads = build_worker_pool(queue, 4)
        start_time = time.time()

        for filename in filenames:
            queue.put(filename)
        for worker in worker_threads:
            queue.put('quit')
        for worker in worker_threads:
            worker.join()

        print time.time() - start_time


    def build_worker_pool(queue, size):
        workers = []
        for _ in range(size):
            worker = Consumer(queue)
            worker.start()
            workers.append(worker)
        return workers


    if __name__ == '__main__':
        Producer()

    map

    Map能够处理集合按顺序遍历,最终将调用产生的结果保存在一个简单的集合当中。

    复制代码 代码如下:

    # -*- coding: utf8 -*-

    import os
    import time
    from multiprocessing import Pool
    from PIL import Image

    def create_thumbnail(filename, size=(128, 128)):
        try:
            fp, fmt = filename.rsplit('.', 1)
            im = Image.open(filename)
            im.thumbnail(size, Image.ANTIALIAS)
            im.save((fp + '_'+'x'.join(str(i) for i in size) + '.'+fmt), im.format)
            return '%s thumbnail success!' % filename
        except Exception:
            return '%s thumbnail failed!' % filename


    def get_image_paths(folder):
        return [os.path.join(folder, f) for f in os.listdir(folder) if 'png' in f]


    def main():
        filenames = get_image_paths('images')
        start_time = time.time()
       
        pool = Pool(4)
        pool.map(create_thumbnail, filenames)
        pool.close()
        pool.join()

        print time.time() - start_time


    if __name__ == '__main__':
        main()

    jsjbwy
    下一篇:没有了