Python之多进程

 1.最近比较喜欢的用法：

from concurrent.futures import ProcessPoolExecutor, as_completed

ex = ProcessPoolExecutor(max_workers=50)


tasks = [ex.submit(process_function, file) for file in file_list]

results = []
finish_num = 0
start_time = time.time()
for future in as_completed(tasks):
results.append(future.result())
used_time = time.time() - start_time
finish_num += 1
if finish_num % 100 == 0:
print('{}/{}, used {:.3}h, {:.3}s/line, ETS:{:.3}h'.format(finish_num, len(file_list),
used_time / 3600, used_time / finish_num,
used_time / finish_num * (
len(file_list) - finish_num) / 3600))

 

 

2.Pool的用法

#!/usr/bin/env python
# -*- coding: utf-8 -*-
'''
@author: Shiyu Huang
@contact: huangsy13@gmail.com
@file: asy_test.py
'''


import numpy as np
import multiprocessing
from multiprocessing import Process, Pool
import time

def test_func(x):
x = x+1
print x
time.sleep(3)
print 'end'
return x

if __name__ == '__main__':
test_size = 20
number = np.zeros(test_size).tolist()
for i in range(len(number)):
number[i] = i

pool = Pool(processes=multiprocessing.cpu_count())

# for i in range(test_size):
# pool.apply_async(test_func, (number[i]))
resultList = pool.map(test_func,number)
pool.close()
pool.join()

print

 

黄世宇/Shiyu Huang's Personal Page：​​https://huangshiyu13.github.io/​​