Python multiprocess vs multithread

1. python GIL (Global Interprinter Lock)

   • 참고
   • 하나의 파이썬 인터프리터에서는 하나의 작업만 실행한다
   • 
   • GIL 회피 방법
     • multiprocess 를 사용한다 == 하나의 interpreter 를 실행하여 여럿의 프로세서를 구동한다
     • 구현 단에서, I/O bound 한 부분과 CPU bound한 부분을 나누는 것
     • GIL이 없는 Jython 이나 IronPython을 이용

2. python multiprocess example

   • multiprocessing 패키지 소개

   • 샘플 테스트 코드

     from multiprocessing import Pool
     from datetime import datetime
     import random
     import time
     
     read_file = open("/Users/mr.sense/Dev/python/test/readfile.txt", 'r') // read 10000 lines
     write_file = open("/Users/mr.sense/Dev/python/test/writefile_multiprocess.txt", 'a')
     
     lines = read_file.readlines()
     read_list = []
     return_list = []
     
     for line in lines:
       line = line.replace('\n','')
       read_list.append(line)
     read_file.close()
     
     # sleep method
     def append_file(msg):
       sleep_time = random.randrange(1,10)
       time.sleep(sleep_time)
       write_file.write(msg+', sleep: '+str(sleep_time)+'\n')
       return msg
     
     # initialize pool
     pool = Pool(processes=1000) # start 1000 worker processes
     
     start = datetime.now()
     print 'start: '+str(start)
     
     # get result process
     for i in pool.imap_unordered(append_file, read_list):
       return_list.append(i)
     
     end = datetime.now()
     print 'end: ' + str(end)
     
     # get time
     print end - start
     

3. python multithread

   • concurrent 패키지 설치 및 활용

   • 샘플코드

     from datetime import datetime
     import concurrent.futures
     import random
     import time
     
     read_file = open("/Users/mr.sense/Dev/python/test/readfile.txt", 'r') // read 10000 lines
     write_file = open("/Users/mr.sense/Dev/python/test/writefile_multithread.txt", 'a')
     
     lines = read_file.readlines()
     read_list = []
     return_list = []
     
     for line in lines:
       line = line.replace('\n','')
       read_list.append(line)
     read_file.close()
     
     def append_file(msg):
       sleep_time = random.randrange(1,10)
       time.sleep(sleep_time)
       write_file.write(msg+', sleep: '+str(sleep_time)+'\n')
       return msg
     
     start = datetime.now()
     print 'start: '+str(start)
     
     with concurrent.futures.ThreadPoolExecutor(max_workers=1000) as exe:
       write_result = {exe.submit(append_file, line): line for line in read_list}
       for future in concurrent.futures.as_completed(write_result):
           line = write_result[future]
           try:
               data = future.result()
           except Exception as exp:
               print("%r generated an exception: %s" % (line, exp))
     
       end = datetime.now()
       print 'end: ' + str(end)
       print end - start