/usr/lib64/python3.9/multiprocessing/__pycache__
Edit: /usr/lib64/python3.9/multiprocessing/__pycache__/pool.cpython-39.pyc (25138B)
a
i+ @ s` d dgZ ddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl
m Z ddl
m Z m
Z
ddlmZ dZd Zd
Zd Ze Zd d
Zdd ZG dd deZG dd dZdd ZG dd deZ d*ddZ d d Z G d d d e Z G d d d e!Z"G d!d" d"e!Z#e#Z$G d#d$ d$e#Z%G d%d& d&e!Z&G d'd( d(e&Z'G d)d de"Z(dS )+Pool
ThreadPool N )util) get_context TimeoutError)waitINITRUNCLOSE TERMINATEc C s t t| S N)listmapargs r ,/usr/lib64/python3.9/multiprocessing/pool.pymapstar/ s r c C s t t| d | d S )Nr r )r itertoolsstarmapr r r r starmapstar2 s r c @ s e Zd Zdd Zdd ZdS )RemoteTracebackc C s
|| _ d S r
tb)selfr r r r __init__: s zRemoteTraceback.__init__c C s | j S r
r r r r r __str__< s zRemoteTraceback.__str__N)__name__
__module__ __qualname__r r r r r r r 9 s r c @ s e Zd Zdd Zdd ZdS )ExceptionWithTracebackc C s0 t t|||}d|}|| _d| | _d S )N z
"""
%s""") tracebackformat_exceptiontypejoinexcr )r r( r r r r r @ s
z ExceptionWithTraceback.__init__c C s t | j| jffS r
) rebuild_excr( r r r r r
__reduce__E s z!ExceptionWithTraceback.__reduce__N)r r r! r r* r r r r r" ? s r" c C s t || _| S r
)r __cause__)r( r r r r r) H s
r) c s0 e Zd ZdZ fddZdd Zdd Z ZS )MaybeEncodingErrorzVWraps possible unpickleable errors, so they can be
safely sent through the socket.c s. t || _t || _tt| | j| j d S r
)reprr( valuesuperr, r )r r( r. __class__r r r T s
zMaybeEncodingError.__init__c C s d| j | jf S )Nz(Error sending result: '%s'. Reason: '%s')r. r( r r r r r Y s zMaybeEncodingError.__str__c C s d| j j| f S )Nz<%s: %s>)r1 r r r r r __repr__] s zMaybeEncodingError.__repr__)r r r! __doc__r r r2
__classcell__r r r0 r r, P s r, r Fc
C s |d ur(t |tr|dks(td||j}| j}t| drR| j |j |d urb|| d}|d u s~|r||k rz
| } W n& t
t fy t
d Y qY n0 | d u rt
d q| \}
} } }
}zd| |
i |f}W nJ ty4 } z0|r| turt||j}d|f}W Y d }~n
d }~0 0 z||
| |f W nT ty } z:t||d }t
d | ||
| d|ff W Y d }~n
d }~0 0 d } }
} } }
}|d7 }qft
d
| d S ) Nr zMaxtasks {!r} is not valid_writerr z)worker got EOFError or OSError -- exitingz worker got sentinel -- exitingTFz0Possible encoding error while sending result: %sz worker exiting after %d tasks)
isinstanceintAssertionErrorformatputgethasattrr5 close_readerEOFErrorOSErrorr debug Exception_helper_reraises_exceptionr"
__traceback__r, )inqueueoutqueue initializerinitargsZmaxtaskswrap_exceptionr: r; Z completedtaskjobifuncr kwdsresultewrappedr r r workera sN
(
rR c C s | dS )z@Pickle-able helper function for use by _guarded_task_generation.Nr )exr r r rC s rC c s2 e Zd ZdZdd fdd
Z fddZ ZS )
_PoolCachez
Class that implements a cache for the Pool class that will notify
the pool management threads every time the cache is emptied. The
notification is done by the use of a queue that is provided when
instantiating the cache.
Nnotifierc s || _ t j|i | d S r
)rV r/ r )r rV r rN r0 r r r s z_PoolCache.__init__c s t | | s | jd d S r
)r/ __delitem__rV r: )r itemr0 r r rW s z_PoolCache.__delitem__)r r r! r3 r rW r4 r r r0 r rT s rT c @ s e Zd ZdZdZedd ZdLddZej e
fd d
Z d d Z d
d Z
edd Zedd Zdd Zedd Zedd Zdd Zdd Zdi fd d ZdMd d ZdNd!d"ZdOd#d$Zd%d& ZdPd(d)ZdQd*d+Zdi ddfd,d-Z dRd.d/Z dSd0d1Z edTd2d3Z e d4d5 Z!ed6d7 Z"ed8d9 Z#ed:d; Z$d
d? Z&d@dA Z'dBdC Z(edDdE Z)e dFdG Z*dHdI Z+dJdK Z,dS )Ur zS
Class which supports an async version of applying functions to arguments.
Tc O s | j |i |S r
Process)ctxr rN r r r rZ s z Pool.ProcessNr c C s g | _ t| _|pt | _| t | _| j | _ t
| j d| _ || _ || _
|| _|d u rjt phd}|dk rztd|d urt|std|| _z | W nF ty | j D ]}|jd u r| q| j D ] }| qւ Y n0 | }tj t j | j | j| j| j | j| j | j | j!| j
| j| j | j"|| j fd| _#d| j#_$t%| j#_| j#& tj t j'| j| j(| j!| j | j fd| _)d| j)_$t%| j)_| j)& tj t j*| j!| j+| j fd| _,d| j,_$t%| j,_| j,& t-j.| | j/| j| j | j!| j | j | j#| j)| j,| j f dd| _0t%| _d S ) NrU r z&Number of processes must be at least 1z initializer must be a callabletargetr T )r exitpriority)1_poolr _stater _ctx
_setup_queuesqueue SimpleQueue
_taskqueue_change_notifierrT _cache_maxtasksperchild _initializer _initargsos cpu_count
ValueErrorcallable TypeError
_processes_repopulate_poolrB exitcode terminater' _get_sentinels threadingThreadr _handle_workersrZ _inqueue _outqueue_wrap_exception_worker_handlerdaemonr
start
_handle_tasks
_quick_put
_task_handler_handle_results
_quick_get_result_handlerr Finalize_terminate_pool
_terminate)r processesrG rH maxtasksperchildcontextp sentinelsr r r r s~
z
Pool.__init__c C s> | j |kr:|d| t| d t| dd d ur:| jd d S )Nz&unclosed running multiprocessing pool )sourcerg )ra ResourceWarninggetattrrg r: )r _warnr
r r r __del__ s
z Pool.__del__c C s0 | j }d|j d|j d| j dt| j d S )N<.z state=z pool_size=>)r1 r r! ra lenr` )r clsr r r r2 s z
Pool.__repr__c C s | j jg}| jjg}g ||S r
)rz r> rg )r Ztask_queue_sentinelsZself_notifier_sentinelsr r r ru s
zPool._get_sentinelsc C s dd | D S )Nc S s g | ]}t |d r|jqS )sentinel)r< r ).0rR r r r
s
z.Pool._get_worker_sentinels..r workersr r r _get_worker_sentinels s zPool._get_worker_sentinelsc C sP d}t tt| D ]6}| | }|jdurtd| | d}| |= q|S )zCleanup after any worker processes which have exited due to reaching
their specified lifetime. Returns True if any workers were cleaned up.
FNcleaning up worker %dT)reversedranger rs r rA r' )poolZcleanedrL rR r r r _join_exited_workers s
zPool._join_exited_workersc C s0 | | j| j| j| j| j| j| j| j| j | j
S r
) _repopulate_pool_staticrb rZ rq r` ry rz rj rk ri r{ r r r r rr . s zPool._repopulate_poolc
C sf t |t| D ]P}
|| t|||||| fd} | jdd| _d| _| || t d qdS )zBring the number of pool processes up to the specified number,
for use after reaping workers which have exited.
r\ rZ Z
PoolWorkerTz added workerN)
r r rR namereplacer} r~ appendr rA ) r[ rZ r r rE rF rG rH r rI rL wr r r r 7 s
z Pool._repopulate_pool_staticc
C s* t |r&t | |||||||||
dS )zEClean up any exited workers and start replacements for them.
N)r r r )
r[ rZ r r rE rF rG rH r rI r r r _maintain_poolJ s
zPool._maintain_poolc C s4 | j | _| j | _| jjj| _| jjj| _ d S r
)
rb re ry rz r5 sendr r> recvr r r r r rc V s zPool._setup_queuesc C s | j tkrtdd S )NzPool not running)ra r
rn r r r r _check_running\ s
zPool._check_runningc C s | ||| S )zT
Equivalent of `func(*args, **kwds)`.
Pool must be running.
) apply_asyncr; )r rM r rN r r r apply` s z
Pool.applyc C s | ||t| S )zx
Apply `func` to each element in `iterable`, collecting the results
in a list that is returned.
)
_map_asyncr r; r rM iterable chunksizer r r r g s zPool.mapc C s | ||t| S )z
Like `map()` method but the elements of the `iterable` are expected to
be iterables as well and will be unpacked as arguments. Hence
`func` and (a, b) becomes func(a, b).
)r r r; r r r r r n s z Pool.starmapc C s | ||t|||S )z=
Asynchronous version of `starmap()` method.
)r r r rM r r callbackerror_callbackr r r
starmap_asyncv s zPool.starmap_asyncc
c sl z,d}t |D ]\}}||||fi fV qW n: tyf } z"||d t|fi fV W Y d}~n
d}~0 0 dS )zProvides a generator of tasks for imap and imap_unordered with
appropriate handling for iterables which throw exceptions during
iteration.r N) enumeraterB rC )r Z
result_jobrM r rL xrP r r r _guarded_task_generation~ s z Pool._guarded_task_generationr c C s | |dkr:t| }| j| |j|||jf |S |dk rPtd|t
|||}t| }| j| |jt ||jf dd |D S dS )zP
Equivalent of `map()` -- can be MUCH slower than `Pool.map()`.
r z Chunksize must be 1+, not {0:n}c s s | ]}|D ]
}|V q
qd S r
r r chunkrX r r r z Pool.imap..N) r IMapIteratorrf r: r _job _set_lengthrn r9 r
_get_tasksr r rM r r rO task_batchesr r r imap s4 z Pool.imapc C s | |dkr:t| }| j| |j|||jf |S |dk rPtd|t
|||}t| }| j| |jt ||jf dd |D S dS )zL
Like `imap()` method but ordering of results is arbitrary.
r z Chunksize must be 1+, not {0!r}c s s | ]}|D ]
}|V q
qd S r
r r r r r r r z&Pool.imap_unordered..N) r IMapUnorderedIteratorrf r: r r r rn r9 r r r r r r r imap_unordered s0 zPool.imap_unorderedc C s6 | t| ||}| j|jd|||fgdf |S )z;
Asynchronous version of `apply()` method.
r N)r ApplyResultrf r: r )r rM r rN r r rO r r r r s zPool.apply_asyncc C s | ||t|||S )z9
Asynchronous version of `map()` method.
)r r r r r r map_async s zPool.map_asyncc
C s | t|dst|}|du rJtt|t| jd \}}|rJ|d7 }t|dkrZd}t|||}t| |t|||d} | j
| | j ||df | S )zY
Helper function to implement map, starmap and their async counterparts.
__len__N r r r )
r r< r divmodr r` r r MapResultrf r: r r )
r rM r Zmapperr r r extrar rO r r r r s,
zPool._map_asyncc C s" t | |d | s | q d S )N)timeout)r emptyr; )r change_notifierr r r r _wait_for_updates s zPool._wait_for_updatesc C st t }|jtks |r\|jtkr\| ||||||| |
| |
g | ||
}| || q|d t
d d S )Nzworker handler exiting) rv current_threadra r
r r r r r: r rA )r cache taskqueuer[ rZ r r rE rF rG rH r rI r r threadZcurrent_sentinelsr r r rx s
zPool._handle_workersc
C s t }t| jd D ]
\}}d }z|D ]}|jtkrDtd qz || W q( ty } zH|d d \}
} z||
| d| f W n t
y Y n0 W Y d } ~ q(d } ~ 0 0 q(|rtd |r|d nd} || d W d } }}
qW d } }}
q,W d } }}
qd } }}
0 qtd z6td | d td |D ]} |d qPW n t y td
Y n0 td d S ) Nz'task handler found thread._state != RUN Fzdoing set_length()r r ztask handler got sentinelz/task handler sending sentinel to result handlerz(task handler sending sentinel to workersz/task handler got OSError when sending sentinelsztask handler exiting)
rv r iterr; ra r
r rA rB _setKeyErrorr: r@ )
r r: rF r r r ZtaskseqZ
set_lengthrJ rP rK idxr r r r r sJ
zPool._handle_tasksc C s t }z
| }W n" ttfy4 td Y d S 0 |jtkr^|jtksRJ dtd q|d u rrtd q|\}}}z|| || W n t
y Y n0 d } }}q|rJ|jtkrJz
| }W n" ttfy td Y d S 0 |d u rtd q|\}}}z|| || W n t
y: Y n0 d } }}qt | drtd z,t dD ] }| j
s q| qjW n ttfy Y n0 td t||j d S )
Nz.result handler got EOFError/OSError -- exitingzThread not in TERMINATEz,result handler found thread._state=TERMINATEzresult handler got sentinelz&result handler ignoring extra sentinelr> z"ensuring that outqueue is not full
z7result handler exiting: len(cache)=%s, thread._state=%s)rv r r@ r? r rA ra r
r r r r< r r> pollr )rF r; r r rJ rK rL objr r r r : s\
zPool._handle_resultsc c s0 t |}tt||}|s d S | |fV qd S r
)r tupler islice)rM itsizer r r r r v s
zPool._get_tasksc C s t dd S )Nz:pool objects cannot be passed between processes or pickled)NotImplementedErrorr r r r r* s zPool.__reduce__c C s2 t d | jtkr.t| _t| j_| jd d S )Nz closing pool)r rA ra r
r r| rg r: r r r r r= s
z
Pool.closec C s t d t| _| d S )Nzterminating pool)r rA r ra r r r r r rt s
zPool.terminatec C sj t d | jtkr tdn| jttfvr4td| j | j | j
| j D ] }| qXd S )Nz joining poolzPool is still runningzIn unknown state) r rA ra r
rn r r r| r' r r r` )r r r r r r' s
z Pool.joinc C s@ t d | j | r<| j r<| j t d qd S )Nz7removing tasks from inqueue until task handler finishedr )
r rA _rlockacquireis_aliver> r r timesleep)rE task_handlerr r r r _help_stuff_finish s
zPool._help_stuff_finishc
C sX t d t|_|d t|_t d | ||t| | sXt| dkrXtdt|_|d |d t d t
|ur| |rt |d drt d |D ]}
|
j
d u r|
qt d t
|ur| t d t
|ur| |rTt |d drTt d
|D ](}
|
r*t d |
j |
q*d S ) Nzfinalizing poolz&helping task handler/workers to finishr z.Cannot have cache with result_hander not alivezjoining worker handlerrt zterminating workerszjoining task handlerzjoining result handlerzjoining pool workersr )r rA r ra r: r r r r8 rv r r' r< rs rt pid) r r rE rF r r Zworker_handlerr Zresult_handlerr r r r r r sB
zPool._terminate_poolc C s | | S r
)r r r r r __enter__ s zPool.__enter__c C s | d S r
)rt )r exc_typeexc_valexc_tbr r r __exit__ s z
Pool.__exit__)NNr NN)N)N)NNN)r )r )NNN)NNN)N)-r r r! r3 r{ staticmethodrZ r warningswarnr
r r2 ru r r rr r r rc r r r r r r r r r r r r classmethodrx r r r r* r= rt r' r r r r r r r r r sv
P
-
;
5c @ sJ e Zd Zdd Zdd Zdd Zddd Zdd
d Zd d
Ze e
j Z dS )r c C s> || _ t | _tt| _|j| _|| _|| _ | | j| j< d S r
)
r` rv Event_eventnext job_counterr rh _callback_error_callback)r r r r r r r r s
zApplyResult.__init__c C s
| j S r
)r is_setr r r r ready s zApplyResult.readyc C s | std| | jS )Nz{0!r} not ready)r rn r9 _successr r r r
successful s zApplyResult.successfulNc C s | j | d S r
)r r r r r r r r s zApplyResult.waitc C s, | | | st| jr"| jS | jd S r
)r r r r _valuer r r r r; s
zApplyResult.getc C sZ |\| _ | _| jr$| j r$| | j | jr<| j s<| | j | j | j| j= d | _d S r
) r r r r r setrh r r` r rL r r r r r s
zApplyResult._set)N)N)
r r r! r r r r r; r r types GenericAlias__class_getitem__r r r r r s
r c @ s e Zd Zdd Zdd ZdS )r c C sh t j| |||d d| _d g| | _|| _|dkrNd| _| j | j| j = n|| t
|| | _d S )Nr Tr ) r r r r
_chunksize _number_leftr r rh r bool)r r r lengthr r r r r r s
zMapResult.__init__c C s | j d8 _ |\}}|rv| jrv|| j|| j |d | j < | j dkr| jrZ| | j | j| j= | j d | _ nL|s| jrd| _|| _| j dkr| j
r|
| j | j| j= | j d | _ d S )Nr r F) r r r r r rh r r r r` r )r rL Zsuccess_resultsuccessrO r r r r & s&
zMapResult._setN)r r r! r r r r r r r s
r c @ s: e Zd Zdd Zdd Zd ddZeZdd Zd
d ZdS )
r c C sT || _ tt | _tt| _|j| _t
| _ d| _ d | _
i | _| | j| j< d S )Nr )r` rv ConditionLock_condr r r rh collectionsdeque_items_index_length _unsorted)r r r r r r D s
zIMapIterator.__init__c C s | S r
r r r r r __iter__O s zIMapIterator.__iter__Nc C s | j z| j }W nv ty | j| jkr r r r r ru s zThreadPool._get_sentinelsc C s g S r
r r r r r r s z ThreadPool._get_worker_sentinelsc C sD z| j dd qW n tjy& Y n0 t|D ]}| d q0d S )NF)block)r; rd Emptyr r: )rE r r rL r r r r s z ThreadPool._help_stuff_finishc C s t | d S r
)r r )r r r r r r r r s z ThreadPool._wait_for_updates)NNr ) r r r! r{ r rZ r rc ru r r r r r r r r s
)Nr NF))__all__r
r rl rd rv r r$ r r r# r r r
connectionr r r
r r countr r r rB r r" r) r, rR rC dictrT objectr r Z AsyncResultr r r r r r r r
sN
- =++E