Mini Shell
"""Future-returning APIs for coroutines."""
# Copyright (c) PyZMQ Developers.
# Distributed under the terms of the Modified BSD License.
from collections import namedtuple, deque
from itertools import chain
from zmq import EVENTS, POLLOUT, POLLIN
import zmq as _zmq
_FutureEvent = namedtuple('_FutureEvent', ('future', 'kind', 'kwargs', 'msg'))
# These are incomplete classes and need a Mixin for compatibility with an eventloop
# defining the followig attributes:
#
# _Future
# _READ
# _WRITE
# _default_loop()
class _AsyncPoller(_zmq.Poller):
"""Poller that returns a Future on poll, instead of blocking."""
def poll(self, timeout=-1):
"""Return a Future for a poll event"""
future = self._Future()
if timeout == 0:
try:
result = super(_AsyncPoller, self).poll(0)
except Exception as e:
future.set_exception(e)
else:
future.set_result(result)
return future
loop = self._default_loop()
# register Future to be called as soon as any event is available on any socket
watcher = self._Future()
# watch raw sockets:
raw_sockets = []
def wake_raw(*args):
if not watcher.done():
watcher.set_result(None)
watcher.add_done_callback(lambda f: self._unwatch_raw_sockets(loop, *raw_sockets))
for socket, mask in self.sockets:
if isinstance(socket, _zmq.Socket):
if not isinstance(socket, self._socket_class):
# it's a blocking zmq.Socket, wrap it in async
socket = self._socket_class.from_socket(socket)
if mask & _zmq.POLLIN:
socket._add_recv_event('poll', future=watcher)
if mask & _zmq.POLLOUT:
socket._add_send_event('poll', future=watcher)
else:
raw_sockets.append(socket)
evt = 0
if mask & _zmq.POLLIN:
evt |= self._READ
if mask & _zmq.POLLOUT:
evt |= self._WRITE
self._watch_raw_socket(loop, socket, evt, wake_raw)
def on_poll_ready(f):
if future.done():
return
if watcher.cancelled():
try:
future.cancel()
except RuntimeError:
# RuntimeError may be called during teardown
pass
return
if watcher.exception():
future.set_exception(watcher.exception())
else:
try:
result = super(_AsyncPoller, self).poll(0)
except Exception as e:
future.set_exception(e)
else:
future.set_result(result)
watcher.add_done_callback(on_poll_ready)
if timeout is not None and timeout > 0:
# schedule cancel to fire on poll timeout, if any
def trigger_timeout():
if not watcher.done():
watcher.set_result(None)
timeout_handle = loop.call_later(
1e-3 * timeout,
trigger_timeout
)
def cancel_timeout(f):
if hasattr(timeout_handle, 'cancel'):
timeout_handle.cancel()
else:
loop.remove_timeout(timeout_handle)
future.add_done_callback(cancel_timeout)
def cancel_watcher(f):
if not watcher.done():
watcher.cancel()
future.add_done_callback(cancel_watcher)
return future
class _AsyncSocket(_zmq.Socket):
# Warning : these class variables are only here to allow to call super().__setattr__.
# They be overridden at instance initialization and not shared in the whole class
_recv_futures = None
_send_futures = None
_state = 0
_shadow_sock = None
_poller_class = _AsyncPoller
io_loop = None
_fd = None
def __init__(self, context=None, socket_type=-1, io_loop=None, **kwargs):
if isinstance(context, _zmq.Socket):
context, from_socket = (None, context)
else:
from_socket = kwargs.pop('_from_socket', None)
if from_socket is not None:
super(_AsyncSocket, self).__init__(shadow=from_socket.underlying)
self._shadow_sock = from_socket
else:
super(_AsyncSocket, self).__init__(context, socket_type, **kwargs)
self._shadow_sock = _zmq.Socket.shadow(self.underlying)
self.io_loop = io_loop or self._default_loop()
self._recv_futures = deque()
self._send_futures = deque()
self._state = 0
self._fd = self._shadow_sock.FD
self._init_io_state()
@classmethod
def from_socket(cls, socket, io_loop=None):
"""Create an async socket from an existing Socket"""
return cls(_from_socket=socket, io_loop=io_loop)
def close(self, linger=None):
if not self.closed:
for event in list(chain(self._recv_futures, self._send_futures)):
if not event.future.done():
try:
event.future.cancel()
except RuntimeError:
# RuntimeError may be called during teardown
pass
self._clear_io_state()
super(_AsyncSocket, self).close(linger=linger)
close.__doc__ = _zmq.Socket.close.__doc__
def get(self, key):
result = super(_AsyncSocket, self).get(key)
if key == EVENTS:
self._schedule_remaining_events(result)
return result
get.__doc__ = _zmq.Socket.get.__doc__
def recv_multipart(self, flags=0, copy=True, track=False):
"""Receive a complete multipart zmq message.
Returns a Future whose result will be a multipart message.
"""
return self._add_recv_event('recv_multipart',
dict(flags=flags, copy=copy, track=track)
)
def recv(self, flags=0, copy=True, track=False):
"""Receive a single zmq frame.
Returns a Future, whose result will be the received frame.
Recommend using recv_multipart instead.
"""
return self._add_recv_event('recv',
dict(flags=flags, copy=copy, track=track)
)
def send_multipart(self, msg, flags=0, copy=True, track=False, **kwargs):
"""Send a complete multipart zmq message.
Returns a Future that resolves when sending is complete.
"""
kwargs['flags'] = flags
kwargs['copy'] = copy
kwargs['track'] = track
return self._add_send_event('send_multipart', msg=msg, kwargs=kwargs)
def send(self, msg, flags=0, copy=True, track=False, **kwargs):
"""Send a single zmq frame.
Returns a Future that resolves when sending is complete.
Recommend using send_multipart instead.
"""
kwargs['flags'] = flags
kwargs['copy'] = copy
kwargs['track'] = track
kwargs.update(dict(flags=flags, copy=copy, track=track))
return self._add_send_event('send', msg=msg, kwargs=kwargs)
def _deserialize(self, recvd, load):
"""Deserialize with Futures"""
f = self._Future()
def _chain(_):
"""Chain result through serialization to recvd"""
if f.done():
return
if recvd.exception():
f.set_exception(recvd.exception())
else:
buf = recvd.result()
try:
loaded = load(buf)
except Exception as e:
f.set_exception(e)
else:
f.set_result(loaded)
recvd.add_done_callback(_chain)
def _chain_cancel(_):
"""Chain cancellation from f to recvd"""
if recvd.done():
return
if f.cancelled():
recvd.cancel()
f.add_done_callback(_chain_cancel)
return f
def poll(self, timeout=None, flags=_zmq.POLLIN):
"""poll the socket for events
returns a Future for the poll results.
"""
if self.closed:
raise _zmq.ZMQError(_zmq.ENOTSUP)
p = self._poller_class()
p.register(self, flags)
f = p.poll(timeout)
future = self._Future()
def unwrap_result(f):
if future.done():
return
if f.cancelled():
try:
future.cancel()
except RuntimeError:
# RuntimeError may be called during teardown
pass
return
if f.exception():
future.set_exception(f.exception())
else:
evts = dict(f.result())
future.set_result(evts.get(self, 0))
if f.done():
# hook up result if
unwrap_result(f)
else:
f.add_done_callback(unwrap_result)
return future
def _add_timeout(self, future, timeout):
"""Add a timeout for a send or recv Future"""
def future_timeout():
if future.done():
# future already resolved, do nothing
return
# raise EAGAIN
future.set_exception(_zmq.Again())
self._call_later(timeout, future_timeout)
def _call_later(self, delay, callback):
"""Schedule a function to be called later
Override for different IOLoop implementations
Tornado and asyncio happen to both have ioloop.call_later
with the same signature.
"""
self.io_loop.call_later(delay, callback)
@staticmethod
def _remove_finished_future(future, event_list):
"""Make sure that futures are removed from the event list when they resolve
Avoids delaying cleanup until the next send/recv event,
which may never come.
"""
for f_idx, (f, kind, kwargs, _) in enumerate(event_list):
if f is future:
break
else:
return
# "future" instance is shared between sockets, but each socket has its own event list.
event_list.remove(event_list[f_idx])
def _add_recv_event(self, kind, kwargs=None, future=None):
"""Add a recv event, returning the corresponding Future"""
f = future or self._Future()
if kind.startswith('recv') and kwargs.get('flags', 0) & _zmq.DONTWAIT:
# short-circuit non-blocking calls
recv = getattr(self._shadow_sock, kind)
try:
r = recv(**kwargs)
except Exception as e:
f.set_exception(e)
else:
f.set_result(r)
return f
# we add it to the list of futures before we add the timeout as the
# timeout will remove the future from recv_futures to avoid leaks
self._recv_futures.append(
_FutureEvent(f, kind, kwargs, msg=None)
)
# Don't let the Future sit in _recv_events after it's done
f.add_done_callback(lambda f: self._remove_finished_future(f, self._recv_futures))
if hasattr(_zmq, 'RCVTIMEO'):
timeout_ms = self._shadow_sock.rcvtimeo
if timeout_ms >= 0:
self._add_timeout(f, timeout_ms * 1e-3)
if self._shadow_sock.get(EVENTS) & POLLIN:
# recv immediately, if we can
self._handle_recv()
if self._recv_futures:
self._add_io_state(POLLIN)
return f
def _add_send_event(self, kind, msg=None, kwargs=None, future=None):
"""Add a send event, returning the corresponding Future"""
f = future or self._Future()
# attempt send with DONTWAIT if no futures are waiting
# short-circuit for sends that will resolve immediately
# only call if no send Futures are waiting
if (
kind in ('send', 'send_multipart')
and not self._send_futures
):
flags = kwargs.get('flags', 0)
nowait_kwargs = kwargs.copy()
nowait_kwargs['flags'] = flags | _zmq.DONTWAIT
# short-circuit non-blocking calls
send = getattr(self._shadow_sock, kind)
# track if the send resolved or not
# (EAGAIN if DONTWAIT is not set should proceed with)
finish_early = True
try:
r = send(msg, **nowait_kwargs)
except _zmq.Again as e:
if flags & _zmq.DONTWAIT:
f.set_exception(e)
else:
# EAGAIN raised and DONTWAIT not requested,
# proceed with async send
finish_early = False
except Exception as e:
f.set_exception(e)
else:
f.set_result(r)
if finish_early:
# short-circuit resolved, return finished Future
# schedule wake for recv if there are any receivers waiting
if self._recv_futures:
self._schedule_remaining_events()
return f
# we add it to the list of futures before we add the timeout as the
# timeout will remove the future from recv_futures to avoid leaks
self._send_futures.append(
_FutureEvent(f, kind, kwargs=kwargs, msg=msg)
)
# Don't let the Future sit in _send_futures after it's done
f.add_done_callback(lambda f: self._remove_finished_future(f, self._send_futures))
if hasattr(_zmq, 'SNDTIMEO'):
timeout_ms = self._shadow_sock.get(_zmq.SNDTIMEO)
if timeout_ms >= 0:
self._add_timeout(f, timeout_ms * 1e-3)
self._add_io_state(POLLOUT)
return f
def _handle_recv(self):
"""Handle recv events"""
if not self._shadow_sock.get(EVENTS) & POLLIN:
# event triggered, but state may have been changed between trigger and callback
return
f = None
while self._recv_futures:
f, kind, kwargs, _ = self._recv_futures.popleft()
# skip any cancelled futures
if f.done():
f = None
else:
break
if not self._recv_futures:
self._drop_io_state(POLLIN)
if f is None:
return
if kind == 'poll':
# on poll event, just signal ready, nothing else.
f.set_result(None)
return
elif kind == 'recv_multipart':
recv = self._shadow_sock.recv_multipart
elif kind == 'recv':
recv = self._shadow_sock.recv
else:
raise ValueError("Unhandled recv event type: %r" % kind)
kwargs['flags'] |= _zmq.DONTWAIT
try:
result = recv(**kwargs)
except Exception as e:
f.set_exception(e)
else:
f.set_result(result)
def _handle_send(self):
if not self._shadow_sock.get(EVENTS) & POLLOUT:
# event triggered, but state may have been changed between trigger and callback
return
f = None
while self._send_futures:
f, kind, kwargs, msg = self._send_futures.popleft()
# skip any cancelled futures
if f.done():
f = None
else:
break
if not self._send_futures:
self._drop_io_state(POLLOUT)
if f is None:
return
if kind == 'poll':
# on poll event, just signal ready, nothing else.
f.set_result(None)
return
elif kind == 'send_multipart':
send = self._shadow_sock.send_multipart
elif kind == 'send':
send = self._shadow_sock.send
else:
raise ValueError("Unhandled send event type: %r" % kind)
kwargs['flags'] |= _zmq.DONTWAIT
try:
result = send(msg, **kwargs)
except Exception as e:
f.set_exception(e)
else:
f.set_result(result)
# event masking from ZMQStream
def _handle_events(self, fd=0, events=0):
"""Dispatch IO events to _handle_recv, etc."""
zmq_events = self._shadow_sock.get(EVENTS)
if zmq_events & _zmq.POLLIN:
self._handle_recv()
if zmq_events & _zmq.POLLOUT:
self._handle_send()
self._schedule_remaining_events()
def _schedule_remaining_events(self, events=None):
"""Schedule a call to handle_events next loop iteration
If there are still events to handle.
"""
# edge-triggered handling
# allow passing events in, in case this is triggered by retrieving events,
# so we don't have to retrieve it twice.
if self._state == 0:
# not watching for anything, nothing to schedule
return
if events is None:
events = self._shadow_sock.get(EVENTS)
if events & self._state:
self._call_later(0, self._handle_events)
def _add_io_state(self, state):
"""Add io_state to poller."""
if self._state != state:
state = self._state = self._state | state
self._update_handler(self._state)
def _drop_io_state(self, state):
"""Stop poller from watching an io_state."""
if self._state & state:
self._state = self._state & (~state)
self._update_handler(self._state)
def _update_handler(self, state):
"""Update IOLoop handler with state.
zmq FD is always read-only.
"""
self._schedule_remaining_events()
def _init_io_state(self):
"""initialize the ioloop event handler"""
self.io_loop.add_handler(self._shadow_sock, self._handle_events, self._READ)
self._call_later(0, self._handle_events)
def _clear_io_state(self):
"""unregister the ioloop event handler
called once during close
"""
fd = self._shadow_sock
if self._shadow_sock.closed:
fd = self._fd
self.io_loop.remove_handler(fd)
Zerion Mini Shell 1.0