From d412ea125107cde07e8ee45781be4b964e927268 Mon Sep 17 00:00:00 2001 From: Stefan Israelsson Tampe Date: Mon, 30 Jul 2018 19:16:58 +0200 Subject: selectors python file imported from main python 3.7 --- modules/language/python/compile.scm | 15 +- modules/language/python/exceptions.scm | 20 +- modules/language/python/module/collections.scm | 2 +- modules/language/python/module/math.scm | 4 +- modules/language/python/module/select.scm | 4 +- modules/language/python/module/selectors.py | 453 +++++++++++++++++++++++++ modules/language/python/module/sys.scm | 7 +- 7 files changed, 486 insertions(+), 19 deletions(-) create mode 100644 modules/language/python/module/selectors.py (limited to 'modules/language') diff --git a/modules/language/python/compile.scm b/modules/language/python/compile.scm index 5d7796b..378138c 100644 --- a/modules/language/python/compile.scm +++ b/modules/language/python/compile.scm @@ -910,10 +910,13 @@ #:select ,(map (lambda (x) (match x ((a . #f) - (exp vs a)) + (let ((s (exp vs a))) + s)) + ((a . b) - (cons (exp vs a) (exp vs b))))) - l)))) + (let ((s (exp vs a))) + (cons s (exp vs b)))))) + l)))) ((_ (#:name ((ids ...) . as)) ...) @@ -2081,7 +2084,7 @@ (define mod (current-module)) (if (module-defined? mod '__all__) (module-export! mod - (for ((x : (module-ref mod '__all__))) ((l '())) - (cons (string->symbol (scm-str x)) l) - #:final l)))) + (for ((x : (module-ref mod '__all__))) ((l '())) + (cons (string->symbol (scm-str x)) l) + #:final l)))) diff --git a/modules/language/python/exceptions.scm b/modules/language/python/exceptions.scm index 0e16e4b..804ee76 100644 --- a/modules/language/python/exceptions.scm +++ b/modules/language/python/exceptions.scm @@ -8,7 +8,8 @@ OSError ProcessLookupError PermissionError None NotImplemented NotImplementedError RunTimeError AssertionError ImportError - ModuleNotFoundError BlockingIOError)) + ModuleNotFoundError BlockingIOError + InterruptedError BaseException)) (define-syntax-rule (aif it p x y) (let ((it p)) (if it x y))) @@ -35,14 +36,22 @@ ((_ nm w k) (define-python-class nm w)))) -(define StopIteration 'StopIteration) -(define GeneratorExit 'GeneratorExit) +(define-syntax define-er2 + (syntax-rules () + ((_ nm k) + (define-python-class nm (BaseException))) + ((_ nm w k) + (define-python-class nm w)))) + +(define StopIteration 'StopIteration) +(define GeneratorExit 'GeneratorExit) +(define-er BaseException 'BaseException) (define-er SystemException 'SystemException) (define-er RuntimeError 'RuntimeError) (define-er IndexError 'IndexError) (define-er ArgumentError 'IndexError) (define-er ValueError 'ValueError) -(define None 'None) +(define None 'None) (define-er KeyError 'KeyError) (define-er TypeError 'TypeError) (define-er AttributeError 'AttributeError) @@ -52,10 +61,11 @@ (define-er PermissionError 'PermissionError) (define-er NotImplementedError 'NotImplementedError) (define-er RunTimeError 'RunTimeError) -(define AssertionError 'AssertionError) +(define AssertionError 'AssertionError) (define-er ImportError 'ImportError) (define-er ModuleNotFoundError (ImportError) 'ModuleNotFoundError) (define-er BlockingIOError 'BlockingIOError) +(define-er InterruptedError 'OSError) (define NotImplemented (list 'NotImplemented)) diff --git a/modules/language/python/module/collections.scm b/modules/language/python/module/collections.scm index 7c2b89f..f8b0b45 100644 --- a/modules/language/python/module/collections.scm +++ b/modules/language/python/module/collections.scm @@ -659,7 +659,7 @@ ,@(map (lambda (key) `(set self ',key ,key)) field_names))) mod))) - + (pylist-set! dict '__getitem__ (object-method (lambda (self i) diff --git a/modules/language/python/module/math.scm b/modules/language/python/module/math.scm index ef137aa..0dfae92 100644 --- a/modules/language/python/module/math.scm +++ b/modules/language/python/module/math.scm @@ -88,7 +88,7 @@ (define trunc py-trunc) ;; Power and logarithms -(define (exp x) (real! 'exp ((@ (guile) exp)) x)) +(define (exp x) (real! 'exp ((@ (guile) exp) x))) (define expm1 (let ((f (pointer->procedure double @@ -96,7 +96,7 @@ (list double)))) (lambda (x) (f x)))) -(define (log x) (real! 'log ((@ (guile) log)) x)) +(define (log x) (real! 'log ((@ (guile) log) x))) (define log1p (let ((f (pointer->procedure double diff --git a/modules/language/python/module/select.scm b/modules/language/python/module/select.scm index 396b6c8..df9c0c4 100644 --- a/modules/language/python/module/select.scm +++ b/modules/language/python/module/select.scm @@ -9,7 +9,7 @@ #:use-module (system foreign) #:use-module (rnrs bytevectors) - #:export (select poll epoll devpoll kqueue kevent error PIPE_BUF + #:export (select poll epoll error PIPE_BUF EPOLLIN EPOLLOUT EPOLLPRI EPOLLERR EPOLLHUP EPOLLET EPOLLONESHOT EPOLLWAKEUP EPOLLEXCLUSIVE EPOLLRDHUP EPOLLRDNORM EPOLLRDBAND EPOLLWRNORM EPOLLWRBAND EPOLLMSG @@ -52,8 +52,6 @@ (define kevent (lambda x (error "kevent not supported"))) -(define select) - (define EPOLL_CTL_ADD 1) (define EPOLL_CTL_MOD 3) (define EPOLL_CTL_DEL 2) diff --git a/modules/language/python/module/selectors.py b/modules/language/python/module/selectors.py new file mode 100644 index 0000000..5e8f3e2 --- /dev/null +++ b/modules/language/python/module/selectors.py @@ -0,0 +1,453 @@ +module(selectors) + +"""Selectors module. + +This module allows high-level and efficient I/O multiplexing, built upon the +`select` module primitives. +""" + + +from abc import ABCMeta, abstractmethod +from collections import namedtuple, Mapping +import math +import select as selectraw +import sys + +# generic events, that must be mapped to implementation-specific ones +EVENT_READ = (1 << 0) +EVENT_WRITE = (1 << 1) + +def _fileobj_to_fd(fileobj): + """Return a file descriptor from a file object. + + Parameters: + fileobj -- file object or file descriptor + + Returns: + corresponding file descriptor + + Raises: + ValueError if the object is invalid + """ + if isinstance(fileobj, int): + fd = fileobj + else: + try: + fd = int(fileobj.fileno()) + except (AttributeError, TypeError, ValueError): + raise ValueError("Invalid file object: " + "{!r}".format(fileobj)) from None + if fd < 0: + raise ValueError("Invalid file descriptor: {}".format(fd)) + return fd + + +SelectorKey = namedtuple('SelectorKey', ['fileobj', 'fd', 'events', 'data']) + + +class _SelectorMapping(Mapping): + """Mapping of file objects to selector keys.""" + + def __init__(self, selector): + self._selector = selector + + def __len__(self): + return len(self._selector._fd_to_key) + + def __getitem__(self, fileobj): + try: + fd = self._selector._fileobj_lookup(fileobj) + return self._selector._fd_to_key[fd] + except KeyError: + raise KeyError("{!r} is not registered".format(fileobj)) from None + + def __iter__(self): + return iter(self._selector._fd_to_key) + +class BaseSelector(metaclass=ABCMeta): + """Selector abstract base class. + + A selector supports registering file objects to be monitored for specific + I/O events. + + A file object is a file descriptor or any object with a `fileno()` method. + An arbitrary object can be attached to the file object, which can be used + for example to store context information, a callback, etc. + + A selector can use various implementations (select(), poll(), epoll()...) + depending on the platform. The default `Selector` class uses the most + efficient implementation on the current platform. + """ + + @abstractmethod + def register(self, fileobj, events, data=None): + """Register a file object. + + Parameters: + fileobj -- file object or file descriptor + events -- events to monitor (bitwise mask of EVENT_READ|EVENT_WRITE) + data -- attached data + + Returns: + SelectorKey instance + + Raises: + ValueError if events is invalid + KeyError if fileobj is already registered + OSError if fileobj is closed or otherwise is unacceptable to + the underlying system call (if a system call is made) + + Note: + OSError may or may not be raised + """ + raise NotImplementedError + + @abstractmethod + def unregister(self, fileobj): + """Unregister a file object. + + Parameters: + fileobj -- file object or file descriptor + + Returns: + SelectorKey instance + + Raises: + KeyError if fileobj is not registered + + Note: + If fileobj is registered but has since been closed this does + *not* raise OSError (even if the wrapped syscall does) + """ + raise NotImplementedError + + def modify(self, fileobj, events, data=None): + """Change a registered file object monitored events or attached data. + + Parameters: + fileobj -- file object or file descriptor + events -- events to monitor (bitwise mask of EVENT_READ|EVENT_WRITE) + data -- attached data + + Returns: + SelectorKey instance + + Raises: + Anything that unregister() or register() raises + """ + self.unregister(fileobj) + return self.register(fileobj, events, data) + + @abstractmethod + def select(self, timeout=None): + """Perform the actual selection, until some monitored file objects are + ready or a timeout expires. + + Parameters: + timeout -- if timeout > 0, this specifies the maximum wait time, in + seconds + if timeout <= 0, the select() call won't block, and will + report the currently ready file objects + if timeout is None, select() will block until a monitored + file object becomes ready + + Returns: + list of (key, events) for ready file objects + `events` is a bitwise mask of EVENT_READ|EVENT_WRITE + """ + raise NotImplementedError + + def close(self): + """Close the selector. + + This must be called to make sure that any underlying resource is freed. + """ + pass + + def get_key(self, fileobj): + """Return the key associated to a registered file object. + + Returns: + SelectorKey for this file object + """ + mapping = self.get_map() + if mapping is None: + raise RuntimeError('Selector is closed') + try: + return mapping[fileobj] + except KeyError: + raise KeyError("{!r} is not registered".format(fileobj)) from None + + @abstractmethod + def get_map(self): + """Return a mapping of file objects to selector keys.""" + raise NotImplementedError + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + +class _BaseSelectorImpl(BaseSelector): + """Base selector implementation.""" + + def __init__(self): + # this maps file descriptors to keys + self._fd_to_key = {} + # read-only mapping returned by get_map() + self._map = _SelectorMapping(self) + + def _fileobj_lookup(self, fileobj): + """Return a file descriptor from a file object. + + This wraps _fileobj_to_fd() to do an exhaustive search in case + the object is invalid but we still have it in our map. This + is used by unregister() so we can unregister an object that + was previously registered even if it is closed. It is also + used by _SelectorMapping. + """ + try: + return _fileobj_to_fd(fileobj) + except ValueError: + # Do an exhaustive search. + for key in self._fd_to_key.values(): + if key.fileobj is fileobj: + return key.fd + # Raise ValueError after all. + raise + + def register(self, fileobj, events, data=None): + if (not events) or (events & ~(EVENT_READ | EVENT_WRITE)): + raise ValueError("Invalid events: {!r}".format(events)) + + key = SelectorKey(fileobj, self._fileobj_lookup(fileobj), events, data) + + if key.fd in self._fd_to_key: + raise KeyError("{!r} (FD {}) is already registered" + .format(fileobj, key.fd)) + + self._fd_to_key[key.fd] = key + return key + + def unregister(self, fileobj): + try: + key = self._fd_to_key.pop(self._fileobj_lookup(fileobj)) + except KeyError: + raise KeyError("{!r} is not registered".format(fileobj)) from None + return key + + def modify(self, fileobj, events, data=None): + # TODO: Subclasses can probably optimize this even further. + try: + key = self._fd_to_key[self._fileobj_lookup(fileobj)] + except KeyError: + raise KeyError("{!r} is not registered".format(fileobj)) from None + if events != key.events: + self.unregister(fileobj) + key = self.register(fileobj, events, data) + elif data != key.data: + # Use a shortcut to update the data. + key = key._replace(data=data) + self._fd_to_key[key.fd] = key + return key + + def close(self): + self._fd_to_key.clear() + self._map = None + + def get_map(self): + return self._map + + def _key_from_fd(self, fd): + """Return the key associated to a given file descriptor. + + Parameters: + fd -- file descriptor + + Returns: + corresponding key, or None if not found + """ + try: + return self._fd_to_key[fd] + except KeyError: + return None + + +class SelectSelector(_BaseSelectorImpl): + """Select-based selector.""" + + def __init__(self): + super().__init__() + self._readers = set() + self._writers = set() + + def register(self, fileobj, events, data=None): + key = super().register(fileobj, events, data) + if events & EVENT_READ: + self._readers.add(key.fd) + if events & EVENT_WRITE: + self._writers.add(key.fd) + return key + + def unregister(self, fileobj): + key = super().unregister(fileobj) + self._readers.discard(key.fd) + self._writers.discard(key.fd) + return key + + if sys.platform == 'win32': + def _select(self, r, w, _, timeout=None): + r, w, x = selectraw.select(r, w, w, timeout) + return r, w + x, [] + else: + _select = selectraw.select + + def select(self, timeout=None): + timeout = None if timeout is None else max(timeout, 0) + ready = [] + try: + r, w, _ = self._select(self._readers, self._writers, [], timeout) + except InterruptedError: + return ready + r = set(r) + w = set(w) + for fd in r | w: + events = 0 + if fd in r: + events |= EVENT_READ + if fd in w: + events |= EVENT_WRITE + + key = self._key_from_fd(fd) + if key: + ready.append((key, events & key.events)) + return ready + +class PollSelector(_BaseSelectorImpl): + """Poll-based selector.""" + + def __init__(self): + super().__init__() + self._poll = selectraw.poll() + + def register(self, fileobj, events, data=None): + key = super().register(fileobj, events, data) + poll_events = 0 + if events & EVENT_READ: + poll_events |= selectraw.POLLIN + if events & EVENT_WRITE: + poll_events |= selectraw.POLLOUT + self._poll.register(key.fd, poll_events) + return key + + def unregister(self, fileobj): + key = super().unregister(fileobj) + self._poll.unregister(key.fd) + return key + + def select(self, timeout=None): + if timeout is None: + timeout = None + elif timeout <= 0: + timeout = 0 + else: + # poll() has a resolution of 1 millisecond, round away from + # zero to wait *at least* timeout seconds. + timeout = math.ceil(timeout * 1e3) + ready = [] + try: + fd_event_list = self._poll.poll(timeout) + except InterruptedError: + return ready + for fd, event in fd_event_list: + events = 0 + if event & ~selectraw.POLLIN: + events |= EVENT_WRITE + if event & ~selectraw.POLLOUT: + events |= EVENT_READ + + key = self._key_from_fd(fd) + if key: + ready.append((key, events & key.events)) + return ready + + +class EpollSelector(_BaseSelectorImpl): + """Epoll-based selector.""" + + def __init__(self): + super().__init__() + self._epoll = selectraw.epoll() + + def fileno(self): + return self._epoll.fileno() + + def register(self, fileobj, events, data=None): + key = super().register(fileobj, events, data) + epoll_events = 0 + if events & EVENT_READ: + epoll_events |= selectraw.EPOLLIN + if events & EVENT_WRITE: + epoll_events |= selectraw.EPOLLOUT + try: + self._epoll.register(key.fd, epoll_events) + except BaseException: + super().unregister(fileobj) + raise + return key + + def unregister(self, fileobj): + key = super().unregister(fileobj) + try: + self._epoll.unregister(key.fd) + except OSError: + # This can happen if the FD was closed since it + # was registered. + pass + return key + + def select(self, timeout=None): + if timeout is None: + timeout = -1 + elif timeout <= 0: + timeout = 0 + else: + # epoll_wait() has a resolution of 1 millisecond, round away + # from zero to wait *at least* timeout seconds. + timeout = math.ceil(timeout * 1e3) * 1e-3 + + # epoll_wait() expects `maxevents` to be greater than zero; + # we want to make sure that `select()` can be called when no + # FD is registered. + max_ev = max(len(self._fd_to_key), 1) + + ready = [] + try: + fd_event_list = self._epoll.poll(timeout, max_ev) + except InterruptedError: + return ready + for fd, event in fd_event_list: + events = 0 + if event & ~selectraw.EPOLLIN: + events |= EVENT_WRITE + if event & ~selectraw.EPOLLOUT: + events |= EVENT_READ + + key = self._key_from_fd(fd) + if key: + ready.append((key, events & key.events)) + return ready + + def close(self): + self._epoll.close() + super().close() + +DefaultSelector = EpollSelector + +__all__ = [ 'BaseSelector' , + 'DefaultSelector' , + 'EpollSelector' , + 'PollSelector' , + 'SelectSelector' ] + diff --git a/modules/language/python/module/sys.scm b/modules/language/python/module/sys.scm index 0df5bb1..1c7d784 100644 --- a/modules/language/python/module/sys.scm +++ b/modules/language/python/module/sys.scm @@ -6,7 +6,9 @@ #:export (argv byteorder copyright implementation stdin stdout stderr __stdin__ __stdout__ __stderr__ - exit)) + exit version_info version api_version + warnoptions winver _xoption + tarcebacklimit platform)) (define-syntax stdin (lambda (x) @@ -64,6 +66,7 @@ (when (not (eq? value None)) (write stdout (repr value)) (set! _ value)))) + (define dont_write_bytecode #f) (define excepthook (lambda (type value traceback) @@ -181,7 +184,7 @@ (define version "0.0.0") (define api_version "0.0.0") -(define version_info '()) +(define version_info '(3 7)) (define warnoptions #f) (define winver 0) (define _xoptions (make-hash-table)) -- cgit v1.2.3