shutil.py

10 files changed, 1430 insertions, 50 deletions

diff --git a/modules/language/python/compile.scm b/modules/language/python/compile.scm
index 354b39d..91c33c1 100644
--- a/modules/language/python/compile.scm
+++ b/modules/language/python/compile.scm
@@ -51,7 +51,7 @@
     (call-with-prompt exit-prompt
       (lambda () code ...)
       (lambda (k val)
-        (if (not (= val 0))
+        (if (not (equal? val 0))
             (format #t "exit with error ~a~%" val))))))
 
 (define (get-exported-symbols x)
@@ -76,20 +76,106 @@
       r)))
 
 
+(define-syntax use-modules--
+  (lambda (x)
+    (define (keyword-like? stx)
+      (let ((dat (syntax->datum stx)))
+        (and (symbol? dat)
+             (eqv? (string-ref (symbol->string dat) 0) #\:))))
+    (define (->keyword sym)
+      (symbol->keyword (string->symbol (substring (symbol->string sym) 1))))
+    
+    (define (quotify-iface args)
+      (let loop ((in args) (out '()))
+        (syntax-case in ()
+          (() (reverse! out))
+          ;; The user wanted #:foo, but wrote :foo. Fix it.
+          ((sym . in) (keyword-like? #'sym)
+           (loop #`(#,(->keyword (syntax->datum #'sym)) . in) out))
+          ((kw . in) (not (keyword? (syntax->datum #'kw)))
+           (syntax-violation 'define-module "expected keyword arg" x #'kw))
+          ((#:renamer renamer . in)
+           (loop #'in (cons* #'renamer #:renamer out)))
+          ((kw val . in)
+           (loop #'in (cons* #''val #'kw out))))))
+
+    (define (quotify specs)
+      (let lp ((in specs) (out '()))
+        (syntax-case in ()
+          (() (reverse out))
+          (((name name* ...) . in)
+           (and-map symbol? (syntax->datum #'(name name* ...)))
+           (lp #'in (cons #''((name name* ...)) out)))
+          ((((name name* ...) arg ...) . in)
+           (and-map symbol? (syntax->datum #'(name name* ...)))
+           (with-syntax (((quoted-arg ...) (quotify-iface #'(arg ...))))
+             (lp #'in (cons #`(list '(name name* ...) quoted-arg ...)
+                            out)))))))
+    
+    (syntax-case x ()
+      ((_ spec ...)
+       (with-syntax (((quoted-args ...) (quotify #'(spec ...))))
+         #'(eval-when (expand)
+             (process-use-modules (list quoted-args ...))
+             *unspecified*))))))
+
+(define-syntax use-modules-
+  (lambda (x)
+    (define (keyword-like? stx)
+      (let ((dat (syntax->datum stx)))
+        (and (symbol? dat)
+             (eqv? (string-ref (symbol->string dat) 0) #\:))))
+    (define (->keyword sym)
+      (symbol->keyword (string->symbol (substring (symbol->string sym) 1))))
+    
+    (define (quotify-iface args)
+      (let loop ((in args) (out '()))
+        (syntax-case in ()
+          (() (reverse! out))
+          ;; The user wanted #:foo, but wrote :foo. Fix it.
+          ((sym . in) (keyword-like? #'sym)
+           (loop #`(#,(->keyword (syntax->datum #'sym)) . in) out))
+          ((kw . in) (not (keyword? (syntax->datum #'kw)))
+           (syntax-violation 'define-module "expected keyword arg" x #'kw))
+          ((#:renamer renamer . in)
+           (loop #'in (cons* #'renamer #:renamer out)))
+          ((kw val . in)
+           (loop #'in (cons* #''val #'kw out))))))
+
+    (define (quotify specs)
+      (let lp ((in specs) (out '()))
+        (syntax-case in ()
+          (() (reverse out))
+          (((name name* ...) . in)
+           (and-map symbol? (syntax->datum #'(name name* ...)))
+           (lp #'in (cons #''((name name* ...)) out)))
+          ((((name name* ...) arg ...) . in)
+           (and-map symbol? (syntax->datum #'(name name* ...)))
+           (with-syntax (((quoted-arg ...) (quotify-iface #'(arg ...))))
+             (lp #'in (cons #`(list '(name name* ...) quoted-arg ...)
+                            out)))))))
+    
+    (syntax-case x ()
+      ((_ spec ...)
+       (with-syntax (((quoted-args ...) (quotify #'(spec ...))))
+         #'(eval-when (eval load)
+             (process-use-modules (list quoted-args ...))
+             *unspecified*))))))
+
 (define-syntax-rule (use p l a ...)
   (begin
-    (eval-when (compile)
+    (eval-when (expand)
       (catch #t
         (lambda ()
           (if (not p) (reload-module (resolve-module 'l)))
-          (use-modules a ...))
+          (use-modules-- a ...))
         (lambda x
           #f)))
     (eval-when (eval load)
       (catch #t
         (lambda ()
           (if (not p) (reload-module (resolve-module 'l)))
-          (use-modules a ...))
+          (use-modules- a ...))
         (lambda x
           (raise (ImportError ((@ (guile) format)
                                #f "failed to import ~a ~a" 'l x))))))))
@@ -684,6 +770,14 @@
 
 (define-syntax-rule (setwrap u)
   (call-with-values (lambda () u)
+    (lambda (x . x*)
+      (if (null? x*)
+          x
+          (cons x x*)))))
+
+#;
+(define-syntax-rule (setwrap u)
+  (call-with-values (lambda () u)
     (case-lambda
       ((x) x)
       (x x))))
@@ -1269,7 +1363,7 @@
   
   ((_ es in code . else)
    (let lp ((es es))
-     (match es
+     (match es       
        (((#:power #f (#:tuple . l) . _))
         (lp l))
        (_            
@@ -1278,7 +1372,10 @@
                (code2 (exp vs2 code))
                (p     (is-ec #t code2 #t (list (C 'continue))))
                (else2 (if else (exp vs2 else) #f))
-               (in2   (map (g vs exp) in)))
+               (in2   (match in
+                        ((in)     (list (exp vs in)))
+                        ((in ...) (list `(,(G 'list)
+                                          ,@ (map (g vs exp) in)))))))
           (list (C 'cfor) es2 in2 code2 else2 p)))))))
 
  (#:sub
@@ -1336,7 +1433,7 @@
  (#:try
   ((_ x (or #f ()) #f . fin)
    (if fin
-       `(,(T 'try) (lambda () ,(exp vs x)) #:finally (lambda () fin))
+       `(,(T 'try) (lambda () ,(exp vs x)) #:finally (lambda () ,(exp vs fin)))
        `(,(T 'try) (lambda () ,(exp vs x)))))
     
   ((_ x exc else . fin)
@@ -1699,14 +1796,14 @@
   ((_  k (and e (#:cfor . _)))
    (let ((dict (gensym "dict")))
      `(,(G 'let) ((,dict (,(Se 'set))))
-        ,(gen-sel vs e `((,(O 'ref) ,dict 'add) ,(exp vs k)))
+        ,(gen-sel vs e `((,(O 'ref) ,dict (,(G 'quote) add)) ,(exp vs k)))
         ,dict)))
   
   ((_  k ...)
    (let ((set (gensym "dict")))
      `(,(G 'let) ((,set (,(Se 'set))))
         ,@(map (lambda (k)
-                 `((,(O 'ref) ,set 'add) ,(exp vs k)))
+                 `((,(O 'ref) ,set (,(G 'quote) add)) ,(exp vs k)))
                k)
         ,set))))
                   
diff --git a/modules/language/python/def.scm b/modules/language/python/def.scm
index 7622b7a..b2a8de2 100644
--- a/modules/language/python/def.scm
+++ b/modules/language/python/def.scm
@@ -77,7 +77,7 @@
 
     (define (->kw x) (symbol->keyword (syntax->datum x)))
     
-    (syntax-case x ()
+    (syntax-case x (*)
       ((_ (arg ...) code ...)
        (let* ((as  (fold get-as '() #'(arg ...)))
               (kw  (fold get-kw '() #'(arg ...)))
@@ -143,7 +143,13 @@
 
 (define-syntax py-apply  
   (lambda (x)
-    (syntax-case x ()
+    (syntax-case x (*)
+      ((_ f a ... (* x))
+       (and-map (lambda (x) (symbol? (syntax->datum x))) #'(a ...))
+       #'(if (pair? x)
+             (apply f a ... x)
+             (apply f a ... (to-list x))))
+
       ((_ f a ...)
        (if (no #'(a ...))
 	   #'(f a ...)
diff --git a/modules/language/python/module/_python.scm b/modules/language/python/module/_python.scm
index 149946e..b8c5118 100644
--- a/modules/language/python/module/_python.scm
+++ b/modules/language/python/module/_python.scm
@@ -249,7 +249,22 @@
 
 (define (idx x) x)
 
-(def (py-min (* l) (= key idx) (= default miss))
+
+(define-syntax min
+  (lambda (x)
+    (syntax-case x ()
+      ((_ x y)
+       #'(if (and (number? x) (number? y))
+             (if (<= x y) x y)
+             (py-min- x y)))
+
+      ((_ l ...)
+       #'(if (and (number? l) ...)
+             ((@ (guile) min) l ...)
+             (py-min- l ...)))
+      (_ #'py-min-))))
+
+(def (py-min- (* l) (= key idx) (= default miss))
      (let lp ((l l))
        (match l
          ((it)
@@ -269,7 +284,20 @@
                    b)))
          (_ (lp ((@ (guile) list) l))))))
 
-(def (py-max (* l) (= key idx) (= default miss))
+(define-syntax max
+  (lambda (x)
+    (syntax-case x ()
+      ((_ x y)
+       #'(if (and (number? x) (number? y))
+             (if (>= x y) x y)
+             (py-max- x y)))
+      ((_ l ...)
+       #'(if (and (number? l) ...)
+             ((@ (guile) max) l ...)
+             (py-max- l ...)))
+      (_ #'py-max-))))
+
+(def (py-max- (* l) (= key idx) (= default miss))
      (let lp ((l l))
        (match l
          ((it)
@@ -303,8 +331,6 @@
 
 (define-syntax-rule (super . l) (py-super-mac . l))       
 
-(define min  py-min)
-(define max  py-max)
 (define list pylist)
 (define reversed py-reversed)
 (define (key-id x) x)
diff --git a/modules/language/python/module/os.scm b/modules/language/python/module/os.scm
index dab8dda..e4dc72c 100644
--- a/modules/language/python/module/os.scm
+++ b/modules/language/python/module/os.scm
@@ -105,7 +105,7 @@
                   sched_getparam sched_rr_get_intervall sched_yield
                   sched_setaffinity sched_getaffinity
 
-                  supports_dir_fs support_effective_ids supports_fd
+                  supports_dir_fd support_effective_ids supports_fd
 
                   confstr confstr_names cpu_count sysconf sysconf_names
                   getloadavg
@@ -116,7 +116,7 @@
                   getrandom urandom GRND_NONBLOCK GRND_RANDOM
                   ))
 
-(define supports_dir_fs
+(define supports_dir_fd
   (py-set '()))
 
 (define support_effective_ids
@@ -151,7 +151,7 @@
      (values))
     ((_ (0 . l) f)
      (begin
-       (py-add supports_dir_fs (symbol->string 'f))
+       (py-add supports_dir_fd (symbol->string 'f))
        (reg l f)))
     ((_ (1 . l) f)
      (begin
@@ -2362,3 +2362,4 @@
        (set! l (cons (symbol->string k) l)))
      p)
     (py-list l)))
+
diff --git a/modules/language/python/module/pwd.scm b/modules/language/python/module/pwd.scm
index 3922ac9..952d4c8 100644
--- a/modules/language/python/module/pwd.scm
+++ b/modules/language/python/module/pwd.scm
@@ -5,7 +5,7 @@
   #:use-module (language python yield)
   #:use-module (language python try)
   #:replace (getpwuid)
-  #:export (getpwname getpwall))
+  #:export (getpwnam getpwall))
 
 (define-python-class PWD (Sequence)
   (define __init__
@@ -42,7 +42,7 @@
 (define (getpwuid uid)
   (ca (apply PWD (vector->list ((@ (guile) getpwuid) uid)))))
                   
-(define (getpwname nm)
+(define (getpwnam nm)
   (ca (apply PWD (vector->list ((@ (guile) getpwnam) nm)))))
 
 (define (getpwall)
diff --git a/modules/language/python/module/random.py b/modules/language/python/module/random.py
index 31783cc..4c24977 100644
--- a/modules/language/python/module/random.py
+++ b/modules/language/python/module/random.py
@@ -69,7 +69,7 @@ RECIP_BPF = 2**-BPF
 
 import _random
 
-class Random(_random.Random):
+class Random (_random.Random):
     """Random number generator base class used by bound module functions.
 
     Used to instantiate instances of Random to get generators that don't
@@ -703,9 +703,9 @@ class SystemRandom(Random):
 
 ## -------------------- test program --------------------
 
-import time
 
 def _test_generator(n, func, args):
+    import time    
     print(n, 'times', func.__name__)
     total = 0.0
     sqsum = 0.0
@@ -727,9 +727,33 @@ def _test_generator(n, func, args):
     print('avg %g, stddev %g, min %g, max %g\n' % \
               (avg, stddev, smallest, largest))
 
+def _test_generator0(n, func):
+    import time    
+    print(n, 'times', func.__name__)
+    total = 0.0
+    sqsum = 0.0
+    smallest = 1e10
+    largest = -1e10
+    t0 = time.time()
+    for i in range(n):
+        x = func()
+        total += x
+        sqsum = sqsum + x*x
+        smallest = min(x, smallest)
+        largest = max(x, largest)
+
+    t1 = time.time()
+    print(round(t1-t0, 3), 'sec,', end=' ')
+    avg = total/n
+    stddev = _sqrt(sqsum/n - avg*avg)
+
+    print('avg %g, stddev %g, min %g, max %g\n' % \
+              (avg, stddev, smallest, largest))
+
 
 def _test(N=2000):
     _test_generator(N, random, ())
+    _test_generator0(N, random)
     _test_generator(N, normalvariate, (0.0, 1.0))
     _test_generator(N, lognormvariate, (0.0, 1.0))
     _test_generator(N, vonmisesvariate, (0.0, 1.0))
diff --git a/modules/language/python/module/shutil.py b/modules/language/python/module/shutil.py
new file mode 100644
index 0000000..c281a37
--- /dev/null
+++ b/modules/language/python/module/shutil.py
@@ -0,0 +1,1153 @@
+module(shutil)
+"""Utility functions for copying and archiving files and directory trees.
+
+XXX The functions here don't copy the resource fork or other metadata on Mac.
+
+"""
+
+import os
+import sys
+import stat
+import fnmatch
+import collections
+import errno
+
+try:
+    import zlib
+    del zlib
+    _ZLIB_SUPPORTED = True
+except ImportError:
+    _ZLIB_SUPPORTED = False
+
+try:
+    import bz2
+    del bz2
+    _BZ2_SUPPORTED = True
+except ImportError:
+    _BZ2_SUPPORTED = False
+
+try:
+    import lzma
+    del lzma
+    _LZMA_SUPPORTED = True
+except ImportError:
+    _LZMA_SUPPORTED = False
+
+try:
+    from pwd import getpwnam as getpwnam_
+    getpwnam = getpwnam_
+except ImportError:
+    getpwnam = None
+
+try:
+    from grp import getgrnam as getgrnam_
+    getgrnam = getgrnam_
+except ImportError:
+    getgrnam = None
+
+__all__ = ["copyfileobj", "copyfile", "copymode", "copystat", "copy", "copy2",
+           "copytree", "move", "rmtree", "Error", "SpecialFileError",
+           "ExecError", "make_archive", "get_archive_formats",
+           "register_archive_format", "unregister_archive_format",
+           "get_unpack_formats", "register_unpack_format",
+           "unregister_unpack_format", "unpack_archive",
+           "ignore_patterns", "chown", "which", "get_terminal_size",
+           "SameFileError"]
+           # disk_usage is added later, if available on the platform
+
+class Error(OSError):
+    pass
+
+class SameFileError(Error):
+    """Raised when source and destination are the same file."""
+
+class SpecialFileError(OSError):
+    """Raised when trying to do a kind of operation (e.g. copying) which is
+    not supported on a special file (e.g. a named pipe)"""
+
+class ExecError(OSError):
+    """Raised when a command could not be executed"""
+
+class ReadError(OSError):
+    """Raised when an archive cannot be read"""
+
+class RegistryError(Exception):
+    """Raised when a registry operation with the archiving
+    and unpacking registries fails"""
+
+
+def copyfileobj(fsrc, fdst, length=16*1024):
+    """copy data from file-like object fsrc to file-like object fdst"""
+    while 1:
+        buf = fsrc.read(length)
+        if not buf:
+            break
+        fdst.write(buf)
+
+def _samefile(src, dst):
+    # Macintosh, Unix.
+    if hasattr(os.path, 'samefile'):
+        try:
+            return os.path.samefile(src, dst)
+        except OSError:
+            return False
+
+    # All other platforms: check for same pathname.
+    return (os.path.normcase(os.path.abspath(src)) ==
+            os.path.normcase(os.path.abspath(dst)))
+
+def copyfile(src, dst, *, follow_symlinks=True):
+    """Copy data from src to dst.
+
+    If follow_symlinks is not set and src is a symbolic link, a new
+    symlink will be created instead of copying the file it points to.
+
+    """
+    if _samefile(src, dst):
+        raise SameFileError("{!r} and {!r} are the same file".format(src, dst))
+
+    for fn in [src, dst]:
+        try:
+            st = os.stat(fn)
+        except OSError:
+            # File most likely does not exist
+            pass
+        else:
+            # XXX What about other special files? (sockets, devices...)
+            if stat.S_ISFIFO(st.st_mode):
+                raise SpecialFileError("`%s` is a named pipe" % fn)
+
+    if not follow_symlinks and os.path.islink(src):
+        os.symlink(os.readlink(src), dst)
+    else:
+        with open(src, 'rb') as fsrc:
+            with open(dst, 'wb') as fdst:
+                copyfileobj(fsrc, fdst)
+    return dst
+
+def copymode(src, dst, *, follow_symlinks=True):
+    """Copy mode bits from src to dst.
+
+    If follow_symlinks is not set, symlinks aren't followed if and only
+    if both `src` and `dst` are symlinks.  If `lchmod` isn't available
+    (e.g. Linux) this method does nothing.
+
+    """
+    if not follow_symlinks and os.path.islink(src) and os.path.islink(dst):
+        if hasattr(os, 'lchmod'):
+            stat_func, chmod_func = os.lstat, os.lchmod
+        else:
+            return
+    elif hasattr(os, 'chmod'):
+        stat_func, chmod_func = os.stat, os.chmod
+    else:
+        return
+
+    st = stat_func(src)
+    chmod_func(dst, stat.S_IMODE(st.st_mode))
+
+if hasattr(os, 'listxattr'):
+    def _copyxattr(src, dst, *, follow_symlinks=True):
+        """Copy extended filesystem attributes from `src` to `dst`.
+
+        Overwrite existing attributes.
+
+        If `follow_symlinks` is false, symlinks won't be followed.
+
+        """
+
+        try:
+            names = os.listxattr(src, follow_symlinks=follow_symlinks)
+        except OSError as e:
+            if e.errno not in (errno.ENOTSUP, errno.ENODATA):
+                raise
+            return
+        for name in names:
+            try:
+                value = os.getxattr(src, name, follow_symlinks=follow_symlinks)
+                os.setxattr(dst, name, value, follow_symlinks=follow_symlinks)
+            except OSError as e:
+                if e.errno not in (errno.EPERM, errno.ENOTSUP, errno.ENODATA):
+                    raise
+else:
+    def _copyxattr(*args, **kwargs):
+        pass
+
+def copystat(src, dst, *, follow_symlinks=True):
+    """Copy all stat info (mode bits, atime, mtime, flags) from src to dst.
+
+    If the optional flag `follow_symlinks` is not set, symlinks aren't followed if and
+    only if both `src` and `dst` are symlinks.
+
+    """
+    def _nop(*args, ns=None, follow_symlinks=None):
+        pass
+
+    # follow symlinks (aka don't not follow symlinks)
+    follow = follow_symlinks or not (os.path.islink(src) and os.path.islink(dst))
+    if follow:
+        # use the real function if it exists
+        def lookup(name):
+            return getattr(os, name, _nop)
+    else:
+        # use the real function only if it exists
+        # *and* it supports follow_symlinks
+        def lookup(name):
+            fn = getattr(os, name, _nop)
+            if fn in os.supports_follow_symlinks:
+                return fn
+            return _nop
+
+    st = lookup("stat")(src, follow_symlinks=follow)
+    mode = stat.S_IMODE(st.st_mode)
+    lookup("utime")(dst, ns=(st.st_atime_ns, st.st_mtime_ns),
+        follow_symlinks=follow)
+    try:
+        lookup("chmod")(dst, mode, follow_symlinks=follow)
+    except NotImplementedError:
+        # if we got a NotImplementedError, it's because
+        #   * follow_symlinks=False,
+        #   * lchown() is unavailable, and
+        #   * either
+        #       * fchownat() is unavailable or
+        #       * fchownat() doesn't implement AT_SYMLINK_NOFOLLOW.
+        #         (it returned ENOSUP.)
+        # therefore we're out of options--we simply cannot chown the
+        # symlink.  give up, suppress the error.
+        # (which is what shutil always did in this circumstance.)
+        pass
+    if hasattr(st, 'st_flags'):
+        try:
+            lookup("chflags")(dst, st.st_flags, follow_symlinks=follow)
+        except OSError as why:
+            for err in 'EOPNOTSUPP', 'ENOTSUP':
+                if hasattr(errno, err) and why.errno == getattr(errno, err):
+                    break
+            else:
+                raise
+    _copyxattr(src, dst, follow_symlinks=follow)
+
+def copy(src, dst, *, follow_symlinks=True):
+    """Copy data and mode bits ("cp src dst"). Return the file's destination.
+
+    The destination may be a directory.
+
+    If follow_symlinks is false, symlinks won't be followed. This
+    resembles GNU's "cp -P src dst".
+
+    If source and destination are the same file, a SameFileError will be
+    raised.
+
+    """
+    if os.path.isdir(dst):
+        dst = os.path.join(dst, os.path.basename(src))
+    copyfile(src, dst, follow_symlinks=follow_symlinks)
+    copymode(src, dst, follow_symlinks=follow_symlinks)
+    return dst
+
+def copy2(src, dst, *, follow_symlinks=True):
+    """Copy data and all stat info ("cp -p src dst"). Return the file's
+    destination."
+
+    The destination may be a directory.
+
+    If follow_symlinks is false, symlinks won't be followed. This
+    resembles GNU's "cp -P src dst".
+
+    """
+    if os.path.isdir(dst):
+        dst = os.path.join(dst, os.path.basename(src))
+    copyfile(src, dst, follow_symlinks=follow_symlinks)
+    copystat(src, dst, follow_symlinks=follow_symlinks)
+    return dst
+
+def ignore_patterns(*patterns):
+    """Function that can be used as copytree() ignore parameter.
+
+    Patterns is a sequence of glob-style patterns
+    that are used to exclude files"""
+    def _ignore_patterns(path, names):
+        ignored_names = []
+        for pattern in patterns:
+            ignored_names.extend(fnmatch.filter(names, pattern))
+        return set(ignored_names)
+    return _ignore_patterns
+
+def copytree(src, dst, symlinks=False, ignore=None, copy_function=copy2,
+             ignore_dangling_symlinks=False):
+    """Recursively copy a directory tree.
+
+    The destination directory must not already exist.
+    If exception(s) occur, an Error is raised with a list of reasons.
+
+    If the optional symlinks flag is true, symbolic links in the
+    source tree result in symbolic links in the destination tree; if
+    it is false, the contents of the files pointed to by symbolic
+    links are copied. If the file pointed by the symlink doesn't
+    exist, an exception will be added in the list of errors raised in
+    an Error exception at the end of the copy process.
+
+    You can set the optional ignore_dangling_symlinks flag to true if you
+    want to silence this exception. Notice that this has no effect on
+    platforms that don't support os.symlink.
+
+    The optional ignore argument is a callable. If given, it
+    is called with the `src` parameter, which is the directory
+    being visited by copytree(), and `names` which is the list of
+    `src` contents, as returned by os.listdir():
+
+        callable(src, names) -> ignored_names
+
+    Since copytree() is called recursively, the callable will be
+    called once for each directory that is copied. It returns a
+    list of names relative to the `src` directory that should
+    not be copied.
+
+    The optional copy_function argument is a callable that will be used
+    to copy each file. It will be called with the source path and the
+    destination path as arguments. By default, copy2() is used, but any
+    function that supports the same signature (like copy()) can be used.
+
+    """
+    names = os.listdir(src)
+    if ignore is not None:
+        ignored_names = ignore(src, names)
+    else:
+        ignored_names = set()
+
+    os.makedirs(dst)
+    errors = []
+    for name in names:
+        if name in ignored_names:
+            continue
+        srcname = os.path.join(src, name)
+        dstname = os.path.join(dst, name)
+        try:
+            if os.path.islink(srcname):
+                linkto = os.readlink(srcname)
+                if symlinks:
+                    # We can't just leave it to `copy_function` because legacy
+                    # code with a custom `copy_function` may rely on copytree
+                    # doing the right thing.
+                    os.symlink(linkto, dstname)
+                    copystat(srcname, dstname, follow_symlinks=not symlinks)
+                else:
+                    # ignore dangling symlink if the flag is on
+                    if not os.path.exists(linkto) and ignore_dangling_symlinks:
+                        continue
+                    # otherwise let the copy occurs. copy2 will raise an error
+                    if os.path.isdir(srcname):
+                        copytree(srcname, dstname, symlinks, ignore,
+                                 copy_function)
+                    else:
+                        copy_function(srcname, dstname)
+            elif os.path.isdir(srcname):
+                copytree(srcname, dstname, symlinks, ignore, copy_function)
+            else:
+                # Will raise a SpecialFileError for unsupported file types
+                copy_function(srcname, dstname)
+        # catch the Error from the recursive copytree so that we can
+        # continue with other files
+        except Error as err:
+            errors.extend(err.args[0])
+        except OSError as why:
+            errors.append((srcname, dstname, str(why)))
+    try:
+        copystat(src, dst)
+    except OSError as why:
+        # Copying file access times may fail on Windows
+        if getattr(why, 'winerror', None) is None:
+            errors.append((src, dst, str(why)))
+    if errors:
+        raise Error(errors)
+    return dst
+
+# version vulnerable to race conditions
+def _rmtree_unsafe(path, onerror):
+    try:
+        if os.path.islink(path):
+            # symlinks to directories are forbidden, see bug #1669
+            raise OSError("Cannot call rmtree on a symbolic link")
+    except OSError:
+        onerror(os.path.islink, path, sys.exc_info())
+        # can't continue even if onerror hook returns
+        return
+    names = []
+    try:
+        names = os.listdir(path)
+    except OSError:
+        onerror(os.listdir, path, sys.exc_info())
+    for name in names:
+        fullname = os.path.join(path, name)
+        try:
+            mode = os.lstat(fullname).st_mode
+        except OSError:
+            mode = 0
+        if stat.S_ISDIR(mode):
+            _rmtree_unsafe(fullname, onerror)
+        else:
+            try:
+                os.unlink(fullname)
+            except OSError:
+                onerror(os.unlink, fullname, sys.exc_info())
+    try:
+        os.rmdir(path)
+    except OSError:
+        onerror(os.rmdir, path, sys.exc_info())
+
+# Version using fd-based APIs to protect against races
+def _rmtree_safe_fd(topfd, path, onerror):
+    names = []
+    try:
+        names = os.listdir(topfd)
+    except OSError as err:
+        err.filename = path
+        onerror(os.listdir, path, sys.exc_info())
+    for name in names:
+        fullname = os.path.join(path, name)
+        try:
+            orig_st = os.stat(name, dir_fd=topfd, follow_symlinks=False)
+            mode = orig_st.st_mode
+        except OSError:
+            mode = 0
+        if stat.S_ISDIR(mode):
+            try:
+                dirfd = os.open(name, os.O_RDONLY, dir_fd=topfd)
+            except OSError:
+                onerror(os.open, fullname, sys.exc_info())
+            else:
+                try:
+                    if os.path.samestat(orig_st, os.fstat(dirfd)):
+                        _rmtree_safe_fd(dirfd, fullname, onerror)
+                        try:
+                            os.rmdir(name, dir_fd=topfd)
+                        except OSError:
+                            onerror(os.rmdir, fullname, sys.exc_info())
+                    else:
+                        try:
+                            # This can only happen if someone replaces
+                            # a directory with a symlink after the call to
+                            # stat.S_ISDIR above.
+                            raise OSError("Cannot call rmtree on a symbolic "
+                                          "link")
+                        except OSError:
+                            onerror(os.path.islink, fullname, sys.exc_info())
+                finally:
+                    os.close(dirfd)
+        else:
+            try:
+                os.unlink(name, dir_fd=topfd)
+            except OSError:
+                onerror(os.unlink, fullname, sys.exc_info())
+
+_use_fd_functions = ({os.open, os.stat, os.unlink, os.rmdir} <=
+                     os.supports_dir_fd and
+                     os.listdir in os.supports_fd and
+                     os.stat in os.supports_follow_symlinks)
+
+def rmtree(path, ignore_errors=False, onerror=None):
+    """Recursively delete a directory tree.
+
+    If ignore_errors is set, errors are ignored; otherwise, if onerror
+    is set, it is called to handle the error with arguments (func,
+    path, exc_info) where func is platform and implementation dependent;
+    path is the argument to that function that caused it to fail; and
+    exc_info is a tuple returned by sys.exc_info().  If ignore_errors
+    is false and onerror is None, an exception is raised.
+
+    """
+    if ignore_errors:
+        def onerror(*args):
+            pass
+    elif onerror is None:
+        def onerror(*args):
+            raise
+    if _use_fd_functions:
+        # While the unsafe rmtree works fine on bytes, the fd based does not.
+        if isinstance(path, bytes):
+            path = os.fsdecode(path)
+        # Note: To guard against symlink races, we use the standard
+        # lstat()/open()/fstat() trick.
+        try:
+            orig_st = os.lstat(path)
+        except Exception:
+            onerror(os.lstat, path, sys.exc_info())
+            return
+        try:
+            fd = os.open(path, os.O_RDONLY)
+        except Exception:
+            onerror(os.lstat, path, sys.exc_info())
+            return
+        try:
+            if os.path.samestat(orig_st, os.fstat(fd)):
+                _rmtree_safe_fd(fd, path, onerror)
+                try:
+                    os.rmdir(path)
+                except OSError:
+                    onerror(os.rmdir, path, sys.exc_info())
+            else:
+                try:
+                    # symlinks to directories are forbidden, see bug #1669
+                    raise OSError("Cannot call rmtree on a symbolic link")
+                except OSError:
+                    onerror(os.path.islink, path, sys.exc_info())
+        finally:
+            os.close(fd)
+    else:
+        return _rmtree_unsafe(path, onerror)
+
+# Allow introspection of whether or not the hardening against symlink
+# attacks is supported on the current platform
+rmtree.avoids_symlink_attacks = _use_fd_functions
+
+def _basename(path):
+    # A basename() variant which first strips the trailing slash, if present.
+    # Thus we always get the last component of the path, even for directories.
+    sep = os.path.sep + (os.path.altsep or '')
+    return os.path.basename(path.rstrip(sep))
+
+def move(src, dst, copy_function=copy2):
+    """Recursively move a file or directory to another location. This is
+    similar to the Unix "mv" command. Return the file or directory's
+    destination.
+
+    If the destination is a directory or a symlink to a directory, the source
+    is moved inside the directory. The destination path must not already
+    exist.
+
+    If the destination already exists but is not a directory, it may be
+    overwritten depending on os.rename() semantics.
+
+    If the destination is on our current filesystem, then rename() is used.
+    Otherwise, src is copied to the destination and then removed. Symlinks are
+    recreated under the new name if os.rename() fails because of cross
+    filesystem renames.
+
+    The optional `copy_function` argument is a callable that will be used
+    to copy the source or it will be delegated to `copytree`.
+    By default, copy2() is used, but any function that supports the same
+    signature (like copy()) can be used.
+
+    A lot more could be done here...  A look at a mv.c shows a lot of
+    the issues this implementation glosses over.
+
+    """
+    real_dst = dst
+    if os.path.isdir(dst):
+        if _samefile(src, dst):
+            # We might be on a case insensitive filesystem,
+            # perform the rename anyway.
+            os.rename(src, dst)
+            return
+
+        real_dst = os.path.join(dst, _basename(src))
+        if os.path.exists(real_dst):
+            raise Error("Destination path '%s' already exists" % real_dst)
+    try:
+        os.rename(src, real_dst)
+    except OSError:
+        if os.path.islink(src):
+            linkto = os.readlink(src)
+            os.symlink(linkto, real_dst)
+            os.unlink(src)
+        elif os.path.isdir(src):
+            if _destinsrc(src, dst):
+                raise Error("Cannot move a directory '%s' into itself"
+                            " '%s'." % (src, dst))
+            copytree(src, real_dst, copy_function=copy_function,
+                     symlinks=True)
+            rmtree(src)
+        else:
+            copy_function(src, real_dst)
+            os.unlink(src)
+    return real_dst
+
+def _destinsrc(src, dst):
+    src = os.path.abspath(src)
+    dst = os.path.abspath(dst)
+    if not src.endswith(os.path.sep):
+        src += os.path.sep
+    if not dst.endswith(os.path.sep):
+        dst += os.path.sep
+    return dst.startswith(src)
+
+def _get_gid(name):
+    """Returns a gid, given a group name."""
+    if getgrnam is None or name is None:
+        return None
+    try:
+        result = getgrnam(name)
+    except KeyError:
+        result = None
+    if result is not None:
+        return result[2]
+    return None
+
+def _get_uid(name):
+    """Returns an uid, given a user name."""
+    if getpwnam is None or name is None:
+        return None
+    try:
+        result = getpwnam(name)
+    except KeyError:
+        result = None
+    if result is not None:
+        return result[2]
+    return None
+
+def _make_tarball(base_name, base_dir, compress="gzip", verbose=0, dry_run=0,
+                  owner=None, group=None, logger=None):
+    """Create a (possibly compressed) tar file from all the files under
+    'base_dir'.
+
+    'compress' must be "gzip" (the default), "bzip2", "xz", or None.
+
+    'owner' and 'group' can be used to define an owner and a group for the
+    archive that is being built. If not provided, the current owner and group
+    will be used.
+
+    The output tar file will be named 'base_name' +  ".tar", possibly plus
+    the appropriate compression extension (".gz", ".bz2", or ".xz").
+
+    Returns the output filename.
+    """
+    if compress is None:
+        tar_compression = ''
+    elif _ZLIB_SUPPORTED and compress == 'gzip':
+        tar_compression = 'gz'
+    elif _BZ2_SUPPORTED and compress == 'bzip2':
+        tar_compression = 'bz2'
+    elif _LZMA_SUPPORTED and compress == 'xz':
+        tar_compression = 'xz'
+    else:
+        raise ValueError("bad value for 'compress', or compression format not "
+                         "supported : {0}".format(compress))
+
+    import tarfile  # late import for breaking circular dependency
+
+    compress_ext = '.' + tar_compression if compress else ''
+    archive_name = base_name + '.tar' + compress_ext
+    archive_dir = os.path.dirname(archive_name)
+
+    if archive_dir and not os.path.exists(archive_dir):
+        if logger is not None:
+            logger.info("creating %s", archive_dir)
+        if not dry_run:
+            os.makedirs(archive_dir)
+
+    # creating the tarball
+    if logger is not None:
+        logger.info('Creating tar archive')
+
+    uid = _get_uid(owner)
+    gid = _get_gid(group)
+
+    def _set_uid_gid(tarinfo):
+        if gid is not None:
+            tarinfo.gid = gid
+            tarinfo.gname = group
+        if uid is not None:
+            tarinfo.uid = uid
+            tarinfo.uname = owner
+        return tarinfo
+
+    if not dry_run:
+        tar = tarfile.open(archive_name, 'w|%s' % tar_compression)
+        try:
+            tar.add(base_dir, filter=_set_uid_gid)
+        finally:
+            tar.close()
+
+    return archive_name
+
+def _make_zipfile(base_name, base_dir, verbose=0, dry_run=0, logger=None):
+    """Create a zip file from all the files under 'base_dir'.
+
+    The output zip file will be named 'base_name' + ".zip".  Returns the
+    name of the output zip file.
+    """
+    import zipfile  # late import for breaking circular dependency
+
+    zip_filename = base_name + ".zip"
+    archive_dir = os.path.dirname(base_name)
+
+    if archive_dir and not os.path.exists(archive_dir):
+        if logger is not None:
+            logger.info("creating %s", archive_dir)
+        if not dry_run:
+            os.makedirs(archive_dir)
+
+    if logger is not None:
+        logger.info("creating '%s' and adding '%s' to it",
+                    zip_filename, base_dir)
+
+    if not dry_run:
+        with zipfile.ZipFile(zip_filename, "w",
+                             compression=zipfile.ZIP_DEFLATED) as zf:
+            path = os.path.normpath(base_dir)
+            if path != os.curdir:
+                zf.write(path, path)
+                if logger is not None:
+                    logger.info("adding '%s'", path)
+            for dirpath, dirnames, filenames in os.walk(base_dir):
+                for name in sorted(dirnames):
+                    path = os.path.normpath(os.path.join(dirpath, name))
+                    zf.write(path, path)
+                    if logger is not None:
+                        logger.info("adding '%s'", path)
+                for name in filenames:
+                    path = os.path.normpath(os.path.join(dirpath, name))
+                    if os.path.isfile(path):
+                        zf.write(path, path)
+                        if logger is not None:
+                            logger.info("adding '%s'", path)
+
+    return zip_filename
+
+_ARCHIVE_FORMATS = {
+    'tar':   (_make_tarball, [('compress', None)], "uncompressed tar file"),
+}
+
+if _ZLIB_SUPPORTED:
+    _ARCHIVE_FORMATS['gztar'] = (_make_tarball, [('compress', 'gzip')],
+                                "gzip'ed tar-file")
+    _ARCHIVE_FORMATS['zip'] = (_make_zipfile, [], "ZIP file")
+
+if _BZ2_SUPPORTED:
+    _ARCHIVE_FORMATS['bztar'] = (_make_tarball, [('compress', 'bzip2')],
+                                "bzip2'ed tar-file")
+
+if _LZMA_SUPPORTED:
+    _ARCHIVE_FORMATS['xztar'] = (_make_tarball, [('compress', 'xz')],
+                                "xz'ed tar-file")
+
+def get_archive_formats():
+    """Returns a list of supported formats for archiving and unarchiving.
+
+    Each element of the returned sequence is a tuple (name, description)
+    """
+    formats = [(name, registry[2]) for name, registry in
+               _ARCHIVE_FORMATS.items()]
+    formats.sort()
+    return formats
+
+def register_archive_format(name, function, extra_args=None, description=''):
+    """Registers an archive format.
+
+    name is the name of the format. function is the callable that will be
+    used to create archives. If provided, extra_args is a sequence of
+    (name, value) tuples that will be passed as arguments to the callable.
+    description can be provided to describe the format, and will be returned
+    by the get_archive_formats() function.
+    """
+    if extra_args is None:
+        extra_args = []
+    if not callable(function):
+        raise TypeError('The %s object is not callable' % function)
+    if not isinstance(extra_args, (tuple, list)):
+        raise TypeError('extra_args needs to be a sequence')
+    for element in extra_args:
+        if not isinstance(element, (tuple, list)) or len(element) !=2:
+            raise TypeError('extra_args elements are : (arg_name, value)')
+
+    _ARCHIVE_FORMATS[name] = (function, extra_args, description)
+
+def unregister_archive_format(name):
+    del _ARCHIVE_FORMATS[name]
+
+def make_archive(base_name, format, root_dir=None, base_dir=None, verbose=0,
+                 dry_run=0, owner=None, group=None, logger=None):
+    """Create an archive file (eg. zip or tar).
+
+    'base_name' is the name of the file to create, minus any format-specific
+    extension; 'format' is the archive format: one of "zip", "tar", "gztar",
+    "bztar", or "xztar".  Or any other registered format.
+
+    'root_dir' is a directory that will be the root directory of the
+    archive; ie. we typically chdir into 'root_dir' before creating the
+    archive.  'base_dir' is the directory where we start archiving from;
+    ie. 'base_dir' will be the common prefix of all files and
+    directories in the archive.  'root_dir' and 'base_dir' both default
+    to the current directory.  Returns the name of the archive file.
+
+    'owner' and 'group' are used when creating a tar archive. By default,
+    uses the current owner and group.
+    """
+    save_cwd = os.getcwd()
+    if root_dir is not None:
+        if logger is not None:
+            logger.debug("changing into '%s'", root_dir)
+        base_name = os.path.abspath(base_name)
+        if not dry_run:
+            os.chdir(root_dir)
+
+    if base_dir is None:
+        base_dir = os.curdir
+
+    kwargs = {'dry_run': dry_run, 'logger': logger}
+
+    try:
+        format_info = _ARCHIVE_FORMATS[format]
+    except KeyError:
+        raise ValueError("unknown archive format '%s'" % format)
+
+    func = format_info[0]
+    for arg, val in format_info[1]:
+        kwargs[arg] = val
+
+    if format != 'zip':
+        kwargs['owner'] = owner
+        kwargs['group'] = group
+
+    try:
+        filename = func(base_name, base_dir, **kwargs)
+    finally:
+        if root_dir is not None:
+            if logger is not None:
+                logger.debug("changing back to '%s'", save_cwd)
+            os.chdir(save_cwd)
+
+    return filename
+
+
+def get_unpack_formats():
+    """Returns a list of supported formats for unpacking.
+
+    Each element of the returned sequence is a tuple
+    (name, extensions, description)
+    """
+    formats = [(name, info[0], info[3]) for name, info in
+               _UNPACK_FORMATS.items()]
+    formats.sort()
+    return formats
+
+def _check_unpack_options(extensions, function, extra_args):
+    """Checks what gets registered as an unpacker."""
+    # first make sure no other unpacker is registered for this extension
+    existing_extensions = {}
+    for name, info in _UNPACK_FORMATS.items():
+        for ext in info[0]:
+            existing_extensions[ext] = name
+
+    for extension in extensions:
+        if extension in existing_extensions:
+            msg = '%s is already registered for "%s"'
+            raise RegistryError(msg % (extension,
+                                       existing_extensions[extension]))
+
+    if not callable(function):
+        raise TypeError('The registered function must be a callable')
+
+
+def register_unpack_format(name, extensions, function, extra_args=None,
+                           description=''):
+    """Registers an unpack format.
+
+    `name` is the name of the format. `extensions` is a list of extensions
+    corresponding to the format.
+
+    `function` is the callable that will be
+    used to unpack archives. The callable will receive archives to unpack.
+    If it's unable to handle an archive, it needs to raise a ReadError
+    exception.
+
+    If provided, `extra_args` is a sequence of
+    (name, value) tuples that will be passed as arguments to the callable.
+    description can be provided to describe the format, and will be returned
+    by the get_unpack_formats() function.
+    """
+    if extra_args is None:
+        extra_args = []
+    _check_unpack_options(extensions, function, extra_args)
+    _UNPACK_FORMATS[name] = extensions, function, extra_args, description
+
+def unregister_unpack_format(name):
+    """Removes the pack format from the registry."""
+    del _UNPACK_FORMATS[name]
+
+def _ensure_directory(path):
+    """Ensure that the parent directory of `path` exists"""
+    dirname = os.path.dirname(path)
+    if not os.path.isdir(dirname):
+        os.makedirs(dirname)
+
+def _unpack_zipfile(filename, extract_dir):
+    """Unpack zip `filename` to `extract_dir`
+    """
+    import zipfile  # late import for breaking circular dependency
+
+    if not zipfile.is_zipfile(filename):
+        raise ReadError("%s is not a zip file" % filename)
+
+    zip = zipfile.ZipFile(filename)
+    try:
+        for info in zip.infolist():
+            name = info.filename
+
+            # don't extract absolute paths or ones with .. in them
+            if name.startswith('/') or '..' in name:
+                continue
+
+            target = os.path.join(extract_dir, *name.split('/'))
+            if not target:
+                continue
+
+            _ensure_directory(target)
+            if not name.endswith('/'):
+                # file
+                data = zip.read(info.filename)
+                f = open(target, 'wb')
+                try:
+                    f.write(data)
+                finally:
+                    f.close()
+                    del data
+    finally:
+        zip.close()
+
+def _unpack_tarfile(filename, extract_dir):
+    """Unpack tar/tar.gz/tar.bz2/tar.xz `filename` to `extract_dir`
+    """
+    import tarfile  # late import for breaking circular dependency
+    try:
+        tarobj = tarfile.open(filename)
+    except tarfile.TarError:
+        raise ReadError(
+            "%s is not a compressed or uncompressed tar file" % filename)
+    try:
+        tarobj.extractall(extract_dir)
+    finally:
+        tarobj.close()
+
+_UNPACK_FORMATS = {
+    'tar':   (['.tar'], _unpack_tarfile, [], "uncompressed tar file"),
+    'zip':   (['.zip'], _unpack_zipfile, [], "ZIP file"),
+}
+
+if _ZLIB_SUPPORTED:
+    _UNPACK_FORMATS['gztar'] = (['.tar.gz', '.tgz'], _unpack_tarfile, [],
+                                "gzip'ed tar-file")
+
+if _BZ2_SUPPORTED:
+    _UNPACK_FORMATS['bztar'] = (['.tar.bz2', '.tbz2'], _unpack_tarfile, [],
+                                "bzip2'ed tar-file")
+
+if _LZMA_SUPPORTED:
+    _UNPACK_FORMATS['xztar'] = (['.tar.xz', '.txz'], _unpack_tarfile, [],
+                                "xz'ed tar-file")
+
+def _find_unpack_format(filename):
+    for name, info in _UNPACK_FORMATS.items():
+        for extension in info[0]:
+            if filename.endswith(extension):
+                return name
+    return None
+
+def unpack_archive(filename, extract_dir=None, format=None):
+    """Unpack an archive.
+
+    `filename` is the name of the archive.
+
+    `extract_dir` is the name of the target directory, where the archive
+    is unpacked. If not provided, the current working directory is used.
+
+    `format` is the archive format: one of "zip", "tar", "gztar", "bztar",
+    or "xztar".  Or any other registered format.  If not provided,
+    unpack_archive will use the filename extension and see if an unpacker
+    was registered for that extension.
+
+    In case none is found, a ValueError is raised.
+    """
+    if extract_dir is None:
+        extract_dir = os.getcwd()
+
+    if format is not None:
+        try:
+            format_info = _UNPACK_FORMATS[format]
+        except KeyError:
+            raise ValueError("Unknown unpack format '{0}'".format(format))
+
+        func = format_info[1]
+        func(filename, extract_dir, **dict(format_info[2]))
+    else:
+        # we need to look at the registered unpackers supported extensions
+        format = _find_unpack_format(filename)
+        if format is None:
+            raise ReadError("Unknown archive format '{0}'".format(filename))
+
+        func = _UNPACK_FORMATS[format][1]
+        kwargs = dict(_UNPACK_FORMATS[format][2])
+        func(filename, extract_dir, **kwargs)
+
+
+if hasattr(os, 'statvfs'):
+
+    __all__.append('disk_usage')
+    _ntuple_diskusage = collections.namedtuple('usage', 'total used free')
+
+    def disk_usage(path):
+        """Return disk usage statistics about the given path.
+
+        Returned value is a named tuple with attributes 'total', 'used' and
+        'free', which are the amount of total, used and free space, in bytes.
+        """
+        st = os.statvfs(path)
+        free = st.f_bavail * st.f_frsize
+        total = st.f_blocks * st.f_frsize
+        used = (st.f_blocks - st.f_bfree) * st.f_frsize
+        return _ntuple_diskusage(total, used, free)
+
+elif os.name == 'nt':
+
+    import nt
+    __all__.append('disk_usage')
+    _ntuple_diskusage = collections.namedtuple('usage', 'total used free')
+
+    def disk_usage(path):
+        """Return disk usage statistics about the given path.
+
+        Returned values is a named tuple with attributes 'total', 'used' and
+        'free', which are the amount of total, used and free space, in bytes.
+        """
+        total, free = nt._getdiskusage(path)
+        used = total - free
+        return _ntuple_diskusage(total, used, free)
+
+
+def chown(path, user=None, group=None):
+    """Change owner user and group of the given path.
+
+    user and group can be the uid/gid or the user/group names, and in that case,
+    they are converted to their respective uid/gid.
+    """
+
+    if user is None and group is None:
+        raise ValueError("user and/or group must be set")
+
+    _user = user
+    _group = group
+
+    # -1 means don't change it
+    if user is None:
+        _user = -1
+    # user can either be an int (the uid) or a string (the system username)
+    elif isinstance(user, str):
+        _user = _get_uid(user)
+        if _user is None:
+            raise LookupError("no such user: {!r}".format(user))
+
+    if group is None:
+        _group = -1
+    elif not isinstance(group, int):
+        _group = _get_gid(group)
+        if _group is None:
+            raise LookupError("no such group: {!r}".format(group))
+
+    os.chown(path, _user, _group)
+
+def get_terminal_size(fallback=(80, 24)):
+    """Get the size of the terminal window.
+
+    For each of the two dimensions, the environment variable, COLUMNS
+    and LINES respectively, is checked. If the variable is defined and
+    the value is a positive integer, it is used.
+
+    When COLUMNS or LINES is not defined, which is the common case,
+    the terminal connected to sys.__stdout__ is queried
+    by invoking os.get_terminal_size.
+
+    If the terminal size cannot be successfully queried, either because
+    the system doesn't support querying, or because we are not
+    connected to a terminal, the value given in fallback parameter
+    is used. Fallback defaults to (80, 24) which is the default
+    size used by many terminal emulators.
+
+    The value returned is a named tuple of type os.terminal_size.
+    """
+    # columns, lines are the working values
+    try:
+        columns = int(os.environ['COLUMNS'])
+    except (KeyError, ValueError):
+        columns = 0
+
+    try:
+        lines = int(os.environ['LINES'])
+    except (KeyError, ValueError):
+        lines = 0
+
+    # only query if necessary
+    if columns <= 0 or lines <= 0:
+        try:
+            size = os.get_terminal_size(sys.__stdout__.fileno())
+        except (AttributeError, ValueError, OSError):
+            # stdout is None, closed, detached, or not a terminal, or
+            # os.get_terminal_size() is unsupported
+            size = os.terminal_size(fallback)
+        if columns <= 0:
+            columns = size.columns
+        if lines <= 0:
+            lines = size.lines
+
+    return os.terminal_size((columns, lines))
+
+def which(cmd, mode=os.F_OK | os.X_OK, path=None):
+    """Given a command, mode, and a PATH string, return the path which
+    conforms to the given mode on the PATH, or None if there is no such
+    file.
+
+    `mode` defaults to os.F_OK | os.X_OK. `path` defaults to the result
+    of os.environ.get("PATH"), or can be overridden with a custom search
+    path.
+
+    """
+    # Check that a given file can be accessed with the correct mode.
+    # Additionally check that `file` is not a directory, as on Windows
+    # directories pass the os.access check.
+    def _access_check(fn, mode):
+        return (os.path.exists(fn) and os.access(fn, mode)
+                and not os.path.isdir(fn))
+
+    # If we're given a path with a directory part, look it up directly rather
+    # than referring to PATH directories. This includes checking relative to the
+    # current directory, e.g. ./script
+    if os.path.dirname(cmd):
+        if _access_check(cmd, mode):
+            return cmd
+        return None
+
+    if path is None:
+        path = os.environ.get("PATH", os.defpath)
+    if not path:
+        return None
+    path = path.split(os.pathsep)
+
+    if sys.platform == "win32":
+        # The current directory takes precedence on Windows.
+        if not os.curdir in path:
+            path.insert(0, os.curdir)
+
+        # PATHEXT is necessary to check on Windows.
+        pathext = os.environ.get("PATHEXT", "").split(os.pathsep)
+        # See if the given file matches any of the expected path extensions.
+        # This will allow us to short circuit when given "python.exe".
+        # If it does match, only test that one, otherwise we have to try
+        # others.
+        if any(cmd.lower().endswith(ext.lower()) for ext in pathext):
+            files = [cmd]
+        else:
+            files = [cmd + ext for ext in pathext]
+    else:
+        # On other platforms you don't have things like PATHEXT to tell you
+        # what file suffixes are executable, so just pass on cmd as-is.
+        files = [cmd]
+
+    seen = set()
+    for dir in path:
+        normdir = os.path.normcase(dir)
+        if not normdir in seen:
+            seen.add(normdir)
+            for thefile in files:
+                name = os.path.join(dir, thefile)
+                if _access_check(name, mode):
+                    return name
+    return None
diff --git a/modules/language/python/procedure.scm b/modules/language/python/procedure.scm
index 1289aae..9417c20 100644
--- a/modules/language/python/procedure.scm
+++ b/modules/language/python/procedure.scm
@@ -10,6 +10,10 @@
   #:use-module (language python dict)
   #:export (function))
 
+(define procedure-property-      (@@ (oop pf-objects) procedure-property-))
+(define procedure-properties-    (@@ (oop pf-objects) procedure-properties-))
+(define set-procedure-property!- (@@ (oop pf-objects) set-procedure-property!-))
+
 (define-syntax-rule (aif it p x y) (let ((it p)) (if it x y)))
 
 (define-python-class function ()
@@ -39,19 +43,19 @@
     (procedure-name f))
 
    ((equal? tag '__qualname__)
-    (aif it (procedure-property f '__qualname__)
+    (aif it (procedure-property- f '__qualname__)
 	 it
 	 (procedure-name f)))
 
    ((equal? tag '__dict__)
-    (dict (let lp ((l (procedure-properties f)))
+    (dict (let lp ((l (procedure-properties- f)))
 	    (if (pair? l)
 		(cons (list (car l) (cdr l))
 		      (lp (cdr l)))
 		'()))))
    
    ((equal? tag '__annotations__)
-    (procedure-property f '__annotations__))
+    (procedure-property- f '__annotations__))
 
    ((equal? tag '__closure__)
     (error "closure property is not implemented"))
@@ -66,7 +70,7 @@
     (error "kwdefaults tag is not implemented"))
    
    (else
-    (let ((r (procedure-property f tag)))
+    (let ((r (procedure-property- f tag)))
       (if (not r)
 	  (if (pair? l) (car l) #f)
 	  r)))))
@@ -97,18 +101,18 @@
 
   (cond
    ((equal? tag '__name__)
-    (set-procedure-property! f 'name
+    (set-procedure-property!- f 'name
 			     (if (symbol? val)
 				 val
 				 (string->symbol val))))
    ((equal? tag '__dict__)
-    (set-procedure-properties! f
+    (set-procedure-properties!- f
 			       (for ((k v : val)) ((l '()))
 				    (cons (cons k v) l)
 				    #:final
 				    (reverse l))))
    (else
-    (set-procedure-property! f tag val))))
+    (set-procedure-property!- f tag val))))
 
 (define-method (dir (o <procedure>))
   (let ((ret (+ (to-pylist '("__name__" "__qualname__"))
@@ -118,7 +122,7 @@
 				    (if (symbol? x)
 					(symbol->string x)
 					x)))
-				(procedure-properties o))))))
+				(procedure-properties- o))))))
     (pylist-sort! ret)
     ret))
 
diff --git a/modules/language/python/set.scm b/modules/language/python/set.scm
index 2f3b7cc..23fb552 100644
--- a/modules/language/python/set.scm
+++ b/modules/language/python/set.scm
@@ -31,6 +31,39 @@
 
 (define miss (list 'miss))
 
+(define-method (< (o1 <set>) ( o2 <set>))
+  (and (not (equal? o1 o2))
+       (for ((k : o1)) ()
+            (if (in k o2)
+                (values)
+                (break #f))
+            #:final #t)))
+
+(define-method (> (o1 <set>) ( o2 <set>))
+  (and (not (equal? o1 o2))
+       (for ((k : o2)) ()
+            (if (in k o1)
+                (values)
+                (break #f))
+            #:final #t)))
+
+(define-method (<= (o1 <set>) ( o2 <set>))
+  (for ((k : o1)) ()
+       (if (in k o2)
+           (values)
+           (break #f))
+       #:final #t))
+
+(define-method (>= (o1 <set>) ( o2 <set>))
+  (for ((k : o2)) ()
+       (if (in k o1)
+           (values)
+           (break #f))
+       #:final #t))
+
+(define-method (in k (o <set>))
+  (in k (slot-ref o 'dict)))
+
 (define-syntax-rule (mk set make-py-hashtable)
 (define-python-class set (<set>)
   (define __init__
diff --git a/modules/oop/pf-objects.scm b/modules/oop/pf-objects.scm
index fd11182..0e2f1d9 100644
--- a/modules/oop/pf-objects.scm
+++ b/modules/oop/pf-objects.scm
@@ -35,10 +35,35 @@ The datastructure is functional but the objects mutate. So one need to
 explicitly tell it to not update etc.
 |#
 
+(define-syntax-rule (aif it p x y) (let ((it p)) (if it x y)))
+
 ;; this is mutated by the dict class
 (define dictNs  '(dictNs))
 (define dictRNs '(dictRNs))
 
+(define prophash (make-hash-table))
+
+(define (procedure-property- o key . l)
+  (define ret (if (pair? l) (car l) #f))    
+  (aif props (hashq-ref prophash o)
+       (aif it (assq key props)
+            (cdr it)
+            ret)
+       ret))
+
+(define (procedure-properties- o)
+  (define ret #f)
+  (aif props (hashq-ref prophash o)
+       props
+       ret))
+
+(define (set-procedure-property!- o key v)
+  (hashq-set! prophash
+              o
+              (aif props (hashq-ref prophash o)
+                   (cons (cons key v) props)
+                   (list (cons key v)))))
+
 #;
 (define (pkk . l)
   (let* ((r (reverse l))
@@ -51,9 +76,6 @@ explicitly tell it to not update etc.
 (define (pkk . l)
   (car (reverse l)))
 
-
-(define-syntax-rule (aif it p x y) (let ((it p)) (if it x y)))
-
 (define (pk-obj o)
   (pk 'start-pk-obj)
   (let ((h (slot-ref o 'h)))
@@ -64,7 +86,7 @@ explicitly tell it to not update etc.
     
     (pk 'finished-obj)
     
-    (let lp ((l (pk 'mro (ref o '__mro__ '()))))
+    (let lp ((l (pk 'mro (rawref o '__mro__ '()))))
       (if (pair? l)
 	  (let ((cl (car l)))
 	    (if (is-a? cl <p>)
@@ -133,24 +155,24 @@ explicitly tell it to not update etc.
        (next-method)))
 
 (define-method (ref (o <procedure>) key . l)
-  (aif it (procedure-property o key)
+  (aif it (procedure-property- o key)
        it
        (if (pair? l)
 	   (car l)
 	   #f)))
 
 (define-method (rawref (o <procedure>) key . l)
-  (aif it (procedure-property o key)
+  (aif it (procedure-property- o key)
        it
        (if (pair? l)
 	   (car l)
 	   #f)))
 
 (define-method (set (o <procedure>) key val)
-  (set-procedure-property! o key val))
+  (set-procedure-property!- o key val))
 
 (define-method (rawset (o <procedure>) key val)
-  (set-procedure-property! o key val))
+  (set-procedure-property!- o key val))
 
 (define-method (find-in-class x key fail) fail)
 (define-method (find-in-class (klass <pf>) key fail)
@@ -520,6 +542,7 @@ explicitly tell it to not update etc.
 	       (apply it class x)
 	       (the-create-object class x))))))
 
+;; This are finished in the _python.scm module
 (define int-cls    #f)
 (define int?       #f)
 (define tuple-cls  #f)
@@ -706,8 +729,14 @@ explicitly tell it to not update etc.
    (else
     #f)))
 
-(define-method (ref (x <pf> )  key . l) (mref     x key l))
-(define-method (ref (x <p>  )  key . l) (mref     x key l))
+(define-syntax-rule (mox o x)
+  (if (procedure? x)
+      (aif it (procedure-property- x '__get__)
+           (it x o (fluid-ref *location*))
+           x)))
+
+(define-method (ref (x <pf> )  key . l) (mox x (mref     x key l)))
+(define-method (ref (x <p>  )  key . l) (mox x (mref     x key l)))
 (define-method (ref (x <pyf>)  key . l) (mref-py  x key l))
 (define-method (ref (x <py> )  key . l) (mref-py  x key l))
 
@@ -717,10 +746,10 @@ explicitly tell it to not update etc.
 
 
 (define-method (set (f <procedure>) key val)
-  (set-procedure-property! f key val))
+  (set-procedure-property!- f key val))
 
 (define-method (ref (f <procedure>) key . l)
-  (aif it (assoc key (procedure-properties f))
+  (aif it (assoc key (procedure-properties- f))
        (cdr it)
        (if (pair? l) (car l) #f)))
 
@@ -1274,16 +1303,23 @@ explicitly tell it to not update etc.
 	   cl)))))
     
 (define type-goops #f)
+(define kind-cache (make-hash-table))
+(define (kind-cache-it type it)
+  (hashq-set! kind-cache type it)
+  type)
+
 (define (kind x)
   (if (not type-goops) (set! type-goops (rawref type '__goops__)))      
   (and (is-a? x <p>)
-       (aif it (find-in-class x '__goops__ #f)
-            (if (or
-                 (not type-goops)
-                 (eq? it type-goops)
-                 (member it (class-subclasses type-goops)))
-                'type
-                'class)
+       (aif it (find-in-class-raw x '__goops__ #f)
+            (aif it2 (hashq-ref kind-cache it)
+                 it2
+                 (if (or                 
+                      (not type-goops)
+                      (eq? it type-goops)
+                      (member it (class-subclasses type-goops)))
+                     (kind-cache-it 'type  it)
+                     (kind-cache-it 'class it)))
             'object)))
 
 (define (pyobject? x) (eq? (kind x) 'object))
@@ -1291,7 +1327,7 @@ explicitly tell it to not update etc.
 (define (pytype?   x) (eq? (kind x) 'type))
 
 (define (mark-fkn tag f)
-  (set-procedure-property! f 'py-special tag)
+  (set-procedure-property!- f 'py-special tag)
   f)
 
 (define-syntax-parameter