diff options
Diffstat (limited to 'modules/language/python')
| -rw-r--r-- | modules/language/python/bytes.scm | 30 | ||||
| -rw-r--r-- | modules/language/python/compile.scm | 56 | ||||
| -rw-r--r-- | modules/language/python/def.scm | 4 | ||||
| -rw-r--r-- | modules/language/python/exceptions.scm | 3 | ||||
| -rwxr-xr-x | modules/language/python/module/base64.py | 598 | ||||
| -rw-r--r-- | modules/language/python/module/binascii.scm | 102 | ||||
| -rwxr-xr-x | modules/language/python/module/quopri.py | 238 | ||||
| -rwxr-xr-x | modules/language/python/module/uu.py | 195 |
8 files changed, 1159 insertions, 67 deletions
diff --git a/modules/language/python/bytes.scm b/modules/language/python/bytes.scm index 8cca953..ceaa5b6 100644 --- a/modules/language/python/bytes.scm +++ b/modules/language/python/bytes.scm @@ -15,7 +15,7 @@ #:use-module (language python bool) #:use-module (language python persist) #:export (<py-bytes> bv-scm pybytes-listing bytes bytearray bytes->bytevector - py-decode + py-decode make_trans <py-bytearray> pybytesarray-listing scm-bytevector)) (define (bv-scm x) @@ -89,6 +89,9 @@ x))) (define-python-class bytes (<py-bytes>) + (define maketrans + (lambda (x y) (make_trans x y))) + (define __init__ (case-lambda ((self) @@ -1143,7 +1146,30 @@ (+ i 1))) (lp (+ i 1) r old))) (reverse r))))) - + +(define (make_trans b1 b2) + (let* ((b1 (bv-scm b1)) + (b2 (bv-scm b2)) + (n1 (len b1)) + (n2 (len b2)) + (n (let lp ((i 0) (r 0)) + (if (< i n1) + (lp (+ i 1) (max (bytevector-u8-ref b1 i) r)) + r)))) + (if (= n1 n2) + (let lp ((i 0) (r '())) + (if (< i n) + (let lp2 ((j 0)) + (if (< j n1) + (if (= (bytevector-u8-ref b1 j) i) + (lp (+ i 1) (cons (bytevector-u8-ref b2 j) r)) + (lp2 (+ j 1))) + (lp (+ i 1) (cons i r)))) + (bytes (list->u8vector (reverse r))))) + (raise + (ValueError + "maketrans: wrong number in second string compared to first"))))) + (define-py* translate (py-translate bytes s n table . l) (let* ((table (b-get table)) (w (b-make n)) diff --git a/modules/language/python/compile.scm b/modules/language/python/compile.scm index 86b30dc..d715c8d 100644 --- a/modules/language/python/compile.scm +++ b/modules/language/python/compile.scm @@ -344,6 +344,15 @@ ((#:lambdef . _) vs) + ((#:comma a) + (scope a vs)) + + ((#:comma a . l) + (union + (scope a vs) + (scope (cons #:comma l) vs))) + + ((#:with (l ...) code) (scope code (union vs (let lp ((l l)) @@ -408,7 +417,7 @@ (() '())))))) (scope final (scope code vs)))) - + ((#:expr-stmt l (#:assign k . u)) (union (union (fold (lambda (x s) @@ -1108,7 +1117,7 @@ (#:suite ((_ #:stmt . l) (cons* (G 'begin) `(,(G 'values)) (map (g vs exp) l))) - ((_ . l) (cons* (G 'begin) `(,(G 'values)) (map (g vs exp) l)))) + ((_ . l) (list (G 'begin) `(,(G 'values)) (map (g vs exp) l)))) (#:classdef ((_ class parents code) @@ -1149,7 +1158,13 @@ (#:scm ((_ (#:string _ s)) (with-input-from-string s read))) - + + (#:comma + ((_ a) + (exp vs a)) + ((_ a . l) + `(,(G 'begin) ,(exp vs a) ,(exp vs (cons #:comma l))))) + (#:import ((_ (#:from (() . nm) . #f)) (let* ((xl (map (lambda (nm) (exp vs nm)) nm)) @@ -1638,9 +1653,7 @@ (#:stmt ((_ l) - (if (> (length l) 1) - (cons cvalues (map (g vs exp) l)) - (exp vs (car l))))) + (exp vs l))) (#:expr-stmt ((_ (l ...) (#:assign)) @@ -1711,7 +1724,7 @@ `(,(G 'if) (,(G 'not) (,(G 'and) ,@(map (lambda (x) `(,(C 'boolit) ,(exp vs x))) x))) - (,(C 'raise) ,(C 'AssertionError) ',f ,n ,m)))) + (,(C 'raise) ,(C 'AssertionError) (,(G 'quote) ,f) ,n ,m)))) @@ -1829,7 +1842,7 @@ (define (exp vs x) - (match x + (match (pr x) ((e) (exp vs e)) ((tag . l) @@ -1852,12 +1865,13 @@ (define start (match x (((#:stmt - ((#:expr-stmt + (#:comma + (#:expr-stmt ((#:test (#:power #f - (#:identifier "module" . _) - ((#:arglist arglist)) - . #f) #f)) + (#:identifier "module" . _) + ((#:arglist arglist)) + . #f) #f)) (#:assign)))) . rest) (let () @@ -1866,7 +1880,7 @@ (exp '() x)) arglist)) - `((,(G 'define-module) (language python module ,@args) + `((define-module (language python module ,@args) #:pure #:use-module ((guile) #:select (@ @@ pk let* lambda call-with-values case-lambda @@ -1892,7 +1906,7 @@ (let* ((globs (get-globals x)) (e (map (g globs exp) x))) - `(,(G 'begin) + `(begin ,@start (,(G 'define) ,fnm (,(G 'make-hash-table))) ,@(map (lambda (s) @@ -1971,7 +1985,7 @@ (define (is-ec ret x tail tags) (match x - (((@ (guile) 'cond) (p a ... b) ...) + ((('@ ('guile) 'cond) (p a ... b) ...) (or (or-map (lambda (x) (or-map (lambda (x) (is-ec ret x #f tags)) x)) a) @@ -1988,12 +2002,12 @@ (or-map (lambda (x) (is-ec ret x #f tags)) a) (is-ec ret b tail tags))) - (((@ (guile) 'begin) a ... b) + ((('@ ('guile) 'begin) a ... b) (or (or-map (lambda (x) (is-ec ret x #f tags)) a) (is-ec ret b tail tags))) - (((@ (guile) 'let) lp ((y x) ...) a ... b) (=> next) + ((('@ ('guile) 'let) lp ((y x) ...) a ... b) (=> next) (if (symbol? lp) (or (or-map (lambda (x) (is-ec ret x #f tags)) x) @@ -2001,7 +2015,7 @@ (is-ec ret b tail tags)) (next))) - (((@ (guile) 'let) ((y x) ...) a ... b) + ((('@ ('guile) 'let) ((y x) ...) a ... b) (or (or-map (lambda (x) (is-ec ret x #f tags)) x) (or-map (lambda (x) (is-ec ret x #f tags)) a) @@ -2013,16 +2027,16 @@ (or-map (lambda (x) (is-ec ret x #f tags)) a) (is-ec ret b tail tags))) - (((@ (guile) 'define) . _) + ((('@ ('guile) 'define) . _) #f) - (((@ (guile) 'if) p a b) + ((('@ ('guile) 'if) p a b) (or (is-ec ret p #f tags) (is-ec ret a tail tags) (is-ec ret b tail tags))) - (((@ (guile) 'if) p a) + ((('@ ('guile) 'if) p a) (or (is-ec ret #'p #f tags) (is-ec ret #'a tail tags))) diff --git a/modules/language/python/def.scm b/modules/language/python/def.scm index 1f527e9..569023c 100644 --- a/modules/language/python/def.scm +++ b/modules/language/python/def.scm @@ -89,7 +89,7 @@ (if (and (pair? u12345678) (not (keyword? (car u12345678)))) (raise (ArgumentError "too many arguments to function"))) - code ...)) + (let () code ...))) (with-syntax ((kw (if (null? kw) (datum->syntax x (gensym "kw")) (car kw))) @@ -107,7 +107,7 @@ ...) (let ((ww ww*) (kw (pytonize kw))) - code ...))))))))))))) + (let () code ...)))))))))))))) (define-syntax-rule (def (f . args) code ...) (define f (lam args code ...))) diff --git a/modules/language/python/exceptions.scm b/modules/language/python/exceptions.scm index 8bc812f..f2cceee 100644 --- a/modules/language/python/exceptions.scm +++ b/modules/language/python/exceptions.scm @@ -16,7 +16,7 @@ ResourceWarning UnicodeDecodeError LookupError IndentationError KeyboardInterrupt MemoryError NameError - EOFError UnicodeError + EOFError UnicodeError UnicodeEncodeError FileExistsError FileNotFoundError IsADirectoryError )) (define-syntax-rule (aif it p x y) (let ((it p)) (if it x y))) @@ -69,6 +69,7 @@ (define StopIteration 'StopIteration) (define GeneratorExit 'GeneratorExit) +(define-er UnicodeEncodeError 'UnicodeEncodeError) (define-er FileExistsError 'FileExistsError) (define-er FileNotFoundError 'FileNotFoundError) (define-er IsADirectoryError 'IsADirectoryError) diff --git a/modules/language/python/module/base64.py b/modules/language/python/module/base64.py new file mode 100755 index 0000000..a61cdbe --- /dev/null +++ b/modules/language/python/module/base64.py @@ -0,0 +1,598 @@ +module(base64) + +"""Base16, Base32, Base64 (RFC 3548), Base85 and Ascii85 data encodings""" + +# Modified 04-Oct-1995 by Jack Jansen to use binascii module +# Modified 30-Dec-2003 by Barry Warsaw to add full RFC 3548 support +# Modified 22-May-2007 by Guido van Rossum to use bytes everywhere + +import re +import struct +import binascii + + +__all__ = [ + # Legacy interface exports traditional RFC 2045 Base64 encodings + 'encode', 'decode', 'encodebytes', 'decodebytes', + # Generalized interface for other encodings + 'b64encode', 'b64decode', 'b32encode', 'b32decode', + 'b16encode', 'b16decode', + # Base85 and Ascii85 encodings + 'b85encode', 'b85decode', 'a85encode', 'a85decode', + # Standard Base64 encoding + 'standard_b64encode', 'standard_b64decode', + # Some common Base64 alternatives. As referenced by RFC 3458, see thread + # starting at: + # + # http://zgp.org/pipermail/p2p-hackers/2001-September/000316.html + 'urlsafe_b64encode', 'urlsafe_b64decode', + ] + + +bytes_types = (bytes, bytearray) # Types acceptable as binary data + +def _bytes_from_decode_data(s): + if isinstance(s, str): + try: + return s.encode('ascii') + except UnicodeEncodeError: + raise ValueError('string argument should contain only ASCII characters') + if isinstance(s, bytes_types): + return s + try: + return memoryview(s).tobytes() + except TypeError: + raise TypeError("argument should be a bytes-like object or ASCII " + "string, not %r" % s.__class__.__name__) from None + + +# Base64 encoding/decoding uses binascii + +def b64encode(s, altchars=None): + """Encode the bytes-like object s using Base64 and return a bytes object. + + Optional altchars should be a byte string of length 2 which specifies an + alternative alphabet for the '+' and '/' characters. This allows an + application to e.g. generate url or filesystem safe Base64 strings. + """ + encoded = binascii.b2a_base64(s, newline=False) + if altchars is not None: + assert len(altchars) == 2, repr(altchars) + return encoded.translate(bytes.maketrans(b'+/', altchars)) + return encoded + + +def b64decode(s, altchars=None, validate=False): + """Decode the Base64 encoded bytes-like object or ASCII string s. + + Optional altchars must be a bytes-like object or ASCII string of length 2 + which specifies the alternative alphabet used instead of the '+' and '/' + characters. + + The result is returned as a bytes object. A binascii.Error is raised if + s is incorrectly padded. + + If validate is False (the default), characters that are neither in the + normal base-64 alphabet nor the alternative alphabet are discarded prior + to the padding check. If validate is True, these non-alphabet characters + in the input result in a binascii.Error. + """ + s = _bytes_from_decode_data(s) + if altchars is not None: + altchars = _bytes_from_decode_data(altchars) + assert len(altchars) == 2, repr(altchars) + s = s.translate(bytes.maketrans(altchars, b'+/')) + if validate and not re.match(b'^[A-Za-z0-9+/]*={0,2}$', s): + raise binascii.Error('Non-base64 digit found') + return binascii.a2b_base64(s) + + +def standard_b64encode(s): + """Encode bytes-like object s using the standard Base64 alphabet. + + The result is returned as a bytes object. + """ + return b64encode(s) + +def standard_b64decode(s): + """Decode bytes encoded with the standard Base64 alphabet. + + Argument s is a bytes-like object or ASCII string to decode. The result + is returned as a bytes object. A binascii.Error is raised if the input + is incorrectly padded. Characters that are not in the standard alphabet + are discarded prior to the padding check. + """ + return b64decode(s) + + +_urlsafe_encode_translation = bytes.maketrans(b'+/', b'-_') +_urlsafe_decode_translation = bytes.maketrans(b'-_', b'+/') + +def urlsafe_b64encode(s): + """Encode bytes using the URL- and filesystem-safe Base64 alphabet. + + Argument s is a bytes-like object to encode. The result is returned as a + bytes object. The alphabet uses '-' instead of '+' and '_' instead of + '/'. + """ + return b64encode(s).translate(_urlsafe_encode_translation) + +def urlsafe_b64decode(s): + """Decode bytes using the URL- and filesystem-safe Base64 alphabet. + + Argument s is a bytes-like object or ASCII string to decode. The result + is returned as a bytes object. A binascii.Error is raised if the input + is incorrectly padded. Characters that are not in the URL-safe base-64 + alphabet, and are not a plus '+' or slash '/', are discarded prior to the + padding check. + + The alphabet uses '-' instead of '+' and '_' instead of '/'. + """ + s = _bytes_from_decode_data(s) + s = s.translate(_urlsafe_decode_translation) + return b64decode(s) + + + +# Base32 encoding/decoding must be done in Python +_b32alphabet = b'ABCDEFGHIJKLMNOPQRSTUVWXYZ234567' +_b32tab2 = None +_b32rev = None + +def b32encode(s): + """Encode the bytes-like object s using Base32 and return a bytes object. + """ + global _b32tab2 + # Delay the initialization of the table to not waste memory + # if the function is never called + if _b32tab2 is None: + b32tab = [bytes((i,)) for i in _b32alphabet] + _b32tab2 = [a + b for a in b32tab for b in b32tab] + b32tab = None + + if not isinstance(s, bytes_types): + s = memoryview(s).tobytes() + leftover = len(s) % 5 + # Pad the last quantum with zero bits if necessary + if leftover: + s = s + b'\0' * (5 - leftover) # Don't use += ! + encoded = bytearray() + from_bytes = int.from_bytes + b32tab2 = _b32tab2 + for i in range(0, len(s), 5): + c = from_bytes(s[i: i + 5], 'big') + encoded += (b32tab2[c >> 30] + # bits 1 - 10 + b32tab2[(c >> 20) & 0x3ff] + # bits 11 - 20 + b32tab2[(c >> 10) & 0x3ff] + # bits 21 - 30 + b32tab2[c & 0x3ff] # bits 31 - 40 + ) + # Adjust for any leftover partial quanta + if leftover == 1: + encoded[-6:] = b'======' + elif leftover == 2: + encoded[-4:] = b'====' + elif leftover == 3: + encoded[-3:] = b'===' + elif leftover == 4: + encoded[-1:] = b'=' + return bytes(encoded) + +def b32decode(s, casefold=False, map01=None): + """Decode the Base32 encoded bytes-like object or ASCII string s. + + Optional casefold is a flag specifying whether a lowercase alphabet is + acceptable as input. For security purposes, the default is False. + + RFC 3548 allows for optional mapping of the digit 0 (zero) to the + letter O (oh), and for optional mapping of the digit 1 (one) to + either the letter I (eye) or letter L (el). The optional argument + map01 when not None, specifies which letter the digit 1 should be + mapped to (when map01 is not None, the digit 0 is always mapped to + the letter O). For security purposes the default is None, so that + 0 and 1 are not allowed in the input. + + The result is returned as a bytes object. A binascii.Error is raised if + the input is incorrectly padded or if there are non-alphabet + characters present in the input. + """ + global _b32rev + # Delay the initialization of the table to not waste memory + # if the function is never called + if _b32rev is None: + _b32rev = {v: k for k, v in enumerate(_b32alphabet)} + s = _bytes_from_decode_data(s) + if len(s) % 8: + raise binascii.Error('Incorrect padding') + # Handle section 2.4 zero and one mapping. The flag map01 will be either + # False, or the character to map the digit 1 (one) to. It should be + # either L (el) or I (eye). + if map01 is not None: + map01 = _bytes_from_decode_data(map01) + assert len(map01) == 1, repr(map01) + s = s.translate(bytes.maketrans(b'01', b'O' + map01)) + if casefold: + s = s.upper() + # Strip off pad characters from the right. We need to count the pad + # characters because this will tell us how many null bytes to remove from + # the end of the decoded string. + l = len(s) + s = s.rstrip(b'=') + padchars = l - len(s) + # Now decode the full quanta + decoded = bytearray() + b32rev = _b32rev + for i in range(0, len(s), 8): + quanta = s[i: i + 8] + acc = 0 + try: + for c in quanta: + acc = (acc << 5) + b32rev[c] + except KeyError: + raise binascii.Error('Non-base32 digit found') from None + decoded += acc.to_bytes(5, 'big') + # Process the last, partial quanta + if padchars: + acc <<= 5 * padchars + last = acc.to_bytes(5, 'big') + if padchars == 1: + decoded[-5:] = last[:-1] + elif padchars == 3: + decoded[-5:] = last[:-2] + elif padchars == 4: + decoded[-5:] = last[:-3] + elif padchars == 6: + decoded[-5:] = last[:-4] + else: + raise binascii.Error('Incorrect padding') + return bytes(decoded) + + + +# RFC 3548, Base 16 Alphabet specifies uppercase, but hexlify() returns +# lowercase. The RFC also recommends against accepting input case +# insensitively. +def b16encode(s): + """Encode the bytes-like object s using Base16 and return a bytes object. + """ + return binascii.hexlify(s).upper() + + +def b16decode(s, casefold=False): + """Decode the Base16 encoded bytes-like object or ASCII string s. + + Optional casefold is a flag specifying whether a lowercase alphabet is + acceptable as input. For security purposes, the default is False. + + The result is returned as a bytes object. A binascii.Error is raised if + s is incorrectly padded or if there are non-alphabet characters present + in the input. + """ + s = _bytes_from_decode_data(s) + if casefold: + s = s.upper() + if re.search(b'[^0-9A-F]', s): + raise binascii.Error('Non-base16 digit found') + return binascii.unhexlify(s) + +# +# Ascii85 encoding/decoding +# + +_a85chars = None +_a85chars2 = None +_A85START = b"<~" +_A85END = b"~>" + +def _85encode(b, chars, chars2, pad=False, foldnuls=False, foldspaces=False): + # Helper function for a85encode and b85encode + if not isinstance(b, bytes_types): + b = memoryview(b).tobytes() + + padding = (-len(b)) % 4 + if padding: + b = b + b'\0' * padding + words = struct.Struct('!%dI' % (len(b) // 4)).unpack(b) + + chunks = [b'z' if foldnuls and not word else + b'y' if foldspaces and word == 0x20202020 else + (chars2[word // 614125] + + chars2[word // 85 % 7225] + + chars[word % 85]) + for word in words] + + if padding and not pad: + if chunks[-1] == b'z': + chunks[-1] = chars[0] * 5 + chunks[-1] = chunks[-1][:-padding] + + return b''.join(chunks) + +def a85encode(b, *, foldspaces=False, wrapcol=0, pad=False, adobe=False): + """Encode bytes-like object b using Ascii85 and return a bytes object. + + foldspaces is an optional flag that uses the special short sequence 'y' + instead of 4 consecutive spaces (ASCII 0x20) as supported by 'btoa'. This + feature is not supported by the "standard" Adobe encoding. + + wrapcol controls whether the output should have newline (b'\\n') characters + added to it. If this is non-zero, each output line will be at most this + many characters long. + + pad controls whether the input is padded to a multiple of 4 before + encoding. Note that the btoa implementation always pads. + + adobe controls whether the encoded byte sequence is framed with <~ and ~>, + which is used by the Adobe implementation. + """ + global _a85chars, _a85chars2 + # Delay the initialization of tables to not waste memory + # if the function is never called + if _a85chars is None: + _a85chars = [bytes((i,)) for i in range(33, 118)] + _a85chars2 = [(a + b) for a in _a85chars for b in _a85chars] + + result = _85encode(b, _a85chars, _a85chars2, pad, True, foldspaces) + + if adobe: + result = _A85START + result + if wrapcol: + wrapcol = max(2 if adobe else 1, wrapcol) + chunks = [result[i: i + wrapcol] + for i in range(0, len(result), wrapcol)] + if adobe: + if len(chunks[-1]) + 2 > wrapcol: + chunks.append(b'') + result = b'\n'.join(chunks) + if adobe: + result += _A85END + + return result + +def a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v'): + """Decode the Ascii85 encoded bytes-like object or ASCII string b. + + foldspaces is a flag that specifies whether the 'y' short sequence should be + accepted as shorthand for 4 consecutive spaces (ASCII 0x20). This feature is + not supported by the "standard" Adobe encoding. + + adobe controls whether the input sequence is in Adobe Ascii85 format (i.e. + is framed with <~ and ~>). + + ignorechars should be a byte string containing characters to ignore from the + input. This should only contain whitespace characters, and by default + contains all whitespace characters in ASCII. + + The result is returned as a bytes object. + """ + b = _bytes_from_decode_data(b) + if adobe: + if not b.endswith(_A85END): + raise ValueError( + "Ascii85 encoded byte sequences must end " + "with {!r}".format(_A85END) + ) + if b.startswith(_A85START): + b = b[2:-2] # Strip off start/end markers + else: + b = b[:-2] + # + # We have to go through this stepwise, so as to ignore spaces and handle + # special short sequences + # + packI = struct.Struct('!I').pack + decoded = [] + decoded_append = decoded.append + curr = [] + curr_append = curr.append + curr_clear = curr.clear + for x in b + b'u' * 4: + if b'!'[0] <= x <= b'u'[0]: + curr_append(x) + if len(curr) == 5: + acc = 0 + for x in curr: + acc = 85 * acc + (x - 33) + try: + decoded_append(packI(acc)) + except struct.error: + raise ValueError('Ascii85 overflow') from None + curr_clear() + elif x == b'z'[0]: + if curr: + raise ValueError('z inside Ascii85 5-tuple') + decoded_append(b'\0\0\0\0') + elif foldspaces and x == b'y'[0]: + if curr: + raise ValueError('y inside Ascii85 5-tuple') + decoded_append(b'\x20\x20\x20\x20') + elif x in ignorechars: + # Skip whitespace + continue + else: + raise ValueError('Non-Ascii85 digit found: %c' % x) + + result = b''.join(decoded) + padding = 4 - len(curr) + if padding: + # Throw away the extra padding + result = result[:-padding] + return result + +# The following code is originally taken (with permission) from Mercurial + +_b85alphabet = (b"0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" + b"abcdefghijklmnopqrstuvwxyz!#$%&()*+-;<=>?@^_`{|}~") +_b85chars = None +_b85chars2 = None +_b85dec = None + +def b85encode(b, pad=False): + """Encode bytes-like object b in base85 format and return a bytes object. + + If pad is true, the input is padded with b'\\0' so its length is a multiple of + 4 bytes before encoding. + """ + global _b85chars, _b85chars2 + # Delay the initialization of tables to not waste memory + # if the function is never called + if _b85chars is None: + _b85chars = [bytes((i,)) for i in _b85alphabet] + _b85chars2 = [(a + b) for a in _b85chars for b in _b85chars] + return _85encode(b, _b85chars, _b85chars2, pad) + +def b85decode(b): + """Decode the base85-encoded bytes-like object or ASCII string b + + The result is returned as a bytes object. + """ + global _b85dec + # Delay the initialization of tables to not waste memory + # if the function is never called + if _b85dec is None: + _b85dec = [None] * 256 + for i, c in enumerate(_b85alphabet): + _b85dec[c] = i + + b = _bytes_from_decode_data(b) + padding = (-len(b)) % 5 + b = b + b'~' * padding + out = [] + packI = struct.Struct('!I').pack + for i in range(0, len(b), 5): + chunk = b[i:i + 5] + acc = 0 + try: + for c in chunk: + acc = acc * 85 + _b85dec[c] + except TypeError: + for j, c in enumerate(chunk): + if _b85dec[c] is None: + raise ValueError('bad base85 character at position %d' + % (i + j)) from None + raise + try: + out.append(packI(acc)) + except struct.error: + raise ValueError('base85 overflow in hunk starting at byte %d' + % i) from None + + result = b''.join(out) + if padding: + result = result[:-padding] + return result + +# Legacy interface. This code could be cleaned up since I don't believe +# binascii has any line length limitations. It just doesn't seem worth it +# though. The files should be opened in binary mode. + +MAXLINESIZE = 76 # Excluding the CRLF +MAXBINSIZE = (MAXLINESIZE//4)*3 + +def encode(input, output): + """Encode a file; input and output are binary files.""" + while True: + s = input.read(MAXBINSIZE) + if not s: + break + while len(s) < MAXBINSIZE: + ns = input.read(MAXBINSIZE-len(s)) + if not ns: + break + s += ns + line = binascii.b2a_base64(s) + output.write(line) + + +def decode(input, output): + """Decode a file; input and output are binary files.""" + while True: + line = input.readline() + if not line: + break + s = binascii.a2b_base64(line) + output.write(s) + +def _input_type_check(s): + try: + m = memoryview(s) + except TypeError as err: + msg = "expected bytes-like object, not %s" % s.__class__.__name__ + raise TypeError(msg) from err + if m.format not in ('c', 'b', 'B'): + msg = ("expected single byte elements, not %r from %s" % + (m.format, s.__class__.__name__)) + raise TypeError(msg) + if m.ndim != 1: + msg = ("expected 1-D data, not %d-D data from %s" % + (m.ndim, s.__class__.__name__)) + raise TypeError(msg) + + +def encodebytes(s): + """Encode a bytestring into a bytes object containing multiple lines + of base-64 data.""" + _input_type_check(s) + pieces = [] + for i in range(0, len(s), MAXBINSIZE): + chunk = s[i : i + MAXBINSIZE] + pieces.append(binascii.b2a_base64(chunk)) + return b"".join(pieces) + +def encodestring(s): + """Legacy alias of encodebytes().""" + import warnings + warnings.warn("encodestring() is a deprecated alias since 3.1, " + "use encodebytes()", + DeprecationWarning, 2) + return encodebytes(s) + + +def decodebytes(s): + """Decode a bytestring of base-64 data into a bytes object.""" + _input_type_check(s) + return binascii.a2b_base64(s) + +def decodestring(s): + """Legacy alias of decodebytes().""" + import warnings + warnings.warn("decodestring() is a deprecated alias since Python 3.1, " + "use decodebytes()", + DeprecationWarning, 2) + return decodebytes(s) + + +# Usable as a script... +def main(): + """Small main program""" + import sys, getopt + try: + opts, args = getopt.getopt(sys.argv[1:], 'deut') + except getopt.error as msg: + sys.stdout = sys.stderr + print(msg) + print("""usage: %s [-d|-e|-u|-t] [file|-] + -d, -u: decode + -e: encode (default) + -t: encode and decode string 'Aladdin:open sesame'"""%sys.argv[0]) + sys.exit(2) + func = encode + for o, a in opts: + if o == '-e': func = encode + if o == '-d': func = decode + if o == '-u': func = decode + if o == '-t': test(); return + if args and args[0] != '-': + with open(args[0], 'rb') as f: + func(f, sys.stdout.buffer) + else: + func(sys.stdin.buffer, sys.stdout.buffer) + + +def test(): + s0 = b"Aladdin:open sesame" + print(repr(s0)) + s1 = encodebytes(s0) + print(repr(s1)) + s2 = decodebytes(s1) + print(repr(s2)) + assert s0 == s2 diff --git a/modules/language/python/module/binascii.scm b/modules/language/python/module/binascii.scm index 364cff8..bda660e 100644 --- a/modules/language/python/module/binascii.scm +++ b/modules/language/python/module/binascii.scm @@ -1,18 +1,20 @@ (define-module (language python module binascii) - #:use-modules (language python list) - #:use-modules (language python bytes) - #:use-modules (language python exceptions) - #:use-modules (language python try) + #:use-module (language python list) + #:use-module (language python bytes) + #:use-module (language python exceptions) + #:use-module (language python try) + #:use-module (language python def) #:use-module (rnrs bytevectors) + #:use-module (oop pf-objects) #:export (Error Incomplete a2b_uu b2a_uu a2b_base64 b2a_base64 a2b_qp b2a_qp a2b_hex b2a_hex)) -(define-python-class Error ()) -(define-python-class Incomplete ()) +(define-python-class Error (Exception)) +(define-python-class Incomplete (Exception)) (define (a2b_uu s) (let ((n (len s)) - (s (bv->scm s))) + (s (bv-scm s))) (define (get n i) (if (< i n) (let ((r (bytevector-u8-ref s i))) @@ -24,10 +26,10 @@ (if (< n 2) (raise (Error "uudecode of malformed uuencoded bytevector")) (let* ((N (get n 0)) - (n (min n (+ (// (* 3 N) 4) 1)))) + (n (min n (+ (ceiling-quotient (* 3 N) 4) 1)))) (let lp ((i 1) (j 0) (r '())) (if (< i n) - (let* ((a1 (get i)) + (let* ((a1 (get n i)) (a2 (get n (+ i 1))) (a3 (get n (+ i 2))) (a4 (get n (+ i 3))) @@ -53,7 +55,7 @@ (def (b2a_uu data (= backtick #f)) (let ((n (len data)) - (data (bv->scm data))) + (data (bv-scm data))) (define (m x) (if (= x 0) @@ -86,7 +88,7 @@ (raise (Error "too many characters at the line"))))) (define (str2vec x) - (u8-list->vector + (list->u8vector (map char->integer (string->list x)))) @@ -96,10 +98,10 @@ (if (< i (len x)) (lp (+ i 1) (max s (bytevector-u8-ref x i))) s)))) - (define (find x) + (define (find y) (let lp ((i 0)) (if (< i (len x)) - (if (= x (bytevector-u8-ref x i)) + (if (= y (bytevector-u8-ref x i)) i (lp (+ i 1))) N))) @@ -107,20 +109,21 @@ (let lp ((i 0) (r '())) (if (< i (+ n 1)) (lp (+ i 1) (cons (find i) r)) - (u8-list->vector (reverse r)))))) + (list->u8vector (reverse r)))))) -(define to64 (str2vec "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/")) -(define from64 (vec2str t064 65)) +(define to64 + (str2vec "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/")) +(define from64 (vec2str to64 65)) (define n64 (len from64)) (define (a2b_base64 s) (define x=x (char->integer #\=)) - (let ((s (bv->scm s)) + (let ((s (bv-scm s)) (n (len s))) (define (get1 n i) (if (< i n) - (butevector-u8-ref s i) + (bytevector-u8-ref s i) (raise (Incomplete "malformed base64 encoding missing values")))) (define (get2 i) @@ -165,15 +168,19 @@ (else (bytes (reverse (cons 10 r))))))))) -(def (b2a_base64 data (= newline True)) +(def (b2a_base64 data (= newline #t)) (define x=x (char->integer #\=)) - (let ((n (len data)) - (s (bv->scm data))) + (let ((n (len data)) + (s (bv-scm data))) (if (= n 0) (if newline (bytes (list 10)) (bytes (list))) (let lp ((i 0) (r '())) + (define (get i) + (bytevector-u8-ref s i)) + (define (get2 i) + (bytevector-u8-ref to64 i)) (cond ((= i (- n 1)) (let* ((b1 (get i)) @@ -211,11 +218,11 @@ (define itoh1 (str2vec "0123456789abcdef")) (define htoi1 (vec2str itoh1 128)) -(define nh1 (len hto1)) +(define nh1 (len htoi1)) (define itoh2 (str2vec "0123456789ABCDEF")) (define htoi2 (vec2str itoh2 128)) -(define nh2 (len hto1)) +(define nh2 (len htoi1)) (def (a2b_qp data (= header #f)) @@ -223,22 +230,22 @@ (define x_x (char->integer #\_)) (define spc (char->integer #\space)) - (define (tr x) - (let ((x (bytevector-u8-ref htoi1))) + (define (tr i) + (let ((x (bytevector-u8-ref htoi1 i))) (if (= x 128) - (let ((x (bytevector-u8-ref htoi2))) + (let ((x (bytevector-u8-ref htoi2 i))) (if (= x 128) (raise (Error "malformed quoted data")) x)) x))) (let ((n (len data)) - (data (bv->scm data))) + (data (bv-scm data))) (let lp ((i 0) (r '())) - (define (get n i) + (define (get i) (if (< i n) (bytevector-u8-ref data i) - (raise (incomplete "malformed quoted data missing values")))) + (raise (Incomplete "malformed quoted data missing values")))) (if (< i n) (let ((x (get i))) @@ -250,9 +257,9 @@ (if (or (= a1 10) (= a1 13)) (lp (+ i 3) r) (lp (+ i 2) r))) - (let ((a1 (tr (get (+ i 1)))) - (a2 (tr (get (+ i 2)))) - (x (logior a1 + (ash a2 4)))) + (let* ((a1 (tr (get (+ i 1)))) + (a2 (tr (get (+ i 2)))) + (x (logior a1 + (ash a2 4)))) (lp (+ i 3) (cons x r)))))) ((and header (= x x_x)) (lp (+ i 1) (cons spc r))) @@ -260,7 +267,7 @@ (lp (+ i 1) (cons x r))))) (bytes (reverse r)))))) -(def (b2a_qp data (= quotetabs #f) (=istext #t) (= header None)) +(def (b2a_qp data (= quotetabs #f) (= istext #t) (= header None)) (define x=x (char->integer #\=)) (define x_x (char->integer #\_)) (define spc (char->integer #\space)) @@ -268,13 +275,16 @@ (define (encode x) (let ((a1 (logand #xf x)) - (a2 (ash (logand #f0 x) -4))) + (a2 (ash (logand #xf0 x) -4))) (values (bytevector-u8-ref itoh1 a1) (bytevector-u8-ref itoh1 a2)))) - (let ((n (len data)) - (data (bv->scm data))) - (let lp ((i 0) (j 0) (tre? #f) (r '())) + (data (bv-scm data))) + + (define (get i) + (bytevector-u8-ref data i)) + + (let lp ((i 0) (j 0) (r '())) (cond ((= j 76) (lp i 3 (cons* (car r) (cadr r) (caddr r) @@ -288,6 +298,16 @@ (else (if (< i n) (let ((x (get i))) + (define (at-end) + (let lp ((i i)) + (if (< i n) + (if (or (eq? (get i) spc) (eq? (get i) tab)) + (lp (+ i 1)) + (if (or (eq? (get i) 10) (eq? (get i) 13)) + #t + #f)) + #t))) + (define (q) (call-with-values (lambda () (encode x)) (lambda (c1 c2) @@ -322,22 +342,22 @@ (define (b2a_hex s) (let ((n (len s)) - (s (bv->scm s))) + (s (bv-scm s))) (define (get i) (bytevector-u8-ref s i)) (define (tr x) (bytevector-u8-ref itoh1 x)) (let lp ((i 0) (r '())) - (if (< i 0) + (if (< i n) (let* ((x (get i)) (a1 (tr (logand #xf x))) (a2 (tr (ash (logand #xf0 x) -4)))) (lp (+ i 1) (cons* a2 a1 r))) (bytes (reverse r)))))) -(define (a2b_hex d) +(define (a2b_hex s) (let ((n (len s)) - (s (bv->scm s))) + (s (bv-scm s))) (define (tr x) (if (< x nh1) diff --git a/modules/language/python/module/quopri.py b/modules/language/python/module/quopri.py new file mode 100755 index 0000000..3531a17 --- /dev/null +++ b/modules/language/python/module/quopri.py @@ -0,0 +1,238 @@ +module(quopri) + +"""Conversions to/from quoted-printable transport encoding as per RFC 1521.""" + +# (Dec 1991 version). + +__all__ = ["encode", "decode", "encodestring", "decodestring"] + +ESCAPE = b'=' +MAXLINESIZE = 76 +HEX = b'0123456789ABCDEF' +EMPTYSTRING = b'' + +try: + from binascii import a2b_qp, b2a_qp +except ImportError: + a2b_qp = None + b2a_qp = None + + +def needsquoting(c, quotetabs, header): + """Decide whether a particular byte ordinal needs to be quoted. + + The 'quotetabs' flag indicates whether embedded tabs and spaces should be + quoted. Note that line-ending tabs and spaces are always encoded, as per + RFC 1521. + """ + assert isinstance(c, bytes) + if c in b' \t': + return quotetabs + # if header, we have to escape _ because _ is used to escape space + if c == b'_': + return header + return c == ESCAPE or not (b' ' <= c <= b'~') + +def quote(c): + """Quote a single character.""" + assert isinstance(c, bytes) and len(c)==1 + c = ord(c) + return ESCAPE + bytes((HEX[c//16], HEX[c%16])) + + + +def encode(input, output, quotetabs, header=False): + """Read 'input', apply quoted-printable encoding, and write to 'output'. + + 'input' and 'output' are binary file objects. The 'quotetabs' flag + indicates whether embedded tabs and spaces should be quoted. Note that + line-ending tabs and spaces are always encoded, as per RFC 1521. + The 'header' flag indicates whether we are encoding spaces as _ as per RFC + 1522.""" + + if b2a_qp is not None: + data = input.read() + odata = b2a_qp(data, quotetabs=quotetabs, header=header) + output.write(odata) + return + + def write(s, output=output, lineEnd=b'\n'): + # RFC 1521 requires that the line ending in a space or tab must have + # that trailing character encoded. + if s and s[-1:] in b' \t': + output.write(s[:-1] + quote(s[-1:]) + lineEnd) + elif s == b'.': + output.write(quote(s) + lineEnd) + else: + output.write(s + lineEnd) + + prevline = None + while 1: + line = input.readline() + if not line: + break + outline = [] + # Strip off any readline induced trailing newline + stripped = b'' + if line[-1:] == b'\n': + line = line[:-1] + stripped = b'\n' + # Calculate the un-length-limited encoded line + for c in line: + c = bytes((c,)) + if needsquoting(c, quotetabs, header): + c = quote(c) + if header and c == b' ': + outline.append(b'_') + else: + outline.append(c) + # First, write out the previous line + if prevline is not None: + write(prevline) + # Now see if we need any soft line breaks because of RFC-imposed + # length limitations. Then do the thisline->prevline dance. + thisline = EMPTYSTRING.join(outline) + while len(thisline) > MAXLINESIZE: + # Don't forget to include the soft line break `=' sign in the + # length calculation! + write(thisline[:MAXLINESIZE-1], lineEnd=b'=\n') + thisline = thisline[MAXLINESIZE-1:] + # Write out the current line + prevline = thisline + # Write out the last line, without a trailing newline + if prevline is not None: + write(prevline, lineEnd=stripped) + +def encodestring(s, quotetabs=False, header=False): + if b2a_qp is not None: + return b2a_qp(s, quotetabs=quotetabs, header=header) + from io import BytesIO + infp = BytesIO(s) + outfp = BytesIO() + encode(infp, outfp, quotetabs, header) + return outfp.getvalue() + + + +def decode(input, output, header=False): + """Read 'input', apply quoted-printable decoding, and write to 'output'. + 'input' and 'output' are binary file objects. + If 'header' is true, decode underscore as space (per RFC 1522).""" + + if a2b_qp is not None: + data = input.read() + odata = a2b_qp(data, header=header) + output.write(odata) + return + + new = b'' + while 1: + line = input.readline() + if not line: break + i, n = 0, len(line) + if n > 0 and line[n-1:n] == b'\n': + partial = 0; n = n-1 + # Strip trailing whitespace + while n > 0 and line[n-1:n] in b" \t\r": + n = n-1 + else: + partial = 1 + while i < n: + c = line[i:i+1] + if c == b'_' and header: + new = new + b' '; i = i+1 + elif c != ESCAPE: + new = new + c; i = i+1 + elif i+1 == n and not partial: + partial = 1; break + elif i+1 < n and line[i+1:i+2] == ESCAPE: + new = new + ESCAPE; i = i+2 + elif i+2 < n and ishex(line[i+1:i+2]) and ishex(line[i+2:i+3]): + new = new + bytes((unhex(line[i+1:i+3]),)); i = i+3 + else: # Bad escape sequence -- leave it in + new = new + c; i = i+1 + if not partial: + output.write(new + b'\n') + new = b'' + if new: + output.write(new) + +def decodestring(s, header=False): + if a2b_qp is not None: + return a2b_qp(s, header=header) + from io import BytesIO + infp = BytesIO(s) + outfp = BytesIO() + decode(infp, outfp, header=header) + return outfp.getvalue() + + + +# Other helper functions +def ishex(c): + """Return true if the byte ordinal 'c' is a hexadecimal digit in ASCII.""" + assert isinstance(c, bytes) + return b'0' <= c <= b'9' or b'a' <= c <= b'f' or b'A' <= c <= b'F' + +def unhex(s): + """Get the integer value of a hexadecimal number.""" + bits = 0 + for c in s: + c = bytes((c,)) + if b'0' <= c <= b'9': + i = ord('0') + elif b'a' <= c <= b'f': + i = ord('a')-10 + elif b'A' <= c <= b'F': + i = ord(b'A')-10 + else: + assert False, "non-hex digit "+repr(c) + bits = bits*16 + (ord(c) - i) + return bits + + + +def main(): + import sys + import getopt + try: + opts, args = getopt.getopt(sys.argv[1:], 'td') + except getopt.error as msg: + sys.stdout = sys.stderr + print(msg) + print("usage: quopri [-t | -d] [file] ...") + print("-t: quote tabs") + print("-d: decode; default encode") + sys.exit(2) + deco = 0 + tabs = 0 + for o, a in opts: + if o == '-t': tabs = 1 + if o == '-d': deco = 1 + if tabs and deco: + sys.stdout = sys.stderr + print("-t and -d are mutually exclusive") + sys.exit(2) + if not args: args = ['-'] + sts = 0 + for file in args: + if file == '-': + fp = sys.stdin.buffer + else: + try: + fp = open(file, "rb") + except OSError as msg: + sys.stderr.write("%s: can't open (%s)\n" % (file, msg)) + sts = 1 + continue + try: + if deco: + decode(fp, sys.stdout.buffer) + else: + encode(fp, sys.stdout.buffer, tabs) + finally: + if file != '-': + fp.close() + if sts: + sys.exit(sts) + diff --git a/modules/language/python/module/uu.py b/modules/language/python/module/uu.py new file mode 100755 index 0000000..dc92840 --- /dev/null +++ b/modules/language/python/module/uu.py @@ -0,0 +1,195 @@ +module(uu) +# Copyright 1994 by Lance Ellinghouse +# Cathedral City, California Republic, United States of America. +# All Rights Reserved +# Permission to use, copy, modify, and distribute this software and its +# documentation for any purpose and without fee is hereby granted, +# provided that the above copyright notice appear in all copies and that +# both that copyright notice and this permission notice appear in +# supporting documentation, and that the name of Lance Ellinghouse +# not be used in advertising or publicity pertaining to distribution +# of the software without specific, written prior permission. +# LANCE ELLINGHOUSE DISCLAIMS ALL WARRANTIES WITH REGARD TO +# THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND +# FITNESS, IN NO EVENT SHALL LANCE ELLINGHOUSE CENTRUM BE LIABLE +# FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT +# OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +# +# Modified by Jack Jansen, CWI, July 1995: +# - Use binascii module to do the actual line-by-line conversion +# between ascii and binary. This results in a 1000-fold speedup. The C +# version is still 5 times faster, though. +# - Arguments more compliant with python standard + +"""Implementation of the UUencode and UUdecode functions. + +encode(in_file, out_file [,name, mode]) +decode(in_file [, out_file, mode]) +""" + +import binascii +import os +import sys + +__all__ = ["Error", "encode", "decode"] + +class Error(Exception): + pass + +def encode(in_file, out_file, name=None, mode=None): + """Uuencode file""" + # + # If in_file is a pathname open it and change defaults + # + opened_files = [] + try: + if in_file == '-': + in_file = sys.stdin.buffer + elif isinstance(in_file, str): + if name is None: + name = os.path.basename(in_file) + if mode is None: + try: + mode = os.stat(in_file).st_mode + except AttributeError: + pass + in_file = open(in_file, 'rb') + opened_files.append(in_file) + # + # Open out_file if it is a pathname + # + if out_file == '-': + out_file = sys.stdout.buffer + elif isinstance(out_file, str): + out_file = open(out_file, 'wb') + opened_files.append(out_file) + # + # Set defaults for name and mode + # + if name is None: + name = '-' + if mode is None: + mode = 0o666 + # + # Write the data + # + out_file.write(('begin %o %s\n' % ((mode & 0o777), name)).encode("ascii")) + data = in_file.read(45) + while len(data) > 0: + out_file.write(binascii.b2a_uu(data)) + data = in_file.read(45) + out_file.write(b' \nend\n') + finally: + for f in opened_files: + f.close() + + +def decode(in_file, out_file=None, mode=None, quiet=False): + """Decode uuencoded file""" + # + # Open the input file, if needed. + # + opened_files = [] + if in_file == '-': + in_file = sys.stdin.buffer + elif isinstance(in_file, str): + in_file = open(in_file, 'rb') + opened_files.append(in_file) + + try: + # + # Read until a begin is encountered or we've exhausted the file + # + while True: + hdr = in_file.readline() + if not hdr: + raise Error('No valid begin line found in input file') + if not hdr.startswith(b'begin'): + continue + hdrfields = hdr.split(b' ', 2) + if len(hdrfields) == 3 and hdrfields[0] == b'begin': + try: + int(hdrfields[1], 8) + break + except ValueError: + pass + if out_file is None: + # If the filename isn't ASCII, what's up with that?!? + out_file = hdrfields[2].rstrip(b' \t\r\n\f').decode("ascii") + if os.path.exists(out_file): + raise Error('Cannot overwrite existing file: %s' % out_file) + if mode is None: + mode = int(hdrfields[1], 8) + # + # Open the output file + # + if out_file == '-': + out_file = sys.stdout.buffer + elif isinstance(out_file, str): + fp = open(out_file, 'wb') + try: + os.path.chmod(out_file, mode) + except AttributeError: + pass + out_file = fp + opened_files.append(out_file) + # + # Main decoding loop + # + s = in_file.readline() + while s and s.strip(b' \t\r\n\f') != b'end': + try: + data = binascii.a2b_uu(s) + except binascii.Error as v: + # Workaround for broken uuencoders by /Fredrik Lundh + nbytes = (((s[0]-32) & 63) * 4 + 5) // 3 + data = binascii.a2b_uu(s[:nbytes]) + if not quiet: + sys.stderr.write("Warning: %s\n" % v) + out_file.write(data) + s = in_file.readline() + if not s: + raise Error('Truncated input file') + finally: + for f in opened_files: + f.close() + +def test(): + """uuencode/uudecode main program""" + + import optparse + parser = optparse.OptionParser(usage='usage: %prog [-d] [-t] [input [output]]') + parser.add_option('-d', '--decode', dest='decode', help='Decode (instead of encode)?', default=False, action='store_true') + parser.add_option('-t', '--text', dest='text', help='data is text, encoded format unix-compatible text?', default=False, action='store_true') + + (options, args) = parser.parse_args() + if len(args) > 2: + parser.error('incorrect number of arguments') + sys.exit(1) + + # Use the binary streams underlying stdin/stdout + input = sys.stdin.buffer + output = sys.stdout.buffer + if len(args) > 0: + input = args[0] + if len(args) > 1: + output = args[1] + + if options.decode: + if options.text: + if isinstance(output, str): + output = open(output, 'wb') + else: + print(sys.argv[0], ': cannot do -t to stdout') + sys.exit(1) + decode(input, output) + else: + if options.text: + if isinstance(input, str): + input = open(input, 'rb') + else: + print(sys.argv[0], ': cannot do -t from stdin') + sys.exit(1) + encode(input, output) |