From 55fa3f634594cedabf75182bd6404463c091ff63 Mon Sep 17 00:00:00 2001 From: Fredrik Tolf Date: Tue, 29 Nov 2011 09:12:47 +0100 Subject: [PATCH] python: Initial porting of the Python code to Python 3. --- python/ashd-wsgi | 119 +++++++++++++++++++++++++----------------------- python/ashd/proto.py | 38 ++++++++++------ python/ashd/scgi.py | 52 +++++++++++++-------- python/ashd/util.py | 25 +++++----- python/ashd/wsgidir.py | 4 +- python/ashd/wsgiutil.py | 1 + python/scgi-wsgi | 6 +-- 7 files changed, 140 insertions(+), 105 deletions(-) diff --git a/python/ashd-wsgi b/python/ashd-wsgi index 894211d..6361111 100755 --- a/python/ashd-wsgi +++ b/python/ashd-wsgi @@ -1,6 +1,6 @@ -#!/usr/bin/python +#!/usr/bin/python3 -import sys, os, getopt, threading, time +import sys, os, getopt, threading, time, locale, collections import ashd.proto, ashd.util def usage(out): @@ -25,8 +25,8 @@ if len(args) < 1: try: handlermod = __import__(args[0], fromlist = ["dummy"]) -except ImportError, exc: - sys.stderr.write("ashd-wsgi: handler %s not found: %s\n" % (args[0], exc.message)) +except ImportError as exc: + sys.stderr.write("ashd-wsgi: handler %s not found: %s\n" % (args[0], exc.args[0])) sys.exit(1) if not modwsgi_compat: if not hasattr(handlermod, "wmain"): @@ -41,7 +41,7 @@ else: class closed(IOError): def __init__(self): - super(closed, self).__init__("The client has closed the connection.") + super().__init__("The client has closed the connection.") cwd = os.getcwd() def absolutify(path): @@ -50,77 +50,80 @@ def absolutify(path): return path def unquoteurl(url): - buf = "" + buf = bytearray() i = 0 while i < len(url): c = url[i] i += 1 - if c == '%': + if c == ord(b'%'): if len(url) >= i + 2: c = 0 - if '0' <= url[i] <= '9': - c |= (ord(url[i]) - ord('0')) << 4 - elif 'a' <= url[i] <= 'f': - c |= (ord(url[i]) - ord('a') + 10) << 4 - elif 'A' <= url[i] <= 'F': - c |= (ord(url[i]) - ord('A') + 10) << 4 + if ord(b'0') <= url[i] <= ord(b'9'): + c |= (url[i] - ord(b'0')) << 4 + elif ord(b'a') <= url[i] <= ord(b'f'): + c |= (url[i] - ord(b'a') + 10) << 4 + elif ord(b'A') <= url[i] <= ord(b'F'): + c |= (url[i] - ord(b'A') + 10) << 4 else: raise ValueError("Illegal URL escape character") - if '0' <= url[i + 1] <= '9': - c |= ord(url[i + 1]) - ord('0') - elif 'a' <= url[i + 1] <= 'f': - c |= ord(url[i + 1]) - ord('a') + 10 - elif 'A' <= url[i + 1] <= 'F': - c |= ord(url[i + 1]) - ord('A') + 10 + if ord(b'0') <= url[i + 1] <= ord(b'9'): + c |= url[i + 1] - ord('0') + elif ord(b'a') <= url[i + 1] <= ord(b'f'): + c |= url[i + 1] - ord(b'a') + 10 + elif ord(b'A') <= url[i + 1] <= ord(b'F'): + c |= url[i + 1] - ord(b'A') + 10 else: raise ValueError("Illegal URL escape character") - buf += chr(c) + buf.append(c) i += 2 else: raise ValueError("Incomplete URL escape character") else: - buf += c + buf.append(c) return buf def dowsgi(req): env = {} env["wsgi.version"] = 1, 0 for key, val in req.headers: - env["HTTP_" + key.upper().replace("-", "_")] = val + env["HTTP_" + key.upper().replace(b"-", b"_").decode("latin-1")] = val.decode("latin-1") env["SERVER_SOFTWARE"] = "ashd-wsgi/1" env["GATEWAY_INTERFACE"] = "CGI/1.1" - env["SERVER_PROTOCOL"] = req.ver - env["REQUEST_METHOD"] = req.method - env["REQUEST_URI"] = req.url - name = req.url + env["SERVER_PROTOCOL"] = req.ver.decode("latin-1") + env["REQUEST_METHOD"] = req.method.decode("latin-1") + try: + rawpi = unquoteurl(req.rest) + except: + rawpi = req.rest + try: + name, rest, pi = (v.decode("utf-8") for v in (req.url, req.rest, rawpi)) + env["wsgi.uri_encoding"] = "utf-8" + except UnicodeError as exc: + name, rest, pi = (v.decode("latin-1") for v in (req.url, req.rest, rawpi)) + env["wsgi.uri_encoding"] = "latin-1" + env["REQUEST_URI"] = name p = name.find('?') if p >= 0: env["QUERY_STRING"] = name[p + 1:] name = name[:p] else: env["QUERY_STRING"] = "" - if name[-len(req.rest):] == req.rest: + if name[-len(rest):] == rest: # This is the same hack used in call*cgi. - name = name[:-len(req.rest)] - try: - pi = unquoteurl(req.rest) - except: - pi = req.rest - if name == '/': + name = name[:-len(rest)] + if name == "/": # This seems to be normal CGI behavior, but see callcgi.c for # details. pi = "/" + pi name = "" env["SCRIPT_NAME"] = name env["PATH_INFO"] = pi - if "Host" in req: env["SERVER_NAME"] = req["Host"] - if "X-Ash-Server-Port" in req: env["SERVER_PORT"] = req["X-Ash-Server-Port"] - if "X-Ash-Protocol" in req and req["X-Ash-Protocol"] == "https": env["HTTPS"] = "on" - if "X-Ash-Address" in req: env["REMOTE_ADDR"] = req["X-Ash-Address"] - if "Content-Type" in req: env["CONTENT_TYPE"] = req["Content-Type"] - if "Content-Length" in req: env["CONTENT_LENGTH"] = req["Content-Length"] - if "X-Ash-File" in req: env["SCRIPT_FILENAME"] = absolutify(req["X-Ash-File"]) - if "X-Ash-Protocol" in req: env["wsgi.url_scheme"] = req["X-Ash-Protocol"] + for src, tgt in [("HTTP_HOST", "SERVER_NAME"), ("HTTP_X_ASH_SERVER_PORT", "SERVER_PORT"), + ("HTTP_X_ASH_ADDRESS", "REMOTE_ADDR"), ("HTTP_CONTENT_TYPE", "CONTENT_TYPE"), + ("HTTP_CONTENT_LENGTH", "CONTENT_LENGTH"), ("HTTP_X_ASH_PROTOCOL", "wsgi.url_scheme")]: + if src in env: env[tgt] = env[src] + if "X-Ash-Protocol" in req and req["X-Ash-Protocol"] == b"https": env["HTTPS"] = "on" + if "X-Ash-File" in req: env["SCRIPT_FILENAME"] = absolutify(req["X-Ash-File"].decode(locale.getpreferredencoding())) env["wsgi.input"] = req.sk env["wsgi.errors"] = sys.stderr env["wsgi.multithread"] = True @@ -130,17 +133,25 @@ def dowsgi(req): resp = [] respsent = [] + def recode(thing): + if isinstance(thing, collections.ByteString): + return thing + else: + return str(thing).encode("latin-1") + def flushreq(): if not respsent: if not resp: - raise Exception, "Trying to write data before starting response." + raise Exception("Trying to write data before starting response.") status, headers = resp respsent[:] = [True] + buf = bytearray() + buf += b"HTTP/1.1 " + recode(status) + b"\n" + for nm, val in headers: + buf += recode(nm) + b": " + recode(val) + b"\n" + buf += b"\n" try: - req.sk.write("HTTP/1.1 %s\n" % status) - for nm, val in headers: - req.sk.write("%s: %s\n" % (nm, val)) - req.sk.write("\n") + req.sk.write(buf) except IOError: raise closed() @@ -159,11 +170,11 @@ def dowsgi(req): if exc_info: # Interesting, this... try: if respsent: - raise exc_info[0], exc_info[1], exc_info[2] + raise exc_info[1] finally: exc_info = None # CPython GC bug? else: - raise Exception, "Can only start responding once." + raise Exception("Can only start responding once.") resp[:] = status, headers return write @@ -185,14 +196,13 @@ inflight = 0 class reqthread(threading.Thread): def __init__(self, req): - super(reqthread, self).__init__(name = "Request handler") + super().__init__(name = "Request handler") self.req = req.dup() def run(self): global inflight try: - flightlock.acquire() - try: + with flightlock: if reqlimit != 0: start = time.time() while inflight >= reqlimit: @@ -200,17 +210,12 @@ class reqthread(threading.Thread): if time.time() - start > 10: os.abort() inflight += 1 - finally: - flightlock.release() try: dowsgi(self.req) finally: - flightlock.acquire() - try: + with flightlock: inflight -= 1 flightlock.notify() - finally: - flightlock.release() finally: self.req.close() diff --git a/python/ashd/proto.py b/python/ashd/proto.py index 4a48304..ab2152e 100644 --- a/python/ashd/proto.py +++ b/python/ashd/proto.py @@ -8,7 +8,7 @@ ashd.util module provides an easier-to-use interface. """ import os, socket -import htlib +from . import htlib __all__ = ["req", "recvreq", "sendreq"] @@ -46,12 +46,14 @@ class req(object): self.ver = ver self.rest = rest self.headers = headers - self.sk = socket.fromfd(fd, socket.AF_UNIX, socket.SOCK_STREAM).makefile('r+') + self.bsk = socket.fromfd(fd, socket.AF_UNIX, socket.SOCK_STREAM) + self.sk = self.bsk.makefile('rwb') os.close(fd) def close(self): "Close this request's response socket." self.sk.close() + self.bsk.close() def __getitem__(self, header): """Find a HTTP header case-insensitively. For example, @@ -59,6 +61,8 @@ class req(object): header regardlessly of whether the client specified it as "Content-Type", "content-type" or "Content-type". """ + if isinstance(header, str): + header = header.encode("ascii") header = header.lower() for key, val in self.headers: if key.lower() == header: @@ -69,6 +73,8 @@ class req(object): """Works analogously to the __getitem__ method for checking header presence case-insensitively. """ + if isinstance(header, str): + header = header.encode("ascii") header = header.lower() for key, val in self.headers: if key.lower() == header: @@ -79,7 +85,7 @@ class req(object): """Creates a duplicate of this request, referring to a duplicate of the response socket. """ - return req(self.method, self.url, self.ver, self.rest, self.headers, os.dup(self.sk.fileno())) + return req(self.method, self.url, self.ver, self.rest, self.headers, os.dup(self.bsk.fileno())) def match(self, match): """If the `match' argument matches exactly the leading part of @@ -95,13 +101,17 @@ class req(object): else: util.respond(req, "Not found", status = "404 Not Found", ctype = "text/plain") """ + if isinstance(match, str): + match = match.encode("utf-8") if self.rest[:len(match)] == match: self.rest = self.rest[len(match):] return True return False def __str__(self): - return "\"%s %s %s\"" % (self.method, self.url, self.ver) + def dec(b): + return b.decode("ascii", errors="replace") + return "\"%s %s %s\"" % (dec(self.method), dec(self.url), dec(self.ver)) def __enter__(self): return self @@ -127,14 +137,14 @@ def recvreq(sock = 0): if fd is None: return None try: - parts = data.split('\0')[:-1] + parts = data.split(b'\0')[:-1] if len(parts) < 5: raise protoerr("Truncated request") method, url, ver, rest = parts[:4] headers = [] i = 4 while True: - if parts[i] == "": break + if parts[i] == b"": break if len(parts) - i < 3: raise protoerr("Truncated request") headers.append((parts[i], parts[i + 1])) @@ -151,13 +161,13 @@ def sendreq(sock, req): This function may raise an OSError if an error occurs on the socket. """ - data = "" - data += req.method + '\0' - data += req.url + '\0' - data += req.ver + '\0' - data += req.rest + '\0' + data = b"" + data += req.method + b'\0' + data += req.url + b'\0' + data += req.ver + b'\0' + data += req.rest + b'\0' for key, val in req.headers: - data += key + '\0' - data += val + '\0' - data += '\0' + data += key + b'\0' + data += val + b'\0' + data += b'\0' htlib.sendfd(sock, req.sk.fileno(), data) diff --git a/python/ashd/scgi.py b/python/ashd/scgi.py index 95325f2..a06267f 100644 --- a/python/ashd/scgi.py +++ b/python/ashd/scgi.py @@ -1,4 +1,4 @@ -import sys +import sys, collections import threading class protoerr(Exception): @@ -12,21 +12,21 @@ def readns(sk): hln = 0 while True: c = sk.read(1) - if c == ':': + if c == b':': break - elif c >= '0' or c <= '9': - hln = (hln * 10) + (ord(c) - ord('0')) + elif c >= b'0' or c <= b'9': + hln = (hln * 10) + (ord(c) - ord(b'0')) else: - raise protoerr, "Invalid netstring length byte: " + c + raise protoerr("Invalid netstring length byte: " + c) ret = sk.read(hln) - if sk.read(1) != ',': - raise protoerr, "Non-terminated netstring" + if sk.read(1) != b',': + raise protoerr("Non-terminated netstring") return ret def readhead(sk): - parts = readns(sk).split('\0')[:-1] + parts = readns(sk).split(b'\0')[:-1] if len(parts) % 2 != 0: - raise protoerr, "Malformed headers" + raise protoerr("Malformed headers") ret = {} i = 0 while i < len(parts): @@ -37,7 +37,7 @@ def readhead(sk): class reqthread(threading.Thread): def __init__(self, sk, handler): super(reqthread, self).__init__(name = "SCGI request handler") - self.sk = sk.dup().makefile("r+") + self.sk = sk.dup().makefile("rwb") self.handler = handler def run(self): @@ -59,9 +59,17 @@ def servescgi(socket, handler): finally: nsk.close() +def decodehead(head, coding): + return {k.decode(coding): v.decode(coding) for k, v in head.items()} + def wrapwsgi(handler): def handle(head, sk): - env = dict(head) + try: + env = decodehead(head, "utf-8") + env["wsgi.uri_encoding"] = "utf-8" + except UnicodeError: + env = decodehead(head, "latin-1") + env["wsgi.uri_encoding"] = "latin-1" env["wsgi.version"] = 1, 0 if "HTTP_X_ASH_PROTOCOL" in env: env["wsgi.url_scheme"] = env["HTTP_X_ASH_PROTOCOL"] @@ -78,17 +86,25 @@ def wrapwsgi(handler): resp = [] respsent = [] + def recode(thing): + if isinstance(thing, collections.ByteString): + return thing + else: + return str(thing).encode("latin-1") + def flushreq(): if not respsent: if not resp: - raise Exception, "Trying to write data before starting response." + raise Exception("Trying to write data before starting response.") status, headers = resp respsent[:] = [True] + buf = bytearray() + buf += b"Status: " + recode(status) + b"\n" + for nm, val in headers: + buf += recode(nm) + b": " + recode(val) + b"\n" + buf += b"\n" try: - sk.write("Status: %s\n" % status) - for nm, val in headers: - sk.write("%s: %s\n" % (nm, val)) - sk.write("\n") + sk.write(buf) except IOError: raise closed() @@ -107,11 +123,11 @@ def wrapwsgi(handler): if exc_info: # Interesting, this... try: if respsent: - raise exc_info[0], exc_info[1], exc_info[2] + raise exc_info[1] finally: exc_info = None # CPython GC bug? else: - raise Exception, "Can only start responding once." + raise Exception("Can only start responding once.") resp[:] = status, headers return write diff --git a/python/ashd/util.py b/python/ashd/util.py index 0ff3878..08945f2 100644 --- a/python/ashd/util.py +++ b/python/ashd/util.py @@ -4,8 +4,8 @@ This module implements a rather convenient interface for writing ashd handlers, wrapping the low-level ashd.proto module. """ -import os, socket -import proto +import os, socket, collections +from . import proto __all__ = ["stdfork", "pchild", "respond", "serveloop"] @@ -27,7 +27,7 @@ def stdfork(argv, chinit = None): if pid == 0: try: os.dup2(csk.fileno(), 0) - for fd in xrange(3, 1024): + for fd in range(3, 1024): try: os.close(fd) except: @@ -131,17 +131,20 @@ def respond(req, body, status = ("200 OK"), ctype = "text/html"): For example: respond(req, "Not found", status = "404 Not Found", ctype = "text/plain") """ - if type(body) == unicode: - body = body.decode("utf-8") - if ctype[:5] == "text/" and ctype.find(';') < 0: - ctype = ctype + "; charset=utf-8" + if isinstance(body, collections.ByteString): + body = bytes(body) else: body = str(body) + body = body.encode("utf-8") + if ctype[:5] == "text/" and ctype.find(';') < 0: + ctype = ctype + "; charset=utf-8" try: - req.sk.write("HTTP/1.1 %s\n" % status) - req.sk.write("Content-Type: %s\n" % ctype) - req.sk.write("Content-Length: %i\n" % len(body)) - req.sk.write("\n") + head = "" + head += "HTTP/1.1 %s\n" % status + head += "Content-Type: %s\n" % ctype + head += "Content-Length: %i\n" % len(body) + head += "\n" + req.sk.write(head.encode("ascii")) req.sk.write(body) finally: req.close() diff --git a/python/ashd/wsgidir.py b/python/ashd/wsgidir.py index 8b473f2..f101117 100644 --- a/python/ashd/wsgidir.py +++ b/python/ashd/wsgidir.py @@ -38,7 +38,7 @@ you will probably want to use the getmod() function in this module. """ import os, threading, types -import wsgiutil +from . import wsgiutil __all__ = ["application", "wmain", "getmod", "cachedmod"] @@ -102,7 +102,7 @@ def getmod(path): code = compile(text, path, "exec") mod = types.ModuleType(mangle(path)) mod.__file__ = path - exec code in mod.__dict__ + exec(code, mod.__dict__) entry = cachedmod(mod, sb.st_mtime) modcache[path] = entry return entry diff --git a/python/ashd/wsgiutil.py b/python/ashd/wsgiutil.py index b947407..5fe7535 100644 --- a/python/ashd/wsgiutil.py +++ b/python/ashd/wsgiutil.py @@ -25,5 +25,6 @@ def simpleerror(env, startreq, code, title, msg):

%s

""" % (title, title, htmlquote(msg)) + buf = buf.encode("ascii") startreq("%i %s" % (code, title), [("Content-Type", "text/html"), ("Content-Length", str(len(buf)))]) return [buf] diff --git a/python/scgi-wsgi b/python/scgi-wsgi index 5ffcf6e..9befb25 100755 --- a/python/scgi-wsgi +++ b/python/scgi-wsgi @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 import sys, os, getopt import socket @@ -41,8 +41,8 @@ if sk is None: try: handlermod = __import__(args[0], fromlist = ["dummy"]) -except ImportError, exc: - sys.stderr.write("scgi-wsgi: handler %s not found: %s\n" % (args[0], exc.message)) +except ImportError as exc: + sys.stderr.write("scgi-wsgi: handler %s not found: %s\n" % (args[0], exc.args[0])) sys.exit(1) if not modwsgi_compat: if not hasattr(handlermod, "wmain"): -- 2.11.0