From: Fredrik Tolf Date: Wed, 8 Apr 2026 00:26:02 +0000 (+0200) Subject: python: Retire Python 2.x package. X-Git-Url: http://www.dolda2000.com/gitweb/?a=commitdiff_plain;h=99c3980c0958b0177fdd041ece0052c099479d57;p=ashd.git python: Retire Python 2.x package. --- diff --git a/python/ashd-wsgi b/python/ashd-wsgi deleted file mode 100755 index d5438fa..0000000 --- a/python/ashd-wsgi +++ /dev/null @@ -1,203 +0,0 @@ -#!/usr/bin/python - -import sys, os, getopt, socket, logging, time, signal -import ashd.util, ashd.serve -try: - import pdm.srv -except: - pdm = None - -def usage(out): - out.write("usage: ashd-wsgi [-hAL] [-m PDM-SPEC] [-p MODPATH] [-t REQUEST-HANDLER[:PAR[=VAL](,PAR[=VAL])...]] HANDLER-MODULE [ARGS...]\n") - -hspec = "free", {} -modwsgi_compat = False -setlog = True -opts, args = getopt.getopt(sys.argv[1:], "+hALp:t:l:m:") -for o, a in opts: - if o == "-h": - usage(sys.stdout) - sys.exit(0) - elif o == "-p": - sys.path.insert(0, a) - elif o == "-L": - setlog = False - elif o == "-A": - modwsgi_compat = True - elif o == "-l": - hspec = "free", {"max": a, "abort": "10"} - elif o == "-t": - hspec = ashd.serve.parsehspec(a) - elif o == "-m": - if pdm is not None: - pdm.srv.listen(a) -if len(args) < 1: - usage(sys.stderr) - sys.exit(1) -if setlog: - logging.basicConfig(format="ashd-wsgi(%(name)s): %(levelname)s: %(message)s") -log = logging.getLogger("ashd-wsgi") - -try: - handlermod = __import__(args[0], fromlist = ["dummy"]) -except ImportError, exc: - sys.stderr.write("ashd-wsgi: handler %s not found: %s\n" % (args[0], exc.message)) - sys.exit(1) -if not modwsgi_compat: - if not hasattr(handlermod, "wmain"): - sys.stderr.write("ashd-wsgi: handler %s has no `wmain' function\n" % args[0]) - sys.exit(1) - handler = handlermod.wmain(*args[1:]) -else: - if not hasattr(handlermod, "application"): - sys.stderr.write("ashd-wsgi: handler %s has no `application' object\n" % args[0]) - sys.exit(1) - handler = handlermod.application - -cwd = os.getcwd() -def absolutify(path): - if path[0] != '/': - return os.path.join(cwd, path) - return path - -def unquoteurl(url): - buf = "" - i = 0 - while i < len(url): - c = url[i] - i += 1 - if c == '%': - if len(url) >= i + 2: - c = 0 - if '0' <= url[i] <= '9': - c |= (ord(url[i]) - ord('0')) << 4 - elif 'a' <= url[i] <= 'f': - c |= (ord(url[i]) - ord('a') + 10) << 4 - elif 'A' <= url[i] <= 'F': - c |= (ord(url[i]) - ord('A') + 10) << 4 - else: - raise ValueError("Illegal URL escape character") - if '0' <= url[i + 1] <= '9': - c |= ord(url[i + 1]) - ord('0') - elif 'a' <= url[i + 1] <= 'f': - c |= ord(url[i + 1]) - ord('a') + 10 - elif 'A' <= url[i + 1] <= 'F': - c |= ord(url[i + 1]) - ord('A') + 10 - else: - raise ValueError("Illegal URL escape character") - buf += chr(c) - i += 2 - else: - raise ValueError("Incomplete URL escape character") - else: - buf += c - return buf - -def mkenv(req): - env = {} - env["wsgi.version"] = 1, 0 - for key, val in req.headers: - env["HTTP_" + key.upper().replace("-", "_")] = val - env["SERVER_SOFTWARE"] = "ashd-wsgi/1" - env["GATEWAY_INTERFACE"] = "CGI/1.1" - env["SERVER_PROTOCOL"] = req.ver - env["REQUEST_METHOD"] = req.method - env["REQUEST_URI"] = req.url - name = req.url - p = name.find('?') - if p >= 0: - env["QUERY_STRING"] = name[p + 1:] - name = name[:p] - else: - env["QUERY_STRING"] = "" - if name[-len(req.rest):] == req.rest: - # This is the same hack used in call*cgi. - name = name[:-len(req.rest)] - try: - pi = unquoteurl(req.rest) - except: - pi = req.rest - if name == '/': - # This seems to be normal CGI behavior, but see callcgi.c for - # details. - pi = "/" + pi - name = "" - env["SCRIPT_NAME"] = name - env["PATH_INFO"] = pi - if "Host" in req: env["SERVER_NAME"] = req["Host"] - if "X-Ash-Server-Address" in req: env["SERVER_ADDR"] = req["X-Ash-Server-Address"] - if "X-Ash-Server-Port" in req: env["SERVER_PORT"] = req["X-Ash-Server-Port"] - if "X-Ash-Protocol" in req and req["X-Ash-Protocol"] == "https": env["HTTPS"] = "on" - if "X-Ash-Address" in req: env["REMOTE_ADDR"] = req["X-Ash-Address"] - if "X-Ash-Port" in req: env["REMOTE_PORT"] = req["X-Ash-Port"] - if "Content-Type" in req: - env["CONTENT_TYPE"] = req["Content-Type"] - # The CGI specification does not strictly require this, but - # many actualy programs and libraries seem to. - del env["HTTP_CONTENT_TYPE"] - if "Content-Length" in req: - env["CONTENT_LENGTH"] = req["Content-Length"] - del env["HTTP_CONTENT_LENGTH"] - if "X-Ash-File" in req: env["SCRIPT_FILENAME"] = absolutify(req["X-Ash-File"]) - if "X-Ash-Protocol" in req: env["wsgi.url_scheme"] = req["X-Ash-Protocol"] - env["wsgi.input"] = req.sk - env["wsgi.errors"] = sys.stderr - env["wsgi.multithread"] = True - env["wsgi.multiprocess"] = False - env["wsgi.run_once"] = False - return env - -class request(ashd.serve.wsgirequest): - def __init__(self, bkreq, **kw): - super(request, self).__init__(**kw) - self.bkreq = bkreq.dup() - - def mkenv(self): - return mkenv(self.bkreq) - - def handlewsgi(self, env, startreq): - return handler(env, startreq) - - def fileno(self): - return self.bkreq.bsk.fileno() - - def writehead(self, status, headers): - w = self.buffer.extend - w("HTTP/1.1 %s\n" % status) - for nm, val in headers: - w("%s: %s\n" % (nm, val)) - w("\n") - - def flush(self): - try: - ret = self.bkreq.bsk.send(self.buffer, socket.MSG_DONTWAIT) - self.buffer[:ret] = "" - except IOError: - raise ashd.serve.closed() - - def close(self): - self.bkreq.close() - -def handle(req): - reqhandler.handle(request(bkreq=req, handler=reqhandler)) - -if hspec[0] not in ashd.serve.names: - sys.stderr.write("ashd-wsgi: no such request handler: %s\n" % hspec[0]) - sys.exit(1) -hclass = ashd.serve.names[hspec[0]] -try: - hargs = hclass.parseargs(**hspec[1]) -except ValueError as exc: - sys.stderr.write("ashd-wsgi: %s\n" % exc) - sys.exit(1) - -def sigterm(sig, frame): - socket.fromfd(0, socket.AF_UNIX, socket.SOCK_SEQPACKET).shutdown(socket.SHUT_RDWR) # :P -for signum in [signal.SIGINT, signal.SIGTERM]: - signal.signal(signum, sigterm) - -reqhandler = hclass(**hargs) -try: - ashd.util.serveloop(handle) -finally: - reqhandler.close() diff --git a/python3/ashd-wsgi3 b/python/ashd-wsgi3 similarity index 100% rename from python3/ashd-wsgi3 rename to python/ashd-wsgi3 diff --git a/python3/ashd/async.py b/python/ashd/async.py similarity index 100% rename from python3/ashd/async.py rename to python/ashd/async.py diff --git a/python3/ashd/asyncio.py b/python/ashd/asyncio.py similarity index 100% rename from python3/ashd/asyncio.py rename to python/ashd/asyncio.py diff --git a/python/ashd/perf.py b/python/ashd/perf.py index c74e443..4adfc54 100644 --- a/python/ashd/perf.py +++ b/python/ashd/perf.py @@ -1,7 +1,13 @@ +import collections.abc try: import pdm.perf except: pdm = None +try: + import time + clock_thread = time.CLOCK_THREAD_CPUTIME_ID +except: + clock_thread = None reqstat = {} @@ -12,7 +18,7 @@ if pdm: class reqstart(pdm.perf.startevent): def __init__(self, env): - super(reqstart, self).__init__() + super().__init__() self.method = env.get("REQUEST_METHOD") self.uri = env.get("REQUEST_URI") self.host = env.get("HTTP_HOST") @@ -23,11 +29,16 @@ if pdm: self.remoteaddr = env.get("REMOTE_ADDR") self.remoteport = env.get("REMOTE_PORT") self.scheme = env.get("wsgi.url_scheme") + if clock_thread is not None: + self.icpu = time.clock_gettime(clock_thread) class reqfinish(pdm.perf.finishevent): def __init__(self, start, aborted, status): - super(reqfinish, self).__init__(start, aborted) + super().__init__(start, aborted) self.status = status + self.cputime = 0 + if clock_thread is not None: + self.cputime = time.clock_gettime(clock_thread) - start.icpu class request(object): def __init__(self, env): @@ -45,6 +56,10 @@ class request(object): try: if len(self.resp) > 0: status = self.resp[0] + if isinstance(status, collections.abc.ByteString): + status = status.decode("latin-1") + else: + status = str(status) p = status.find(" ") if p < 0: key = status diff --git a/python/ashd/proto.py b/python/ashd/proto.py index e18023d..aa6b686 100644 --- a/python/ashd/proto.py +++ b/python/ashd/proto.py @@ -8,7 +8,7 @@ ashd.util module provides an easier-to-use interface. """ import os, socket -import htlib +from . import htlib __all__ = ["req", "recvreq", "sendreq"] @@ -32,10 +32,13 @@ class req(object): Python stream object in the `sk' variable. Again, see the ashd(7) manpage for what to receive and transmit on the response socket. - Note that instances of this class contain a reference to the live - socket used for responding to requests, which should be closed - when you are done with the request. The socket can be closed - manually by calling the close() method on this + Note that all request parts are stored in byte, rather than + string, form. The response socket is also opened in binary mode. + + Note also that instances of this class contain a reference to the + live socket used for responding to requests, which should be + closed when you are done with the request. The socket can be + closed manually by calling the close() method on this object. Alternatively, this class implements the resource-manager interface, so that it can be used in `with' statements. """ @@ -47,7 +50,7 @@ class req(object): self.rest = rest self.headers = headers self.bsk = socket.fromfd(fd, socket.AF_UNIX, socket.SOCK_STREAM) - self.sk = self.bsk.makefile('r+') + self.sk = self.bsk.makefile('rwb') os.close(fd) def close(self): @@ -60,7 +63,12 @@ class req(object): req["Content-Type"] returns the value of the content-type header regardlessly of whether the client specified it as "Content-Type", "content-type" or "Content-type". + + If the header is given as a (Unicode) string, it is encoded + into Ascii for use in matching. """ + if isinstance(header, str): + header = header.encode("ascii") header = header.lower() for key, val in self.headers: if key.lower() == header: @@ -71,6 +79,8 @@ class req(object): """Works analogously to the __getitem__ method for checking header presence case-insensitively. """ + if isinstance(header, str): + header = header.encode("ascii") header = header.lower() for key, val in self.headers: if key.lower() == header: @@ -89,6 +99,9 @@ class req(object): string off and returns True. Otherwise, it returns False without doing anything. + If the `match' argument is given as a (Unicode) string, it is + encoded into UTF-8. + This can be used for simple dispatching. For example: if req.match("foo/"): handle(req) @@ -97,13 +110,17 @@ class req(object): else: util.respond(req, "Not found", status = "404 Not Found", ctype = "text/plain") """ + if isinstance(match, str): + match = match.encode("utf-8") if self.rest[:len(match)] == match: self.rest = self.rest[len(match):] return True return False def __str__(self): - return "\"%s %s %s\"" % (self.method, self.url, self.ver) + def dec(b): + return b.decode("ascii", errors="replace") + return "\"%s %s %s\"" % (dec(self.method), dec(self.url), dec(self.ver)) def __enter__(self): return self @@ -129,14 +146,14 @@ def recvreq(sock = 0): if fd is None: return None try: - parts = data.split('\0')[:-1] + parts = data.split(b'\0')[:-1] if len(parts) < 5: raise protoerr("Truncated request") method, url, ver, rest = parts[:4] headers = [] i = 4 while True: - if parts[i] == "": break + if parts[i] == b"": break if len(parts) - i < 3: raise protoerr("Truncated request") headers.append((parts[i], parts[i + 1])) @@ -153,13 +170,13 @@ def sendreq(sock, req): This function may raise an OSError if an error occurs on the socket. """ - data = "" - data += req.method + '\0' - data += req.url + '\0' - data += req.ver + '\0' - data += req.rest + '\0' + data = b"" + data += req.method + b'\0' + data += req.url + b'\0' + data += req.ver + b'\0' + data += req.rest + b'\0' for key, val in req.headers: - data += key + '\0' - data += val + '\0' - data += '\0' - htlib.sendfd(sock, req.sk.fileno(), data) + data += key + b'\0' + data += val + b'\0' + data += b'\0' + htlib.sendfd(sock, req.bsk.fileno(), data) diff --git a/python/ashd/scgi.py b/python/ashd/scgi.py index 1f0c5ab..c00c5a3 100644 --- a/python/ashd/scgi.py +++ b/python/ashd/scgi.py @@ -5,24 +5,27 @@ def readns(sk): hln = 0 while True: c = sk.read(1) - if c == ':': + if c == b':': break - elif c >= '0' or c <= '9': - hln = (hln * 10) + (ord(c) - ord('0')) + elif c >= b'0' or c <= b'9': + hln = (hln * 10) + (ord(c) - ord(b'0')) else: - raise protoerr, "Invalid netstring length byte: " + c + raise protoerr("Invalid netstring length byte: " + c) ret = sk.read(hln) - if sk.read(1) != ',': - raise protoerr, "Non-terminated netstring" + if sk.read(1) != b',': + raise protoerr("Non-terminated netstring") return ret def readhead(sk): - parts = readns(sk).split('\0')[:-1] + parts = readns(sk).split(b'\0')[:-1] if len(parts) % 2 != 0: - raise protoerr, "Malformed headers" + raise protoerr("Malformed headers") ret = {} i = 0 while i < len(parts): ret[parts[i]] = parts[i + 1] i += 2 return ret + +def decodehead(head, coding): + return {k.decode(coding): v.decode(coding) for k, v in head.items()} diff --git a/python/ashd/serve.py b/python/ashd/serve.py index 3de5861..0927710 100644 --- a/python/ashd/serve.py +++ b/python/ashd/serve.py @@ -1,5 +1,5 @@ -import sys, os, threading, time, logging, select, Queue -import perf +import sys, os, threading, time, logging, select, queue, collections +from . import perf log = logging.getLogger("ashd.serve") seq = 1 @@ -14,16 +14,16 @@ def reqseq(): class closed(IOError): def __init__(self): - super(closed, self).__init__("The client has closed the connection.") + super().__init__("The client has closed the connection.") class reqthread(threading.Thread): - def __init__(self, name=None, **kw): + def __init__(self, *, name=None, **kw): if name is None: name = "Request handler %i" % reqseq() - super(reqthread, self).__init__(name=name, **kw) + super().__init__(name=name, **kw) class wsgirequest(object): - def __init__(self, handler): + def __init__(self, *, handler): self.status = None self.headers = [] self.respsent = False @@ -86,7 +86,7 @@ class handler(object): @classmethod def parseargs(cls, **args): if len(args) > 0: - raise ValueError("unknown handler argument: " + iter(args).next()) + raise ValueError("unknown handler argument: " + next(iter(args))) return {} class single(handler): @@ -110,11 +110,21 @@ class single(handler): finally: req.close() +def dbg(*a): + f = True + for o in a: + if not f: + sys.stderr.write(" ") + sys.stderr.write(str(a)) + f = False + sys.stderr.write("\n") + sys.stderr.flush() + class freethread(handler): cname = "free" - def __init__(self, max=None, timeout=None, **kw): - super(freethread, self).__init__(**kw) + def __init__(self, *, max=None, timeout=None, **kw): + super().__init__(**kw) self.current = set() self.lk = threading.Lock() self.tcond = threading.Condition(self.lk) @@ -122,8 +132,8 @@ class freethread(handler): self.timeout = timeout @classmethod - def parseargs(cls, max=None, abort=None, **args): - ret = super(freethread, cls).parseargs(**args) + def parseargs(cls, *, max=None, abort=None, **args): + ret = super().parseargs(**args) if max: ret["max"] = int(max) if abort: @@ -181,7 +191,122 @@ class freethread(handler): while True: with self.lk: if len(self.current) > 0: - th = iter(self.current).next() + th = next(iter(self.current)) + else: + return + th.join() + +class threadpool(handler): + cname = "pool" + + def __init__(self, *, max=25, qsz=100, timeout=None, **kw): + super().__init__(**kw) + self.current = set() + self.clk = threading.Lock() + self.ccond = threading.Condition(self.clk) + self.queue = collections.deque() + self.waiting = set() + self.waitlimit = 5 + self.wlstart = 0.0 + self.qlk = threading.Lock() + self.qfcond = threading.Condition(self.qlk) + self.qecond = threading.Condition(self.qlk) + self.max = max + self.qsz = qsz + self.timeout = timeout + + @classmethod + def parseargs(cls, *, max=None, queue=None, abort=None, **args): + ret = super().parseargs(**args) + if max: + ret["max"] = int(max) + if queue: + ret["qsz"] = int(queue) + if abort: + ret["timeout"] = int(abort) + return ret + + def handle(self, req): + spawn = False + with self.qlk: + if self.timeout is not None: + now = start = time.time() + while len(self.queue) >= self.qsz: + self.qecond.wait(start + self.timeout - now) + now = time.time() + if now - start > self.timeout: + os.abort() + else: + while len(self.queue) >= self.qsz: + self.qecond.wait() + self.queue.append(req) + self.qfcond.notify() + if len(self.waiting) < 1: + spawn = True + if spawn: + with self.clk: + if len(self.current) < self.max: + th = reqthread(target=self.run) + th.registered = False + th.start() + while not th.registered: + self.ccond.wait() + + def handle1(self, req): + try: + env = req.mkenv() + with perf.request(env) as reqevent: + respiter = req.handlewsgi(env, req.startreq) + for data in respiter: + req.write(data) + if req.status: + reqevent.response([req.status, req.headers]) + req.flushreq() + self.ckflush(req) + except closed: + pass + except: + log.error("exception occurred when handling request", exc_info=True) + + def run(self): + timeout = 10.0 + th = threading.current_thread() + with self.clk: + self.current.add(th) + th.registered = True + self.ccond.notify_all() + try: + while True: + start = now = time.time() + with self.qlk: + while len(self.queue) < 1: + if len(self.waiting) >= self.waitlimit and now - self.wlstart >= timeout: + return + self.waiting.add(th) + try: + if len(self.waiting) == self.waitlimit: + self.wlstart = now + self.qfcond.wait(start + timeout - now) + finally: + self.waiting.remove(th) + now = time.time() + if now - start > timeout: + return + req = self.queue.popleft() + self.qecond.notify() + try: + self.handle1(req) + finally: + req.close() + finally: + with self.clk: + self.current.remove(th) + + def close(self): + while True: + with self.clk: + if len(self.current) > 0: + th = next(iter(self.current)) else: return th.join() @@ -189,20 +314,20 @@ class freethread(handler): class resplex(handler): cname = "rplex" - def __init__(self, max=None, **kw): - super(resplex, self).__init__(**kw) + def __init__(self, *, max=None, **kw): + super().__init__(**kw) self.current = set() self.lk = threading.Lock() self.tcond = threading.Condition(self.lk) self.max = max - self.cqueue = Queue.Queue(5) + self.cqueue = queue.Queue(5) self.cnpipe = os.pipe() self.rthread = reqthread(name="Response thread", target=self.handle2) self.rthread.start() @classmethod - def parseargs(cls, max=None, **args): - ret = super(resplex, cls).parseargs(**args) + def parseargs(cls, *, max=None, **args): + ret = super().parseargs(**args) if max: ret["max"] = int(max) return ret @@ -239,7 +364,7 @@ class resplex(handler): return else: self.cqueue.put((req, respiter)) - os.write(self.cnpipe[1], " ") + os.write(self.cnpipe[1], b" ") req = None finally: with self.lk: @@ -275,7 +400,7 @@ class resplex(handler): if respiter is not None: rem = False try: - data = respiter.next() + data = next(respiter) except StopIteration: rem = True try: @@ -305,7 +430,7 @@ class resplex(handler): closereq(req) while True: - bufl = list(req for req in current.iterkeys() if req.buffer) + bufl = list(req for req in current.keys() if req.buffer) rls, wls, els = select.select([rp], bufl, [rp] + bufl) if rp in rls: ret = os.read(rp, 1024) @@ -317,7 +442,7 @@ class resplex(handler): req, respiter = self.cqueue.get(False) current[req] = respiter ckiter(req) - except Queue.Empty: + except queue.Empty: pass for req in wls: try: @@ -338,17 +463,17 @@ class resplex(handler): while True: with self.lk: if len(self.current) > 0: - th = iter(self.current).next() + th = next(iter(self.current)) else: break th.join() os.close(self.cnpipe[1]) self.rthread.join() -names = dict((cls.cname, cls) for cls in globals().itervalues() if - isinstance(cls, type) and - issubclass(cls, handler) and - hasattr(cls, "cname")) +names = {cls.cname: cls for cls in globals().values() if + isinstance(cls, type) and + issubclass(cls, handler) and + hasattr(cls, "cname")} def parsehspec(spec): if ":" not in spec: diff --git a/python3/ashd/ssi.py b/python/ashd/ssi.py similarity index 100% rename from python3/ashd/ssi.py rename to python/ashd/ssi.py diff --git a/python/ashd/util.py b/python/ashd/util.py index 0ff3878..bf32637 100644 --- a/python/ashd/util.py +++ b/python/ashd/util.py @@ -4,8 +4,8 @@ This module implements a rather convenient interface for writing ashd handlers, wrapping the low-level ashd.proto module. """ -import os, socket -import proto +import os, socket, collections +from . import proto __all__ = ["stdfork", "pchild", "respond", "serveloop"] @@ -27,7 +27,7 @@ def stdfork(argv, chinit = None): if pid == 0: try: os.dup2(csk.fileno(), 0) - for fd in xrange(3, 1024): + for fd in range(3, 1024): try: os.close(fd) except: @@ -126,22 +126,26 @@ def respond(req, body, status = ("200 OK"), ctype = "text/html"): and the `ctype' argument can be used to specify a non-HTML MIME-type. - If `body' is a Unicode object, it will be encoded as UTF-8. + If `body' is not a byte string, its string representation will be + encoded as UTF-8. For example: respond(req, "Not found", status = "404 Not Found", ctype = "text/plain") """ - if type(body) == unicode: - body = body.decode("utf-8") - if ctype[:5] == "text/" and ctype.find(';') < 0: - ctype = ctype + "; charset=utf-8" + if isinstance(body, collections.ByteString): + body = bytes(body) else: body = str(body) + body = body.encode("utf-8") + if ctype[:5] == "text/" and ctype.find(';') < 0: + ctype = ctype + "; charset=utf-8" try: - req.sk.write("HTTP/1.1 %s\n" % status) - req.sk.write("Content-Type: %s\n" % ctype) - req.sk.write("Content-Length: %i\n" % len(body)) - req.sk.write("\n") + head = "" + head += "HTTP/1.1 %s\n" % status + head += "Content-Type: %s\n" % ctype + head += "Content-Length: %i\n" % len(body) + head += "\n" + req.sk.write(head.encode("ascii")) req.sk.write(body) finally: req.close() @@ -157,7 +161,10 @@ def serveloop(handler, sock = 0): and is called once for each received request. """ while True: - req = proto.recvreq(sock) + try: + req = proto.recvreq(sock) + except InterruptedError: + continue if req is None: break try: diff --git a/python/ashd/wsgidir.py b/python/ashd/wsgidir.py index 1fc66f9..9fc3649 100644 --- a/python/ashd/wsgidir.py +++ b/python/ashd/wsgidir.py @@ -17,7 +17,7 @@ omitted (such that the name is a string with no dots), in which case the handler object is looked up from this module. By default, this module will handle files with the extensions `.wsgi' -or `.wsgi2' using the `chain' handler, which chainloads such files and +or `.wsgi3' using the `chain' handler, which chainloads such files and runs them as independent WSGI applications. See its documentation for details. @@ -33,8 +33,8 @@ argument `.fpy=my.module.foohandler' can be given to pass requests for functions, you may want to use the getmod() function in this module. """ -import sys, os, threading, types, logging, getopt -import wsgiutil +import sys, os, threading, types, logging, importlib, getopt +from . import wsgiutil __all__ = ["application", "wmain", "getmod", "cachedmod", "chain"] @@ -58,6 +58,20 @@ class cachedmod(object): self.mod = mod self.mtime = mtime +class current(object): + def __init__(self): + self.cond = threading.Condition() + self.current = True + def wait(self, timeout=None): + with self.cond: + self.cond.wait(timeout) + def uncurrent(self): + with self.cond: + self.current = False + self.cond.notify_all() + def __bool__(self): + return self.current + modcache = {} cachelock = threading.Lock() @@ -84,31 +98,30 @@ def getmod(path): about the module. See its documentation for details. """ sb = os.stat(path) - cachelock.acquire() - try: + with cachelock: if path in modcache: entry = modcache[path] else: entry = [threading.Lock(), None] modcache[path] = entry - finally: - cachelock.release() - entry[0].acquire() - try: + with entry[0]: if entry[1] is None or sb.st_mtime > entry[1].mtime: - f = open(path, "r") - try: + with open(path, "rb") as f: text = f.read() - finally: - f.close() code = compile(text, path, "exec") mod = types.ModuleType(mangle(path)) mod.__file__ = path - exec code in mod.__dict__ - entry[1] = cachedmod(mod, sb.st_mtime) + mod.__current__ = current() + try: + exec(code, mod.__dict__) + except: + mod.__current__.uncurrent() + raise + else: + if entry[1] is not None: + entry[1].mod.__current__.uncurrent() + entry[1] = cachedmod(mod, sb.st_mtime) return entry[1] - finally: - entry[0].release() def importlocal(filename): import inspect @@ -137,11 +150,10 @@ class handler(object): self.handlers = {} self.exts = {} self.addext("wsgi", "chain") - self.addext("wsgi2", "chain") + self.addext("wsgi3", "chain") def resolve(self, name): - self.lock.acquire() - try: + with self.lock: if name in self.handlers: return self.handlers[name] p = name.rfind('.') @@ -149,12 +161,10 @@ class handler(object): return globals()[name] mname = name[:p] hname = name[p + 1:] - mod = __import__(mname, fromlist = ["dummy"]) + mod = importlib.import_module(mname) ret = getattr(mod, hname) self.handlers[name] = ret return ret - finally: - self.lock.release() def addext(self, ext, handler): self.exts[ext] = self.resolve(handler) @@ -195,7 +205,7 @@ def wmain(*argv): hnd = handler() ret = hnd.handle - opts, args = getopt.getopt(argv, "V") + opts, args = getopt.getopt(argv, "-V") for o, a in opts: if o == "-V": import wsgiref.validate @@ -230,8 +240,7 @@ def chain(env, startreq): return wsgiutil.simpleerror(env, startreq, 500, "Internal Error", "Could not load WSGI handler.") entry = None if mod is not None: - mod.lock.acquire() - try: + with mod.lock: if hasattr(mod, "entry"): entry = mod.entry else: @@ -240,8 +249,6 @@ def chain(env, startreq): elif hasattr(mod.mod, "application"): entry = mod.mod.application mod.entry = entry - finally: - mod.lock.release() if entry is not None: return entry(env, startreq) return wsgiutil.simpleerror(env, startreq, 500, "Internal Error", "Invalid WSGI handler.") diff --git a/python/ashd/wsgiutil.py b/python/ashd/wsgiutil.py index 711435f..3686748 100644 --- a/python/ashd/wsgiutil.py +++ b/python/ashd/wsgiutil.py @@ -1,4 +1,4 @@ -import time +import time, sys, io def htmlquote(text): ret = "" @@ -27,6 +27,7 @@ def simpleerror(env, startreq, code, title, msg):

%s

""" % (title, title, htmlquote(msg)) + buf = buf.encode("ascii") startreq("%i %s" % (code, title), [("Content-Type", "text/html"), ("Content-Length", str(len(buf)))]) return [buf] @@ -41,3 +42,63 @@ def phttpdate(dstr): tz = int(tz[1:]) tz = (((tz / 100) * 60) + (tz % 100)) * 60 return time.mktime(time.strptime(dstr, "%a, %d %b %Y %H:%M:%S")) - tz - time.altzone + +def testenviron(uri, qs="", pi="", method=None, filename=None, host="localhost", data=None, ctype=None, head={}): + if method is None: + method = "GET" if data is None else "POST" + if ctype is None and data is not None: + ctype = "application/x-www-form-urlencoded" + ret = {} + ret["wsgi.version"] = 1, 0 + ret["SERVER_SOFTWARE"] = "ashd-test/1" + ret["GATEWAY_INTERFACE"] = "CGI/1.1" + ret["SERVER_PROTOCOL"] = "HTTP/1.1" + ret["REQUEST_METHOD"] = method + ret["wsgi.uri_encoding"] = "utf-8" + ret["SCRIPT_NAME"] = uri + ret["PATH_INFO"] = pi + ret["QUERY_STRING"] = qs + full = uri + pi + if qs: + full = full + "?" + qs + ret["REQUEST_URI"] = full + if filename is not None: + ret["SCRIPT_FILENAME"] = filename + ret["HTTP_HOST"] = ret["SERVER_NAME"] = host + ret["wsgi.url_scheme"] = "http" + ret["SERVER_ADDR"] = "127.0.0.1" + ret["SERVER_PORT"] = "80" + ret["REMOTE_ADDR"] = "127.0.0.1" + ret["REMOTE_PORT"] = "12345" + if data is not None: + ret["CONTENT_TYPE"] = ctype + ret["CONTENT_LENGTH"] = len(data) + ret["wsgi.input"] = io.BytesIO(data) + else: + ret["wsgi.input"] = io.BytesIO(b"") + ret["wsgi.errors"] = sys.stderr + ret["wsgi.multithread"] = True + ret["wsgi.multiprocess"] = False + ret["wsgi.run_once"] = False + for key, val in head.items(): + ret["HTTP_" + key.upper().replace("-", "_")] = val + return ret + +class testrequest(object): + def __init__(self): + self.wbuf = io.BytesIO() + self.headers = None + self.status = None + + def __call__(self, status, headers): + self.status = status + self.headers = headers + return self.wbuf.write + + def __repr__(self): + return "" % (self.status, + "None" if self.headers is None else ("[%i]" % len(self.headers)), + "(no data)" if len(self.wbuf.getvalue()) == 0 else "(with data)") + + def __str__(self): + return repr(self) diff --git a/python/doc/ashd-wsgi.doc b/python/doc/ashd-wsgi.doc deleted file mode 100644 index 566238c..0000000 --- a/python/doc/ashd-wsgi.doc +++ /dev/null @@ -1,189 +0,0 @@ -ashd-wsgi(1) -============ - -NAME ----- -ashd-wsgi - WSGI adapter for ashd(7) - -SYNOPSIS --------- -*ashd-wsgi* [*-hAL*] [*-m* 'PDM-SPEC'] [*-p* 'MODPATH'] [*-t* 'HANDLING-MODEL'] 'HANDLER-MODULE' ['ARGS'...] - -DESCRIPTION ------------ - -The *ashd-wsgi* handler translates *ashd*(7) requests to WSGI -requests, and passes them to a specified Python handler module. The -precise Python convention for doing so is described in the PROTOCOL -section, below. - -*ashd-wsgi* is a persistent handler, as defined in *ashd*(7). It uses -multithreaded dispatching in a single Python interpreter, which means -that WSGI applications that use it need to be thread-safe, but that -they can also share all Python data structures and global variables -between requests. More precisely, *ashd-wsgi* implements a couple of -slightly different ways to handle requests and threads, which can be -configured using the *-t* option, as described in the REQUEST HANDLING -section, below. - -The Python module that *ashd-wsgi* comes with also contains a standard -handler module, `ashd.wsgidir`, which serves individual WSGI -applications directly from the files in which they reside and as such -makes this program useful as a *dirplex*(1) handler. Please see its -Python documentation for further details. - -*ashd-wsgi* requires the `ashd.proto` and `ashd.util` modules, which -are only available for CPython. If you want to use some other Python -implementation instead, you may want to use the *scgi-wsgi*(1) program -instead, along with *callscgi*(1). - -OPTIONS -------- - -*-h*:: - - Print a brief help message to standard output and exit. - -*-A*:: - - Use the convention used by Apache's mod_wsgi module to find - the WSGI application object. See the PROTOCOL section, below, - for details. - -*-L*:: - By default, *ashd-wsgi* sets up the Python logging with a - logging format and for logging to standard error. The *-L* - option suppresses that behavior, so that any handler module - may set up logging itself. - -*-p* 'MODPATH':: - - Prepend 'MODPATH' to Python's `sys.path`; can be given multiple - times. Note that the working directory of *ashd-wsgi* is not - on Python's module path by default, so if you want to use a - module in that directory, you will need to specify "`-p .`". - -*-t* 'HANDLING-MODEL':: - - Specify the way *ashd-wsgi* handles requests. See below, under - REQUEST HANDLING. - -*-m* 'PDM-SPEC':: - - If the PDM library is installed on the system, create a - listening socket for connecting PDM clients according to - 'PDM-SPEC'. - -PROTOCOL --------- - -When starting, *ashd-wsgi* will attempt to import the module named by -'HANDLER-MODULE', look for an object named `wmain` in that module, -call that object passing the 'ARGS' (as Python strings) as positional -parameters, and use the returned object as the WSGI application -object. If the *-A* option was specified, it will look for an object -named `application` instead of `wmain`, and use that object directly -as the WSGI application object. - -When calling the WSGI application, a new thread is started for each -request, in which the WSGI application object is called (but see -below, under REQUEST HANDLING, for details). All requests run in the -same interpreter, so it is guaranteed that data structures and global -variables can be shared between requests. - -The WSGI environment is the standard CGI environment, including the -`SCRIPT_FILENAME` variable whenever the `X-Ash-File` header was -included in the request. - -REQUEST HANDLING ----------------- - -*ashd-wsgi* can be configured to handle requests in various ways, -using the *-t* command-line option. The argument to the *-t* option -takes the form 'HANDLER'[*:*'PAR'[*=*'VAL'][(*,*'PAR'[*=*'VAL'])...]], -in order to specify the handler model, along with parameters to the -same (using the same syntax as the port specifications of -*htparser*(1)). The 'HANDLER' can be any of the following: - -*free*[*:max=*'MAX-THREADS'*,timeout=*'TIMEOUT']:: - - The *free* handler, which is the default, starts a new thread - for every incoming request, which runs the whole request in - its entirety, from running the WSGI handler function to - sending the contents of the response iterator. Optionally, - 'MAX-THREADS' may be specified to an integer, in which case no - more than that many request-handler threads will be allowed to - run at any one time (by default, any number of threads are - allowed to run, without limit). If further requests come in - while 'MAX-THREADS' handlers are running, the request dispatch - thread itself will block until one exits, making new requests - queue up in the socket over which they arrive, eventually - filling up its buffers if no threads exit, in turn making the - parent handler either block or receive *EAGAIN* errors. Also, - if 'MAX-THREADS' is specified, 'TIMEOUT' may also be - specified, to tell the dispatcher thread to never block more - than so many seconds for a handler thread to exit. If it is - forced to wait longer than 'TIMEOUT' seconds, it will assume - the whole process is somehow foobar and will *abort*(3). - -*rplex*[*:max=*'MAX-THREADS']:: - - The *rplex* handler starts a new thread for every incoming - request, but unlike the *free* handler, only the WSGI handler - function runs in that thread. Whenever any such thread, then, - returns its response iterator, all such iterators will be - passed to a single independent thread which sends their - contents to the clients, multiplexing between them whenever - their respective clients are ready to receive data. Like the - *free* handler, a 'MAX-THREADS' argument may be given to - specify how many handler threads are allowed to run at the - same time. The main advantage, compared to the *free* handler, - is that the *rplex* handler allows an arbitrary number of - response iterators to run simultaneously without tying up - handler threads, therefore not counting towards 'MAX-THREADS', - which may be necessary for applications handling large - files. However, it must be noted that no response iterators in - the application may block on returning data, since that would - also block all other running responses. Also, the *rplex* - handler does not support the `write` function returned by - `start_request`, according to the WSGI specification. - -*single*:: - - The *single* handler starts no threads at all, running all - received requests directly in the main dispatch thread. It is - probably not good for much except as the simplest possible - example of a request handling model. - -EXAMPLES --------- - -The following *dirplex*(1) configuration can be used for serving WSGI -modules directly from the filesystem. - --------- -child wsgidir - exec ashd-wsgi ashd.wsgidir -match - filename *.wsgi - xset python-handler chain - handler wsgidir --------- - -Since *ashd-wsgi* is a persistent handler, it can be used directly as -a root handler for *htparser*(1). For instance, if the directory -`/srv/www/foo` contains a `wsgi.py` file, which declares a standard -WSGI `application` object, it can be served with the following -command: - --------- -htparser plain:port=8080 -- ashd-wsgi -Ap /srv/www/foo wsgi --------- - -AUTHOR ------- -Fredrik Tolf - -SEE ALSO --------- -*scgi-wsgi*(1), *ashd*(7), diff --git a/python3/doc/ashd-wsgi3.doc b/python/doc/ashd-wsgi3.doc similarity index 100% rename from python3/doc/ashd-wsgi3.doc rename to python/doc/ashd-wsgi3.doc diff --git a/python/doc/scgi-wsgi.doc b/python/doc/scgi-wsgi.doc deleted file mode 100644 index 08fc31e..0000000 --- a/python/doc/scgi-wsgi.doc +++ /dev/null @@ -1,80 +0,0 @@ -scgi-wsgi(1) -============ - -NAME ----- -scgi-wsgi - WSGI adapter for SCGI - -SYNOPSIS --------- -*scgi-wsgi* [*-hAL*] [*-m* 'PDM-SPEC'] [*-p* 'MODPATH'] [*-t* 'HANDLING-MODEL'] [*-T* \[HOST:]'PORT'] 'HANDLER-MODULE' ['ARGS'...] - -DESCRIPTION ------------ - -The *scgi-wsgi* program translates SCGI requests to WSGI requests, and -passes them to a specified Python module. It is mainly written to -emulate the behavior of *ashd-wsgi*(1), but over SCGI instead of the -native *ashd*(7) protocol, so please see its documentation for details -of Python interoperation. Unlike *ashd-wsgi* which requires CPython, -however, *scgi-wsgi* is written in pure Python using only the standard -library, and so should be usable by any Python implementation. If -using it under *ashd*(7), please see the documentation for -*callscgi*(1) as well. - -Following *callscgi*(1) conventions, *scgi-wsgi* will, by default, -accept connections on a socket passed on its standard input (a -behavior which is, obviously, not available on all Python -implementations). Use the *-T* option to listen to a TCP address -instead. - -OPTIONS -------- - -*-h*:: - - Print a brief help message to standard output and exit. - -*-A*:: - - Use the convention used by Apache's mod_wsgi module to find - the WSGI application object. See the PROTOCOL section of - *ashd-wsgi*(1) for details. - -*-L*:: - By default, *scgi-wsgi* sets up the Python logging with a - logging format and for logging to standard error. The *-L* - option suppresses that behavior, so that any handler module - may set up logging itself. - -*-p* 'MODPATH':: - - Prepend 'MODPATH' to Python's `sys.path`; can be given multiple - times. - -*-t* 'HANDLING-MODEL':: - - Specify the way *scgi-wsgi* handles requests. See the REQUEST - HANDLING section of *ashd-wsgi*(1) for details. - -*-T* \[HOST:]'PORT':: - - Instead of using a listening socket passed on standard input - to accept SCGI connections, bind a TCP socket to the 'HOST' - address listening for connections on 'PORT' instead. If 'HOST' - is not given, `localhost` is used by default. - -*-m* 'PDM-SPEC':: - - If the PDM library is installed on the system, create a - listening socket for connecting PDM clients according to - 'PDM-SPEC'. - -AUTHOR ------- -Fredrik Tolf - -SEE ALSO --------- -*ashd-wsgi*(1), *callscgi*(1), , - diff --git a/python3/doc/scgi-wsgi3.doc b/python/doc/scgi-wsgi3.doc similarity index 100% rename from python3/doc/scgi-wsgi3.doc rename to python/doc/scgi-wsgi3.doc diff --git a/python/htp.c b/python/htp.c index 2daeddf..3a091d4 100644 --- a/python/htp.c +++ b/python/htp.c @@ -47,7 +47,7 @@ static PyObject *p_recvfd(PyObject *self, PyObject *args) PyErr_SetFromErrno(PyExc_OSError); return(NULL); } - ro = Py_BuildValue("Ni", PyString_FromStringAndSize(data, dlen), ret); + ro = Py_BuildValue("Ni", PyBytes_FromStringAndSize(data, dlen), ret); free(data); return(ro); } @@ -56,18 +56,15 @@ static PyObject *p_recvfd(PyObject *self, PyObject *args) static PyObject *p_sendfd(PyObject *self, PyObject *args) { int sock, fd, ret; - PyObject *data; + Py_buffer data; - if(!PyArg_ParseTuple(args, "iiO", &sock, &fd, &data)) + if(!PyArg_ParseTuple(args, "iiy*", &sock, &fd, &data)) return(NULL); - if(!PyString_Check(data)) { - PyErr_SetString(PyExc_TypeError, "datagram must be a string"); - return(NULL); - } while(1) { Py_BEGIN_ALLOW_THREADS; - ret = sendfd(sock, fd, PyString_AsString(data), PyString_Size(data)); + ret = sendfd(sock, fd, data.buf, data.len); Py_END_ALLOW_THREADS; + PyBuffer_Release(&data); if(ret < 0) { if(errno == EINTR) { if(PyErr_CheckSignals()) @@ -87,7 +84,14 @@ static PyMethodDef methods[] = { {NULL, NULL, 0, NULL} }; -PyMODINIT_FUNC inithtlib(void) +static struct PyModuleDef module = { + PyModuleDef_HEAD_INIT, + .m_name = "htlib", + .m_size = -1, + .m_methods = methods, +}; + +PyMODINIT_FUNC PyInit_htlib(void) { - Py_InitModule("ashd.htlib", methods); + return(PyModule_Create(&module)); } diff --git a/python/htredir b/python/htredir deleted file mode 100755 index 1e03499..0000000 --- a/python/htredir +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/python - -import sys, os, getopt - -def destructurl(url): - if "://" in url: - p = url.index("://") - scheme, url = url[:p], url[p + 3] - if "/" in url: - p = url.index("/") - host, url = url[:p], url[p + 1:] - else: - host, url = url, "" - else: - scheme = None - host = None - return scheme, host, url - -def usage(out): - out.write("usage: htredir [-hp] TARGET METHOD URL REST\n") - -status = "302 Found" -opts, args = getopt.getopt(sys.argv[1:], "hp") -for o, a in opts: - if o == "-h": - usage(sys.stdout) - sys.exit(0) - elif o == "-p": - status = "301 Moved Permanently" -if len(args) != 4: - usage(sys.stderr) - sys.exit(1) -target, method, url, rest = args -scheme = os.getenv("REQ_X_ASH_PROTOCOL") -host = os.getenv("REQ_HOST") -me = url -if me[-len(rest):] == rest: - me = me[:-len(rest)] -tscheme, thost, target = destructurl(target) -if tscheme: scheme = tscheme -if thost: host = thost -if len(target) > 0 and target[0] == "/": - pass -else: - if "/" in me: - p = me.rindex("/") - target = me[:p + 1] + target -if len(target) > 0 and target[0] == "/": - target = target[1:] -if scheme and host: - target = "%s://%s/%s" % (scheme, host, target) -else: - # Illegal, but the only option (the premises are illegal anyway) - pass - -try: - sys.stdout.write("HTTP/1.1 %s\n" % status) - sys.stdout.write("Location: %s\n" % target) - sys.stdout.write("Content-Length: 0\n") - sys.stdout.write("\n") -except IOError: - sys.exit(1) diff --git a/python/scgi-wsgi b/python/scgi-wsgi deleted file mode 100755 index 99e003c..0000000 --- a/python/scgi-wsgi +++ /dev/null @@ -1,154 +0,0 @@ -#!/usr/bin/python - -import sys, os, getopt, logging, platform -import socket -import ashd.scgi, ashd.serve -try: - import pdm.srv -except: - pdm = None - -def usage(out): - out.write("usage: scgi-wsgi [-hAL] [-m PDM-SPEC] [-p MODPATH] [-t REQUEST-HANDLER[:PAR[=VAL](,PAR[=VAL])...]] [-T [HOST:]PORT] HANDLER-MODULE [ARGS...]\n") - -sk = None -hspec = "free", {} -modwsgi_compat = False -setlog = True -opts, args = getopt.getopt(sys.argv[1:], "+hALp:t:T:m:") -for o, a in opts: - if o == "-h": - usage(sys.stdout) - sys.exit(0) - elif o == "-p": - sys.path.insert(0, a) - elif o == "-L": - setlog = False - elif o == "-T": - sk = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - sk.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - p = a.rfind(":") - if p < 0: - bindhost = "localhost" - bindport = int(a) - else: - bindhost = a[:p] - bindport = int(a[p + 1:]) - sk.bind((bindhost, bindport)) - sk.listen(32) - elif o == "-A": - modwsgi_compat = True - elif o == "-m": - if pdm is not None: - pdm.srv.listen(a) - elif o == "-t": - hspec = ashd.serve.parsehspec(a) -if len(args) < 1: - usage(sys.stderr) - sys.exit(1) -if setlog: - logging.basicConfig(format="scgi-wsgi(%(name)s): %(levelname)s: %(message)s") - -if sk is None: - # This is suboptimal, since the socket on stdin is not necessarily - # AF_UNIX, but Python does not seem to offer any way around it, - # that I can find. - sk = socket.fromfd(0, socket.AF_UNIX, socket.SOCK_STREAM) - -try: - handlermod = __import__(args[0], fromlist = ["dummy"]) -except ImportError, exc: - sys.stderr.write("scgi-wsgi: handler %s not found: %s\n" % (args[0], exc.message)) - sys.exit(1) -if not modwsgi_compat: - if not hasattr(handlermod, "wmain"): - sys.stderr.write("scgi-wsgi: handler %s has no `wmain' function\n" % args[0]) - sys.exit(1) - handler = handlermod.wmain(*args[1:]) -else: - if not hasattr(handlermod, "application"): - sys.stderr.write("scgi-wsgi: handler %s has no `application' object\n" % args[0]) - sys.exit(1) - handler = handlermod.application - -def mkenv(head, sk): - env = dict(head) - env["wsgi.version"] = 1, 0 - if "HTTP_X_ASH_PROTOCOL" in env: - env["wsgi.url_scheme"] = env["HTTP_X_ASH_PROTOCOL"] - elif "HTTPS" in env: - env["wsgi.url_scheme"] = "https" - else: - env["wsgi.url_scheme"] = "http" - env["wsgi.input"] = sk - env["wsgi.errors"] = sys.stderr - env["wsgi.multithread"] = True - env["wsgi.multiprocess"] = False - env["wsgi.run_once"] = False - return env - -class request(ashd.serve.wsgirequest): - def __init__(self, sk, **kw): - super(request, self).__init__(**kw) - self.bsk = sk.dup() - self.sk = self.bsk.makefile("r+") - - def mkenv(self): - return mkenv(ashd.scgi.readhead(self.sk), self.sk) - - def handlewsgi(self, env, startreq): - return handler(env, startreq) - - _onjython = None - @staticmethod - def onjython(): - if request._onjython is None: - request._onjython = ("java" in platform.system().lower()) - return request._onjython - - def fileno(self): - if request.onjython(): - self.bsk.setblocking(False) - return self.bsk.fileno() - - def writehead(self, status, headers): - w = self.buffer.extend - w("Status: %s\n" % status) - for nm, val in headers: - w("%s: %s\n" % (nm, val)) - w("\n") - - def flush(self): - try: - if not request.onjython(): - ret = self.bsk.send(self.buffer, socket.MSG_DONTWAIT) - else: - ret = self.bsk.send(str(self.buffer)) - self.buffer[:ret] = "" - except IOError: - raise ashd.serve.closed() - - def close(self): - self.sk.close() - self.bsk.close() - -if hspec[0] not in ashd.serve.names: - sys.stderr.write("scgi-wsgi: no such request handler: %s\n" % hspec[0]) - sys.exit(1) -hclass = ashd.serve.names[hspec[0]] -try: - hargs = hclass.parseargs(**hspec[1]) -except ValueError as exc: - sys.stderr.write("scgi-wsgi: %s\n" % exc) - sys.exit(1) - -reqhandler = hclass(**hargs) -try: - while True: - nsk, addr = sk.accept() - try: - reqhandler.handle(request(sk=nsk, handler=reqhandler)) - finally: - nsk.close() -finally: - reqhandler.close() diff --git a/python3/scgi-wsgi3 b/python/scgi-wsgi3 similarity index 100% rename from python3/scgi-wsgi3 rename to python/scgi-wsgi3 diff --git a/python3/serve-ssi b/python/serve-ssi similarity index 100% rename from python3/serve-ssi rename to python/serve-ssi diff --git a/python/setup.py b/python/setup.py index 3ffdcd8..6c6e16c 100755 --- a/python/setup.py +++ b/python/setup.py @@ -1,11 +1,11 @@ -#!/usr/bin/python +#!/usr/bin/python3 from distutils.core import setup, Extension htlib = Extension("ashd.htlib", ["htp.c"], libraries = ["ht"]) -setup(name = "ashd-py", +setup(name = "ashd-py3", version = "0.6", description = "Python module for handling ashd requests", author = "Fredrik Tolf", @@ -13,5 +13,5 @@ setup(name = "ashd-py", url = "http://www.dolda2000.com/~fredrik/ashd/", ext_modules = [htlib], packages = ["ashd"], - scripts = ["ashd-wsgi", "scgi-wsgi", "htredir"], + scripts = ["ashd-wsgi3", "scgi-wsgi3", "serve-ssi"], license = "GPL-3") diff --git a/python3/.gitignore b/python3/.gitignore deleted file mode 100644 index 21e5002..0000000 --- a/python3/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -*.pyc -/build -/ashd/htlib.so diff --git a/python3/ashd/__init__.py b/python3/ashd/__init__.py deleted file mode 100644 index eea633c..0000000 --- a/python3/ashd/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Base module for ashd(7)-related functions. - -This module implements nothing. Please see the ashd.util or ashd.proto -modules. -""" diff --git a/python3/ashd/perf.py b/python3/ashd/perf.py deleted file mode 100644 index 4adfc54..0000000 --- a/python3/ashd/perf.py +++ /dev/null @@ -1,79 +0,0 @@ -import collections.abc -try: - import pdm.perf -except: - pdm = None -try: - import time - clock_thread = time.CLOCK_THREAD_CPUTIME_ID -except: - clock_thread = None - -reqstat = {} - -if pdm: - statistics = pdm.perf.staticdir() - statistics["req"] = pdm.perf.valueattr(reqstat) - requests = pdm.perf.eventobj() - - class reqstart(pdm.perf.startevent): - def __init__(self, env): - super().__init__() - self.method = env.get("REQUEST_METHOD") - self.uri = env.get("REQUEST_URI") - self.host = env.get("HTTP_HOST") - self.script_uri = env.get("SCRIPT_NAME") - self.script_path = env.get("SCRIPT_FILENAME") - self.pathinfo = env.get("PATH_INFO") - self.querystring = env.get("QUERY_STRING") - self.remoteaddr = env.get("REMOTE_ADDR") - self.remoteport = env.get("REMOTE_PORT") - self.scheme = env.get("wsgi.url_scheme") - if clock_thread is not None: - self.icpu = time.clock_gettime(clock_thread) - - class reqfinish(pdm.perf.finishevent): - def __init__(self, start, aborted, status): - super().__init__(start, aborted) - self.status = status - self.cputime = 0 - if clock_thread is not None: - self.cputime = time.clock_gettime(clock_thread) - start.icpu - -class request(object): - def __init__(self, env): - self.resp = None - if pdm: - self.startev = reqstart(env) - requests.notify(self.startev) - - def response(self, resp): - self.resp = resp - - def finish(self, aborted): - key = None - status = None - try: - if len(self.resp) > 0: - status = self.resp[0] - if isinstance(status, collections.abc.ByteString): - status = status.decode("latin-1") - else: - status = str(status) - p = status.find(" ") - if p < 0: - key = status - else: - key = status[:p] - except: - pass - reqstat[key] = reqstat.setdefault(key, 0) + 1 - if pdm: - requests.notify(reqfinish(self.startev, aborted, status)) - - def __enter__(self): - return self - - def __exit__(self, *excinfo): - self.finish(bool(excinfo[0])) - return False diff --git a/python3/ashd/proto.py b/python3/ashd/proto.py deleted file mode 100644 index aa6b686..0000000 --- a/python3/ashd/proto.py +++ /dev/null @@ -1,182 +0,0 @@ -"""Low-level protocol module for ashd(7) - -This module provides primitive functions that speak the raw ashd(7) -protocol. Primarily, it implements the `req' class that is used to -represent ashd requests. The functions it provides can also be used to -create ashd handlers, but unless you require very precise control, the -ashd.util module provides an easier-to-use interface. -""" - -import os, socket -from . import htlib - -__all__ = ["req", "recvreq", "sendreq"] - -class protoerr(Exception): - pass - -class req(object): - """Represents a single ashd request. Normally, you would not - create instances of this class manually, but receive them from the - recvreq function. - - For the abstract structure of ashd requests, please see the - ashd(7) manual page. This class provides access to the HTTP - method, raw URL, HTTP version and rest string via the `method', - `url', `ver' and `rest' variables respectively. It also implements - a dict-like interface for case-independent access to the HTTP - headers. The raw headers are available as a list of (name, value) - tuples in the `headers' variable. - - For responding, the response socket is available as a standard - Python stream object in the `sk' variable. Again, see the ashd(7) - manpage for what to receive and transmit on the response socket. - - Note that all request parts are stored in byte, rather than - string, form. The response socket is also opened in binary mode. - - Note also that instances of this class contain a reference to the - live socket used for responding to requests, which should be - closed when you are done with the request. The socket can be - closed manually by calling the close() method on this - object. Alternatively, this class implements the resource-manager - interface, so that it can be used in `with' statements. - """ - - def __init__(self, method, url, ver, rest, headers, fd): - self.method = method - self.url = url - self.ver = ver - self.rest = rest - self.headers = headers - self.bsk = socket.fromfd(fd, socket.AF_UNIX, socket.SOCK_STREAM) - self.sk = self.bsk.makefile('rwb') - os.close(fd) - - def close(self): - "Close this request's response socket." - self.sk.close() - self.bsk.close() - - def __getitem__(self, header): - """Find a HTTP header case-insensitively. For example, - req["Content-Type"] returns the value of the content-type - header regardlessly of whether the client specified it as - "Content-Type", "content-type" or "Content-type". - - If the header is given as a (Unicode) string, it is encoded - into Ascii for use in matching. - """ - if isinstance(header, str): - header = header.encode("ascii") - header = header.lower() - for key, val in self.headers: - if key.lower() == header: - return val - raise KeyError(header) - - def __contains__(self, header): - """Works analogously to the __getitem__ method for checking - header presence case-insensitively. - """ - if isinstance(header, str): - header = header.encode("ascii") - header = header.lower() - for key, val in self.headers: - if key.lower() == header: - return True - return False - - def dup(self): - """Creates a duplicate of this request, referring to a - duplicate of the response socket. - """ - return req(self.method, self.url, self.ver, self.rest, self.headers, os.dup(self.bsk.fileno())) - - def match(self, match): - """If the `match' argument matches exactly the leading part of - the rest string, this method strips that part of the rest - string off and returns True. Otherwise, it returns False - without doing anything. - - If the `match' argument is given as a (Unicode) string, it is - encoded into UTF-8. - - This can be used for simple dispatching. For example: - if req.match("foo/"): - handle(req) - elif req.match("bar/"): - handle_otherwise(req) - else: - util.respond(req, "Not found", status = "404 Not Found", ctype = "text/plain") - """ - if isinstance(match, str): - match = match.encode("utf-8") - if self.rest[:len(match)] == match: - self.rest = self.rest[len(match):] - return True - return False - - def __str__(self): - def dec(b): - return b.decode("ascii", errors="replace") - return "\"%s %s %s\"" % (dec(self.method), dec(self.url), dec(self.ver)) - - def __enter__(self): - return self - - def __exit__(self, *excinfo): - self.sk.close() - return False - -def recvreq(sock = 0): - """Receive a single ashd request on the specified socket file - descriptor (or standard input if unspecified). - - The returned value is an instance of the `req' class. As per its - description, care should be taken to close() the request when - done, to avoid leaking response sockets. If end-of-file is - received on the socket, None is returned. - - This function may either raise an OSError if an error occurs on - the socket, or an ashd.proto.protoerr if the incoming request is - invalidly encoded. - """ - data, fd = htlib.recvfd(sock) - if fd is None: - return None - try: - parts = data.split(b'\0')[:-1] - if len(parts) < 5: - raise protoerr("Truncated request") - method, url, ver, rest = parts[:4] - headers = [] - i = 4 - while True: - if parts[i] == b"": break - if len(parts) - i < 3: - raise protoerr("Truncated request") - headers.append((parts[i], parts[i + 1])) - i += 2 - return req(method, url, ver, rest, headers, os.dup(fd)) - finally: - os.close(fd) - -def sendreq(sock, req): - """Encode and send a single request to the specified socket file - descriptor using the ashd protocol. The request should be an - instance of the `req' class. - - This function may raise an OSError if an error occurs on the - socket. - """ - data = b"" - data += req.method + b'\0' - data += req.url + b'\0' - data += req.ver + b'\0' - data += req.rest + b'\0' - for key, val in req.headers: - data += key + b'\0' - data += val + b'\0' - data += b'\0' - htlib.sendfd(sock, req.bsk.fileno(), data) diff --git a/python3/ashd/scgi.py b/python3/ashd/scgi.py deleted file mode 100644 index c00c5a3..0000000 --- a/python3/ashd/scgi.py +++ /dev/null @@ -1,31 +0,0 @@ -class protoerr(Exception): - pass - -def readns(sk): - hln = 0 - while True: - c = sk.read(1) - if c == b':': - break - elif c >= b'0' or c <= b'9': - hln = (hln * 10) + (ord(c) - ord(b'0')) - else: - raise protoerr("Invalid netstring length byte: " + c) - ret = sk.read(hln) - if sk.read(1) != b',': - raise protoerr("Non-terminated netstring") - return ret - -def readhead(sk): - parts = readns(sk).split(b'\0')[:-1] - if len(parts) % 2 != 0: - raise protoerr("Malformed headers") - ret = {} - i = 0 - while i < len(parts): - ret[parts[i]] = parts[i + 1] - i += 2 - return ret - -def decodehead(head, coding): - return {k.decode(coding): v.decode(coding) for k, v in head.items()} diff --git a/python3/ashd/serve.py b/python3/ashd/serve.py deleted file mode 100644 index 0927710..0000000 --- a/python3/ashd/serve.py +++ /dev/null @@ -1,493 +0,0 @@ -import sys, os, threading, time, logging, select, queue, collections -from . import perf - -log = logging.getLogger("ashd.serve") -seq = 1 -seqlk = threading.Lock() - -def reqseq(): - global seq - with seqlk: - s = seq - seq += 1 - return s - -class closed(IOError): - def __init__(self): - super().__init__("The client has closed the connection.") - -class reqthread(threading.Thread): - def __init__(self, *, name=None, **kw): - if name is None: - name = "Request handler %i" % reqseq() - super().__init__(name=name, **kw) - -class wsgirequest(object): - def __init__(self, *, handler): - self.status = None - self.headers = [] - self.respsent = False - self.handler = handler - self.buffer = bytearray() - - def handlewsgi(self): - raise Exception() - def fileno(self): - raise Exception() - def writehead(self, status, headers): - raise Exception() - def flush(self): - raise Exception() - def close(self): - pass - def writedata(self, data): - self.buffer.extend(data) - - def flushreq(self): - if not self.respsent: - if not self.status: - raise Exception("Cannot send response body before starting response.") - self.respsent = True - self.writehead(self.status, self.headers) - - def write(self, data): - if not data: - return - self.flushreq() - self.writedata(data) - self.handler.ckflush(self) - - def startreq(self, status, headers, exc_info=None): - if self.status: - if exc_info: - try: - if self.respsent: - raise exc_info[1] - finally: - exc_info = None - else: - raise Exception("Can only start responding once.") - self.status = status - self.headers = headers - return self.write - -class handler(object): - def handle(self, request): - raise Exception() - def ckflush(self, req): - p = select.poll() - p.register(req, select.POLLOUT) - while len(req.buffer) > 0: - p.poll() - req.flush() - def close(self): - pass - - @classmethod - def parseargs(cls, **args): - if len(args) > 0: - raise ValueError("unknown handler argument: " + next(iter(args))) - return {} - -class single(handler): - cname = "single" - - def handle(self, req): - try: - env = req.mkenv() - with perf.request(env) as reqevent: - respiter = req.handlewsgi(env, req.startreq) - for data in respiter: - req.write(data) - if req.status: - reqevent.response([req.status, req.headers]) - req.flushreq() - self.ckflush(req) - except closed: - pass - except: - log.error("exception occurred when handling request", exc_info=True) - finally: - req.close() - -def dbg(*a): - f = True - for o in a: - if not f: - sys.stderr.write(" ") - sys.stderr.write(str(a)) - f = False - sys.stderr.write("\n") - sys.stderr.flush() - -class freethread(handler): - cname = "free" - - def __init__(self, *, max=None, timeout=None, **kw): - super().__init__(**kw) - self.current = set() - self.lk = threading.Lock() - self.tcond = threading.Condition(self.lk) - self.max = max - self.timeout = timeout - - @classmethod - def parseargs(cls, *, max=None, abort=None, **args): - ret = super().parseargs(**args) - if max: - ret["max"] = int(max) - if abort: - ret["timeout"] = int(abort) - return ret - - def handle(self, req): - with self.lk: - if self.max is not None: - if self.timeout is not None: - now = start = time.time() - while len(self.current) >= self.max: - self.tcond.wait(start + self.timeout - now) - now = time.time() - if now - start > self.timeout: - os.abort() - else: - while len(self.current) >= self.max: - self.tcond.wait() - th = reqthread(target=self.run, args=[req]) - th.registered = False - th.start() - while not th.registered: - self.tcond.wait() - - def run(self, req): - try: - th = threading.current_thread() - with self.lk: - self.current.add(th) - th.registered = True - self.tcond.notify_all() - try: - env = req.mkenv() - with perf.request(env) as reqevent: - respiter = req.handlewsgi(env, req.startreq) - for data in respiter: - req.write(data) - if req.status: - reqevent.response([req.status, req.headers]) - req.flushreq() - self.ckflush(req) - except closed: - pass - except: - log.error("exception occurred when handling request", exc_info=True) - finally: - with self.lk: - self.current.remove(th) - self.tcond.notify_all() - finally: - req.close() - - def close(self): - while True: - with self.lk: - if len(self.current) > 0: - th = next(iter(self.current)) - else: - return - th.join() - -class threadpool(handler): - cname = "pool" - - def __init__(self, *, max=25, qsz=100, timeout=None, **kw): - super().__init__(**kw) - self.current = set() - self.clk = threading.Lock() - self.ccond = threading.Condition(self.clk) - self.queue = collections.deque() - self.waiting = set() - self.waitlimit = 5 - self.wlstart = 0.0 - self.qlk = threading.Lock() - self.qfcond = threading.Condition(self.qlk) - self.qecond = threading.Condition(self.qlk) - self.max = max - self.qsz = qsz - self.timeout = timeout - - @classmethod - def parseargs(cls, *, max=None, queue=None, abort=None, **args): - ret = super().parseargs(**args) - if max: - ret["max"] = int(max) - if queue: - ret["qsz"] = int(queue) - if abort: - ret["timeout"] = int(abort) - return ret - - def handle(self, req): - spawn = False - with self.qlk: - if self.timeout is not None: - now = start = time.time() - while len(self.queue) >= self.qsz: - self.qecond.wait(start + self.timeout - now) - now = time.time() - if now - start > self.timeout: - os.abort() - else: - while len(self.queue) >= self.qsz: - self.qecond.wait() - self.queue.append(req) - self.qfcond.notify() - if len(self.waiting) < 1: - spawn = True - if spawn: - with self.clk: - if len(self.current) < self.max: - th = reqthread(target=self.run) - th.registered = False - th.start() - while not th.registered: - self.ccond.wait() - - def handle1(self, req): - try: - env = req.mkenv() - with perf.request(env) as reqevent: - respiter = req.handlewsgi(env, req.startreq) - for data in respiter: - req.write(data) - if req.status: - reqevent.response([req.status, req.headers]) - req.flushreq() - self.ckflush(req) - except closed: - pass - except: - log.error("exception occurred when handling request", exc_info=True) - - def run(self): - timeout = 10.0 - th = threading.current_thread() - with self.clk: - self.current.add(th) - th.registered = True - self.ccond.notify_all() - try: - while True: - start = now = time.time() - with self.qlk: - while len(self.queue) < 1: - if len(self.waiting) >= self.waitlimit and now - self.wlstart >= timeout: - return - self.waiting.add(th) - try: - if len(self.waiting) == self.waitlimit: - self.wlstart = now - self.qfcond.wait(start + timeout - now) - finally: - self.waiting.remove(th) - now = time.time() - if now - start > timeout: - return - req = self.queue.popleft() - self.qecond.notify() - try: - self.handle1(req) - finally: - req.close() - finally: - with self.clk: - self.current.remove(th) - - def close(self): - while True: - with self.clk: - if len(self.current) > 0: - th = next(iter(self.current)) - else: - return - th.join() - -class resplex(handler): - cname = "rplex" - - def __init__(self, *, max=None, **kw): - super().__init__(**kw) - self.current = set() - self.lk = threading.Lock() - self.tcond = threading.Condition(self.lk) - self.max = max - self.cqueue = queue.Queue(5) - self.cnpipe = os.pipe() - self.rthread = reqthread(name="Response thread", target=self.handle2) - self.rthread.start() - - @classmethod - def parseargs(cls, *, max=None, **args): - ret = super().parseargs(**args) - if max: - ret["max"] = int(max) - return ret - - def ckflush(self, req): - raise Exception("resplex handler does not support the write() function") - - def handle(self, req): - with self.lk: - if self.max is not None: - while len(self.current) >= self.max: - self.tcond.wait() - th = reqthread(target=self.handle1, args=[req]) - th.registered = False - th.start() - while not th.registered: - self.tcond.wait() - - def handle1(self, req): - try: - th = threading.current_thread() - with self.lk: - self.current.add(th) - th.registered = True - self.tcond.notify_all() - try: - env = req.mkenv() - respobj = req.handlewsgi(env, req.startreq) - respiter = iter(respobj) - if not req.status: - log.error("request handler returned without calling start_request") - if hasattr(respiter, "close"): - respiter.close() - return - else: - self.cqueue.put((req, respiter)) - os.write(self.cnpipe[1], b" ") - req = None - finally: - with self.lk: - self.current.remove(th) - self.tcond.notify_all() - except closed: - pass - except: - log.error("exception occurred when handling request", exc_info=True) - finally: - if req is not None: - req.close() - - def handle2(self): - try: - rp = self.cnpipe[0] - current = {} - - def closereq(req): - respiter = current[req] - try: - if respiter is not None and hasattr(respiter, "close"): - respiter.close() - except: - log.error("exception occurred when closing iterator", exc_info=True) - try: - req.close() - except: - log.error("exception occurred when closing request", exc_info=True) - del current[req] - def ckiter(req): - respiter = current[req] - if respiter is not None: - rem = False - try: - data = next(respiter) - except StopIteration: - rem = True - try: - req.flushreq() - except: - log.error("exception occurred when handling response data", exc_info=True) - except: - rem = True - log.error("exception occurred when iterating response", exc_info=True) - if not rem: - if data: - try: - req.flushreq() - req.writedata(data) - except: - log.error("exception occurred when handling response data", exc_info=True) - rem = True - if rem: - current[req] = None - try: - if hasattr(respiter, "close"): - respiter.close() - except: - log.error("exception occurred when closing iterator", exc_info=True) - respiter = None - if respiter is None and not req.buffer: - closereq(req) - - while True: - bufl = list(req for req in current.keys() if req.buffer) - rls, wls, els = select.select([rp], bufl, [rp] + bufl) - if rp in rls: - ret = os.read(rp, 1024) - if not ret: - os.close(rp) - return - try: - while True: - req, respiter = self.cqueue.get(False) - current[req] = respiter - ckiter(req) - except queue.Empty: - pass - for req in wls: - try: - req.flush() - except closed: - closereq(req) - except: - log.error("exception occurred when writing response", exc_info=True) - closereq(req) - else: - if len(req.buffer) < 65536: - ckiter(req) - except: - log.critical("unexpected exception occurred in response handler thread", exc_info=True) - os.abort() - - def close(self): - while True: - with self.lk: - if len(self.current) > 0: - th = next(iter(self.current)) - else: - break - th.join() - os.close(self.cnpipe[1]) - self.rthread.join() - -names = {cls.cname: cls for cls in globals().values() if - isinstance(cls, type) and - issubclass(cls, handler) and - hasattr(cls, "cname")} - -def parsehspec(spec): - if ":" not in spec: - return spec, {} - nm, spec = spec.split(":", 1) - args = {} - while spec: - if "," in spec: - part, spec = spec.split(",", 1) - else: - part, spec = spec, None - if "=" in part: - key, val = part.split("=", 1) - else: - key, val = part, "" - args[key] = val - return nm, args diff --git a/python3/ashd/util.py b/python3/ashd/util.py deleted file mode 100644 index bf32637..0000000 --- a/python3/ashd/util.py +++ /dev/null @@ -1,173 +0,0 @@ -"""High-level utility module for ashd(7) - -This module implements a rather convenient interface for writing ashd -handlers, wrapping the low-level ashd.proto module. -""" - -import os, socket, collections -from . import proto - -__all__ = ["stdfork", "pchild", "respond", "serveloop"] - -def stdfork(argv, chinit = None): - """Fork a persistent handler process using the `argv' argument - list, as per the standard ashd(7) calling convention. For an - easier-to-use interface, see the `pchild' class. - - If a callable object of no arguments is provided in the `chinit' - argument, it will be called in the child process before exec()'ing - the handler program, and can be used to set parameters for the new - process, such as working directory, nice level or ulimits. - - Returns the file descriptor of the socket for sending requests to - the new child. - """ - csk, psk = socket.socketpair(socket.AF_UNIX, socket.SOCK_SEQPACKET) - pid = os.fork() - if pid == 0: - try: - os.dup2(csk.fileno(), 0) - for fd in range(3, 1024): - try: - os.close(fd) - except: - pass - if chinit is not None: - chinit() - os.execvp(argv[0], argv) - finally: - os._exit(127) - csk.close() - fd = os.dup(psk.fileno()) - psk.close() - return fd - -class pchild(object): - """class pchild(argv, autorespawn=False, chinit=None) - - Represents a persistent child handler process, started as per the - standard ashd(7) calling convention. It will be called with the - `argv' argument lest, which should be a list (or other iterable) - of strings. - - If `autorespawn' is specified as True, the child process will be - automatically restarted if a request cannot be successfully sent - to it. - - For a description of the `chinit' argument, see `stdfork'. - - When this child handler should be disposed of, care should be - taken to call the close() method to release its socket and let it - exit. This class also implements the resource-manager interface, - so that it can be used in `with' statements. - """ - - def __init__(self, argv, autorespawn = False, chinit = None): - self.argv = argv - self.chinit = chinit - self.fd = -1 - self.respawn = autorespawn - self.spawn() - - def spawn(self): - """Start the child handler, or restart it if it is already - running. You should not have to call this method manually - unless you explicitly want to manage the process' lifecycle. - """ - self.close() - self.fd = stdfork(self.argv, self.chinit) - - def close(self): - """Close this child handler's socket. For normal child - handlers, this will make the program terminate normally. - """ - if self.fd >= 0: - os.close(self.fd) - self.fd = -1 - - def __del__(self): - self.close() - - def passreq(self, req): - """Pass the specified request (which should be an instance of - the ashd.proto.req class) to this child handler. If the child - handler fails for some reason, and `autorespawn' was specified - as True when creating this handler, one attempt will be made - to restart it. - - Note: You still need to close the request normally. - - This method may raise an OSError if the request fails and - autorespawning was either not requested, or if the - autorespawning fails. - """ - try: - proto.sendreq(self.fd, req) - except OSError: - if self.respawn: - self.spawn() - proto.sendreq(self.fd, req) - - def __enter__(self): - return self - - def __exit__(self, *excinfo): - self.close() - return False - -def respond(req, body, status = ("200 OK"), ctype = "text/html"): - """Simple function for conveniently responding to a request. - - Sends the specified body text to the request's response socket, - prepending an HTTP header with the appropriate Content-Type and - Content-Length headers, and then closes the response socket. - - The `status' argument can be used to specify a non-200 response, - and the `ctype' argument can be used to specify a non-HTML - MIME-type. - - If `body' is not a byte string, its string representation will be - encoded as UTF-8. - - For example: - respond(req, "Not found", status = "404 Not Found", ctype = "text/plain") - """ - if isinstance(body, collections.ByteString): - body = bytes(body) - else: - body = str(body) - body = body.encode("utf-8") - if ctype[:5] == "text/" and ctype.find(';') < 0: - ctype = ctype + "; charset=utf-8" - try: - head = "" - head += "HTTP/1.1 %s\n" % status - head += "Content-Type: %s\n" % ctype - head += "Content-Length: %i\n" % len(body) - head += "\n" - req.sk.write(head.encode("ascii")) - req.sk.write(body) - finally: - req.close() - -def serveloop(handler, sock = 0): - """Implements a simple loop for serving requests sequentially, by - receiving requests from standard input (or the specified socket), - passing them to the specified handler function, and finally making - sure to close them. Returns when end-of-file is received on the - incoming socket. - - The handler function should be a callable object of one argument, - and is called once for each received request. - """ - while True: - try: - req = proto.recvreq(sock) - except InterruptedError: - continue - if req is None: - break - try: - handler(req) - finally: - req.close() diff --git a/python3/ashd/wsgidir.py b/python3/ashd/wsgidir.py deleted file mode 100644 index 9fc3649..0000000 --- a/python3/ashd/wsgidir.py +++ /dev/null @@ -1,256 +0,0 @@ -"""WSGI handler for serving chained WSGI modules from physical files - -The WSGI handler in this module ensures that the SCRIPT_FILENAME -variable is properly set in every request and points out a file that -exists and is readable. It then dispatches the request in one of two -ways: If the header X-Ash-Python-Handler is set in the request, its -value is used as the name of a handler object to dispatch the request -to; otherwise, the file extension of the SCRIPT_FILENAME is used to -determine the handler object. - -The name of a handler object is specified as a string, which is split -along its last constituent dot. The part left of the dot is the name -of a module, which is imported; and the part right of the dot is the -name of an object in that module, which should be a callable adhering -to the WSGI specification. Alternatively, the module part may be -omitted (such that the name is a string with no dots), in which case -the handler object is looked up from this module. - -By default, this module will handle files with the extensions `.wsgi' -or `.wsgi3' using the `chain' handler, which chainloads such files and -runs them as independent WSGI applications. See its documentation for -details. - -This module itself contains both an `application' and a `wmain' -object. If this module is used by ashd-wsgi(1) or scgi-wsgi(1) so that -its wmain function is called, arguments can be specified to it to -install handlers for other file extensions. Such arguments take the -form `.EXT=HANDLER', where EXT is the file extension to be handled, -and HANDLER is a handler name, as described above. For example, the -argument `.fpy=my.module.foohandler' can be given to pass requests for -`.fpy' files to the function `foohandler' in the module `my.module' -(which must, of course, be importable). When writing such handler -functions, you may want to use the getmod() function in this module. -""" - -import sys, os, threading, types, logging, importlib, getopt -from . import wsgiutil - -__all__ = ["application", "wmain", "getmod", "cachedmod", "chain"] - -log = logging.getLogger("wsgidir") - -class cachedmod(object): - """Cache entry for modules loaded by getmod() - - Instances of this class are returned by the getmod() - function. They contain three data attributes: - * mod - The loaded module - * lock - A threading.Lock object, which can be used for - manipulating this instance in a thread-safe manner - * mtime - The time the file was last modified - - Additional data attributes can be arbitrarily added for recording - any meta-data about the module. - """ - def __init__(self, mod = None, mtime = -1): - self.lock = threading.Lock() - self.mod = mod - self.mtime = mtime - -class current(object): - def __init__(self): - self.cond = threading.Condition() - self.current = True - def wait(self, timeout=None): - with self.cond: - self.cond.wait(timeout) - def uncurrent(self): - with self.cond: - self.current = False - self.cond.notify_all() - def __bool__(self): - return self.current - -modcache = {} -cachelock = threading.Lock() - -def mangle(path): - ret = "" - for c in path: - if c.isalnum(): - ret += c - else: - ret += "_" - return ret - -def getmod(path): - """Load the given file as a module, caching it appropriately - - The given file is loaded and compiled into a Python module. The - compiled module is cached and returned upon subsequent requests - for the same file, unless the file has changed (as determined by - its mtime), in which case the cached module is discarded and the - new file contents are reloaded in its place. - - The return value is an instance of the cachedmod class, which can - be used for locking purposes and for storing arbitrary meta-data - about the module. See its documentation for details. - """ - sb = os.stat(path) - with cachelock: - if path in modcache: - entry = modcache[path] - else: - entry = [threading.Lock(), None] - modcache[path] = entry - with entry[0]: - if entry[1] is None or sb.st_mtime > entry[1].mtime: - with open(path, "rb") as f: - text = f.read() - code = compile(text, path, "exec") - mod = types.ModuleType(mangle(path)) - mod.__file__ = path - mod.__current__ = current() - try: - exec(code, mod.__dict__) - except: - mod.__current__.uncurrent() - raise - else: - if entry[1] is not None: - entry[1].mod.__current__.uncurrent() - entry[1] = cachedmod(mod, sb.st_mtime) - return entry[1] - -def importlocal(filename): - import inspect - cf = inspect.currentframe() - if cf is None: raise ImportError("could not get current frame") - if cf.f_back is None: raise ImportError("could not get caller frame") - cfile = cf.f_back.f_code.co_filename - if not os.path.exists(cfile): - raise ImportError("caller is not in a proper file") - path = os.path.realpath(os.path.join(os.path.dirname(cfile), filename)) - if '.' not in os.path.basename(path): - for ext in [".pyl", ".py"]: - if os.path.exists(path + ext): - path += ext - break - else: - raise ImportError("could not resolve file: " + filename) - else: - if not os.path.exists(cfile): - raise ImportError("no such file: " + filename) - return getmod(path).mod - -class handler(object): - def __init__(self): - self.lock = threading.Lock() - self.handlers = {} - self.exts = {} - self.addext("wsgi", "chain") - self.addext("wsgi3", "chain") - - def resolve(self, name): - with self.lock: - if name in self.handlers: - return self.handlers[name] - p = name.rfind('.') - if p < 0: - return globals()[name] - mname = name[:p] - hname = name[p + 1:] - mod = importlib.import_module(mname) - ret = getattr(mod, hname) - self.handlers[name] = ret - return ret - - def addext(self, ext, handler): - self.exts[ext] = self.resolve(handler) - - def handle(self, env, startreq): - if not "SCRIPT_FILENAME" in env: - log.error("wsgidir called without SCRIPT_FILENAME set") - return wsgiutil.simpleerror(env, startreq, 500, "Internal Error", "The server is erroneously configured.") - path = env["SCRIPT_FILENAME"] - if not os.access(path, os.R_OK): - log.error("%s: not readable" % path) - return wsgiutil.simpleerror(env, startreq, 500, "Internal Error", "The server is erroneously configured.") - if "HTTP_X_ASH_PYTHON_HANDLER" in env: - try: - handler = self.resolve(env["HTTP_X_ASH_PYTHON_HANDLER"]) - except Exception: - log.error("could not load handler %s" % env["HTTP_X_ASH_PYTHON_HANDLER"], exc_info=sys.exc_info()) - return wsgiutil.simpleerror(env, startreq, 500, "Internal Error", "The server is erroneously configured.") - else: - base = os.path.basename(path) - p = base.rfind('.') - if p < 0: - log.error("wsgidir called with neither X-Ash-Python-Handler nor a file extension: %s" % path) - return wsgiutil.simpleerror(env, startreq, 500, "Internal Error", "The server is erroneously configured.") - ext = base[p + 1:] - if not ext in self.exts: - log.error("unregistered file extension: %s" % ext) - return wsgiutil.simpleerror(env, startreq, 500, "Internal Error", "The server is erroneously configured.") - handler = self.exts[ext] - return handler(env, startreq) - -def wmain(*argv): - """Main function for ashd(7)-compatible WSGI handlers - - Returns the `application' function. If any arguments are given, - they are parsed according to the module documentation. - """ - hnd = handler() - ret = hnd.handle - - opts, args = getopt.getopt(argv, "-V") - for o, a in opts: - if o == "-V": - import wsgiref.validate - ret = wsgiref.validate.validator(ret) - - for arg in args: - if arg[0] == '.': - p = arg.index('=') - hnd.addext(arg[1:p], arg[p + 1:]) - return ret - -def chain(env, startreq): - """Chain-loading WSGI handler - - This handler loads requested files, compiles them and loads them - into their own modules. The compiled modules are cached and reused - until the file is modified, in which case the previous module is - discarded and the new file contents are loaded into a new module - in its place. When chaining such modules, an object named `wmain' - is first looked for and called with no arguments if found. The - object it returns is then used as the WSGI application object for - that module, which is reused until the module is reloaded. If - `wmain' is not found, an object named `application' is looked for - instead. If found, it is used directly as the WSGI application - object. - """ - path = env["SCRIPT_FILENAME"] - try: - mod = getmod(path) - except Exception: - log.error("Exception occurred when loading %s" % path, exc_info=sys.exc_info()) - return wsgiutil.simpleerror(env, startreq, 500, "Internal Error", "Could not load WSGI handler.") - entry = None - if mod is not None: - with mod.lock: - if hasattr(mod, "entry"): - entry = mod.entry - else: - if hasattr(mod.mod, "wmain"): - entry = mod.mod.wmain() - elif hasattr(mod.mod, "application"): - entry = mod.mod.application - mod.entry = entry - if entry is not None: - return entry(env, startreq) - return wsgiutil.simpleerror(env, startreq, 500, "Internal Error", "Invalid WSGI handler.") - -application = handler().handle diff --git a/python3/ashd/wsgiutil.py b/python3/ashd/wsgiutil.py deleted file mode 100644 index 3686748..0000000 --- a/python3/ashd/wsgiutil.py +++ /dev/null @@ -1,104 +0,0 @@ -import time, sys, io - -def htmlquote(text): - ret = "" - for c in text: - if c == '&': - ret += "&" - elif c == '<': - ret += "<" - elif c == '>': - ret += ">" - elif c == '"': - ret += """ - else: - ret += c - return ret - -def simpleerror(env, startreq, code, title, msg): - buf = """ - - - -%s - - -

%s

-

%s

- -""" % (title, title, htmlquote(msg)) - buf = buf.encode("ascii") - startreq("%i %s" % (code, title), [("Content-Type", "text/html"), ("Content-Length", str(len(buf)))]) - return [buf] - -def httpdate(ts): - return time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime(ts)) - -def phttpdate(dstr): - tz = dstr[-6:] - dstr = dstr[:-6] - if tz[0] != " " or (tz[1] != "+" and tz[1] != "-") or not tz[2:].isdigit(): - return None - tz = int(tz[1:]) - tz = (((tz / 100) * 60) + (tz % 100)) * 60 - return time.mktime(time.strptime(dstr, "%a, %d %b %Y %H:%M:%S")) - tz - time.altzone - -def testenviron(uri, qs="", pi="", method=None, filename=None, host="localhost", data=None, ctype=None, head={}): - if method is None: - method = "GET" if data is None else "POST" - if ctype is None and data is not None: - ctype = "application/x-www-form-urlencoded" - ret = {} - ret["wsgi.version"] = 1, 0 - ret["SERVER_SOFTWARE"] = "ashd-test/1" - ret["GATEWAY_INTERFACE"] = "CGI/1.1" - ret["SERVER_PROTOCOL"] = "HTTP/1.1" - ret["REQUEST_METHOD"] = method - ret["wsgi.uri_encoding"] = "utf-8" - ret["SCRIPT_NAME"] = uri - ret["PATH_INFO"] = pi - ret["QUERY_STRING"] = qs - full = uri + pi - if qs: - full = full + "?" + qs - ret["REQUEST_URI"] = full - if filename is not None: - ret["SCRIPT_FILENAME"] = filename - ret["HTTP_HOST"] = ret["SERVER_NAME"] = host - ret["wsgi.url_scheme"] = "http" - ret["SERVER_ADDR"] = "127.0.0.1" - ret["SERVER_PORT"] = "80" - ret["REMOTE_ADDR"] = "127.0.0.1" - ret["REMOTE_PORT"] = "12345" - if data is not None: - ret["CONTENT_TYPE"] = ctype - ret["CONTENT_LENGTH"] = len(data) - ret["wsgi.input"] = io.BytesIO(data) - else: - ret["wsgi.input"] = io.BytesIO(b"") - ret["wsgi.errors"] = sys.stderr - ret["wsgi.multithread"] = True - ret["wsgi.multiprocess"] = False - ret["wsgi.run_once"] = False - for key, val in head.items(): - ret["HTTP_" + key.upper().replace("-", "_")] = val - return ret - -class testrequest(object): - def __init__(self): - self.wbuf = io.BytesIO() - self.headers = None - self.status = None - - def __call__(self, status, headers): - self.status = status - self.headers = headers - return self.wbuf.write - - def __repr__(self): - return "" % (self.status, - "None" if self.headers is None else ("[%i]" % len(self.headers)), - "(no data)" if len(self.wbuf.getvalue()) == 0 else "(with data)") - - def __str__(self): - return repr(self) diff --git a/python3/doc/.gitignore b/python3/doc/.gitignore deleted file mode 100644 index 494c1f8..0000000 --- a/python3/doc/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -/*.1 -/*.html -/*.css diff --git a/python3/htp.c b/python3/htp.c deleted file mode 100644 index 3a091d4..0000000 --- a/python3/htp.c +++ /dev/null @@ -1,97 +0,0 @@ -/* - ashd - A Sane HTTP Daemon - Copyright (C) 2008 Fredrik Tolf - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -*/ - -#include -#include - -#include -#include - -static PyObject *p_recvfd(PyObject *self, PyObject *args) -{ - int fd, ret; - char *data; - size_t dlen; - PyObject *ro; - - fd = 0; - if(!PyArg_ParseTuple(args, "|i", &fd)) - return(NULL); - while(1) { - Py_BEGIN_ALLOW_THREADS; - ret = recvfd(fd, &data, &dlen); - Py_END_ALLOW_THREADS; - if(ret < 0) { - if(errno == 0) - return(Py_BuildValue("OO", Py_None, Py_None)); - if(errno == EINTR) { - if(PyErr_CheckSignals()) - return(NULL); - continue; - } - PyErr_SetFromErrno(PyExc_OSError); - return(NULL); - } - ro = Py_BuildValue("Ni", PyBytes_FromStringAndSize(data, dlen), ret); - free(data); - return(ro); - } -} - -static PyObject *p_sendfd(PyObject *self, PyObject *args) -{ - int sock, fd, ret; - Py_buffer data; - - if(!PyArg_ParseTuple(args, "iiy*", &sock, &fd, &data)) - return(NULL); - while(1) { - Py_BEGIN_ALLOW_THREADS; - ret = sendfd(sock, fd, data.buf, data.len); - Py_END_ALLOW_THREADS; - PyBuffer_Release(&data); - if(ret < 0) { - if(errno == EINTR) { - if(PyErr_CheckSignals()) - return(NULL); - continue; - } - PyErr_SetFromErrno(PyExc_OSError); - return(NULL); - } - Py_RETURN_NONE; - } -} - -static PyMethodDef methods[] = { - {"recvfd", p_recvfd, METH_VARARGS, "Receive a datagram and a file descriptor"}, - {"sendfd", p_sendfd, METH_VARARGS, "Send a datagram and a file descriptor"}, - {NULL, NULL, 0, NULL} -}; - -static struct PyModuleDef module = { - PyModuleDef_HEAD_INIT, - .m_name = "htlib", - .m_size = -1, - .m_methods = methods, -}; - -PyMODINIT_FUNC PyInit_htlib(void) -{ - return(PyModule_Create(&module)); -} diff --git a/python3/setup.py b/python3/setup.py deleted file mode 100755 index 6c6e16c..0000000 --- a/python3/setup.py +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/python3 - -from distutils.core import setup, Extension - -htlib = Extension("ashd.htlib", ["htp.c"], - libraries = ["ht"]) - -setup(name = "ashd-py3", - version = "0.6", - description = "Python module for handling ashd requests", - author = "Fredrik Tolf", - author_email = "fredrik@dolda2000.com", - url = "http://www.dolda2000.com/~fredrik/ashd/", - ext_modules = [htlib], - packages = ["ashd"], - scripts = ["ashd-wsgi3", "scgi-wsgi3", "serve-ssi"], - license = "GPL-3")