From: Fredrik Tolf Date: Tue, 13 Dec 2011 01:57:27 +0000 (+0100) Subject: Merge branch 'master' into python3 X-Git-Tag: 0.10~14 X-Git-Url: http://www.dolda2000.com/gitweb/?p=ashd.git;a=commitdiff_plain;h=c221b22d61df5882462eab41e5c79ae2a3ea9642;hp=50140c89086cdbc748250c87dd5b0671bf829369 Merge branch 'master' into python3 --- diff --git a/python3/.gitignore b/python3/.gitignore new file mode 100644 index 0000000..21e5002 --- /dev/null +++ b/python3/.gitignore @@ -0,0 +1,3 @@ +*.pyc +/build +/ashd/htlib.so diff --git a/python3/ashd-wsgi3 b/python3/ashd-wsgi3 new file mode 100755 index 0000000..da60cfa --- /dev/null +++ b/python3/ashd-wsgi3 @@ -0,0 +1,226 @@ +#!/usr/bin/python3 + +import sys, os, getopt, threading, time, locale, collections +import ashd.proto, ashd.util + +def usage(out): + out.write("usage: ashd-wsgi3 [-hA] [-p MODPATH] [-l REQLIMIT] HANDLER-MODULE [ARGS...]\n") + +reqlimit = 0 +modwsgi_compat = False +opts, args = getopt.getopt(sys.argv[1:], "+hAp:l:") +for o, a in opts: + if o == "-h": + usage(sys.stdout) + sys.exit(0) + elif o == "-p": + sys.path.insert(0, a) + elif o == "-A": + modwsgi_compat = True + elif o == "-l": + reqlimit = int(a) +if len(args) < 1: + usage(sys.stderr) + sys.exit(1) + +try: + handlermod = __import__(args[0], fromlist = ["dummy"]) +except ImportError as exc: + sys.stderr.write("ashd-wsgi3: handler %s not found: %s\n" % (args[0], exc.args[0])) + sys.exit(1) +if not modwsgi_compat: + if not hasattr(handlermod, "wmain"): + sys.stderr.write("ashd-wsgi3: handler %s has no `wmain' function\n" % args[0]) + sys.exit(1) + handler = handlermod.wmain(*args[1:]) +else: + if not hasattr(handlermod, "application"): + sys.stderr.write("ashd-wsgi3: handler %s has no `application' object\n" % args[0]) + sys.exit(1) + handler = handlermod.application + +class closed(IOError): + def __init__(self): + super().__init__("The client has closed the connection.") + +cwd = os.getcwd() +def absolutify(path): + if path[0] != '/': + return os.path.join(cwd, path) + return path + +def unquoteurl(url): + buf = bytearray() + i = 0 + while i < len(url): + c = url[i] + i += 1 + if c == ord(b'%'): + if len(url) >= i + 2: + c = 0 + if ord(b'0') <= url[i] <= ord(b'9'): + c |= (url[i] - ord(b'0')) << 4 + elif ord(b'a') <= url[i] <= ord(b'f'): + c |= (url[i] - ord(b'a') + 10) << 4 + elif ord(b'A') <= url[i] <= ord(b'F'): + c |= (url[i] - ord(b'A') + 10) << 4 + else: + raise ValueError("Illegal URL escape character") + if ord(b'0') <= url[i + 1] <= ord(b'9'): + c |= url[i + 1] - ord('0') + elif ord(b'a') <= url[i + 1] <= ord(b'f'): + c |= url[i + 1] - ord(b'a') + 10 + elif ord(b'A') <= url[i + 1] <= ord(b'F'): + c |= url[i + 1] - ord(b'A') + 10 + else: + raise ValueError("Illegal URL escape character") + buf.append(c) + i += 2 + else: + raise ValueError("Incomplete URL escape character") + else: + buf.append(c) + return buf + +def dowsgi(req): + env = {} + env["wsgi.version"] = 1, 0 + for key, val in req.headers: + env["HTTP_" + key.upper().replace(b"-", b"_").decode("latin-1")] = val.decode("latin-1") + env["SERVER_SOFTWARE"] = "ashd-wsgi/1" + env["GATEWAY_INTERFACE"] = "CGI/1.1" + env["SERVER_PROTOCOL"] = req.ver.decode("latin-1") + env["REQUEST_METHOD"] = req.method.decode("latin-1") + try: + rawpi = unquoteurl(req.rest) + except: + rawpi = req.rest + try: + name, rest, pi = (v.decode("utf-8") for v in (req.url, req.rest, rawpi)) + env["wsgi.uri_encoding"] = "utf-8" + except UnicodeError as exc: + name, rest, pi = (v.decode("latin-1") for v in (req.url, req.rest, rawpi)) + env["wsgi.uri_encoding"] = "latin-1" + env["REQUEST_URI"] = name + p = name.find('?') + if p >= 0: + env["QUERY_STRING"] = name[p + 1:] + name = name[:p] + else: + env["QUERY_STRING"] = "" + if name[-len(rest):] == rest: + # This is the same hack used in call*cgi. + name = name[:-len(rest)] + if name == "/": + # This seems to be normal CGI behavior, but see callcgi.c for + # details. + pi = "/" + pi + name = "" + env["SCRIPT_NAME"] = name + env["PATH_INFO"] = pi + for src, tgt in [("HTTP_HOST", "SERVER_NAME"), ("HTTP_X_ASH_SERVER_PORT", "SERVER_PORT"), + ("HTTP_X_ASH_ADDRESS", "REMOTE_ADDR"), ("HTTP_CONTENT_TYPE", "CONTENT_TYPE"), + ("HTTP_CONTENT_LENGTH", "CONTENT_LENGTH"), ("HTTP_X_ASH_PROTOCOL", "wsgi.url_scheme")]: + if src in env: env[tgt] = env[src] + if "X-Ash-Protocol" in req and req["X-Ash-Protocol"] == b"https": env["HTTPS"] = "on" + if "X-Ash-File" in req: env["SCRIPT_FILENAME"] = absolutify(req["X-Ash-File"].decode(locale.getpreferredencoding())) + env["wsgi.input"] = req.sk + env["wsgi.errors"] = sys.stderr + env["wsgi.multithread"] = True + env["wsgi.multiprocess"] = False + env["wsgi.run_once"] = False + + resp = [] + respsent = [] + + def recode(thing): + if isinstance(thing, collections.ByteString): + return thing + else: + return str(thing).encode("latin-1") + + def flushreq(): + if not respsent: + if not resp: + raise Exception("Trying to write data before starting response.") + status, headers = resp + respsent[:] = [True] + buf = bytearray() + buf += b"HTTP/1.1 " + recode(status) + b"\n" + for nm, val in headers: + buf += recode(nm) + b": " + recode(val) + b"\n" + buf += b"\n" + try: + req.sk.write(buf) + except IOError: + raise closed() + + def write(data): + if not data: + return + flushreq() + try: + req.sk.write(data) + req.sk.flush() + except IOError: + raise closed() + + def startreq(status, headers, exc_info = None): + if resp: + if exc_info: # Interesting, this... + try: + if respsent: + raise exc_info[1] + finally: + exc_info = None # CPython GC bug? + else: + raise Exception("Can only start responding once.") + resp[:] = status, headers + return write + + respiter = handler(env, startreq) + try: + try: + for data in respiter: + write(data) + if resp: + flushreq() + except closed: + pass + finally: + if hasattr(respiter, "close"): + respiter.close() + +flightlock = threading.Condition() +inflight = 0 + +class reqthread(threading.Thread): + def __init__(self, req): + super().__init__(name = "Request handler") + self.req = req.dup() + + def run(self): + global inflight + try: + with flightlock: + if reqlimit != 0: + start = time.time() + while inflight >= reqlimit: + flightlock.wait(10) + if time.time() - start > 10: + os.abort() + inflight += 1 + try: + dowsgi(self.req) + finally: + with flightlock: + inflight -= 1 + flightlock.notify() + finally: + self.req.close() + sys.stderr.flush() + +def handle(req): + reqthread(req).start() + +ashd.util.serveloop(handle) diff --git a/python3/ashd/__init__.py b/python3/ashd/__init__.py new file mode 100644 index 0000000..c918ad6 --- /dev/null +++ b/python3/ashd/__init__.py @@ -0,0 +1,5 @@ +"""Base module for ashd(7)-related fucntions. + +This module implements nothing. Please see the ashd.util or ashd.proto +modules. +""" diff --git a/python3/ashd/proto.py b/python3/ashd/proto.py new file mode 100644 index 0000000..e695751 --- /dev/null +++ b/python3/ashd/proto.py @@ -0,0 +1,182 @@ +"""Low-level protocol module for ashd(7) + +This module provides primitive functions that speak the raw ashd(7) +protocol. Primarily, it implements the `req' class that is used to +represent ashd requests. The functions it provides can also be used to +create ashd handlers, but unless you require very precise control, the +ashd.util module provides an easier-to-use interface. +""" + +import os, socket +from . import htlib + +__all__ = ["req", "recvreq", "sendreq"] + +class protoerr(Exception): + pass + +class req(object): + """Represents a single ashd request. Normally, you would not + create instances of this class manually, but receive them from the + recvreq function. + + For the abstract structure of ashd requests, please see the + ashd(7) manual page. This class provides access to the HTTP + method, raw URL, HTTP version and rest string via the `method', + `url', `ver' and `rest' variables respectively. It also implements + a dict-like interface for case-independent access to the HTTP + headers. The raw headers are available as a list of (name, value) + tuples in the `headers' variable. + + For responding, the response socket is available as a standard + Python stream object in the `sk' variable. Again, see the ashd(7) + manpage for what to receive and transmit on the response socket. + + Note that all request parts are stored in byte, rather than + string, form. The response socket is also opened in binary mode. + + Note also that instances of this class contain a reference to the + live socket used for responding to requests, which should be + closed when you are done with the request. The socket can be + closed manually by calling the close() method on this + object. Alternatively, this class implements the resource-manager + interface, so that it can be used in `with' statements. + """ + + def __init__(self, method, url, ver, rest, headers, fd): + self.method = method + self.url = url + self.ver = ver + self.rest = rest + self.headers = headers + self.bsk = socket.fromfd(fd, socket.AF_UNIX, socket.SOCK_STREAM) + self.sk = self.bsk.makefile('rwb') + os.close(fd) + + def close(self): + "Close this request's response socket." + self.sk.close() + self.bsk.close() + + def __getitem__(self, header): + """Find a HTTP header case-insensitively. For example, + req["Content-Type"] returns the value of the content-type + header regardlessly of whether the client specified it as + "Content-Type", "content-type" or "Content-type". + + If the header is given as a (Unicode) string, it is encoded + into Ascii for use in matching. + """ + if isinstance(header, str): + header = header.encode("ascii") + header = header.lower() + for key, val in self.headers: + if key.lower() == header: + return val + raise KeyError(header) + + def __contains__(self, header): + """Works analogously to the __getitem__ method for checking + header presence case-insensitively. + """ + if isinstance(header, str): + header = header.encode("ascii") + header = header.lower() + for key, val in self.headers: + if key.lower() == header: + return True + return False + + def dup(self): + """Creates a duplicate of this request, referring to a + duplicate of the response socket. + """ + return req(self.method, self.url, self.ver, self.rest, self.headers, os.dup(self.bsk.fileno())) + + def match(self, match): + """If the `match' argument matches exactly the leading part of + the rest string, this method strips that part of the rest + string off and returns True. Otherwise, it returns False + without doing anything. + + If the `match' argument is given as a (Unicode) string, it is + encoded into UTF-8. + + This can be used for simple dispatching. For example: + if req.match("foo/"): + handle(req) + elif req.match("bar/"): + handle_otherwise(req) + else: + util.respond(req, "Not found", status = "404 Not Found", ctype = "text/plain") + """ + if isinstance(match, str): + match = match.encode("utf-8") + if self.rest[:len(match)] == match: + self.rest = self.rest[len(match):] + return True + return False + + def __str__(self): + def dec(b): + return b.decode("ascii", errors="replace") + return "\"%s %s %s\"" % (dec(self.method), dec(self.url), dec(self.ver)) + + def __enter__(self): + return self + + def __exit__(self, *excinfo): + self.sk.close() + return False + +def recvreq(sock = 0): + """Receive a single ashd request on the specified socket file + descriptor (or standard input if unspecified). + + The returned value is an instance of the `req' class. As per its + description, care should be taken to close() the request when + done, to avoid leaking response sockets. If end-of-file is + received on the socket, None is returned. + + This function may either raise on OSError if an error occurs on + the socket, or a ashd.proto.protoerr if the incoming request is + invalidly encoded. + """ + data, fd = htlib.recvfd(sock) + if fd is None: + return None + try: + parts = data.split(b'\0')[:-1] + if len(parts) < 5: + raise protoerr("Truncated request") + method, url, ver, rest = parts[:4] + headers = [] + i = 4 + while True: + if parts[i] == b"": break + if len(parts) - i < 3: + raise protoerr("Truncated request") + headers.append((parts[i], parts[i + 1])) + i += 2 + return req(method, url, ver, rest, headers, os.dup(fd)) + finally: + os.close(fd) + +def sendreq(sock, req): + """Encode and send a single request to the specified socket file + descriptor using the ashd protocol. The request should be an + instance of the `req' class. + + This function may raise an OSError if an error occurs on the + socket. + """ + data = b"" + data += req.method + b'\0' + data += req.url + b'\0' + data += req.ver + b'\0' + data += req.rest + b'\0' + for key, val in req.headers: + data += key + b'\0' + data += val + b'\0' + data += b'\0' + htlib.sendfd(sock, req.sk.fileno(), data) diff --git a/python3/ashd/scgi.py b/python3/ashd/scgi.py new file mode 100644 index 0000000..a06267f --- /dev/null +++ b/python3/ashd/scgi.py @@ -0,0 +1,146 @@ +import sys, collections +import threading + +class protoerr(Exception): + pass + +class closed(IOError): + def __init__(self): + super(closed, self).__init__("The client has closed the connection.") + +def readns(sk): + hln = 0 + while True: + c = sk.read(1) + if c == b':': + break + elif c >= b'0' or c <= b'9': + hln = (hln * 10) + (ord(c) - ord(b'0')) + else: + raise protoerr("Invalid netstring length byte: " + c) + ret = sk.read(hln) + if sk.read(1) != b',': + raise protoerr("Non-terminated netstring") + return ret + +def readhead(sk): + parts = readns(sk).split(b'\0')[:-1] + if len(parts) % 2 != 0: + raise protoerr("Malformed headers") + ret = {} + i = 0 + while i < len(parts): + ret[parts[i]] = parts[i + 1] + i += 2 + return ret + +class reqthread(threading.Thread): + def __init__(self, sk, handler): + super(reqthread, self).__init__(name = "SCGI request handler") + self.sk = sk.dup().makefile("rwb") + self.handler = handler + + def run(self): + try: + head = readhead(self.sk) + self.handler(head, self.sk) + finally: + self.sk.close() + +def handlescgi(sk, handler): + t = reqthread(sk, handler) + t.start() + +def servescgi(socket, handler): + while True: + nsk, addr = socket.accept() + try: + handlescgi(nsk, handler) + finally: + nsk.close() + +def decodehead(head, coding): + return {k.decode(coding): v.decode(coding) for k, v in head.items()} + +def wrapwsgi(handler): + def handle(head, sk): + try: + env = decodehead(head, "utf-8") + env["wsgi.uri_encoding"] = "utf-8" + except UnicodeError: + env = decodehead(head, "latin-1") + env["wsgi.uri_encoding"] = "latin-1" + env["wsgi.version"] = 1, 0 + if "HTTP_X_ASH_PROTOCOL" in env: + env["wsgi.url_scheme"] = env["HTTP_X_ASH_PROTOCOL"] + elif "HTTPS" in env: + env["wsgi.url_scheme"] = "https" + else: + env["wsgi.url_scheme"] = "http" + env["wsgi.input"] = sk + env["wsgi.errors"] = sys.stderr + env["wsgi.multithread"] = True + env["wsgi.multiprocess"] = False + env["wsgi.run_once"] = False + + resp = [] + respsent = [] + + def recode(thing): + if isinstance(thing, collections.ByteString): + return thing + else: + return str(thing).encode("latin-1") + + def flushreq(): + if not respsent: + if not resp: + raise Exception("Trying to write data before starting response.") + status, headers = resp + respsent[:] = [True] + buf = bytearray() + buf += b"Status: " + recode(status) + b"\n" + for nm, val in headers: + buf += recode(nm) + b": " + recode(val) + b"\n" + buf += b"\n" + try: + sk.write(buf) + except IOError: + raise closed() + + def write(data): + if not data: + return + flushreq() + try: + sk.write(data) + sk.flush() + except IOError: + raise closed() + + def startreq(status, headers, exc_info = None): + if resp: + if exc_info: # Interesting, this... + try: + if respsent: + raise exc_info[1] + finally: + exc_info = None # CPython GC bug? + else: + raise Exception("Can only start responding once.") + resp[:] = status, headers + return write + + respiter = handler(env, startreq) + try: + try: + for data in respiter: + write(data) + if resp: + flushreq() + except closed: + pass + finally: + if hasattr(respiter, "close"): + respiter.close() + return handle diff --git a/python3/ashd/util.py b/python3/ashd/util.py new file mode 100644 index 0000000..3818e4b --- /dev/null +++ b/python3/ashd/util.py @@ -0,0 +1,170 @@ +"""High-level utility module for ashd(7) + +This module implements a rather convenient interface for writing ashd +handlers, wrapping the low-level ashd.proto module. +""" + +import os, socket, collections +from . import proto + +__all__ = ["stdfork", "pchild", "respond", "serveloop"] + +def stdfork(argv, chinit = None): + """Fork a persistent handler process using the `argv' argument + list, as per the standard ashd(7) calling convention. For an + easier-to-use interface, see the `pchild' class. + + If a callable object of no arguments is provided in the `chinit' + argument, it will be called in the child process before exec()'ing + the handler program, and can be used to set parameters for the new + process, such as working directory, nice level or ulimits. + + Returns the file descriptor of the socket for sending requests to + the new child. + """ + csk, psk = socket.socketpair(socket.AF_UNIX, socket.SOCK_SEQPACKET) + pid = os.fork() + if pid == 0: + try: + os.dup2(csk.fileno(), 0) + for fd in range(3, 1024): + try: + os.close(fd) + except: + pass + if chinit is not None: + chinit() + os.execvp(argv[0], argv) + finally: + os._exit(127) + csk.close() + fd = os.dup(psk.fileno()) + psk.close() + return fd + +class pchild(object): + """class pchild(argv, autorespawn=False, chinit=None) + + Represents a persistent child handler process, started as per the + standard ashd(7) calling convention. It will be called with the + `argv' argument lest, which should be a list (or other iterable) + of strings. + + If `autorespawn' is specified as True, the child process will be + automatically restarted if a request cannot be successfully sent + to it. + + For a description of the `chinit' argument, see `stdfork'. + + When this child handler should be disposed of, care should be + taken to call the close() method to release its socket and let it + exit. This class also implements the resource-manager interface, + so that it can be used in `with' statements. + """ + + def __init__(self, argv, autorespawn = False, chinit = None): + self.argv = argv + self.chinit = chinit + self.fd = -1 + self.respawn = autorespawn + self.spawn() + + def spawn(self): + """Start the child handler, or restart it if it is already + running. You should not have to call this method manually + unless you explicitly want to manage the process' lifecycle. + """ + self.close() + self.fd = stdfork(self.argv, self.chinit) + + def close(self): + """Close this child handler's socket. For normal child + handlers, this will make the program terminate normally. + """ + if self.fd >= 0: + os.close(self.fd) + self.fd = -1 + + def __del__(self): + self.close() + + def passreq(self, req): + """Pass the specified request (which should be an instance of + the ashd.proto.req class) to this child handler. If the child + handler fails for some reason, and `autorespawn' was specified + as True when creating this handler, one attempt will be made + to restart it. + + Note: You still need to close the request normally. + + This method may raise an OSError if the request fails and + autorespawning was either not requested, or if the + autorespawning fails. + """ + try: + proto.sendreq(self.fd, req) + except OSError: + if self.respawn: + self.spawn() + proto.sendreq(self.fd, req) + + def __enter__(self): + return self + + def __exit__(self, *excinfo): + self.close() + return False + +def respond(req, body, status = ("200 OK"), ctype = "text/html"): + """Simple function for conveniently responding to a request. + + Sends the specified body text to the request's response socket, + prepending an HTTP header with the appropriate Content-Type and + Content-Length headers, and then closes the response socket. + + The `status' argument can be used to specify a non-200 response, + and the `ctype' argument can be used to specify a non-HTML + MIME-type. + + If `body' is not a byte string, its string representation will be + encoded as UTF-8. + + For example: + respond(req, "Not found", status = "404 Not Found", ctype = "text/plain") + """ + if isinstance(body, collections.ByteString): + body = bytes(body) + else: + body = str(body) + body = body.encode("utf-8") + if ctype[:5] == "text/" and ctype.find(';') < 0: + ctype = ctype + "; charset=utf-8" + try: + head = "" + head += "HTTP/1.1 %s\n" % status + head += "Content-Type: %s\n" % ctype + head += "Content-Length: %i\n" % len(body) + head += "\n" + req.sk.write(head.encode("ascii")) + req.sk.write(body) + finally: + req.close() + +def serveloop(handler, sock = 0): + """Implements a simple loop for serving requests sequentially, by + receiving requests from standard input (or the specified socket), + passing them to the specified handler function, and finally making + sure to close them. Returns when end-of-file is received on the + incoming socket. + + The handler function should be a callable object of one argument, + and is called once for each received request. + """ + while True: + req = proto.recvreq(sock) + if req is None: + break + try: + handler(req) + finally: + req.close() diff --git a/python3/ashd/wsgidir.py b/python3/ashd/wsgidir.py new file mode 100644 index 0000000..cc17b9b --- /dev/null +++ b/python3/ashd/wsgidir.py @@ -0,0 +1,169 @@ +"""WSGI handler for serving chained WSGI modules from physical files + +The WSGI handler in this module examines the SCRIPT_FILENAME variable +of the requests it handles -- that is, the physical file corresponding +to the request, as determined by the webserver -- determining what to +do with the request based on the extension of that file. + +By default, it handles files named `.wsgi' by compiling them into +Python modules and using them, in turn, as chained WSGI handlers, but +handlers for other extensions can be installed as well. + +When handling `.wsgi' files, the compiled modules are cached and +reused until the file is modified, in which case the previous module +is discarded and the new file contents are loaded into a new module in +its place. When chaining such modules, an object named `wmain' is +first looked for and called with no arguments if found. The object it +returns is then used as the WSGI application object for that module, +which is reused until the module is reloaded. If `wmain' is not found, +an object named `application' is looked for instead. If found, it is +used directly as the WSGI application object. + +This module itself contains both an `application' and a `wmain' +object. If this module is used by ashd-wsgi(1) or scgi-wsgi(1) so that +its wmain function is called, arguments can be specified to it to +install handlers for other file extensions. Such arguments take the +form `.EXT=MODULE.HANDLER', where EXT is the file extension to be +handled, and the MODULE.HANDLER string is treated by splitting it +along its last constituent dot. The part left of the dot is the name +of a module which is imported, and the part right of the dot is the +name of an object in that module, which should be a callable adhering +to the WSGI specification. When called, this module will have made +sure that the WSGI environment contains the SCRIPT_FILENAME parameter +and that it is properly working. For example, the argument +`.fpy=my.module.foohandler' can be given to pass requests for `.fpy' +files to the function `foohandler' in the module `my.module' (which +must, of course, be importable). When writing such handler functions, +you will probably want to use the getmod() function in this module. +""" + +import os, threading, types +from . import wsgiutil + +__all__ = ["application", "wmain", "getmod", "cachedmod"] + +class cachedmod(object): + """Cache entry for modules loaded by getmod() + + Instances of this class are returned by the getmod() + function. They contain three data attributes: + * mod - The loaded module + * lock - A threading.Lock object, which can be used for + manipulating this instance in a thread-safe manner + * mtime - The time the file was last modified + + Additional data attributes can be arbitrarily added for recording + any meta-data about the module. + """ + def __init__(self, mod, mtime): + self.lock = threading.Lock() + self.mod = mod + self.mtime = mtime + +exts = {} +modcache = {} +cachelock = threading.Lock() + +def mangle(path): + ret = "" + for c in path: + if c.isalnum(): + ret += c + else: + ret += "_" + return ret + +def getmod(path): + """Load the given file as a module, caching it appropriately + + The given file is loaded and compiled into a Python module. The + compiled module is cached and returned upon subsequent requests + for the same file, unless the file has changed (as determined by + its mtime), in which case the cached module is discarded and the + new file contents are reloaded in its place. + + The return value is an instance of the cachedmod class, which can + be used for locking purposes and for storing arbitrary meta-data + about the module. See its documentation for details. + """ + sb = os.stat(path) + cachelock.acquire() + try: + if path in modcache: + entry = modcache[path] + if sb.st_mtime <= entry.mtime: + return entry + + f = open(path) + try: + text = f.read() + finally: + f.close() + code = compile(text, path, "exec") + mod = types.ModuleType(mangle(path)) + mod.__file__ = path + exec(code, mod.__dict__) + entry = cachedmod(mod, sb.st_mtime) + modcache[path] = entry + return entry + finally: + cachelock.release() + +def chain(env, startreq): + path = env["SCRIPT_FILENAME"] + mod = getmod(path) + entry = None + if mod is not None: + mod.lock.acquire() + try: + if hasattr(mod, "entry"): + entry = mod.entry + else: + if hasattr(mod.mod, "wmain"): + entry = mod.mod.wmain() + elif hasattr(mod.mod, "application"): + entry = mod.mod.application + mod.entry = entry + finally: + mod.lock.release() + if entry is not None: + return entry(env, startreq) + return wsgiutil.simpleerror(env, startreq, 500, "Internal Error", "Invalid WSGI handler.") +exts["wsgi"] = chain +exts["wsgi3"] = chain + +def addext(ext, handler): + p = handler.rindex('.') + mname = handler[:p] + hname = handler[p + 1:] + mod = __import__(mname, fromlist = ["dummy"]) + exts[ext] = getattr(mod, hname) + +def application(env, startreq): + """WSGI handler function + + Handles WSGI requests as per the module documentation. + """ + if not "SCRIPT_FILENAME" in env: + return wsgiutil.simpleerror(env, startreq, 500, "Internal Error", "The server is erroneously configured.") + path = env["SCRIPT_FILENAME"] + base = os.path.basename(path) + p = base.rfind('.') + if p < 0 or not os.access(path, os.R_OK): + return wsgiutil.simpleerror(env, startreq, 500, "Internal Error", "The server is erroneously configured.") + ext = base[p + 1:] + if not ext in exts: + return wsgiutil.simpleerror(env, startreq, 500, "Internal Error", "The server is erroneously configured.") + return(exts[ext](env, startreq)) + +def wmain(*argv): + """Main function for ashd(7)-compatible WSGI handlers + + Returns the `application' function. If any arguments are given, + they are parsed according to the module documentation. + """ + for arg in argv: + if arg[0] == '.': + p = arg.index('=') + addext(arg[1:p], arg[p + 1:]) + return application diff --git a/python3/ashd/wsgiutil.py b/python3/ashd/wsgiutil.py new file mode 100644 index 0000000..5fe7535 --- /dev/null +++ b/python3/ashd/wsgiutil.py @@ -0,0 +1,30 @@ +def htmlquote(text): + ret = "" + for c in text: + if c == '&': + ret += "&" + elif c == '<': + ret += "<" + elif c == '>': + ret += ">" + elif c == '"': + ret += """ + else: + ret += c + return ret + +def simpleerror(env, startreq, code, title, msg): + buf = """ + + + +%s + + +

%s

+

%s

+ +""" % (title, title, htmlquote(msg)) + buf = buf.encode("ascii") + startreq("%i %s" % (code, title), [("Content-Type", "text/html"), ("Content-Length", str(len(buf)))]) + return [buf] diff --git a/python3/doc/.gitignore b/python3/doc/.gitignore new file mode 100644 index 0000000..494c1f8 --- /dev/null +++ b/python3/doc/.gitignore @@ -0,0 +1,3 @@ +/*.1 +/*.html +/*.css diff --git a/python3/doc/ashd-wsgi3.doc b/python3/doc/ashd-wsgi3.doc new file mode 100644 index 0000000..0af0882 --- /dev/null +++ b/python3/doc/ashd-wsgi3.doc @@ -0,0 +1,115 @@ +ashd-wsgi3(1) +============= + +NAME +---- +ashd-wsgi3 - WSGI adapter for ashd(7) + +SYNOPSIS +-------- +*ashd-wsgi3* [*-hA*] [*-p* 'MODPATH'] [*-l* 'LIMIT'] 'HANDLER-MODULE' ['ARGS'...] + +DESCRIPTION +----------- + +The *ashd-wsgi3* handler translates *ashd*(7) requests to WSGI +requests, and passes them to a specified Python handler module. The +precise Python convention for doing so is described in the PROTOCOL +section, below. + +*ashd-wsgi3* is a persistent handler, as defined in *ashd*(7). It uses +multithreaded dispatching in a single Python interpreter, which means +that WSGI applications that use it need to be thread-safe, but that +they can also share all Python data structures and global variables +between requests. + +The Python module that *ashd-wsgi3* comes with also contains a +standard handler module, `ashd.wsgidir`, which serves individual WSGI +applications directly from the files in which they reside and as such +makes this program useful as a *dirplex*(1) handler. Please see its +Python documentation for further details. + +*ashd-wsgi3* requires the `ashd.proto` and `ashd.util` modules, which +are only available for CPython. If you want to use some other Python +implementation instead, you may want to use the *scgi-wsgi*(1) program +instead, along with *callscgi*(1). + +OPTIONS +------- + +*-h*:: + + Print a brief help message to standard output and exit. + +*-A*:: + + Use the convention used by Apache's mod_wsgi module to find + the WSGI application object. See the PROTOCOL section, below, + for details. + +*-p* 'MODPATH':: + + Prepend 'MODPATH' to Python's `sys.path`; can be given multiple + times. Note that the working directory of *ashd-wsgi3* is not + on Python's module path by default, so if you want to use a + module in that directory, you will need to specify "`-p .`". + +*-l* 'LIMIT':: + + Allow at most 'LIMIT' requests to run concurrently. If a new + request is made when 'LIMIT' requests are executing, the new + request will wait up to ten seconds for one of them to + complete; if none does, *ashd-wsgi3* will assume that the + process is foobar and *abort*(3). + +PROTOCOL +-------- + +When starting, *ashd-wsgi3* will attempt to import the module named by +'HANDLER-MODULE', look for an object named `wmain` in that module, +call that object passing the 'ARGS' (as Python strings) as positional +parameters, and use the returned object as the WSGI application +object. If the *-A* option was specified, it will look for an object +named `application` instead of `wmain`, and use that object directly +as the WSGI application object. + +When calling the WSGI application, a new thread is started for each +request, in which the WSGI application object is called. All requests +run in the same interpreter, so it is guaranteed that data structures +and global variables can be shared between requests. + +The WSGI environment is the standard CGI environment, including the +`SCRIPT_FILENAME` variable whenever the `X-Ash-File` header was +included in the request. + +EXAMPLES +-------- + +The following *dirplex*(1) configuration can be used for serving WSGI +modules directly from the filesystem. + +-------- +child wsgidir + exec ashd-wsgi3 ashd.wsgidir +match + filename *.wsgi + handler wsgidir +-------- + +Since *ashd-wsgi3* is a persistent handler, it can be used directly as +a root handler for *htparser*(1). For instance, if the directory +`/srv/www/foo` contains a `wsgi.py` file, which declares a standard +WSGI `application` object, it can be served with the following +command: + +-------- +htparser plain:port=8080 -- ashd-wsgi3 -Ap /srv/www/foo wsgi +-------- + +AUTHOR +------ +Fredrik Tolf + +SEE ALSO +-------- +*scgi-wsgi3*(1), *ashd*(7), diff --git a/python3/doc/scgi-wsgi3.doc b/python3/doc/scgi-wsgi3.doc new file mode 100644 index 0000000..df91477 --- /dev/null +++ b/python3/doc/scgi-wsgi3.doc @@ -0,0 +1,63 @@ +scgi-wsgi3(1) +============ + +NAME +---- +scgi-wsgi3 - WSGI adapter for SCGI + +SYNOPSIS +-------- +*scgi-wsgi3* [*-hA*] [*-p* 'MODPATH'] [*-T* \[HOST:]'PORT'] 'HANDLER-MODULE' ['ARGS'...] + +DESCRIPTION +----------- + +The *scgi-wsgi3* program translates SCGI requests to WSGI requests, +and passes them to a specified Python module. It is mainly written to +emulate the behavior of *ashd-wsgi*(1), but over SCGI instead of the +native *ashd*(7) protocol, so please see its documentation for details +of Python interoperation. Unlike *ashd-wsgi* which requires CPython, +however, *scgi-wsgi3* is written in pure Python using only the +standard library, and so should be usable by any Python +implementation. If using it under *ashd*(7), please see the +documentation for *callscgi*(1) as well. + +Following *callscgi*(1) conventions, *scgi-wsgi3* will, by default, +accept connections on a socket passed on its standard input (a +behavior which is, obviously, not available on all Python +implementations). Use the *-T* option to listen to a TCP address +instead. + +OPTIONS +------- + +*-h*:: + + Print a brief help message to standard output and exit. + +*-A*:: + + Use the convention used by Apache's mod_wsgi module to find + the WSGI application object. See the PROTOCOL section of + *ashd-wsgi*(1) for details. + +*-p* 'MODPATH':: + + Prepend 'MODPATH' to Python's `sys.path`; can be given multiple + times. + +*-T* \[HOST:]'PORT':: + + Instead of using a listening socket passed on standard input + to accept SCGI connections, bind a TCP socket to the 'HOST' + address listening for connections on 'PORT' instead. If 'HOST' + is not given, `localhost` is used by default. + +AUTHOR +------ +Fredrik Tolf + +SEE ALSO +-------- +*ashd-wsgi3*(1), *callscgi*(1), , + diff --git a/python3/htp.c b/python3/htp.c new file mode 100644 index 0000000..ec4ebab --- /dev/null +++ b/python3/htp.c @@ -0,0 +1,83 @@ +/* + ashd - A Sane HTTP Daemon + Copyright (C) 2008 Fredrik Tolf + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +*/ + +#include +#include + +#include +#include + +static PyObject *p_recvfd(PyObject *self, PyObject *args) +{ + int fd, ret; + char *data; + size_t dlen; + PyObject *ro; + + fd = 0; + if(!PyArg_ParseTuple(args, "|i", &fd)) + return(NULL); + Py_BEGIN_ALLOW_THREADS; + ret = recvfd(fd, &data, &dlen); + Py_END_ALLOW_THREADS; + if(ret < 0) { + if(errno == 0) + return(Py_BuildValue("OO", Py_None, Py_None)); + PyErr_SetFromErrno(PyExc_OSError); + return(NULL); + } + ro = Py_BuildValue("Ni", PyBytes_FromStringAndSize(data, dlen), ret); + free(data); + return(ro); +} + +static PyObject *p_sendfd(PyObject *self, PyObject *args) +{ + int sock, fd, ret; + Py_buffer data; + + if(!PyArg_ParseTuple(args, "iiy*", &sock, &fd, &data)) + return(NULL); + Py_BEGIN_ALLOW_THREADS; + ret = sendfd(sock, fd, data.buf, data.len); + Py_END_ALLOW_THREADS; + PyBuffer_Release(&data); + if(ret < 0) { + PyErr_SetFromErrno(PyExc_OSError); + return(NULL); + } + Py_RETURN_NONE; +} + +static PyMethodDef methods[] = { + {"recvfd", p_recvfd, METH_VARARGS, "Receive a datagram and a file descriptor"}, + {"sendfd", p_sendfd, METH_VARARGS, "Send a datagram and a file descriptor"}, + {NULL, NULL, 0, NULL} +}; + +static struct PyModuleDef module = { + PyModuleDef_HEAD_INIT, + .m_name = "htlib", + .m_size = -1, + .m_methods = methods, +}; + +PyMODINIT_FUNC PyInit_htlib(void) +{ + return(PyModule_Create(&module)); +} diff --git a/python3/scgi-wsgi3 b/python3/scgi-wsgi3 new file mode 100755 index 0000000..4f5714e --- /dev/null +++ b/python3/scgi-wsgi3 @@ -0,0 +1,58 @@ +#!/usr/bin/python3 + +import sys, os, getopt +import socket +import ashd.scgi + +def usage(out): + out.write("usage: scgi-wsgi3 [-hA] [-p MODPATH] [-T [HOST:]PORT] HANDLER-MODULE [ARGS...]\n") + +sk = None +modwsgi_compat = False +opts, args = getopt.getopt(sys.argv[1:], "+hAp:T:") +for o, a in opts: + if o == "-h": + usage(sys.stdout) + sys.exit(0) + elif o == "-p": + sys.path.insert(0, a) + elif o == "-T": + sk = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + p = a.rfind(":") + if p < 0: + bindhost = "localhost" + bindport = int(a) + else: + bindhost = a[:p] + bindport = int(a[p + 1:]) + sk.bind((bindhost, bindport)) + sk.listen(32) + elif o == "-A": + modwsgi_compat = True +if len(args) < 1: + usage(sys.stderr) + sys.exit(1) + +if sk is None: + # This is suboptimal, since the socket on stdin is not necessarily + # AF_UNIX, but Python does not seem to offer any way around it, + # that I can find. + sk = socket.fromfd(0, socket.AF_UNIX, socket.SOCK_STREAM) + +try: + handlermod = __import__(args[0], fromlist = ["dummy"]) +except ImportError as exc: + sys.stderr.write("scgi-wsgi3: handler %s not found: %s\n" % (args[0], exc.args[0])) + sys.exit(1) +if not modwsgi_compat: + if not hasattr(handlermod, "wmain"): + sys.stderr.write("scgi-wsgi3: handler %s has no `wmain' function\n" % args[0]) + sys.exit(1) + handler = handlermod.wmain(*args[1:]) +else: + if not hasattr(handlermod, "application"): + sys.stderr.write("scgi-wsgi3: handler %s has no `application' object\n" % args[0]) + sys.exit(1) + handler = handlermod.application + +ashd.scgi.servescgi(sk, ashd.scgi.wrapwsgi(handler)) diff --git a/python3/setup.py b/python3/setup.py new file mode 100755 index 0000000..f8924eb --- /dev/null +++ b/python3/setup.py @@ -0,0 +1,17 @@ +#!/usr/bin/python3 + +from distutils.core import setup, Extension + +htlib = Extension("ashd.htlib", ["htp.c"], + libraries = ["ht"]) + +setup(name = "ashd-py3", + version = "0.4", + description = "Python module for handling ashd requests", + author = "Fredrik Tolf", + author_email = "fredrik@dolda2000.com", + url = "http://www.dolda2000.com/~fredrik/ashd/", + ext_modules = [htlib], + packages = ["ashd"], + scripts = ["ashd-wsgi3", "scgi-wsgi3"], + license = "GPL-3")