From: Fredrik Tolf Date: Sat, 31 Dec 2016 17:53:33 +0000 (+0100) Subject: Merge branch 'master' into timeheap X-Git-Url: http://www.dolda2000.com/gitweb/?p=ashd.git;a=commitdiff_plain;h=bcad6b0c48d516ddc920b52f06083ceaa242e1ca;hp=589987f8218c9aa61d65f582a3b3e1bbd32bda81 Merge branch 'master' into timeheap Conflicts: lib/mtio-epoll.c --- diff --git a/ChangeLog b/ChangeLog index 69a2025..d8c848a 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,21 @@ +Version 0.13: + + * FreeBSD support. + * Support upgrading connections to full-duplex, for eg. websockets. + * More useful dirplex/patplex configuration options. + * Improved behavior under overload conditions. + * Expanded accesslog's logging capabilities. + * Added httrcall. + * Quite a slew of random bugfixes and implementation improvements. + +Version 0.12: + + * Support chunked request-bodies from clients. + * Improved general behavior when handlers are overloaded. + * Improved shutdown behavior. + * Properly support URL percent-escapes in patplex. + * Added psendfile. + Version 0.11: * Some useful configuration options for dirplex. diff --git a/configure.in b/configure.in index 74a1a99..80d2389 100644 --- a/configure.in +++ b/configure.in @@ -1,6 +1,8 @@ -AC_INIT(src/htparser.c) -AM_INIT_AUTOMAKE([ashd], [0.12]) +AC_INIT([ashd], [0.13]) +AC_CONFIG_SRCDIR(src/htparser.c) +AM_INIT_AUTOMAKE AM_CONFIG_HEADER(config.h) +AC_USE_SYSTEM_EXTENSIONS AC_PROG_CC AM_PROG_CC_C_O @@ -19,11 +21,29 @@ if test "$HAS_MAGIC" = no; then AC_MSG_ERROR([*** cannot find libmagic on this system]) fi +AH_TEMPLATE(HAVE_GLIBC_STDIO, [define to indicate system support for glibc cookie streams]) +AH_TEMPLATE(HAVE_BSD_STDIO, [define to indicate system support for BSD-style funopen streams]) + +HAS_FOPENCOOKIE=yes +AC_CHECK_FUNC(fopencookie, [], [HAS_FOPENCOOKIE=no]) +AC_CHECK_MEMBER([cookie_io_functions_t.read], [], [HAS_FOPENCOOKIE=no]) + +HAS_FUNOPEN=yes +AC_CHECK_FUNC(funopen, [], [HAS_FUNOPEN=no]) + +if test "$HAS_FOPENCOOKIE" = yes; then + AC_DEFINE(HAVE_GLIBC_STDIO) +elif test "$HAS_FUNOPEN" = yes; then + AC_DEFINE(HAVE_BSD_STDIO) +else + AC_MSG_ERROR([*** libc support for custom stdio streams is required]) +fi + AH_TEMPLATE(HAVE_VALGRIND, [define to include debugging support for Valgrind]) AC_CHECK_HEADER(valgrind/memcheck.h, [AC_DEFINE(HAVE_VALGRIND)], []) AH_TEMPLATE(HAVE_EPOLL, [define to enable epoll support]) -AC_ARG_WITH(epoll, [ --with-epoll Enable epoll(2) support]) +AC_ARG_WITH(epoll, AS_HELP_STRING([--with-epoll], [enable epoll(2) support])) HAS_EPOLL="" if test "$with_epoll" = no; then HAS_EPOLL=no; fi if test -z "$HAS_EPOLL"; then @@ -39,10 +59,30 @@ fi if test "$HAS_EPOLL" = yes; then AC_DEFINE(HAVE_EPOLL) fi -AM_CONDITIONAL(USE_EPOLL, [test "$HAS_EPOLL" = yes ]) + +AH_TEMPLATE(HAVE_KQUEUE, [define to enable kqueue support]) +AC_ARG_WITH(kqueue, AS_HELP_STRING([--with-kqueue], [enable kqueue(2) support])) +HAS_KQUEUE="" +if test "$with_kqueue" = no; then HAS_QUEUE=no; fi +if test -z "$HAS_KQUEUE"; then + AC_CHECK_FUNC(kqueue, [], [HAS_KQUEUE=no]) +fi +if test -z "$HAS_KQUEUE"; then + AC_CHECK_HEADER(sys/event.h, [], [HAS_KQUEUE=no]) +fi +if test "$HAS_KQUEUE" != no; then HAS_KQUEUE=yes; fi +if test "$with_kqueue" = yes -a "$HAS_KQUEUE" = no; then + AC_MSG_ERROR([*** cannot find kqueue support on this system]) +fi +if test "$HAS_KQUEUE" = yes; then + AC_DEFINE(HAVE_KQUEUE) +fi + +AM_CONDITIONAL(USE_EPOLL, [test "$HAS_EPOLL" = yes]) +AM_CONDITIONAL(USE_KQUEUE, [test "$HAS_KQUEUE" = yes]) AH_TEMPLATE(HAVE_XATTR, [define to compile support for filesystem extended attributes]) -AC_ARG_WITH(xattr, [ --with-xattr Enable XATTR support]) +AC_ARG_WITH(xattr, AS_HELP_STRING([--with-xattr], [enable XATTR support])) HAS_XATTR="" if test "$with_xattr" = no; then HAS_XATTR=no; fi if test -z "$HAS_XATTR"; then @@ -62,7 +102,7 @@ fi AC_SUBST(XATTR_LIBS) AH_TEMPLATE(HAVE_GNUTLS, [define to use the GnuTLS library for SSL support]) -AC_ARG_WITH(gnutls, [ --with-gnutls Enable SSL support with the GnuTLS library]) +AC_ARG_WITH(gnutls, AS_HELP_STRING([--with-gnutls], [enable SSL support with the GnuTLS library])) HAS_GNUTLS="" if test "$with_gnutls" = no; then HAS_GNUTLS=no; fi if test -z "$HAS_GNUTLS"; then diff --git a/doc/Makefile.am b/doc/Makefile.am index dc88692..baf94f3 100644 --- a/doc/Makefile.am +++ b/doc/Makefile.am @@ -1,17 +1,19 @@ dist_man1_MANS = callcgi.1 dirplex.1 htparser.1 patplex.1 sendfile.1 \ userplex.1 htls.1 callscgi.1 accesslog.1 htextauth.1 \ - callfcgi.1 multifscgi.1 errlogger.1 httimed.1 + callfcgi.1 multifscgi.1 errlogger.1 httimed.1 \ + psendfile.1 httrcall.1 dist_man7_MANS = ashd.7 -%.7 %.1: %.doc +.doc.1: a2x -f manpage $< - -%.html: %.doc +.doc.7: + a2x -f manpage $< +.doc.html: a2x -f xhtml $< manpages: $(dist_man1_MANS) $(dist_man7_MANS) -htmldoc: $(patsubst %.doc, %.html, *.doc) +htmldoc: ${dist_man1_MANS:.1=.html} ${dist_man7_MANS:.7=.html} EXTRA_DIST = *.doc diff --git a/doc/accesslog.doc b/doc/accesslog.doc index c33c6d6..2baf508 100644 --- a/doc/accesslog.doc +++ b/doc/accesslog.doc @@ -7,7 +7,7 @@ accesslog - Access logger for ashd(7) SYNOPSIS -------- -*accesslog* [*-hFaL*] [*-f* 'FORMAT'] [*-p* 'PIDFILE'] 'OUTFILE' 'CHILD' ['ARGS'...] +*accesslog* [*-hFaeL*] [*-f* 'FORMAT'] [*-p* 'PIDFILE'] 'OUTFILE' 'CHILD' ['ARGS'...] *accesslog* *-P* 'LOGFILE' @@ -51,7 +51,7 @@ OPTIONS Use the specified 'FORMAT' string instead of the default log record format. See the FORMAT section, below, for a - description of the 'FORMAT' string. + description of the 'FORMAT' string. See also the *-e* option. *-p* 'PIDFILE':: @@ -68,9 +68,18 @@ OPTIONS is used: -------- -%A - - [%{%d/%b/%Y:%H:%M:%S %z}t] "%m %u %v" - - "%R" "%G" +%A - - [%{%d/%b/%Y:%H:%M:%S %z}t] "%m %u %v" %c %o "%R" "%G" -------- +*-e*:: + + Make extended log data available. This option makes + *accesslog* run in a different mode where it looks at not only + the request, but also the (entire) response, which requires + quite a bit more CPU time per request. However, some log items + are only available in this mode; these have been marked as + such under the FORMAT section, below. + *-L*:: Do not attempt to lock the logfile. Note that this switch @@ -164,6 +173,32 @@ The following log items are currently specified: Expands into the `User-Agent` header. +The following log items are only available when running in extended +mode, requested by the *-e* option, as described above. If unavailable +due to not running in extended mode, each of the log items below will +instead expand into a dash. + +*%c*:: + + Expands into the HTTP status code of the response. + +*%i*:: + + Expands into the number of bytes sent by the client as a + request-body. HTTP headers are not counted. + +*%o*:: + + Expands into the number of bytes sent back by the handler, to + the client, as the response-body. HTTP headers are not + counted, and neither are overhead as part of any required + transfer-encoding, such as chunking. + +*%d*:: + + Expands into the time it took for the handler to complete the + response, expressed as seconds with 6 decimals precision. + In any expanded field, any "unsafe" characters are escaped. Currently, this means that double-quotes and backslashes are prepended with a backslash, newlines and tabs are expressed as, respectively, `\n` and diff --git a/doc/callscgi.doc b/doc/callscgi.doc index 755e7e9..77cf496 100644 --- a/doc/callscgi.doc +++ b/doc/callscgi.doc @@ -33,7 +33,7 @@ on its standard input, expecting the program to start accepting connections on that socket. *callscgi* will then connect to that socket for each request. If such a connection is refused, the child program will be assumed to have crashed, and *callscgi* will restart -it. Also in anonymous mode, *callscgi*, will try and kill the child +it. Also in anonymous mode, *callscgi* will try and kill the child program whenever *callscgi* itself exits for whatever reason (commonly because its own control socket is closed by its parent handler). diff --git a/doc/dirplex.doc b/doc/dirplex.doc index 10b7609..e2769d7 100644 --- a/doc/dirplex.doc +++ b/doc/dirplex.doc @@ -85,12 +85,13 @@ CONFIGURATION Configuration in *dirplex* comes from several sources. When *dirplex* starts, unless the *-N* option is given, it tries to find a global -configuration file named `dirplex.rc`. It looks in all directories -named by the *PATH* environment variable, appended with -`../etc/ashd`. For example, then, if *PATH* is -`/usr/local/bin:/bin:/usr/bin`, the directories `/usr/local/etc/ashd`, -`/etc/ashd` and `/usr/etc/ashd` are searched for `dirplex.rc`, in that -order. Only the first file found is used, should there exist several. +configuration file named `dirplex.rc`. It looks in `$HOME/.ashd/etc`, +and then in all directories named by the *PATH* environment variable, +appended with `../etc/ashd`. For example, then, if *PATH* is +`/usr/local/bin:/bin:/usr/bin`, the directories `$HOME/.ashd/etc`, +`/usr/local/etc/ashd`, `/etc/ashd` and `/usr/etc/ashd` are searched +for `dirplex.rc`, in that order. Only the first file found is used, +should there exist several. If the *-c* option is given to *dirplex*, it too specifies a configuration file to load. If the name given contains any slashes, it @@ -176,7 +177,7 @@ The following configuration directives are recognized: program will be started in the same directory as the `.htrc` file itself. -*match* [*directory*]:: +*match* ['TYPE']:: Specifies a filename pattern-matching rule. The pattern-matching procedure and the follow-up lines accepted by @@ -192,9 +193,10 @@ The following configuration directives are recognized: be a named request handler specified either in the same `.htrc` file or elsewhere. The *capture* directive accepts no follow-up lines. Note that the `X-Ash-File` header is not - added to requests passed via *capture* directives. If 'FLAGS' - contain the character `R`, this *capture* directive will be - ignored if it is in the root directory that *dirplex* serves. + added to requests passed via *capture* directives. Normally, + *capture* directives will be ignored if they appear in the + root directory that *dirplex* serves, but not if 'FLAGS' + contain the character `D`. MATCHING -------- @@ -208,10 +210,13 @@ ignored. To match a file, any *match* stanzas specified by any `.htrc` file or in the global configuration files are searched in order of their -"distance" (see CONFIGURATION above) from the actual file. If it is a -directory which is being considered, only *match* stanzas with the -*directory* parameter are considered; otherwise, if it is a file, only -*match* stanzas without the *directory* parameter are considered. +"distance" (see CONFIGURATION above) from the actual file. Which +*match* stanzas are considered depends on the type of the file being +matched: if an ordinary file is being matched, only *match* stanzas +without any 'TYPE' parameter are considered, while if it is a +directory, only those with the 'TYPE' parameter specified as +*directory* are considered. 'TYPE' can also take the value *notfound*, +described below under 404 RESPONSES. A *match* stanza must contain at least one follow-up line specifying match rules. All rules must match for the stanza as a whole to match. @@ -295,11 +300,23 @@ A HTTP 404 response is sent to the client if * The mapping procedure results in a file which is not matched by any *match* stanza. -By default, *dirplex* will send a built-in 404 response, but any -`.htrc` file or global configuration may define a request handler -named `.notfound` to customize the behavior. Note that, unlike -successful requests, such a handler will not be passed the -`X-Ash-File` header. +By default, *dirplex* will send a built-in 404 response, but there are +two ways to customize the response: + +First, *match* stanzas with the type *notfound* will be matched +against any request that would result in a 404 error. The filename for +such matching is that of the last succesfully found component, which +may be a directory, for example in case a name component could not be +found in the real filesystem; or a file, for example in case a file +was found, but not matched by any *match* stanzas. + +Otherwise, any request that would result in a 404 response but is +matched by no *notfound* stanza is instead passed to a default handler +named `.notfound`, which is handled internally in *dirplex* by +default, but may be overridden just as any other handler may be in a +`.htrc` file or by global configuration. Note, however, that any +request not matched by a *notfound* stanza will not have the +`X-Ash-File` header added to it. The built-in `.notfound` handler can also be used in *match* or *capture* stanzas (for example, to restrict access to certain files or diff --git a/doc/htextauth.doc b/doc/htextauth.doc index b985492..8c3b6f7 100644 --- a/doc/htextauth.doc +++ b/doc/htextauth.doc @@ -78,6 +78,16 @@ the client. Note that *htextauth* will wait for the authentication program to exit and not process any other requests until then. +FILES +----- +The file `etc/extauth/vhtpasswd` in the *ashd* source distribution is +a simple authenticator program (written in Python) that can be used +with *htextauth*, which verifies the given credentials against a +simple database of users with encrypted passwords. It can be used as +is, or as a simple example of how to produce authenticator +programs. The accompanying `mkhtpasswd` program can be used to +maintain the password database. + AUTHOR ------ Fredrik Tolf diff --git a/doc/httrcall.doc b/doc/httrcall.doc new file mode 100644 index 0000000..555284e --- /dev/null +++ b/doc/httrcall.doc @@ -0,0 +1,47 @@ +httrcall(1) +========== + +NAME +---- +httrcall - Call transient ashd handlers + +SYNOPSIS +-------- +*httrcall* [*-h*] [*-l* 'LIMIT'] 'PROGRAM' ['ARGS'...] + +DESCRIPTION +----------- + +*httrcall* is a persistent handler, as defined in *ashd*(7), but works +by starting a specified transient handler for every incoming +request. Thus, it can be used to run transient handlers where normally +only persistent handlers are accepted, such as for the program +specified to *accesslog*(1), *htextauth*(1) or even *htparser*(1). + +The transient handler to call is specified by the 'PROGRAM' +argument. Any 'ARGS' given are prepended to the usual arguments for +transient handlers, as described in *ashd*(7). + +OPTIONS +------- + +*-h*:: + + Print a brief help message to standard output and exit. + +*-l* 'LIMIT':: + + If specified, only 'LIMIT' copies of the handler program are + allowed to run at one time. If furter request are received + while 'LIMIT' children are running, *httrcall* will block, + waiting for one to exit before processing further requests. If + no *-l* option is specified, *httrcall* imposes no limit on + the number of children that may run. + +AUTHOR +------ +Fredrik Tolf + +SEE ALSO +-------- +*ashd*(7) diff --git a/doc/patplex.doc b/doc/patplex.doc index 65e2d11..86d6ace 100644 --- a/doc/patplex.doc +++ b/doc/patplex.doc @@ -35,12 +35,16 @@ CONFIGURATION In addition to the 'CONFIGFILE' specified on the command-line, *patplex* also attempts to find and read a global configuration file -called `patplex.rc`, unless the *-N* option is given. It looks in all -directories named by the *PATH* environment variable, appended with -`../etc`. For example, then, if *PATH* is -`/usr/local/bin:/bin:/usr/bin`, the directories `/usr/local/etc`, -`/etc` and `/usr/etc` are searched for `patplex.rc`, in that -order. Only the first file found is used, should there exist several. +called `patplex.rc`, unless the *-N* option is given. It looks in +`$HOME/.ashd/etc`, and then in all directories named by the *PATH* +environment variable, appended with `../etc/ashd`. For example, then, +if *PATH* is `/usr/local/bin:/bin:/usr/bin`, the directories +`$HOME/.ashd/etc`, `/usr/local/etc/ashd`, `/etc/ashd` and +`/usr/etc/ashd` are searched for `patplex.rc`, in that order. Only the +first file found is used, should there exist several. If the given +'CONFIGFILE' contains any slashes, it is opened by that exact +name. Otherwise, it is searched for in the same manner as the global +configuration file. Should the global and the given configuration files conflict, the directives from the given file take precedence. @@ -63,14 +67,15 @@ rules are recognized: matched case-independently. If the *match* stanza as a whole matches and contains no *restpat* line (as described below), the rest string of the request is replaced by the remainder of - the rest string after the portion that was matched by 'REGEX'. + the rest string after the portion that was matched by + 'REGEX'. See also URL UNQUOTING, below. *url* 'REGEX' 'FLAGS':: 'REGEX' must be an extended regular expression. The rule is considered to match if 'REGEX' matches the raw URL of the request. If 'FLAGS' contain the character `i`, 'REGEX' is - matched case-independently. + matched case-independently. See also URL UNQUOTING, below. *method* 'REGEX' 'FLAGS':: @@ -151,6 +156,20 @@ optional lines: If no *match* stanza matches, a 404 response is returned to the client. +URL UNQUOTING +------------- + +If the 'FLAGS' of a *point* or *url* rule contain the character `q`, +then the rule's pattern will be matched against a copy of the input +string where URL percent-escapes have been decoded so that, for +example, the regular expression `^~` will match an input string that +begins with either `~`, `%7E` or `%7e`. + +Even if such percent-escapes were decoded, however, the original +version of the string will be used for any *restpat* expansion, +regardlessly of whether the escapes were unquoted inside or outside +the matched part of the string. + SIGNALS ------- diff --git a/doc/psendfile.doc b/doc/psendfile.doc new file mode 100644 index 0000000..46a58b5 --- /dev/null +++ b/doc/psendfile.doc @@ -0,0 +1,43 @@ +psendfile(1) +============ + +NAME +---- +psendfile - Persistent static file handler for ashd(7) + +SYNOPSIS +-------- +*psendfile* [*-h*] + +DESCRIPTION +----------- + +*psendfile* is a variant of *sendfile*(1) that runs as a persistent +handler. Except those actions of *sendfile* explicitly triggered by +command-line options, *psendfile* does the exact same things as +*sendfile*, the only difference being that it runs persistently and +handles all requests in the same process, instead of spawning a +process per request. In doing so, it uses more persistent resources in +order to use less resources per request. As such, *psendfile* is more +suitable for a system under heavier load, while *sendfile* is more +suitable for a system under light load. + +Accordingly, *psendfile* is a persistent handler, as defined in +*ashd*(7). See the *sendfile*(1) manpage for all the details of its +operations, as it mirrors that program exactly except in the special +options it accepts. + +OPTIONS +------- + +*-h*:: + + Print a brief help message to standard output and exit. + +AUTHOR +------ +Fredrik Tolf + +SEE ALSO +-------- +*sendfile*(1), *dirplex*(1), *ashd*(7) diff --git a/doc/sendfile.doc b/doc/sendfile.doc index a958030..57467b1 100644 --- a/doc/sendfile.doc +++ b/doc/sendfile.doc @@ -52,7 +52,7 @@ OPTIONS *-c* 'CONTENT-TYPE':: - Sends 'CONTENT-TYPE' as the file's MIME-type instead of trying + Send 'CONTENT-TYPE' as the file's MIME-type instead of trying to auto-detect the file type. *-f* 'FILE':: @@ -66,4 +66,4 @@ Fredrik Tolf SEE ALSO -------- -*dirplex*(1), *ashd*(7) +*dirplex*(1), *psendfile*(1), *ashd*(7) diff --git a/etc/ashd/dirplex.d/python.rc b/etc/ashd/dirplex.d/python.rc new file mode 100644 index 0000000..602833f --- /dev/null +++ b/etc/ashd/dirplex.d/python.rc @@ -0,0 +1,18 @@ +child wsgidir + exec ashd-wsgi ashd.wsgidir +child wsgidir3 + exec ashd-wsgi3 ashd.wsgidir + +match + filename *.wsgi + xset python-handler chain + handler wsgidir +match + filename *.wsgi2 + xset python-handler chain + handler wsgidir + +match + filename *.wsgi3 + xset python-handler chain + handler wsgidir3 diff --git a/etc/extauth/mkhtpasswd b/etc/extauth/mkhtpasswd new file mode 100755 index 0000000..923ab07 --- /dev/null +++ b/etc/extauth/mkhtpasswd @@ -0,0 +1,37 @@ +#!/usr/bin/python + +import sys, os, termios, hmac, hashlib, getopt, getpass + +def usage(out): + out.write("usage: mkhtpasswd [-h] FILE USERNAME\n") + +opts, args = getopt.getopt(sys.argv[1:], "h") +for o, a in opts: + if o == "-h": + usage(sys.stdout) + sys.exit(0) +if len(args) < 2: + usage(sys.stderr) + sys.exit(1) + +def hashpw(usr, pw): + dig = hmac.new(pw, digestmod=hashlib.sha1) + dig.update(usr) + return dig.hexdigest() + +if ':' in args[1]: + sys.stderr.write("mkhtpasswd: username cannot contain `:'\n") + sys.exit(1) + +passwds = {} +if os.path.exists(args[0]): + with open(args[0]) as fp: + for line in fp: + usr, pw = line.strip().split(':') + passwds[usr] = pw + +passwds[args[1]] = hashpw(args[1], getpass.getpass()) + +with open(args[0], "w") as fp: + for usr, pw in passwds.iteritems(): + fp.write("%s:%s\n" % (usr, pw)) diff --git a/etc/extauth/vhtpasswd b/etc/extauth/vhtpasswd new file mode 100755 index 0000000..422206d --- /dev/null +++ b/etc/extauth/vhtpasswd @@ -0,0 +1,34 @@ +#!/usr/bin/python + +import sys, hmac, hashlib, getopt + +def usage(out): + out.write("usage: vhtpasswd [-h] FILE\n") + +opts, args = getopt.getopt(sys.argv[1:], "h") +for o, a in opts: + if o == "-h": + usage(sys.stdout) + sys.exit(0) +if len(args) < 1: + usage(sys.stderr) + sys.exit(1) + +def hashpw(usr, pw): + dig = hmac.new(pw, digestmod=hashlib.sha1) + dig.update(usr) + return dig.hexdigest() + +def findpw(fn, name): + with open(fn) as fp: + for line in fp: + usr, pw = line.strip().split(':') + if usr == name: + return pw + return None + +usr = sys.stdin.readline().strip() +gpw = sys.stdin.readline().strip() +if findpw(args[0], usr) == hashpw(usr, gpw): + sys.exit(0) +sys.exit(1) diff --git a/examples/python/dynhosts/dynhosts b/examples/python/dynhosts/dynhosts index 9a69a70..0e10b73 100755 --- a/examples/python/dynhosts/dynhosts +++ b/examples/python/dynhosts/dynhosts @@ -20,6 +20,6 @@ def serve(req): children[dname] = util.pchild(["dirplex", path], autorespawn = True) children[dname].passreq(req) return - util.respond(req, "No such host in configured.\n", status = "404 Not Found", ctype = "text/plain") + util.respond(req, "No such host is configured.\n", status = "404 Not Found", ctype = "text/plain") util.serveloop(serve) diff --git a/examples/python/dynhosts/run b/examples/python/dynhosts/run index d4203ea..07f6673 100755 --- a/examples/python/dynhosts/run +++ b/examples/python/dynhosts/run @@ -4,9 +4,5 @@ set -e cd "$(dirname "$0")" -# Start htparser running this dynhosts script. The setsid command -# ensures that SIGINT is only received by htparser and not by -# dynhosts; it is not of grave importance, but makes shutdown slightly -# more clean, and hides the KeyboardInterrupt otherwise raised by -# Python. -htparser plain:port=8080 -- setsid ./dynhosts . +# Start htparser running this dynhosts script. +htparser plain:port=8080 -- ./dynhosts . diff --git a/examples/python/wsgidir/run b/examples/python/wsgidir/run index 5dbe5d2..8931d4c 100755 --- a/examples/python/wsgidir/run +++ b/examples/python/wsgidir/run @@ -7,9 +7,4 @@ cd "$(dirname "$0")" # Invoke dirplex running in this directory, loading the wsgidir.rc # configuration file. The same configuration can be put in # e.g. /etc/ashd/dirplex.d or in any .htrc file. - -# The setsid command ensures that SIGINT is only received by htparser -# and not by dirplex or its children; it is not of any importance, but -# makes shutdown slightly cleaner, and hides the KeyboardInterrupt -# otherwise raised by Python. -htparser plain:port=8080 -- setsid dirplex -c ./wsgidir.rc . +htparser plain:port=8080 -- dirplex -c ./wsgidir.rc . diff --git a/lib/Makefile.am b/lib/Makefile.am index d48cfb6..0f13d2b 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -1,12 +1,17 @@ lib_LIBRARIES = libht.a -libht_a_SOURCES = utils.c mt.c log.c req.c proc.c mtio.c resp.c cf.c +libht_a_SOURCES = utils.c mt.c log.c req.c proc.c mtio.c resp.c \ + cf.c bufio.c libht_a_CFLAGS = -fPIC -libht_a_CPPFLAGS = -D_GNU_SOURCE if USE_EPOLL libht_a_SOURCES += mtio-epoll.c else +if USE_KQUEUE +libht_a_SOURCES += mtio-kqueue.c +else libht_a_SOURCES += mtio-select.c endif +endif -pkginclude_HEADERS = utils.h mt.h log.h req.h proc.h mtio.h resp.h cf.h +pkginclude_HEADERS = utils.h mt.h log.h req.h proc.h mtio.h resp.h \ + cf.h bufio.h diff --git a/lib/bufio.c b/lib/bufio.c new file mode 100644 index 0000000..11d8c04 --- /dev/null +++ b/lib/bufio.c @@ -0,0 +1,329 @@ +/* + ashd - A Sane HTTP Daemon + Copyright (C) 2008 Fredrik Tolf + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +*/ + +#ifdef HAVE_CONFIG_H +#include +#endif +#include +#include +#include + +#include +#include + +struct bufio *bioopen(void *pdata, struct bufioops *ops) +{ + struct bufio *bio; + + omalloc(bio); + bio->pdata = pdata; + bio->ops = ops; + bio->bufhint = 4096; + return(bio); +} + +int bioclose(struct bufio *bio) +{ + int rv; + + bioflush(bio); + if(bio->ops->close) + rv = bio->ops->close(bio->pdata); + else + rv = 0; + buffree(bio->rbuf); + buffree(bio->wbuf); + free(bio); + return(rv); +} + +size_t biordata(struct bufio *bio) +{ + return(bio->rbuf.d - bio->rh); +} + +size_t biorspace(struct bufio *bio) +{ + if((bio->rbuf.d - bio->rh) >= bio->bufhint) + return(0); + return(bio->bufhint - (bio->rbuf.d - bio->rh)); +} + +int bioeof(struct bufio *bio) +{ + return(bio->eof && (bio->rh >= bio->rbuf.d)); +} + +static ssize_t biofill(struct bufio *bio) +{ + size_t ns; + ssize_t ret; + + if(!bio->ops->read) { + bio->eof = 1; + return(0); + } + if(bio->eof) + return(0); + if(bio->rh == bio->rbuf.d) + bio->rh = bio->rbuf.d = 0; + if(bio->rbuf.d == bio->rbuf.s) { + if(bio->rh > 0) { + memmove(bio->rbuf.b, bio->rbuf.b + bio->rh, bio->rbuf.d -= bio->rh); + bio->rh = 0; + } else { + if((ns = bio->rbuf.s * 2) < bio->bufhint) + ns = bio->bufhint; + sizebuf(bio->rbuf, ns); + } + } + if((bio->rbuf.s > bio->bufhint) && (bio->rbuf.d < bio->bufhint)) + bio->rbuf.b = srealloc(bio->rbuf.b, bio->rbuf.s = bio->bufhint); + ret = bio->ops->read(bio->pdata, bio->rbuf.b + bio->rbuf.d, bio->rbuf.s - bio->rbuf.d); + if(ret < 0) { + bio->err = errno; + return(-1); + } else if(ret == 0) { + bio->eof = 1; + return(0); + } + bio->rbuf.d += ret; + return(bio->rbuf.d - bio->rh); +} + +ssize_t biorensure(struct bufio *bio, size_t bytes) +{ + ssize_t ret; + + while(bio->rbuf.d - bio->rh < bytes) { + if((ret = biofill(bio)) <= 0) + return(ret); + } + return(bio->rbuf.d - bio->rh); +} + +ssize_t biofillsome(struct bufio *bio) +{ + return(biofill(bio)); +} + +int biogetc(struct bufio *bio) +{ + ssize_t ret; + + while(bio->rbuf.d <= bio->rh) { + if((ret = biofill(bio)) <= 0) + return(EOF); + } + return((unsigned char)bio->rbuf.b[bio->rh++]); +} + +ssize_t bioreadsome(struct bufio *bio, void *buf, size_t len) +{ + ssize_t ret; + + if((bio->rh >= bio->rbuf.d) && ((ret = biofill(bio)) <= 0)) + return(ret); + ret = min(len, bio->rbuf.d - bio->rh); + memcpy(buf, bio->rbuf.b + bio->rh, ret); + bio->rh += ret; + return(ret); +} + +size_t biowdata(struct bufio *bio) +{ + return(bio->wbuf.d - bio->wh); +} + +size_t biowspace(struct bufio *bio) +{ + if((bio->wbuf.d - bio->wh) >= bio->bufhint) + return(0); + return(bio->bufhint - (bio->wbuf.d - bio->wh)); +} + +int bioflush(struct bufio *bio) +{ + ssize_t ret; + + while(bio->wh < bio->wbuf.d) { + ret = bio->ops->write(bio->pdata, bio->wbuf.b + bio->wh, bio->wbuf.d - bio->wh); + if(ret < 0) { + bio->err = errno; + return(-1); + } + bio->wh += ret; + } + return(0); +} + +int bioflushsome(struct bufio *bio) +{ + ssize_t ret; + + if(bio->wh < bio->wbuf.d) { + ret = bio->ops->write(bio->pdata, bio->wbuf.b + bio->wh, bio->wbuf.d - bio->wh); + if(ret < 0) { + bio->err = errno; + return(-1); + } + bio->wh += ret; + return(1); + } else { + return(0); + } +} + +ssize_t biowensure(struct bufio *bio, size_t bytes) +{ + if(bio->wbuf.s - bio->wbuf.d < bytes) { + if(!bio->ops->write) { + errno = bio->err = EPIPE; + return(-1); + } + if(bioflush(bio) < 0) + return(-1); + bio->wh = bio->wbuf.d = 0; + if((bio->wbuf.s > bio->bufhint) && (bytes <= bio->bufhint)) + bio->wbuf.b = srealloc(bio->wbuf.b, bio->wbuf.s = bio->bufhint); + else + sizebuf(bio->wbuf, (bytes < bio->bufhint)?bio->bufhint:bytes); + } + return(0); +} + +int bioputc(struct bufio *bio, int c) +{ + if(biowensure(bio, 1) < 0) + return(-1); + bio->wbuf.b[bio->wbuf.d++] = c; + return(0); +} + +ssize_t biowrite(struct bufio *bio, const void *data, size_t len) +{ + ssize_t wb, ret; + + wb = 0; + while(len > 0) { + if(biowensure(bio, min(len, bio->bufhint)) < 0) { + if(wb > 0) + return(wb); + return(-1); + } + if(len < bio->wbuf.s - bio->wbuf.d) { + memcpy(bio->wbuf.b + bio->wbuf.d, data, len); + bio->wbuf.d += len; + wb += len; + len = 0; + } else { + if(bioflush(bio) < 0) { + if(wb > 0) + return(wb); + return(-1); + } + bio->wh = bio->wbuf.d = 0; + ret = bio->ops->write(bio->pdata, data, len); + if(ret < 0) { + if(wb > 0) + return(wb); + bio->err = errno; + return(-1); + } + data += ret; len -= ret; wb += ret; + } + } + return(wb); +} + +ssize_t biowritesome(struct bufio *bio, const void *data, size_t len) +{ + ssize_t ret; + + sizebuf(bio->wbuf, bio->bufhint); + if(bio->wh == bio->wbuf.d) + bio->wh = bio->wbuf.d = 0; + if(bio->wbuf.d == bio->wbuf.s) { + if(bio->wh > 0) { + memmove(bio->wbuf.b, bio->wbuf.b + bio->wh, bio->wbuf.d -= bio->wh); + bio->wh = 0; + } + } + ret = min(len, bio->wbuf.s - bio->wbuf.d); + memcpy(bio->wbuf.b + bio->wbuf.d, data, ret); + bio->wbuf.d += ret; + if(bioflushsome(bio) < 0) { + if(ret == 0) + return(-1); + if(ret < bio->wbuf.d - bio->wh) { /* Should never be false */ + bio->wbuf.d -= ret; + return(-1); + } + } + return(ret); +} + +int bioprintf(struct bufio *bio, const char *format, ...) +{ + va_list args; + int ret; + + if(biowensure(bio, strlen(format)) < 0) + return(-1); + while(1) { + va_start(args, format); + ret = vsnprintf(bio->wbuf.b + bio->wbuf.d, bio->wbuf.s - bio->wbuf.d, format, args); + va_end(args); + if(ret < bio->wbuf.s - bio->wbuf.d) { + bio->wbuf.d += ret; + return(0); + } + if(biowensure(bio, ret + 1) < 0) + return(-1); + } +} + +ssize_t biocopysome(struct bufio *dst, struct bufio *src) +{ + ssize_t ret; + + if(src->rh >= src->rbuf.d) + return(0); + if((ret = biowritesome(dst, src->rbuf.b + src->rh, src->rbuf.d - src->rh)) < 0) + return(-1); + src->rh += ret; + return(ret); +} + +ssize_t biocopybuf(struct bufio *dst, struct bufio *src) +{ + ssize_t ret; + + sizebuf(dst->wbuf, dst->bufhint); + if(dst->wbuf.d == dst->wbuf.s) { + if(dst->wh > 0) { + memmove(dst->wbuf.b, dst->wbuf.b + dst->wh, dst->wbuf.d -= dst->wh); + dst->wh = 0; + } + } + ret = min(src->rbuf.d - src->rh, dst->wbuf.s - dst->wbuf.d); + memcpy(dst->wbuf.b + dst->wbuf.d, src->rbuf.b + src->rh, ret); + src->rh += ret; + dst->wbuf.d += ret; + return(ret); +} diff --git a/lib/bufio.h b/lib/bufio.h new file mode 100644 index 0000000..5874577 --- /dev/null +++ b/lib/bufio.h @@ -0,0 +1,41 @@ +#ifndef _LIB_BUFIO_H +#define _LIB_BUFIO_H + +struct bufioops { + ssize_t (*read)(void *pdata, void *buf, size_t len); + ssize_t (*write)(void *pdata, const void *buf, size_t len); + int (*close)(void *pdata); +}; + +struct bufio { + struct charbuf rbuf, wbuf; + size_t rh, wh, bufhint; + int err, eof; + void *pdata; + struct bufioops *ops; +}; + +struct bufio *bioopen(void *pdata, struct bufioops *ops); +int bioclose(struct bufio *bio); + +size_t biordata(struct bufio *bio); +size_t biorspace(struct bufio *bio); +int bioeof(struct bufio *bio); +ssize_t biorensure(struct bufio *bio, size_t bytes); +ssize_t biofillsome(struct bufio *bio); +int biogetc(struct bufio *bio); +ssize_t bioreadsome(struct bufio *bio, void *buf, size_t len); + +size_t biowdata(struct bufio *bio); +size_t biowspace(struct bufio *bio); +int bioflush(struct bufio *bio); +int bioflushsome(struct bufio *bio); +ssize_t biowensure(struct bufio *bio, size_t bytes); +int bioputc(struct bufio *bio, int c); +ssize_t biowrite(struct bufio *bio, const void *data, size_t len); +ssize_t biowritesome(struct bufio *bio, const void *data, size_t len); +int bioprintf(struct bufio *bio, const char *format, ...); +ssize_t biocopysome(struct bufio *dst, struct bufio *src); +ssize_t biocopybuf(struct bufio *dst, struct bufio *src); + +#endif diff --git a/lib/cf.c b/lib/cf.c index 38d9808..b809d33 100644 --- a/lib/cf.c +++ b/lib/cf.c @@ -23,6 +23,8 @@ #include #include #include +#include +#include #include #ifdef HAVE_CONFIG_H @@ -40,7 +42,10 @@ struct stdchild { int type; char **argv; + char **envp; int fd; + int agains; + time_t lastrep; }; static int parsefile(struct cfstate *s, FILE *in); @@ -232,22 +237,27 @@ void freecfparser(struct cfstate *s) char *findstdconf(char *name) { - char *path, *p, *p2, *t; + char *home, *path, *p, *p2, *t; - if((path = getenv("PATH")) == NULL) - return(NULL); - path = sstrdup(path); - for(p = strtok(path, ":"); p != NULL; p = strtok(NULL, ":")) { - if((p2 = strrchr(p, '/')) == NULL) - continue; - *p2 = 0; - if(!access(t = sprintf2("%s/etc/%s", p, name), R_OK)) { - free(path); + if((home = getenv("HOME")) != NULL) { + if(!access(t = sprintf2("%s/.ashd/etc/%s", home, name), R_OK)) return(t); - } free(t); } - free(path); + if((path = getenv("PATH")) != NULL) { + path = sstrdup(path); + for(p = strtok(path, ":"); p != NULL; p = strtok(NULL, ":")) { + if((p2 = strrchr(p, '/')) == NULL) + continue; + *p2 = 0; + if(!access(t = sprintf2("%s/etc/%s", p, name), R_OK)) { + free(path); + return(t); + } + free(t); + } + free(path); + } return(NULL); } @@ -302,29 +312,120 @@ static struct chandler stdhandler = { .destroy = stddestroy, }; -static int stdhandle(struct child *ch, struct hthead *req, int fd, void (*chinit)(void *), void *idata) +static char **expandargs(struct stdchild *sd) +{ + int i; + char **ret, *p, *p2, *p3, *np, *env; + struct charbuf exp; + + ret = szmalloc(sizeof(*ret) * (calen(sd->argv) + 1)); + bufinit(exp); + for(i = 0; sd->argv[i] != NULL; i++) { + if((p = strchr(sd->argv[i], '$')) == NULL) { + ret[i] = sstrdup(sd->argv[i]); + } else { + exp.d = 0; + for(p2 = sd->argv[i]; p != NULL; p2 = np, p = strchr(np, '$')) { + bufcat(exp, p2, p - p2); + if(p[1] == '{') { + if((p3 = strchr((p += 2), '}')) == NULL) { + np = p; + break; + } + np = p3 + 1; + } else { + for(p3 = ++p; *p3; p3++) { + if(!(((*p3 >= 'a') && (*p3 <= 'z')) || + ((*p3 >= 'A') && (*p3 <= 'Z')) || + ((*p3 >= '0') && (*p3 <= '9')) || + (*p3 == '_'))) { + break; + } + } + np = p3; + } + char temp[(p3 - p) + 1]; + memcpy(temp, p, p3 - p); + temp[p3 - p] = 0; + if((env = getenv(temp)) != NULL) + bufcatstr(exp, env); + } + bufcatstr2(exp, np); + ret[i] = sstrdup(exp.b); + } + } + ret[i] = NULL; + buffree(exp); + return(ret); +} + +struct sidata { + struct stdchild *sd; + void (*sinit)(void *); + void *sdata; +}; + +static void stdinit(void *data) +{ + struct sidata *d = data; + int i; + + for(i = 0; d->sd->envp[i]; i += 2) + putenv(sprintf2("%s=%s", d->sd->envp[i], d->sd->envp[i + 1])); + if(d->sinit != NULL) + d->sinit(d->sdata); +} + +static int stdhandle(struct child *ch, struct hthead *req, int fd, void (*chinit)(void *), void *sdata) { - struct stdchild *i = ch->pdata; + struct stdchild *sd = ch->pdata; + int serr; + char **args; + struct sidata idat; - if(i->type == CH_SOCKET) { - if(i->fd < 0) - i->fd = stdmkchild(i->argv, chinit, idata); - if(sendreq(i->fd, req, fd)) { - if((errno == EPIPE) || (errno == ECONNRESET)) { + if(sd->type == CH_SOCKET) { + idat = (struct sidata) {.sd = sd, .sinit = chinit, .sdata = sdata}; + if(sd->fd < 0) { + args = expandargs(sd); + sd->fd = stdmkchild(args, stdinit, &idat); + freeca(args); + } + if(sendreq2(sd->fd, req, fd, MSG_NOSIGNAL | MSG_DONTWAIT)) { + serr = errno; + if((serr == EPIPE) || (serr == ECONNRESET)) { /* Assume that the child has crashed and restart it. */ - close(i->fd); - i->fd = stdmkchild(i->argv, chinit, idata); - if(!sendreq(i->fd, req, fd)) - return(0); + close(sd->fd); + args = expandargs(sd); + sd->fd = stdmkchild(args, stdinit, &idat); + freeca(args); + if(!sendreq2(sd->fd, req, fd, MSG_NOSIGNAL | MSG_DONTWAIT)) + goto ok; + serr = errno; + } + if(serr == EAGAIN) { + if(sd->agains++ == 0) { + flog(LOG_WARNING, "request to child %s denied due to buffer overload", ch->name); + sd->lastrep = time(NULL); + } + } else { + flog(LOG_ERR, "could not pass on request to child %s: %s", ch->name, strerror(serr)); + close(sd->fd); + sd->fd = -1; } - flog(LOG_ERR, "could not pass on request to child %s: %s", ch->name, strerror(errno)); - close(i->fd); - i->fd = -1; return(-1); } - } else if(i->type == CH_FORK) { - if(stdforkserve(i->argv, req, fd, chinit, idata) < 0) + ok: + if((sd->agains > 0) && ((time(NULL) - sd->lastrep) > 10)) { + flog(LOG_WARNING, "%i requests to child %s were denied due to buffer overload", sd->agains, ch->name); + sd->agains = 0; + } + } else if(sd->type == CH_FORK) { + args = expandargs(sd); + if(stdforkserve(args, req, fd, chinit, sdata) < 0) { + freeca(args); return(-1); + } + freeca(args); } return(0); } @@ -349,6 +450,8 @@ static void stddestroy(struct child *ch) close(d->fd); if(d->argv) freeca(d->argv); + if(d->envp) + freeca(d->envp); free(d); } @@ -356,6 +459,7 @@ struct child *parsechild(struct cfstate *s) { struct child *ch; struct stdchild *d; + struct charvbuf envbuf; int i; int sl; @@ -383,6 +487,7 @@ struct child *parsechild(struct cfstate *s) } d->fd = -1; + bufinit(envbuf); while(1) { getcfline(s); if(!strcmp(s->argv[0], "exec")) { @@ -393,12 +498,21 @@ struct child *parsechild(struct cfstate *s) d->argv = szmalloc(sizeof(*d->argv) * s->argc); for(i = 0; i < s->argc - 1; i++) d->argv[i] = sstrdup(s->argv[i + 1]); + } else if(!strcmp(s->argv[0], "env")) { + if(s->argc < 3) { + flog(LOG_WARNING, "%s:%i: too few parameters to `env'", s->file, s->lno); + continue; + } + bufadd(envbuf, sstrdup(s->argv[1])); + bufadd(envbuf, sstrdup(s->argv[2])); } else if(!strcmp(s->argv[0], "end") || !strcmp(s->argv[0], "eof")) { break; } else { flog(LOG_WARNING, "%s:%i: unknown directive `%s' in child declaration", s->file, s->lno, s->argv[0]); } } + bufadd(envbuf, NULL); + d->envp = envbuf.b; if(d->argv == NULL) { flog(LOG_WARNING, "%s:%i: missing `exec' in child declaration %s", s->file, sl, ch->name); freechild(ch); diff --git a/lib/mtio-epoll.c b/lib/mtio-epoll.c index 2a25992..3082eaf 100644 --- a/lib/mtio-epoll.c +++ b/lib/mtio-epoll.c @@ -37,7 +37,7 @@ static struct blocker *blockers; struct blocker { struct blocker *n, *p, *n2, *p2; int fd, reg; - int ev; + int ev, rev, id; int thpos; time_t to; struct muth *th; @@ -195,36 +195,76 @@ static void deltimeout(struct blocker *bl) thlower(ent, n); } -int block(int fd, int ev, time_t to) +static int addblock(struct blocker *bl) { - struct blocker *bl; - int rv; - - omalloc(bl); - bl->fd = fd; - bl->ev = ev; - bl->th = current; - if((epfd >= 0) && regfd(bl)) { - free(bl); + if((epfd >= 0) && regfd(bl)) return(-1); - } bl->n = blockers; if(blockers) blockers->p = bl; blockers = bl; - if(to > 0) - addtimeout(bl, bl->to = (time(NULL) + to)); - rv = yield(); + if(bl->to > 0) + addtimeout(bl, bl->to); + return(0); +} + +static void remblock(struct blocker *bl) +{ if(bl->to > 0) deltimeout(bl); if(bl->n) bl->n->p = bl->p; if(bl->p) bl->p->n = bl->n; - if(bl == blockers) + if(blockers == bl) blockers = bl->n; remfd(bl); - free(bl); +} + +struct selected mblock(time_t to, int n, struct selected *spec) +{ + int i, id; + struct blocker bls[n]; + + to = (to > 0)?(time(NULL) + to):0; + for(i = 0; i < n; i++) { + bls[i] = (struct blocker) { + .fd = spec[i].fd, + .ev = spec[i].ev, + .id = i, + .to = to, + .th = current, + }; + if(addblock(&bls[i])) { + for(i--; i >= 0; i--) + remblock(&bls[i]); + return((struct selected){.fd = -1, .ev = -1}); + } + } + id = yield(); + for(i = 0; i < n; i++) + remblock(&bls[i]); + if(id < 0) + return((struct selected){.fd = -1, .ev = -1}); + return((struct selected){.fd = bls[id].fd, .ev = bls[id].rev}); +} + +int block(int fd, int ev, time_t to) +{ + struct blocker bl; + int rv; + + bl = (struct blocker) { + .fd = fd, + .ev = ev, + .id = -1, + .to = (to > 0)?(time(NULL) + to):0, + .th = current, + }; + if(addblock(&bl)) + return(-1); + rv = yield(); + remblock(&bl); return(rv); } @@ -275,13 +315,25 @@ int ioloop(void) ev = -1; for(bl = fdlist[fd]; bl; bl = nbl) { nbl = bl->n2; - if((ev < 0) || (ev & bl->ev)) - resume(bl->th, ev); + if((ev < 0) || (ev & bl->ev)) { + if(bl->id < 0) { + resume(bl->th, ev); + } else { + bl->rev = ev; + resume(bl->th, bl->id); + } + } } } now = time(NULL); - while((timeheap.d > 0) && (timeheap.b[0].to <= now)) - resume(timeheap.b[0].bl->th, 0); + while((timeheap.d > 0) && (timeheap.b[0].to <= now)) { + if(bl->id < 0) { + resume(timeheap.b[0].bl->th, 0); + } else { + bl->rev = 0; + resume(bl->th, bl->id); + } + } } for(bl = blockers; bl; bl = bl->n) remfd(bl); diff --git a/lib/mtio-kqueue.c b/lib/mtio-kqueue.c new file mode 100644 index 0000000..e317153 --- /dev/null +++ b/lib/mtio-kqueue.c @@ -0,0 +1,287 @@ +/* + ashd - A Sane HTTP Daemon + Copyright (C) 2008 Fredrik Tolf + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +*/ + +#include +#include +#include +#include +#include +#include +#include + +#ifdef HAVE_CONFIG_H +#include +#endif +#include +#include +#include +#include + +static struct blocker *blockers; + +struct blocker { + struct blocker *n, *p, *n2, *p2; + int fd, reg; + int ev, rev, id; + time_t to; + struct muth *th; +}; + +static int qfd = -1, fdln = 0; +static int exitstatus; +static struct blocker **fdlist; + +static int regfd(struct blocker *bl) +{ + struct blocker *o; + int prev; + struct kevent evd; + + if(bl->fd >= fdln) { + if(fdlist) { + fdlist = srealloc(fdlist, sizeof(*fdlist) * (bl->fd + 1)); + memset(fdlist + fdln, 0, sizeof(*fdlist) * (bl->fd + 1 - fdln)); + fdln = bl->fd + 1; + } else { + fdlist = szmalloc(sizeof(*fdlist) * (fdln = (bl->fd + 1))); + } + } + for(prev = 0, o = fdlist[bl->fd]; o; o = o->n2) + prev |= o->ev; + if((bl->ev & EV_READ) && !(prev & EV_READ)) { + evd = (struct kevent) { + .flags = EV_ADD, + .ident = bl->fd, + .filter = EVFILT_READ, + }; + if(kevent(qfd, &evd, 1, NULL, 0, NULL) < 0) { + /* XXX?! Whatever to do, really? */ + flog(LOG_ERR, "kevent(EV_ADD, EVFILT_READ) on fd %i: %s", bl->fd, strerror(errno)); + return(-1); + } + } + if((bl->ev & EV_WRITE) && !(prev & EV_WRITE)) { + evd = (struct kevent) { + .flags = EV_ADD, + .ident = bl->fd, + .filter = EVFILT_WRITE, + }; + if(kevent(qfd, &evd, 1, NULL, 0, NULL) < 0) { + /* XXX?! Whatever to do, really? */ + flog(LOG_ERR, "kevent(EV_ADD, EVFILT_WRITE) on fd %i: %s", bl->fd, strerror(errno)); + return(-1); + } + } + bl->n2 = fdlist[bl->fd]; + bl->p2 = NULL; + if(fdlist[bl->fd] != NULL) + fdlist[bl->fd]->p2 = bl; + fdlist[bl->fd] = bl; + bl->reg = 1; + return(0); +} + +static void remfd(struct blocker *bl) +{ + struct blocker *o; + struct kevent evd; + int left; + + if(!bl->reg) + return; + if(bl->n2) + bl->n2->p2 = bl->p2; + if(bl->p2) + bl->p2->n2 = bl->n2; + if(bl == fdlist[bl->fd]) + fdlist[bl->fd] = bl->n2; + for(left = 0, o = fdlist[bl->fd]; o; o = o->n2) + left |= o->ev; + if((bl->ev & EV_READ) && !(left & EV_READ)) { + evd = (struct kevent) { + .flags = EV_DELETE, + .ident = bl->fd, + .filter = EVFILT_READ, + }; + if(kevent(qfd, &evd, 1, NULL, 0, NULL) < 0) { + /* XXX?! Whatever to do, really? */ + flog(LOG_ERR, "kevent(EV_DELETE, EVFILT_READ) on fd %i: %s", bl->fd, strerror(errno)); + } + } + if((bl->ev & EV_WRITE) && !(left & EV_WRITE)) { + evd = (struct kevent) { + .flags = EV_DELETE, + .ident = bl->fd, + .filter = EVFILT_WRITE, + }; + if(kevent(qfd, &evd, 1, NULL, 0, NULL) < 0) { + /* XXX?! Whatever to do, really? */ + flog(LOG_ERR, "kevent(EV_DELETE, EVFILT_WRITE) on fd %i: %s", bl->fd, strerror(errno)); + } + } + bl->reg = 0; +} + +static int addblock(struct blocker *bl) +{ + if((qfd >= 0) && regfd(bl)) + return(-1); + bl->n = blockers; + if(blockers) + blockers->p = bl; + blockers = bl; + return(0); +} + +static void remblock(struct blocker *bl) +{ + if(bl->n) + bl->n->p = bl->p; + if(bl->p) + bl->p->n = bl->n; + if(bl == blockers) + blockers = bl->n; + remfd(bl); +} + +struct selected mblock(time_t to, int n, struct selected *spec) +{ + int i, id; + struct blocker bls[n]; + + to = (to > 0)?(time(NULL) + to):0; + for(i = 0; i < n; i++) { + bls[i] = (struct blocker) { + .fd = spec[i].fd, + .ev = spec[i].ev, + .id = i, + .to = to, + .th = current, + }; + if(addblock(&bls[i])) { + for(i--; i >= 0; i--) + remblock(&bls[i]); + return((struct selected){.fd = -1, .ev = -1}); + } + } + id = yield(); + for(i = 0; i < n; i++) + remblock(&bls[i]); + if(id < 0) + return((struct selected){.fd = -1, .ev = -1}); + return((struct selected){.fd = bls[id].fd, .ev = bls[id].rev}); +} + +int block(int fd, int ev, time_t to) +{ + struct blocker bl; + int rv; + + bl = (struct blocker) { + .fd = fd, + .ev = ev, + .id = -1, + .to = (to > 0)?(time(NULL) + to):0, + .th = current, + }; + addblock(&bl); + rv = yield(); + remblock(&bl); + return(rv); +} + +int ioloop(void) +{ + struct blocker *bl, *nbl; + struct kevent evs[16]; + int i, fd, nev, ev; + time_t now, timeout; + struct timespec *toval; + + exitstatus = 0; + qfd = kqueue(); + fcntl(qfd, F_SETFD, FD_CLOEXEC); + for(bl = blockers; bl; bl = nbl) { + nbl = bl->n; + if(regfd(bl)) + resume(bl->th, -1); + } + while(blockers != NULL) { + timeout = 0; + for(bl = blockers; bl; bl = bl->n) { + if((bl->to != 0) && ((timeout == 0) || (timeout > bl->to))) + timeout = bl->to; + } + now = time(NULL); + if(timeout == 0) + toval = NULL; + else if(timeout > now) + toval = &(struct timespec){.tv_sec = timeout - now}; + else + toval = &(struct timespec){.tv_sec = 1}; + if(exitstatus) + break; + nev = kevent(qfd, NULL, 0, evs, sizeof(evs) / sizeof(*evs), toval); + if(nev < 0) { + if(errno != EINTR) { + flog(LOG_CRIT, "ioloop: kevent errored out: %s", strerror(errno)); + /* To avoid CPU hogging in case it's bad, which it + * probably is. */ + sleep(1); + } + continue; + } + for(i = 0; i < nev; i++) { + fd = (int)evs[i].ident; + ev = (evs[i].filter == EVFILT_READ)?EV_READ:EV_WRITE; + for(bl = fdlist[fd]; bl; bl = nbl) { + nbl = bl->n2; + if(ev & bl->ev) { + if(bl->id < 0) { + resume(bl->th, ev); + } else { + bl->rev = ev; + resume(bl->th, bl->id); + } + } + } + } + now = time(NULL); + for(bl = blockers; bl; bl = nbl) { + nbl = bl->n; + if((bl->to != 0) && (bl->to <= now)) { + if(bl->id < 0) { + resume(bl->th, 0); + } else { + bl->rev = 0; + resume(bl->th, bl->id); + } + } + } + } + for(bl = blockers; bl; bl = bl->n) + remfd(bl); + close(qfd); + qfd = -1; + return(exitstatus); +} + +void exitioloop(int status) +{ + exitstatus = status; +} diff --git a/lib/mtio-select.c b/lib/mtio-select.c index 26e6785..e0a4177 100644 --- a/lib/mtio-select.c +++ b/lib/mtio-select.c @@ -36,40 +36,84 @@ static int exitstatus; struct blocker { struct blocker *n, *p; + struct iterator *it; int fd; - int ev; + int ev, rev, id; time_t to; struct muth *th; }; -int block(int fd, int ev, time_t to) -{ +struct iterator { struct blocker *bl; - int rv; - - if(fd >= FD_SETSIZE) { - flog(LOG_ERR, "tried to use more file descriptors than select() can handle: fd %i", fd); - errno = EMFILE; - return(-1); - } - omalloc(bl); - bl->fd = fd; - bl->ev = ev; - if(to > 0) - bl->to = time(NULL) + to; - bl->th = current; +}; + +static void addblock(struct blocker *bl) +{ bl->n = blockers; if(blockers) blockers->p = bl; blockers = bl; - rv = yield(); +} + +static void remblock(struct blocker *bl) +{ if(bl->n) bl->n->p = bl->p; if(bl->p) bl->p->n = bl->n; if(bl == blockers) blockers = bl->n; - free(bl); + if(bl->it) { + if((bl->it->bl = bl->n) != NULL) + bl->it->bl->it = bl->it; + bl->it = NULL; + } +} + +struct selected mblock(time_t to, int n, struct selected *spec) +{ + int i, id; + struct blocker bls[n]; + + to = (to > 0)?(time(NULL) + to):0; + for(i = 0; i < n; i++) { + bls[i] = (struct blocker){ + .fd = spec[i].fd, + .ev = spec[i].ev, + .id = i, + .to = to, + .th = current, + }; + addblock(&bls[i]); + } + id = yield(); + for(i = 0; i < n; i++) + remblock(&bls[i]); + if(id < 0) + return((struct selected){.fd = -1, .ev = -1}); + return((struct selected){.fd = bls[id].fd, .ev = bls[id].rev}); +} + +int block(int fd, int ev, time_t to) +{ + struct blocker bl; + int rv; + + if(fd >= FD_SETSIZE) { + flog(LOG_ERR, "tried to use more file descriptors than select() can handle: fd %i", fd); + errno = EMFILE; + return(-1); + } + bl = (struct blocker) { + .fd = fd, + .ev = ev, + .id = -1, + .to = (to > 0)?(time(NULL) + to):0, + .th = current, + }; + addblock(&bl); + rv = yield(); + remblock(&bl); return(rv); } @@ -77,7 +121,8 @@ int ioloop(void) { int ret; fd_set rfds, wfds, efds; - struct blocker *bl, *nbl; + struct blocker *bl; + struct iterator it; struct timeval toval; time_t now, timeout; int maxfd; @@ -116,8 +161,9 @@ int ioloop(void) } } else { now = time(NULL); - for(bl = blockers; bl; bl = nbl) { - nbl = bl->n; + for(bl = it.bl = blockers; bl; bl = it.bl) { + if((it.bl = bl->n) != NULL) + it.bl->it = ⁢ ev = 0; if(FD_ISSET(bl->fd, &rfds)) ev |= EV_READ; @@ -125,10 +171,23 @@ int ioloop(void) ev |= EV_WRITE; if(FD_ISSET(bl->fd, &efds)) ev = -1; - if((ev < 0) || (ev & bl->ev)) - resume(bl->th, ev); - else if((bl->to != 0) && (bl->to <= now)) - resume(bl->th, 0); + if((ev < 0) || (ev & bl->ev)) { + if(bl->id < 0) { + resume(bl->th, ev); + } else { + bl->rev = ev; + resume(bl->th, bl->id); + } + } else if((bl->to != 0) && (bl->to <= now)) { + if(bl->id < 0) { + resume(bl->th, 0); + } else { + bl->rev = 0; + resume(bl->th, bl->id); + } + } + if(it.bl) + it.bl->it = NULL; } } } diff --git a/lib/mtio.c b/lib/mtio.c index 3b728d0..b911f72 100644 --- a/lib/mtio.c +++ b/lib/mtio.c @@ -16,6 +16,9 @@ along with this program. If not, see . */ +#ifdef HAVE_CONFIG_H +#include +#endif #include #include #include @@ -24,28 +27,57 @@ #include #include -#ifdef HAVE_CONFIG_H -#include -#endif #include #include #include #include +#include -struct stdiofd { - int fd; - int sock; - int timeout; -}; +static ssize_t mtrecv(struct stdiofd *d, void *buf, size_t len) +{ + struct msghdr msg; + char cbuf[512]; + struct cmsghdr *cmsg; + struct iovec bufvec; + socklen_t clen; + ssize_t ret; + int i, *fds; + + msg = (struct msghdr){}; + msg.msg_iov = &bufvec; + msg.msg_iovlen = 1; + bufvec.iov_base = buf; + bufvec.iov_len = len; + msg.msg_control = cbuf; + msg.msg_controllen = sizeof(cbuf); + if((ret = recvmsg(d->fd, &msg, MSG_DONTWAIT)) < 0) + return(ret); + for(cmsg = CMSG_FIRSTHDR(&msg); cmsg != NULL; cmsg = CMSG_NXTHDR(&msg, cmsg)) { + if((cmsg->cmsg_level == SOL_SOCKET) && (cmsg->cmsg_type == SCM_RIGHTS)) { + fds = (int *)CMSG_DATA(cmsg); + clen = (cmsg->cmsg_len - ((char *)fds - (char *)cmsg)) / sizeof(*fds); + for(i = 0; i < clen; i++) { + if(d->rights < 0) + d->rights = fds[i]; + else + close(fds[i]); + } + } + } + return(ret); +} -static ssize_t mtread(void *cookie, char *buf, size_t len) +static ssize_t mtread(void *cookie, void *buf, size_t len) { struct stdiofd *d = cookie; int ev; ssize_t ret; while(1) { - ret = read(d->fd, buf, len); + if(d->sock) + ret = mtrecv(d, buf, len); + else + ret = read(d->fd, buf, len); if((ret < 0) && (errno == EAGAIN)) { ev = block(d->fd, EV_READ, d->timeout); if(ev < 0) { @@ -63,39 +95,63 @@ static ssize_t mtread(void *cookie, char *buf, size_t len) } } -static ssize_t mtwrite(void *cookie, const char *buf, size_t len) +static ssize_t mtsend(struct stdiofd *d, const void *buf, size_t len) +{ + struct msghdr msg; + struct cmsghdr *cmsg; + char cbuf[CMSG_SPACE(sizeof(int))]; + struct iovec bufvec; + ssize_t ret; + int cr; + + msg = (struct msghdr){}; + msg.msg_iov = &bufvec; + msg.msg_iovlen = 1; + bufvec.iov_base = (void *)buf; + bufvec.iov_len = len; + cr = -1; + if(d->sendrights >= 0) { + msg.msg_control = cbuf; + msg.msg_controllen = sizeof(cbuf); + cmsg = CMSG_FIRSTHDR(&msg); + cmsg->cmsg_level = SOL_SOCKET; + cmsg->cmsg_type = SCM_RIGHTS; + cmsg->cmsg_len = CMSG_LEN(sizeof(int)); + *((int *)CMSG_DATA(cmsg)) = d->sendrights; + cr = d->sendrights; + d->sendrights = -1; + msg.msg_controllen = cmsg->cmsg_len; + } + ret = sendmsg(d->fd, &msg, MSG_DONTWAIT | MSG_NOSIGNAL); + if(cr >= 0) + close(cr); + return(ret); +} + +static ssize_t mtwrite(void *cookie, const void *buf, size_t len) { struct stdiofd *d = cookie; int ev; - size_t off; ssize_t ret; - off = 0; - while(off < len) { + while(1) { if(d->sock) - ret = send(d->fd, buf + off, len - off, MSG_DONTWAIT | MSG_NOSIGNAL); + ret = mtsend(d, buf, len); else - ret = write(d->fd, buf + off, len - off); - if(ret < 0) { - if(errno == EAGAIN) { - ev = block(d->fd, EV_WRITE, d->timeout); - if(ev < 0) { - /* If we just go on, we should get the real error. */ - continue; - } else if(ev == 0) { - errno = ETIMEDOUT; - return(off); - } else { - continue; - } - } else { - return(off); + ret = write(d->fd, buf, len); + if((ret < 0) && (errno == EAGAIN)) { + ev = block(d->fd, EV_WRITE, d->timeout); + if(ev < 0) { + /* If we just go on, we should get the real error. */ + continue; + } else if(ev == 0) { + errno = ETIMEDOUT; + return(-1); } } else { - off += ret; + return(ret); } } - return(off); } static int mtclose(void *cookie) @@ -103,29 +159,175 @@ static int mtclose(void *cookie) struct stdiofd *d = cookie; close(d->fd); + if(d->rights >= 0) + close(d->rights); + if(d->sendrights >= 0) + close(d->sendrights); free(d); return(0); } -static cookie_io_functions_t iofuns = { - .read = mtread, - .write = mtwrite, - .close = mtclose, -}; - -FILE *mtstdopen(int fd, int issock, int timeout, char *mode) +FILE *mtstdopen(int fd, int issock, int timeout, char *mode, struct stdiofd **infop) { struct stdiofd *d; FILE *ret; + int r, w; + + if(!strcmp(mode, "r")) { + r = 1; w = 0; + } else if(!strcmp(mode, "w")) { + r = 0; w = 1; + } else if(!strcmp(mode, "r+")) { + r = w = 1; + } else { + return(NULL); + } + omalloc(d); + d->fd = fd; + d->sock = issock; + d->timeout = timeout; + d->rights = d->sendrights = -1; + if(!(ret = funstdio(d, r?mtread:NULL, w?mtwrite:NULL, NULL, mtclose))) { + free(d); + return(NULL); + } + fcntl(fd, F_SETFL, fcntl(fd, F_GETFL) | O_NONBLOCK); + if(infop) + *infop = d; + return(ret); +} + +struct bufio *mtbioopen(int fd, int issock, int timeout, char *mode, struct stdiofd **infop) +{ + static struct bufioops ops = { + .read = mtread, .write = mtwrite, .close = mtclose, + }; + struct stdiofd *d; + struct bufio *ret; + if(!strcmp(mode, "r")) { + } else if(!strcmp(mode, "w")) { + } else if(!strcmp(mode, "r+")) { + } else { + return(NULL); + } omalloc(d); d->fd = fd; d->sock = issock; d->timeout = timeout; - ret = fopencookie(d, mode, iofuns); - if(!ret) + d->rights = d->sendrights = -1; + if(!(ret = bioopen(d, &ops))) { free(d); - else - fcntl(fd, F_SETFL, fcntl(fd, F_GETFL) | O_NONBLOCK); + return(NULL); + } + fcntl(fd, F_SETFL, fcntl(fd, F_GETFL) | O_NONBLOCK); + if(infop) + *infop = d; return(ret); } + +struct pipe { + struct charbuf data; + size_t bufmax; + int closed; + struct muth *r, *w; +}; + +static void freepipe(struct pipe *p) +{ + buffree(p->data); + free(p); +} + +static ssize_t piperead(void *pdata, void *buf, size_t len) +{ + struct pipe *p = pdata; + ssize_t ret; + + while(p->data.d == 0) { + if(p->closed & 2) + return(0); + if(p->r) { + errno = EBUSY; + return(-1); + } + p->r = current; + yield(); + p->r = NULL; + } + ret = min(len, p->data.d); + memcpy(buf, p->data.b, ret); + memmove(p->data.b, p->data.b + ret, p->data.d -= ret); + if(p->w) + resume(p->w, 0); + return(ret); +} + +static int piperclose(void *pdata) +{ + struct pipe *p = pdata; + + if(p->closed & 2) { + freepipe(p); + } else { + p->closed |= 1; + if(p->w) + resume(p->w, 0); + } + return(0); +} + +static ssize_t pipewrite(void *pdata, const void *buf, size_t len) +{ + struct pipe *p = pdata; + ssize_t ret; + + if(p->closed & 1) { + errno = EPIPE; + return(-1); + } + while(p->data.d >= p->bufmax) { + if(p->w) { + errno = EBUSY; + return(-1); + } + p->w = current; + yield(); + p->w = NULL; + if(p->closed & 1) { + errno = EPIPE; + return(-1); + } + } + ret = min(len, p->bufmax - p->data.d); + sizebuf(p->data, p->data.d + ret); + memcpy(p->data.b + p->data.d, buf, ret); + p->data.d += ret; + if(p->r) + resume(p->r, 0); + return(ret); +} + +static int pipewclose(void *pdata) +{ + struct pipe *p = pdata; + + if(p->closed & 1) { + freepipe(p); + } else { + p->closed |= 2; + if(p->r) + resume(p->r, 0); + } + return(0); +} + +void mtiopipe(FILE **read, FILE **write) +{ + struct pipe *p; + + omalloc(p); + p->bufmax = 4096; + *read = funstdio(p, piperead, NULL, NULL, piperclose); + *write = funstdio(p, NULL, pipewrite, NULL, pipewclose); +} diff --git a/lib/mtio.h b/lib/mtio.h index 2ea0eb5..ffb57da 100644 --- a/lib/mtio.h +++ b/lib/mtio.h @@ -6,9 +6,23 @@ #define EV_READ 1 #define EV_WRITE 2 +struct stdiofd { + int fd; + int sock; + int timeout; + int rights, sendrights; +}; + +struct selected { + int fd, ev; +}; + +struct selected mblock(time_t to, int n, struct selected *spec); int block(int fd, int ev, time_t to); int ioloop(void); void exitioloop(int status); -FILE *mtstdopen(int fd, int issock, int timeout, char *mode); +FILE *mtstdopen(int fd, int issock, int timeout, char *mode, struct stdiofd **infop); +struct bufio *mtbioopen(int fd, int issock, int timeout, char *mode, struct stdiofd **infop); +void mtiopipe(FILE **read, FILE **write); #endif diff --git a/lib/proc.c b/lib/proc.c index 2c05608..ac777d4 100644 --- a/lib/proc.c +++ b/lib/proc.c @@ -56,7 +56,7 @@ int stdmkchild(char **argv, void (*chinit)(void *), void *idata) return(fd[1]); } -int sendfd(int sock, int fd, char *data, size_t datalen) +int sendfd2(int sock, int fd, char *data, size_t datalen, int flags) { struct msghdr msg; struct cmsghdr *cmsg; @@ -78,7 +78,12 @@ int sendfd(int sock, int fd, char *data, size_t datalen) *((int *)CMSG_DATA(cmsg)) = fd; msg.msg_controllen = cmsg->cmsg_len; - return(sendmsg(sock, &msg, MSG_NOSIGNAL | MSG_DONTWAIT)); + return(sendmsg(sock, &msg, flags)); +} + +int sendfd(int sock, int fd, char *data, size_t datalen) +{ + return(sendfd2(sock, fd, data, datalen, MSG_NOSIGNAL)); } int recvfd(int sock, char **data, size_t *datalen) diff --git a/lib/proc.h b/lib/proc.h index 6d2ea92..7236595 100644 --- a/lib/proc.h +++ b/lib/proc.h @@ -4,6 +4,7 @@ #include "req.h" int stdmkchild(char **argv, void (*chinit)(void *), void *idata); +int sendfd2(int sock, int fd, char *data, size_t datalen, int flags); int sendfd(int sock, int fd, char *data, size_t datalen); int recvfd(int sock, char **data, size_t *datalen); pid_t stdforkserve(char **argv, struct hthead *req, int fd, void (*chinit)(void *), void *idata); diff --git a/lib/req.c b/lib/req.c index 662df0f..074e7df 100644 --- a/lib/req.c +++ b/lib/req.c @@ -32,6 +32,7 @@ #include #include #include +#include struct hthead *mkreq(char *method, char *url, char *ver) { @@ -164,6 +165,68 @@ fail: return(-1); } +int parseheadersb(struct hthead *head, struct bufio *in) +{ + int c, state; + struct charbuf name, val; + size_t tsz; + + bufinit(name); + bufinit(val); + state = 0; + tsz = 0; + while(1) { + c = biogetc(in); + if(++tsz >= 65536) + goto fail; + again: + if(state == 0) { + if(c == '\r') { + } else if(c == '\n') { + break; + } else if(c == EOF) { + goto fail; + } else { + state = 1; + goto again; + } + } else if(state == 1) { + if(c == ':') { + trim(&name); + bufadd(name, 0); + state = 2; + } else if(c == '\r') { + } else if(c == '\n') { + goto fail; + } else if(c == EOF) { + goto fail; + } else { + bufadd(name, c); + } + } else if(state == 2) { + if(c == '\r') { + } else if(c == '\n') { + trim(&val); + bufadd(val, 0); + headappheader(head, name.b, val.b); + buffree(name); + buffree(val); + state = 0; + } else if(c == EOF) { + goto fail; + } else { + bufadd(val, c); + } + } + } + return(0); + +fail: + buffree(name); + buffree(val); + return(-1); +} + struct hthead *parseresponse(FILE *in) { struct hthead *req; @@ -230,6 +293,72 @@ out: return(req); } +struct hthead *parseresponseb(struct bufio *in) +{ + struct hthead *req; + int code; + struct charbuf ver, msg; + int c; + + req = NULL; + bufinit(ver); + bufinit(msg); + code = 0; + while(1) { + c = biogetc(in); + if(c == ' ') { + break; + } else if((c == EOF) || (c < 32) || (c >= 128)) { + goto fail; + } else { + bufadd(ver, c); + if(ver.d >= 128) + goto fail; + } + } + while(1) { + c = biogetc(in); + if(c == ' ') { + break; + } else if((c == EOF) || (c < '0') || (c > '9')) { + goto fail; + } else { + code = (code * 10) + (c - '0'); + if(code >= 10000) + goto fail; + } + } + while(1) { + c = biogetc(in); + if(c == 10) { + break; + } else if(c == 13) { + } else if((c == EOF) || (c < 32)) { + goto fail; + } else { + bufadd(msg, c); + if(msg.d >= 512) + goto fail; + } + } + bufadd(msg, 0); + bufadd(ver, 0); + req = mkresp(code, msg.b, ver.b); + if(parseheadersb(req, in)) + goto fail; + goto out; + +fail: + if(req != NULL) { + freehthead(req); + req = NULL; + } +out: + buffree(msg); + buffree(ver); + return(req); +} + void replrest(struct hthead *head, char *rest) { char *tmp; @@ -290,7 +419,20 @@ int writeresp(FILE *out, struct hthead *resp) return(0); } -int sendreq(int sock, struct hthead *req, int fd) +int writerespb(struct bufio *out, struct hthead *resp) +{ + int i; + + if(bioprintf(out, "%s %i %s\r\n", resp->ver, resp->code, resp->msg) < 0) + return(-1); + for(i = 0; i < resp->noheaders; i++) { + if(bioprintf(out, "%s: %s\r\n", resp->headers[i][0], resp->headers[i][1]) < 0) + return(-1); + } + return(0); +} + +int sendreq2(int sock, struct hthead *req, int fd, int flags) { int ret, i; struct charbuf buf; @@ -305,7 +447,7 @@ int sendreq(int sock, struct hthead *req, int fd) bufcatstr2(buf, req->headers[i][1]); } bufcatstr2(buf, ""); - ret = sendfd(sock, fd, buf.b, buf.d); + ret = sendfd2(sock, fd, buf.b, buf.d, flags); buffree(buf); if(ret < 0) return(-1); @@ -313,6 +455,11 @@ int sendreq(int sock, struct hthead *req, int fd) return(0); } +int sendreq(int sock, struct hthead *req, int fd) +{ + return(sendreq2(sock, req, fd, MSG_NOSIGNAL)); +} + int recvreq(int sock, struct hthead **reqp) { int fd; diff --git a/lib/req.h b/lib/req.h index 71a7a06..9ae60ab 100644 --- a/lib/req.h +++ b/lib/req.h @@ -3,6 +3,8 @@ #include +struct bufio; + struct hthead { char *method, *url, *ver, *msg; int code; @@ -18,12 +20,16 @@ char *getheader(struct hthead *head, char *name); void headpreheader(struct hthead *head, const char *name, const char *val); void headappheader(struct hthead *head, const char *name, const char *val); void headrmheader(struct hthead *head, const char *name); +int sendreq2(int sock, struct hthead *req, int fd, int flags); int sendreq(int sock, struct hthead *req, int fd); int recvreq(int sock, struct hthead **reqp); void replrest(struct hthead *head, char *rest); int parseheaders(struct hthead *head, FILE *in); +int parseheadersb(struct hthead *head, struct bufio *in); struct hthead *parseresponse(FILE *in); +struct hthead *parseresponseb(struct bufio *in); int writeresp(FILE *out, struct hthead *resp); +int writerespb(struct bufio *out, struct hthead *resp); char *unquoteurl(char *in); #endif diff --git a/lib/resp.c b/lib/resp.c index 52fbd4b..be2f7bd 100644 --- a/lib/resp.c +++ b/lib/resp.c @@ -53,7 +53,7 @@ char *urlquote(char *text) bufinit(buf); for(; *text; text++) { c = *text; - if(!c < 128 && safechars[(int)c]) + if((c < 128) && safechars[(int)c]) bufadd(buf, *text); else bprintf(&buf, "%%%02X", (int)c); @@ -190,6 +190,22 @@ char *fmthttpdate(time_t time) return(sprintf3("%s, %i %s %i %02i:%02i:%02i GMT", days[(tm->tm_wday + 6) % 7], tm->tm_mday, months[tm->tm_mon], tm->tm_year + 1900, tm->tm_hour, tm->tm_min, tm->tm_sec)); } +static int gtoi(char *bstr, regmatch_t g) +{ + int i, n; + + for(i = g.rm_so, n = 0; i < g.rm_eo; i++) + n = (n * 10) + (bstr[i] - '0'); + return(n); +} + +static int gstrcmp(char *bstr, regmatch_t g, char *str) +{ + if(g.rm_eo - g.rm_so != strlen(str)) + return(1); + return(strncasecmp(bstr + g.rm_so, str, g.rm_eo - g.rm_so)); +} + time_t parsehttpdate(char *date) { static regex_t *spec = NULL; @@ -200,21 +216,6 @@ time_t parsehttpdate(char *date) struct tm tm; int tz; - int gtoi(regmatch_t g) - { - int i, n; - - for(i = g.rm_so, n = 0; i < g.rm_eo; i++) - n = (n * 10) + (date[i] - '0'); - return(n); - } - - int gstrcmp(regmatch_t g, char *str) { - if(g.rm_eo - g.rm_so != strlen(str)) - return(1); - return(strncasecmp(date + g.rm_so, str, g.rm_eo - g.rm_so)); - } - if(spec == NULL) { omalloc(spec); if(regcomp(spec, "^[A-Z]{3}, +([0-9]+) +([A-Z]{3}) +([0-9]+) +([0-9]{2}):([0-9]{2}):([0-9]{2}) +(([A-Z]+)|[+-]([0-9]{2})([0-9]{2}))$", REG_EXTENDED | REG_ICASE)) { @@ -225,15 +226,15 @@ time_t parsehttpdate(char *date) } if(regexec(spec, date, 11, g, 0)) return(0); - tm.tm_mday = gtoi(g[1]); - tm.tm_year = gtoi(g[3]) - 1900; - tm.tm_hour = gtoi(g[4]); - tm.tm_min = gtoi(g[5]); - tm.tm_sec = gtoi(g[6]); + tm.tm_mday = gtoi(date, g[1]); + tm.tm_year = gtoi(date, g[3]) - 1900; + tm.tm_hour = gtoi(date, g[4]); + tm.tm_min = gtoi(date, g[5]); + tm.tm_sec = gtoi(date, g[6]); tm.tm_mon = -1; for(i = 0; i < 12; i++) { - if(!gstrcmp(g[2], months[i])) { + if(!gstrcmp(date, g[2], months[i])) { tm.tm_mon = i; break; } @@ -242,12 +243,12 @@ time_t parsehttpdate(char *date) return(0); if(g[8].rm_so > 0) { - if(!gstrcmp(g[8], "GMT")) + if(!gstrcmp(date, g[8], "GMT")) tz = 0; else return(0); } else if((g[9].rm_so > 0) && (g[10].rm_so > 0)) { - tz = gtoi(g[9]) * 3600 + gtoi(g[10]) * 60; + tz = gtoi(date, g[9]) * 3600 + gtoi(date, g[10]) * 60; if(date[g[7].rm_so] == '-') tz = -tz; } else { @@ -256,3 +257,45 @@ time_t parsehttpdate(char *date) return(timegm(&tm) - tz); } + +char *httpdefstatus(int code) +{ + switch(code) { + case 200: + return("OK"); + case 201: + return("Created"); + case 202: + return("Accepted"); + case 204: + return("No Content"); + case 300: + return("Multiple Choices"); + case 301: + return("Moved Permanently"); + case 302: + return("Found"); + case 303: + return("See Other"); + case 304: + return("Not Modified"); + case 307: + return("Moved Temporarily"); + case 400: + return("Bad Request"); + case 401: + return("Unauthorized"); + case 403: + return("Forbidden"); + case 404: + return("Not Found"); + case 500: + return("Internal Server Error"); + case 501: + return("Not Implemented"); + case 503: + return("Service Unavailable"); + default: + return("Unknown status"); + } +} diff --git a/lib/resp.h b/lib/resp.h index 8730584..6262244 100644 --- a/lib/resp.h +++ b/lib/resp.h @@ -10,5 +10,6 @@ void simpleerror2(FILE *out, int code, char *msg, char *fmt, ...); void stdredir(struct hthead *req, int fd, int code, char *dst); char *fmthttpdate(time_t time); time_t parsehttpdate(char *date); +char *httpdefstatus(int code); #endif diff --git a/lib/utils.c b/lib/utils.c index 33e55c7..b37430d 100644 --- a/lib/utils.c +++ b/lib/utils.c @@ -16,14 +16,15 @@ along with this program. If not, see . */ +#ifdef HAVE_CONFIG_H +#include +#endif #include #include #include #include +#include -#ifdef HAVE_CONFIG_H -#include -#endif #include static char *base64set = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; @@ -307,6 +308,14 @@ char *base64decode(char *data, size_t *datalen) return(buf.b); } +int hexdigit(char c) +{ + if((c >= '0') && (c <= '9')) return(c - '0'); + else if((c >= 'a') && (c <= 'f')) return(c - 'a' + 10); + else if((c >= 'A') && (c <= 'F')) return(c - 'A' + 10); + return(-1); +} + static int btheight(struct btree *tree) { if(tree == NULL) @@ -466,3 +475,146 @@ void *btreeget(struct btree *tree, void *key, int (*cmp)(void *, void *)) return(tree->d); } } + +struct stdif { + ssize_t (*read)(void *pdata, void *buf, size_t len); + ssize_t (*write)(void *pdata, const void *buf, size_t len); + off_t (*seek)(void *pdata, off_t offset, int whence); + int (*close)(void *pdata); + void *pdata; +}; + +#if defined(HAVE_GLIBC_STDIO) +static ssize_t wrapread(void *pdata, char *buf, size_t len) +{ + struct stdif *nf = pdata; + + return(nf->read(nf->pdata, buf, len)); +} + +static ssize_t wrapwrite(void *pdata, const char *buf, size_t len) +{ + struct stdif *nf = pdata; + size_t off; + ssize_t ret; + + /* + * XXX? In seeming violation of its own manual, glibc requires the + * cookie-write function to complete writing the entire buffer, + * rather than working like write(2). + */ + off = 0; + while(off < len) { + ret = nf->write(nf->pdata, buf + off, len - off); + if(ret < 0) + return(off); + off += ret; + } + return(off); +} + +static int wrapseek(void *pdata, off64_t *pos, int whence) +{ + struct stdif *nf = pdata; + off_t ret; + + ret = nf->seek(nf->pdata, *pos, whence); + if(ret < 0) + return(-1); + *pos = ret; + return(0); +} + +static int wrapclose(void *pdata) +{ + struct stdif *nf = pdata; + int ret; + + if(nf->close != NULL) + ret = nf->close(nf->pdata); + else + ret = 0; + free(nf); + return(ret); +} + +FILE *funstdio(void *pdata, + ssize_t (*read)(void *pdata, void *buf, size_t len), + ssize_t (*write)(void *pdata, const void *buf, size_t len), + off_t (*seek)(void *pdata, off_t offset, int whence), + int (*close)(void *pdata)) +{ + struct stdif *nf; + cookie_io_functions_t io; + char *mode; + + if(read && write) { + mode = "r+"; + } else if(read) { + mode = "r"; + } else if(write) { + mode = "w"; + } else { + errno = EINVAL; + return(NULL); + } + omalloc(nf); + *nf = (struct stdif){.read = read, .write = write, .seek = seek, .close = close, .pdata = pdata}; + io = (cookie_io_functions_t) { + .read = read?wrapread:NULL, + .write = write?wrapwrite:NULL, + .seek = seek?wrapseek:NULL, + .close = wrapclose, + }; + return(fopencookie(nf, mode, io)); +} +#elif defined(HAVE_BSD_STDIO) +static int wrapread(void *pdata, char *buf, int len) +{ + struct stdif *nf = pdata; + + return(nf->read(nf->pdata, buf, len)); +} + +static int wrapwrite(void *pdata, const char *buf, int len) +{ + struct stdif *nf = pdata; + + return(nf->write(nf->pdata, buf, len)); +} + +static fpos_t wrapseek(void *pdata, fpos_t pos, int whence) +{ + struct stdif *nf = pdata; + + return(nf->seek(nf->pdata, pos, whence)); +} + +static int wrapclose(void *pdata) +{ + struct stdif *nf = pdata; + int ret; + + if(nf->close != NULL) + ret = nf->close(nf->pdata); + else + ret = 0; + free(nf); + return(ret); +} + +FILE *funstdio(void *pdata, + ssize_t (*read)(void *pdata, void *buf, size_t len), + ssize_t (*write)(void *pdata, const void *buf, size_t len), + off_t (*seek)(void *pdata, off_t offset, int whence), + int (*close)(void *pdata)) +{ + struct stdif *nf; + + omalloc(nf); + *nf = (struct stdif){.read = read, .write = write, .seek = seek, .close = close, .pdata = pdata}; + return(funopen(nf, read?wrapread:NULL, write?wrapwrite:NULL, seek?wrapseek:NULL, wrapclose)); +} +#else +#error "No stdio implementation for this system" +#endif diff --git a/lib/utils.h b/lib/utils.h index 2066918..658eea2 100644 --- a/lib/utils.h +++ b/lib/utils.h @@ -1,14 +1,15 @@ #ifndef _UTILS_H #define _UTILS_H +#include #include #include #define max(a, b) (((b) > (a))?(b):(a)) #define min(a, b) (((b) < (a))?(b):(a)) -#define smalloc(size) ({void *__result__; ((__result__ = malloc(size)) == NULL)?({exit(-1); (void *)0;}):__result__;}) -#define srealloc(ptr, size) ({void *__result__; ((__result__ = realloc((ptr), (size))) == NULL)?({exit(-1); (void *)0;}):__result__;}) +#define smalloc(size) ({void *__result__; ((__result__ = malloc(size)) == NULL)?({abort(); (void *)0;}):__result__;}) +#define srealloc(ptr, size) ({void *__result__; ((__result__ = realloc((ptr), (size))) == NULL)?({abort(); (void *)0;}):__result__;}) #define szmalloc(size) memset(smalloc(size), 0, size) #define sstrdup(str) ({const char *__strbuf__ = (str); strcpy(smalloc(strlen(__strbuf__) + 1), __strbuf__);}) #define omalloc(o) ((o) = szmalloc(sizeof(*(o)))) @@ -81,9 +82,15 @@ void bprintf(struct charbuf *buf, char *format, ...); void replstr(char **p, char *n); char *base64encode(char *data, size_t datalen); char *base64decode(char *data, size_t *datalen); +int hexdigit(char c); int bbtreedel(struct btree **tree, void *item, int (*cmp)(void *, void *)); void freebtree(struct btree **tree, void (*ffunc)(void *)); int bbtreeput(struct btree **tree, void *item, int (*cmp)(void *, void *)); void *btreeget(struct btree *tree, void *key, int (*cmp)(void *, void *)); +FILE *funstdio(void *pdata, + ssize_t (*read)(void *pdata, void *buf, size_t len), + ssize_t (*write)(void *pdata, const void *buf, size_t len), + off_t (*seek)(void *pdata, off_t offset, int whence), + int (*close)(void *pdata)); #endif diff --git a/python/ashd-wsgi b/python/ashd-wsgi index 2e4a2fe..d5438fa 100755 --- a/python/ashd-wsgi +++ b/python/ashd-wsgi @@ -1,19 +1,19 @@ #!/usr/bin/python -import sys, os, getopt, threading, logging, time -import ashd.proto, ashd.util, ashd.perf +import sys, os, getopt, socket, logging, time, signal +import ashd.util, ashd.serve try: import pdm.srv except: pdm = None def usage(out): - out.write("usage: ashd-wsgi [-hAL] [-m PDM-SPEC] [-p MODPATH] [-l REQLIMIT] HANDLER-MODULE [ARGS...]\n") + out.write("usage: ashd-wsgi [-hAL] [-m PDM-SPEC] [-p MODPATH] [-t REQUEST-HANDLER[:PAR[=VAL](,PAR[=VAL])...]] HANDLER-MODULE [ARGS...]\n") -reqlimit = 0 +hspec = "free", {} modwsgi_compat = False setlog = True -opts, args = getopt.getopt(sys.argv[1:], "+hAp:l:m:") +opts, args = getopt.getopt(sys.argv[1:], "+hALp:t:l:m:") for o, a in opts: if o == "-h": usage(sys.stdout) @@ -25,7 +25,9 @@ for o, a in opts: elif o == "-A": modwsgi_compat = True elif o == "-l": - reqlimit = int(a) + hspec = "free", {"max": a, "abort": "10"} + elif o == "-t": + hspec = ashd.serve.parsehspec(a) elif o == "-m": if pdm is not None: pdm.srv.listen(a) @@ -52,10 +54,6 @@ else: sys.exit(1) handler = handlermod.application -class closed(IOError): - def __init__(self): - super(closed, self).__init__("The client has closed the connection.") - cwd = os.getcwd() def absolutify(path): if path[0] != '/': @@ -95,7 +93,7 @@ def unquoteurl(url): buf += c return buf -def dowsgi(req): +def mkenv(req): env = {} env["wsgi.version"] = 1, 0 for key, val in req.headers: @@ -147,107 +145,59 @@ def dowsgi(req): env["wsgi.multithread"] = True env["wsgi.multiprocess"] = False env["wsgi.run_once"] = False + return env + +class request(ashd.serve.wsgirequest): + def __init__(self, bkreq, **kw): + super(request, self).__init__(**kw) + self.bkreq = bkreq.dup() + + def mkenv(self): + return mkenv(self.bkreq) + + def handlewsgi(self, env, startreq): + return handler(env, startreq) + + def fileno(self): + return self.bkreq.bsk.fileno() - resp = [] - respsent = [] - - def flushreq(): - if not respsent: - if not resp: - raise Exception, "Trying to write data before starting response." - status, headers = resp - respsent[:] = [True] - try: - req.sk.write("HTTP/1.1 %s\n" % status) - for nm, val in headers: - req.sk.write("%s: %s\n" % (nm, val)) - req.sk.write("\n") - except IOError: - raise closed() - - def write(data): - if not data: - return - flushreq() + def writehead(self, status, headers): + w = self.buffer.extend + w("HTTP/1.1 %s\n" % status) + for nm, val in headers: + w("%s: %s\n" % (nm, val)) + w("\n") + + def flush(self): try: - req.sk.write(data) - req.sk.flush() + ret = self.bkreq.bsk.send(self.buffer, socket.MSG_DONTWAIT) + self.buffer[:ret] = "" except IOError: - raise closed() - - def startreq(status, headers, exc_info = None): - if resp: - if exc_info: # Interesting, this... - try: - if respsent: - raise exc_info[0], exc_info[1], exc_info[2] - finally: - exc_info = None # CPython GC bug? - else: - raise Exception, "Can only start responding once." - resp[:] = status, headers - return write + raise ashd.serve.closed() + + def close(self): + self.bkreq.close() - reqevent = ashd.perf.request(env) - exc = (None, None, None) - try: - try: - respiter = handler(env, startreq) - try: - for data in respiter: - write(data) - if resp: - flushreq() - finally: - if hasattr(respiter, "close"): - respiter.close() - except closed: - pass - if resp: - reqevent.response(resp) - except: - exc = sys.exc_info() - raise - finally: - reqevent.__exit__(*exc) - -flightlock = threading.Condition() -inflight = 0 - -class reqthread(threading.Thread): - def __init__(self, req): - super(reqthread, self).__init__(name = "Request handler") - self.req = req.dup() - - def run(self): - global inflight - try: - flightlock.acquire() - try: - if reqlimit != 0: - start = time.time() - while inflight >= reqlimit: - flightlock.wait(10) - if time.time() - start > 10: - os.abort() - inflight += 1 - finally: - flightlock.release() - try: - dowsgi(self.req) - finally: - flightlock.acquire() - try: - inflight -= 1 - flightlock.notify() - finally: - flightlock.release() - except: - log.error("exception occurred in handler thread", exc_info=True) - finally: - self.req.close() - def handle(req): - reqthread(req).start() + reqhandler.handle(request(bkreq=req, handler=reqhandler)) -ashd.util.serveloop(handle) +if hspec[0] not in ashd.serve.names: + sys.stderr.write("ashd-wsgi: no such request handler: %s\n" % hspec[0]) + sys.exit(1) +hclass = ashd.serve.names[hspec[0]] +try: + hargs = hclass.parseargs(**hspec[1]) +except ValueError as exc: + sys.stderr.write("ashd-wsgi: %s\n" % exc) + sys.exit(1) + +def sigterm(sig, frame): + socket.fromfd(0, socket.AF_UNIX, socket.SOCK_SEQPACKET).shutdown(socket.SHUT_RDWR) # :P +for signum in [signal.SIGINT, signal.SIGTERM]: + signal.signal(signum, sigterm) + +reqhandler = hclass(**hargs) +try: + ashd.util.serveloop(handle) +finally: + reqhandler.close() diff --git a/python/ashd/perf.py b/python/ashd/perf.py index 97146a8..c74e443 100644 --- a/python/ashd/perf.py +++ b/python/ashd/perf.py @@ -16,8 +16,13 @@ if pdm: self.method = env.get("REQUEST_METHOD") self.uri = env.get("REQUEST_URI") self.host = env.get("HTTP_HOST") + self.script_uri = env.get("SCRIPT_NAME") + self.script_path = env.get("SCRIPT_FILENAME") + self.pathinfo = env.get("PATH_INFO") + self.querystring = env.get("QUERY_STRING") self.remoteaddr = env.get("REMOTE_ADDR") self.remoteport = env.get("REMOTE_PORT") + self.scheme = env.get("wsgi.url_scheme") class reqfinish(pdm.perf.finishevent): def __init__(self, start, aborted, status): diff --git a/python/ashd/proto.py b/python/ashd/proto.py index 8dc5ecd..e18023d 100644 --- a/python/ashd/proto.py +++ b/python/ashd/proto.py @@ -46,12 +46,14 @@ class req(object): self.ver = ver self.rest = rest self.headers = headers - self.sk = socket.fromfd(fd, socket.AF_UNIX, socket.SOCK_STREAM).makefile('r+') + self.bsk = socket.fromfd(fd, socket.AF_UNIX, socket.SOCK_STREAM) + self.sk = self.bsk.makefile('r+') os.close(fd) def close(self): "Close this request's response socket." self.sk.close() + self.bsk.close() def __getitem__(self, header): """Find a HTTP header case-insensitively. For example, @@ -79,7 +81,7 @@ class req(object): """Creates a duplicate of this request, referring to a duplicate of the response socket. """ - return req(self.method, self.url, self.ver, self.rest, self.headers, os.dup(self.sk.fileno())) + return req(self.method, self.url, self.ver, self.rest, self.headers, os.dup(self.bsk.fileno())) def match(self, match): """If the `match' argument matches exactly the leading part of diff --git a/python/ashd/scgi.py b/python/ashd/scgi.py index f7ba3a8..1f0c5ab 100644 --- a/python/ashd/scgi.py +++ b/python/ashd/scgi.py @@ -1,13 +1,6 @@ -import sys -import threading - class protoerr(Exception): pass -class closed(IOError): - def __init__(self): - super(closed, self).__init__("The client has closed the connection.") - def readns(sk): hln = 0 while True: @@ -33,100 +26,3 @@ def readhead(sk): ret[parts[i]] = parts[i + 1] i += 2 return ret - -class reqthread(threading.Thread): - def __init__(self, sk, handler): - super(reqthread, self).__init__(name = "SCGI request handler") - self.bsk = sk.dup() - self.sk = self.bsk.makefile("r+") - self.handler = handler - - def run(self): - try: - head = readhead(self.sk) - self.handler(head, self.sk) - finally: - self.sk.close() - self.bsk.close() - -def handlescgi(sk, handler): - t = reqthread(sk, handler) - t.start() - -def servescgi(socket, handler): - while True: - nsk, addr = socket.accept() - try: - handlescgi(nsk, handler) - finally: - nsk.close() - -def wrapwsgi(handler): - def handle(head, sk): - env = dict(head) - env["wsgi.version"] = 1, 0 - if "HTTP_X_ASH_PROTOCOL" in env: - env["wsgi.url_scheme"] = env["HTTP_X_ASH_PROTOCOL"] - elif "HTTPS" in env: - env["wsgi.url_scheme"] = "https" - else: - env["wsgi.url_scheme"] = "http" - env["wsgi.input"] = sk - env["wsgi.errors"] = sys.stderr - env["wsgi.multithread"] = True - env["wsgi.multiprocess"] = False - env["wsgi.run_once"] = False - - resp = [] - respsent = [] - - def flushreq(): - if not respsent: - if not resp: - raise Exception, "Trying to write data before starting response." - status, headers = resp - respsent[:] = [True] - try: - sk.write("Status: %s\n" % status) - for nm, val in headers: - sk.write("%s: %s\n" % (nm, val)) - sk.write("\n") - except IOError: - raise closed() - - def write(data): - if not data: - return - flushreq() - try: - sk.write(data) - sk.flush() - except IOError: - raise closed() - - def startreq(status, headers, exc_info = None): - if resp: - if exc_info: # Interesting, this... - try: - if respsent: - raise exc_info[0], exc_info[1], exc_info[2] - finally: - exc_info = None # CPython GC bug? - else: - raise Exception, "Can only start responding once." - resp[:] = status, headers - return write - - respiter = handler(env, startreq) - try: - try: - for data in respiter: - write(data) - if resp: - flushreq() - except closed: - pass - finally: - if hasattr(respiter, "close"): - respiter.close() - return handle diff --git a/python/ashd/serve.py b/python/ashd/serve.py new file mode 100644 index 0000000..e9f92b0 --- /dev/null +++ b/python/ashd/serve.py @@ -0,0 +1,366 @@ +import sys, os, threading, time, logging, select, Queue +import perf + +log = logging.getLogger("ashd.serve") +seq = 1 +seqlk = threading.Lock() + +def reqseq(): + global seq + with seqlk: + s = seq + seq += 1 + return s + +class closed(IOError): + def __init__(self): + super(closed, self).__init__("The client has closed the connection.") + +class reqthread(threading.Thread): + def __init__(self, name=None, **kw): + if name is None: + name = "Request handler %i" % reqseq() + super(reqthread, self).__init__(name=name, **kw) + +class wsgirequest(object): + def __init__(self, handler): + self.status = None + self.headers = [] + self.respsent = False + self.handler = handler + self.buffer = bytearray() + + def handlewsgi(self): + raise Exception() + def fileno(self): + raise Exception() + def writehead(self, status, headers): + raise Exception() + def flush(self): + raise Exception() + def close(self): + pass + def writedata(self, data): + self.buffer.extend(data) + + def flushreq(self): + if not self.respsent: + if not self.status: + raise Exception("Cannot send response body before starting response.") + self.respsent = True + self.writehead(self.status, self.headers) + + def write(self, data): + if not data: + return + self.flushreq() + self.writedata(data) + self.handler.ckflush(self) + + def startreq(self, status, headers, exc_info=None): + if self.status: + if exc_info: + try: + if self.respsent: + raise exc_info[1] + finally: + exc_info = None + else: + raise Exception("Can only start responding once.") + self.status = status + self.headers = headers + return self.write + +class handler(object): + def handle(self, request): + raise Exception() + def ckflush(self, req): + while len(req.buffer) > 0: + rls, wls, els = select.select([], [req], [req]) + req.flush() + def close(self): + pass + + @classmethod + def parseargs(cls, **args): + if len(args) > 0: + raise ValueError("unknown handler argument: " + iter(args).next()) + return {} + +class single(handler): + cname = "single" + + def handle(self, req): + try: + env = req.mkenv() + with perf.request(env) as reqevent: + respiter = req.handlewsgi(env, req.startreq) + for data in respiter: + req.write(data) + if req.status: + reqevent.response([req.status, req.headers]) + req.flushreq() + self.ckflush(req) + except closed: + pass + except: + log.error("exception occurred when handling request", exc_info=True) + finally: + req.close() + +class freethread(handler): + cname = "free" + + def __init__(self, max=None, timeout=None, **kw): + super(freethread, self).__init__(**kw) + self.current = set() + self.lk = threading.Lock() + self.tcond = threading.Condition(self.lk) + self.max = max + self.timeout = timeout + + @classmethod + def parseargs(cls, max=None, abort=None, **args): + ret = super(freethread, cls).parseargs(**args) + if max: + ret["max"] = int(max) + if abort: + ret["timeout"] = int(abort) + return ret + + def handle(self, req): + with self.lk: + if self.max is not None: + if self.timeout is not None: + now = start = time.time() + while len(self.current) >= self.max: + self.tcond.wait(start + self.timeout - now) + now = time.time() + if now - start > self.timeout: + os.abort() + else: + while len(self.current) >= self.max: + self.tcond.wait() + th = reqthread(target=self.run, args=[req]) + th.registered = False + th.start() + while not th.registered: + self.tcond.wait() + + def run(self, req): + try: + th = threading.current_thread() + with self.lk: + self.current.add(th) + th.registered = True + self.tcond.notify_all() + try: + env = req.mkenv() + with perf.request(env) as reqevent: + respiter = req.handlewsgi(env, req.startreq) + for data in respiter: + req.write(data) + if req.status: + reqevent.response([req.status, req.headers]) + req.flushreq() + self.ckflush(req) + except closed: + pass + except: + log.error("exception occurred when handling request", exc_info=True) + finally: + with self.lk: + self.current.remove(th) + self.tcond.notify_all() + finally: + req.close() + + def close(self): + while True: + with self.lk: + if len(self.current) > 0: + th = iter(self.current).next() + else: + return + th.join() + +class resplex(handler): + cname = "rplex" + + def __init__(self, max=None, **kw): + super(resplex, self).__init__(**kw) + self.current = set() + self.lk = threading.Lock() + self.tcond = threading.Condition(self.lk) + self.max = max + self.cqueue = Queue.Queue(5) + self.cnpipe = os.pipe() + self.rthread = reqthread(name="Response thread", target=self.handle2) + self.rthread.start() + + @classmethod + def parseargs(cls, max=None, **args): + ret = super(resplex, cls).parseargs(**args) + if max: + ret["max"] = int(max) + return ret + + def ckflush(self, req): + raise Exception("resplex handler does not support the write() function") + + def handle(self, req): + with self.lk: + if self.max is not None: + while len(self.current) >= self.max: + self.tcond.wait() + th = reqthread(target=self.handle1, args=[req]) + th.registered = False + th.start() + while not th.registered: + self.tcond.wait() + + def handle1(self, req): + try: + th = threading.current_thread() + with self.lk: + self.current.add(th) + th.registered = True + self.tcond.notify_all() + try: + env = req.mkenv() + respobj = req.handlewsgi(env, req.startreq) + respiter = iter(respobj) + if not req.status: + log.error("request handler returned without calling start_request") + if hasattr(respiter, "close"): + respiter.close() + return + else: + self.cqueue.put((req, respiter)) + os.write(self.cnpipe[1], " ") + req = None + finally: + with self.lk: + self.current.remove(th) + self.tcond.notify_all() + except closed: + pass + except: + log.error("exception occurred when handling request", exc_info=True) + finally: + if req is not None: + req.close() + + def handle2(self): + try: + rp = self.cnpipe[0] + current = {} + + def closereq(req): + respiter = current[req] + try: + if respiter is not None and hasattr(respiter, "close"): + respiter.close() + except: + log.error("exception occurred when closing iterator", exc_info=True) + try: + req.close() + except: + log.error("exception occurred when closing request", exc_info=True) + del current[req] + def ckiter(req): + respiter = current[req] + if respiter is not None: + rem = False + try: + data = respiter.next() + except StopIteration: + rem = True + try: + req.flushreq() + except: + log.error("exception occurred when handling response data", exc_info=True) + except: + rem = True + log.error("exception occurred when iterating response", exc_info=True) + if not rem: + if data: + try: + req.flushreq() + req.writedata(data) + except: + log.error("exception occurred when handling response data", exc_info=True) + rem = True + if rem: + current[req] = None + try: + if hasattr(respiter, "close"): + respiter.close() + except: + log.error("exception occurred when closing iterator", exc_info=True) + respiter = None + if respiter is None and not req.buffer: + closereq(req) + + while True: + bufl = list(req for req in current.iterkeys() if req.buffer) + rls, wls, els = select.select([rp], bufl, [rp] + bufl) + if rp in rls: + ret = os.read(rp, 1024) + if not ret: + os.close(rp) + return + try: + while True: + req, respiter = self.cqueue.get(False) + current[req] = respiter + ckiter(req) + except Queue.Empty: + pass + for req in wls: + try: + req.flush() + except closed: + closereq(req) + except: + log.error("exception occurred when writing response", exc_info=True) + closereq(req) + else: + if len(req.buffer) < 65536: + ckiter(req) + except: + log.critical("unexpected exception occurred in response handler thread", exc_info=True) + os.abort() + + def close(self): + while True: + with self.lk: + if len(self.current) > 0: + th = iter(self.current).next() + else: + break + th.join() + os.close(self.cnpipe[1]) + self.rthread.join() + +names = dict((cls.cname, cls) for cls in globals().itervalues() if + isinstance(cls, type) and + issubclass(cls, handler) and + hasattr(cls, "cname")) + +def parsehspec(spec): + if ":" not in spec: + return spec, {} + nm, spec = spec.split(":", 1) + args = {} + while spec: + if "," in spec: + part, spec = spec.split(",", 1) + else: + part, spec = spec, None + if "=" in part: + key, val = part.split("=", 1) + else: + key, val = part, "" + args[key] = val + return nm, args diff --git a/python/doc/ashd-wsgi.doc b/python/doc/ashd-wsgi.doc index 2102557..566238c 100644 --- a/python/doc/ashd-wsgi.doc +++ b/python/doc/ashd-wsgi.doc @@ -7,7 +7,7 @@ ashd-wsgi - WSGI adapter for ashd(7) SYNOPSIS -------- -*ashd-wsgi* [*-hA*] [*-m* 'PDM-SPEC'] [*-p* 'MODPATH'] [*-l* 'LIMIT'] 'HANDLER-MODULE' ['ARGS'...] +*ashd-wsgi* [*-hAL*] [*-m* 'PDM-SPEC'] [*-p* 'MODPATH'] [*-t* 'HANDLING-MODEL'] 'HANDLER-MODULE' ['ARGS'...] DESCRIPTION ----------- @@ -21,7 +21,10 @@ section, below. multithreaded dispatching in a single Python interpreter, which means that WSGI applications that use it need to be thread-safe, but that they can also share all Python data structures and global variables -between requests. +between requests. More precisely, *ashd-wsgi* implements a couple of +slightly different ways to handle requests and threads, which can be +configured using the *-t* option, as described in the REQUEST HANDLING +section, below. The Python module that *ashd-wsgi* comes with also contains a standard handler module, `ashd.wsgidir`, which serves individual WSGI @@ -47,6 +50,12 @@ OPTIONS the WSGI application object. See the PROTOCOL section, below, for details. +*-L*:: + By default, *ashd-wsgi* sets up the Python logging with a + logging format and for logging to standard error. The *-L* + option suppresses that behavior, so that any handler module + may set up logging itself. + *-p* 'MODPATH':: Prepend 'MODPATH' to Python's `sys.path`; can be given multiple @@ -54,18 +63,15 @@ OPTIONS on Python's module path by default, so if you want to use a module in that directory, you will need to specify "`-p .`". -*-l* 'LIMIT':: +*-t* 'HANDLING-MODEL':: - Allow at most 'LIMIT' requests to run concurrently. If a new - request is made when 'LIMIT' requests are executing, the new - request will wait up to ten seconds for one of them to - complete; if none does, *ashd-wsgi* will assume that the - process is foobar and *abort*(3). + Specify the way *ashd-wsgi* handles requests. See below, under + REQUEST HANDLING. *-m* 'PDM-SPEC':: If the PDM library is installed on the system, create a - listening socket for connection PDM clients according to + listening socket for connecting PDM clients according to 'PDM-SPEC'. PROTOCOL @@ -80,14 +86,75 @@ named `application` instead of `wmain`, and use that object directly as the WSGI application object. When calling the WSGI application, a new thread is started for each -request, in which the WSGI application object is called. All requests -run in the same interpreter, so it is guaranteed that data structures -and global variables can be shared between requests. +request, in which the WSGI application object is called (but see +below, under REQUEST HANDLING, for details). All requests run in the +same interpreter, so it is guaranteed that data structures and global +variables can be shared between requests. The WSGI environment is the standard CGI environment, including the `SCRIPT_FILENAME` variable whenever the `X-Ash-File` header was included in the request. +REQUEST HANDLING +---------------- + +*ashd-wsgi* can be configured to handle requests in various ways, +using the *-t* command-line option. The argument to the *-t* option +takes the form 'HANDLER'[*:*'PAR'[*=*'VAL'][(*,*'PAR'[*=*'VAL'])...]], +in order to specify the handler model, along with parameters to the +same (using the same syntax as the port specifications of +*htparser*(1)). The 'HANDLER' can be any of the following: + +*free*[*:max=*'MAX-THREADS'*,timeout=*'TIMEOUT']:: + + The *free* handler, which is the default, starts a new thread + for every incoming request, which runs the whole request in + its entirety, from running the WSGI handler function to + sending the contents of the response iterator. Optionally, + 'MAX-THREADS' may be specified to an integer, in which case no + more than that many request-handler threads will be allowed to + run at any one time (by default, any number of threads are + allowed to run, without limit). If further requests come in + while 'MAX-THREADS' handlers are running, the request dispatch + thread itself will block until one exits, making new requests + queue up in the socket over which they arrive, eventually + filling up its buffers if no threads exit, in turn making the + parent handler either block or receive *EAGAIN* errors. Also, + if 'MAX-THREADS' is specified, 'TIMEOUT' may also be + specified, to tell the dispatcher thread to never block more + than so many seconds for a handler thread to exit. If it is + forced to wait longer than 'TIMEOUT' seconds, it will assume + the whole process is somehow foobar and will *abort*(3). + +*rplex*[*:max=*'MAX-THREADS']:: + + The *rplex* handler starts a new thread for every incoming + request, but unlike the *free* handler, only the WSGI handler + function runs in that thread. Whenever any such thread, then, + returns its response iterator, all such iterators will be + passed to a single independent thread which sends their + contents to the clients, multiplexing between them whenever + their respective clients are ready to receive data. Like the + *free* handler, a 'MAX-THREADS' argument may be given to + specify how many handler threads are allowed to run at the + same time. The main advantage, compared to the *free* handler, + is that the *rplex* handler allows an arbitrary number of + response iterators to run simultaneously without tying up + handler threads, therefore not counting towards 'MAX-THREADS', + which may be necessary for applications handling large + files. However, it must be noted that no response iterators in + the application may block on returning data, since that would + also block all other running responses. Also, the *rplex* + handler does not support the `write` function returned by + `start_request`, according to the WSGI specification. + +*single*:: + + The *single* handler starts no threads at all, running all + received requests directly in the main dispatch thread. It is + probably not good for much except as the simplest possible + example of a request handling model. + EXAMPLES -------- @@ -99,6 +166,7 @@ child wsgidir exec ashd-wsgi ashd.wsgidir match filename *.wsgi + xset python-handler chain handler wsgidir -------- diff --git a/python/doc/scgi-wsgi.doc b/python/doc/scgi-wsgi.doc index 1aab621..08fc31e 100644 --- a/python/doc/scgi-wsgi.doc +++ b/python/doc/scgi-wsgi.doc @@ -7,7 +7,7 @@ scgi-wsgi - WSGI adapter for SCGI SYNOPSIS -------- -*scgi-wsgi* [*-hA*] [*-p* 'MODPATH'] [*-T* \[HOST:]'PORT'] 'HANDLER-MODULE' ['ARGS'...] +*scgi-wsgi* [*-hAL*] [*-m* 'PDM-SPEC'] [*-p* 'MODPATH'] [*-t* 'HANDLING-MODEL'] [*-T* \[HOST:]'PORT'] 'HANDLER-MODULE' ['ARGS'...] DESCRIPTION ----------- @@ -41,11 +41,22 @@ OPTIONS the WSGI application object. See the PROTOCOL section of *ashd-wsgi*(1) for details. +*-L*:: + By default, *scgi-wsgi* sets up the Python logging with a + logging format and for logging to standard error. The *-L* + option suppresses that behavior, so that any handler module + may set up logging itself. + *-p* 'MODPATH':: Prepend 'MODPATH' to Python's `sys.path`; can be given multiple times. +*-t* 'HANDLING-MODEL':: + + Specify the way *scgi-wsgi* handles requests. See the REQUEST + HANDLING section of *ashd-wsgi*(1) for details. + *-T* \[HOST:]'PORT':: Instead of using a listening socket passed on standard input @@ -53,6 +64,12 @@ OPTIONS address listening for connections on 'PORT' instead. If 'HOST' is not given, `localhost` is used by default. +*-m* 'PDM-SPEC':: + + If the PDM library is installed on the system, create a + listening socket for connecting PDM clients according to + 'PDM-SPEC'. + AUTHOR ------ Fredrik Tolf diff --git a/python/htp.c b/python/htp.c index 33c0361..2daeddf 100644 --- a/python/htp.c +++ b/python/htp.c @@ -32,18 +32,25 @@ static PyObject *p_recvfd(PyObject *self, PyObject *args) fd = 0; if(!PyArg_ParseTuple(args, "|i", &fd)) return(NULL); - Py_BEGIN_ALLOW_THREADS; - ret = recvfd(fd, &data, &dlen); - Py_END_ALLOW_THREADS; - if(ret < 0) { - if(errno == 0) - return(Py_BuildValue("OO", Py_None, Py_None)); - PyErr_SetFromErrno(PyExc_OSError); - return(NULL); + while(1) { + Py_BEGIN_ALLOW_THREADS; + ret = recvfd(fd, &data, &dlen); + Py_END_ALLOW_THREADS; + if(ret < 0) { + if(errno == 0) + return(Py_BuildValue("OO", Py_None, Py_None)); + if(errno == EINTR) { + if(PyErr_CheckSignals()) + return(NULL); + continue; + } + PyErr_SetFromErrno(PyExc_OSError); + return(NULL); + } + ro = Py_BuildValue("Ni", PyString_FromStringAndSize(data, dlen), ret); + free(data); + return(ro); } - ro = Py_BuildValue("Ni", PyString_FromStringAndSize(data, dlen), ret); - free(data); - return(ro); } static PyObject *p_sendfd(PyObject *self, PyObject *args) @@ -57,14 +64,21 @@ static PyObject *p_sendfd(PyObject *self, PyObject *args) PyErr_SetString(PyExc_TypeError, "datagram must be a string"); return(NULL); } - Py_BEGIN_ALLOW_THREADS; - ret = sendfd(sock, fd, PyString_AsString(data), PyString_Size(data)); - Py_END_ALLOW_THREADS; - if(ret < 0) { - PyErr_SetFromErrno(PyExc_OSError); - return(NULL); + while(1) { + Py_BEGIN_ALLOW_THREADS; + ret = sendfd(sock, fd, PyString_AsString(data), PyString_Size(data)); + Py_END_ALLOW_THREADS; + if(ret < 0) { + if(errno == EINTR) { + if(PyErr_CheckSignals()) + return(NULL); + continue; + } + PyErr_SetFromErrno(PyExc_OSError); + return(NULL); + } + Py_RETURN_NONE; } - Py_RETURN_NONE; } static PyMethodDef methods[] = { diff --git a/python/scgi-wsgi b/python/scgi-wsgi index e2689d4..99e003c 100755 --- a/python/scgi-wsgi +++ b/python/scgi-wsgi @@ -1,16 +1,21 @@ #!/usr/bin/python -import sys, os, getopt, logging +import sys, os, getopt, logging, platform import socket -import ashd.scgi +import ashd.scgi, ashd.serve +try: + import pdm.srv +except: + pdm = None def usage(out): - out.write("usage: scgi-wsgi [-hAL] [-p MODPATH] [-T [HOST:]PORT] HANDLER-MODULE [ARGS...]\n") + out.write("usage: scgi-wsgi [-hAL] [-m PDM-SPEC] [-p MODPATH] [-t REQUEST-HANDLER[:PAR[=VAL](,PAR[=VAL])...]] [-T [HOST:]PORT] HANDLER-MODULE [ARGS...]\n") sk = None +hspec = "free", {} modwsgi_compat = False setlog = True -opts, args = getopt.getopt(sys.argv[1:], "+hALp:T:") +opts, args = getopt.getopt(sys.argv[1:], "+hALp:t:T:m:") for o, a in opts: if o == "-h": usage(sys.stdout) @@ -33,6 +38,11 @@ for o, a in opts: sk.listen(32) elif o == "-A": modwsgi_compat = True + elif o == "-m": + if pdm is not None: + pdm.srv.listen(a) + elif o == "-t": + hspec = ashd.serve.parsehspec(a) if len(args) < 1: usage(sys.stderr) sys.exit(1) @@ -61,4 +71,84 @@ else: sys.exit(1) handler = handlermod.application -ashd.scgi.servescgi(sk, ashd.scgi.wrapwsgi(handler)) +def mkenv(head, sk): + env = dict(head) + env["wsgi.version"] = 1, 0 + if "HTTP_X_ASH_PROTOCOL" in env: + env["wsgi.url_scheme"] = env["HTTP_X_ASH_PROTOCOL"] + elif "HTTPS" in env: + env["wsgi.url_scheme"] = "https" + else: + env["wsgi.url_scheme"] = "http" + env["wsgi.input"] = sk + env["wsgi.errors"] = sys.stderr + env["wsgi.multithread"] = True + env["wsgi.multiprocess"] = False + env["wsgi.run_once"] = False + return env + +class request(ashd.serve.wsgirequest): + def __init__(self, sk, **kw): + super(request, self).__init__(**kw) + self.bsk = sk.dup() + self.sk = self.bsk.makefile("r+") + + def mkenv(self): + return mkenv(ashd.scgi.readhead(self.sk), self.sk) + + def handlewsgi(self, env, startreq): + return handler(env, startreq) + + _onjython = None + @staticmethod + def onjython(): + if request._onjython is None: + request._onjython = ("java" in platform.system().lower()) + return request._onjython + + def fileno(self): + if request.onjython(): + self.bsk.setblocking(False) + return self.bsk.fileno() + + def writehead(self, status, headers): + w = self.buffer.extend + w("Status: %s\n" % status) + for nm, val in headers: + w("%s: %s\n" % (nm, val)) + w("\n") + + def flush(self): + try: + if not request.onjython(): + ret = self.bsk.send(self.buffer, socket.MSG_DONTWAIT) + else: + ret = self.bsk.send(str(self.buffer)) + self.buffer[:ret] = "" + except IOError: + raise ashd.serve.closed() + + def close(self): + self.sk.close() + self.bsk.close() + +if hspec[0] not in ashd.serve.names: + sys.stderr.write("scgi-wsgi: no such request handler: %s\n" % hspec[0]) + sys.exit(1) +hclass = ashd.serve.names[hspec[0]] +try: + hargs = hclass.parseargs(**hspec[1]) +except ValueError as exc: + sys.stderr.write("scgi-wsgi: %s\n" % exc) + sys.exit(1) + +reqhandler = hclass(**hargs) +try: + while True: + nsk, addr = sk.accept() + try: + reqhandler.handle(request(sk=nsk, handler=reqhandler)) + finally: + nsk.close() +finally: + reqhandler.close() diff --git a/python/setup.py b/python/setup.py index ed6f192..3ffdcd8 100755 --- a/python/setup.py +++ b/python/setup.py @@ -6,7 +6,7 @@ htlib = Extension("ashd.htlib", ["htp.c"], libraries = ["ht"]) setup(name = "ashd-py", - version = "0.5", + version = "0.6", description = "Python module for handling ashd requests", author = "Fredrik Tolf", author_email = "fredrik@dolda2000.com", diff --git a/python3/ashd-wsgi3 b/python3/ashd-wsgi3 index 8944b5c..fa0b8b5 100755 --- a/python3/ashd-wsgi3 +++ b/python3/ashd-wsgi3 @@ -1,19 +1,19 @@ #!/usr/bin/python3 -import sys, os, getopt, threading, logging, time, locale, collections -import ashd.proto, ashd.util, ashd.perf +import sys, os, getopt, socket, logging, time, locale, collections, signal +import ashd.util, ashd.serve, ashd.htlib try: import pdm.srv except: pdm = None def usage(out): - out.write("usage: ashd-wsgi3 [-hAL] [-m PDM-SPEC] [-p MODPATH] [-l REQLIMIT] HANDLER-MODULE [ARGS...]\n") + out.write("usage: ashd-wsgi3 [-hAL] [-m PDM-SPEC] [-p MODPATH] [-t REQUEST-HANDLER[:PAR[=VAL](,PAR[=VAL])...]] HANDLER-MODULE [ARGS...]\n") -reqlimit = 0 +hspec = "free", {} modwsgi_compat = False setlog = True -opts, args = getopt.getopt(sys.argv[1:], "+hALp:l:m:") +opts, args = getopt.getopt(sys.argv[1:], "+hALp:t:l:m:") for o, a in opts: if o == "-h": usage(sys.stdout) @@ -25,7 +25,9 @@ for o, a in opts: elif o == "-A": modwsgi_compat = True elif o == "-l": - reqlimit = int(a) + hspec = "free", {"max": a, "abort": "10"} + elif o == "-t": + hspec = ashd.serve.parsehspec(a) elif o == "-m": if pdm is not None: pdm.srv.listen(a) @@ -52,10 +54,6 @@ else: sys.exit(1) handler = handlermod.application -class closed(IOError): - def __init__(self): - super().__init__("The client has closed the connection.") - cwd = os.getcwd() def absolutify(path): if path[0] != '/': @@ -95,7 +93,7 @@ def unquoteurl(url): buf.append(c) return buf -def dowsgi(req): +def mkenv(req): env = {} env["wsgi.version"] = 1, 0 for key, val in req.headers: @@ -147,103 +145,82 @@ def dowsgi(req): env["wsgi.multithread"] = True env["wsgi.multiprocess"] = False env["wsgi.run_once"] = False + return env - resp = [] - respsent = [] - - def recode(thing): - if isinstance(thing, collections.ByteString): - return thing - else: - return str(thing).encode("latin-1") - - def flushreq(): - if not respsent: - if not resp: - raise Exception("Trying to write data before starting response.") - status, headers = resp - respsent[:] = [True] - buf = bytearray() - buf += b"HTTP/1.1 " + recode(status) + b"\n" - for nm, val in headers: - buf += recode(nm) + b": " + recode(val) + b"\n" - buf += b"\n" - try: - req.sk.write(buf) - except IOError: - raise closed() - - def write(data): - if not data: - return - flushreq() +def recode(thing): + if isinstance(thing, collections.ByteString): + return thing + else: + return str(thing).encode("latin-1") + +class request(ashd.serve.wsgirequest): + def __init__(self, *, bkreq, **kw): + super().__init__(**kw) + self.bkreq = bkreq.dup() + self.sendrights = None + + def mkenv(self): + return mkenv(self.bkreq) + + def handlewsgi(self, env, startreq): + return handler(env, startreq) + + def fileno(self): + return self.bkreq.bsk.fileno() + + def writehead(self, status, headers): + headers = list(headers) + for header in headers: + nm, val = header + if nm.lower() == "x-ash-send-rights": + self.sendrights = val + headers.remove(header) + break + w = self.buffer.extend + w(b"HTTP/1.1 " + recode(status) + b"\n") + for nm, val in headers: + w(recode(nm) + b": " + recode(val) + b"\n") + w(b"\n") + + def flush(self): try: - req.sk.write(data) - req.sk.flush() - except IOError: - raise closed() - - def startreq(status, headers, exc_info = None): - if resp: - if exc_info: # Interesting, this... - try: - if respsent: - raise exc_info[1] - finally: - exc_info = None # CPython GC bug? + if self.sendrights is not None: + ret = ashd.htlib.sendfd(self.bkreq.bsk.fileno(), self.sendrights.fileno(), self.buffer) + self.sendrights.close() + self.sendrights = None else: - raise Exception("Can only start responding once.") - resp[:] = status, headers - return write + ret = self.bkreq.bsk.send(self.buffer, socket.MSG_DONTWAIT) + self.buffer[:ret] = b"" + except IOError: + raise ashd.serve.closed() - with ashd.perf.request(env) as reqevent: - try: - respiter = handler(env, startreq) - try: - for data in respiter: - write(data) - if resp: - flushreq() - finally: - if hasattr(respiter, "close"): - respiter.close() - except closed: - pass - if resp: - reqevent.response(resp) - -flightlock = threading.Condition() -inflight = 0 - -class reqthread(threading.Thread): - def __init__(self, req): - super().__init__(name = "Request handler") - self.req = req.dup() - - def run(self): - global inflight + def close(self): try: - with flightlock: - if reqlimit != 0: - start = time.time() - while inflight >= reqlimit: - flightlock.wait(10) - if time.time() - start > 10: - os.abort() - inflight += 1 - try: - dowsgi(self.req) - finally: - with flightlock: - inflight -= 1 - flightlock.notify() - except: - log.error("exception occurred in handler thread", exc_info=True) + self.bkreq.close() finally: - self.req.close() - sys.stderr.flush() - + if self.sendrights is not None: + self.sendrights.close() + def handle(req): - reqthread(req).start() + reqhandler.handle(request(bkreq=req, handler=reqhandler)) -ashd.util.serveloop(handle) +if hspec[0] not in ashd.serve.names: + sys.stderr.write("ashd-wsgi3: no such request handler: %s\n" % hspec[0]) + sys.exit(1) +hclass = ashd.serve.names[hspec[0]] +try: + hargs = hclass.parseargs(**hspec[1]) +except ValueError as exc: + sys.stderr.write("ashd-wsgi3: %s\n" % exc) + sys.exit(1) + +def sigterm(sig, frame): + socket.fromfd(0, socket.AF_UNIX, socket.SOCK_SEQPACKET).shutdown(socket.SHUT_RDWR) # :P +for signum in [signal.SIGINT, signal.SIGTERM]: + signal.signal(signum, sigterm) + +reqhandler = hclass(**hargs) +try: + ashd.util.serveloop(handle) +finally: + reqhandler.close() diff --git a/python3/ashd/async.py b/python3/ashd/async.py new file mode 100644 index 0000000..99da89a --- /dev/null +++ b/python3/ashd/async.py @@ -0,0 +1,289 @@ +import sys, os, errno, threading, select, traceback + +class epoller(object): + exc_handler = None + + def __init__(self, check=None): + self.registered = {} + self.lock = threading.RLock() + self.ep = None + self.th = None + self.stopped = False + self.loopcheck = set() + if check is not None: + self.loopcheck.add(check) + self._daemon = True + + @staticmethod + def _evsfor(ch): + return ((select.EPOLLIN if ch.readable else 0) | + (select.EPOLLOUT if ch.writable else 0)) + + def _ckrun(self): + if self.registered and self.th is None: + th = threading.Thread(target=self._run, name="Async epoll thread") + th.daemon = self._daemon + th.start() + self.th = th + + def exception(self, ch, *exc): + self.remove(ch) + if self.exc_handler is None: + traceback.print_exception(*exc) + else: + self.exc_handler(ch, *exc) + + def _cb(self, ch, nm): + try: + m = getattr(ch, nm, None) + if m is None: + raise AttributeError("%r has no %s method" % (ch, nm)) + m() + except Exception as exc: + self.exception(ch, *sys.exc_info()) + + def _closeall(self): + with self.lock: + while self.registered: + fd, (ch, evs) = next(iter(self.registered.items())) + del self.registered[fd] + self.ep.unregister(fd) + self._cb(ch, "close") + + def _run(self): + ep = select.epoll() + try: + with self.lock: + for fd, (ob, evs) in self.registered.items(): + ep.register(fd, evs) + self.ep = ep + + while self.registered: + for ck in self.loopcheck: + ck(self) + if self.stopped: + self._closeall() + break + try: + evlist = ep.poll(10) + except IOError as exc: + if exc.errno == errno.EINTR: + continue + raise + for fd, evs in evlist: + with self.lock: + if fd not in self.registered: + continue + ch, cevs = self.registered[fd] + if fd in self.registered and evs & (select.EPOLLIN | select.EPOLLHUP | select.EPOLLERR): + self._cb(ch, "read") + if fd in self.registered and evs & select.EPOLLOUT: + self._cb(ch, "write") + if fd in self.registered: + nevs = self._evsfor(ch) + if nevs == 0: + del self.registered[fd] + ep.unregister(fd) + self._cb(ch, "close") + elif nevs != cevs: + self.registered[fd] = ch, nevs + ep.modify(fd, nevs) + + finally: + with self.lock: + self.th = None + self.ep = None + self._ckrun() + ep.close() + + @property + def daemon(self): return self._daemon + @daemon.setter + def daemon(self, value): + self._daemon = bool(value) + with self.lock: + if self.th is not None: + self.th = daemon = self._daemon + + def add(self, ch): + with self.lock: + fd = ch.fileno() + if fd in self.registered: + raise KeyError("fd %i is already registered" % fd) + evs = self._evsfor(ch) + if evs == 0: + ch.close() + return + ch.watcher = self + self.registered[fd] = (ch, evs) + if self.ep: + self.ep.register(fd, evs) + self._ckrun() + + def remove(self, ch, ignore=False): + with self.lock: + fd = ch.fileno() + if fd not in self.registered: + if ignore: + return + raise KeyError("fd %i is not registered" % fd) + pch, cevs = self.registered[fd] + if pch is not ch: + raise ValueError("fd %i registered via object %r, cannot remove with %r" % (pch, ch)) + del self.registered[fd] + if self.ep: + self.ep.unregister(fd) + ch.close() + + def update(self, ch, ignore=False): + with self.lock: + fd = ch.fileno() + if fd not in self.registered: + if ignore: + return + raise KeyError("fd %i is not registered" % fd) + pch, cevs = self.registered[fd] + if pch is not ch: + raise ValueError("fd %i registered via object %r, cannot update with %r" % (pch, ch)) + evs = self._evsfor(ch) + if evs == 0: + del self.registered[fd] + if self.ep: + self.ep.unregister(fd) + ch.close() + elif evs != cevs: + self.registered[fd] = ch, evs + if self.ep: + self.ep.modify(fd, evs) + + def stop(self): + if threading.current_thread() == self.th: + self.stopped = True + else: + def tgt(): + self.stopped = True + cb = callbuffer() + cb.call(tgt) + cb.stop() + self.add(cb) + +def watcher(): + return epoller() + +class channel(object): + readable = False + writable = False + + def __init__(self): + self.watcher = None + + def fileno(self): + raise NotImplementedError("fileno()") + + def close(self): + pass + +class sockbuffer(channel): + def __init__(self, socket, **kwargs): + super().__init__(**kwargs) + self.sk = socket + self.eof = False + self.obuf = bytearray() + + def fileno(self): + return self.sk.fileno() + + def close(self): + self.sk.close() + + def gotdata(self, data): + if data == b"": + self.eof = True + + def send(self, data, eof=False): + self.obuf.extend(data) + if eof: + self.eof = True + if self.watcher is not None: + self.watcher.update(self, True) + + @property + def readable(self): + return not self.eof + def read(self): + try: + data = self.sk.recv(1024) + self.gotdata(data) + except IOError: + self.obuf[:] = b"" + self.eof = True + + @property + def writable(self): + return bool(self.obuf); + def write(self): + try: + ret = self.sk.send(self.obuf) + self.obuf[:ret] = b"" + except IOError: + self.obuf[:] = b"" + self.eof = True + +class callbuffer(channel): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.queue = [] + self.rp, self.wp = os.pipe() + self.lock = threading.Lock() + self.eof = False + + def fileno(self): + return self.rp + + def close(self): + with self.lock: + try: + if self.wp >= 0: + os.close(self.wp) + self.wp = -1 + finally: + if self.rp >= 0: + os.close(self.rp) + self.rp = -1 + + @property + def readable(self): + return not self.eof + def read(self): + with self.lock: + try: + data = os.read(self.rp, 1024) + if data == b"": + self.eof = True + except IOError: + self.eof = True + cbs = list(self.queue) + self.queue[:] = [] + for cb in cbs: + cb() + + writable = False + + def call(self, cb): + with self.lock: + if self.wp < 0: + raise Exception("stopped") + self.queue.append(cb) + os.write(self.wp, b"a") + + def stop(self): + with self.lock: + if self.wp >= 0: + os.close(self.wp) + self.wp = -1 + +def currentwatcher(io, current): + def check(io): + if not current: + io.stop() + io.loopcheck.add(check) diff --git a/python3/ashd/perf.py b/python3/ashd/perf.py index 6942424..86fe43c 100644 --- a/python3/ashd/perf.py +++ b/python3/ashd/perf.py @@ -3,6 +3,11 @@ try: import pdm.perf except: pdm = None +try: + import time + clock_thread = time.CLOCK_THREAD_CPUTIME_ID +except: + clock_thread = None reqstat = {} @@ -17,13 +22,23 @@ if pdm: self.method = env.get("REQUEST_METHOD") self.uri = env.get("REQUEST_URI") self.host = env.get("HTTP_HOST") + self.script_uri = env.get("SCRIPT_NAME") + self.script_path = env.get("SCRIPT_FILENAME") + self.pathinfo = env.get("PATH_INFO") + self.querystring = env.get("QUERY_STRING") self.remoteaddr = env.get("REMOTE_ADDR") self.remoteport = env.get("REMOTE_PORT") + self.scheme = env.get("wsgi.url_scheme") + if clock_thread is not None: + self.icpu = time.clock_gettime(clock_thread) class reqfinish(pdm.perf.finishevent): def __init__(self, start, aborted, status): super().__init__(start, aborted) self.status = status + self.cputime = 0 + if clock_thread is not None: + self.cputime = time.clock_gettime(clock_thread) - start.icpu class request(object): def __init__(self, env): diff --git a/python3/ashd/proto.py b/python3/ashd/proto.py index 7a078de..aa6b686 100644 --- a/python3/ashd/proto.py +++ b/python3/ashd/proto.py @@ -179,4 +179,4 @@ def sendreq(sock, req): data += key + b'\0' data += val + b'\0' data += b'\0' - htlib.sendfd(sock, req.sk.fileno(), data) + htlib.sendfd(sock, req.bsk.fileno(), data) diff --git a/python3/ashd/scgi.py b/python3/ashd/scgi.py index 8fa5767..c00c5a3 100644 --- a/python3/ashd/scgi.py +++ b/python3/ashd/scgi.py @@ -1,13 +1,6 @@ -import sys, collections -import threading - class protoerr(Exception): pass -class closed(IOError): - def __init__(self): - super(closed, self).__init__("The client has closed the connection.") - def readns(sk): hln = 0 while True: @@ -34,115 +27,5 @@ def readhead(sk): i += 2 return ret -class reqthread(threading.Thread): - def __init__(self, sk, handler): - super(reqthread, self).__init__(name = "SCGI request handler") - self.bsk = sk.dup() - self.sk = self.bsk.makefile("rwb") - self.handler = handler - - def run(self): - try: - head = readhead(self.sk) - self.handler(head, self.sk) - finally: - self.sk.close() - self.bsk.close() - -def handlescgi(sk, handler): - t = reqthread(sk, handler) - t.start() - -def servescgi(socket, handler): - while True: - nsk, addr = socket.accept() - try: - handlescgi(nsk, handler) - finally: - nsk.close() - def decodehead(head, coding): return {k.decode(coding): v.decode(coding) for k, v in head.items()} - -def wrapwsgi(handler): - def handle(head, sk): - try: - env = decodehead(head, "utf-8") - env["wsgi.uri_encoding"] = "utf-8" - except UnicodeError: - env = decodehead(head, "latin-1") - env["wsgi.uri_encoding"] = "latin-1" - env["wsgi.version"] = 1, 0 - if "HTTP_X_ASH_PROTOCOL" in env: - env["wsgi.url_scheme"] = env["HTTP_X_ASH_PROTOCOL"] - elif "HTTPS" in env: - env["wsgi.url_scheme"] = "https" - else: - env["wsgi.url_scheme"] = "http" - env["wsgi.input"] = sk - env["wsgi.errors"] = sys.stderr - env["wsgi.multithread"] = True - env["wsgi.multiprocess"] = False - env["wsgi.run_once"] = False - - resp = [] - respsent = [] - - def recode(thing): - if isinstance(thing, collections.ByteString): - return thing - else: - return str(thing).encode("latin-1") - - def flushreq(): - if not respsent: - if not resp: - raise Exception("Trying to write data before starting response.") - status, headers = resp - respsent[:] = [True] - buf = bytearray() - buf += b"Status: " + recode(status) + b"\n" - for nm, val in headers: - buf += recode(nm) + b": " + recode(val) + b"\n" - buf += b"\n" - try: - sk.write(buf) - except IOError: - raise closed() - - def write(data): - if not data: - return - flushreq() - try: - sk.write(data) - sk.flush() - except IOError: - raise closed() - - def startreq(status, headers, exc_info = None): - if resp: - if exc_info: # Interesting, this... - try: - if respsent: - raise exc_info[1] - finally: - exc_info = None # CPython GC bug? - else: - raise Exception("Can only start responding once.") - resp[:] = status, headers - return write - - respiter = handler(env, startreq) - try: - try: - for data in respiter: - write(data) - if resp: - flushreq() - except closed: - pass - finally: - if hasattr(respiter, "close"): - respiter.close() - return handle diff --git a/python3/ashd/serve.py b/python3/ashd/serve.py new file mode 100644 index 0000000..e156cd6 --- /dev/null +++ b/python3/ashd/serve.py @@ -0,0 +1,489 @@ +import sys, os, threading, time, logging, select, queue, collections +from . import perf + +log = logging.getLogger("ashd.serve") +seq = 1 +seqlk = threading.Lock() + +def reqseq(): + global seq + with seqlk: + s = seq + seq += 1 + return s + +class closed(IOError): + def __init__(self): + super().__init__("The client has closed the connection.") + +class reqthread(threading.Thread): + def __init__(self, *, name=None, **kw): + if name is None: + name = "Request handler %i" % reqseq() + super().__init__(name=name, **kw) + +class wsgirequest(object): + def __init__(self, *, handler): + self.status = None + self.headers = [] + self.respsent = False + self.handler = handler + self.buffer = bytearray() + + def handlewsgi(self): + raise Exception() + def fileno(self): + raise Exception() + def writehead(self, status, headers): + raise Exception() + def flush(self): + raise Exception() + def close(self): + pass + def writedata(self, data): + self.buffer.extend(data) + + def flushreq(self): + if not self.respsent: + if not self.status: + raise Exception("Cannot send response body before starting response.") + self.respsent = True + self.writehead(self.status, self.headers) + + def write(self, data): + if not data: + return + self.flushreq() + self.writedata(data) + self.handler.ckflush(self) + + def startreq(self, status, headers, exc_info=None): + if self.status: + if exc_info: + try: + if self.respsent: + raise exc_info[1] + finally: + exc_info = None + else: + raise Exception("Can only start responding once.") + self.status = status + self.headers = headers + return self.write + +class handler(object): + def handle(self, request): + raise Exception() + def ckflush(self, req): + while len(req.buffer) > 0: + rls, wls, els = select.select([], [req], [req]) + req.flush() + def close(self): + pass + + @classmethod + def parseargs(cls, **args): + if len(args) > 0: + raise ValueError("unknown handler argument: " + next(iter(args))) + return {} + +class single(handler): + cname = "single" + + def handle(self, req): + try: + env = req.mkenv() + with perf.request(env) as reqevent: + respiter = req.handlewsgi(env, req.startreq) + for data in respiter: + req.write(data) + if req.status: + reqevent.response([req.status, req.headers]) + req.flushreq() + self.ckflush(req) + except closed: + pass + except: + log.error("exception occurred when handling request", exc_info=True) + finally: + req.close() + +def dbg(*a): + f = True + for o in a: + if not f: + sys.stderr.write(" ") + sys.stderr.write(str(a)) + f = False + sys.stderr.write("\n") + sys.stderr.flush() + +class freethread(handler): + cname = "free" + + def __init__(self, *, max=None, timeout=None, **kw): + super().__init__(**kw) + self.current = set() + self.lk = threading.Lock() + self.tcond = threading.Condition(self.lk) + self.max = max + self.timeout = timeout + + @classmethod + def parseargs(cls, *, max=None, abort=None, **args): + ret = super().parseargs(**args) + if max: + ret["max"] = int(max) + if abort: + ret["timeout"] = int(abort) + return ret + + def handle(self, req): + with self.lk: + if self.max is not None: + if self.timeout is not None: + now = start = time.time() + while len(self.current) >= self.max: + self.tcond.wait(start + self.timeout - now) + now = time.time() + if now - start > self.timeout: + os.abort() + else: + while len(self.current) >= self.max: + self.tcond.wait() + th = reqthread(target=self.run, args=[req]) + th.registered = False + th.start() + while not th.registered: + self.tcond.wait() + + def run(self, req): + try: + th = threading.current_thread() + with self.lk: + self.current.add(th) + th.registered = True + self.tcond.notify_all() + try: + env = req.mkenv() + with perf.request(env) as reqevent: + respiter = req.handlewsgi(env, req.startreq) + for data in respiter: + req.write(data) + if req.status: + reqevent.response([req.status, req.headers]) + req.flushreq() + self.ckflush(req) + except closed: + pass + except: + log.error("exception occurred when handling request", exc_info=True) + finally: + with self.lk: + self.current.remove(th) + self.tcond.notify_all() + finally: + req.close() + + def close(self): + while True: + with self.lk: + if len(self.current) > 0: + th = next(iter(self.current)) + else: + return + th.join() + +class threadpool(handler): + cname = "pool" + + def __init__(self, *, max=25, qsz=100, timeout=None, **kw): + super().__init__(**kw) + self.current = set() + self.clk = threading.Lock() + self.ccond = threading.Condition(self.clk) + self.queue = collections.deque() + self.waiting = set() + self.waitlimit = 5 + self.wlstart = 0.0 + self.qlk = threading.Lock() + self.qcond = threading.Condition(self.qlk) + self.max = max + self.qsz = qsz + self.timeout = timeout + + @classmethod + def parseargs(cls, *, max=None, queue=None, abort=None, **args): + ret = super().parseargs(**args) + if max: + ret["max"] = int(max) + if queue: + ret["qsz"] = int(queue) + if abort: + ret["timeout"] = int(abort) + return ret + + def handle(self, req): + spawn = False + with self.qlk: + if self.timeout is not None: + now = start = time.time() + while len(self.queue) >= self.qsz: + self.qcond.wait(start + self.timeout - now) + now = time.time() + if now - start > self.timeout: + os.abort() + else: + while len(self.current) >= self.qsz: + self.qcond.wait() + self.queue.append(req) + self.qcond.notify() + if len(self.waiting) < 1: + spawn = True + if spawn: + with self.clk: + if len(self.current) < self.max: + th = reqthread(target=self.run) + th.registered = False + th.start() + while not th.registered: + self.ccond.wait() + + def handle1(self, req): + try: + env = req.mkenv() + with perf.request(env) as reqevent: + respiter = req.handlewsgi(env, req.startreq) + for data in respiter: + req.write(data) + if req.status: + reqevent.response([req.status, req.headers]) + req.flushreq() + self.ckflush(req) + except closed: + pass + except: + log.error("exception occurred when handling request", exc_info=True) + + def run(self): + timeout = 10.0 + th = threading.current_thread() + with self.clk: + self.current.add(th) + th.registered = True + self.ccond.notify_all() + try: + while True: + start = now = time.time() + with self.qlk: + while len(self.queue) < 1: + if len(self.waiting) >= self.waitlimit and now - self.wlstart >= timeout: + return + self.waiting.add(th) + try: + if len(self.waiting) == self.waitlimit: + self.wlstart = now + self.qcond.wait(start + timeout - now) + finally: + self.waiting.remove(th) + now = time.time() + if now - start > timeout: + return + req = self.queue.popleft() + try: + self.handle1(req) + finally: + req.close() + finally: + with self.clk: + self.current.remove(th) + + def close(self): + while True: + with self.clk: + if len(self.current) > 0: + th = next(iter(self.current)) + else: + return + th.join() + +class resplex(handler): + cname = "rplex" + + def __init__(self, *, max=None, **kw): + super().__init__(**kw) + self.current = set() + self.lk = threading.Lock() + self.tcond = threading.Condition(self.lk) + self.max = max + self.cqueue = queue.Queue(5) + self.cnpipe = os.pipe() + self.rthread = reqthread(name="Response thread", target=self.handle2) + self.rthread.start() + + @classmethod + def parseargs(cls, *, max=None, **args): + ret = super().parseargs(**args) + if max: + ret["max"] = int(max) + return ret + + def ckflush(self, req): + raise Exception("resplex handler does not support the write() function") + + def handle(self, req): + with self.lk: + if self.max is not None: + while len(self.current) >= self.max: + self.tcond.wait() + th = reqthread(target=self.handle1, args=[req]) + th.registered = False + th.start() + while not th.registered: + self.tcond.wait() + + def handle1(self, req): + try: + th = threading.current_thread() + with self.lk: + self.current.add(th) + th.registered = True + self.tcond.notify_all() + try: + env = req.mkenv() + respobj = req.handlewsgi(env, req.startreq) + respiter = iter(respobj) + if not req.status: + log.error("request handler returned without calling start_request") + if hasattr(respiter, "close"): + respiter.close() + return + else: + self.cqueue.put((req, respiter)) + os.write(self.cnpipe[1], b" ") + req = None + finally: + with self.lk: + self.current.remove(th) + self.tcond.notify_all() + except closed: + pass + except: + log.error("exception occurred when handling request", exc_info=True) + finally: + if req is not None: + req.close() + + def handle2(self): + try: + rp = self.cnpipe[0] + current = {} + + def closereq(req): + respiter = current[req] + try: + if respiter is not None and hasattr(respiter, "close"): + respiter.close() + except: + log.error("exception occurred when closing iterator", exc_info=True) + try: + req.close() + except: + log.error("exception occurred when closing request", exc_info=True) + del current[req] + def ckiter(req): + respiter = current[req] + if respiter is not None: + rem = False + try: + data = next(respiter) + except StopIteration: + rem = True + try: + req.flushreq() + except: + log.error("exception occurred when handling response data", exc_info=True) + except: + rem = True + log.error("exception occurred when iterating response", exc_info=True) + if not rem: + if data: + try: + req.flushreq() + req.writedata(data) + except: + log.error("exception occurred when handling response data", exc_info=True) + rem = True + if rem: + current[req] = None + try: + if hasattr(respiter, "close"): + respiter.close() + except: + log.error("exception occurred when closing iterator", exc_info=True) + respiter = None + if respiter is None and not req.buffer: + closereq(req) + + while True: + bufl = list(req for req in current.keys() if req.buffer) + rls, wls, els = select.select([rp], bufl, [rp] + bufl) + if rp in rls: + ret = os.read(rp, 1024) + if not ret: + os.close(rp) + return + try: + while True: + req, respiter = self.cqueue.get(False) + current[req] = respiter + ckiter(req) + except queue.Empty: + pass + for req in wls: + try: + req.flush() + except closed: + closereq(req) + except: + log.error("exception occurred when writing response", exc_info=True) + closereq(req) + else: + if len(req.buffer) < 65536: + ckiter(req) + except: + log.critical("unexpected exception occurred in response handler thread", exc_info=True) + os.abort() + + def close(self): + while True: + with self.lk: + if len(self.current) > 0: + th = next(iter(self.current)) + else: + break + th.join() + os.close(self.cnpipe[1]) + self.rthread.join() + +names = {cls.cname: cls for cls in globals().values() if + isinstance(cls, type) and + issubclass(cls, handler) and + hasattr(cls, "cname")} + +def parsehspec(spec): + if ":" not in spec: + return spec, {} + nm, spec = spec.split(":", 1) + args = {} + while spec: + if "," in spec: + part, spec = spec.split(",", 1) + else: + part, spec = spec, None + if "=" in part: + key, val = part.split("=", 1) + else: + key, val = part, "" + args[key] = val + return nm, args diff --git a/python3/ashd/wsgidir.py b/python3/ashd/wsgidir.py index c6fa1ad..9fc3649 100644 --- a/python3/ashd/wsgidir.py +++ b/python3/ashd/wsgidir.py @@ -58,6 +58,20 @@ class cachedmod(object): self.mod = mod self.mtime = mtime +class current(object): + def __init__(self): + self.cond = threading.Condition() + self.current = True + def wait(self, timeout=None): + with self.cond: + self.cond.wait(timeout) + def uncurrent(self): + with self.cond: + self.current = False + self.cond.notify_all() + def __bool__(self): + return self.current + modcache = {} cachelock = threading.Lock() @@ -97,8 +111,16 @@ def getmod(path): code = compile(text, path, "exec") mod = types.ModuleType(mangle(path)) mod.__file__ = path - exec(code, mod.__dict__) - entry[1] = cachedmod(mod, sb.st_mtime) + mod.__current__ = current() + try: + exec(code, mod.__dict__) + except: + mod.__current__.uncurrent() + raise + else: + if entry[1] is not None: + entry[1].mod.__current__.uncurrent() + entry[1] = cachedmod(mod, sb.st_mtime) return entry[1] def importlocal(filename): diff --git a/python3/ashd/wsgiutil.py b/python3/ashd/wsgiutil.py index 896af61..3686748 100644 --- a/python3/ashd/wsgiutil.py +++ b/python3/ashd/wsgiutil.py @@ -1,4 +1,4 @@ -import time +import time, sys, io def htmlquote(text): ret = "" @@ -42,3 +42,63 @@ def phttpdate(dstr): tz = int(tz[1:]) tz = (((tz / 100) * 60) + (tz % 100)) * 60 return time.mktime(time.strptime(dstr, "%a, %d %b %Y %H:%M:%S")) - tz - time.altzone + +def testenviron(uri, qs="", pi="", method=None, filename=None, host="localhost", data=None, ctype=None, head={}): + if method is None: + method = "GET" if data is None else "POST" + if ctype is None and data is not None: + ctype = "application/x-www-form-urlencoded" + ret = {} + ret["wsgi.version"] = 1, 0 + ret["SERVER_SOFTWARE"] = "ashd-test/1" + ret["GATEWAY_INTERFACE"] = "CGI/1.1" + ret["SERVER_PROTOCOL"] = "HTTP/1.1" + ret["REQUEST_METHOD"] = method + ret["wsgi.uri_encoding"] = "utf-8" + ret["SCRIPT_NAME"] = uri + ret["PATH_INFO"] = pi + ret["QUERY_STRING"] = qs + full = uri + pi + if qs: + full = full + "?" + qs + ret["REQUEST_URI"] = full + if filename is not None: + ret["SCRIPT_FILENAME"] = filename + ret["HTTP_HOST"] = ret["SERVER_NAME"] = host + ret["wsgi.url_scheme"] = "http" + ret["SERVER_ADDR"] = "127.0.0.1" + ret["SERVER_PORT"] = "80" + ret["REMOTE_ADDR"] = "127.0.0.1" + ret["REMOTE_PORT"] = "12345" + if data is not None: + ret["CONTENT_TYPE"] = ctype + ret["CONTENT_LENGTH"] = len(data) + ret["wsgi.input"] = io.BytesIO(data) + else: + ret["wsgi.input"] = io.BytesIO(b"") + ret["wsgi.errors"] = sys.stderr + ret["wsgi.multithread"] = True + ret["wsgi.multiprocess"] = False + ret["wsgi.run_once"] = False + for key, val in head.items(): + ret["HTTP_" + key.upper().replace("-", "_")] = val + return ret + +class testrequest(object): + def __init__(self): + self.wbuf = io.BytesIO() + self.headers = None + self.status = None + + def __call__(self, status, headers): + self.status = status + self.headers = headers + return self.wbuf.write + + def __repr__(self): + return "" % (self.status, + "None" if self.headers is None else ("[%i]" % len(self.headers)), + "(no data)" if len(self.wbuf.getvalue()) == 0 else "(with data)") + + def __str__(self): + return repr(self) diff --git a/python3/doc/ashd-wsgi3.doc b/python3/doc/ashd-wsgi3.doc index 0be02b1..4350693 100644 --- a/python3/doc/ashd-wsgi3.doc +++ b/python3/doc/ashd-wsgi3.doc @@ -7,7 +7,7 @@ ashd-wsgi3 - WSGI adapter for ashd(7) SYNOPSIS -------- -*ashd-wsgi3* [*-hA*] [*-m* 'PDM-SPEC'] [*-p* 'MODPATH'] [*-l* 'LIMIT'] 'HANDLER-MODULE' ['ARGS'...] +*ashd-wsgi3* [*-hAL*] [*-m* 'PDM-SPEC'] [*-p* 'MODPATH'] [*-t* 'HANDLING-MODEL'] 'HANDLER-MODULE' ['ARGS'...] DESCRIPTION ----------- @@ -21,7 +21,10 @@ section, below. multithreaded dispatching in a single Python interpreter, which means that WSGI applications that use it need to be thread-safe, but that they can also share all Python data structures and global variables -between requests. +between requests. More precisely, *ashd-wsgi* implements a couple of +slightly different ways to handle requests and threads, which can be +configured using the *-t* option, as described in the REQUEST HANDLING +section, below. The Python module that *ashd-wsgi3* comes with also contains a standard handler module, `ashd.wsgidir`, which serves individual WSGI @@ -47,6 +50,12 @@ OPTIONS the WSGI application object. See the PROTOCOL section, below, for details. +*-L*:: + By default, *ashd-wsgi3* sets up the Python logging with a + logging format and for logging to standard error. The *-L* + option suppresses that behavior, so that any handler module + may set up logging itself. + *-p* 'MODPATH':: Prepend 'MODPATH' to Python's `sys.path`; can be given multiple @@ -54,18 +63,15 @@ OPTIONS on Python's module path by default, so if you want to use a module in that directory, you will need to specify "`-p .`". -*-l* 'LIMIT':: +*-t* 'HANDLING-MODEL':: - Allow at most 'LIMIT' requests to run concurrently. If a new - request is made when 'LIMIT' requests are executing, the new - request will wait up to ten seconds for one of them to - complete; if none does, *ashd-wsgi3* will assume that the - process is foobar and *abort*(3). + Specify the way *ashd-wsgi* handles requests. See below, under + REQUEST HANDLING. *-m* 'PDM-SPEC':: If the PDM library is installed on the system, create a - listening socket for connection PDM clients according to + listening socket for connecting PDM clients according to 'PDM-SPEC'. PROTOCOL @@ -80,14 +86,75 @@ named `application` instead of `wmain`, and use that object directly as the WSGI application object. When calling the WSGI application, a new thread is started for each -request, in which the WSGI application object is called. All requests -run in the same interpreter, so it is guaranteed that data structures -and global variables can be shared between requests. +request, in which the WSGI application object is called (but see +below, under REQUEST HANDLING, for details). All requests run in the +same interpreter, so it is guaranteed that data structures and global +variables can be shared between requests. The WSGI environment is the standard CGI environment, including the `SCRIPT_FILENAME` variable whenever the `X-Ash-File` header was included in the request. +REQUEST HANDLING +---------------- + +*ashd-wsgi3* can be configured to handle requests in various ways, +using the *-t* command-line option. The argument to the *-t* option +takes the form 'HANDLER'[*:*'PAR'[*=*'VAL'][(*,*'PAR'[*=*'VAL'])...]], +in order to specify the handler model, along with parameters to the +same (using the same syntax as the port specifications of +*htparser*(1)). The 'HANDLER' can be any of the following: + +*free*[*:max=*'MAX-THREADS'*,timeout=*'TIMEOUT']:: + + The *free* handler, which is the default, starts a new thread + for every incoming request, which runs the whole request in + its entirety, from running the WSGI handler function to + sending the contents of the response iterator. Optionally, + 'MAX-THREADS' may be specified to an integer, in which case no + more than that many request-handler threads will be allowed to + run at any one time (by default, any number of threads are + allowed to run, without limit). If further requests come in + while 'MAX-THREADS' handlers are running, the request dispatch + thread itself will block until one exits, making new requests + queue up in the socket over which they arrive, eventually + filling up its buffers if no threads exit, in turn making the + parent handler either block or receive *EAGAIN* errors. Also, + if 'MAX-THREADS' is specified, 'TIMEOUT' may also be + specified, to tell the dispatcher thread to never block more + than so many seconds for a handler thread to exit. If it is + forced to wait longer than 'TIMEOUT' seconds, it will assume + the whole process is somehow foobar and will *abort*(3). + +*rplex*[*:max=*'MAX-THREADS']:: + + The *rplex* handler starts a new thread for every incoming + request, but unlike the *free* handler, only the WSGI handler + function runs in that thread. Whenever any such thread, then, + returns its response iterator, all such iterators will be + passed to a single independent thread which sends their + contents to the clients, multiplexing between them whenever + their respective clients are ready to receive data. Like the + *free* handler, a 'MAX-THREADS' argument may be given to + specify how many handler threads are allowed to run at the + same time. The main advantage, compared to the *free* handler, + is that the *rplex* handler allows an arbitrary number of + response iterators to run simultaneously without tying up + handler threads, therefore not counting towards 'MAX-THREADS', + which may be necessary for applications handling large + files. However, it must be noted that no response iterators in + the application may block on returning data, since that would + also block all other running responses. Also, the *rplex* + handler does not support the `write` function returned by + `start_request`, according to the WSGI specification. + +*single*:: + + The *single* handler starts no threads at all, running all + received requests directly in the main dispatch thread. It is + probably not good for much except as the simplest possible + example of a request handling model. + EXAMPLES -------- @@ -99,6 +166,7 @@ child wsgidir exec ashd-wsgi3 ashd.wsgidir match filename *.wsgi + xset python-handler chain handler wsgidir -------- diff --git a/python3/doc/scgi-wsgi3.doc b/python3/doc/scgi-wsgi3.doc index df91477..d516859 100644 --- a/python3/doc/scgi-wsgi3.doc +++ b/python3/doc/scgi-wsgi3.doc @@ -7,7 +7,7 @@ scgi-wsgi3 - WSGI adapter for SCGI SYNOPSIS -------- -*scgi-wsgi3* [*-hA*] [*-p* 'MODPATH'] [*-T* \[HOST:]'PORT'] 'HANDLER-MODULE' ['ARGS'...] +*scgi-wsgi3* [*-hAL*] [*-m* 'PDM-SPEC'] [*-p* 'MODPATH'] [*-t* 'HANDLING-MODEL'] [*-T* \[HOST:]'PORT'] 'HANDLER-MODULE' ['ARGS'...] DESCRIPTION ----------- @@ -41,11 +41,22 @@ OPTIONS the WSGI application object. See the PROTOCOL section of *ashd-wsgi*(1) for details. +*-L*:: + By default, *scgi-wsgi3* sets up the Python logging with a + logging format and for logging to standard error. The *-L* + option suppresses that behavior, so that any handler module + may set up logging itself. + *-p* 'MODPATH':: Prepend 'MODPATH' to Python's `sys.path`; can be given multiple times. +*-t* 'HANDLING-MODEL':: + + Specify the way *scgi-wsgi3* handles requests. See the REQUEST + HANDLING section of *ashd-wsgi3*(1) for details. + *-T* \[HOST:]'PORT':: Instead of using a listening socket passed on standard input @@ -53,6 +64,12 @@ OPTIONS address listening for connections on 'PORT' instead. If 'HOST' is not given, `localhost` is used by default. +*-m* 'PDM-SPEC':: + + If the PDM library is installed on the system, create a + listening socket for connecting PDM clients according to + 'PDM-SPEC'. + AUTHOR ------ Fredrik Tolf diff --git a/python3/htp.c b/python3/htp.c index ec4ebab..3a091d4 100644 --- a/python3/htp.c +++ b/python3/htp.c @@ -32,18 +32,25 @@ static PyObject *p_recvfd(PyObject *self, PyObject *args) fd = 0; if(!PyArg_ParseTuple(args, "|i", &fd)) return(NULL); - Py_BEGIN_ALLOW_THREADS; - ret = recvfd(fd, &data, &dlen); - Py_END_ALLOW_THREADS; - if(ret < 0) { - if(errno == 0) - return(Py_BuildValue("OO", Py_None, Py_None)); - PyErr_SetFromErrno(PyExc_OSError); - return(NULL); + while(1) { + Py_BEGIN_ALLOW_THREADS; + ret = recvfd(fd, &data, &dlen); + Py_END_ALLOW_THREADS; + if(ret < 0) { + if(errno == 0) + return(Py_BuildValue("OO", Py_None, Py_None)); + if(errno == EINTR) { + if(PyErr_CheckSignals()) + return(NULL); + continue; + } + PyErr_SetFromErrno(PyExc_OSError); + return(NULL); + } + ro = Py_BuildValue("Ni", PyBytes_FromStringAndSize(data, dlen), ret); + free(data); + return(ro); } - ro = Py_BuildValue("Ni", PyBytes_FromStringAndSize(data, dlen), ret); - free(data); - return(ro); } static PyObject *p_sendfd(PyObject *self, PyObject *args) @@ -53,15 +60,22 @@ static PyObject *p_sendfd(PyObject *self, PyObject *args) if(!PyArg_ParseTuple(args, "iiy*", &sock, &fd, &data)) return(NULL); - Py_BEGIN_ALLOW_THREADS; - ret = sendfd(sock, fd, data.buf, data.len); - Py_END_ALLOW_THREADS; - PyBuffer_Release(&data); - if(ret < 0) { - PyErr_SetFromErrno(PyExc_OSError); - return(NULL); + while(1) { + Py_BEGIN_ALLOW_THREADS; + ret = sendfd(sock, fd, data.buf, data.len); + Py_END_ALLOW_THREADS; + PyBuffer_Release(&data); + if(ret < 0) { + if(errno == EINTR) { + if(PyErr_CheckSignals()) + return(NULL); + continue; + } + PyErr_SetFromErrno(PyExc_OSError); + return(NULL); + } + Py_RETURN_NONE; } - Py_RETURN_NONE; } static PyMethodDef methods[] = { diff --git a/python3/scgi-wsgi3 b/python3/scgi-wsgi3 index c66f6e3..a8fccf0 100755 --- a/python3/scgi-wsgi3 +++ b/python3/scgi-wsgi3 @@ -1,16 +1,21 @@ #!/usr/bin/python3 -import sys, os, getopt, logging +import sys, os, getopt, logging, collections import socket -import ashd.scgi +import ashd.scgi, ashd.serve +try: + import pdm.srv +except: + pdm = None def usage(out): - out.write("usage: scgi-wsgi3 [-hAL] [-p MODPATH] [-T [HOST:]PORT] HANDLER-MODULE [ARGS...]\n") + out.write("usage: scgi-wsgi3 [-hAL] [-m PDM-SPEC] [-p MODPATH] [-t REQUEST-HANDLER[:PAR[=VAL](,PAR[=VAL])...]] [-T [HOST:]PORT] HANDLER-MODULE [ARGS...]\n") sk = None +hspec = "free", {} modwsgi_compat = False setlog = True -opts, args = getopt.getopt(sys.argv[1:], "+hALp:T:") +opts, args = getopt.getopt(sys.argv[1:], "+hALp:t:T:m:") for o, a in opts: if o == "-h": usage(sys.stdout) @@ -33,6 +38,11 @@ for o, a in opts: sk.listen(32) elif o == "-A": modwsgi_compat = True + elif o == "-m": + if pdm is not None: + pdm.srv.listen(a) + elif o == "-t": + hspec = ashd.serve.parsehspec(a) if len(args) < 1: usage(sys.stderr) sys.exit(1) @@ -61,4 +71,83 @@ else: sys.exit(1) handler = handlermod.application -ashd.scgi.servescgi(sk, ashd.scgi.wrapwsgi(handler)) +def mkenv(head, sk): + try: + env = ashd.scgi.decodehead(head, "utf-8") + env["wsgi.uri_encoding"] = "utf-8" + except UnicodeError: + env = ashd.scgi.decodehead(head, "latin-1") + env["wsgi.uri_encoding"] = "latin-1" + env["wsgi.version"] = 1, 0 + if "HTTP_X_ASH_PROTOCOL" in env: + env["wsgi.url_scheme"] = env["HTTP_X_ASH_PROTOCOL"] + elif "HTTPS" in env: + env["wsgi.url_scheme"] = "https" + else: + env["wsgi.url_scheme"] = "http" + env["wsgi.input"] = sk + env["wsgi.errors"] = sys.stderr + env["wsgi.multithread"] = True + env["wsgi.multiprocess"] = False + env["wsgi.run_once"] = False + return env + +def recode(thing): + if isinstance(thing, collections.ByteString): + return thing + else: + return str(thing).encode("latin-1") + +class request(ashd.serve.wsgirequest): + def __init__(self, *, sk, **kw): + super().__init__(**kw) + self.bsk = sk.dup() + self.sk = self.bsk.makefile("rwb") + + def mkenv(self): + return mkenv(ashd.scgi.readhead(self.sk), self.sk) + + def handlewsgi(self, env, startreq): + return handler(env, startreq) + + def fileno(self): + return self.bsk.fileno() + + def writehead(self, status, headers): + w = self.buffer.extend + w(b"Status: " + recode(status) + b"\n") + for nm, val in headers: + w(recode(nm) + b": " + recode(val) + b"\n") + w(b"\n") + + def flush(self): + try: + ret = self.bsk.send(self.buffer, socket.MSG_DONTWAIT) + self.buffer[:ret] = b"" + except IOError: + raise ashd.serve.closed() + + def close(self): + self.sk.close() + self.bsk.close() + +if hspec[0] not in ashd.serve.names: + sys.stderr.write("scgi-wsgi3: no such request handler: %s\n" % hspec[0]) + sys.exit(1) +hclass = ashd.serve.names[hspec[0]] +try: + hargs = hclass.parseargs(**hspec[1]) +except ValueError as exc: + sys.stderr.write("scgi-wsgi3: %s\n" % exc) + sys.exit(1) + +reqhandler = hclass(**hargs) +try: + while True: + nsk, addr = sk.accept() + try: + reqhandler.handle(request(sk=nsk, handler=reqhandler)) + finally: + nsk.close() +finally: + reqhandler.close() diff --git a/python3/setup.py b/python3/setup.py index 79c30bb..6c6e16c 100755 --- a/python3/setup.py +++ b/python3/setup.py @@ -6,7 +6,7 @@ htlib = Extension("ashd.htlib", ["htp.c"], libraries = ["ht"]) setup(name = "ashd-py3", - version = "0.5", + version = "0.6", description = "Python module for handling ashd requests", author = "Fredrik Tolf", author_email = "fredrik@dolda2000.com", diff --git a/src/.gitignore b/src/.gitignore index 582f3e7..1ecc66e 100644 --- a/src/.gitignore +++ b/src/.gitignore @@ -12,3 +12,4 @@ /errlogger /psendfile /httimed +/httrcall diff --git a/src/Makefile.am b/src/Makefile.am index 8a08744..5422e74 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -2,9 +2,7 @@ SUBDIRS = dirplex bin_PROGRAMS = htparser sendfile callcgi patplex userplex htls \ callscgi accesslog htextauth callfcgi multifscgi \ - errlogger httimed - -noinst_PROGRAMS=psendfile + errlogger httimed psendfile httrcall htparser_SOURCES = htparser.c htparser.h plaintcp.c ssl-gnutls.c diff --git a/src/accesslog.c b/src/accesslog.c index 0c292c8..af43373 100644 --- a/src/accesslog.c +++ b/src/accesslog.c @@ -26,7 +26,9 @@ #include #include #include +#include #include +#include #ifdef HAVE_CONFIG_H #include @@ -35,19 +37,32 @@ #include #include #include +#include +#include +#include #define DEFFORMAT "%{%Y-%m-%d %H:%M:%S}t %m %u %A \"%G\"" -static int ch; +static struct logdata { + struct hthead *req, *resp; + struct timeval start, end; + off_t bytesin, bytesout; +} defdata = { + .bytesin = -1, + .bytesout = -1, +}; + +static int ch, filter; static char *outname = NULL; static FILE *out; static int flush = 1, locklog = 1; static char *format; -static struct timeval now; static volatile int reopen = 0; -static void qputs(char *s, FILE *o) +static void qputs(char *sp, FILE *o) { + unsigned char *s = (unsigned char *)sp; + for(; *s; s++) { if(*s == '\"') { fputs("\\\"", o); @@ -58,14 +73,14 @@ static void qputs(char *s, FILE *o) } else if(*s == '\t') { fputs("\\t", o); } else if((*s < 32) || (*s >= 128)) { - fprintf(o, "\\x%02x", (int)(unsigned char)*s); + fprintf(o, "\\x%02x", (int)*s); } else { fputc(*s, o); } } } -static void logitem(struct hthead *req, char o, char *d) +static void logitem(struct logdata *data, char o, char *d) { char *h, *p; char buf[1024]; @@ -75,61 +90,86 @@ static void logitem(struct hthead *req, char o, char *d) putc('%', out); break; case 'h': - if((h = getheader(req, d)) == NULL) { + if((h = getheader(data->req, d)) == NULL) { putc('-', out); } else { qputs(h, out); } break; case 'u': - qputs(req->url, out); + qputs(data->req->url, out); break; case 'U': - strcpy(buf, req->url); + strncpy(buf, data->req->url, sizeof(buf)); + buf[sizeof(buf) - 1] = 0; if((p = strchr(buf, '?')) != NULL) *p = 0; qputs(buf, out); break; case 'm': - qputs(req->method, out); + qputs(data->req->method, out); break; case 'r': - qputs(req->rest, out); + qputs(data->req->rest, out); break; case 'v': - qputs(req->ver, out); + qputs(data->req->ver, out); break; case 't': if(!*d) d = "%a, %d %b %Y %H:%M:%S %z"; - strftime(buf, sizeof(buf), d, localtime(&now.tv_sec)); + strftime(buf, sizeof(buf), d, localtime(&data->start.tv_sec)); qputs(buf, out); break; case 'T': if(!*d) d = "%a, %d %b %Y %H:%M:%S %z"; - strftime(buf, sizeof(buf), d, gmtime(&now.tv_sec)); + strftime(buf, sizeof(buf), d, gmtime(&data->start.tv_sec)); qputs(buf, out); break; case 's': - fprintf(out, "%06i", (int)now.tv_usec); + fprintf(out, "%06i", (int)data->start.tv_usec); + break; + case 'c': + if(!data->resp) + putc('-', out); + else + fprintf(out, "%i", data->resp->code); + break; + case 'i': + if(data->bytesin < 0) + putc('-', out); + else + fprintf(out, "%ji", (intmax_t)data->bytesin); + break; + case 'o': + if(data->bytesout < 0) + putc('-', out); + else + fprintf(out, "%ji", (intmax_t)data->bytesout); + break; + case 'd': + if((data->end.tv_sec == 0) && (data->end.tv_usec == 0)) + fputc('-', out); + else + fprintf(out, "%.6f", (data->end.tv_sec - data->start.tv_sec) + ((data->end.tv_usec - data->start.tv_usec) / 1000000.0)); break; case 'A': - logitem(req, 'h', "X-Ash-Address"); + logitem(data, 'h', "X-Ash-Address"); break; case 'H': - logitem(req, 'h', "Host"); + logitem(data, 'h', "Host"); break; case 'R': - logitem(req, 'h', "Referer"); + logitem(data, 'h', "Referer"); break; case 'G': - logitem(req, 'h', "User-Agent"); + logitem(data, 'h', "User-Agent"); break; } } -static void logreq(struct hthead *req) +static void logreq(struct logdata *data) { char *p, *p2; char d[strlen(format)]; @@ -152,7 +192,7 @@ static void logreq(struct hthead *req) o = *p++; if(o == 0) break; - logitem(req, o, d); + logitem(data, o, d); } else { fputc(*p++, out); } @@ -164,18 +204,99 @@ static void logreq(struct hthead *req) static void serve(struct hthead *req, int fd) { - gettimeofday(&now, NULL); + struct logdata data; + + data = defdata; + data.req = req; + gettimeofday(&data.start, NULL); if(sendreq(ch, req, fd)) { flog(LOG_ERR, "accesslog: could not pass request to child: %s", strerror(errno)); exit(1); } - logreq(req); + logreq(&data); +} + +static int passdata(struct bufio *in, struct bufio *out, off_t *passed) +{ + ssize_t read; + off_t total; + + total = 0; + while(!bioeof(in)) { + if((read = biordata(in)) > 0) { + if((read = biowritesome(out, in->rbuf.b + in->rh, read)) < 0) + return(-1); + in->rh += read; + total += read; + } + if(biorspace(in) && (biofillsome(in) < 0)) + return(-1); + } + if(passed) + *passed = total; + return(0); +} + +static void filterreq(struct muth *mt, va_list args) +{ + vavar(struct hthead *, req); + vavar(int, fd); + int pfds[2]; + struct hthead *resp; + struct bufio *cl, *hd; + struct stdiofd *cli, *hdi; + struct logdata data; + + hd = NULL; + resp = NULL; + data = defdata; + data.req = req; + gettimeofday(&data.start, NULL); + cl = mtbioopen(fd, 1, 600, "r+", &cli); + if(socketpair(PF_UNIX, SOCK_STREAM, 0, pfds)) + goto out; + hd = mtbioopen(pfds[1], 1, 600, "r+", &hdi); + if(sendreq(ch, req, pfds[0])) { + close(pfds[0]); + goto out; + } + close(pfds[0]); + + if(passdata(cl, hd, &data.bytesin)) + goto out; + if(bioflush(hd)) + goto out; + shutdown(pfds[1], SHUT_WR); + if((resp = parseresponseb(hd)) == NULL) + goto out; + cli->sendrights = hdi->rights; + hdi->rights = -1; + data.resp = resp; + writerespb(cl, resp); + bioprintf(cl, "\r\n"); + if(passdata(hd, cl, &data.bytesout)) + goto out; + gettimeofday(&data.end, NULL); + +out: + logreq(&data); + + freehthead(req); + if(resp != NULL) + freehthead(resp); + bioclose(cl); + if(hd != NULL) + bioclose(hd); } static void sighandler(int sig) { - if(sig == SIGHUP) - reopen = 1; + if(sig == SIGHUP) { + if(filter) + exitioloop(2); + else + reopen = 1; + } } static int lockfile(FILE *file) @@ -271,6 +392,85 @@ static void reopenlog(void) out = new; } +static void listenloop(struct muth *mt, va_list args) +{ + vavar(int, lfd); + int fd; + struct hthead *req; + + while(1) { + block(lfd, EV_READ, 0); + if((fd = recvreq(lfd, &req)) < 0) { + if(errno != 0) + flog(LOG_ERR, "accesslog: error in recvreq: %s", strerror(errno)); + exit(1); + } + mustart(filterreq, req, fd); + } +} + +static void chwatch(struct muth *mt, va_list args) +{ + vavar(int, cfd); + + block(cfd, EV_READ, 0); + exitioloop(1); +} + +static void floop(void) +{ + mustart(listenloop, 0); + mustart(chwatch, ch); + while(1) { + switch(ioloop()) { + case 0: + case 1: + return; + case 2: + reopenlog(); + break; + } + } +} + +static void sloop(void) +{ + int fd, ret; + struct hthead *req; + struct pollfd pfd[2]; + + while(1) { + if(reopen) { + reopenlog(); + reopen = 0; + } + memset(pfd, 0, sizeof(pfd)); + pfd[0].fd = 0; + pfd[0].events = POLLIN; + pfd[1].fd = ch; + pfd[1].events = POLLHUP; + if((ret = poll(pfd, 2, -1)) < 0) { + if(errno != EINTR) { + flog(LOG_ERR, "accesslog: error in poll: %s", strerror(errno)); + exit(1); + } + } + if(pfd[0].revents) { + if((fd = recvreq(0, &req)) < 0) { + if(errno == 0) + return; + flog(LOG_ERR, "accesslog: error in recvreq: %s", strerror(errno)); + exit(1); + } + serve(req, fd); + freehthead(req); + close(fd); + } + if(pfd[1].revents & POLLHUP) + return; + } +} + static void usage(FILE *out) { fprintf(out, "usage: accesslog [-hFaL] [-f FORMAT] [-p PIDFILE] OUTFILE CHILD [ARGS...]\n"); @@ -279,15 +479,12 @@ static void usage(FILE *out) int main(int argc, char **argv) { - int c, ret; - struct hthead *req; - int fd; - struct pollfd pfd[2]; + int c; char *pidfile; FILE *pidout; pidfile = NULL; - while((c = getopt(argc, argv, "+hFaLf:p:P:")) >= 0) { + while((c = getopt(argc, argv, "+hFaeLf:p:P:")) >= 0) { switch(c) { case 'h': usage(stdout); @@ -298,6 +495,9 @@ int main(int argc, char **argv) case 'L': locklog = 0; break; + case 'e': + filter = 1; + break; case 'f': format = optarg; break; @@ -308,7 +508,7 @@ int main(int argc, char **argv) pidfile = optarg; break; case 'a': - format = "%A - - [%{%d/%b/%Y:%H:%M:%S %z}t] \"%m %u %v\" - - \"%R\" \"%G\""; + format = "%A - - [%{%d/%b/%Y:%H:%M:%S %z}t] \"%m %u %v\" %c %o \"%R\" \"%G\""; break; default: usage(stderr); @@ -365,36 +565,11 @@ int main(int argc, char **argv) fprintf(pidout, "%i\n", (int)getpid()); fclose(pidout); } - while(1) { - if(reopen) { - reopenlog(); - reopen = 0; - } - memset(pfd, 0, sizeof(pfd)); - pfd[0].fd = 0; - pfd[0].events = POLLIN; - pfd[1].fd = ch; - pfd[1].events = POLLHUP; - if((ret = poll(pfd, 2, -1)) < 0) { - if(errno != EINTR) { - flog(LOG_ERR, "accesslog: error in poll: %s", strerror(errno)); - exit(1); - } - } - if(pfd[0].revents) { - if((fd = recvreq(0, &req)) < 0) { - if(errno == 0) - break; - flog(LOG_ERR, "accesslog: error in recvreq: %s", strerror(errno)); - exit(1); - } - serve(req, fd); - freehthead(req); - close(fd); - } - if(pfd[1].revents & POLLHUP) - break; - } + if(filter) + floop(); + else + sloop(); + fclose(out); if(pidfile != NULL) unlink(pidfile); return(0); diff --git a/src/callcgi.c b/src/callcgi.c index 11fb213..67a6d14 100644 --- a/src/callcgi.c +++ b/src/callcgi.c @@ -62,7 +62,8 @@ static int passdata(FILE *in, FILE *out) return(1); } if(fwrite(buf, 1, ret, out) != ret) { - flog(LOG_ERR, "callcgi: could not write output: %s", strerror(errno)); + if(errno != EPIPE) + flog(LOG_ERR, "callcgi: could not write output: %s", strerror(errno)); return(1); } } @@ -84,12 +85,13 @@ static char *absolutify(char *file) return(sstrdup(file)); } -static pid_t forkchild(int inpath, char *prog, char *file, char *method, char *url, char *rest, int *infd, int *outfd) +static pid_t forkchild(int inpath, char **prog, char *file, char *method, char *url, char *rest, int *infd, int *outfd) { char *qp, **env, *name; int inp[2], outp[2]; pid_t pid; char *pi; + int (*execfun)(const char *, char *const []); pipe(inp); pipe(outp); @@ -169,11 +171,10 @@ static pid_t forkchild(int inpath, char *prog, char *file, char *method, char *u * This is (understandably) missing from the CGI * specification, but PHP seems to require it. */ - putenv(sprintf2("SCRIPT_FILENAME=%s", absolutify(file))); - if(inpath) - execlp(prog, prog, file, NULL); - else - execl(prog, prog, file, NULL); + execfun = inpath?execvp:execv; + if(file != NULL) + putenv(sprintf2("SCRIPT_FILENAME=%s", absolutify(file))); + execfun(prog[0], prog); exit(127); } close(inp[0]); @@ -335,14 +336,15 @@ static void sendheaders(char **headers, FILE *out) static void usage(void) { - flog(LOG_ERR, "usage: callcgi [-c] [-p PROGRAM] METHOD URL REST"); + flog(LOG_ERR, "usage: callcgi [-c] [-p PROGRAM] [-P PROGRAM ARGS... ;] METHOD URL REST"); } int main(int argc, char **argv, char **envp) { int c; - char *file, *prog, *sp; - int inpath, cd; + char *file, *sp; + struct charvbuf prog; + int inpath, addfile, cd; int infd, outfd; FILE *in, *out; char **headers; @@ -352,17 +354,39 @@ int main(int argc, char **argv, char **envp) environ = envp; signal(SIGPIPE, SIG_IGN); - prog = NULL; + bufinit(prog); inpath = 0; + addfile = 1; cd = 0; - while((c = getopt(argc, argv, "cp:")) >= 0) { + while((c = getopt(argc, argv, "cp:P:")) >= 0) { switch(c) { case 'c': cd = 1; break; case 'p': - prog = optarg; + bufadd(prog, optarg); + inpath = 1; + break; + case 'P': + prog.d = 0; + bufadd(prog, optarg); + while(1) { + if(optind >= argc) { + flog(LOG_ERR, "callcgi: unterminated argument list for -P"); + exit(1); + } + if(!strcmp(argv[optind], ";")) { + optind++; + break; + } + bufadd(prog, argv[optind++]); + } + if(prog.d == 0) { + flog(LOG_ERR, "callcgi: -P option needs at least a program name"); + exit(1); + } inpath = 1; + addfile = 0; break; default: usage(); @@ -374,7 +398,7 @@ int main(int argc, char **argv, char **envp) usage(); exit(1); } - if((file = getenv("REQ_X_ASH_FILE")) == NULL) { + if(((file = getenv("REQ_X_ASH_FILE")) == NULL) && (prog.d == 0)) { flog(LOG_ERR, "callcgi: needs to be called with the X-Ash-File header"); exit(1); } @@ -393,9 +417,12 @@ int main(int argc, char **argv, char **envp) } } - if(prog == NULL) - prog = file; - child = forkchild(inpath, prog, file, argv[optind], argv[optind + 1], argv[optind + 2], &infd, &outfd); + if(prog.d == 0) + bufadd(prog, file); + if(addfile && (file != NULL)) + bufadd(prog, file); + bufadd(prog, NULL); + child = forkchild(inpath, prog.b, file, argv[optind], argv[optind + 1], argv[optind + 2], &infd, &outfd); in = fdopen(infd, "w"); passdata(stdin, in); /* Ignore errors, perhaps? */ fclose(in); @@ -411,7 +438,7 @@ int main(int argc, char **argv, char **envp) kill(child, SIGINT); if(waitpid(child, &estat, 0) == child) { if(WCOREDUMP(estat)) - flog(LOG_WARNING, "CGI handler `%s' dumped core", prog); + flog(LOG_WARNING, "CGI handler `%s' dumped core", prog.b[0]); if(WIFEXITED(estat) && !WEXITSTATUS(estat)) return(0); else diff --git a/src/callfcgi.c b/src/callfcgi.c index 3320f5d..8d33eef 100644 --- a/src/callfcgi.c +++ b/src/callfcgi.c @@ -37,7 +37,7 @@ #include #include #include -#include +#include #include #ifdef HAVE_CONFIG_H @@ -45,6 +45,7 @@ #endif #include #include +#include #include #include #include @@ -123,9 +124,16 @@ static char *mkanonid(void) return(tmpl); } +static void setupchild(void) +{ + /* PHP appears to not expect to inherit SIGCHLD set to SIG_IGN, so + * reset it for it. */ + signal(SIGCHLD, SIG_DFL); +} + static void startlisten(void) { - int i, fd; + int fd; struct addrinfo *ai, *cai; char *unpath; struct sockaddr_un unm; @@ -203,9 +211,9 @@ static void startlisten(void) exit(1); } if(child == 0) { + setupchild(); dup2(fd, 0); - for(i = 3; i < FD_SETSIZE; i++) - close(i); + close(fd); execvp(*progspec, progspec); flog(LOG_ERR, "callfcgi: %s: %s", *progspec, strerror(errno)); _exit(127); @@ -215,15 +223,14 @@ static void startlisten(void) static void startnolisten(void) { - int i, fd; + int fd; if((child = fork()) < 0) { flog(LOG_ERR, "could not fork: %s", strerror(errno)); exit(1); } if(child == 0) { - for(i = 3; i < FD_SETSIZE; i++) - close(i); + setupchild(); if((fd = open("/dev/null", O_RDONLY)) < 0) { flog(LOG_ERR, "/dev/null: %s", strerror(errno)); _exit(127); @@ -466,8 +473,8 @@ static void mkcgienv(struct hthead *req, struct charbuf *dst) pi = sprintf2("/%s", tmp = pi); free(tmp); } - bufaddenv(dst, "PATH_INFO", pi); - bufaddenv(dst, "SCRIPT_NAME", url); + bufaddenv(dst, "PATH_INFO", "%s", pi); + bufaddenv(dst, "SCRIPT_NAME", "%s", url); bufaddenv(dst, "QUERY_STRING", "%s", qp?qp:""); free(pi); free(url); @@ -509,46 +516,6 @@ static void mkcgienv(struct hthead *req, struct charbuf *dst) } } -static char *defstatus(int code) -{ - if(code == 200) - return("OK"); - else if(code == 201) - return("Created"); - else if(code == 202) - return("Accepted"); - else if(code == 204) - return("No Content"); - else if(code == 300) - return("Multiple Choices"); - else if(code == 301) - return("Moved Permanently"); - else if(code == 302) - return("Found"); - else if(code == 303) - return("See Other"); - else if(code == 304) - return("Not Modified"); - else if(code == 307) - return("Moved Temporarily"); - else if(code == 400) - return("Bad Request"); - else if(code == 401) - return("Unauthorized"); - else if(code == 403) - return("Forbidden"); - else if(code == 404) - return("Not Found"); - else if(code == 500) - return("Internal Server Error"); - else if(code == 501) - return("Not Implemented"); - else if(code == 503) - return("Service Unavailable"); - else - return("Unknown status"); -} - static struct hthead *parseresp(FILE *in) { struct hthead *resp; @@ -567,7 +534,7 @@ static struct hthead *parseresp(FILE *in) resp->msg = sstrdup(p); } else { resp->code = atoi(st); - resp->msg = sstrdup(defstatus(resp->code)); + resp->msg = sstrdup(httpdefstatus(resp->code)); } headrmheader(resp, "Status"); } else if(getheader(resp, "Location")) { @@ -608,35 +575,26 @@ static int sendrec(FILE *out, int type, int rid, char *data, size_t dlen) return(0); } -#define fgetc2(f) ({int __c__ = fgetc(f); if(__c__ == EOF) return(-1); __c__;}) - static int recvrec(FILE *in, int *type, int *rid, char **data, size_t *dlen) { - int b1, b2, pl; + unsigned char header[8]; + int tl; - if(fgetc2(in) != 1) + if(fread(header, 1, 8, in) != 8) + return(-1); + if(header[0] != 1) return(-1); - *type = fgetc2(in); - b1 = fgetc2(in); - b2 = fgetc2(in); - *rid = (b1 << 8) | b2; - b1 = fgetc2(in); - b2 = fgetc2(in); - *dlen = (b1 << 8) | b2; - pl = fgetc2(in); - if(fgetc2(in) != 0) + *type = header[1]; + *rid = (header[2] << 8) | header[3]; + *dlen = (header[4] << 8) | header[5]; + tl = *dlen + header[6]; + if(header[7] != 0) return(-1); - *data = smalloc(max(*dlen, 1)); - if(fread(*data, 1, *dlen, in) != *dlen) { + *data = smalloc(max(tl, 1)); + if(fread(*data, 1, tl, in) != tl) { free(*data); return(-1); } - for(; pl > 0; pl--) { - if(fgetc(in) == EOF) { - free(*data); - return(-1); - } - } return(0); } @@ -647,15 +605,6 @@ static int begreq(FILE *out, int rid) return(sendrec(out, FCGI_BEGIN_REQUEST, rid, rec, 8)); } -static void mtiopipe(FILE **read, FILE **write) -{ - int fds[2]; - - pipe(fds); - *read = mtstdopen(fds[0], 0, 600, "r"); - *write = mtstdopen(fds[1], 0, 600, "w"); -} - static void outplex(struct muth *muth, va_list args) { vavar(FILE *, sk); @@ -741,12 +690,12 @@ static void serve(struct muth *muth, va_list args) char buf[8192]; sfd = reconn(); - is = mtstdopen(fd, 1, 60, "r+"); - os = mtstdopen(sfd, 1, 600, "r+"); + is = mtstdopen(fd, 1, 60, "r+", NULL); + os = mtstdopen(sfd, 1, 600, "r+", NULL); outi = NULL; mtiopipe(&outi, &outo); mtiopipe(&erri, &erro); - mustart(outplex, mtstdopen(dup(sfd), 1, 600, "r+"), outo, FCGI_STDOUT, erro, FCGI_STDERR, NULL); + mustart(outplex, mtstdopen(dup(sfd), 1, 600, "r+", NULL), outo, FCGI_STDOUT, erro, FCGI_STDERR, NULL); mustart(errhandler, erri); if(begreq(os, 1)) @@ -815,7 +764,6 @@ static void sigign(int sig) static void sigexit(int sig) { shutdown(0, SHUT_RDWR); - exit(0); } static void usage(FILE *out) @@ -859,7 +807,7 @@ int main(int argc, char **argv) signal(SIGINT, sigexit); signal(SIGTERM, sigexit); mustart(listenloop, 0); - atexit(killcuraddr); ioloop(); + killcuraddr(); return(0); } diff --git a/src/callscgi.c b/src/callscgi.c index e388ab2..2483c49 100644 --- a/src/callscgi.c +++ b/src/callscgi.c @@ -32,7 +32,7 @@ #include #include #include -#include +#include #include #ifdef HAVE_CONFIG_H @@ -40,6 +40,7 @@ #endif #include #include +#include #include #include #include @@ -110,9 +111,16 @@ static char *mkanonid(void) return(tmpl); } +static void setupchild(void) +{ + /* PHP appears to not expect to inherit SIGCHLD set to SIG_IGN, so + * reset it for it. */ + signal(SIGCHLD, SIG_DFL); +} + static void startlisten(void) { - int i, fd; + int fd; struct addrinfo *ai, *cai; char *unpath; struct sockaddr_un unm; @@ -190,9 +198,9 @@ static void startlisten(void) exit(1); } if(child == 0) { + setupchild(); dup2(fd, 0); - for(i = 3; i < FD_SETSIZE; i++) - close(i); + close(fd); execvp(*progspec, progspec); flog(LOG_ERR, "callscgi: %s: %s", *progspec, strerror(errno)); _exit(127); @@ -202,15 +210,14 @@ static void startlisten(void) static void startnolisten(void) { - int i, fd; + int fd; if((child = fork()) < 0) { flog(LOG_ERR, "could not fork: %s", strerror(errno)); exit(1); } if(child == 0) { - for(i = 3; i < FD_SETSIZE; i++) - close(i); + setupchild(); if((fd = open("/dev/null", O_RDONLY)) < 0) { flog(LOG_ERR, "/dev/null: %s", strerror(errno)); _exit(127); @@ -429,8 +436,8 @@ static void mkcgienv(struct hthead *req, struct charbuf *dst) pi = sprintf2("/%s", tmp = pi); free(tmp); } - bufaddenv(dst, "PATH_INFO", pi); - bufaddenv(dst, "SCRIPT_NAME", url); + bufaddenv(dst, "PATH_INFO", "%s", pi); + bufaddenv(dst, "SCRIPT_NAME", "%s", url); bufaddenv(dst, "QUERY_STRING", "%s", qp?qp:""); free(pi); free(url); @@ -473,46 +480,6 @@ static void mkcgienv(struct hthead *req, struct charbuf *dst) } } -static char *defstatus(int code) -{ - if(code == 200) - return("OK"); - else if(code == 201) - return("Created"); - else if(code == 202) - return("Accepted"); - else if(code == 204) - return("No Content"); - else if(code == 300) - return("Multiple Choices"); - else if(code == 301) - return("Moved Permanently"); - else if(code == 302) - return("Found"); - else if(code == 303) - return("See Other"); - else if(code == 304) - return("Not Modified"); - else if(code == 307) - return("Moved Temporarily"); - else if(code == 400) - return("Bad Request"); - else if(code == 401) - return("Unauthorized"); - else if(code == 403) - return("Forbidden"); - else if(code == 404) - return("Not Found"); - else if(code == 500) - return("Internal Server Error"); - else if(code == 501) - return("Not Implemented"); - else if(code == 503) - return("Service Unavailable"); - else - return("Unknown status"); -} - static struct hthead *parseresp(FILE *in) { struct hthead *resp; @@ -531,7 +498,7 @@ static struct hthead *parseresp(FILE *in) resp->msg = sstrdup(p); } else { resp->code = atoi(st); - resp->msg = sstrdup(defstatus(resp->code)); + resp->msg = sstrdup(httpdefstatus(resp->code)); } headrmheader(resp, "Status"); } else if(getheader(resp, "Location")) { @@ -554,8 +521,8 @@ static void serve(struct muth *muth, va_list args) struct hthead *resp; sfd = reconn(); - is = mtstdopen(fd, 1, 60, "r+"); - os = mtstdopen(sfd, 1, 600, "r+"); + is = mtstdopen(fd, 1, 60, "r+", NULL); + os = mtstdopen(sfd, 1, 600, "r+", NULL); bufinit(head); mkcgienv(req, &head); @@ -604,7 +571,6 @@ static void sigign(int sig) static void sigexit(int sig) { shutdown(0, SHUT_RDWR); - exit(0); } static void usage(FILE *out) @@ -648,7 +614,7 @@ int main(int argc, char **argv) signal(SIGINT, sigexit); signal(SIGTERM, sigexit); mustart(listenloop, 0); - atexit(killcuraddr); ioloop(); + killcuraddr(); return(0); } diff --git a/src/dirplex/conf.c b/src/dirplex/conf.c index dec84b5..9ae4ca7 100644 --- a/src/dirplex/conf.c +++ b/src/dirplex/conf.c @@ -86,6 +86,8 @@ static void freeconfig(struct config *cf) freeca(cf->index); if(cf->capture != NULL) free(cf->capture); + if(cf->reparse != NULL) + free(cf->reparse); free(cf); } @@ -150,6 +152,10 @@ static struct pattern *parsepattern(struct cfstate *s) if((s->argc > 1) && !strcmp(s->argv[1], "directory")) pat->type = PT_DIR; + else if((s->argc > 1) && !strcmp(s->argv[1], "notfound")) + pat->type = PT_NOTFOUND; + else + pat->type = PT_FILE; sl = s->lno; while(1) { getcfline(s); @@ -251,9 +257,7 @@ struct config *readconfig(char *file) cf->patterns = pat; } else if(!strcmp(s->argv[0], "index-file")) { freeca(cf->index); - cf->index = NULL; - if(s->argc > 1) - cf->index = cadup(s->argv + 1); + cf->index = cadup(s->argv + 1); } else if(!strcmp(s->argv[0], "capture")) { if(s->argc < 2) { flog(LOG_WARNING, "%s:%i: missing argument to capture declaration", s->file, s->lno); @@ -262,9 +266,20 @@ struct config *readconfig(char *file) if(cf->capture != NULL) free(cf->capture); cf->capture = sstrdup(s->argv[1]); - cf->caproot = 1; - if((s->argc > 2) && strchr(s->argv[2], 'R')) - cf->caproot = 0; + cf->caproot = 0; + if((s->argc > 2) && strchr(s->argv[2], 'D')) + cf->caproot = 1; + } else if(!strcmp(s->argv[0], "reparse")) { + if(s->argc < 2) { + flog(LOG_WARNING, "%s:%i: missing argument to reparse declaration", s->file, s->lno); + continue; + } + if(cf->reparse != NULL) + free(cf->reparse); + cf->reparse = sstrdup(s->argv[1]); + cf->parsecomb = 0; + if((s->argc > 2) && strchr(s->argv[2], 'c')) + cf->parsecomb = 1; } else if(!strcmp(s->argv[0], "eof")) { break; } else { @@ -374,7 +389,7 @@ struct child *findchild(char *file, char *name, struct config **cf) return(NULL); } -struct pattern *findmatch(char *file, int trydefault, int dir) +struct pattern *findmatch(char *file, int trydefault, int type) { int i, o, c; char *bn, *ln; @@ -399,9 +414,7 @@ struct pattern *findmatch(char *file, int trydefault, int dir) ln = file; /* This should only happen in the base directory. */ } for(pat = cfs[c]->patterns; pat != NULL; pat = pat->next) { - if(!dir && (pat->type == PT_DIR)) - continue; - if(dir && (pat->type != PT_DIR)) + if(pat->type != type) continue; for(i = 0; (rule = pat->rules[i]) != NULL; i++) { if(rule->type == PAT_BASENAME) { @@ -432,7 +445,7 @@ struct pattern *findmatch(char *file, int trydefault, int dir) } } if(!trydefault) - return(findmatch(file, 1, dir)); + return(findmatch(file, 1, type)); return(NULL); } diff --git a/src/dirplex/dirplex.c b/src/dirplex/dirplex.c index d687957..73bb9cc 100644 --- a/src/dirplex/dirplex.c +++ b/src/dirplex/dirplex.c @@ -55,6 +55,14 @@ static void chinit(void *idata) } } +static void childerror(struct hthead *req, int fd) +{ + if(errno == EAGAIN) + simpleerror(fd, 500, "Server Error", "The request handler is overloaded."); + else + simpleerror(fd, 500, "Server Error", "The request handler crashed."); +} + static void handle(struct hthead *req, int fd, char *path, struct pattern *pat) { struct child *ch; @@ -91,7 +99,7 @@ static void handle(struct hthead *req, int fd, char *path, struct pattern *pat) } headappheader(req, "X-Ash-File", path); if(childhandle(ch, req, fd, chinit, twd)) - simpleerror(fd, 500, "Server Error", "The request handler crashed."); + childerror(req, fd); } } @@ -99,20 +107,24 @@ static void handle404(struct hthead *req, int fd, char *path) { struct child *ch; struct config *ccf; - char *tmp; + struct pattern *pat; - tmp = sstrdup(path); - ch = findchild(tmp, ".notfound", &ccf); - if(childhandle(ch, req, fd, chinit, ccf?ccf->path:NULL)) - simpleerror(fd, 500, "Server Error", "The request handler crashed."); - free(tmp); + char tmp[strlen(path) + 1]; + strcpy(tmp, path); + if((pat = findmatch(tmp, 0, PT_NOTFOUND)) != NULL) { + handle(req, fd, tmp, pat); + } else { + ch = findchild(tmp, ".notfound", &ccf); + if(childhandle(ch, req, fd, chinit, ccf?ccf->path:NULL)) + childerror(req, fd); + } } static void handlefile(struct hthead *req, int fd, char *path) { struct pattern *pat; - if((pat = findmatch(path, 0, 0)) == NULL) { + if((pat = findmatch(path, 0, PT_FILE)) == NULL) { handle404(req, fd, path); return; } @@ -186,7 +198,7 @@ static void handledir(struct hthead *req, int fd, char *path) break; } } - if((pat = findmatch(cpath, 0, 1)) != NULL) { + if((pat = findmatch(cpath, 0, PT_DIR)) != NULL) { handle(req, fd, cpath, pat); goto out; } @@ -196,18 +208,16 @@ out: free(cpath); } -static int checkpath(struct hthead *req, int fd, char *path, char *rest); +static int checkpath(struct hthead *req, int fd, char *path, char *rest, int final); -static int checkentry(struct hthead *req, int fd, char *path, char *rest, char *el) +static int checkentry(struct hthead *req, int fd, char *path, char *rest, char *el, int final) { struct stat sb; char *newpath; int rv; - if(*el == '.') { - handle404(req, fd, sprintf3("%s/", path)); - return(1); - } + if(*el == '.') + return(0); if(!stat(sprintf3("%s/%s", path, el), &sb)) { if(S_ISDIR(sb.st_mode)) { if(!*rest) { @@ -215,7 +225,7 @@ static int checkentry(struct hthead *req, int fd, char *path, char *rest, char * return(1); } newpath = sprintf2("%s/%s", path, el); - rv = checkpath(req, fd, newpath, rest + 1); + rv = checkpath(req, fd, newpath, rest + 1, final); free(newpath); return(rv); } else if(S_ISREG(sb.st_mode)) { @@ -239,9 +249,11 @@ static int checkentry(struct hthead *req, int fd, char *path, char *rest, char * static int checkdir(struct hthead *req, int fd, char *path, char *rest) { - char *cpath; + char *cpath, *newpath; struct config *cf, *ccf; struct child *ch; + struct stat sb; + int rv; cf = getconfig(path); if((cf->capture != NULL) && (cf->caproot || !cf->path || strcmp(cf->path, "."))) { @@ -257,13 +269,29 @@ static int checkdir(struct hthead *req, int fd, char *path, char *rest) rest++; replrest(req, rest); if(childhandle(ch, req, fd, chinit, ccf?ccf->path:NULL)) - simpleerror(fd, 500, "Server Error", "The request handler crashed."); + childerror(req, fd); return(1); } + if(cf->reparse != NULL) { + newpath = (cf->reparse[0] == '/')?sstrdup(cf->reparse):sprintf2("%s/%s", path, cf->reparse); + rv = stat(newpath, &sb); + if(!rv && S_ISDIR(sb.st_mode)) { + rv = checkpath(req, fd, newpath, rest, !cf->parsecomb); + } else if(!rv && S_ISREG(sb.st_mode)) { + replrest(req, rest); + handlefile(req, fd, newpath); + rv = 1; + } else { + rv = !cf->parsecomb; + } + free(newpath); + if(rv) + return(rv); + } return(0); } -static int checkpath(struct hthead *req, int fd, char *path, char *rest) +static int checkpath(struct hthead *req, int fd, char *path, char *rest, int final) { char *p, *el; int rv; @@ -292,8 +320,7 @@ static int checkpath(struct hthead *req, int fd, char *path, char *rest) goto out; } if(strchr(el, '/') || (!*el && *rest)) { - handle404(req, fd, sprintf3("%s/", path)); - rv = 1; + rv = 0; goto out; } if(!*el) { @@ -302,9 +329,13 @@ static int checkpath(struct hthead *req, int fd, char *path, char *rest) rv = 1; goto out; } - rv = checkentry(req, fd, path, rest, el); + rv = checkentry(req, fd, path, rest, el, final); out: + if(final && !rv) { + handle404(req, fd, sprintf3("%s/", path)); + rv = 1; + } if(el != NULL) free(el); return(rv); @@ -313,8 +344,7 @@ out: static void serve(struct hthead *req, int fd) { now = time(NULL); - if(!checkpath(req, fd, ".", req->rest)) - handle404(req, fd, "."); + checkpath(req, fd, ".", req->rest, 1); } static void chldhandler(int sig) diff --git a/src/dirplex/dirplex.h b/src/dirplex/dirplex.h index 14b5454..ffe12a6 100644 --- a/src/dirplex/dirplex.h +++ b/src/dirplex/dirplex.h @@ -9,6 +9,7 @@ #define PT_FILE 0 #define PT_DIR 1 +#define PT_NOTFOUND 2 struct config { struct config *next, *prev; @@ -17,8 +18,8 @@ struct config { struct child *children; struct pattern *patterns; char **index; - char *capture; - int caproot; + char *capture, *reparse; + int caproot, parsecomb; }; struct rule { @@ -45,7 +46,7 @@ struct config *readconfig(char *file); struct config *getconfig(char *path); struct config **getconfigs(char *file); struct child *findchild(char *file, char *name, struct config **cf); -struct pattern *findmatch(char *file, int trydefault, int dir); +struct pattern *findmatch(char *file, int trydefault, int type); void modheaders(struct hthead *req, struct pattern *pat); extern time_t now; diff --git a/src/htparser.c b/src/htparser.c index 8d0b515..d10121c 100644 --- a/src/htparser.c +++ b/src/htparser.c @@ -35,6 +35,7 @@ #include #include #include +#include #include "htparser.h" @@ -60,7 +61,7 @@ static void trimx(struct hthead *req) } } -static struct hthead *parsereq(FILE *in) +static struct hthead *parsereq(struct bufio *in) { struct hthead *req; struct charbuf method, url, ver; @@ -71,7 +72,7 @@ static struct hthead *parsereq(FILE *in) bufinit(url); bufinit(ver); while(1) { - c = getc(in); + c = biogetc(in); if(c == ' ') { break; } else if((c == EOF) || (c < 32) || (c >= 128)) { @@ -83,7 +84,7 @@ static struct hthead *parsereq(FILE *in) } } while(1) { - c = getc(in); + c = biogetc(in); if(c == ' ') { break; } else if((c == EOF) || (c < 32)) { @@ -95,7 +96,7 @@ static struct hthead *parsereq(FILE *in) } } while(1) { - c = getc(in); + c = biogetc(in); if(c == 10) { break; } else if(c == 13) { @@ -111,7 +112,7 @@ static struct hthead *parsereq(FILE *in) bufadd(url, 0); bufadd(ver, 0); req = mkreq(method.b, url.b, ver.b); - if(parseheaders(req, in)) + if(parseheadersb(req, in)) goto fail; trimx(req); goto out; @@ -128,38 +129,37 @@ out: return(req); } -static off_t passdata(FILE *in, FILE *out, off_t max) +static off_t passdata(struct bufio *in, struct bufio *out, off_t max) { - size_t read; + ssize_t read; off_t total; - char buf[8192]; total = 0; - while(!feof(in) && ((max < 0) || (total < max))) { - read = sizeof(buf); - if(max >= 0) - read = min(max - total, read); - read = fread(buf, 1, read, in); - if(ferror(in)) - return(-1); - if(fwrite(buf, 1, read, out) != read) + while(!bioeof(in) && ((max < 0) || (total < max))) { + if((read = biordata(in)) > 0) { + if(max >= 0) + read = min(max - total, read); + if((read = biowritesome(out, in->rbuf.b + in->rh, read)) < 0) + return(-1); + in->rh += read; + total += read; + } + if(biorspace(in) && ((max < 0) || (biordata(in) < max - total)) && (biofillsome(in) < 0)) return(-1); - total += read; } return(total); } -static int recvchunks(FILE *in, FILE *out) +static int recvchunks(struct bufio *in, struct bufio *out) { - char buf[8192]; - size_t read, chlen; + ssize_t read, chlen; int c, r; while(1) { chlen = 0; r = 0; while(1) { - c = getc(in); + c = biogetc(in); if(c == 10) { if(!r) return(-1); @@ -185,37 +185,43 @@ static int recvchunks(FILE *in, FILE *out) if(chlen == 0) break; while(chlen > 0) { - read = fread(buf, 1, min(sizeof(buf), chlen), in); - if(feof(in) || ferror(in)) - return(-1); - if(fwrite(buf, 1, read, out) != read) + if((read = biordata(in)) > 0) { + if((read = biowritesome(out, in->rbuf.b + in->rh, min(read, chlen))) < 0) + return(-1); + in->rh += read; + chlen -= read; + } + if(biorspace(in) && (biordata(in) < chlen) && (biofillsome(in) <= 0)) return(-1); - chlen -= read; } - if((getc(in) != 13) || (getc(in) != 10)) + if((biogetc(in) != 13) || (biogetc(in) != 10)) return(-1); } /* XXX: Technically, there may be trailers to be read, but that's * just about as likely as chunk extensions. */ - if((getc(in) != 13) || (getc(in) != 10)) + if((biogetc(in) != 13) || (biogetc(in) != 10)) return(-1); return(0); } -static int passchunks(FILE *in, FILE *out) +static int passchunks(struct bufio *in, struct bufio *out) { - char buf[8192]; size_t read; - do { - read = fread(buf, 1, sizeof(buf), in); - if(ferror(in)) - return(-1); - fprintf(out, "%zx\r\n", read); - if(fwrite(buf, 1, read, out) != read) + while(!bioeof(in)) { + if((read = biordata(in)) > 0) { + bioprintf(out, "%zx\r\n", read); + if(biowrite(out, in->rbuf.b + in->rh, read) != read) + return(-1); + in->rh += read; + bioprintf(out, "\r\n"); + if(bioflush(out) < 0) + return(-1); + } + if(biorspace(in) && (biofillsome(in) < 0)) return(-1); - fprintf(out, "\r\n"); - } while(read > 0); + } + bioprintf(out, "0\r\n\r\n"); return(0); } @@ -280,23 +286,89 @@ static int http10keep(struct hthead *req, struct hthead *resp) } } -void serve(FILE *in, struct conn *conn) +static char *connid(void) +{ + static struct charbuf cur; + int i; + char *ret; + + for(i = 0; i < cur.d; i++) { + if((++cur.b[i]) > 'Z') + cur.b[i] = 'A'; + else + goto done; + } + bufadd(cur, 'A'); +done: + ret = memcpy(smalloc(cur.d + 1), cur.b, cur.d); + ret[cur.d] = 0; + return(ret); +} + +static void passduplex(struct bufio *a, int afd, struct bufio *b, int bfd) +{ + struct selected pfd[4], sel; + struct bufio *sio; + int n, ev; + + while(!bioeof(a) && !bioeof(b)) { + biocopybuf(b, a); + biocopybuf(a, b); + n = 0; + if(!a->eof) { + ev = 0; + if(biorspace(a)) + ev |= EV_READ; + if(biowdata(a)) + ev |= EV_WRITE; + if(ev) + pfd[n++] = (struct selected){.fd = afd, .ev = ev}; + } + if(!b->eof) { + ev = 0; + if(!b->eof && biorspace(b)) + ev |= EV_READ; + if(biowdata(b)) + ev |= EV_WRITE; + if(ev) + pfd[n++] = (struct selected){.fd = bfd, .ev = ev}; + } + if((sel = mblock(600, n, pfd)).ev == 0) + break; + if(sel.fd == afd) + sio = a; + else if(sel.fd == bfd) + sio = b; + else + break; + if((sel.ev & EV_READ) && (biofillsome(sio) < 0)) + break; + if((sel.ev & EV_WRITE) && (bioflushsome(sio) < 0)) + break; + } +} + +void serve(struct bufio *in, int infd, struct conn *conn) { int pfds[2]; - FILE *out; + struct bufio *out, *dout; + struct stdiofd *outi; struct hthead *req, *resp; - char *hd; + char *hd, *id; off_t dlen; - int keep; + int keep, duplex; + id = connid(); out = NULL; req = resp = NULL; while(plex >= 0) { + bioflush(in); if((req = parsereq(in)) == NULL) break; if(!canonreq(req)) break; + headappheader(req, "X-Ash-Connection-ID", id); if((conn->initreq != NULL) && conn->initreq(conn, req)) break; @@ -307,7 +379,7 @@ void serve(FILE *in, struct conn *conn) if(sendreq(plex, req, pfds[0])) break; close(pfds[0]); - out = mtstdopen(pfds[1], 1, 600, "r+"); + out = mtbioopen(pfds[1], 1, 600, "r+", &outi); if(getheader(req, "content-type") != NULL) { if((hd = getheader(req, "content-length")) != NULL) { @@ -324,34 +396,46 @@ void serve(FILE *in, struct conn *conn) headrmheader(req, "content-type"); } } - if(fflush(out)) + if(bioflush(out)) break; /* Make sure to send EOF */ shutdown(pfds[1], SHUT_WR); - if((resp = parseresponse(out)) == NULL) + if((resp = parseresponseb(out)) == NULL) break; replstr(&resp->ver, req->ver); if(!getheader(resp, "server")) headappheader(resp, "Server", sprintf3("ashd/%s", VERSION)); + duplex = hasheader(resp, "x-ash-switch", "duplex"); + trimx(resp); - if(!strcasecmp(req->ver, "HTTP/1.0")) { + if(duplex) { + if(outi->rights < 0) + break; + writerespb(in, resp); + bioprintf(in, "\r\n"); + dout = mtbioopen(outi->rights, 1, 600, "r+", NULL); + passduplex(in, infd, dout, outi->rights); + outi->rights = -1; + bioclose(dout); + break; + } else if(!strcasecmp(req->ver, "HTTP/1.0")) { if(!strcasecmp(req->method, "head")) { keep = http10keep(req, resp); - writeresp(in, resp); - fprintf(in, "\r\n"); + writerespb(in, resp); + bioprintf(in, "\r\n"); } else if((hd = getheader(resp, "content-length")) != NULL) { keep = http10keep(req, resp); dlen = atoo(hd); - writeresp(in, resp); - fprintf(in, "\r\n"); + writerespb(in, resp); + bioprintf(in, "\r\n"); if(passdata(out, in, dlen) != dlen) break; } else { headrmheader(resp, "connection"); - writeresp(in, resp); - fprintf(in, "\r\n"); + writerespb(in, resp); + bioprintf(in, "\r\n"); passdata(out, in, -1); break; } @@ -359,23 +443,23 @@ void serve(FILE *in, struct conn *conn) break; } else if(!strcasecmp(req->ver, "HTTP/1.1")) { if(!strcasecmp(req->method, "head")) { - writeresp(in, resp); - fprintf(in, "\r\n"); + writerespb(in, resp); + bioprintf(in, "\r\n"); } else if((hd = getheader(resp, "content-length")) != NULL) { - writeresp(in, resp); - fprintf(in, "\r\n"); + writerespb(in, resp); + bioprintf(in, "\r\n"); dlen = atoo(hd); if(passdata(out, in, dlen) != dlen) break; } else if(!getheader(resp, "transfer-encoding")) { headappheader(resp, "Transfer-Encoding", "chunked"); - writeresp(in, resp); - fprintf(in, "\r\n"); + writerespb(in, resp); + bioprintf(in, "\r\n"); if(passchunks(out, in)) break; } else { - writeresp(in, resp); - fprintf(in, "\r\n"); + writerespb(in, resp); + bioprintf(in, "\r\n"); passdata(out, in, -1); break; } @@ -385,7 +469,7 @@ void serve(FILE *in, struct conn *conn) break; } - fclose(out); + bioclose(out); out = NULL; freehthead(req); freehthead(resp); @@ -393,20 +477,22 @@ void serve(FILE *in, struct conn *conn) } if(out != NULL) - fclose(out); + bioclose(out); if(req != NULL) freehthead(req); if(resp != NULL) freehthead(resp); - fclose(in); + bioclose(in); + free(id); } static void plexwatch(struct muth *muth, va_list args) { vavar(int, fd); char *buf; - int i, ret; + int i, s, ret; + s = 0; while(1) { if(block(fd, EV_READ, 0) == 0) break; @@ -416,6 +502,7 @@ static void plexwatch(struct muth *muth, va_list args) flog(LOG_WARNING, "received error on rootplex read channel: %s", strerror(errno)); exit(1); } else if(ret == 0) { + s = 1; free(buf); break; } @@ -423,15 +510,16 @@ static void plexwatch(struct muth *muth, va_list args) * some day... */ free(buf); } - close(plex); - plex = -1; + shutdown(plex, SHUT_RDWR); for(i = 0; i < listeners.d; i++) { if(listeners.b[i] == muth) bufdel(listeners, i); } - flog(LOG_INFO, "root handler exited, so shutting down listening..."); - while(listeners.d > 0) - resume(listeners.b[0], 0); + if(s) { + flog(LOG_INFO, "root handler exited, so shutting down listening..."); + while(listeners.d > 0) + resume(listeners.b[0], 0); + } } static void initroot(void *uu) diff --git a/src/htparser.h b/src/htparser.h index 7bd60a0..d9f014d 100644 --- a/src/htparser.h +++ b/src/htparser.h @@ -11,7 +11,7 @@ struct mtbuf { size_t s, d; }; -void serve(FILE *in, struct conn *conn); +void serve(struct bufio *in, int infd, struct conn *conn); int listensock4(int port); int listensock6(int port); diff --git a/src/httimed.c b/src/httimed.c index 2c939c5..dc8ba99 100644 --- a/src/httimed.c +++ b/src/httimed.c @@ -23,6 +23,7 @@ #include #include #include +#include #ifdef HAVE_CONFIG_H #include @@ -39,21 +40,21 @@ static void usage(FILE *out) int main(int argc, char **argv) { - int c, t, ret; + int c, timeout, ret; int ch, fd; struct hthead *req; struct pollfd pfd[2]; time_t lreq, now; - t = 300; + timeout = 300; while((c = getopt(argc, argv, "+ht:")) >= 0) { switch(c) { case 'h': usage(stdout); exit(0); case 't': - t = atoi(optarg); - if(t < 1) { + timeout = atoi(optarg); + if(timeout < 1) { fprintf(stderr, "httimed: timeout must be positive\n"); exit(1); } @@ -75,7 +76,7 @@ int main(int argc, char **argv) pfd[0].events = POLLIN; pfd[1].fd = ch; pfd[1].events = POLLHUP; - if((ret = poll(pfd, 2, (t + 1 - (time(NULL) - lreq)) * 1000)) < 0) { + if((ret = poll(pfd, 2, (timeout < 0)?-1:((timeout + 1 - (time(NULL) - lreq)) * 1000))) < 0) { if(errno != EINTR) { flog(LOG_ERR, "httimed: error in poll: %s", strerror(errno)); exit(1); @@ -96,10 +97,10 @@ int main(int argc, char **argv) freehthead(req); close(fd); } - if(pfd[1].revents & POLLHUP) - break; - if(now - lreq > t) - break; + if((pfd[1].revents & POLLHUP) || (now - lreq > timeout)) { + timeout = -1; + shutdown(0, SHUT_RDWR); + } lreq = now; } return(0); diff --git a/src/httrcall.c b/src/httrcall.c new file mode 100644 index 0000000..c3e7e0d --- /dev/null +++ b/src/httrcall.c @@ -0,0 +1,144 @@ +/* + ashd - A Sane HTTP Daemon + Copyright (C) 2008 Fredrik Tolf + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +*/ + +#include +#include +#include +#include +#include +#include +#include + +#ifdef HAVE_CONFIG_H +#include +#endif +#include +#include +#include +#include +#include + +struct current { + struct current *next, *prev; + pid_t pid; +}; + +static char **prog; +static struct current *running = NULL; +static int nrunning = 0, limit = 0; +static volatile int exited; + +static void checkexit(int block) +{ + pid_t pid; + int st; + struct current *rec; + + exited = 0; + while((pid = waitpid(-1, &st, block?0:WNOHANG)) > 0) { + if(WCOREDUMP(st)) + flog(LOG_WARNING, "child process %i dumped core", pid); + for(rec = running; rec != NULL; rec = rec->next) { + if(rec->pid == pid) { + if(rec->next) + rec->next->prev = rec->prev; + if(rec->prev) + rec->prev->next = rec->next; + if(rec == running) + running = rec->next; + free(rec); + nrunning--; + break; + } + } + } +} + +static void serve(struct hthead *req, int fd) +{ + pid_t new; + struct current *rec; + + while((limit > 0) && (nrunning >= limit)) + checkexit(1); + if((new = stdforkserve(prog, req, fd, NULL, NULL)) < 0) { + simpleerror(fd, 500, "Server Error", "The server appears to be overloaded."); + return; + } + omalloc(rec); + rec->pid = new; + rec->next = running; + if(running != NULL) + running->prev = rec; + running = rec; + nrunning++; +} + +static void chldhandler(int sig) +{ + exited = 1; +} + +static void usage(FILE *out) +{ + fprintf(out, "usage: httrcall [-h] [-l LIMIT] PROGRAM [ARGS...]\n"); +} + +int main(int argc, char **argv) +{ + int c; + struct hthead *req; + int fd; + + while((c = getopt(argc, argv, "+hl:")) >= 0) { + switch(c) { + case 'h': + usage(stdout); + exit(0); + case 'l': + limit = atoi(optarg); + break; + default: + usage(stderr); + exit(1); + } + } + if(argc < optind - 1) { + usage(stderr); + exit(1); + } + prog = argv + optind; + sigaction(SIGCHLD, &(struct sigaction) { + .sa_handler = chldhandler, + }, NULL); + while(1) { + if(exited) + checkexit(0); + if((fd = recvreq(0, &req)) < 0) { + if(errno == EINTR) + continue; + if(errno != 0) + flog(LOG_ERR, "recvreq: %s", strerror(errno)); + break; + } + serve(req, fd); + freehthead(req); + close(fd); + } + return(0); +} diff --git a/src/patplex.c b/src/patplex.c index 13fa062..9939543 100644 --- a/src/patplex.c +++ b/src/patplex.c @@ -44,6 +44,7 @@ #define PAT_DEFAULT 5 #define PATFL_MSS 1 +#define PATFL_UNQ 2 struct config { struct child *children; @@ -188,6 +189,7 @@ static struct pattern *parsepattern(struct cfstate *s) flog(LOG_WARNING, "%s:%i: missing pattern for `%s' match", s->file, s->lno, s->argv[0]); continue; } + rxfl = 0; if(s->argc >= 3) { if(strchr(s->argv[2], 'i')) rxfl |= REG_ICASE; @@ -207,12 +209,15 @@ static struct pattern *parsepattern(struct cfstate *s) if(s->argc >= 3) { if(strchr(s->argv[2], 's')) rule->fl |= PATFL_MSS; + if(strchr(s->argv[2], 'q')) + rule->fl |= PATFL_UNQ; } } else if(!strcmp(s->argv[0], "header")) { if(s->argc < 3) { flog(LOG_WARNING, "%s:%i: missing header name or pattern for `header' match", s->file, s->lno); continue; } + rxfl = 0; if(s->argc >= 4) { if(strchr(s->argv[3], 'i')) rxfl |= REG_ICASE; @@ -360,12 +365,42 @@ static void exprestpat(struct hthead *req, struct pattern *pat, char **mstr) buffree(buf); } +static void qoffsets(char *buf, int *obuf, char *pstr, int unquote) +{ + int i, o, d1, d2; + + if(unquote) { + i = o = 0; + while(pstr[i]) { + obuf[o] = i; + if((pstr[i] == '%') && ((d1 = hexdigit(pstr[i + 1])) >= 0) && ((d2 = hexdigit(pstr[i + 2])) >= 0)) { + buf[o] = (d1 << 4) | d2; + i += 3; + } else { + buf[o] = pstr[i]; + i++; + } + o++; + } + buf[o] = 0; + obuf[o] = i; + } else { + for(i = 0; pstr[i]; i++) { + buf[i] = pstr[i]; + obuf[i] = i; + } + buf[i] = 0; + obuf[i] = i; + } +} + static struct pattern *findmatch(struct config *cf, struct hthead *req, int trydefault) { int i, o; struct pattern *pat; struct rule *rule; - int rmo, matched; + int rmo; + regex_t *rx; char *pstr; char **mstr; regmatch_t gr[10]; @@ -374,46 +409,52 @@ static struct pattern *findmatch(struct config *cf, struct hthead *req, int tryd for(pat = cf->patterns; pat != NULL; pat = pat->next) { rmo = -1; for(i = 0; (rule = pat->rules[i]) != NULL; i++) { - matched = 0; + rx = NULL; if(rule->type == PAT_REST) { - if((matched = !regexec(rule->pattern, pstr = req->rest, 10, gr, 0))) - rmo = gr[0].rm_eo; - else - break; + rx = rule->pattern; + pstr = req->rest; } else if(rule->type == PAT_URL) { - if(!(matched = !regexec(rule->pattern, pstr = req->url, 10, gr, 0))) - break; + rx = rule->pattern; + pstr = req->url; } else if(rule->type == PAT_METHOD) { - if(!(matched = !regexec(rule->pattern, pstr = req->method, 10, gr, 0))) - break; + rx = rule->pattern; + pstr = req->method; } else if(rule->type == PAT_HEADER) { + rx = rule->pattern; if(!(pstr = getheader(req, rule->header))) break; - if(!(matched = !regexec(rule->pattern, pstr, 10, gr, 0))) + } + if(rx != NULL) { + char pbuf[strlen(pstr) + 1]; + int obuf[strlen(pstr) + 1]; + qoffsets(pbuf, obuf, pstr, !!(rule->fl & PATFL_UNQ)); + if(regexec(rx, pbuf, 10, gr, 0)) break; + else if(rule->type == PAT_REST) + rmo = obuf[gr[0].rm_eo]; + if(rule->fl & PATFL_MSS) { + if(mstr) { + flog(LOG_WARNING, "two pattern rules marked with `s' flag found (for handler %s)", pat->childnm); + freeca(mstr); + } + for(o = 0; o < 10; o++) { + if(gr[o].rm_so < 0) + break; + } + mstr = szmalloc((o + 1) * sizeof(*mstr)); + for(o = 0; o < 10; o++) { + if(gr[o].rm_so < 0) + break; + mstr[o] = smalloc(obuf[gr[o].rm_eo] - obuf[gr[o].rm_so] + 1); + memcpy(mstr[o], pstr + obuf[gr[o].rm_so], obuf[gr[o].rm_eo] - obuf[gr[o].rm_so]); + mstr[o][obuf[gr[o].rm_eo] - obuf[gr[o].rm_so]] = 0; + } + } } else if(rule->type == PAT_ALL) { } else if(rule->type == PAT_DEFAULT) { if(!trydefault) break; } - if(matched && (rule->fl & PATFL_MSS)) { - if(mstr) { - flog(LOG_WARNING, "two pattern rules marked with `s' flag found (for handler %s)", pat->childnm); - freeca(mstr); - } - for(o = 0; o < 10; o++) { - if(gr[o].rm_so < 0) - break; - } - mstr = szmalloc((o + 1) * sizeof(*mstr)); - for(o = 0; o < 10; o++) { - if(gr[o].rm_so < 0) - break; - mstr[o] = smalloc(gr[o].rm_eo - gr[o].rm_so + 1); - memcpy(mstr[o], pstr + gr[o].rm_so, gr[o].rm_eo - gr[o].rm_so); - mstr[o][gr[o].rm_eo - gr[o].rm_so] = 0; - } - } } if(!rule) { if(pat->restpat) { @@ -433,6 +474,14 @@ static struct pattern *findmatch(struct config *cf, struct hthead *req, int tryd return(NULL); } +static void childerror(struct hthead *req, int fd) +{ + if(errno == EAGAIN) + simpleerror(fd, 500, "Server Error", "The request handler is overloaded."); + else + simpleerror(fd, 500, "Server Error", "The request handler crashed."); +} + static void serve(struct hthead *req, int fd) { struct pattern *pat; @@ -472,7 +521,7 @@ static void serve(struct hthead *req, int fd) headappheader(req, head->name, head->value); } if(childhandle(ch, req, fd, NULL, NULL)) - simpleerror(fd, 500, "Server Error", "The request handler crashed."); + childerror(req, fd); } static void reloadconf(char *nm) @@ -514,7 +563,7 @@ int main(int argc, char **argv) { int c; int nodef; - char *gcf; + char *gcf, *lcf; struct hthead *req; int fd; @@ -542,8 +591,14 @@ int main(int argc, char **argv) free(gcf); } } - if((lconfig = readconfig(argv[optind])) == NULL) { - flog(LOG_ERR, "could not read `%s'", argv[optind]); + if((strchr(lcf = argv[optind], '/')) == NULL) { + if((lcf = findstdconf(sprintf3("ashd/%s", lcf))) == NULL) { + flog(LOG_ERR, "could not find requested configuration file `%s'", argv[optind]); + exit(1); + } + } + if((lconfig = readconfig(lcf)) == NULL) { + flog(LOG_ERR, "could not read `%s'", lcf); exit(1); } signal(SIGCHLD, chldhandler); @@ -551,7 +606,7 @@ int main(int argc, char **argv) signal(SIGPIPE, sighandler); while(1) { if(reload) { - reloadconf(argv[optind]); + reloadconf(lcf); reload = 0; } if((fd = recvreq(0, &req)) < 0) { diff --git a/src/plaintcp.c b/src/plaintcp.c index 7d36a1d..56fc81c 100644 --- a/src/plaintcp.c +++ b/src/plaintcp.c @@ -150,38 +150,48 @@ void servetcp(struct muth *muth, va_list args) vavar(int, fd); vavar(struct sockaddr_storage, name); vavar(struct tcpport *, stcp); - FILE *in; + struct bufio *in; struct conn conn; struct tcpconn tcp; memset(&conn, 0, sizeof(conn)); memset(&tcp, 0, sizeof(tcp)); - in = mtstdopen(fd, 1, 60, "r+"); + in = mtbioopen(fd, 1, 60, "r+", NULL); conn.pdata = &tcp; conn.initreq = initreq; tcp.fd = fd; tcp.name = name; tcp.port = stcp; - serve(in, &conn); + serve(in, fd, &conn); } static void listenloop(struct muth *muth, va_list args) { vavar(struct tcpport *, tcp); - int i, ns; + int i, ns, n; struct sockaddr_storage name; socklen_t namelen; + fcntl(tcp->fd, F_SETFL, fcntl(tcp->fd, F_GETFL) | O_NONBLOCK); while(1) { namelen = sizeof(name); if(block(tcp->fd, EV_READ, 0) == 0) goto out; - ns = accept(tcp->fd, (struct sockaddr *)&name, &namelen); - if(ns < 0) { - flog(LOG_ERR, "accept: %s", strerror(errno)); - goto out; + n = 0; + while(1) { + ns = accept(tcp->fd, (struct sockaddr *)&name, &namelen); + if(ns < 0) { + if(errno == EAGAIN) + break; + if(errno == ECONNABORTED) + continue; + flog(LOG_ERR, "accept: %s", strerror(errno)); + goto out; + } + mustart(servetcp, ns, name, tcp); + if(++n >= 100) + break; } - mustart(servetcp, ns, name, tcp); } out: diff --git a/src/psendfile.c b/src/psendfile.c index 08cd1d2..3e26f8f 100644 --- a/src/psendfile.c +++ b/src/psendfile.c @@ -227,7 +227,7 @@ static void serve(struct muth *muth, va_list args) sfile = NULL; contype = NULL; - out = mtstdopen(fd, 1, 60, "r+"); + out = mtstdopen(fd, 1, 60, "r+", NULL); if((file = getheader(req, "X-Ash-File")) == NULL) { flog(LOG_ERR, "psendfile: needs to be called with the X-Ash-File header"); @@ -238,7 +238,7 @@ static void serve(struct muth *muth, va_list args) simpleerror2(out, 404, "Not Found", "The requested URL has no corresponding resource."); goto out; } - if(((sfile = fopen(file, "r")) < 0) || fstat(fileno(sfile), &sb)) { + if(((sfile = fopen(file, "r")) == NULL) || fstat(fileno(sfile), &sb)) { flog(LOG_ERR, "psendfile: could not stat input file %s: %s", file, strerror(errno)); simpleerror2(out, 500, "Internal Error", "The server could not access its own data."); goto out; diff --git a/src/ssl-gnutls.c b/src/ssl-gnutls.c index 0d4dd22..7aa1df0 100644 --- a/src/ssl-gnutls.c +++ b/src/ssl-gnutls.c @@ -34,6 +34,7 @@ #include #include #include +#include #include "htparser.h" @@ -56,6 +57,7 @@ struct sslport { int fd; int sport; gnutls_certificate_credentials_t creds; + gnutls_priority_t ciphers; struct namedcreds **ncreds; }; @@ -187,7 +189,7 @@ static int tlsblock(int fd, gnutls_session_t sess, time_t to) return(block(fd, EV_READ, to)); } -static ssize_t sslread(void *cookie, char *buf, size_t len) +static ssize_t sslread(void *cookie, void *buf, size_t len) { struct sslconn *ssl = cookie; ssize_t xf; @@ -216,7 +218,7 @@ static ssize_t sslread(void *cookie, char *buf, size_t len) return(xf); } -static ssize_t sslwrite(void *cookie, const char *buf, size_t len) +static ssize_t sslwrite(void *cookie, const void *buf, size_t len) { struct sslconn *ssl = cookie; int ret; @@ -254,7 +256,7 @@ static int sslclose(void *cookie) return(0); } -static cookie_io_functions_t iofuns = { +static struct bufioops iofuns = { .read = sslread, .write = sslwrite, .close = sslclose, @@ -265,16 +267,12 @@ static int initreq(struct conn *conn, struct hthead *req) struct sslconn *ssl = conn->pdata; struct sockaddr_storage sa; socklen_t salen; - char nmbuf[256]; headappheader(req, "X-Ash-Address", formathaddress((struct sockaddr *)&ssl->name, sizeof(sa))); - if(ssl->name.ss_family == AF_INET) { - headappheader(req, "X-Ash-Address", inet_ntop(AF_INET, &((struct sockaddr_in *)&ssl->name)->sin_addr, nmbuf, sizeof(nmbuf))); + if(ssl->name.ss_family == AF_INET) headappheader(req, "X-Ash-Port", sprintf3("%i", ntohs(((struct sockaddr_in *)&ssl->name)->sin_port))); - } else if(ssl->name.ss_family == AF_INET6) { - headappheader(req, "X-Ash-Address", inet_ntop(AF_INET6, &((struct sockaddr_in6 *)&ssl->name)->sin6_addr, nmbuf, sizeof(nmbuf))); + else if(ssl->name.ss_family == AF_INET6) headappheader(req, "X-Ash-Port", sprintf3("%i", ntohs(((struct sockaddr_in6 *)&ssl->name)->sin6_port))); - } salen = sizeof(sa); if(!getsockname(ssl->fd, (struct sockaddr *)&sa, &salen)) headappheader(req, "X-Ash-Server-Address", formathaddress((struct sockaddr *)&sa, sizeof(sa))); @@ -292,7 +290,6 @@ static void servessl(struct muth *muth, va_list args) struct sslconn ssl; gnutls_session_t sess; int ret; - FILE *in; int setcreds(gnutls_session_t sess) { @@ -325,7 +322,7 @@ static void servessl(struct muth *muth, va_list args) numconn++; fcntl(fd, F_SETFL, fcntl(fd, F_GETFL) | O_NONBLOCK); gnutls_init(&sess, GNUTLS_SERVER); - gnutls_set_default_priority(sess); + gnutls_priority_set(sess, pd->ciphers); gnutls_db_set_retrieve_function(sess, sessdbfetch); gnutls_db_set_store_function(sess, sessdbstore); gnutls_db_set_remove_function(sess, sessdbdel); @@ -347,8 +344,7 @@ static void servessl(struct muth *muth, va_list args) ssl.name = name; ssl.sess = sess; bufinit(ssl.in); - in = fopencookie(&ssl, "r+", iofuns); - serve(in, &conn); + serve(bioopen(&ssl, &iofuns), fd, &conn); out: gnutls_deinit(sess); @@ -359,20 +355,30 @@ out: static void listenloop(struct muth *muth, va_list args) { vavar(struct sslport *, pd); - int i, ns; + int i, ns, n; struct sockaddr_storage name; socklen_t namelen; + fcntl(pd->fd, F_SETFL, fcntl(pd->fd, F_GETFL) | O_NONBLOCK); while(1) { namelen = sizeof(name); if(block(pd->fd, EV_READ, 0) == 0) goto out; - ns = accept(pd->fd, (struct sockaddr *)&name, &namelen); - if(ns < 0) { - flog(LOG_ERR, "accept: %s", strerror(errno)); - goto out; + n = 0; + while(1) { + ns = accept(pd->fd, (struct sockaddr *)&name, &namelen); + if(ns < 0) { + if(errno == EAGAIN) + break; + if(errno == ECONNABORTED) + continue; + flog(LOG_ERR, "accept: %s", strerror(errno)); + goto out; + } + mustart(servessl, ns, name, pd); + if(++n >= 100) + break; } - mustart(servessl, ns, name, pd); } out: @@ -512,15 +518,17 @@ void handlegnussl(int argc, char **argp, char **argv) { int i, ret, port, fd; gnutls_certificate_credentials_t creds; + gnutls_priority_t ciphers; struct ncredbuf ncreds; struct sslport *pd; - char *crtfile, *keyfile; + char *crtfile, *keyfile, *perr; init(); port = 443; bufinit(ncreds); gnutls_certificate_allocate_credentials(&creds); keyfile = crtfile = NULL; + ciphers = NULL; for(i = 0; i < argc; i++) { if(!strcmp(argp[i], "help")) { printf("ssl handler parameters:\n"); @@ -528,6 +536,8 @@ void handlegnussl(int argc, char **argp, char **argv) printf("\t\tThe name of the file to read the certificate from.\n"); printf("\tkey=KEY-FILE [same as CERT-FILE]\n"); printf("\t\tThe name of the file to read the private key from.\n"); + printf("\tprio=PRIORITIES [NORMAL]\n"); + printf("\t\tCiphersuite priorities, as a GnuTLS priority string.\n"); printf("\ttrust=CA-FILE [no default]\n"); printf("\t\tThe name of a file to read trusted certificates from.\n"); printf("\t\tMay be given multiple times.\n"); @@ -557,6 +567,17 @@ void handlegnussl(int argc, char **argp, char **argv) crtfile = argv[i]; } else if(!strcmp(argp[i], "key")) { keyfile = argv[i]; + } else if(!strcmp(argp[i], "prio")) { + if(ciphers != NULL) + gnutls_priority_deinit(ciphers); + ret = gnutls_priority_init(&ciphers, argv[i], (const char **)&perr); + if(ret == GNUTLS_E_INVALID_REQUEST) { + flog(LOG_ERR, "ssl: invalid cipher priority string, at `%s'", perr); + exit(1); + } else if(ret != 0) { + flog(LOG_ERR, "ssl: could not initialize cipher priorities: %s", gnutls_strerror(ret)); + exit(1); + } } else if(!strcmp(argp[i], "trust")) { if((ret = gnutls_certificate_set_x509_trust_file(creds, argv[i], GNUTLS_X509_FMT_PEM)) != 0) { flog(LOG_ERR, "ssl: could not load trust file `%s': %s", argv[i], gnutls_strerror(ret)); @@ -604,6 +625,10 @@ void handlegnussl(int argc, char **argp, char **argv) flog(LOG_ERR, "ssl: could not load certificate or key: %s", gnutls_strerror(ret)); exit(1); } + if((ciphers == NULL) && ((ret = gnutls_priority_init(&ciphers, "NORMAL", NULL)) != 0)) { + flog(LOG_ERR, "ssl: could not initialize cipher priorities: %s", gnutls_strerror(ret)); + exit(1); + } gnutls_certificate_set_dh_params(creds, dhparams()); bufadd(ncreds, NULL); omalloc(pd); @@ -611,6 +636,7 @@ void handlegnussl(int argc, char **argp, char **argv) pd->sport = port; pd->creds = creds; pd->ncreds = ncreds.b; + pd->ciphers = ciphers; bufadd(listeners, mustart(listenloop, pd)); if((fd = listensock4(port)) < 0) { if(errno != EADDRINUSE) { @@ -622,6 +648,8 @@ void handlegnussl(int argc, char **argp, char **argv) pd->fd = fd; pd->sport = port; pd->creds = creds; + pd->ncreds = ncreds.b; + pd->ciphers = ciphers; bufadd(listeners, mustart(listenloop, pd)); } } diff --git a/src/userplex.c b/src/userplex.c index c50a15c..48a8ed3 100644 --- a/src/userplex.c +++ b/src/userplex.c @@ -158,19 +158,24 @@ static int forkchild(char *usrnm, struct hthead *forreq, int reqfd) static void serve2(struct user *usr, struct hthead *req, int fd) { + int serr; + if(usr->fd < 0) usr->fd = forkchild(usr->name, req, fd); - if(sendreq(usr->fd, req, fd)) { - if((errno == EPIPE) || (errno == ECONNRESET)) { + if(sendreq2(usr->fd, req, fd, MSG_NOSIGNAL | MSG_DONTWAIT)) { + serr = errno; + if((serr == EPIPE) || (serr == ECONNRESET)) { /* Assume that the child has crashed and restart it. */ close(usr->fd); usr->fd = forkchild(usr->name, req, fd); - if(!sendreq(usr->fd, req, fd)) + if(!sendreq2(usr->fd, req, fd, MSG_NOSIGNAL | MSG_DONTWAIT)) return; } - flog(LOG_ERR, "could not pass on request to user `%s': %s", usr->name, strerror(errno)); - close(usr->fd); - usr->fd = -1; + flog(LOG_ERR, "could not pass on request to user `%s': %s", usr->name, strerror(serr)); + if(serr != EAGAIN) { + close(usr->fd); + usr->fd = -1; + } } }