From: Fredrik Tolf Date: Tue, 20 Jan 2026 05:28:15 +0000 (+0100) Subject: sendfile: Add transparent compression support. X-Git-Url: http://www.dolda2000.com/gitweb/?a=commitdiff_plain;h=b83945860b5be303b0f0d79f8a8dafc4a1337ae2;p=ashd.git sendfile: Add transparent compression support. --- diff --git a/configure.ac b/configure.ac index fcecf84..5919259 100644 --- a/configure.ac +++ b/configure.ac @@ -96,6 +96,26 @@ if test "$HAS_XATTR" = yes; then fi AC_SUBST(XATTR_LIBS) +AH_TEMPLATE(HAVE_ZLIB, [define to compile support for libz content compression]) +AC_ARG_WITH(zlib, AS_HELP_STRING([--with-zlib], [enable zlib support])) +HAS_ZLIB="" +if test "$with_zlib" = no; then HAS_ZLIB=no; fi +if test -z "$HAS_ZLIB"; then + AC_CHECK_LIB(z, deflate, [:], [HAS_ZLIB=no]) +fi +if test -z "$HAS_ZLIB"; then + AC_CHECK_HEADER(zlib.h, [], [HAS_ZLIB=no]) +fi +if test "$HAS_ZLIB" != no; then HAS_ZLIB=yes; fi +if test "$with_zlib" = yes -a "$HAS_ZLIB" = no; then + AC_MSG_ERROR([*** cannot find zlib support on this system]) +fi +if test "$HAS_ZLIB" = yes; then + ZLIB_LIBS=-lz + AC_DEFINE(HAVE_ZLIB) +fi +AC_SUBST(ZLIB_LIBS) + AH_TEMPLATE(HAVE_GNUTLS, [define to use the GnuTLS library for SSL support]) AH_TEMPLATE(HAVE_OPENSSL, [define to use the OpenSSL library for SSL support]) AC_ARG_WITH(gnutls, AS_HELP_STRING([--with-gnutls], [enable SSL support with the GnuTLS library])) diff --git a/etc/ashd/dirplex.d/send.rc b/etc/ashd/dirplex.d/send.rc index ae540a2..b834c22 100644 --- a/etc/ashd/dirplex.d/send.rc +++ b/etc/ashd/dirplex.d/send.rc @@ -4,18 +4,22 @@ fchild send match filename *.html xset content-type text/html + xset compress 1 handler send match filename *.css xset content-type text/css + xset compress 1 handler send match filename *.txt xset content-type text/plain + xset compress 1 handler send match filename *.js xset content-type text/javascript + xset compress 1 handler send # Image types diff --git a/src/Makefile.am b/src/Makefile.am index 3bc8963..91bde61 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -5,11 +5,13 @@ bin_PROGRAMS = htparser sendfile callcgi patplex userplex htls \ errlogger httimed psendfile httrcall htpipe ratequeue htparser_SOURCES = htparser.c htparser.h plaintcp.c ssl-gnutls.c ssl-openssl.c +sendfile_SOURCES = sendfile.c compress.c +psendfile_SOURCES = psendfile.c compress.c LDADD = $(top_srcdir)/lib/libht.a AM_CPPFLAGS = -I$(top_srcdir)/lib htparser_CPPFLAGS = $(AM_CPPFLAGS) @GNUTLS_CPPFLAGS@ @OPENSSL_CPPFLAGS@ htparser_LDADD = $(LDADD) @GNUTLS_LIBS@ @OPENSSL_LIBS@ -sendfile_LDADD = $(LDADD) -lmagic @XATTR_LIBS@ -psendfile_LDADD = $(LDADD) -lmagic @XATTR_LIBS@ +sendfile_LDADD = $(LDADD) -lmagic @XATTR_LIBS@ @ZLIB_LIBS@ +psendfile_LDADD = $(LDADD) -lmagic @XATTR_LIBS@ @ZLIB_LIBS@ diff --git a/src/compress.c b/src/compress.c new file mode 100644 index 0000000..f0fb688 --- /dev/null +++ b/src/compress.c @@ -0,0 +1,301 @@ +/* + ashd - A Sane HTTP Daemon + Copyright (C) 2008 Fredrik Tolf + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +*/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef HAVE_CONFIG_H +#include +#endif +#include +#include + +#include "compress.h" + +struct etype { + char *name; + int (*compress)(int, int); +}; + +static char *cachedir(void) +{ + static char *ret = NULL; + + if(ret == NULL) + ret = sprintf2("/tmp/sendfile-cc-%i", getuid()); + return(ret); +} + +#ifdef HAVE_ZLIB + +#include + +static int deflatefile(int dfd, int sfd) +{ + int rv, zr, b, w; + char *ib, *ob; + z_stream zs; + + ib = smalloc(65536); + ob = smalloc(65536); + memset(&zs, 0, sizeof(zs)); + deflateInit(&zs, 9); + while(1) { + rv = read(sfd, ib, 6553); + if(rv < 0) + goto out; + if(rv == 0) + break; + zs.next_in = (Bytef *)ib; + zs.avail_in = rv; + while(zs.avail_in > 0) { + zs.next_out = (Bytef *)ob; + zs.avail_out = 65536; + if(deflate(&zs, Z_NO_FLUSH) != Z_OK) { + rv = -1; + goto out; + } + for(w = 0, b = (char *)zs.next_out - ob; w < b; w += rv) { + rv = write(dfd, ob + w, b - w); + if(rv < 0) + goto out; + } + } + } + zs.next_in = (Bytef *)ib; + zs.avail_in = 0; + do { + zs.next_out = (Bytef *)ob; + zs.avail_out = 65536; + zr = deflate(&zs, Z_FINISH); + if((zr != Z_OK) && (zr != Z_STREAM_END)) { + rv = -1; + goto out; + } + for(w = 0, b = (char *)zs.next_out - ob; w < b; w += rv) { + rv = write(dfd, ob + w, b - w); + if(rv < 0) + goto out; + } + } while(zr != Z_STREAM_END); + rv = 0; +out: + free(ib); + free(ob); + deflateEnd(&zs); + return(rv); +} + +static struct etype t_gz = { + .name = "deflate", + .compress = deflatefile, +}; + +#endif + +static struct etype *types[] = { +#ifdef HAVE_ZLIB + &t_gz, +#endif + NULL, +}; + +static struct etype *findtype(char *name) +{ + int i; + + for(i = 0; types[i] != NULL; i++) { + if(!strcmp(name, types[i]->name)) + return(types[i]); + } + return(NULL); +} + +static void checkclean(void) +{ + char *path; + int fd; + struct stat sb; + DIR *dp; + struct dirent *ent; + time_t now; + + if(stat(sprintf3("%s/lastclean", cachedir()), &sb)) + sb.st_mtime = 0; + now = time(NULL); + if(now - sb.st_mtime < 3600 * 24) + return; + if((fd = open(path = sprintf3("%s/cleaning", cachedir()), O_WRONLY | O_CREAT | O_EXCL, 0600)) < 0) { + if(errno == EEXIST) { + if(stat(path, &sb)) + return; + if(now - sb.st_mtime < 3600) + return; + } else { + return; + } + } + close(fd); + if((fd = open(sprintf3("%s/lastclean", cachedir()), O_WRONLY | O_CREAT, 0600)) >= 0) + close(fd); + if((dp = opendir(cachedir())) == NULL) + return; + while((ent = readdir(dp)) != NULL) { + if((ent->d_name[0] == '.') || !strcmp(ent->d_name, "cleaning") || !strcmp(ent->d_name, "lastclean")) + continue; + path = sprintf3("%s/%s", cachedir(), ent->d_name); + if(stat(path, &sb)) + continue; + if(now - sb.st_atime > 3600 * 48) + unlink(path); + } + closedir(dp); + unlink(sprintf3("%s/cleaning", cachedir())); +} + +static int openbytype(struct etype *type, int ofd, struct stat *info, struct stat *oinfo) +{ + char *epath, *npath; + int fd; + + npath = NULL; + epath = sprintf2("%s/%jx:%jx:%s", cachedir(), (uintmax_t)info->st_dev, (uintmax_t)info->st_ino, type->name); + if((fd = open(epath, O_RDONLY)) >= 0) { + if(fstat(fd, oinfo)) + goto error; + if(oinfo->st_mtime >= info->st_mtime) + goto out; + close(fd); + fd = -1; + } + npath = sprintf2("%s/tmp-%ji", cachedir(), (intmax_t)getpid()); + if((fd = open(npath, O_RDWR | O_CREAT | O_TRUNC, 0600)) < 0) { + if(errno != ENOENT) + goto error; + if(!mkdir(cachedir(), 0700)) { + if((fd = open(sprintf3("%s/lastclean", cachedir()), O_WRONLY, O_CREAT, O_TRUNC, 0600)) < 0) + goto error; + close(fd); + } + if((fd = open(npath, O_RDWR | O_CREAT | O_TRUNC, 0600)) < 0) + goto error; + } + if(type->compress(fd, ofd)) { + unlink(npath); + goto error; + } + if(rename(npath, epath)) { + unlink(npath); + goto error; + } + fstat(fd, oinfo); + lseek(fd, 0, SEEK_SET); + checkclean(); + goto out; + +error: + if(fd >= 0) { + close(fd); + fd = -1; + } +out: + free(epath); + if(npath) + free(npath); + return(fd); +} + +static void parsetypes(struct charvbuf *dst, char *head) +{ + char *p; + struct charbuf type; + + p = head; + while(*p) { + for(; *p && isspace(*p); p++); + for(bufinit(type); *p && (*p != ',') && (*p != ';') && !isspace(*p); p++) + bufadd(type, *p); + if(type.b) { + bufadd(type, 0); + bufadd(*dst, type.b); + } + for(; *p && (*p != ','); p++); + if(*p) + p++; + } +} + +int ccopen(char *path, struct stat *sb, char *accept, const char **encoding) +{ + int i, fd, efd, mfd; + off_t minsz; + struct charvbuf types; + struct etype *type, *mtype; + struct stat esb, msb; + + *encoding = NULL; + bufinit(types); + mfd = -1; + if((fd = open(path, O_RDONLY)) < 0) + return(-1); + if(fstat(fd, sb)) + goto error; + if(accept == NULL) + goto out; + parsetypes(&types, accept); + for(i = 0; i < types.d; i++) { + if((type = findtype(types.b[i])) != NULL) { + if((efd = openbytype(type, fd, sb, &esb)) >= 0) { + if((mfd < 0) || (esb.st_size < minsz)) { + if(mfd >= 0) + close(mfd); + mfd = efd; + minsz = esb.st_size; + msb = esb; + mtype = type; + } + } + } + } + if(mfd < 0) + goto out; + close(fd); + *sb = msb; + fd = mfd; + mfd = -1; + *encoding = mtype->name; + goto out; +error: + close(fd); + fd = -1; +out: + if(mfd >= 0) + close(mfd); + for(i = 0; i < types.d; i++) + free(types.b[i]); + buffree(types); + return(fd); +} diff --git a/src/compress.h b/src/compress.h new file mode 100644 index 0000000..7112b1d --- /dev/null +++ b/src/compress.h @@ -0,0 +1,6 @@ +#ifndef SENDFILE_COMPRESS_H +#define SENDFILE_COMPRESS_H + +int ccopen(char *path, struct stat *sb, char *accept, const char **encoding); + +#endif diff --git a/src/psendfile.c b/src/psendfile.c index ae5f7d2..ea00b3e 100644 --- a/src/psendfile.c +++ b/src/psendfile.c @@ -45,6 +45,8 @@ #include #endif +#include "compress.h" + static magic_t cookie; static char *attrmimetype(char *file) @@ -138,10 +140,12 @@ static off_t passdata(FILE *in, FILE *out, off_t max) return(total); } -static void sendwhole(struct hthead *req, FILE *out, FILE *sfile, struct stat *sb, char *contype, int head) +static void sendwhole(struct hthead *req, FILE *out, FILE *sfile, struct stat *sb, char *contype, const char *enctype, int head) { fprintf(out, "HTTP/1.1 200 OK\n"); fprintf(out, "Content-Type: %s\n", contype); + if(enctype != NULL) + fprintf(out, "Content-Encoding: %s\n", enctype); fprintf(out, "Content-Length: %ji\n", (intmax_t)sb->st_size); fprintf(out, "Last-Modified: %s\n", fmthttpdate(sb->st_mtime)); fprintf(out, "Date: %s\n", fmthttpdate(time(NULL))); @@ -150,7 +154,7 @@ static void sendwhole(struct hthead *req, FILE *out, FILE *sfile, struct stat *s passdata(sfile, out, -1); } -static void sendrange(struct hthead *req, FILE *out, FILE *sfile, struct stat *sb, char *contype, char *spec, int head) +static void sendrange(struct hthead *req, FILE *out, FILE *sfile, struct stat *sb, char *contype, const char *enctype, char *spec, int head) { char buf[strlen(spec) + 1]; char *p, *e; @@ -214,7 +218,7 @@ static void sendrange(struct hthead *req, FILE *out, FILE *sfile, struct stat *s return; error: - sendwhole(req, out, sfile, sb, contype, head); + sendwhole(req, out, sfile, sb, contype, enctype, head); } static void serve(struct muth *muth, va_list args) @@ -222,8 +226,9 @@ static void serve(struct muth *muth, va_list args) vavar(struct hthead *, req); vavar(int, fd); FILE *out, *sfile; - int ishead; + int ishead, sfd; char *file, *contype, *hdr; + const char *enctype; struct stat sb; sfile = NULL; @@ -239,8 +244,11 @@ static void serve(struct muth *muth, va_list args) simpleerror2(out, 404, "Not Found", "The requested URL has no corresponding resource."); goto out; } - if(((sfile = fopen(file, "r")) == NULL) || fstat(fileno(sfile), &sb)) { - flog(LOG_ERR, "psendfile: could not stat input file %s: %s", file, strerror(errno)); + hdr = getheader(req, "X-Ash-Compress") ? getheader(req, "Accept-Encoding") : ""; + if(((sfd = ccopen(file, &sb, hdr, &enctype)) < 0) || ((sfile = fdopen(sfd, "r")) == NULL)) { + if(sfd >= 0) + close(sfd); + flog(LOG_ERR, "psendfile: could not open input file %s: %s", file, strerror(errno)); simpleerror2(out, 500, "Internal Error", "The server could not access its own data."); goto out; } @@ -262,9 +270,9 @@ static void serve(struct muth *muth, va_list args) goto out; if((hdr = getheader(req, "Range")) != NULL) - sendrange(req, out, sfile, &sb, contype, hdr, ishead); + sendrange(req, out, sfile, &sb, contype, enctype, hdr, ishead); else - sendwhole(req, out, sfile, &sb, contype, ishead); + sendwhole(req, out, sfile, &sb, contype, enctype, ishead); out: if(sfile != NULL) diff --git a/src/sendfile.c b/src/sendfile.c index 3a81e80..f58ba8f 100644 --- a/src/sendfile.c +++ b/src/sendfile.c @@ -40,6 +40,8 @@ #include #endif +#include "compress.h" + static magic_t cookie = NULL; static void passdata(int in, int out, off_t maxlen) @@ -141,10 +143,12 @@ static void checkcache(char *file, struct stat *sb) } } -static void sendwhole(int fd, struct stat *sb, const char *contype, int head) +static void sendwhole(int fd, struct stat *sb, const char *contype, const char *enctype, int head) { printf("HTTP/1.1 200 OK\n"); printf("Content-Type: %s\n", contype); + if(enctype != NULL) + printf("Content-Encoding: %s\n", enctype); printf("Content-Length: %ji\n", (intmax_t)sb->st_size); printf("Last-Modified: %s\n", fmthttpdate(sb->st_mtime)); printf("Date: %s\n", fmthttpdate(time(NULL))); @@ -154,7 +158,7 @@ static void sendwhole(int fd, struct stat *sb, const char *contype, int head) passdata(fd, 1, -1); } -static void sendrange(int fd, struct stat *sb, const char *contype, char *spec, int head) +static void sendrange(int fd, struct stat *sb, const char *contype, const char *enctype, char *spec, int head) { char buf[strlen(spec) + 1]; char *p, *e; @@ -210,6 +214,8 @@ static void sendrange(int fd, struct stat *sb, const char *contype, char *spec, printf("Content-Range: bytes %ji-%ji/%ji\n", (intmax_t)start, (intmax_t)(end - 1), (intmax_t)sb->st_size); printf("Content-Length: %ji\n", (intmax_t)(end - start)); printf("Content-Type: %s\n", contype); + if(enctype != NULL) + printf("Content-Encoding: %s\n", enctype); printf("Last-Modified: %s\n", fmthttpdate(sb->st_mtime)); printf("Date: %s\n", fmthttpdate(time(NULL))); printf("\n"); @@ -219,7 +225,7 @@ static void sendrange(int fd, struct stat *sb, const char *contype, char *spec, return; error: - sendwhole(fd, sb, contype, head); + sendwhole(fd, sb, contype, enctype, head); } static void usage(void) @@ -233,7 +239,7 @@ int main(int argc, char **argv) char *file, *hdr; struct stat sb; int fd, ishead, ignrest; - const char *contype; + const char *contype, *enctype; setlocale(LC_ALL, ""); contype = NULL; @@ -266,8 +272,9 @@ int main(int argc, char **argv) simpleerror(1, 404, "Not Found", "The requested URL has no corresponding resource."); exit(0); } - if(stat(file, &sb) || ((fd = open(file, O_RDONLY)) < 0)) { - flog(LOG_ERR, "sendfile: could not stat input file %s: %s", file, strerror(errno)); + hdr = getenv("REQ_X_ASH_COMPRESS") ? getenv("REQ_ACCEPT_ENCODING") : ""; + if((fd = ccopen(file, &sb, hdr, &enctype)) < 0) { + flog(LOG_ERR, "sendfile: could not open input file %s: %s", file, strerror(errno)); simpleerror(1, 500, "Internal Error", "The server could not access its own data."); exit(1); } @@ -290,8 +297,8 @@ int main(int argc, char **argv) checkcache(file, &sb); if((hdr = getenv("REQ_RANGE")) != NULL) - sendrange(fd, &sb, contype, hdr, ishead); + sendrange(fd, &sb, contype, enctype, hdr, ishead); else - sendwhole(fd, &sb, contype, ishead); + sendwhole(fd, &sb, contype, enctype, ishead); return(0); }