From 3b0a901275e63ca7a0d46a5f58bdc6d1f706ce06 Mon Sep 17 00:00:00 2001 From: Laurent Bercot Date: Thu, 12 Oct 2023 10:17:47 +0000 Subject: New logging system; not used yet in tipideed Signed-off-by: Laurent Bercot --- doc/tipidee.conf.html | 91 +++++++++++++++++++ doc/tipideed.html | 41 +++++++-- package/deps.mak | 4 +- src/config/defaults.c | 159 ++++++++++++++++++---------------- src/config/lexparse.c | 55 ++++++++++-- src/include/tipidee/log.h | 4 + src/libtipidee/tipidee_log_answer.c | 2 +- src/libtipidee/tipidee_log_resource.c | 4 +- 8 files changed, 264 insertions(+), 96 deletions(-) diff --git a/doc/tipidee.conf.html b/doc/tipidee.conf.html index 73620bc..57c5c29 100644 --- a/doc/tipidee.conf.html +++ b/doc/tipidee.conf.html @@ -322,6 +322,97 @@ only the index.html file will be looked up when a resource resolves to a directory. +
+

The log directive

+
+ +

+ log is also a global directive, but is introduced by the +keyword log, without prepending global. It allows +the user to control what will appear in +tipideed's log output. +

+ +

+ log nothing
+ log keyword1 keyword2 ... +

+ + + +

+ Here are the informational log lines printed by tipideed, +depending on the keywords in the log directive: +

+ +
+
nothing
Don't log anything else than warning and error messages. This +keyword cannot be given with other keywords.
+
start
Log a start line when tipideed starts + and an exit exitcode line when it exits.
+
ip
Add an ip client_ip field to the start line. +This is potentially PII, so make sure to stay compliant with your local laws if you activate it. +client_ip is read from the TCPREMOTEIP environment variable. +This keyword has no effect when given without the start keyword.
+
hostname
Add a host client_hostname field to the start line. +This is potentially PII, so make sure to stay compliant with your local laws if you activate it. +client_hostname is read from the TCPREMOTEHOST environment variable if it exists, or made up from +TCPREMOTEIP otherwise. Make sure to invoke +s6-tcpserver-access before +tipideed in order to get meaningful values for this field. +This keyword has no effect when given without the start keyword.
+
host_as_prefix
Prepend all request, resource and answer +lines with a host host field. This field will not be repeated in the request +line, so it changes the order of the fields. host is the virtual host the request is addressed +to. host_as_prefix is useful when you want to log entries for different virtual hosts to +different locations. For instance, if you're using +s6-log, and want entries for example.com and +example.org to be logged to different backends, you would use the host_as_prefix directive, +and use - +"^tipidee: pid [[:digit:]]*: info: host example\\.com " to select example.com-related +lines, and - +"^tipidee: pid [[:digit:]]*: info: host example\\.org " +to select example.org-related lines. Note that warning and error messages would still need an additional +backend, as well as start and exit lines if you add the start directive +to your log configuration.
+
request
Log a request line when tipideed +receives a request from its client. The line looks like request method host host path "path" +http version. The path is decoded, but if there are non-printable characters in it, they are +encoded as hexadecimal values \0xab. If the request line includes a query, a query query field +is added before the http field.
+
referrer
Add a referrer "referrer" field to the request line, for +requests that include a Referrer: header. referrer is quoted like path, to avoid +malicious clients messing with log lines. +This keyword has no effect when given without the request keyword.
+
user-agent
Add a user-agent "user-agent" field to the request line, for +requests that include a User-Agent: header. user-agent is quoted like path, to avoid +malicious clients messing with log lines. +This keyword has no effect when given without the request keyword.
+
resource
Log a resource line when tipideed +has found a resource corresponding to the URI and is willing to serve it. The line looks like +resource docroot docroot file file type type. docroot is +the document root of the virtual host; file is the path to the served file; type +is nph for an NPH script, cgi for a CGI script, or the Content-Type for a regular file.
+
answer
Log an answer line when tipideed +answers the currently processed request. The line looks like answer status, where status is +the 3-digit status code returned to the client.
+
size
Add a size size field to the answer line, +containing the Content-Length of the answer. +This keyword has no effect when given without the answer keyword.
+
debug
Log debug information. You should not need this in regular use.
+ +

The content-type directive

diff --git a/doc/tipideed.html b/doc/tipideed.html index 8f080bc..977317c 100644 --- a/doc/tipideed.html +++ b/doc/tipideed.html @@ -28,7 +28,7 @@ a web server package: it serves files over HTTP. 
-     tipideed [ -v verbosity ] [ -f cdbfile ] [ -d basedir ] [ -R ] [ -U ]
+     tipideed [ -f cdbfile ] [ -d basedir ] [ -R ] [ -U ]
    @@ -201,11 +201,6 @@ cannot be used by CGI scripts in a portable way.
    -
    -v verbosity
    -
    The level of log verbosity. This is the same as the global verbosity -setting in the configuration file; an explicit -command line option overrides any setting present in the configuration file.
    -
    -f file
    Use file as the compiled configuration database, typically obtained by running tipidee-config -o file. @@ -335,6 +330,40 @@ at all, your documents will most likely be accessible for HTTP/1.0 clients under @:80 or @:443.

    +
    +

    Logging

    +
    + +
      +
    • tipideed uses stderr for all its logging. All its log lines are prefixed +with "tipideed: pid pid: ".
      • +
    • The log lines continue with "fatal: " for fatal error messages (meaning +that tipideed exits right after writing the message), or "warning: " for +warnings (meaning that tipideed continues operating after writing the message). +In normal operation, you should not see any fatal or warning line.
      • +
    • In normal operation, tipidee can log informational lines, and the continuing +prefix is "info: ". It can potentially log: +
        +
      • One line when it starts (i.e. a client has connected)
        • +
      • Up to three lines for every request: +
          +
        • One when the request is received
          • +
        • One when a suitable resource is found
          • +
        • One when an answer is sent
          • +
        • +
      • One line when it exits normally
        • +
      • +
    • What informational lines to log is configured via the +log directive in the +configuration file. By default, only +a minimal request line and an answer line are printed.
      • +
    • The log format is designed to be readable by a human, but still +easily processable by automation. For instance, the regular prefix structure +makes it easy for s6-log +to select different lines to send them to various backends for archiving or +processing.
      • +
    +

    Detailed operation

    diff --git a/package/deps.mak b/package/deps.mak index 016ff9e..bc7bee1 100644 --- a/package/deps.mak +++ b/package/deps.mak @@ -10,8 +10,8 @@ src/include/tipidee/tipidee.h: src/include/tipidee/conf.h src/include/tipidee/co src/tipideed/tipideed-internal.h: src/include/tipidee/tipidee.h src/config/confnode.o src/config/confnode.lo: src/config/confnode.c src/config/tipidee-config-internal.h src/config/conftree.o src/config/conftree.lo: src/config/conftree.c src/config/tipidee-config-internal.h -src/config/defaults.o src/config/defaults.lo: src/config/defaults.c src/config/tipidee-config-internal.h -src/config/lexparse.o src/config/lexparse.lo: src/config/lexparse.c src/config/tipidee-config-internal.h src/include/tipidee/config.h +src/config/defaults.o src/config/defaults.lo: src/config/defaults.c src/config/tipidee-config-internal.h src/include/tipidee/log.h +src/config/lexparse.o src/config/lexparse.lo: src/config/lexparse.c src/config/tipidee-config-internal.h src/include/tipidee/config.h src/include/tipidee/log.h src/config/tipidee-config-preprocess.o src/config/tipidee-config-preprocess.lo: src/config/tipidee-config-preprocess.c src/config/tipidee-config.o src/config/tipidee-config.lo: src/config/tipidee-config.c src/config/tipidee-config-internal.h src/include/tipidee/config.h src/libtipidee/tipidee_conf_free.o src/libtipidee/tipidee_conf_free.lo: src/libtipidee/tipidee_conf_free.c src/include/tipidee/conf.h diff --git a/src/config/defaults.c b/src/config/defaults.c index 390bb81..1017d29 100644 --- a/src/config/defaults.c +++ b/src/config/defaults.c @@ -2,6 +2,9 @@ #include +#include + +#include #include "tipidee-config-internal.h" struct defaults_s @@ -11,88 +14,90 @@ struct defaults_s size_t vlen ; } ; -#define RECB(k, v, n) { .key = k, .value = v, .vlen = n } -#define REC(k, v) RECB(k, v, sizeof(v)) +#define REC(k, v, n) { .key = (k), .value = (v), .vlen = (n) } +#define RECS(k, v) REC(k, v, sizeof(v)) +#define RECU32(k, v) REC(k, (char const *)(uint32_t)UINT32_BIG(v), 4) struct defaults_s const defaults[] = { - RECB("G:verbosity", "\0\0\0\001", 4), - RECB("G:read_timeout", "\0\0\0", 4), - RECB("G:write_timeout", "\0\0\0", 4), - RECB("G:cgi_timeout", "\0\0\0", 4), - RECB("G:max_request_body_length", "\0\0 ", 4), - RECB("G:max_cgi_body_length", "\0@\0", 4), - REC("G:index_file", "index.html"), + RECU32("G:verbosity", 1), + RECU32("G:read_timeout", 0), + RECU32("G:write_timeout", 0), + RECU32("G:cgi_timeout", 0), + RECU32("G:max_request_body_length", 8192), + RECU32("G:max_cgi_body_length", 4194304), + RECS("G:index_file", "index.html"), + RECU32("G:logv", TIPIDEE_LOG_DEFAULT), - REC("T:html", "text/html"), - REC("T:htm", "text/html"), - REC("T:txt", "text/plain"), - REC("T:h", "text/plain"), - REC("T:c", "text/plain"), - REC("T:cc", "text/plain"), - REC("T:cpp", "text/plain"), - REC("T:ass", "text/plain"), - REC("T:java", "text/plain"), - REC("T:mjs", "text/javascript"), - REC("T:css", "text/css"), - REC("T:csv", "text/csv"), - REC("T:sub", "text/vnd.dvb.subtitle"), - REC("T:doc", "application/msword"), - REC("T:docx", "application/vnd.openxmlformats-officedocument.wordprocessingml.document"), - REC("T:js", "application/javascript"), - REC("T:jpg", "image/jpeg"), - REC("T:jpeg", "image/jpeg"), - REC("T:gif", "image/gif"), - REC("T:png", "image/png"), - REC("T:bmp", "image/bmp"), - REC("T:svg", "image/svg+xml"), - REC("T:tif", "image/tiff"), - REC("T:tiff", "image/tiff"), - REC("T:ico", "image/vnd.microsoft.icon"), - REC("T:au", "audio/basic"), - REC("T:aac", "audio/aac"), - REC("T:wav", "audio/wav"), - REC("T:mid", "audio/midi"), - REC("T:midi", "audio/midi"), - REC("T:mp3", "audio/mpeg"), - REC("T:ogg", "audio/ogg"), - REC("T:oga", "audio/ogg"), - REC("T:ogv", "video/ogg"), - REC("T:avi", "video/x-msvideo"), - REC("T:wmv", "video/x-ms-wmv"), - REC("T:qt", "video/quicktime"), - REC("T:mov", "video/quicktime"), - REC("T:mpe", "video/mpeg"), - REC("T:mpeg", "video/mpeg"), - REC("T:mp4", "video/mp4"), - REC("T:mkv", "video/x-matroska"), - REC("T:otf", "font/otf"), - REC("T:ttf", "font/ttf"), - REC("T:epub", "application/epub+zip"), - REC("T:jar", "application/java-archive"), - REC("T:json", "application/json"), - REC("T:jsonld", "application/ld+json"), - REC("T:pdf", "application/pdf"), - REC("T:ppt", "application/vnd.ms-powerpoint"), - REC("T:pptx", "application/vnd.openxmlformats-officedocument.presentationml.presentation"), - REC("T:odp", "application/vnd.oasis.opendocument.presentation"), - REC("T:ods", "application/vnd.oasis.opendocument.spreadsheet"), - REC("T:odt", "application/vnd.oasis.opendocument.text"), - REC("T:oggx", "application/ogg"), - REC("T:rar", "application/vnd.rar"), - REC("T:rtf", "application/rtf"), - REC("T:sh", "application/x-sh"), - REC("T:srt", "application/x-subrip"), - REC("T:tar", "application/x-tar"), - REC("T:xhtml", "application/xhtml+xml"), - REC("T:xls", "application/vnd.ms-excel"), - REC("T:xlsx", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"), - REC("T:xml", "application/xml"), - REC("T:xul", "application/vnd.mozilla.xul+xml"), - REC("T:zip", "application/zip"), - REC("T:7z", "application/x-7z-compressed"), + RECS("T:html", "text/html"), + RECS("T:htm", "text/html"), + RECS("T:txt", "text/plain"), + RECS("T:h", "text/plain"), + RECS("T:c", "text/plain"), + RECS("T:cc", "text/plain"), + RECS("T:cpp", "text/plain"), + RECS("T:ass", "text/plain"), + RECS("T:java", "text/plain"), + RECS("T:mjs", "text/javascript"), + RECS("T:css", "text/css"), + RECS("T:csv", "text/csv"), + RECS("T:sub", "text/vnd.dvb.subtitle"), + RECS("T:doc", "application/msword"), + RECS("T:docx", "application/vnd.openxmlformats-officedocument.wordprocessingml.document"), + RECS("T:js", "application/javascript"), + RECS("T:jpg", "image/jpeg"), + RECS("T:jpeg", "image/jpeg"), + RECS("T:gif", "image/gif"), + RECS("T:png", "image/png"), + RECS("T:bmp", "image/bmp"), + RECS("T:svg", "image/svg+xml"), + RECS("T:tif", "image/tiff"), + RECS("T:tiff", "image/tiff"), + RECS("T:ico", "image/vnd.microsoft.icon"), + RECS("T:au", "audio/basic"), + RECS("T:aac", "audio/aac"), + RECS("T:wav", "audio/wav"), + RECS("T:mid", "audio/midi"), + RECS("T:midi", "audio/midi"), + RECS("T:mp3", "audio/mpeg"), + RECS("T:ogg", "audio/ogg"), + RECS("T:oga", "audio/ogg"), + RECS("T:ogv", "video/ogg"), + RECS("T:avi", "video/x-msvideo"), + RECS("T:wmv", "video/x-ms-wmv"), + RECS("T:qt", "video/quicktime"), + RECS("T:mov", "video/quicktime"), + RECS("T:mpe", "video/mpeg"), + RECS("T:mpeg", "video/mpeg"), + RECS("T:mp4", "video/mp4"), + RECS("T:mkv", "video/x-matroska"), + RECS("T:otf", "font/otf"), + RECS("T:ttf", "font/ttf"), + RECS("T:epub", "application/epub+zip"), + RECS("T:jar", "application/java-archive"), + RECS("T:json", "application/json"), + RECS("T:jsonld", "application/ld+json"), + RECS("T:pdf", "application/pdf"), + RECS("T:ppt", "application/vnd.ms-powerpoint"), + RECS("T:pptx", "application/vnd.openxmlformats-officedocument.presentationml.presentation"), + RECS("T:odp", "application/vnd.oasis.opendocument.presentation"), + RECS("T:ods", "application/vnd.oasis.opendocument.spreadsheet"), + RECS("T:odt", "application/vnd.oasis.opendocument.text"), + RECS("T:oggx", "application/ogg"), + RECS("T:rar", "application/vnd.rar"), + RECS("T:rtf", "application/rtf"), + RECS("T:sh", "application/x-sh"), + RECS("T:srt", "application/x-subrip"), + RECS("T:tar", "application/x-tar"), + RECS("T:xhtml", "application/xhtml+xml"), + RECS("T:xls", "application/vnd.ms-excel"), + RECS("T:xlsx", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"), + RECS("T:xml", "application/xml"), + RECS("T:xul", "application/vnd.mozilla.xul+xml"), + RECS("T:zip", "application/zip"), + RECS("T:7z", "application/x-7z-compressed"), - RECB(0, 0, 0) + REC(0, 0, 0) } ; void conf_defaults (void) diff --git a/src/config/lexparse.c b/src/config/lexparse.c index d541888..b048031 100644 --- a/src/config/lexparse.c +++ b/src/config/lexparse.c @@ -12,10 +12,12 @@ #include #include +#include #include #include "tipidee-config-internal.h" #define dietoobig() strerr_diefu1sys(100, "read configuration") +#define BSEARCH(type, key, array) bsearch(key, (array), sizeof(array)/sizeof(type), sizeof(type), (int (*)(void const *, void const *))&strcmp) typedef struct mdt_s mdt, *mdt_ref ; struct mdt_s @@ -33,15 +35,17 @@ struct globalkey_s uint32_t type ; } ; -static int globalkey_cmp (void const *a, void const *b) +struct logkey_s { - return strcmp((char const *)a, ((struct globalkey_s const *)b)->name) ; -} + char const *name ; + uint32_t value ; +} ; enum token_e { T_BANG, T_GLOBAL, + T_LOG, T_CONTENTTYPE, T_DOMAIN, T_NPHPREFIX, @@ -61,10 +65,6 @@ struct directive_s enum token_e token ; } ; -static int directive_cmp (void const *a, void const *b) -{ - return strcmp((char const *)a, ((struct directive_s const *)b)->s) ; -} static void check_unique (char const *key, mdt const *md) { @@ -101,7 +101,7 @@ static inline void parse_global (char const *s, size_t const *word, size_t n, md struct globalkey_s const *gl ; if (n < 2) strerr_dief8x(1, "too ", "few", " arguments to directive ", "global", " in file ", g.storage.s + md->filepos, " line ", md->linefmt) ; - gl = bsearch(s + word[0], globalkeys, sizeof(globalkeys)/sizeof(struct globalkey_s), sizeof(struct globalkey_s), &globalkey_cmp) ; + gl = BSEARCH(struct globalkey_s, s + word[0], globalkeys) ; if (!gl) strerr_dief6x(1, "unrecognized global setting ", s + word[0], " in file ", g.storage.s + md->filepos, " line ", md->linefmt) ; switch (gl->type) { @@ -130,6 +130,39 @@ static inline void parse_global (char const *s, size_t const *word, size_t n, md } } +static inline void parse_log (char const *s, size_t const *word, size_t n, mdt const *md) +{ + static struct logkey_s const logkeys[] = + { + { .name = "answer", .value = TIPIDEE_LOG_ANSWER }, + { .name = "debug", .value = TIPIDEE_LOG_DEBUG }, + { .name = "host_as_prefix", .value = TIPIDEE_LOG_HOSTASPREFIX }, + { .name = "hostname", .value = TIPIDEE_LOG_CLIENTHOST }, + { .name = "ip", .value = TIPIDEE_LOG_CLIENTIP }, + { .name = "nothing", .value = 0 }, + { .name = "referrer", .value = TIPIDEE_LOG_REFERRER }, + { .name = "request", .value = TIPIDEE_LOG_REQUEST }, + { .name = "resource", .value = TIPIDEE_LOG_RESOURCE }, + { .name = "response_size", .value = TIPIDEE_LOG_SIZE }, + { .name = "start", .value = TIPIDEE_LOG_START }, + { .name = "user-agent", .value = TIPIDEE_LOG_UA } + } ; + uint32_t v = 0 ; + char pack[4] ; + if (!n) + strerr_dief8x(1, "too ", "few", " arguments to directive ", "log", " in file ", g.storage.s + md->filepos, " line ", md->linefmt) ; + for (size_t i = 0 ; i < n ; i++) + { + struct logkey_s const *l = BSEARCH(struct logkey_s, s + word[i], logkeys) ; + if (!l) strerr_dief6x(1, "unrecognized log setting ", s + word[i], " in file ", g.storage.s + md->filepos, " line ", md->linefmt) ; + if (l->value) v |= l->value ; + else if (n == 1) v = 0 ; + else strerr_dief5x(1, "log nothing cannot be mixed with other log values", " in file ", g.storage.s + md->filepos, " line ", md->linefmt) ; + } + uint32_pack_big(pack, v) ; + add_unique("G:logv", pack, 4, md) ; +} + static inline void parse_contenttype (char const *s, size_t const *word, size_t n, mdt const *md) { char const *ct ; @@ -295,6 +328,7 @@ static inline void process_line (char const *s, size_t const *word, size_t n, st { .s = "domain", .token = T_DOMAIN }, { .s = "file-type", .token = T_FILETYPE }, { .s = "global", .token = T_GLOBAL }, + { .s = "log", .token = T_LOG }, { .s = "no-auth", .token = T_NOAUTH }, { .s = "noncgi", .token = T_NONCGI }, { .s = "nonnph", .token = T_NONNPH }, @@ -306,7 +340,7 @@ static inline void process_line (char const *s, size_t const *word, size_t n, st char const *word0 ; if (!n--) return ; word0 = s + *word++ ; - directive = bsearch(word0, directives, sizeof(directives)/sizeof(struct directive_s), sizeof(struct directive_s), &directive_cmp) ; + directive = BSEARCH(struct directive_s, word0, directives) ; if (!directive) strerr_dief6x(1, "unrecognized word ", word0, " in file ", g.storage.s + md->filepos, " line ", md->linefmt) ; switch (directive->token) @@ -328,6 +362,9 @@ static inline void process_line (char const *s, size_t const *word, size_t n, st case T_GLOBAL : parse_global(s, word, n, md) ; break ; + case T_LOG : + parse_log(s, word, n, md) ; + break ; case T_CONTENTTYPE : parse_contenttype(s, word, n, md) ; break ; diff --git a/src/include/tipidee/log.h b/src/include/tipidee/log.h index 29d621f..98e6d0e 100644 --- a/src/include/tipidee/log.h +++ b/src/include/tipidee/log.h @@ -6,6 +6,7 @@ #include #include +#include #include #include @@ -20,6 +21,7 @@ #define TIPIDEE_LOG_CLIENTIP 0x0200 #define TIPIDEE_LOG_CLIENTHOST 0x0400 #define TIPIDEE_LOG_HOSTASPREFIX 0x1000 +#define TIPIDEE_LOG_DEBUG 0x10000 #define TIPIDEE_LOG_DEFAULT (TIPIDEE_LOG_REQUEST | TIPIDEE_LOG_ANSWER | TIPIDEE_LOG_SIZE) @@ -39,4 +41,6 @@ extern void tipidee_log_request (uint32_t, tipidee_rql const *, char const *, ch extern void tipidee_log_resource (uint32_t, tipidee_rql const *, char const *, char const *, tipidee_resattr const *) ; extern void tipidee_log_answer (uint32_t, tipidee_rql const *, unsigned int, off_t) ; +#define tipidee_log_debug(v, ...) do { if ((v) & TIPIDEE_LOG_DEBUG) strerr_warn(PROG, ": debug: ", __VA_ARGS__) ; } while (0) + #endif diff --git a/src/libtipidee/tipidee_log_answer.c b/src/libtipidee/tipidee_log_answer.c index b6e4617..ae4e880 100644 --- a/src/libtipidee/tipidee_log_answer.c +++ b/src/libtipidee/tipidee_log_answer.c @@ -1,6 +1,6 @@ /* ISC license. */ -#include +#include #include #include diff --git a/src/libtipidee/tipidee_log_resource.c b/src/libtipidee/tipidee_log_resource.c index 6387027..b730612 100644 --- a/src/libtipidee/tipidee_log_resource.c +++ b/src/libtipidee/tipidee_log_resource.c @@ -1,5 +1,7 @@ /* ISC license. */ +#include + #include #include @@ -14,7 +16,7 @@ void tipidee_log_resource (uint32_t v, tipidee_rql const *rql, char const *docro a[m++] = " host " ; a[m++] = rql->uri.host ; } - a[m++] = " docroot " ; + a[m++] = " resource docroot " ; a[m++] = docroot ; a[m++] = " file " ; a[m++] = file ; -- cgit v1.2.3