diff options
26 files changed, 514 insertions, 356 deletions
diff --git a/package/deps.mak b/package/deps.mak index 19bdf1a..a9813a3 100644 --- a/package/deps.mak +++ b/package/deps.mak @@ -13,7 +13,7 @@ src/cache/clientaccess.o src/cache/clientaccess.lo: src/cache/clientaccess.c src src/cache/conf.o src/cache/conf.lo: src/cache/conf.c src/cache/shibari-cache-internal.h src/cache/dns.o src/cache/dns.lo: src/cache/dns.c src/cache/shibari-cache-internal.h src/cache/log.o src/cache/log.lo: src/cache/log.c src/cache/shibari-cache-internal.h -src/cache/query.o src/cache/query.lo: src/cache/query.c src/cache/shibari-cache-internal.h +src/cache/query.o src/cache/query.lo: src/cache/query.c src/cache/shibari-cache-internal.h src/include/shibari/dcache.h src/cache/shibari-cache.o src/cache/shibari-cache.lo: src/cache/shibari-cache.c src/cache/shibari-cache-internal.h src/include/shibari/common.h src/include/shibari/config.h src/cache/tcpconnection.o src/cache/tcpconnection.lo: src/cache/tcpconnection.c src/cache/shibari-cache-internal.h src/cache/udpqueue.o src/cache/udpqueue.lo: src/cache/udpqueue.c src/cache/shibari-cache-internal.h @@ -32,14 +32,16 @@ src/config/lexparse.o src/config/lexparse.lo: src/config/lexparse.c src/config/s src/config/repo.o src/config/repo.lo: src/config/repo.c src/config/shibari-cache-config-internal.h src/config/shibari-cache-config.o src/config/shibari-cache-config.lo: src/config/shibari-cache-config.c src/config/shibari-cache-config-internal.h src/include/shibari/config.h src/config/util.o src/config/util.lo: src/config/util.c src/config/shibari-cache-config-internal.h -src/libdcache/dcache_add.o src/libdcache/dcache_add.lo: src/libdcache/dcache_add.c src/libdcache/dcache-internal.h src/include/shibari/dcache.h +src/libdcache/dcache_add_data.o src/libdcache/dcache_add_data.lo: src/libdcache/dcache_add_data.c src/libdcache/dcache-internal.h src/include/shibari/dcache.h src/libdcache/dcache_clean_expired.o src/libdcache/dcache_clean_expired.lo: src/libdcache/dcache_clean_expired.c src/libdcache/dcache-internal.h src/include/shibari/dcache.h src/libdcache/dcache_delete.o src/libdcache/dcache_delete.lo: src/libdcache/dcache_delete.c src/libdcache/dcache-internal.h src/include/shibari/dcache.h src/libdcache/dcache_free.o src/libdcache/dcache_free.lo: src/libdcache/dcache_free.c src/include/shibari/dcache.h +src/libdcache/dcache_get_data.o src/libdcache/dcache_get_data.lo: src/libdcache/dcache_get_data.c src/include/shibari/dcache.h src/libdcache/dcache_init.o src/libdcache/dcache_init.lo: src/libdcache/dcache_init.c src/include/shibari/dcache.h src/libdcache/dcache_load.o src/libdcache/dcache_load.lo: src/libdcache/dcache_load.c src/include/shibari/dcache.h +src/libdcache/dcache_node_new.o src/libdcache/dcache_node_new.lo: src/libdcache/dcache_node_new.c src/libdcache/dcache-internal.h src/include/shibari/dcache.h src/libdcache/dcache_save.o src/libdcache/dcache_save.lo: src/libdcache/dcache_save.c src/include/shibari/dcache.h -src/libdcache/dcache_search.o src/libdcache/dcache_search.lo: src/libdcache/dcache_search.c src/libdcache/dcache-internal.h src/include/shibari/dcache.h +src/libdcache/dcache_searchnode.o src/libdcache/dcache_searchnode.lo: src/libdcache/dcache_searchnode.c src/libdcache/dcache-internal.h src/include/shibari/dcache.h src/server/shibari-server-tcp.o src/server/shibari-server-tcp.lo: src/server/shibari-server-tcp.c src/include/shibari/common.h src/include/shibari/server.h src/server/shibari-server-udp.o src/server/shibari-server-udp.lo: src/server/shibari-server-udp.c src/include/shibari/common.h src/include/shibari/server.h src/server/shibari_packet_add_glue.o src/server/shibari_packet_add_glue.lo: src/server/shibari_packet_add_glue.c src/include/shibari/constants.h src/include/shibari/packet.h src/include/shibari/tdb.h src/include/shibari/util.h @@ -67,12 +69,12 @@ libshibari-common.so.xyzzy: src/common/shibari_log_answer.lo src/common/shibari_ shibari-cache-config: EXTRA_LIBS := -ls6dns -lskarnet shibari-cache-config: src/config/shibari-cache-config.o src/config/repo.o src/config/defaults.o src/config/lexparse.o src/config/util.o ifeq ($(strip $(STATIC_LIBS_ARE_PIC)),) -libdcache.a.xyzzy: src/libdcache/dcache_add.o src/libdcache/dcache_clean_expired.o src/libdcache/dcache_delete.o src/libdcache/dcache_free.o src/libdcache/dcache_init.o src/libdcache/dcache_load.o src/libdcache/dcache_save.o src/libdcache/dcache_search.o +libdcache.a.xyzzy: src/libdcache/dcache_add_data.o src/libdcache/dcache_clean_expired.o src/libdcache/dcache_delete.o src/libdcache/dcache_free.o src/libdcache/dcache_init.o src/libdcache/dcache_load.o src/libdcache/dcache_save.o else -libdcache.a.xyzzy: src/libdcache/dcache_add.lo src/libdcache/dcache_clean_expired.lo src/libdcache/dcache_delete.lo src/libdcache/dcache_free.lo src/libdcache/dcache_init.lo src/libdcache/dcache_load.lo src/libdcache/dcache_save.lo src/libdcache/dcache_search.lo +libdcache.a.xyzzy: src/libdcache/dcache_add_data.lo src/libdcache/dcache_clean_expired.lo src/libdcache/dcache_delete.lo src/libdcache/dcache_free.lo src/libdcache/dcache_init.lo src/libdcache/dcache_load.lo src/libdcache/dcache_save.lo endif libdcache.so.xyzzy: EXTRA_LIBS := -libdcache.so.xyzzy: src/libdcache/dcache_add.lo src/libdcache/dcache_clean_expired.lo src/libdcache/dcache_delete.lo src/libdcache/dcache_free.lo src/libdcache/dcache_init.lo src/libdcache/dcache_load.lo src/libdcache/dcache_save.lo src/libdcache/dcache_search.lo +libdcache.so.xyzzy: src/libdcache/dcache_add_data.lo src/libdcache/dcache_clean_expired.lo src/libdcache/dcache_delete.lo src/libdcache/dcache_free.lo src/libdcache/dcache_init.lo src/libdcache/dcache_load.lo src/libdcache/dcache_save.lo ifeq ($(strip $(STATIC_LIBS_ARE_PIC)),) libshibari-server.a.xyzzy: src/server/shibari_packet_init.o src/server/shibari_packet_begin.o src/server/shibari_packet_end.o src/server/shibari_packet_add_rr.o src/server/shibari_tdb_entry_parse.o src/server/shibari_tdb_extract_domain.o src/server/shibari_tdb_find_authority.o src/server/shibari_tdb_read_entry.o src/server/shibari_packet_add_glue.o src/server/shibari_packet_assert_authority.o src/server/shibari_packet_tdb_answer_query.o src/server/shibari_packet_tdb_axfr.o else diff --git a/package/targets.mak b/package/targets.mak index 42e8691..094fa88 100644 --- a/package/targets.mak +++ b/package/targets.mak @@ -1,8 +1,8 @@ BIN_TARGETS := \ shibari-server-tcp \ shibari-server-udp \ -shibari-cache-config \ -shibari-cache +# shibari-cache \ +shibari-cache-config LIBEXEC_TARGETS := diff --git a/src/cache/deps-exe/shibari-cache b/src/cache/deps-exe/shibari-cache index e363f19..44d74a0 100644 --- a/src/cache/deps-exe/shibari-cache +++ b/src/cache/deps-exe/shibari-cache @@ -1,6 +1,7 @@ cache.o clientaccess.o conf.o +dns.o log.o query.o tcpconnection.o diff --git a/src/cache/dns.c b/src/cache/dns.c new file mode 100644 index 0000000..f027180 --- /dev/null +++ b/src/cache/dns.c @@ -0,0 +1,89 @@ +/* ISC license. */ + +#include <stdint.h> + +#include <skalibs/uint16.h> +#include <skalibs/uint32.h> + + +#include <s6-dns/s6dns.h> + +#include "shibari-cache-internal.h" + +int dns_ask (dcache_string *answer, uint8_t source, uint16_t sid, char const *q, uint16_t qlen, uint16_t qtype) +{ + uint32_t i ; + int r = dcache_searchnode_g(&g->cache, &i, q, qlen, qtype) ; + if (r > 0) + { + dcache_string data ; + dcache_node_get_data(&g->cache, i, answer) ; + return 1 ; + } + else if (!r) + { + dns_task_new() + } +} + +uint16_t dnstask_new (uint8_t source, uint16_t sid, char const *ip, uint16_t port, uint16_t qid, char const *q, uint16_t qlen, uint16_t qtype) +{ + uint16_t i = genset_new(&g->dnstasks) ; + dnstask *task = DNSTASK(i) ; + if (!stralloc_ready(&task->sa, 6 + qlen)) dienomem() ; + task->source = source ; + task->sid = sid ; + if (source < 2) + { + memcpy(task->ip, ip, source ? 16 : 4) ; + task->port = port ; + } + else + { + memset(task->ip, 0, SKALIBS_IP_SIZE) ; + task->port = 0 ; + } + task->spin = 0 ; + task->prefixlen = 0 ; + uint16_pack_big(task->sa.s, qtype) ; + memcpy(task->sa.s + 2, q, qlen) ; + uint32_pack_big(task->sa.s + 2 + qlen, 6 + qlen) ; + task->sa.len = 6 + qlen ; + return i ; +} + +int dns_start_query (uint8_t source, uint16_t sid, char const *ip, uint16_t port, uint16_t qid, char const *q, uint16_t qlen, uint16_t qtype) +{ + uint16_t tid = dnstask_new(source, sid, ip, port, qid, q, qlen, qtype) ; + dnstask *task = DNSTASK(i) ; + +} + + +int dns_start (uint8_t source, uint16_t i, char const *ip, uint16_t port, char const *buf, uint16_t len) +{ + s6dns_message_header_t hdr ; + s6dns_message_counts_t counts ; + s6dns_domain_t name ; + unsigned int pos ; + unsigned int rcode ; + int r; + uint16_t qtype ; + + if (!s6dns_message_parse_init(&hdr, &counts, buf, len, &pos) + || !s6dns_message_parse_question(&counts, &name, &qtype, buf, len, &pos) + || !s6dns_domain_encode(&name) + || counts.qd || counts.an || counts.ns || counts.nr) + return 0 ; + if (hdr.opcode) return dns_error(source, i, ip, port, &name, qtype, hdr.id, 4) ; + if (!hdr.rd) return dns_error(source, i, ip, port, &name, qtype, hdr.id, 9) ; + + r = dns_ask(&data, source, i, name.s, name.len, qtype) ; + return r > 0 ? + dns_need_glue(name.s, name.len, qtype, data.s, data.len) ? + dns_start_glue(source, i, ip, port, hdr.id, name.s, name.len, qtype, data.s, data.len) : + dns_answer(source, i, ip, port, hdr.id, name.s, name.len, qtype, data.s, data.len) : + r < 0 ? + dns_start_query(source, i, ip, port, hdr.id, name.s, name.len, qtype) : + dns_start_wait(source, i, ip, port, hdr.id, name.s, name.len, qtype, (dcache_node *)data.s) ; +} diff --git a/src/cache/log.c b/src/cache/log.c index 0e20f91..06da0ec 100644 --- a/src/cache/log.c +++ b/src/cache/log.c @@ -1,4 +1,6 @@ -/* ISC license. */ +/* ISC license. */ + +#include <stdint.h> #include <skalibs/fmtscan.h> #include <skalibs/strerr.h> @@ -7,7 +9,6 @@ #include "shibari-cache-internal.h" - void log_udp4bad (char const *ip, uint16_t port) { if (g->verbosity >= 3) @@ -53,3 +54,11 @@ void log_newtcp6 (char const *ip, uint16_t port) } #endif + +void log_warn_unexpected_answer(char const *q, uint16_t qlen, uint16_t qtype, int present) +{ + if (g->verbosity) + { + } +} + diff --git a/src/cache/query.c b/src/cache/query.c index 7fb27c4..7fa5af3 100644 --- a/src/cache/query.c +++ b/src/cache/query.c @@ -1,13 +1,25 @@ /* ISC license. */ #include <stdint.h> +#include <string.h> +#include <errno.h> -#include <s6-dns/s6dns.h> +#include <skalibs/bitarray.h> +#include <skalibs/error.h> +#include <skalibs/ip46.h> +#include <skalibs/tai.h> +#include <skalibs/random.h> +#include <skalibs/gensetdyn.h> -#include <shibari/constants.h> +#include <s6-dns/s6dns-engine.h> +#include <s6-dns/s6dns-ip46.h> + +#include <shibari/dcache.h> #include "shibari-cache-internal.h" -static uint16_t query_delete (query *q) +#include <skalibs/posixishard.h> + +static inline uint16_t query_delete (query *q) { uint16_t newi = q->prev ; QUERY(newi)->next = q->next ; @@ -17,105 +29,88 @@ static uint16_t query_delete (query *q) return newi ; } -uint16_t query_abort (uint16_t id) -{ - query *q = QUERY(id) ; - s6dns_engine_recycle(&q->dt) ; - return query_delete(q) ; -} - -uint16_t query_fail (uint16_t id) -{ - query *q = QUERY(id) ; - - if (q->source == 2) tcpconnection_removequery(TCPCONNECTION(q->i), id) ; - return query_delete(q) ; -} - -uint16_t query_succeed (uint16_t id) -{ - query *q = QUERY(id) ; - - if (q->source == 2) tcpconnection_removequery(TCPCONNECTION(q->i), id) ; - return query_delete(q) ; -} - -int query_end (uint8_t source, uint16_t i, char const *ip, uint16_t port, char const *buf, uint16_t len) +uint16_t query_event (uint16_t qid) { - return source < 2 ? - udpqueue_add(g->udpqueues[source] + i, source, ip, port, buf, len) : - tcpconnection_add(g->tcpconnections + i, buf, len) ; -} - -int query_error (uint8_t source, uint16_t i, char const *ip, uint16_t port, s6dns_domain_t *name, uint16_t qtype, uint16_t id, unsigned int rcode) -{ - s6dns_message_header_t hdr = S6DNS_MESSAGE_HEADER_ZERO ; - unsigned int pos = 12 ; - char pkt[name->len + 16] ; - hdr.id = id ; - hdr.qr = 1 ; - hdr.ra = 1 ; - hdr.rcode = rcode ; - hdr.counts.qd = 1 ; - s6dns_message_header_pack(pkt, &hdr) ; - memcpy(pkt + pos, name->s, name->len) ; pos += name->len ; - uint16_pack_big(pkt + pos, qtype) ; pos += 2 ; - uint16_pack_big(pkt + pos, SHIBARI_C_IN) ; pos += 2 ; - return query_end(source, i, ip, port, pkt, pos) ; -} - -static void query_init (query *q, uint8_t source, uint16_t i, char const *ip, uint16_t port, s6dns_domain_t const *name, uint16_t qtype) -{ - q->source = source ; - q->i = i ; - if (source < 2) + query *q = QUERY(qid) ; + dcache_string question ; + int r ; + uint32_t nodeid ; + uint16_t qtype ; + uint16_t rcode = 0 ; + switch (q->dt.status) + { + case EAGAIN : + case EWOULDBLOCK : return qid ; + case 0 : break ; + case EOPNOTSUP : rcode = 4 ; break ; + case EPROTO : rcode = 1 ; break ; + default : rcode = 2 ; break ; + } + s6dns_engine_query(&q->dt, &question.s, &question.len, &qtype) ; + r = dcache_searchnode_g(&g->dcache, &nodeid, question.s, question.len, qtype) ; + switch (r) { - memcpy(q->ip, ip, source ? 16 : 4) ; - q->port = port ; + case -1 : + log_warn_unexpected_answer(question.s, question.len, qtype, 0) ; + if (!rcode) dcache_add_new_answer(&g->dcache, question.s, question.len, qtype, s6dns_engine_packet(&q->dt), s6dns_engine_packetlen(&q->dt)) ; + break ; + case 1 : + log_warn_unexpected_answer(question.s, question.len, qtype, 1) ; + if (!rcode) dcache_refresh_answer(&g->dcache, nodeid, s6dns_engine_packet(&q->dt), s6dns_engine_packetlen(&q->dt)) ; + break ; + case 0 : + { + uint16_t n = dcache_get_taskn(&g->cache, nodeid) ; + uint16_t tasks[n ? n : 1] ; + dcache_get_tasks(&g->cache, nodeid, tasks, taskn) ; + if (rcode) dcache_delete(&g->cache, nodeid) ; + else + { + dcache_add_answer(&g->dcache, nodeid, s6dns_engine_packet(&q->dt), s6dns_engine_packetlen(&q->dt)) ; + s6dns_engine_recycle(&q->dt) ; + } + for (uint16_t i = 0 ; i < n ; i++) dnstask_wakeup(tasks[i], rcode, nodeid) ; + break ; + } } - q->port = port ; - if (!stralloc_catb(&q->qname, name->s, name->len)) dienomem() ; - q->qtype = qtype ; - q->prefixlen = 0 ; + return query_delete(q) ; } -static query *query_new (uint8_t source, uint16_t i, char const *ip, uint16_t port, s6dns_domain_t const *name, uint16_t qtype) +static inline uint16_t query_new (void) { - uint16_t n = genset_new(&g->queries) ; + uint32_t qid ; + if (!gensetdyn_new(&g->queries, &qid) || n > UINT16_MAX) dienomem() ; query *sentinel = QUERY(g->qsentinel) ; - query *q = QUERY(n) ; - query_init(q, source, i, ip, port, name, type) ; + query *q = QUERY(qid) ; q->prev = g->qsentinel ; q->next = sentinel->next ; - QUERY(sentinel->next)->prev = n ; - sentinel->next = n ; - return q ; + QUERY(sentinel->next)->prev = qid ; + sentinel->next = qid ; + return qid ; } -int query_start (uint8_t source, uint16_t i, char const *ip, uint16_t port, char const *buf, uint16_t len) +uint16_t query_start (uint16_t tid, char const *q, uint16_t qlen, uint16_t qtype, char const *ip4, uint16_t n4, char const *ip6, uint16_t n6, uint32_t flags) { - dcache_key_t data ; - s6dns_message_header_t hdr ; - s6dns_message_counts_t counts ; - s6dns_domain_t name ; - unsigned int pos ; - unsigned int rcode ; - uint16_t qtype ; - - if (!s6dns_message_parse_init(&hdr, &counts, buf, len, &pos) - || !s6dns_message_parse_question(&counts, &name, &qtype, buf, len, &pos) - || !s6dns_domain_encode(&name)) return 0 ; - if (hdr.opcode) return query_error(source, i, ip, port, &name, qtype, hdr.id, 4) ; - if (!hdr.rd) return query_error(source, i, ip, port, &name, qtype, hdr.id, 9) ; - - if (cache_search(&name, qtype, &data)) - return query_end(source, i, ip, port, data.s, data.len) ; - + query *p ; + tain qdeadline ; + uint16_t qid ; + uint16_t n = n4 + n6 ; + s6dns_ip46list_t servers = S6DNS_IP46LIST_ZERO ; /* TODO: away with all this goofiness */ { - uint16_t j = genset_new(&g->queries) ; - query *q = QUERY(j) ; + ip46 list[n] ; + for (uint16_t i = 0 ; i < n4 ; i++) ip46_from_ip4(list + i, ip4 + (i<<2)) ; + for (uint16_t i = 0 ; i < n6 ; i++) ip46_from_ip6(list + n4 + i, ip6 + (i<<4)) ; + random_unsort(list, n, sizeof(ip46)) ; + for (uint16_t i = 0 ; i < n ; i++) + { + memcpy(servers.ip + i * SKALIBS_IP_SIZE, list[i].ip, ip46_is6(list + i) ? 16 : 4) ; + if (ip46_is6(list + i)) bitarray_set(servers.is6, i) ; + } } - - return 1 ; + tain_add_g(&qdeadline, &g->qtto) ; + qid = query_new() ; + p = QUERY(qid) ; + if (!dcache_add_new_entry(&g->cache, q, qlen, qtype, tid)) dienomem() ; + if (!s6dns_engine_init_g(&q->dt, servers, flags, q, qlen, qtype, &qdeadline)) dienewquery() ; + return qid ; } - diff --git a/src/cache/shibari-cache-internal.h b/src/cache/shibari-cache-internal.h index 950bfdc..c4208f9 100644 --- a/src/cache/shibari-cache-internal.h +++ b/src/cache/shibari-cache-internal.h @@ -22,6 +22,7 @@ #include <shibari/dcache.h> #define dienomem() strerr_diefu1sys(111, "concatenate data") ; +#define dienewquery() strerr_diefu1sys(111, "prepare new query") ; /* cache */ @@ -50,6 +51,29 @@ extern int conf_get_uint64 (char const *, uint64_t *) ; extern char const *conf_get_string (char const *) ; + /* dns */ + +typedef struct dnstask_s dnstask, *dnstask_ref ; +struct dnstask_s +{ + stralloc sa ; + char ip[SKALIBS_IP_SIZE] ; + uint16_t qtype ; + uint16_t prefixlen ; + uint16_t state : 13 ; + uint16_t spin : 1 ; + uint16_t source : 2 ; + uint16_t sid ; + uint16 query[2] ; +} ; +#define DNSTASK_ZERO { .sa = STRALLOC_ZERO, .ip = { 0 }, .qtype = 0, .prefixlen = 0, .state = 0, .spin = 0, .source = 0, .sid = 0, query = { 0, 0 } } +#define ntasks genset_n(&g->tasks) +#define DNSTASK(i) genset_p(dnstask, &g->dnstasks, (i)) + +extern void dnstask_wakeup (uint16_t, uint16_t, uint32_t) ; +extern int dns_start (uint8_t, uint16_t, char const *, uint16_t, char const *, uint16_t) ; + + /* log */ extern void log_udp4bad (char const *, uint16_t) ; @@ -68,28 +92,27 @@ typedef struct query_s query, *query_ref ; struct query_s { s6dns_engine_t dt ; - stralloc qname ; uint16_t prev ; uint16_t next ; uint16_t xindex ; - uint16_t i ; - uint16_t port ; - uint16_t qtype ; - uint8_t source ; - char ip[SKALIBS_IP_SIZE] ; } ; -#define QUERY_ZERO { .dt = S6DNS_ENGINE_ZERO, .qname = STRALLOC_ZERO, .prev = 0, .next = 0, .xindex = UINT16_MAX, .i = 0, .port = 0, qtype = 0, name = 0, .source = 0, .ip = { 0 } } -#define nq (genset_n(&g->queries) - 1) -#define QUERY(i) genset_p(query, &g->queries, (i)) +#define QUERY_ZERO { .dt = S6DNS_ENGINE_ZERO, .prev = 0, .next = 0, .xindex = UINT16_MAX } +#define nq (gensetdyn_n(&g->queries) - 1) +#define QUERY(i) GENSETDYN_P(query, &g->queries, (i)) #define qstart (QUERY(g->qsentinel)->next) extern uint16_t query_abort (uint16_t) ; -extern uint16_t query_fail (uint16_t) ; -extern uint16_t query_succeed (uint16_t) ; +extern uint16_t query_event (uint16_t) ; -extern int query_start (uint8_t, uint16_t, char const *, uint16_t, char const *, uint16_t) ; +extern int query_start (uint16_t, char const *, uint16_t, uint16_t, char const *, uint16_t, char const *, uint16_t, uint32_t) ; extern int query_end (uint8_t, uint16_t, char const *, uint16_t, char const *, uint16_t) ; + + /* dns */ + +extern void dns_start (query *) ; + + /* tcpconnection */ typedef struct tcpconnection_s tcpconnection, *tcpconnection_ref ; @@ -100,7 +123,7 @@ struct tcpconnection_s uint32_t instate ; tain rdeadline ; tain wdeadline ; - genalloc queries ; /* uint16_t */ + genalloc tasks ; /* uint16_t */ uint16_t prev ; uint16_t next ; uint16_t xindex ; @@ -150,26 +173,30 @@ struct global_s { cdb confdb ; char const *dumpfile ; - uint16_t verbosity ; tain rtto ; tain wtto ; + tain qtto ; udpqueue *udpqueues[2] ; - genset tcpconnections ; /* tcpconnection */ - genset queries ; /* query */ + genset tcpconnections ; + genset dnstasks ; + gensetdyn queries ; uint16_t tcpsentinel ; uint16_t qsentinel ; + uint16_t verbosity ; } ; #define GLOBAL_ZERO { \ .confdb = CDB_ZERO, \ .dumpfile = 0, \ - .verbosity = 1, \ .rtto = TAIN_INFINITE, \ .wtto = TAIN_INFINITE, \ + .qtto = TAIN_INFINITE, \ .udpqueues = { 0, 0 }, \ .tcpconnections = GENSET_ZERO, \ - .queries = GENSET_ZERO, \ + .dnstasks = GENSET_ZERO, \ + .queries = GENSETDYN_INIT(query, 3, 3, 8), \ .tcpsentinel = 0, \ .qsentinel = 0, \ + .verbosity = 1, \ } extern global *g ; diff --git a/src/cache/shibari-cache.c b/src/cache/shibari-cache.c index d9391f2..54abeae 100644 --- a/src/cache/shibari-cache.c +++ b/src/cache/shibari-cache.c @@ -45,7 +45,7 @@ static int flagwantfinaldump = 1 ; static unsigned int cont = 2 ; -static inline void conf_init (char const *conffile, uint16_t *n4, uint16_t *n6, char const **ip4, char const **ip6, uint16_t *maxtcp, uint16_t *maxqueries) +static inline void conf_init (char const *conffile, uint16_t *n4, uint16_t *n6, char const **ip4, char const **ip6, uint16_t *maxtcp, uint16_t *maxtasks) { cdb_data data ; uint32_t u ; @@ -63,16 +63,19 @@ static inline void conf_init (char const *conffile, uint16_t *n4, uint16_t *n6, strerr_diefu4sys(102, "read ", "G:maxtcp", " configuration key from ", conffile) ; if (*maxtcp > 4096 || *maxtcp < 1) strerr_dief2x(102, "invalid G:maxtcp in ", conffile) ; - if (!conf_get_uint16("G:maxqueries", maxqueries)) - strerr_diefu4sys(102, "read ", "G:maxqueries", " configuration key from ", conffile) ; - if (*maxqueries > 8192 || *maxqueries < 1) - strerr_dief2x(102, "invalid G:maxqueries in ", conffile) ; + if (!conf_get_uint16("G:maxtasks", maxtasks)) + strerr_diefu4sys(102, "read ", "G:maxtasks", " configuration key from ", conffile) ; + if (*maxtasks > 8192 || *maxtasks < 1) + strerr_dief2x(102, "invalid G:maxtasks in ", conffile) ; if (!conf_get_uint32("G:rtimeout", &u)) if (u) tain_from_millisecs(&g->rtto, u) ; strerr_diefu4sys(102, "read ", "G:rtimeout", " configuration key from ", conffile) ; if (!conf_get_uint32("G:wtimeout", &u)) strerr_diefu4sys(102, "read ", "G:wtimeout", " configuration key from ", conffile) ; if (u) tain_from_millisecs(&g->wtto, u) ; + if (!conf_get_uint32("G:qtimeout", &u)) + strerr_diefu4sys(102, "read ", "G:qtimeout", " configuration key from ", conffile) ; + if (u) tain_from_millisecs(&g->qtto, u) ; g->dumpfile = conf_get_string("G:cachefile") ; if (!g->dumpfile && errno != ENOENT) strerr_diefu4sys(102, "read ", "G:cachefile", " configuration key from ", conffile) ; @@ -122,7 +125,7 @@ int main (int argc, char const *const *argv) { global globals = GLOBAL_ZERO ; char const *conffile = SHIBARI_SYSCONFPREFIX "/shibari-cache.conf.cdb" ; - uint16_t n4 = 0, n6 = 0, maxtcp, maxqueries ; + uint16_t n4 = 0, n6 = 0, maxtcp, maxtasks ; char const *ip4 = 0, *ip6 = 0 ; unsigned int cont = 2 ; int spfd = -1 ; @@ -168,7 +171,7 @@ int main (int argc, char const *const *argv) close(1) ; if (!cdb_init(&g->confdb, conffile)) strerr_diefu2sys(111, "open ", conffile) ; - conf_init(conffile, &n4, &n6, &ip4, &ip6, &maxtcp, &maxqueries) ; + conf_init(conffile, &n4, &n6, &ip4, &ip6, &maxtcp, &maxtasks) ; spfd = selfpipe_init() ; if (spfd == -1) strerr_diefu1sys(111, "create selfpipe") ; @@ -192,19 +195,18 @@ int main (int argc, char const *const *argv) uint16_t tcp6xindex[n4 ? n4 : 1] ; tcpconnection tcpconnection_storage[maxtcp + 1] ; uint32_t tcpconnection_freelist[maxtcp + 1] ; - query query_storage[maxqueries + 1] ; - uint32_t query_freelist[maxqueries + 1] ; + dnstask dnstask_storage[maxtasks] ; + uint32_t dnstask_freelist[maxtasks] ; memset(udpq4, 0, n4 * sizeof(udpqueue)) ; memset(udpq6, 0, n6 * sizeof(udpqueue)) ; g->udpqueues[0] = udpq4 ; g->udpqueues[1] = udpq6 ; memset(tcpconnection_storage, 0, (maxtcp + 1) * sizeof(tcpconnection)) ; - memset(query_storage, 0, (maxqueries + 1) * sizeof(query)) ; + memset(dnstasks_storage, 0, maxtasks * sizeof(dnstask)) ; GENSET_init(&g->tcpconnections, tcpconnection, tcpconnection_storage, tcpconnection_freelist, maxtcp + 1) ; g->tcpsentinel = genset_new(&g->tcpconnections) ; - GENSET_init(&g->queries, query, query_storage, query_freelist, maxqueries + 1) ; - g->qsentinel = genset_new(&g->queries) ; + GENSET_init(&g->dnstasks, dnstask, dnstask_storage, dnstask_freelist, maxtasks) ; { tcpconnection *p = TCPCONNECTION(g->tcpsentinel) ; query *q = QUERY(g->qsentinel) ; @@ -285,7 +287,7 @@ int main (int argc, char const *const *argv) for (uint16_t i = 0 ; i < n4 ; i++) { x[j].fd = udpq4[i].fd ; - x[j].events = nq < maxqueries && cont >= 2 ? IOPAUSE_READ : 0 ; + x[j].events = ntasks < maxtasks && cont >= 2 ? IOPAUSE_READ : 0 ; if (genalloc_len(udp4msg, &udpq4[i].messages)) { x[j].events |= IOPAUSE_WRITE ; @@ -307,7 +309,7 @@ int main (int argc, char const *const *argv) for (uint16_t i = 0 ; i < n6 ; i++) { x[j].fd = udpq6[i].fd ; - x[j].events = nq < maxqueries && cont >= 2 ? IOPAUSE_READ : 0 ; + x[j].events = ntasks < maxtasks && cont >= 2 ? IOPAUSE_READ : 0 ; if (genalloc_len(udp6msg, &udpq6[i].messages)) { x[j].events |= IOPAUSE_WRITE ; @@ -330,7 +332,7 @@ int main (int argc, char const *const *argv) { tcpconnection *p = TCPCONNECTION(i) ; x[j].fd = bufalloc_fd(&p->out) ; - if (nq < maxqueries && cont >= 2) + if (ntasks < maxtasks && cont >= 2) { x[j].events = IOPAUSE_READ ; if (tain_less(&p->rdeadline, &deadline)) deadline = p->rdeadline ; @@ -354,10 +356,7 @@ int main (int argc, char const *const *argv) if (x[j].events) p->xindex = j++ ; else p->xindex = UINT16_MAX ; } - - /* normal exit condition */ - - if (cont < 2 && !r && !nq) break ; + if (cont < 2 && !r && !ntasks) break ; /* normal exit */ /* poll() */ @@ -370,9 +369,9 @@ int main (int argc, char const *const *argv) if (!r) { - if (cont == 1 && !tain_future(&lameduckt)) break ; /* too lame */ + if (cont == 1 && !tain_future(&lameduckt)) break ; /* lameduck exit */ for (uint16_t i = qstart ; i != g->qsentinel ; i = QUERY(i)->next) - if (s6dns_engine_timeout_g(&QUERY(i)->dt)) i = query_fail(i) ; + if (s6dns_engine_timeout_g(&QUERY(i)->dt)) i = query_event(i) ; for (uint16_t i = tcpstart ; i != g->tcpsentinel ; i = TCPCONNECTION(i)->next) { tcpconnection *p = TCPCONNECTION(i) ; @@ -440,12 +439,8 @@ int main (int argc, char const *const *argv) /* process in-flight queries */ for (uint16_t i = qstart ; i != g->qsentinel ; i = QUERY(i)->next) - { - if (QUERY(i)->xindex == UINT16_MAX) continue ; - r = s6dns_engine_event_g(&QUERY(i)->dt) ; - if (r < 0) i = query_fail(i) ; - else if (r > 0) i = query_succeed(i) ; - } + if (QUERY(i)->xindex < UINT16_MAX && s6dns_engine_event_g(&QUERY(i)->dt)) + i = query_event(i) ; /* read udp */ @@ -454,11 +449,11 @@ int main (int argc, char const *const *argv) { if (udpq4[i].xindex < UINT16_MAX && x[udpq4[i].xindex].revents & IOPAUSE_READ) { + uint16_t port ; uint16_t n = MAXSAME ; - char buf[513] ; char ip[4] ; - uint16_t port ; - while (n-- && nq < maxqueries) + char buf[513] ; + while (n-- && ntasks < maxtasks) { ssize_t len = sanitize_read(socket_recv4(udpq4[i].fd, buf, 512, ip, &port)) ; if (len == -1) @@ -470,10 +465,7 @@ int main (int argc, char const *const *argv) if (!len) break ; if (len < 12 || len > 512 || !clientaccess_ip4(ip) - || !query_start(0, i, ip, port, buf, len)) - { - if (g->verbosity >= 3) log_udpbad(ip, port) ; - } + || !dns_start(0, i, ip, port, buf, len)) log_udpbad(ip, port) ; } } } @@ -483,11 +475,11 @@ int main (int argc, char const *const *argv) { if (udpq6[i].xindex < UINT16_MAX && x[udpq6[i].xindex].revents & IOPAUSE_READ) { + uint16_t port ; uint16_t n = MAXSAME ; - char buf[513] ; char ip[16] ; - uint16_t port ; - while (n-- && nq < maxqueries) + char buf[513] ; + while (n-- && ntasks < maxtasks) { ssize_t len = sanitize_read(socket_recv6(udpq6[i].fd, buf, 512, ip, &port)) ; if (len == -1) @@ -499,10 +491,7 @@ int main (int argc, char const *const *argv) if (!len) break ; if (len < 12 || len > 512 || !clientaccess_ip6(ip) - || !query_start(1, i, ip, port, buf, len)) - { - if (g->verbosity >= 3) log_udpbad(ip, port) ; - } + || !dns_start(1, i, ip, port, buf, len)) log_udpbad(ip, port) ; } } } @@ -523,9 +512,9 @@ int main (int argc, char const *const *argv) if (l == -1) { i = tcpconnection_delete(p) ; break ; } if (!l) break ; if (p->in.len < 12 || p->in.len > 65536 - || !query_start(2, i, 0, 0, p->in.s, p->in.len)) + || !dns_start(2, i, 0, 0, p->in.s, p->in.len)) { - if (g->verbosity >= 3) log_tcpbad(i) ; + log_tcpbad(i) ; i = tcpconnection_delete(p) ; break ; } diff --git a/src/cache/tcpconnection.c b/src/cache/tcpconnection.c index 40c3035..87ac69c 100644 --- a/src/cache/tcpconnection.c +++ b/src/cache/tcpconnection.c @@ -13,15 +13,15 @@ #include "shibari-cache-internal.h" -void tcpconnection_removequery (tcpconnection *p, uint16_t id) +void tcpconnection_removetask (tcpconnection *p, uint16_t id) { - uint16_t *tab = genalloc_s(uint16_t, &p->queries) ; - uint16_t n = genalloc_len(uint16_t, &p->queries) ; + uint16_t *tab = genalloc_s(uint16_t, &p->tasks) ; + uint16_t n = genalloc_len(uint16_t, &p->tasks) ; uint16_t i = 0 ; for (; i < n ; i++) if (id == tab[i]) break ; if (i >= n) return ; tab[i] = tab[--n] ; - genalloc_setlen(uint16_t, &p->queries, n) ; + genalloc_setlen(uint16_t, &p->tasks, n) ; } uint16_t tcpconnection_delete (tcpconnection *p) @@ -31,9 +31,9 @@ uint16_t tcpconnection_delete (tcpconnection *p) p->in.len = 0 ; p->instate = 0 ; fd_close(p->out.fd) ; - for (uint16_t i = 0 ; i < genalloc_len(uint16_t, &p->queries) ; i++) - query_abort(genalloc_s(uint16_t, &p->queries)[i]) ; - genalloc_setlen(uint16_t, &p->queries, 0) ; + for (uint16_t i = 0 ; i < genalloc_len(uint16_t, &p->tasks) ; i++) + dnstask_abort(genalloc_s(uint16_t, &p->tasks)[i]) ; + genalloc_setlen(uint16_t, &p->tasks, 0) ; TCPCONNECTION(newi)->next = p->next ; TCPCONNECTION(p->next)->prev = p->prev ; p->xindex = UINT16_MAX ; diff --git a/src/config/defaults.c b/src/config/defaults.c index 12e404a..bd9dea2 100644 --- a/src/config/defaults.c +++ b/src/config/defaults.c @@ -22,9 +22,10 @@ static struct defaults_s const defaults[] = RECU16("G:logv", 1), RECU64("G:cachesize", 1048576ull), RECU16("G:maxtcp", 64), - RECU16("G:maxqueries", 256), + RECU16("G:maxtasks", 256), RECU32("G:rtimeout", 0), RECU32("G:wtimeout", 0), + RECU32("G:qtimeout", 0), REC("G:listen4", "\177\0\0\1", 4), REC("G:listen6", "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\1", 16), diff --git a/src/config/lexparse.c b/src/config/lexparse.c index 7884fee..b4316c2 100644 --- a/src/config/lexparse.c +++ b/src/config/lexparse.c @@ -31,9 +31,10 @@ enum directivevalue_e T_VERBOSITY, T_CACHESIZE, T_MAXTCP, - T_MAXQUERIES, + T_MAXTASKS, T_RTIMEOUT, T_WTIMEOUT, + T_QTIMEOUT, T_CACHEFILE, T_LISTEN, T_ACCEPT, @@ -219,8 +220,9 @@ static inline void process_line (char const *s, size_t const *word, size_t n, ch { .name = "cache_size", .value = T_CACHESIZE }, { .name = "forward", .value = T_FORWARD }, { .name = "listen", .value = T_LISTEN }, - { .name = "maxqueries", .value = T_MAXQUERIES }, + { .name = "maxtasks", .value = T_MAXTASKS }, { .name = "maxtcp", .value = T_MAXTCP }, + { .name = "query_timeout", .value = T_QTIMEOUT }, { .name = "read_timeout", .value = T_RTIMEOUT }, { .name = "server", .value = T_SERVER }, { .name = "verbosity", .value = T_VERBOSITY }, @@ -243,8 +245,8 @@ static inline void process_line (char const *s, size_t const *word, size_t n, ch case T_MAXTCP : parse_u16(s, word, n, ifile, line, "maxtcp", "G:maxtcp") ; break ; - case T_MAXQUERIES : - parse_u16(s, word, n, ifile, line, "maxqueries", "G:maxqueries") ; + case T_MAXTASKS : + parse_u16(s, word, n, ifile, line, "maxtasks", "G:maxtasks") ; break ; case T_RTIMEOUT : parse_u32(s, word, n, ifile, line, "read_timeout", "G:rtimeout") ; @@ -252,6 +254,9 @@ static inline void process_line (char const *s, size_t const *word, size_t n, ch case T_WTIMEOUT : parse_u32(s, word, n, ifile, line, "write_timeout", "G:wtimeout") ; break ; + case T_QTIMEOUT : + parse_u32(s, word, n, ifile, line, "query_timeout", "G:qtimeout") ; + break ; case T_CACHEFILE : parse_string(s, word, n, ifile, line, "cache_file", "G:cachefile") ; break ; diff --git a/src/include/shibari/dcache.h b/src/include/shibari/dcache.h index 1ec9fb6..1127249 100644 --- a/src/include/shibari/dcache.h +++ b/src/include/shibari/dcache.h @@ -12,26 +12,26 @@ #define DCACHE_MAGIC "--DCACHE--\n" -typedef struct dcache_key_s dcache_key_t, *dcache_key_t_ref ; -struct dcache_key_s +typedef struct dcache_string_s dcache_string, *dcache_string_ref ; +struct dcache_string_s { char *s ; uint16_t len ; } ; -typedef struct dcache_node_s dcache_node_t, *dcache_node_t_ref ; +typedef struct dcache_node_s dcache_node, *dcache_node_ref ; struct dcache_node_s { - dcache_key_t key ; - uint16_t datalen ; - tain entry ; - tain expire ; + stralloc sa ; + tai entry ; + tai expire ; } ; +#define DCACHE_NODE_ZERO = { .sa = STRALLOC_ZERO, .entry = TAI_INFINITE, .expire = TAI_INFINITE } -typedef struct dcache_s dcache_t, *dcache_t_ref ; +typedef struct dcache_s dcache, *dcache_ref ; struct dcache_s { - gensetdyn storage ; /* dcache_node_t */ + gensetdyn storage ; /* dcache_node */ avltree by_key ; avltree by_entry ; avltree by_expire ; @@ -41,16 +41,17 @@ struct dcache_s } ; #define DCACHE_ZERO { .storage = GENSETDYN_ZERO, .by_key = AVLTREE_ZERO, .by_entry = AVLTREE_ZERO, .by_expire = AVLTREE_ZERO, .max = 0, .size = 0, .motion = 0 } -extern void dcache_init (dcache_t *, uint64_t) ; -extern dcache_node_t *dcache_search (dcache_t *, char const *, uint16_t, tain const *) ; -#define dcache_search_g(d, key, keylen) dcache_search(d, key, (keylen), &STAMP) -extern int dcache_add (dcache_t *, char const *, uint16_t, char const *, uint16_t, tain const *, tain const *) ; -#define dcache_add_g(d, key, keylen, data, datalen, expire) dcache_add(d, key, keylen, data, datalen, (expire), &STAMP) -extern void dcache_clean_expired (dcache_t *, tain const *) ; -#define dcache_clean_expired_g(d) dcache_clean_expired((d), &STAMP) -extern void dcache_free (dcache_t *) ; - -extern int dcache_save (dcache_t const *, char const *) ; -extern int dcache_load (dcache_t *, char const *) ; +extern void dcache_init (dcache *, uint64_t) ; + +extern int dcache_searchnode (dcache *, uint32_t *, char const *, uint16_t, uint16_t, tai const *) ; +#define dcache_searchnode_g(d, idx, q, qlen, qtype) dcache_searchnode(d, idx, q, qlen, (qtype), tain_secp(&STAMP)) + +extern void dcache_clean_expired (dcache *, tai const *) ; +#define dcache_clean_expired_g(d) dcache_clean_expired((d), tain_secp(&STAMP)) + +extern void dcache_free (dcache *) ; + +extern int dcache_save (dcache const *, char const *) ; +extern int dcache_load (dcache *, char const *) ; #endif diff --git a/src/libdcache/dcache-internal.h b/src/libdcache/dcache-internal.h index 7a21044..49ec504 100644 --- a/src/libdcache/dcache-internal.h +++ b/src/libdcache/dcache-internal.h @@ -5,14 +5,15 @@ #include <stdint.h> +#include <skalibs/tai.h> #include <skalibs/avlnode.h> #include <skalibs/gensetdyn.h> #include <shibari/dcache.h> -#define DNODE(z, i) GENSETDYN_P(dcache_node_t, &(z)->storage, i) -#define DCACHE_NODE_OVERHEAD (32 + sizeof(dcache_node_t) + 3 * sizeof(avlnode)) +#define DNODE(z, i) GENSETDYN_P(dcache_node, &(z)->storage, i) +#define DCACHE_NODE_OVERHEAD (32 + sizeof(dcache_node) + 3 * sizeof(avlnode)) -extern void dcache_delete (dcache_t *, uint32_t) ; +extern void dcache_delete (dcache *, uint32_t) ; #endif diff --git a/src/libdcache/dcache_add.c b/src/libdcache/dcache_add.c deleted file mode 100644 index 7e1d871..0000000 --- a/src/libdcache/dcache_add.c +++ /dev/null @@ -1,85 +0,0 @@ -/* ISC license. */ - -#include <stdint.h> -#include <string.h> -#include <errno.h> - -#include <skalibs/uint64.h> -#include <skalibs/alloc.h> -#include <skalibs/tai.h> -#include <skalibs/gensetdyn.h> -#include <skalibs/avlnode.h> -#include <skalibs/avltree.h> - -#include <shibari/dcache.h> -#include "dcache-internal.h" - -static void uniquify (avltree const *tree, tain *stamp) -{ - static tain const nano = { .sec = TAI_ZERO, .nano = 1 } ; - uint32_t dummy ; - while (avltree_search(tree, stamp, &dummy)) - tain_add(stamp, stamp, &nano) ; -} - -static inline void dcache_gc_by_entry (dcache_t *z, uint64_t max) -{ - while (z->size > max) - { - uint32_t oldest ; - if (!avltree_min(&z->by_entry, &oldest)) break ; - dcache_delete(z, oldest) ; - } -} - -static inline int dcache_add_node (dcache_t *z, dcache_node_t const *node) -{ - uint32_t i ; - dcache_node_t *y ; - if (!gensetdyn_new(&z->storage, &i)) return 0 ; - y = DNODE(z, i) ; *y = *node ; - uniquify(&z->by_entry, &y->entry) ; - uniquify(&z->by_expire, &y->expire) ; - if (!avltree_insert(&z->by_key, i)) goto err1 ; - if (!avltree_insert(&z->by_entry, i)) goto err2 ; - if (!avltree_insert(&z->by_expire, i)) goto err3 ; - return 1 ; - - err3: - avltree_delete(&z->by_entry, &y->entry) ; - err2: - avltree_delete(&z->by_key, &y->key) ; - err1: - gensetdyn_delete(&z->storage, i) ; - return 0 ; -} - -static inline int dcache_add_unbounded (dcache_t *z, char const *key, uint16_t keylen, char const *data, uint16_t datalen, tain const *expire, tain const *stamp) -{ - uint32_t len = (uint32_t)keylen + (uint32_t)datalen ; - dcache_node_t y = { .key = { .s = alloc(len) } } ; - if (!y.key.s) return 0 ; - memcpy(y.key.s, key, keylen) ; - memcpy(y.key.s + keylen, data, datalen) ; - y.key.len = keylen ; - y.datalen = datalen ; - y.entry = *stamp ; - y.expire = *expire ; - if (!dcache_add_node(z, &y)) - { - alloc_free(y.key.s) ; - return 0 ; - } - z->size += DCACHE_NODE_OVERHEAD + len ; - z->motion += DCACHE_NODE_OVERHEAD + len ; - return 1 ; -} - -int dcache_add (dcache_t *z, char const *key, uint16_t keylen, char const *data, uint16_t datalen, tain const *expire, tain const *stamp) -{ - uint64_t size = DCACHE_NODE_OVERHEAD + keylen + datalen ; - if (size > z->max) return (errno = EINVAL, 0) ; - if (z->size > z->max - size) dcache_clean_expired(z, stamp) ; - if (z->size > z->max - size) dcache_gc_by_entry(z, z->max - size) ; - return dcache_add_unbounded(z, key, keylen, data, datalen, expire, stamp) ; -} diff --git a/src/libdcache/dcache_add_data.c b/src/libdcache/dcache_add_data.c new file mode 100644 index 0000000..aa1a110 --- /dev/null +++ b/src/libdcache/dcache_add_data.c @@ -0,0 +1,64 @@ +/* ISC license. */ + +#include <stdint.h> +#include <string.h> +#include <errno.h> + +#include <skalibs/uint64.h> +#include <skalibs/alloc.h> +#include <skalibs/tai.h> +#include <skalibs/gensetdyn.h> +#include <skalibs/avlnode.h> +#include <skalibs/avltree.h> + +#include <shibari/dcache.h> +#include "dcache-internal.h" + +#include <skalibs/posixishard.h> + +static inline void dcache_gc_by_entry (dcache *z, uint64_t max) +{ + while (z->size > max) + { + uint32_t oldest ; + if (!avltree_min(&z->by_entry, &oldest)) break ; + dcache_delete(z, oldest) ; + } +} + +static inline int dcache_add_data_to_node (dcache *z, uint32_t i, uint16_t qlen, char const *data, uint16_t datalen, tai const *expire, tai const *stamp) +{ + dcache_node *node = DNODE(z, i) ; + size_t len = 4 + qlen + datalen ; + if (node->sa.len == node->sa.a) return (errno = EDOM, 0) ; + if (!stralloc_ready_tuned(&node->sa, len, 0, 0, 1)) return 0 ; + node->entry = *stamp ; + node->expire = *expire ; + if (!avltree_insert(&z->by_entry, i)) return 0 ; + if (!avltree_insert(&z->by_expire, i)) + { + avltree_delete(&z->by_entry, &node->entry) ; + return 0 ; + } + memcpy(node->sa.s + 4 + qlen, data, datalen) ; + node->sa.len = len ; + if (!stralloc_shrink(&node->sa)) node->sa.a = len ; + z->size += DCACHE_NODE_OVERHEAD + len ; + z->motion += DCACHE_NODE_OVERHEAD + len ; + return 1 ; +} + +int dcache_add_data (dcache *z, char const *q, uint16_t qlen, uint16_t qtype, char const *data, uint16_t datalen, tai const *expire, tai const *stamp) +{ + uint64_t size = DCACHE_NODE_OVERHEAD + qlen + datalen + 4 ; + uint32_t i ; + if (size > z->max) return (errno = EINVAL, 0) ; + if (z->size > z->max - size) dcache_clean_expired(z, stamp) ; + if (z->size > z->max - size) dcache_gc_by_entry(z, z->max - size) ; + if (z->size > z->max - size) return (errno = ENOBUFS, 0) ; + if (!dcache_searchnode(z, &i, q, qlen, qtype, stamp)) return 0 ; + if (!dcache_add_data_to_node(z, i, qlen, data, datalen, expire, stamp)) return 0 ; + z->size += size ; + z->motion += size ; + return 1 ; +} diff --git a/src/libdcache/dcache_clean_expired.c b/src/libdcache/dcache_clean_expired.c index 0e23443..d81bec8 100644 --- a/src/libdcache/dcache_clean_expired.c +++ b/src/libdcache/dcache_clean_expired.c @@ -8,13 +8,13 @@ #include <shibari/dcache.h> #include "dcache-internal.h" -void dcache_clean_expired (dcache_t *z, tain const *stamp) +void dcache_clean_expired (dcache *z, tai const *stamp) { for (;;) { uint32_t i ; if (!avltree_min(&z->by_expire, &i)) break ; - if (tain_less(stamp, &DNODE(z, i)->expire)) break ; + if (tai_less(stamp, &DNODE(z, i)->expire)) break ; dcache_delete(z, i) ; } } diff --git a/src/libdcache/dcache_delete.c b/src/libdcache/dcache_delete.c index 92a5fcc..861a30e 100644 --- a/src/libdcache/dcache_delete.c +++ b/src/libdcache/dcache_delete.c @@ -1,19 +1,22 @@ /* ISC license. */ -#include <skalibs/alloc.h> +#include <skalibs/stralloc.h> #include <skalibs/gensetdyn.h> #include <skalibs/avltree.h> #include <shibari/dcache.h> #include "dcache-internal.h" -void dcache_delete (dcache_t *z, uint32_t i) +void dcache_delete (dcache *z, uint32_t i) { - dcache_node_t *y = DNODE(z, i) ; - avltree_delete(&z->by_expire, &y->expire) ; - avltree_delete(&z->by_entry, &y->entry) ; - avltree_delete(&z->by_key, &y->key) ; - alloc_free(y->key.s) ; - z->size -= DCACHE_NODE_OVERHEAD + y->key.len + y->datalen ; + dcache_node *node = DNODE(z, i) ; + if (node->sa.len == node->sa.a) + { + avltree_delete(&z->by_expire, &node->expire) ; + avltree_delete(&z->by_entry, &node->entry) ; + } + avltree_delete(&z->by_key, &node->sa.s) ; + z->size -= DCACHE_NODE_OVERHEAD + node->sa.len ; + node->sa.len = 0 ; gensetdyn_delete(&z->storage, i) ; } diff --git a/src/libdcache/dcache_free.c b/src/libdcache/dcache_free.c index 16c074e..07d6982 100644 --- a/src/libdcache/dcache_free.c +++ b/src/libdcache/dcache_free.c @@ -1,6 +1,6 @@ /* ISC license. */ -#include <skalibs/alloc.h> +#include <skalibs/stralloc.h> #include <skalibs/gensetdyn.h> #include <skalibs/avltree.h> @@ -8,12 +8,13 @@ static void dcache_node_free (void *p) { - alloc_free(((dcache_node_t *)p)->key.s) ; + dcache_node *node = p ; + stralloc_free(&node->sa) ; } -void dcache_free (dcache_t *z) +void dcache_free (dcache *z) { - static dcache_t const dcache_zero = DCACHE_ZERO ; + static dcache const dcache_zero = DCACHE_ZERO ; avltree_free(&z->by_expire) ; avltree_free(&z->by_entry) ; avltree_free(&z->by_key) ; diff --git a/src/libdcache/dcache_get_data.c b/src/libdcache/dcache_get_data.c new file mode 100644 index 0000000..f14977c --- /dev/null +++ b/src/libdcache/dcache_get_data.c @@ -0,0 +1,16 @@ +/* ISC license. */ + +#include <stdint.h> + +#include <skalibs/uint16.h> + +#include <shibari/dcache.h> + +void dcache_get_data (dcache *z, uint32_t nid, dcache_string *data) +{ + dcache_node *node = DNODE(z, nid) ; + uint16_t qlen ; + uint16_unpack_big(node->sa.s + 2, &qlen) ; + data->s = node->sa.s + 4 + qlen ; + data->len = node->sa.len - qlen - 4 ; +} diff --git a/src/libdcache/dcache_init.c b/src/libdcache/dcache_init.c index 91c529e..8995f81 100644 --- a/src/libdcache/dcache_init.c +++ b/src/libdcache/dcache_init.c @@ -3,7 +3,7 @@ #include <stdint.h> #include <string.h> -#include <skalibs/uint64.h> +#include <skalibs/uint16.h> #include <skalibs/tai.h> #include <skalibs/gensetdyn.h> #include <skalibs/avltree.h> @@ -12,44 +12,47 @@ static int key_cmp (void const *a, void const *b, void *x) { - dcache_key_t const *ka = a ; - dcache_key_t const *kb = b ; - if (ka->len < kb->len) return -1 ; - if (kb->len < ka->len) return 1 ; - (void)x ; - return memcmp(ka->s, kb->s, ka->len) ; + int r = memcmp(a, b, 4) ; + if (r) return r ; + { + char const *aa = a ; + char const *bb = b ; + uint16_t len ; + uint16_unpack_big(aa+2, &len) ; + return memcmp(aa+4, bb+4, len) ; + } } -static int tain_cmp (void const *a, void const *b, void *x) +static int tai_cmp (void const *a, void const *b, void *x) { - tain const *ta = a ; - tain const *tb = b ; + tai const *ta = a ; + tai const *tb = b ; (void)x ; - return tain_less(ta, tb) ? -1 : tain_less(tb, ta) ; + return tai_less(ta, tb) ? -1 : tai_less(tb, ta) ; } static void *key_dtok (uint32_t d, void *x) { - return &GENSETDYN_P(dcache_node_t, (gensetdyn *)x, d)->key ; + return &GENSETDYN_P(dcache_node, (gensetdyn *)x, d)->sa.s ; } static void *entry_dtok (uint32_t d, void *x) { - return &GENSETDYN_P(dcache_node_t, (gensetdyn *)x, d)->entry ; + return &GENSETDYN_P(dcache_node, (gensetdyn *)x, d)->entry ; } static void *expire_dtok (uint32_t d, void *x) { - return &GENSETDYN_P(dcache_node_t, (gensetdyn *)x, d)->expire ; + return &GENSETDYN_P(dcache_node, (gensetdyn *)x, d)->expire ; } -void dcache_init (dcache_t *z, uint64_t max) +void dcache_init (dcache *z, uint64_t max) { - gensetdyn_init(&z->storage, sizeof(dcache_node_t), max >> 9, 3, 8) ; + gensetdyn_init(&z->storage, sizeof(dcache_node), max >> 9, 3, 8) ; avltree_init(&z->by_key, max >> 9, 3, 8, &key_dtok, &key_cmp, &z->storage) ; - avltree_init(&z->by_entry, max >> 9, 3, 8, &entry_dtok, &tain_cmp, &z->storage) ; - avltree_init(&z->by_expire, max >> 9, 3, 8, &expire_dtok, &tain_cmp, &z->storage) ; + avltree_init(&z->by_entry, max >> 9, 3, 8, &entry_dtok, &tai_cmp, &z->storage) ; + avltree_init(&z->by_expire, max >> 9, 3, 8, &expire_dtok, &tai_cmp, &z->storage) ; z->max = max ; z->size = 0 ; z->motion = 0 ; diff --git a/src/libdcache/dcache_load.c b/src/libdcache/dcache_load.c index 176a722..0693b21 100644 --- a/src/libdcache/dcache_load.c +++ b/src/libdcache/dcache_load.c @@ -14,18 +14,17 @@ #include <skalibs/posixishard.h> -static inline int dcache_load_node (dcache_t *z, buffer *b) +static inline int dcache_load_node (dcache *z, buffer *b) { - tain entry = { .nano = 0 } ; - tain expire = { .nano = 0 } ; + tai entry, expire ; uint16_t keylen ; uint16_t datalen ; char pack[TAI_PACK * 2 + 4] ; ssize_t r = buffer_get(b, pack, TAI_PACK * 2 + 4) ; if (!r) return 0 ; if (r < TAI_PACK * 2 + 4) return -1 ; - tai_unpack(pack, tain_secp(&entry)) ; - tai_unpack(pack + TAI_PACK, tain_secp(&expire)) ; + tai_unpack(pack, &entry) ; + tai_unpack(pack + TAI_PACK, &expire) ; uint16_unpack_big(pack + TAI_PACK * 2, &keylen) ; uint16_unpack_big(pack + TAI_PACK * 2 + 2, &datalen) ; { @@ -35,12 +34,12 @@ static inline int dcache_load_node (dcache_t *z, buffer *b) if (!r) return (errno = EPIPE, -1) ; if (r < len) return -1 ; if (blob[len]) return (errno = EPROTO, -1) ; - if (!dcache_add(z, blob, keylen, blob + keylen, datalen, &expire, &entry)) return -1 ; +// if (!dcache_add(z, blob, keylen, blob + keylen, datalen, &expire, &entry)) return -1 ; } return 1 ; } -static inline int dcache_load_from_buffer (dcache_t *z, buffer *b) +static inline int dcache_load_from_buffer (dcache *z, buffer *b) { { char banner[sizeof(DCACHE_MAGIC) - 1] ; @@ -64,7 +63,7 @@ static inline int dcache_load_from_buffer (dcache_t *z, buffer *b) #define N 8192 -int dcache_load (dcache_t *z, char const *file) +int dcache_load (dcache *z, char const *file) { char buf[N] ; buffer b ; diff --git a/src/libdcache/dcache_node_new.c b/src/libdcache/dcache_node_new.c new file mode 100644 index 0000000..1d13eaf --- /dev/null +++ b/src/libdcache/dcache_node_new.c @@ -0,0 +1,34 @@ +/* ISC license. */ + +#include <stdint.h> +#include <string.h> + +#include <skalibs/uint16.h> +#include <skalibs/stralloc.h> +#include <skalibs/gensetdyn.h> +#include <skalibs/avltree.h> + +#include <shibari/dcache.h> +#include "dcache-internal.h" + +dcache_node *dcache_node_new (dcache *z, char const *key, uint16_t keylen) +{ + static tai const tai_infinite = TAI_INFINITE ; + dcache_node *node ; + uint32_t i ; + if (!gensetdyn_new(&z->storage, i)) return 0 ; + node = DNODE(z, i) ; + if (!stralloc_ready(&node->sa, 6 + keylen)) goto err0 ; + uint16_pack_big(node->sa.s, keylen) ; + memcpy(node->sa.s + 2, key, keylen) ; + node->sa.len = 2 + keylen ; + node->entry = tai_infinite ; + node->expire = tai_infinite ; + return node ; + + err1: + node->sa.len = 0 ; + err0: + gensetdyn_delete(&z->storage, i) ; + return 0 ; +} diff --git a/src/libdcache/dcache_save.c b/src/libdcache/dcache_save.c index 7277771..62c55a2 100644 --- a/src/libdcache/dcache_save.c +++ b/src/libdcache/dcache_save.c @@ -19,20 +19,20 @@ static int write_node_iter (void *data, void *aux) { - dcache_node_t *y = data ; + dcache_node *y = data ; buffer *b = aux ; char pack[TAI_PACK * 2 + 4] ; - tai_pack(pack, tain_secp(&y->entry)) ; - tai_pack(pack + TAI_PACK, tain_secp(&y->expire)) ; - uint16_pack(pack + TAI_PACK * 2, y->key.len) ; - uint16_pack(pack + TAI_PACK * 2 + 2, y->datalen) ; + tai_pack(pack, &y->entry) ; + tai_pack(pack + TAI_PACK, &y->expire) ; +// uint16_pack(pack + TAI_PACK * 2, y->key.len) ; +// uint16_pack(pack + TAI_PACK * 2 + 2, y->datalen) ; if (buffer_put(b, pack, TAI_PACK * 2 + 4) == -1) return 0 ; - if (buffer_put(b, y->key.s, y->key.len + y->datalen) == -1) return 0 ; +// if (buffer_put(b, y->key.s, y->key.len + y->datalen) == -1) return 0 ; if (buffer_put(b, "", 1) == -1) return 0 ; return 1 ; } -static inline int dcache_save_to_buffer (dcache_t const *z, buffer *b) +static inline int dcache_save_to_buffer (dcache const *z, buffer *b) { char pack[16] ; if (buffer_puts(b, DCACHE_MAGIC) == -1) return 0 ; @@ -48,7 +48,7 @@ static inline int dcache_save_to_buffer (dcache_t const *z, buffer *b) #define N 8192 -int dcache_save (dcache_t const *z, char const *file) +int dcache_save (dcache const *z, char const *file) { size_t len = strlen(file) ; int fd ; @@ -60,7 +60,7 @@ int dcache_save (dcache_t const *z, char const *file) fd = mkstemp(tmp) ; if (fd == -1) return 0 ; buffer_init(&b, &buffer_write, fd, buf, N) ; - if (!dcache_save_to_buffer(z, &b) || fsync(fd) < 0) goto err2 ; + if (!dcache_save_to_buffer(z, &b) || fsync(fd) == -1) goto err2 ; fd_close(fd) ; if (rename(tmp, file) == -1) goto err1 ; return 1 ; diff --git a/src/libdcache/dcache_search.c b/src/libdcache/dcache_search.c deleted file mode 100644 index 3d79dd6..0000000 --- a/src/libdcache/dcache_search.c +++ /dev/null @@ -1,24 +0,0 @@ -/* ISC license. */ - -#include <stdint.h> - -#include <skalibs/avltree.h> - -#include <shibari/dcache.h> -#include "dcache-internal.h" - -dcache_node_t *dcache_search (dcache_t *z, char const *key, uint16_t keylen, tain const *stamp) -{ - uint32_t i ; - dcache_key_t k = { .s = (char *)key, .len = keylen } ; - if (avltree_search(&z->by_key, &k, &i)) - { - if (tain_less(&DNODE(z, i)->expire, stamp)) - { - dcache_clean_expired(z, stamp) ; - if (!avltree_search(&z->by_key, &k, &i)) return 0 ; - } - return DNODE(z, i) ; - } - else return 0 ; -} diff --git a/src/libdcache/dcache_searchnode.c b/src/libdcache/dcache_searchnode.c new file mode 100644 index 0000000..ef7341c --- /dev/null +++ b/src/libdcache/dcache_searchnode.c @@ -0,0 +1,28 @@ +/* ISC license. */ + +#include <stdint.h> + +#include <skalibs/uint16.h> +#include <skalibs/avltree.h> + +#include <shibari/dcache.h> +#include "dcache-internal.h" + +int dcache_searchnode (dcache *z, uint32_t *idx, char const *q, uint16_t qlen, uint16_t qtype, tai const *stamp) +{ + dcache_node *node ; + uint32_t i ; + char key[4 + qlen] ; + uint16_pack_big(key, qtype) ; + uint16_pack_big(key+2, qlen) ; + memcpy(key+4, q, qlen) ; + if (!avltree_search(&z->by_key, &k, &i)) return -1 ; + node = DNODE(z, i) ; + if (node->sa.len == node->sa.a && !tai_less(stamp, &node->expire)) + { + dcache_delete(z, i) ; + return -1 ; + } + *idx = i ; + return node->sa.len == node->sa.a ; +} diff --git a/src/libdcache/deps-lib/dcache b/src/libdcache/deps-lib/dcache index fcc09a0..07f9f51 100644 --- a/src/libdcache/deps-lib/dcache +++ b/src/libdcache/deps-lib/dcache @@ -1,8 +1,7 @@ -dcache_add.o +dcache_add_data.o dcache_clean_expired.o dcache_delete.o dcache_free.o dcache_init.o dcache_load.o dcache_save.o -dcache_search.o |