Add a lot of stuff, disable shibari-cache build for now

Signed-off-by: Laurent Bercot <ska@appnovation.com>

14 files changed, 202 insertions, 163 deletions

diff --git a/src/libdcache/dcache-internal.h b/src/libdcache/dcache-internal.h
index 7a21044..49ec504 100644
--- a/src/libdcache/dcache-internal.h
+++ b/src/libdcache/dcache-internal.h
@@ -5,14 +5,15 @@
 
 #include <stdint.h>
 
+#include <skalibs/tai.h>
 #include <skalibs/avlnode.h>
 #include <skalibs/gensetdyn.h>
 
 #include <shibari/dcache.h>
 
-#define DNODE(z, i) GENSETDYN_P(dcache_node_t, &(z)->storage, i)
-#define DCACHE_NODE_OVERHEAD (32 + sizeof(dcache_node_t) + 3 * sizeof(avlnode))
+#define DNODE(z, i) GENSETDYN_P(dcache_node, &(z)->storage, i)
+#define DCACHE_NODE_OVERHEAD (32 + sizeof(dcache_node) + 3 * sizeof(avlnode))
 
-extern void dcache_delete (dcache_t *, uint32_t) ;
+extern void dcache_delete (dcache *, uint32_t) ;
 
 #endif
diff --git a/src/libdcache/dcache_add.c b/src/libdcache/dcache_add.c
deleted file mode 100644
index 7e1d871..0000000
--- a/src/libdcache/dcache_add.c
+++ /dev/null
@@ -1,85 +0,0 @@
-/* ISC license. */
-
-#include <stdint.h>
-#include <string.h>
-#include <errno.h>
-
-#include <skalibs/uint64.h>
-#include <skalibs/alloc.h>
-#include <skalibs/tai.h>
-#include <skalibs/gensetdyn.h>
-#include <skalibs/avlnode.h>
-#include <skalibs/avltree.h>
-
-#include <shibari/dcache.h>
-#include "dcache-internal.h"
-
-static void uniquify (avltree const *tree, tain *stamp)
-{
-  static tain const nano = { .sec = TAI_ZERO, .nano = 1 } ;
-  uint32_t dummy ;
-  while (avltree_search(tree, stamp, &dummy))
-    tain_add(stamp, stamp, &nano) ;
-}
-
-static inline void dcache_gc_by_entry (dcache_t *z, uint64_t max)
-{
-  while (z->size > max)
-  {
-    uint32_t oldest ;
-    if (!avltree_min(&z->by_entry, &oldest)) break ;
-    dcache_delete(z, oldest) ;
-  }
-}
-
-static inline int dcache_add_node (dcache_t *z, dcache_node_t const *node)
-{
-  uint32_t i ;
-  dcache_node_t *y ;
-  if (!gensetdyn_new(&z->storage, &i)) return 0 ;
-  y = DNODE(z, i) ; *y = *node ;
-  uniquify(&z->by_entry, &y->entry) ;
-  uniquify(&z->by_expire, &y->expire) ;
-  if (!avltree_insert(&z->by_key, i)) goto err1 ;
-  if (!avltree_insert(&z->by_entry, i)) goto err2 ;
-  if (!avltree_insert(&z->by_expire, i)) goto err3 ;
-  return 1 ;
-
- err3:
-  avltree_delete(&z->by_entry, &y->entry) ;
- err2:
-  avltree_delete(&z->by_key, &y->key) ;
- err1:
-  gensetdyn_delete(&z->storage, i) ;
-  return 0 ;
-}
-
-static inline int dcache_add_unbounded (dcache_t *z, char const *key, uint16_t keylen, char const *data, uint16_t datalen, tain const *expire, tain const *stamp)
-{
-  uint32_t len = (uint32_t)keylen + (uint32_t)datalen ;
-  dcache_node_t y = { .key = { .s = alloc(len) } } ;
-  if (!y.key.s) return 0 ;
-  memcpy(y.key.s, key, keylen) ;
-  memcpy(y.key.s + keylen, data, datalen) ;
-  y.key.len = keylen ;
-  y.datalen = datalen ;
-  y.entry = *stamp ;
-  y.expire = *expire ;
-  if (!dcache_add_node(z, &y))
-  {
-    alloc_free(y.key.s) ;
-    return 0 ;
-  }
-  z->size += DCACHE_NODE_OVERHEAD + len ;
-  z->motion += DCACHE_NODE_OVERHEAD + len ;
-  return 1 ;
-}
-
-int dcache_add (dcache_t *z, char const *key, uint16_t keylen, char const *data, uint16_t datalen, tain const *expire, tain const *stamp)
-{
-  uint64_t size = DCACHE_NODE_OVERHEAD + keylen + datalen ;
-  if (size > z->max) return (errno = EINVAL, 0) ;
-  if (z->size > z->max - size) dcache_clean_expired(z, stamp) ;
-  if (z->size > z->max - size) dcache_gc_by_entry(z, z->max - size) ;
-  return dcache_add_unbounded(z, key, keylen, data, datalen, expire, stamp) ;
-}
diff --git a/src/libdcache/dcache_add_data.c b/src/libdcache/dcache_add_data.c
new file mode 100644
index 0000000..aa1a110
--- /dev/null
+++ b/src/libdcache/dcache_add_data.c
@@ -0,0 +1,64 @@
+/* ISC license. */
+
+#include <stdint.h>
+#include <string.h>
+#include <errno.h>
+
+#include <skalibs/uint64.h>
+#include <skalibs/alloc.h>
+#include <skalibs/tai.h>
+#include <skalibs/gensetdyn.h>
+#include <skalibs/avlnode.h>
+#include <skalibs/avltree.h>
+
+#include <shibari/dcache.h>
+#include "dcache-internal.h"
+
+#include <skalibs/posixishard.h>
+
+static inline void dcache_gc_by_entry (dcache *z, uint64_t max)
+{
+  while (z->size > max)
+  {
+    uint32_t oldest ;
+    if (!avltree_min(&z->by_entry, &oldest)) break ;
+    dcache_delete(z, oldest) ;
+  }
+}
+
+static inline int dcache_add_data_to_node (dcache *z, uint32_t i, uint16_t qlen, char const *data, uint16_t datalen, tai const *expire, tai const *stamp)
+{
+  dcache_node *node = DNODE(z, i) ;
+  size_t len = 4 + qlen + datalen ;
+  if (node->sa.len == node->sa.a) return (errno = EDOM, 0) ;
+  if (!stralloc_ready_tuned(&node->sa, len, 0, 0, 1)) return 0 ;
+  node->entry = *stamp ;
+  node->expire = *expire ;
+  if (!avltree_insert(&z->by_entry, i)) return 0 ;
+  if (!avltree_insert(&z->by_expire, i))
+  {
+    avltree_delete(&z->by_entry, &node->entry) ;
+    return 0 ;
+  }
+  memcpy(node->sa.s + 4 + qlen, data, datalen) ;
+  node->sa.len = len ;
+  if (!stralloc_shrink(&node->sa)) node->sa.a = len ;
+  z->size += DCACHE_NODE_OVERHEAD + len ;
+  z->motion += DCACHE_NODE_OVERHEAD + len ;
+  return 1 ;
+}
+
+int dcache_add_data (dcache *z, char const *q, uint16_t qlen, uint16_t qtype, char const *data, uint16_t datalen, tai const *expire, tai const *stamp)
+{
+  uint64_t size = DCACHE_NODE_OVERHEAD + qlen + datalen + 4 ;
+  uint32_t i ;
+  if (size > z->max) return (errno = EINVAL, 0) ;
+  if (z->size > z->max - size) dcache_clean_expired(z, stamp) ;
+  if (z->size > z->max - size) dcache_gc_by_entry(z, z->max - size) ;
+  if (z->size > z->max - size) return (errno = ENOBUFS, 0) ;
+  if (!dcache_searchnode(z, &i, q, qlen, qtype, stamp)) return 0 ;
+  if (!dcache_add_data_to_node(z, i, qlen, data, datalen, expire, stamp)) return 0 ;
+  z->size += size ;
+  z->motion += size ;
+  return 1 ;
+}
diff --git a/src/libdcache/dcache_clean_expired.c b/src/libdcache/dcache_clean_expired.c
index 0e23443..d81bec8 100644
--- a/src/libdcache/dcache_clean_expired.c
+++ b/src/libdcache/dcache_clean_expired.c
@@ -8,13 +8,13 @@
 #include <shibari/dcache.h>
 #include "dcache-internal.h"
 
-void dcache_clean_expired (dcache_t *z, tain const *stamp)
+void dcache_clean_expired (dcache *z, tai const *stamp)
 {
   for (;;)
   {
     uint32_t i ;
     if (!avltree_min(&z->by_expire, &i)) break ;
-    if (tain_less(stamp, &DNODE(z, i)->expire)) break ;
+    if (tai_less(stamp, &DNODE(z, i)->expire)) break ;
     dcache_delete(z, i) ;
   }
 }
diff --git a/src/libdcache/dcache_delete.c b/src/libdcache/dcache_delete.c
index 92a5fcc..861a30e 100644
--- a/src/libdcache/dcache_delete.c
+++ b/src/libdcache/dcache_delete.c
@@ -1,19 +1,22 @@
 /* ISC license. */
 
-#include <skalibs/alloc.h>
+#include <skalibs/stralloc.h>
 #include <skalibs/gensetdyn.h>
 #include <skalibs/avltree.h>
 
 #include <shibari/dcache.h>
 #include "dcache-internal.h"
 
-void dcache_delete (dcache_t *z, uint32_t i)
+void dcache_delete (dcache *z, uint32_t i)
 {
-  dcache_node_t *y = DNODE(z, i) ;
-  avltree_delete(&z->by_expire, &y->expire) ;
-  avltree_delete(&z->by_entry, &y->entry) ;
-  avltree_delete(&z->by_key, &y->key) ;
-  alloc_free(y->key.s) ;
-  z->size -= DCACHE_NODE_OVERHEAD + y->key.len + y->datalen ;
+  dcache_node *node = DNODE(z, i) ;
+  if (node->sa.len == node->sa.a)
+  {
+    avltree_delete(&z->by_expire, &node->expire) ;
+    avltree_delete(&z->by_entry, &node->entry) ;
+  }
+  avltree_delete(&z->by_key, &node->sa.s) ;
+  z->size -= DCACHE_NODE_OVERHEAD + node->sa.len ;
+  node->sa.len = 0 ;
   gensetdyn_delete(&z->storage, i) ;
 }
diff --git a/src/libdcache/dcache_free.c b/src/libdcache/dcache_free.c
index 16c074e..07d6982 100644
--- a/src/libdcache/dcache_free.c
+++ b/src/libdcache/dcache_free.c
@@ -1,6 +1,6 @@
 /* ISC license. */
 
-#include <skalibs/alloc.h>
+#include <skalibs/stralloc.h>
 #include <skalibs/gensetdyn.h>
 #include <skalibs/avltree.h>
 
@@ -8,12 +8,13 @@
 
 static void dcache_node_free (void *p)
 {
-  alloc_free(((dcache_node_t *)p)->key.s) ;
+  dcache_node *node = p ;
+  stralloc_free(&node->sa) ;
 }
 
-void dcache_free (dcache_t *z)
+void dcache_free (dcache *z)
 {
-  static dcache_t const dcache_zero = DCACHE_ZERO ;
+  static dcache const dcache_zero = DCACHE_ZERO ;
   avltree_free(&z->by_expire) ;
   avltree_free(&z->by_entry) ;
   avltree_free(&z->by_key) ;
diff --git a/src/libdcache/dcache_get_data.c b/src/libdcache/dcache_get_data.c
new file mode 100644
index 0000000..f14977c
--- /dev/null
+++ b/src/libdcache/dcache_get_data.c
@@ -0,0 +1,16 @@
+/* ISC license. */
+
+#include <stdint.h>
+
+#include <skalibs/uint16.h>
+
+#include <shibari/dcache.h>
+
+void dcache_get_data (dcache *z, uint32_t nid, dcache_string *data)
+{
+  dcache_node *node = DNODE(z, nid) ;
+  uint16_t qlen ;
+  uint16_unpack_big(node->sa.s + 2, &qlen) ;
+  data->s = node->sa.s + 4 + qlen ;
+  data->len = node->sa.len - qlen - 4 ;
+}
diff --git a/src/libdcache/dcache_init.c b/src/libdcache/dcache_init.c
index 91c529e..8995f81 100644
--- a/src/libdcache/dcache_init.c
+++ b/src/libdcache/dcache_init.c
@@ -3,7 +3,7 @@
 #include <stdint.h>
 #include <string.h>
 
-#include <skalibs/uint64.h>
+#include <skalibs/uint16.h>
 #include <skalibs/tai.h>
 #include <skalibs/gensetdyn.h>
 #include <skalibs/avltree.h>
@@ -12,44 +12,47 @@
 
 static int key_cmp (void const *a, void const *b, void *x)
 {
-  dcache_key_t const *ka = a ;
-  dcache_key_t const *kb = b ;
-  if (ka->len < kb->len) return -1 ;
-  if (kb->len < ka->len) return 1 ;
-  (void)x ;
-  return memcmp(ka->s, kb->s, ka->len) ;
+  int r = memcmp(a, b, 4) ;
+  if (r) return r ;
+  {
+    char const *aa = a ;
+    char const *bb = b ;
+    uint16_t len ;
+    uint16_unpack_big(aa+2, &len) ;
+    return memcmp(aa+4, bb+4, len) ;
+  }
 }
 
-static int tain_cmp (void const *a, void const *b, void *x)
+static int tai_cmp (void const *a, void const *b, void *x)
 {
-  tain const *ta = a ;
-  tain const *tb = b ;
+  tai const *ta = a ;
+  tai const *tb = b ;
   (void)x ;
-  return tain_less(ta, tb) ? -1 : tain_less(tb, ta) ;
+  return tai_less(ta, tb) ? -1 : tai_less(tb, ta) ;
 }
 
 static void *key_dtok (uint32_t d, void *x)
 {
-  return &GENSETDYN_P(dcache_node_t, (gensetdyn *)x, d)->key ;
+  return &GENSETDYN_P(dcache_node, (gensetdyn *)x, d)->sa.s ;
 }
 
 static void *entry_dtok (uint32_t d, void *x)
 {
-  return &GENSETDYN_P(dcache_node_t, (gensetdyn *)x, d)->entry ;
+  return &GENSETDYN_P(dcache_node, (gensetdyn *)x, d)->entry ;
 }
 
 static void *expire_dtok (uint32_t d, void *x)
 {
-  return &GENSETDYN_P(dcache_node_t, (gensetdyn *)x, d)->expire ;
+  return &GENSETDYN_P(dcache_node, (gensetdyn *)x, d)->expire ;
 }
 
 
-void dcache_init (dcache_t *z, uint64_t max)
+void dcache_init (dcache *z, uint64_t max)
 {
-  gensetdyn_init(&z->storage, sizeof(dcache_node_t), max >> 9, 3, 8) ;
+  gensetdyn_init(&z->storage, sizeof(dcache_node), max >> 9, 3, 8) ;
   avltree_init(&z->by_key, max >> 9, 3, 8, &key_dtok, &key_cmp, &z->storage) ;
-  avltree_init(&z->by_entry, max >> 9, 3, 8, &entry_dtok, &tain_cmp, &z->storage) ;
-  avltree_init(&z->by_expire, max >> 9, 3, 8, &expire_dtok, &tain_cmp, &z->storage) ;
+  avltree_init(&z->by_entry, max >> 9, 3, 8, &entry_dtok, &tai_cmp, &z->storage) ;
+  avltree_init(&z->by_expire, max >> 9, 3, 8, &expire_dtok, &tai_cmp, &z->storage) ;
   z->max = max ;
   z->size = 0 ;
   z->motion = 0 ;
diff --git a/src/libdcache/dcache_load.c b/src/libdcache/dcache_load.c
index 176a722..0693b21 100644
--- a/src/libdcache/dcache_load.c
+++ b/src/libdcache/dcache_load.c
@@ -14,18 +14,17 @@
 
 #include <skalibs/posixishard.h>
 
-static inline int dcache_load_node (dcache_t *z, buffer *b)
+static inline int dcache_load_node (dcache *z, buffer *b)
 {
-  tain entry = { .nano = 0 } ;
-  tain expire = { .nano = 0 } ;
+  tai entry, expire ;
   uint16_t keylen ;
   uint16_t datalen ;
   char pack[TAI_PACK * 2 + 4] ;
   ssize_t r = buffer_get(b, pack, TAI_PACK * 2 + 4) ;
   if (!r) return 0 ;
   if (r < TAI_PACK * 2 + 4) return -1 ;
-  tai_unpack(pack, tain_secp(&entry)) ;
-  tai_unpack(pack + TAI_PACK, tain_secp(&expire)) ;
+  tai_unpack(pack, &entry) ;
+  tai_unpack(pack + TAI_PACK, &expire) ;
   uint16_unpack_big(pack + TAI_PACK * 2, &keylen) ;
   uint16_unpack_big(pack + TAI_PACK * 2 + 2, &datalen) ;
   {
@@ -35,12 +34,12 @@ static inline int dcache_load_node (dcache_t *z, buffer *b)
     if (!r) return (errno = EPIPE, -1) ;
     if (r < len) return -1 ;
     if (blob[len]) return (errno = EPROTO, -1) ;
-    if (!dcache_add(z, blob, keylen, blob + keylen, datalen, &expire, &entry)) return -1 ;
+//    if (!dcache_add(z, blob, keylen, blob + keylen, datalen, &expire, &entry)) return -1 ;
   }
   return 1 ;
 }
 
-static inline int dcache_load_from_buffer (dcache_t *z, buffer *b)
+static inline int dcache_load_from_buffer (dcache *z, buffer *b)
 {
   {
     char banner[sizeof(DCACHE_MAGIC) - 1] ;
@@ -64,7 +63,7 @@ static inline int dcache_load_from_buffer (dcache_t *z, buffer *b)
 
 #define N 8192
 
-int dcache_load (dcache_t *z, char const *file)
+int dcache_load (dcache *z, char const *file)
 {
   char buf[N] ;
   buffer b ;
diff --git a/src/libdcache/dcache_node_new.c b/src/libdcache/dcache_node_new.c
new file mode 100644
index 0000000..1d13eaf
--- /dev/null
+++ b/src/libdcache/dcache_node_new.c
@@ -0,0 +1,34 @@
+/* ISC license. */
+
+#include <stdint.h>
+#include <string.h>
+
+#include <skalibs/uint16.h>
+#include <skalibs/stralloc.h>
+#include <skalibs/gensetdyn.h>
+#include <skalibs/avltree.h>
+
+#include <shibari/dcache.h>
+#include "dcache-internal.h"
+
+dcache_node *dcache_node_new (dcache *z, char const *key, uint16_t keylen)
+{
+  static tai const tai_infinite = TAI_INFINITE ;
+  dcache_node *node ;
+  uint32_t i ;
+  if (!gensetdyn_new(&z->storage, i)) return 0 ;
+  node = DNODE(z, i) ;
+  if (!stralloc_ready(&node->sa, 6 + keylen)) goto err0 ;
+  uint16_pack_big(node->sa.s, keylen) ;
+  memcpy(node->sa.s + 2, key, keylen) ;
+  node->sa.len = 2 + keylen ;
+  node->entry = tai_infinite ;
+  node->expire = tai_infinite ;
+  return node ;
+
+ err1:
+  node->sa.len = 0 ;
+ err0:
+  gensetdyn_delete(&z->storage, i) ;
+  return 0 ;
+}
diff --git a/src/libdcache/dcache_save.c b/src/libdcache/dcache_save.c
index 7277771..62c55a2 100644
--- a/src/libdcache/dcache_save.c
+++ b/src/libdcache/dcache_save.c
@@ -19,20 +19,20 @@
 
 static int write_node_iter (void *data, void *aux)
 {
-  dcache_node_t *y = data ;
+  dcache_node *y = data ;
   buffer *b = aux ;
   char pack[TAI_PACK * 2 + 4] ;
-  tai_pack(pack, tain_secp(&y->entry)) ;
-  tai_pack(pack + TAI_PACK, tain_secp(&y->expire)) ;
-  uint16_pack(pack + TAI_PACK * 2, y->key.len) ;
-  uint16_pack(pack + TAI_PACK * 2 + 2, y->datalen) ;
+  tai_pack(pack, &y->entry) ;
+  tai_pack(pack + TAI_PACK, &y->expire) ;
+//  uint16_pack(pack + TAI_PACK * 2, y->key.len) ;
+//  uint16_pack(pack + TAI_PACK * 2 + 2, y->datalen) ;
   if (buffer_put(b, pack, TAI_PACK * 2 + 4) == -1) return 0 ;
-  if (buffer_put(b, y->key.s, y->key.len + y->datalen) == -1) return 0 ;
+//  if (buffer_put(b, y->key.s, y->key.len + y->datalen) == -1) return 0 ;
   if (buffer_put(b, "", 1) == -1) return 0 ;
   return 1 ;
 }
 
-static inline int dcache_save_to_buffer (dcache_t const *z, buffer *b)
+static inline int dcache_save_to_buffer (dcache const *z, buffer *b)
 {
   char pack[16] ;
   if (buffer_puts(b, DCACHE_MAGIC) == -1) return 0 ;
@@ -48,7 +48,7 @@ static inline int dcache_save_to_buffer (dcache_t const *z, buffer *b)
 
 #define N 8192
 
-int dcache_save (dcache_t const *z, char const *file)
+int dcache_save (dcache const *z, char const *file)
 {
   size_t len = strlen(file) ;
   int fd ;
@@ -60,7 +60,7 @@ int dcache_save (dcache_t const *z, char const *file)
   fd = mkstemp(tmp) ;
   if (fd == -1) return 0 ;
   buffer_init(&b, &buffer_write, fd, buf, N) ;
-  if (!dcache_save_to_buffer(z, &b) || fsync(fd) < 0) goto err2 ;
+  if (!dcache_save_to_buffer(z, &b) || fsync(fd) == -1) goto err2 ;
   fd_close(fd) ;
   if (rename(tmp, file) == -1) goto err1 ;
   return 1 ;
diff --git a/src/libdcache/dcache_search.c b/src/libdcache/dcache_search.c
deleted file mode 100644
index 3d79dd6..0000000
--- a/src/libdcache/dcache_search.c
+++ /dev/null
@@ -1,24 +0,0 @@
-/* ISC license. */
-
-#include <stdint.h>
-
-#include <skalibs/avltree.h>
-
-#include <shibari/dcache.h>
-#include "dcache-internal.h"
-
-dcache_node_t *dcache_search (dcache_t *z, char const *key, uint16_t keylen, tain const *stamp)
-{
-  uint32_t i ;
-  dcache_key_t k = { .s = (char *)key, .len = keylen } ;
-  if (avltree_search(&z->by_key, &k, &i))
-  {
-    if (tain_less(&DNODE(z, i)->expire, stamp))
-    {
-      dcache_clean_expired(z, stamp) ;
-      if (!avltree_search(&z->by_key, &k, &i)) return 0 ;
-    }
-    return DNODE(z, i) ;
-  }
-  else return 0 ;
-}
diff --git a/src/libdcache/dcache_searchnode.c b/src/libdcache/dcache_searchnode.c
new file mode 100644
index 0000000..ef7341c
--- /dev/null
+++ b/src/libdcache/dcache_searchnode.c
@@ -0,0 +1,28 @@
+/* ISC license. */
+
+#include <stdint.h>
+
+#include <skalibs/uint16.h>
+#include <skalibs/avltree.h>
+
+#include <shibari/dcache.h>
+#include "dcache-internal.h"
+
+int dcache_searchnode (dcache *z, uint32_t *idx, char const *q, uint16_t qlen, uint16_t qtype, tai const *stamp)
+{
+  dcache_node *node ;
+  uint32_t i ;
+  char key[4 + qlen] ;
+  uint16_pack_big(key, qtype) ;
+  uint16_pack_big(key+2, qlen) ;
+  memcpy(key+4, q, qlen) ;
+  if (!avltree_search(&z->by_key, &k, &i)) return -1 ;
+  node = DNODE(z, i) ;
+  if (node->sa.len == node->sa.a && !tai_less(stamp, &node->expire))
+  {
+    dcache_delete(z, i) ;
+    return -1 ;
+  }
+  *idx = i ;
+  return node->sa.len == node->sa.a ;
+}
diff --git a/src/libdcache/deps-lib/dcache b/src/libdcache/deps-lib/dcache
index fcc09a0..07f9f51 100644
--- a/src/libdcache/deps-lib/dcache
+++ b/src/libdcache/deps-lib/dcache
@@ -1,8 +1,7 @@
-dcache_add.o
+dcache_add_data.o
 dcache_clean_expired.o
 dcache_delete.o
 dcache_free.o
 dcache_init.o
 dcache_load.o
 dcache_save.o
-dcache_search.o