Refactor cdbmake, add cdbmake_addv, cdb_hashv

Signed-off-by: Laurent Bercot <ska@appnovation.com>

13 files changed, 164 insertions, 61 deletions

diff --git a/package/deps.mak b/package/deps.mak
index fe15281..dbef6b6 100644
--- a/package/deps.mak
+++ b/package/deps.mak
@@ -15,7 +15,7 @@ src/include/skalibs/buffer.h: src/include/skalibs/allreadwrite.h src/include/ska
 src/include/skalibs/bytestr.h: src/include/skalibs/gccattributes.h
 src/include/skalibs/cbuffer.h: src/include/skalibs/gccattributes.h
 src/include/skalibs/cdb.h: src/include/skalibs/gccattributes.h
-src/include/skalibs/cdbmake.h: src/include/skalibs/allreadwrite.h src/include/skalibs/buffer.h src/include/skalibs/diuint32.h src/include/skalibs/genalloc.h
+src/include/skalibs/cdbmake.h: src/include/skalibs/buffer.h src/include/skalibs/genalloc.h
 src/include/skalibs/datastruct.h: src/include/skalibs/avlnode.h src/include/skalibs/avltree.h src/include/skalibs/avltreen.h src/include/skalibs/bigkv.h src/include/skalibs/genqdyn.h src/include/skalibs/genset.h src/include/skalibs/gensetdyn.h
 src/include/skalibs/djbtime.h: src/include/skalibs/tai.h src/include/skalibs/uint64.h
 src/include/skalibs/djbunix.h: src/include/skalibs/devino.h src/include/skalibs/gccattributes.h src/include/skalibs/stralloc.h
@@ -63,6 +63,7 @@ src/libstdcrypto/sha1-internal.h: src/include/skalibs/sha1.h
 src/libstdcrypto/sha256-internal.h: src/include/skalibs/sha256.h
 src/libstdcrypto/sha512-internal.h: src/include/skalibs/sha512.h
 src/libstddjb/cdb-internal.h: src/include/skalibs/cdb.h src/include/skalibs/gccattributes.h
+src/libstddjb/cdbmake-internal.h: src/include/skalibs/cdbmake.h
 src/libstddjb/child_spawn-internal.h: src/include/skalibs/sysdeps.h
 src/libstddjb/djbtime-internal.h: src/include/skalibs/uint64.h
 src/libstddjb/fmtscan-internal.h: src/include/skalibs/fmtscan.h src/include/skalibs/uint64.h
@@ -287,13 +288,20 @@ src/libstddjb/cdb_find.o src/libstddjb/cdb_find.lo: src/libstddjb/cdb_find.c src
 src/libstddjb/cdb_findnext.o src/libstddjb/cdb_findnext.lo: src/libstddjb/cdb_findnext.c src/libstddjb/cdb-internal.h src/include/skalibs/cdb.h src/include/skalibs/uint32.h
 src/libstddjb/cdb_free.o src/libstddjb/cdb_free.lo: src/libstddjb/cdb_free.c src/include/skalibs/cdb.h src/include/skalibs/posixplz.h
 src/libstddjb/cdb_hash.o src/libstddjb/cdb_hash.lo: src/libstddjb/cdb_hash.c src/libstddjb/cdb-internal.h
+src/libstddjb/cdb_hashadd.o src/libstddjb/cdb_hashadd.lo: src/libstddjb/cdb_hashadd.c src/libstddjb/cdb-internal.h
+src/libstddjb/cdb_hashv.o src/libstddjb/cdb_hashv.lo: src/libstddjb/cdb_hashv.c src/libstddjb/cdb-internal.h
 src/libstddjb/cdb_init.o src/libstddjb/cdb_init.lo: src/libstddjb/cdb_init.c src/include/skalibs/cdb.h src/include/skalibs/djbunix.h
 src/libstddjb/cdb_init_at.o src/libstddjb/cdb_init_at.lo: src/libstddjb/cdb_init_at.c src/include/skalibs/cdb.h src/include/skalibs/djbunix.h src/include/skalibs/unix-transactional.h
 src/libstddjb/cdb_init_fromfd.o src/libstddjb/cdb_init_fromfd.lo: src/libstddjb/cdb_init_fromfd.c src/include/skalibs/bsdsnowflake.h src/include/skalibs/cdb.h
 src/libstddjb/cdb_p.o src/libstddjb/cdb_p.lo: src/libstddjb/cdb_p.c src/libstddjb/cdb-internal.h src/include/skalibs/cdb.h
 src/libstddjb/cdb_traverse_next.o src/libstddjb/cdb_traverse_next.lo: src/libstddjb/cdb_traverse_next.c src/libstddjb/cdb-internal.h src/include/skalibs/cdb.h src/include/skalibs/uint32.h
 src/libstddjb/cdb_zero.o src/libstddjb/cdb_zero.lo: src/libstddjb/cdb_zero.c src/include/skalibs/cdb.h
-src/libstddjb/cdbmake.o src/libstddjb/cdbmake.lo: src/libstddjb/cdbmake.c src/include/skalibs/buffer.h src/libstddjb/cdb-internal.h src/include/skalibs/cdbmake.h src/include/skalibs/diuint32.h src/include/skalibs/genalloc.h src/include/skalibs/uint32.h
+src/libstddjb/cdbmake_add.o src/libstddjb/cdbmake_add.lo: src/libstddjb/cdbmake_add.c src/include/skalibs/buffer.h src/libstddjb/cdb-internal.h src/libstddjb/cdbmake-internal.h src/include/skalibs/cdbmake.h src/include/skalibs/diuint32.h src/include/skalibs/genalloc.h src/include/skalibs/uint32.h
+src/libstddjb/cdbmake_addbeginend.o src/libstddjb/cdbmake_addbeginend.lo: src/libstddjb/cdbmake_addbeginend.c src/include/skalibs/buffer.h src/libstddjb/cdbmake-internal.h src/include/skalibs/diuint32.h src/include/skalibs/genalloc.h src/include/skalibs/uint32.h
+src/libstddjb/cdbmake_addv.o src/libstddjb/cdbmake_addv.lo: src/libstddjb/cdbmake_addv.c src/include/skalibs/buffer.h src/libstddjb/cdb-internal.h src/libstddjb/cdbmake-internal.h src/include/skalibs/cdbmake.h src/include/skalibs/diuint32.h src/include/skalibs/genalloc.h src/include/skalibs/siovec.h src/include/skalibs/uint32.h
+src/libstddjb/cdbmake_finish.o src/libstddjb/cdbmake_finish.lo: src/libstddjb/cdbmake_finish.c src/include/skalibs/buffer.h src/libstddjb/cdbmake-internal.h src/include/skalibs/cdbmake.h src/include/skalibs/diuint32.h src/include/skalibs/genalloc.h src/include/skalibs/uint32.h
+src/libstddjb/cdbmake_posplus.o src/libstddjb/cdbmake_posplus.lo: src/libstddjb/cdbmake_posplus.c src/libstddjb/cdbmake-internal.h
+src/libstddjb/cdbmake_start.o src/libstddjb/cdbmake_start.lo: src/libstddjb/cdbmake_start.c src/include/skalibs/buffer.h src/include/skalibs/cdbmake.h src/include/skalibs/genalloc.h
 src/libstddjb/child_spawn.o src/libstddjb/child_spawn.lo: src/libstddjb/child_spawn.c src/include/skalibs/allreadwrite.h src/include/skalibs/config.h src/include/skalibs/djbunix.h src/include/skalibs/env.h src/include/skalibs/exec.h src/include/skalibs/sig.h src/include/skalibs/sysdeps.h src/include/skalibs/types.h
 src/libstddjb/child_spawn0.o src/libstddjb/child_spawn0.lo: src/libstddjb/child_spawn0.c src/include/skalibs/allreadwrite.h src/include/skalibs/config.h src/include/skalibs/djbunix.h src/include/skalibs/exec.h src/include/skalibs/sig.h src/include/skalibs/sysdeps.h
 src/libstddjb/child_spawn1_internal.o src/libstddjb/child_spawn1_internal.lo: src/libstddjb/child_spawn1_internal.c src/include/skalibs/allreadwrite.h src/include/skalibs/config.h src/include/skalibs/djbunix.h src/include/skalibs/exec.h src/include/skalibs/sig.h src/include/skalibs/sysdeps.h
diff --git a/src/include/skalibs/cdbmake.h b/src/include/skalibs/cdbmake.h
index 0da0a86..18b798a 100644
--- a/src/include/skalibs/cdbmake.h
+++ b/src/include/skalibs/cdbmake.h
@@ -4,9 +4,8 @@
 #define SKALIBS_CDBMAKE_H
 
 #include <stdint.h>
+#include <sys/uio.h>
 
-#include <skalibs/diuint32.h>
-#include <skalibs/allreadwrite.h>
 #include <skalibs/genalloc.h>
 #include <skalibs/buffer.h>
 
@@ -18,11 +17,11 @@ struct cdbmaker_s
   buffer b ;
   char buf[BUFFER_OUTSIZE] ;
 } ;
-
-#define CDBMAKER_ZERO { .hplist = GENALLOC_ZERO, .pos = 2048, .b = BUFFER_INIT(&fd_writev, -1, 0, 0) }
+#define CDBMAKER_ZERO { .hplist = GENALLOC_ZERO, .pos = 2048, .b = BUFFER_ZERO, .buf = { 0 } }
 
 extern int cdbmake_start (cdbmaker *, int) ;
 extern int cdbmake_add (cdbmaker *, char const *, uint32_t, char const *, uint32_t) ;
+extern int cdbmake_addv (cdbmaker *, struct iovec const *, unsigned int, struct iovec const *, unsigned int) ;
 extern int cdbmake_finish (cdbmaker *) ;
 
 #endif
diff --git a/src/libstddjb/cdb-internal.h b/src/libstddjb/cdb-internal.h
index 0db5a5d..d2b7324 100644
--- a/src/libstddjb/cdb-internal.h
+++ b/src/libstddjb/cdb-internal.h
@@ -4,11 +4,16 @@
 #define SKALIBS_CDB_INTERNAL_H
 
 #include <stdint.h>
+#include <sys/uio.h>
 
 #include <skalibs/gccattributes.h>
 #include <skalibs/cdb.h>
 
+#define CDB_HASHSTART 5381
+
+extern uint32_t cdb_hashadd (uint32_t, uint8_t) ;
 extern uint32_t cdb_hash (char const *, uint32_t) gccattr_pure ;
+extern uint32_t cdb_hashv (struct iovec const *, unsigned int) gccattr_pure ;
 extern char const *cdb_p (cdb const *, uint32_t, uint32_t) gccattr_pure ;
 
 #endif
diff --git a/src/libstddjb/cdb_hash.c b/src/libstddjb/cdb_hash.c
index a7b626b..3849f02 100644
--- a/src/libstddjb/cdb_hash.c
+++ b/src/libstddjb/cdb_hash.c
@@ -4,14 +4,6 @@
 
 #include "cdb-internal.h"
 
-#define CDB_HASHSTART 5381
-
-static inline uint32_t cdb_hashadd (uint32_t h, unsigned char c)
-{
-  h += (h << 5) ;
-  return h ^ c ;
-}
-
 uint32_t cdb_hash (char const *buf, uint32_t len)
 {
   uint32_t h = CDB_HASHSTART ;
diff --git a/src/libstddjb/cdb_hashadd.c b/src/libstddjb/cdb_hashadd.c
new file mode 100644
index 0000000..426fb95
--- /dev/null
+++ b/src/libstddjb/cdb_hashadd.c
@@ -0,0 +1,9 @@
+/* ISC license. */
+
+#include "cdb-internal.h"
+
+uint32_t cdb_hashadd (uint32_t h, uint8_t c)
+{
+  h += (h << 5) ;
+  return h ^ c ;
+}
diff --git a/src/libstddjb/cdb_hashv.c b/src/libstddjb/cdb_hashv.c
new file mode 100644
index 0000000..e2b80d9
--- /dev/null
+++ b/src/libstddjb/cdb_hashv.c
@@ -0,0 +1,14 @@
+/* ISC license. */
+
+#include <stdint.h>
+
+#include "cdb-internal.h"
+
+uint32_t cdb_hashv (struct iovec const *v, unsigned int n)
+{
+  uint32_t h = CDB_HASHSTART ;
+  for (unsigned int i = 0 ; i < n ; i++)
+    for (size_t j = 0 ; j < v[i].iov_len ; j++)
+      h = cdb_hashadd(h, ((uint8_t const *)v[i].iov_base)[j]) ;
+  return h ;
+}
diff --git a/src/libstddjb/cdbmake-internal.h b/src/libstddjb/cdbmake-internal.h
new file mode 100644
index 0000000..1d5bd24
--- /dev/null
+++ b/src/libstddjb/cdbmake-internal.h
@@ -0,0 +1,14 @@
+/* ISC license. */
+
+#ifndef SKALIBS_CDBMAKE_INTERNAL_H
+#define SKALIBS_CDBMAKE_INTERNAL_H
+
+#include <stdint.h>
+
+#include <skalibs/cdbmake.h>
+
+extern int cdbmake_posplus (cdbmaker *, uint32_t) ;
+extern int cdbmake_addend (cdbmaker *, uint32_t, uint32_t, uint32_t) ;
+extern int cdbmake_addbegin (cdbmaker *, uint32_t, uint32_t) ;
+
+#endif
diff --git a/src/libstddjb/cdbmake_add.c b/src/libstddjb/cdbmake_add.c
new file mode 100644
index 0000000..f7fd096
--- /dev/null
+++ b/src/libstddjb/cdbmake_add.c
@@ -0,0 +1,22 @@
+/* ISC license. */
+
+#include <skalibs/uint32.h>
+#include <skalibs/diuint32.h>
+#include <skalibs/buffer.h>
+#include <skalibs/genalloc.h>
+#include <skalibs/cdbmake.h>
+#include "cdb-internal.h"
+#include "cdbmake-internal.h"
+
+int cdbmake_add (cdbmaker *c, char const *key, uint32_t keylen, char const *data, uint32_t datalen)
+{
+  if (!cdbmake_addbegin(c, keylen, datalen)
+   || buffer_put(&c->b, key, keylen) < keylen
+   || buffer_put(&c->b, data, datalen) < datalen
+   || !cdbmake_addend(c, keylen, datalen, cdb_hash(key, keylen)))
+  {
+    genalloc_free(diuint32, &c->hplist) ;
+    return 0 ;
+  }
+  return 1 ;
+}
diff --git a/src/libstddjb/cdbmake_addbeginend.c b/src/libstddjb/cdbmake_addbeginend.c
new file mode 100644
index 0000000..37d35c8
--- /dev/null
+++ b/src/libstddjb/cdbmake_addbeginend.c
@@ -0,0 +1,24 @@
+/* ISC license. */
+
+#include <stdint.h>
+#include <errno.h>
+
+#include <skalibs/uint32.h>
+#include <skalibs/diuint32.h>
+#include <skalibs/buffer.h>
+#include <skalibs/genalloc.h>
+#include "cdbmake-internal.h"
+
+int cdbmake_addend (cdbmaker *c, uint32_t keylen, uint32_t datalen, uint32_t h)
+{
+  diuint32 blah = { .left = h, .right = c->pos } ;
+  return genalloc_append(diuint32, &c->hplist, &blah) && cdbmake_posplus(c, 8) && cdbmake_posplus(c, keylen) && cdbmake_posplus(c, datalen) ;
+}
+
+int cdbmake_addbegin (cdbmaker *c, uint32_t keylen, uint32_t datalen)
+{
+  char buf[8] ;
+  uint32_pack(buf, keylen) ;
+  uint32_pack(buf + 4, datalen) ;
+  return buffer_put(&c->b, buf, 8) == 8 ;
+}
diff --git a/src/libstddjb/cdbmake_addv.c b/src/libstddjb/cdbmake_addv.c
new file mode 100644
index 0000000..d8719a4
--- /dev/null
+++ b/src/libstddjb/cdbmake_addv.c
@@ -0,0 +1,30 @@
+/* ISC license. */
+
+#include <errno.h>
+#include <sys/uio.h>
+
+#include <skalibs/uint32.h>
+#include <skalibs/diuint32.h>
+#include <skalibs/buffer.h>
+#include <skalibs/genalloc.h>
+#include <skalibs/siovec.h>
+#include <skalibs/cdbmake.h>
+#include "cdb-internal.h"
+#include "cdbmake-internal.h"
+
+int cdbmake_addv (cdbmaker *c, struct iovec const *kv, unsigned int kn, struct iovec const *dv, unsigned int dn)
+{
+  size_t keylen = siovec_len(kv, kn) ;
+  size_t datalen = siovec_len(dv, dn) ;
+  if (keylen > UINT32_MAX || datalen > UINT32_MAX) return (errno = EOVERFLOW, 0) ;
+
+  if (!cdbmake_addbegin(c, keylen, datalen)
+   || buffer_putv(&c->b, kv, kn) < keylen
+   || buffer_putv(&c->b, dv, dn) < datalen
+   || !cdbmake_addend(c, keylen, datalen, cdb_hashv(kv, kn)))
+  {
+    genalloc_free(diuint32, &c->hplist) ;
+    return 0 ;
+  }
+  return 1 ;
+}
diff --git a/src/libstddjb/cdbmake.c b/src/libstddjb/cdbmake_finish.c
index 4cc4fc8..fe0b35a 100644
--- a/src/libstddjb/cdbmake.c
+++ b/src/libstddjb/cdbmake_finish.c
@@ -9,50 +9,7 @@
 #include <skalibs/buffer.h>
 #include <skalibs/genalloc.h>
 #include <skalibs/cdbmake.h>
-#include "cdb-internal.h"
-
-int cdbmake_start (cdbmaker *c, int fd)
-{
-  c->hplist = genalloc_zero ;
-  c->pos = 2048 ;
-  buffer_init(&c->b, &buffer_write, fd, c->buf, BUFFER_OUTSIZE) ;
-  return lseek(fd, c->pos, SEEK_SET) >= 0 ;
-}
-
-static int posplus (cdbmaker *c, uint32_t len)
-{
-  uint32_t newpos = c->pos + len ;
-  if (newpos < len) return (errno = ENOMEM, 0) ;
-  c->pos = newpos ;
-  return 1 ;
-}
-
-static inline int cdbmake_addend (cdbmaker *c, uint32_t keylen, uint32_t datalen, uint32_t h)
-{
-  diuint32 blah = { .left = h, .right = c->pos } ;
-  return genalloc_append(diuint32, &c->hplist, &blah) && posplus(c, 8) && posplus(c, keylen) && posplus(c, datalen) ;
-}
-
-static inline ssize_t cdbmake_addbegin (cdbmaker *c, uint32_t keylen, uint32_t datalen)
-{
-  char buf[8] ;
-  uint32_pack(buf, keylen) ;
-  uint32_pack(buf + 4, datalen) ;
-  return buffer_put(&c->b, buf, 8) == 8 ;
-}
-
-int cdbmake_add (cdbmaker *c, char const *key, uint32_t keylen, char const *data, uint32_t datalen)
-{
-  if (!cdbmake_addbegin(c, keylen, datalen)
-   || buffer_put(&c->b, key, keylen) < 0
-   || buffer_put(&c->b, data, datalen) < 0
-   || !cdbmake_addend(c, keylen, datalen, cdb_hash(key, keylen)))
-  {
-    genalloc_free(diuint32, &c->hplist) ;
-    return 0 ;
-  }
-  return 1 ;
-}
+#include "cdbmake-internal.h"
 
 int cdbmake_finish (cdbmaker *c)
 {
@@ -109,13 +66,13 @@ int cdbmake_finish (cdbmaker *c)
         uint32_pack(buf, hp[j].left) ;
         uint32_pack(buf + 4, hp[j].right) ;
         if (buffer_put(&c->b, buf, 8) < 0) return 0 ;
-        if (!posplus(c, 8)) return 0 ;
+        if (!cdbmake_posplus(c, 8)) return 0 ;
       }
     }
   }
 
   if (!buffer_flush(&c->b)
-   || lseek(buffer_fd(&c->b), 0, SEEK_SET) < 0
-   || buffer_putflush(&c->b, final, 2048) < 0) return 0 ;
+   || lseek(buffer_fd(&c->b), 0, SEEK_SET) == -1
+   || buffer_putflush(&c->b, final, 2048) < 2048) return 0 ;
   return 1 ;
 }
diff --git a/src/libstddjb/cdbmake_posplus.c b/src/libstddjb/cdbmake_posplus.c
new file mode 100644
index 0000000..a2f4468
--- /dev/null
+++ b/src/libstddjb/cdbmake_posplus.c
@@ -0,0 +1,14 @@
+/* ISC license. */
+
+#include <stdint.h>
+#include <errno.h>
+
+#include "cdbmake-internal.h"
+
+int cdbmake_posplus (cdbmaker *c, uint32_t len)
+{
+  uint32_t newpos = c->pos + len ;
+  if (newpos < len) return (errno = ENOMEM, 0) ;
+  c->pos = newpos ;
+  return 1 ;
+}
diff --git a/src/libstddjb/cdbmake_start.c b/src/libstddjb/cdbmake_start.c
new file mode 100644
index 0000000..94f1635
--- /dev/null
+++ b/src/libstddjb/cdbmake_start.c
@@ -0,0 +1,15 @@
+/* ISC license. */
+
+#include <unistd.h>
+
+#include <skalibs/buffer.h>
+#include <skalibs/genalloc.h>
+#include <skalibs/cdbmake.h>
+
+int cdbmake_start (cdbmaker *c, int fd)
+{
+  c->hplist = genalloc_zero ;
+  c->pos = 2048 ;
+  buffer_init(&c->b, &buffer_write, fd, c->buf, BUFFER_OUTSIZE) ;
+  return lseek(fd, c->pos, SEEK_SET) >= 0 ;
+}