New db design with mmap()

Signed-off-by: Laurent Bercot <ska@appnovation.com>

7 files changed, 183 insertions, 220 deletions

diff --git a/src/include-local/db.h b/src/include-local/db.h
index a937e7a..be7ef20 100644
--- a/src/include-local/db.h
+++ b/src/include-local/db.h
@@ -5,46 +5,33 @@
 
 #include <stdint.h>
 
+#include <skalibs/cdb.h>
 
- /* Service types and db representation in memory */
-
-#define S6RC_INSTANCES_MAX 0xffffffU
-#define S6RC_INSTANCEPARAM_MAXLEN 0xffffffU
+#define S6RC_ARGV_MAX 0x00ffffffu
+#define S6RC_INSTANCES_MAX 0x00ffffffu
+#define S6RC_INSTANCEPARAM_MAXLEN 0x00ffffffu
 
-typedef enum s6rc_stype_e s6rc_stype_t, *s6rc_stype_t_ref ;
-enum s6rc_stype_e
-{
-  S6RC_STYPE_LONGRUN,
-  S6RC_STYPE_ONESHOT,
-  S6RC_STYPE_EXTERNAL,
-  S6RC_STYPE_BUNDLE,
-  S6RC_STYPE_VIRTUAL,
-  S6RC_STYPE_PHAIL
-} ;
+ /* Service types and db representation in memory */
 
+#define S6RC_STYPE_LONGRUN 0
+#define S6RC_STYPE_ONESHOT 1
+#define S6RC_STYPE_EXTERNAL 2
+#define S6RC_STYPE_BUNDLE 3
+#define S6RC_STYPE_VIRTUAL 4
+#define S6RC_STYPE_N 5
 
-typedef struct s6rc_baseid_s s6rc_baseid_t, *s6rc_baseid_t_ref ;
-struct s6rc_baseid_s
-{
-  uint32_t i ;
-  s6rc_stype_t stype ;
-} ;
+typedef uint32_t s6rc_id_t, *s6rc_id_t_ref ;
 
-typedef struct s6rc_id_s s6rc_id_t, *s6rc_id_t_ref ;
-struct s6rc_id_s
-{
-  char const *param ;
-  s6rc_baseid_t baseid ;
-} ;
+#define stype(sid) ((sid) >> 28)
+#define snum(sid) ((sid) & 0x0fffffffu)
 
 typedef struct s6rc_common_s s6rc_common_t, *s6rc_common_t_ref ;
 struct s6rc_common_s
 {
   uint32_t name ;
+  uint32_t flags ;
   uint32_t deps[2] ;
   uint32_t ndeps[2] ;
-  uint32_t flag_essential : 1 ;
-  uint32_t flags : 31 ;
 } ;
 
 typedef struct s6rc_atomic_s s6rc_atomic_t, *s6rc_atomic_t_ref ;
@@ -58,7 +45,6 @@ typedef struct s6rc_oneshot_s s6rc_oneshot_t, *s6rc_oneshot_t_ref ;
 struct s6rc_oneshot_s
 {
   s6rc_atomic_t satomic ;
-  uint32_t argc[2] ;
   uint32_t argv[2] ;
 } ;
 
@@ -66,9 +52,9 @@ typedef struct s6rc_longrun_s s6rc_longrun_t, *s6rc_longrun_t_ref ;
 struct s6rc_longrun_s
 {
   s6rc_atomic_t satomic ;
-  s6rc_sid_t consumer ;
   uint32_t nproducers ;
   uint32_t producers ;
+  s6rc_id_t consumer ;
 } ;
 
 typedef struct s6rc_external_s s6rc_external_t, *s6rc_external_t_ref ;
@@ -85,30 +71,31 @@ struct s6rc_bundle_s
   uint32_t contents ;
 } ;
 
-typedef struct s6rc_deptype_s s6rc_deptype_t, *s6rc_deptype_t_ref ;
-struct s6rc_deptype_s
-{
-  uint8_t passive : 1 ;
-  uint8_t soft : 1 ;
-  uint8_t loose : 1 ;
-} ;
+#define s6rc_deptype_passive(dep) ((dep) & 0x01u)
+#define s6rc_deptype_soft(dep) ((dep) & 0x02u)
+#define s6rc_deptype_loose(dep) ((dep) & 0x04u)
 
 typedef struct s6rc_db_s s6rc_db_t, *s6rc_db_t_ref ;
 struct s6rc_db_s
 {
-  uint32_t n[STYPE_PHAIL << 1] ;
-  s6rc_longrun_t *longruns ;
-  s6rc_oneshot_t *oneshots ;
-  s6rc_external_t *externals ;
-  s6rc_bundle_t *bundles ;
-  s6rc_bundle_t *virtuals ;
-  s6rc_baseid_t *deps[2] ;
-  s6rc_deptype_t *deptypes[2] ;
-  s6rc_baseid_t *producers ;
-  char const **argvs ;
-  char *storage ;
+  char const *map ;
+  size_t size ;
+  uint32_t const *n ;
+  s6rc_longrun_t const *longruns ;
+  s6rc_oneshot_t const *oneshots ;
+  s6rc_external_t const *externals ;
+  s6rc_bundle_t const *bundles ;
+  s6rc_bundle_t const *virtuals ;
+  s6rc_id_t const *deps[2] ;
+  uint8_t const *deptypes[2] ;
+  s6rc_id_t const *producers ;
+  char const *storage ;
+  char const **argvs ;  /* alloced */
 } ;
+#define S6RC_DB_ZERO = { .map = 0, .len = 0 }
+
 
-extern s6rc_common_t const *s6rc_service_common (s6rc_db_t const *, s6rc_sid_t const *) ;
+extern int s6rc_service_resolve (cdb_t *, char const *, s6rc_id_t *, char const **) ;
+extern s6rc_common_t const *s6rc_service_common (s6rc_db_t const *, s6rc_id_t const *) ;
 
 #endif
diff --git a/src/libs6rcd/s6rc_db_free.c b/src/libs6rcd/s6rc_db_free.c
index b8eb54e..07f4ccd 100644
--- a/src/libs6rcd/s6rc_db_free.c
+++ b/src/libs6rcd/s6rc_db_free.c
@@ -1,18 +1,16 @@
 /* ISC license. */
 
+#include <errno.h>
+
 #include <skalibs/alloc.h>
 
 #include "db.h"
 
 void s6rc_db_free (s6rc_db_t *db)
 {
-  alloc_free(db->storage) ;
+  int e = errno ;
   alloc_free(db->argvs) ;
-  alloc_free(db->producers) ;
-  alloc_free(db->deptypes[0]) ;
-  alloc_free(db->deps[0]) ;
-  alloc_free(db->bundles) ;
-  alloc_free(db->externals) ;
-  alloc_free(db->oneshots) ;
-  alloc_free(db->longruns) ;
+  munmap(db->map, db->size) ;
+  db->map = 0 ;
+  errno = e ;
 }
diff --git a/src/libs6rcd/s6rc_db_load.c b/src/libs6rcd/s6rc_db_load.c
new file mode 100644
index 0000000..e42afc9
--- /dev/null
+++ b/src/libs6rcd/s6rc_db_load.c
@@ -0,0 +1,129 @@
+/* ISC license. */
+
+#include <stdint.h>
+#include <string.h>
+#include <errno.h>
+#include <sys/stat.h>
+#include <sys/mman.h>
+
+#include <skalibs/uint32.h>
+#include <skalibs/alloc.h>
+#include <skalibs/buffer.h>
+#include <skalibs/djbunix.h>
+#include <skalibs/sha256.h>
+
+#include "db.h"
+
+#include <skalibs/posixishard.h>
+
+static int gethu32 (buffer *b, SHA256Schedule *ctx, uint32_t *n)
+{
+  if (buffer_get(b, (char *)n, 4) < 4) return 0 ;
+  sha256_update(ctx, (char *)n, 4) ;
+  return 1 ;
+}
+
+int s6rc_db_load (char const *dir, s6rc_db_t *db)
+{
+  SHA256Schedule ctx = SHA256_INIT() ;
+  uint32_t ntotal, ndeps, nproducers, storagelen, nargv ;
+  size_t len = strlen(dir) ;
+  buffer b ;
+  int fd ;
+  char buf[4096] ;
+  char fn[len + 10] ;
+  memcpy(fn, dir, len) ;
+  memcpy(fn + len, "/db_nomap", 10) ;
+  fd = openc_read(fn) ;
+  if (fd == -1) return 0 ;
+  buffer_init(&b, &buffer_read, fd, buf, 4096) ;
+  {
+    uint32_t canary ;
+    if (!gethu32(&b, &canary)) goto err0 ;
+    if (canary != 0x11223344u) { errno = EILSEQ ; goto err0 ; }
+  }
+  if (!gethu32(&b, &ctx, &ntotal)) goto err0 ;
+  if (!gethu32(&b, &ctx, &ndeps)) goto err0 ;
+  if (!gethu32(&b, &ctx, &nproducers)) goto err0 ;
+  if (!gethu32(&b, &ctx, &storagelen)) goto err0 ;
+  if (!gethu32(&b, &ctx, &nargv)) goto err0 ;
+  if (nargv > S6RC_ARGV_MAX) goto eproto0 ;
+
+  {
+    uint32_t argvs[nargv ? nargv : 1] ;
+    if (buffer_get(&b, (char *)argvs, nargv * 4) < nargv * 4) goto err0 ;
+    {
+      char c ;
+      ssize_t r = buffer_get(&b, &c, 1) ;
+      if (r < 0) goto err0 ;
+      if (r) goto eproto0 ;
+    }
+    fd_close(fd) ;
+    sha256_update(&ctx, (char *)argvs, nargv * 4) ;
+    memcpy(fn + len, "/db", 4) ;
+
+    {
+      struct stat st ;
+      void *map ;
+      fd = openc_read(fn) ;
+      if (fd == -1) return 0 ;
+      if (fstat(fd, &st) == -1) goto err0 ;
+      if (!S_ISREG(st.st_mode)) goto eproto0 ;
+      if (st.st_size < 8 * S6RC_STYPE_N) goto eproto0 ;
+      map = mmap(0, st.st_size, PROT_READ, MAP_SHARED, fd, 0) ;
+      if (map == MAP_FAILED) goto err0 ;
+      fd_close(fd) ;
+      db->size = st.st_size ;
+      db->map = map ;
+    }
+    db->n = (uint32_t const *)db->map ;
+    if (ntotal != db->n[0] + db->n[1] + db->n[2] + db->n[3] + db->n[4] + db->n[5] + db->n[6] + db->n[7] + db->n[8] + db->n[9]) goto eproto1 ;
+
+    {
+      ssize_t r ;
+      memcpy(fn + len, "/hash", 6) ;
+      r = openreadnclose(fn, buf+32, 33) ;
+      if (r == 33) goto eproto1 ;
+      if (r < 32) goto err1 ;
+      sha256_update(&ctx, db->map, db->size) ;
+      sha256_final(&ctx, buf) ;
+      if (memcmp(buf, buf+32, 32)) goto eproto1 ;
+    }
+
+    db->longruns = (s6rc_longrun_t const *)(db->map + 4 * 2 * S6RC_STYPE_N) ;
+    db->oneshots = (s6rc_oneshot_t const *)(db->longruns + db->n[S6RC_STYPE_LONGRUN] + db->n[S6RC_STYPE_N + S6RC_STYPE_LONGRUN]) ;
+    db->externals = (s6rc_external_t const *)(db->oneshots + db->n[S6RC_STYPE_ONESHOT] + db->n[S6RC_STYPE_N + S6RC_STYPE_ONESHOT]) ;
+    db->bundles = (s6rc_bundle_t const *)(db->externals + db->n[S6RC_STYPE_EXTERNAL] + db->n[S6RC_STYPE_N + S6RC_STYPE_EXTERNAL]) ;
+    db->virtuals = (s6rc_bundle_t const *)(db->bundles + db->n[S6RC_STYPE_BUNDLE] + db->n[S6RC_STYPE_N + S6RC_STYPE_BUNDLE]) ;
+    db->deps[0] = (s6rc_id_t const *)(db->virtuals + db->n[S6RC_STYPE_VIRTUAL] + db->n[S6RC_STYPE_N + S6RC_STYPE_VIRTUAL]) ;
+    db->deps[1] = db->deps[0] + ndeps ;
+    db->producers = db->deps[1] + ndeps ;
+    db->deptypes[0] = (uint8_t const *)(db->producers + nproducers) ;
+    db->deptypes[1] = db->deptypes[0] + ndeps ;
+    db->storage = (char const *)(db->deptypes[1] + ndeps) ;
+    if (db->storage + storagelen != db->map + db->size) goto eproto1 ;
+
+    db->argvs = (char const **)alloc(sizeof(char const *) * nargv) ;
+    if (!db->argvs) goto err1 ;
+    for (uint32_t i = 0 ; i < nargv ; i++)
+      db->argvs[i] = argvs[i] ? db->storage + argvs[i] : 0 ;
+  }
+  return 1 ;
+
+ eproto1:
+  errno = EPROTO ;
+ err1:
+  {
+    int e = errno ;
+    munmap(db->map, db->size) ;
+    errno = e ;
+  }
+  db->map = 0 ;
+  return 0 ;
+
+ eproto0:
+  errno = EPROTO ;
+ err0:
+  fd_close(fd) ;
+  return 0 ;
+}
diff --git a/src/libs6rcd/s6rc_db_read.c b/src/libs6rcd/s6rc_db_read.c
deleted file mode 100644
index 9b60184..0000000
--- a/src/libs6rcd/s6rc_db_read.c
+++ /dev/null
@@ -1,131 +0,0 @@
-/* ISC license. */
-
-#include <stdint.h>
-#include <string.h>
-#include <errno.h>
-
-#include <skalibs/alloc.h>
-#include <skalibs/buffer.h>
-#include <skalibs/djbunix.h>
-#include <skalibs/sha256.h>
-
-#include "db.h"
-
-#include <skalibs/posixishard.h>
-
-static int geth (buffer *b, SHA256Schedule *ctx, char *s, size_t len)
-{
-  if (buffer_get(b, s, len) < len) return 0 ;
-  sha256_update(ctx, s, len) ;
-  return 1 ;
-}
-
-static int readu32 (buffer *b, SHA256Schedule *ctx, uint32_t *x)
-{
-  char pack[4] ;
-  if (!geth(b, ctx, pack, 4)) return 0 ;
-  uint32_unpack_big(pack, x) ;
-  return 1 ;
-}
-
-static int readdeps (buffer *b, SHA256Schedule *ctx, s6rc_baseid_t *deps, uint32_t ndeps, uint32_t const *dbn)
-{
-  for (uint32_t i = 0 ; i < (ndeps << 1) ; i++)
-  {
-    char pack[5] ;
-    if (!geth(b, ctx, pack, 5)) return 0 ;
-    if ((unsigned char)pack[0] >= S6RC_STYPE_PHAIL) return (errno = EPROTO, 0) ;
-    deps[i].stype = pack[0] ;
-    uint32_unpack_big(pack + 1, &deps[i].i) ;
-  }
-  return 1 ;
-}
-
-int s6rc_db_read_main (char const *dir, s6rc_db_t const *db, char *dbhash)
-{
-  SHA256Schedule ctx = SHA256_INIT() ;
-  int fd ;
-  buffer b ;
-  size_t len = strlen(dir) ;
-  uint32_t ndeps, nproducers, nargvs, storagelen ;
-  char buf[1024] ;
-  char fn[len + 4] ;
-  memcpy(fn, dir, len) ;
-  memcpy(fn + len, "/db", 4) ;
-  fd = openc_read(fn) ;
-  if (fd == -1) return 0 ;
-  buffer_init(&b, &buffer_read, fd, buf, 1024) ;
-
-  {
-    char banner[S6RC_DB_BANNER_START_LEN] ;
-    if (!geth(&b, &ctx, banner, S6RC_DB_BANNER_START_LEN)) goto err0 ;
-    if (memcmp(banner, S6RC_DB_BANNER_START, S6RC_DB_BANNER_START_LEN)) { errno = EPROTO ; goto err0 ; }
-  }
-
-  {
-    uint32_t ntotal ;
-    uint32_t nacc = 0 ;
-    if (!readu32(&b, &ctx, &ntotal)) goto err0 ;
-    for (s6rc_stype_t type = 0 ; (type < S6RC_STYPE_PHAIL << 1) ; type++)
-    {
-      if (!readu32(&b, &ctx, db->n + type)) goto err0 ;
-      nacc += db->n[type] ;
-    }
-    if (ntotal != nacc) { errno = EPROTO ; goto err0 ; }
-  }
-  if (!readu32(&b, &ctx, &ndeps)) goto err0 ;
-  if (!readu32(&b, &ctx, &nproducers)) goto err0 ;
-  if (!readu32(&b, &ctx, &nargvs)) goto err0 ;
-  if (!readu32(&b, &ctx, &storagelen)) goto err0 ;
-
-  db->longruns = alloc(sizeof(s6rc_longrun_t) * (db->n[S6RC_STYPE_LONGRUN] + db->n[S6RC_STYPE_PHAIL + S6RC_STYPE_LONGRUN])) ;
-  if (!db->longruns) goto err0 ;
-  db->oneshots = alloc(sizeof(s6rc_oneshot_t) * (db->n[S6RC_STYPE_ONESHOT] + db->n[S6RC_STYPE_PHAIL + S6RC_STYPE_ONESHOT])) ;
-  if (!db->oneshots) goto err1 ;
-  db->externals = alloc(sizeof(s6rc_external_t) * (db->n[S6RC_STYPE_EXTERNAL] + db->n[S6RC_STYPE_PHAIL + S6RC_STYPE_EXTERNAL])) ;
-  if (!db->externals) goto err2 ;
-  db->bundles = alloc(sizeof(s6rc_bundle_t) * (db->n[S6RC_STYPE_BUNDLE] + db->n[S6RC_STYPE_PHAIL + S6RC_STYPE_BUNDLE] + db->n[S6RC_STYPE_VIRTUAL] + db->n[S6RC_STYPE_PHAIL + S6RC_TYPE_VIRTUAL])) ;
-  if (!db->bundles) goto err3 ;
-  db->virtuals = db->bundles + db->n[S6RC_STYPE_BUNDLE] + db->n[S6RC_STYPE_PHAIL + S6RC_STYPE_BUNDLE])) ;
-  db->deps[0] = alloc(sizeof(s6rc_baseid_t) * (ndeps << 1)) ;
-  if (!db->deps[0]) goto err4 ;
-  db->deps[1] = db->deps[0] + ndeps ;
-  db->deptypes[0] = alloc(sizeof(s6rc_deptype_t) * (ndeps << 1)) ;
-  if (!db->deptypes[0]) goto err5 ;
-  db->deptypes[1] = db->deptypes[0] + ndeps ;
-  db->producers = alloc(sizeof(s6rc_baseid_t) * nproducers) ;
-  if (!db->producers) goto err6 ;
-  db->argvs = alloc(sizeof(char const *) * nargvs) ;
-  if (!db->longruns) goto err7 ;
-  db->storage = alloc(storagelen) ;
-  if (!db->storage) goto err8 ;
-
-  if (!geth(&b, &ctx, db->storage, storagelen)) goto err ;
-  if (!geth(&b, &ctx, db->deptypes[0], sizeof(s6rc_deptype_t) * (ndeps << 1))) goto err ;
-  if (!readdeps(&b, &ctx, db->deps[0], ndeps, db->n)) goto err ;
-
-  sha256_final(&ctx, dbhash) ;
-  return 1 ;
-
- err:
-  alloc_free(db->storage) ;
- err8:
-  alloc_free(db->argvs) ;
- err7:
-  alloc_free(db->producers) ;
- err6:
-  alloc_free(db->deptypes[0]) ;
- err5:
-  alloc_free(db->deps[0]) ;
- err4:
-  alloc_free(db->bundles) ;
- err3:
-  alloc_free(db->externals) ;
- err2:
-  alloc_free(db->oneshots) ;
- err1:
-  alloc_free(db->longruns) ;
- err0:
-  fd_close(fd) ;
-  return 0 ;
-}
diff --git a/src/libs6rcd/s6rc_db_read_hash.c b/src/libs6rcd/s6rc_db_read_hash.c
deleted file mode 100644
index f33e657..0000000
--- a/src/libs6rcd/s6rc_db_read_hash.c
+++ /dev/null
@@ -1,15 +0,0 @@
-/* ISC license. */
-
-#include <string.h>
-#include <skalibs/djbunix.h>
-
-#include "db.h"
-
-int s6rc_db_read_hash (char const *dir, char *dbhash)
-{
-  size_t len = strlen(dir) ;
-  char fn[len + 6] ;
-  memcpy(fn, dir, len) ;
-  memcpy(fn + len, "/hash", 6) ;
-  return openreadnclose(fn, dbhash, 32) == 32 ;
-}
diff --git a/src/libs6rcd/s6rc_service_common.c b/src/libs6rcd/s6rc_service_common.c
index 93c6ee3..3361695 100644
--- a/src/libs6rcd/s6rc_service_common.c
+++ b/src/libs6rcd/s6rc_service_common.c
@@ -1,19 +1,16 @@
 /* ISC license. */
 
-#include <stdint.h>
-
 #include "db.h"
 
-s6rc_common_t const *s6rc_service_common (s6rc_db_t const *db, s6rc_sid_t const *id)
+s6rc_common_t const *s6rc_service_common (s6rc_db_t const *db, s6rc_id_t const *id)
 {
-  uint32_t i = id->i + (id->param ? db->n[id->stype] : 0) ;
-  switch (id->stype)
+  switch (stype(id))
   {
-    case STYPE_LONGRUN :  return &db->longruns[i].common ;
-    case STYPE_ONESHOT :  return &db->oneshots[i].common ;
-    case STYPE_EXTERNAL : return &db->externals[i].common ;
-    case STYPE_BUNDLE :   return &db->bundles[i].common ;
-    case STYPE_VIRTUAL :  return &db->virtuals[i].common ;
+    case STYPE_LONGRUN :  return &db->longruns[snum(id)].common ;
+    case STYPE_ONESHOT :  return &db->oneshots[snum(id)].common ;
+    case STYPE_EXTERNAL : return &db->externals[snum(id)].common ;
+    case STYPE_BUNDLE :   return &db->bundles[snum(id)].common ;
+    case STYPE_VIRTUAL :  return &db->virtuals[snum(id)].common ;
     default : return 0 ;
   }
 }
diff --git a/src/libs6rcd/s6rc_service_resolve.c b/src/libs6rcd/s6rc_service_resolve.c
index a786542..fe863be 100644
--- a/src/libs6rcd/s6rc_service_resolve.c
+++ b/src/libs6rcd/s6rc_service_resolve.c
@@ -9,10 +9,10 @@
 
 #include "db.h"
 
-int s6rc_service_resolve (cdb_t *c, s6rc_sid_t *id, char const *s)
+int s6rc_service_resolve (cdb_t *c, char const *s, s6rc_id_t *id, char const **param)
 {
   size_t len = strlen(s) ;
-  char const *param = 0 ;
+  char const *p = 0 ;
   char pack[5] ;
   int r = cdb_find(c, s, len) ;
   if (r < 0) return r ;
@@ -23,11 +23,9 @@ int s6rc_service_resolve (cdb_t *c, s6rc_sid_t *id, char const *s)
     if (at == len - 1) return (errno = EINVAL, -1) ;
     r = cdb_find(c, s, at + 1) ;
     if (r <= 0) return r ;
-    param = s + at + 1 ;
+    p = s + at + 1 ;
   }
-  if (cdb_read(c, pack, 5, cdb_datapos(c)) < 0) return -1 ;
-  id->stype = pack[0] ;
-  uint32_unpack_big(pack + 1, &id->i) ;
-  id->param = param ;
+  uint32_unpack_big(cdb_datapos(c), id) ;
+  *param = p ;
   return 1 ;
 }