From 84044171f5aa29fbce94b5ed85345cd50a8ddf56 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Wed, 3 Sep 2003 23:43:32 +0000 Subject: libz support added git-svn-id: file:///home/lennart/svn/public/syrep/trunk@20 07ea20a6-d2c5-0310-9e02-9ef735347d72 --- doc/todo.txt | 6 +- src/context.c | 2 +- src/dbutil.c | 42 +++++-- src/diff.c | 8 +- src/package.c | 348 +++++++++++++++++++++++++++++++++++++++++++++++++--------- test/Makefile | 2 +- 6 files changed, 337 insertions(+), 71 deletions(-) diff --git a/doc/todo.txt b/doc/todo.txt index a3bd5d4..a88dc60 100644 --- a/doc/todo.txt +++ b/doc/todo.txt @@ -13,9 +13,9 @@ pre-0.1: - package.c: progressive, sendfile, mmap DONE - 64bit support DONE - progress on package packing and unpacking DONE -- replace path names by their hashes -- package.c: gzip support, atomicity -- replace -z by -Z +- replace path names by their hashes DONE +- package.c: gzip support DONE +- package.c: atomicity - fix makepatch to not include already existing files (by md5) - better usage info on --help diff --git a/src/context.c b/src/context.c index 1249669..b0cfc78 100644 --- a/src/context.c +++ b/src/context.c @@ -80,7 +80,7 @@ static DB* open_db(const char*path, int dup, int recno) { return NULL; } - if (dup) + if (dup && !recno) db->set_flags(db, DB_DUPSORT); //db->set_pagesize(db, 4096*8); diff --git a/src/dbutil.c b/src/dbutil.c index 535b665..d7aa319 100644 --- a/src/dbutil.c +++ b/src/dbutil.c @@ -198,10 +198,11 @@ int get_name_by_nrecno(struct syrep_db_context *c, const struct syrep_nrecno *nr key.data = (void*) &nrecno->recno; key.size = sizeof(nrecno->recno); - + if ((ret = c->db_nrecno_name->get(c->db_nrecno_name, NULL, &key, &data, 0))) { - if (ret == DB_NOTFOUND) + if (ret == DB_NOTFOUND) { return 0; + } c->db_nrecno_name->err(c->db_nrecno_name, ret, "nrecno_name::get"); return -1; @@ -216,13 +217,15 @@ int get_name_by_nrecno(struct syrep_db_context *c, const struct syrep_nrecno *nr } static uint32_t csum_name(const struct syrep_name *name) { - uint32_t a; + uint32_t a = 0; assert(name); - a = adler32(0, NULL, 0); - a = adler32(a, name->path, strlen(name->path)); - + a = adler32(0, NULL, 0); + a = adler32(a, name->path, strlen(name->path)); + + /*fprintf(stderr, "csum: %s -> %u\n", name->path, a);*/ + return a; } @@ -251,9 +254,12 @@ int get_nrecno_by_name(struct syrep_db_context *c, const struct syrep_name *rnam for(;;) { struct syrep_nrecno nrecno; struct syrep_name name; + +/* struct syrep_nhash *foo1; */ +/* struct syrep_nrecno *foo2; */ int f; - - if (ret != 0) { + + if (ret) { if (ret == DB_NOTFOUND) { r = 0; @@ -264,6 +270,17 @@ int get_nrecno_by_name(struct syrep_db_context *c, const struct syrep_name *rnam goto finish; } + + if (memcmp(key.data, &nhash, sizeof(struct syrep_nhash))) { + r = 0; + goto finish; + } + +/* foo1 = key.data; */ +/* foo2 = data.data; */ + +/* fprintf(stderr, "cursor: %u ?= %u, %u\n", nhash.hash, foo1->hash, foo2->recno); */ + memcpy(&nrecno, data.data, sizeof(nrecno)); if ((f = get_name_by_nrecno(c, &nrecno, &name)) < 0) @@ -285,8 +302,9 @@ finish: if (cursor) cursor->c_close(cursor); - if (!r && create) + if (!r && create) { r = new_name(c, rname, rnrecno) < 0 ? -1 : 1; + } return r; } @@ -309,7 +327,7 @@ int new_name(struct syrep_db_context *c, const struct syrep_name *name, struct s return -1; } - nrecno.recno = *((db_recno_t*) data.data); + nrecno.recno = *((db_recno_t*) key.data); memset(&key, 0, sizeof(key)); memset(&data, 0, sizeof(data)); @@ -320,8 +338,8 @@ int new_name(struct syrep_db_context *c, const struct syrep_name *name, struct s data.data = &nrecno; data.size = sizeof(nrecno); - - if ((ret = c->db_nhash_nrecno->put(c->db_nhash_nrecno, NULL, &key, &data, 0))) { + + if ((ret = c->db_nhash_nrecno->put(c->db_nhash_nrecno, NULL, &key, &data, DB_NODUPDATA))) { c->db_nhash_nrecno->err(c->db_nhash_nrecno, ret, "nhash_nrecno::put"); return -1; } diff --git a/src/diff.c b/src/diff.c index b722198..1be9c63 100644 --- a/src/diff.c +++ b/src/diff.c @@ -62,7 +62,7 @@ static int add_diff_entry(DB *ddb, struct syrep_name *name, int action, struct s } if (data.size != data2.size || memcmp(data.data, data2.data, data.size)) { - fprintf(stderr, "Snapshot inconsistency\n"); + fprintf(stderr, "Snapshot inconsistency %u %u\n", data.size, data2.size); return -1; } } @@ -77,14 +77,14 @@ static int foreach(DB *ddb, struct syrep_db_context *c1, struct syrep_db_context int nrecno1_valid, nrecno2_valid; if ((nrecno1_valid = get_nrecno_by_name(c1, name, &nrecno1, 0)) < 0) - return 1; + return -1; if (nrecno1_valid) if ((md1_valid = get_current_md_by_nrecno(c1, &nrecno1, &md1)) < 0) return -1; - if ((nrecno1_valid = get_nrecno_by_name(c1, name, &nrecno2, 0)) < 0) - return 1; + if ((nrecno2_valid = get_nrecno_by_name(c2, name, &nrecno2, 0)) < 0) + return -1; if (nrecno2_valid) if ((md2_valid = get_current_md_by_nrecno(c2, &nrecno2, &md2)) < 0) diff --git a/src/package.c b/src/package.c index 0c26fef..c684020 100644 --- a/src/package.c +++ b/src/package.c @@ -30,6 +30,8 @@ #include #include +#include + #include "package.h" #include "util.h" #include "syrep.h" @@ -43,10 +45,14 @@ struct package_item { struct package_item *next; }; +#define ZBUFSIZE (64*1024) + struct package { char *base; int count; int read_fd, write_fd; + z_stream read_z, write_z; + void *read_zbuf, *write_zbuf; int x_endianess; int compressed; struct package_item *items; @@ -54,37 +60,205 @@ struct package { }; +static int alloc_read_zbuf(struct package *p) { + if (!p->read_zbuf) + if (!(p->read_zbuf = malloc(ZBUFSIZE))) { + fprintf(stderr, "malloc(): %s\n", strerror(errno)); + return -1; + } + + return 0; +} + +static int alloc_write_zbuf(struct package *p) { + if (!p->write_zbuf) + if (!(p->write_zbuf = malloc(ZBUFSIZE))) { + fprintf(stderr, "malloc(): %s\n", strerror(errno)); + return -1; + } + + return 0; +} + static ssize_t package_read(struct package *p, void *d, size_t l) { - if (!p->compressed) - return loop_read(p->read_fd, d, l); + ssize_t r = -1; + + if (!p->compressed) { + + if ((r = loop_read(p->read_fd, d, l)) < 0) + fprintf(stderr, "read(): %s\n", strerror(errno)); + + goto finish; + } + + if (!p->read_zbuf && alloc_read_zbuf(p) < 0) + return -1; + + p->read_z.next_out = d; + p->read_z.avail_out = l; + + while (p->read_z.avail_out > 0) { + int z; + + if (p->read_z.avail_in <= 0) { + ssize_t n; + + if ((n = read(p->read_fd, p->read_zbuf, ZBUFSIZE)) < 0) { + fprintf(stderr, "read(): %s\n", strerror(errno)); + goto finish; + } + + if (!n) { + r = (void*) p->read_z.next_out - d; + goto finish; + } + + p->read_z.next_in = p->read_zbuf; + p->read_z.avail_in = n; + } + + if ((z = inflate(&p->read_z, 0)) != Z_OK) { + + if (z == Z_STREAM_END) { + r = (void*) p->read_z.next_out - d; + goto finish; + } + + + fprintf(stderr, "XXX inflate() failed %i\n", z); + goto finish; + } + } + + r = l; - assert(0); +finish: - return -1; + return r; } static ssize_t package_write(struct package *p, void *d, size_t l) { - if (!args.compress_flag) - return loop_write(p->write_fd, d, l); + ssize_t r = -1; + + if (!args.compress_flag) { + + if ((r = loop_write(p->write_fd, d, l)) < 0) + fprintf(stderr, "write(): %s\n", strerror(errno)); + + goto finish; + } - assert(0); + if (!p->write_zbuf && alloc_write_zbuf(p) < 0) + return -1; - return -1; + p->write_z.next_in = d; + p->write_z.avail_in = l; + + while (p->write_z.avail_in > 0) { + size_t t; + + p->write_z.next_out = p->write_zbuf; + p->write_z.avail_out = ZBUFSIZE; + + + if (deflate(&p->write_z, Z_NO_FLUSH) != Z_OK) { + fprintf(stderr, "deflate() failed\n"); + goto finish; + } + + t = (void*) p->write_z.next_out - p->write_zbuf; + + if (t) { + ssize_t n; + + if ((n = loop_write(p->write_fd, p->write_zbuf, t)) < 0) { + fprintf(stderr, "loop_write(): %s\n", strerror(errno)); + goto finish; + } + + if (n != t) { + if ((r = (void*) p->write_z.next_in - d) > 0) + r --; + + goto finish; + } + } + } + + r = l; + +finish: + + return r; + } +#define CBUFSIZE ZBUFSIZE static int copy_deflate(struct package *p, int sfd, off_t l) { + void *buf = NULL; + int r = -1; - assert(0); + if (!(buf = malloc(CBUFSIZE))) { + fprintf(stderr, "malloc(): %s\n", strerror(errno)); + goto finish; + } - return 0; + while (l > 0) { + size_t t = MIN(l, CBUFSIZE); + ssize_t n; + + if ((n = loop_read(sfd, buf, t)) != t) { + fprintf(stderr, "read() : %s\n", n < 0 ? strerror(errno) : "EOF"); + goto finish; + } + + if (package_write(p, buf, n) != n) + goto finish; + + l -= n; + } + + r = 0; + +finish: + if (buf) + free(buf); + + return r; } static int copy_inflate(struct package *p, int dfd, off_t l) { + void *buf = NULL; + int r = -1; - assert(0); + if (!(buf = malloc(CBUFSIZE))) { + fprintf(stderr, "malloc(): %s\n", strerror(errno)); + goto finish; + } - return 0; + while (l > 0) { + size_t t = MIN(l, CBUFSIZE); + ssize_t n; + + if (package_read(p, buf, t) != t) + goto finish; + + if ((n = loop_write(dfd, buf, t)) != t) { + fprintf(stderr, "write() : %s\n", n < 0 ? strerror(errno) : "EOF"); + goto finish; + } + + l -= n; + } + + r = 0; + +finish: + if (buf) + free(buf); + + return r; } static char *tmp(char *fn, int l) { @@ -147,6 +321,67 @@ static void append_item(struct package *p, struct package_item *i) { p->items = p->last = i; } +static void close_read_fd(struct package *p) { + assert(p); + + if (p->read_fd >= 0) { + if (p->compressed) + inflateEnd(&p->read_z); + + if (p->read_fd != STDIN_FILENO) + close(p->read_fd); + p->read_fd = -1; + } +} + +static int close_write_fd(struct package *p) { + int r = 0; + assert(p); + + if (p->write_fd >= 0) { + if (args.compress_flag) { + int z; + + p->write_z.next_in = NULL; + p->write_z.avail_in = 0; + + do { + size_t t; + ssize_t n; + + p->write_z.next_out = p->write_zbuf; + p->write_z.avail_out = ZBUFSIZE; + + z = deflate(&p->write_z, Z_FINISH); + + t = (void*) p->write_z.next_out - p->write_zbuf; + if (t) { + if ((n = loop_write(p->write_fd, p->write_zbuf, t)) != t) { + fprintf(stderr, "loop_write(): %s\n", n < 0 ? strerror(errno) : "EOF"); + r = -1; + break; + } + } + + } while (z == Z_OK); + + if (!r && z != Z_STREAM_END) { + fprintf(stderr, "Final deflate() failure\n"); + r = -1; + } + + deflateEnd(&p->write_z); + } + + if (p->write_fd != STDOUT_FILENO) + close(p->write_fd); + + p->write_fd = -1; + } + + return r; +} + #define X32(i) (((i)<<24)|(((i)<<8)&0x00FF0000)|(((i)>>8)&0x0000FF00)|((i)>>24)) #define X64(i) (X32((i)>>32)|X32((i)>>32)) @@ -164,22 +399,17 @@ static int read_item(struct package *p) { if ((r = package_read(p, name, PACKAGE_ITEMNAMELEN)) != PACKAGE_ITEMNAMELEN) { if (r == 0) { - close(p->read_fd); - p->read_fd = -1; + close_read_fd(p); return 0; } else if (r > 0) fprintf(stderr, "Short read\n"); - else - fprintf(stderr, "read(): %s\n", strerror(errno)); goto fail; } - if ((r = package_read(p, &l, 8) != 8)) { + if ((r = package_read(p, &l, 8)) != 8) { if (r >= 0) fprintf(stderr, "Short read\n"); - else - fprintf(stderr, "read(): %s\n", strerror(errno)); goto fail; } @@ -247,21 +477,17 @@ static int write_item(struct package *p, struct package_item *i) { memset(name, 0, sizeof(name)); strncpy(name, i->name, PACKAGE_ITEMNAMELEN); - if ((r = package_write(p, name, PACKAGE_ITEMNAMELEN) != PACKAGE_ITEMNAMELEN)) { + if ((r = package_write(p, name, PACKAGE_ITEMNAMELEN)) != PACKAGE_ITEMNAMELEN) { if (r >= 0) fprintf(stderr, "Short write\n"); - else - fprintf(stderr, "write(): %s\n", strerror(errno)); goto fail; } l = (uint64_t) size; - if ((r = package_write(p, &l, 8) != 8)) { + if ((r = package_write(p, &l, 8)) != 8) { if (r >= 0) fprintf(stderr, "Short write\n"); - else - fprintf(stderr, "write(): %s\n", strerror(errno)); goto fail; } @@ -312,30 +538,31 @@ struct package* package_open(const char *fn, int force) { if (fn) { - if ((p->read_fd = open(fn, O_RDONLY)) < 0) { + + if (!strcmp(fn, "-")) + p->read_fd = STDIN_FILENO; + else if ((p->read_fd = open(fn, O_RDONLY)) < 0) { if (errno != ENOENT || !force) { fprintf(stderr, "Failed to open <%s>: %s\n", fn, strerror(errno)); goto finish; } - - } else if (p->read_fd >= 0) { + } + + if (p->read_fd >= 0) { uint32_t id; ssize_t r; if ((r = loop_read(p->read_fd, &id, 4)) != 4) { - if (r < 0) - fprintf(stderr, "read(): %s\n", strerror(errno)); - else - fprintf(stderr, "Short read\n"); - + fprintf(stderr, "read(): %s\n", r < 0 ? strerror(errno) : "EOF"); + goto finish; } if (id == PACKAGE_FILEID) - p->x_endianess = 0; + p->x_endianess = p->compressed = 0; else if (id == X32(PACKAGE_FILEID)) - p->x_endianess = 1; + p->x_endianess = !(p->compressed = 0); else if (id == PACKAGE_FILEIDCOMPRESSED) p->x_endianess = !(p->compressed = 1); else if (id == X32(PACKAGE_FILEIDCOMPRESSED)) @@ -348,6 +575,19 @@ struct package* package_open(const char *fn, int force) { } + if (p->compressed) { + + memset(&p->read_z, 0, sizeof(p->read_z)); + + if (inflateInit(&p->read_z) != Z_OK) { + fprintf(stderr, "zlib initialisation failure\n"); + goto finish; + } + + + } + + return p; finish: @@ -385,25 +625,32 @@ int package_save(struct package *p, const char *fn) { if (load_complete(p) < 0) goto finish; - if (!fn) + if (!fn || !strcmp(fn, "-")) p->write_fd = STDOUT_FILENO; else if ((p->write_fd = open(fn, O_RDWR|O_TRUNC|O_CREAT, 0666)) < 0) { /* RDWR for mmap compatibility */ fprintf(stderr, "open(\"%s\") XX: %s\n", fn, strerror(errno)); goto finish; } - id = p->compressed ? PACKAGE_FILEIDCOMPRESSED : PACKAGE_FILEID; + id = args.compress_flag ? PACKAGE_FILEIDCOMPRESSED : PACKAGE_FILEID; - if ((n = loop_write(p->write_fd, &id, 4) != 4)) { + if ((n = loop_write(p->write_fd, &id, 4)) != 4) { if (n < 0) - fprintf(stderr, "write(): %s\n", strerror(errno)); - else - fprintf(stderr, "Short write\n"); + fprintf(stderr, "write(): %s\n", r < 0 ? strerror(errno) : "EOF"); goto finish; } - + if (args.compress_flag) { + + memset(&p->write_z, 0, sizeof(p->write_z)); + + if (deflateInit(&p->write_z, Z_DEFAULT_COMPRESSION) != Z_OK) { + fprintf(stderr, "zlib initialisation failure\n"); + goto finish; + } + } + for (i = p->items; i; i = i->next) { if (!i->name || !i->path) @@ -419,10 +666,8 @@ int package_save(struct package *p, const char *fn) { finish: - if (fn && p->write_fd >= 0) - close(p->write_fd); - - p->write_fd = -1; + if (close_write_fd(p) < 0) + r = -1; if (r < 0 && fn) unlink(fn); @@ -507,11 +752,14 @@ void package_remove(struct package *p) { free(p->base); } - if (p->read_fd >= 0) - close(p->read_fd); + close_read_fd(p); + close_write_fd(p); - if (p->write_fd >= 0) - close(p->write_fd); + if (p->read_zbuf) + free(p->read_zbuf); + + if (p->write_zbuf) + free(p->write_zbuf); free(p); } diff --git a/test/Makefile b/test/Makefile index fc92e0e..39c51ae 100644 --- a/test/Makefile +++ b/test/Makefile @@ -4,7 +4,7 @@ ifdef VERBOSE VERBOSE=-v endif -ARGS=-T --output-directory=extract -opatch --show-deleted --show-times --prune-empty +ARGS+=-z -T --output-directory=extract -opatch --show-deleted --show-times --prune-empty %: @mkdir -p rep1 rep2 -- cgit