libz support added

git-svn-id: file:///home/lennart/svn/public/syrep/trunk@20 07ea20a6-d2c5-0310-9e02-9ef735347d72

6 files changed, 337 insertions, 71 deletions

diff --git a/doc/todo.txt b/doc/todo.txt
index a3bd5d4..a88dc60 100644
--- a/doc/todo.txt
+++ b/doc/todo.txt
@@ -13,9 +13,9 @@ pre-0.1:
 - package.c: progressive, sendfile, mmap               DONE
 - 64bit support                                        DONE
 - progress on package packing and unpacking            DONE
-- replace path names by their hashes
-- package.c: gzip support, atomicity
-- replace -z by -Z
+- replace path names by their hashes                   DONE
+- package.c: gzip support                              DONE
+- package.c: atomicity
 - fix makepatch to not include already existing files (by md5)
 
 - better usage info on --help
diff --git a/src/context.c b/src/context.c
index 1249669..b0cfc78 100644
--- a/src/context.c
+++ b/src/context.c
@@ -80,7 +80,7 @@ static DB* open_db(const char*path, int dup, int recno) {
         return NULL;
     }
 
-    if (dup)
+    if (dup && !recno)
         db->set_flags(db, DB_DUPSORT);
 
     //db->set_pagesize(db, 4096*8);
diff --git a/src/dbutil.c b/src/dbutil.c
index 535b665..d7aa319 100644
--- a/src/dbutil.c
+++ b/src/dbutil.c
@@ -198,10 +198,11 @@ int get_name_by_nrecno(struct syrep_db_context *c, const struct syrep_nrecno *nr
 
     key.data = (void*) &nrecno->recno;
     key.size = sizeof(nrecno->recno);
-    
+
     if ((ret = c->db_nrecno_name->get(c->db_nrecno_name, NULL, &key, &data, 0))) {
-        if (ret == DB_NOTFOUND)
+        if (ret == DB_NOTFOUND) {
             return 0;
+        }
         
         c->db_nrecno_name->err(c->db_nrecno_name, ret, "nrecno_name::get");
         return -1;
@@ -216,13 +217,15 @@ int get_name_by_nrecno(struct syrep_db_context *c, const struct syrep_nrecno *nr
 }
 
 static uint32_t csum_name(const struct syrep_name *name) {
-    uint32_t a;
+    uint32_t a = 0;
 
     assert(name);
     
-    a = adler32(0, NULL, 0);
-    a = adler32(a, name->path, strlen(name->path));
-
+    a = adler32(0, NULL, 0); 
+    a = adler32(a, name->path, strlen(name->path)); 
+    
+    /*fprintf(stderr, "csum: %s -> %u\n", name->path, a);*/
+    
     return a;
 }
 
@@ -251,9 +254,12 @@ int get_nrecno_by_name(struct syrep_db_context *c, const struct syrep_name *rnam
     for(;;) {
         struct syrep_nrecno nrecno;
         struct syrep_name name;
+
+/*         struct syrep_nhash *foo1; */
+/*         struct syrep_nrecno *foo2; */
         int f;
-        
-        if (ret != 0) {
+
+        if (ret) {
             if (ret == DB_NOTFOUND) {
 
                 r = 0;
@@ -264,6 +270,17 @@ int get_nrecno_by_name(struct syrep_db_context *c, const struct syrep_name *rnam
             goto finish;
         }
 
+
+        if (memcmp(key.data, &nhash, sizeof(struct syrep_nhash))) {
+            r = 0;
+            goto finish;
+        }
+            
+/*         foo1 = key.data; */
+/*         foo2 = data.data; */
+
+/*         fprintf(stderr, "cursor: %u ?= %u, %u\n", nhash.hash, foo1->hash, foo2->recno); */
+        
         memcpy(&nrecno, data.data, sizeof(nrecno));
 
         if ((f = get_name_by_nrecno(c,  &nrecno, &name)) < 0)
@@ -285,8 +302,9 @@ finish:
     if (cursor)
         cursor->c_close(cursor);
 
-    if (!r && create)
+    if (!r && create) {
         r = new_name(c, rname, rnrecno) < 0 ? -1 : 1;
+    }
     
     return r;
 }
@@ -309,7 +327,7 @@ int new_name(struct syrep_db_context *c, const struct syrep_name *name, struct s
         return -1;
     }
 
-    nrecno.recno = *((db_recno_t*) data.data);
+    nrecno.recno = *((db_recno_t*) key.data);
 
     memset(&key, 0, sizeof(key));
     memset(&data, 0, sizeof(data));
@@ -320,8 +338,8 @@ int new_name(struct syrep_db_context *c, const struct syrep_name *name, struct s
 
     data.data = &nrecno;
     data.size = sizeof(nrecno);
-    
-    if ((ret = c->db_nhash_nrecno->put(c->db_nhash_nrecno, NULL, &key, &data, 0))) {
+
+    if ((ret = c->db_nhash_nrecno->put(c->db_nhash_nrecno, NULL, &key, &data, DB_NODUPDATA))) {
         c->db_nhash_nrecno->err(c->db_nhash_nrecno, ret, "nhash_nrecno::put");
         return -1;
     }
diff --git a/src/diff.c b/src/diff.c
index b722198..1be9c63 100644
--- a/src/diff.c
+++ b/src/diff.c
@@ -62,7 +62,7 @@ static int add_diff_entry(DB *ddb, struct syrep_name *name, int action, struct s
         }
         
         if (data.size != data2.size || memcmp(data.data, data2.data, data.size)) {
-            fprintf(stderr, "Snapshot inconsistency\n");
+            fprintf(stderr, "Snapshot inconsistency %u %u\n", data.size, data2.size);
             return -1;
         }
     }
@@ -77,14 +77,14 @@ static int foreach(DB *ddb, struct syrep_db_context *c1, struct syrep_db_context
     int nrecno1_valid, nrecno2_valid;
 
     if ((nrecno1_valid = get_nrecno_by_name(c1, name, &nrecno1, 0)) < 0)
-        return 1;
+        return -1;
 
     if (nrecno1_valid)
         if ((md1_valid = get_current_md_by_nrecno(c1, &nrecno1, &md1)) < 0)
             return -1;
 
-    if ((nrecno1_valid = get_nrecno_by_name(c1, name, &nrecno2, 0)) < 0)
-        return 1;
+    if ((nrecno2_valid = get_nrecno_by_name(c2, name, &nrecno2, 0)) < 0)
+        return -1;
 
     if (nrecno2_valid)
         if ((md2_valid = get_current_md_by_nrecno(c2, &nrecno2, &md2)) < 0)
diff --git a/src/package.c b/src/package.c
index 0c26fef..c684020 100644
--- a/src/package.c
+++ b/src/package.c
@@ -30,6 +30,8 @@
 #include <sys/stat.h>
 #include <fcntl.h>
 
+#include <zlib.h>
+
 #include "package.h"
 #include "util.h"
 #include "syrep.h"
@@ -43,10 +45,14 @@ struct package_item {
     struct package_item *next;
 };
 
+#define ZBUFSIZE (64*1024)
+
 struct package {
     char *base;
     int count;
     int read_fd, write_fd;
+    z_stream read_z, write_z;
+    void *read_zbuf, *write_zbuf;
     int x_endianess;
     int compressed;
     struct package_item *items;
@@ -54,37 +60,205 @@ struct package {
 };
 
 
+static int alloc_read_zbuf(struct package *p) {
+    if (!p->read_zbuf)
+        if (!(p->read_zbuf = malloc(ZBUFSIZE))) {
+            fprintf(stderr, "malloc(): %s\n", strerror(errno));
+            return -1;
+        }
+
+    return 0;
+}
+
+static int alloc_write_zbuf(struct package *p) {
+    if (!p->write_zbuf)
+        if (!(p->write_zbuf = malloc(ZBUFSIZE))) {
+            fprintf(stderr, "malloc(): %s\n", strerror(errno));
+            return -1;
+        }
+
+    return 0;
+}
+
 static ssize_t package_read(struct package *p, void *d, size_t l) {
-    if (!p->compressed)
-        return loop_read(p->read_fd, d, l);
+    ssize_t r = -1;
+    
+    if (!p->compressed) {
+        
+        if ((r = loop_read(p->read_fd, d, l)) < 0)
+            fprintf(stderr, "read(): %s\n", strerror(errno));
+
+        goto finish;
+    }
+        
+    if (!p->read_zbuf && alloc_read_zbuf(p) < 0)
+        return -1;    
+
+    p->read_z.next_out = d;
+    p->read_z.avail_out = l;
+
+    while (p->read_z.avail_out > 0) {
+        int z;
+        
+        if (p->read_z.avail_in <= 0) {
+            ssize_t n;
+            
+            if ((n = read(p->read_fd, p->read_zbuf, ZBUFSIZE)) < 0) {
+                fprintf(stderr, "read(): %s\n", strerror(errno));
+                goto finish;
+            }
+
+            if (!n) {
+                r = (void*) p->read_z.next_out - d;
+                goto finish;
+            }
+
+            p->read_z.next_in = p->read_zbuf;
+            p->read_z.avail_in = n;
+        }
+        
+        if ((z = inflate(&p->read_z, 0)) != Z_OK) {
+
+            if (z == Z_STREAM_END) {
+                r = (void*) p->read_z.next_out - d;
+                goto finish;
+            }
+                
+            
+            fprintf(stderr, "XXX inflate() failed %i\n", z);
+            goto finish;
+        }
+    }
+
+    r = l;
 
-    assert(0);
+finish:
     
-    return -1;
+    return r;
 }
 
 
 static ssize_t package_write(struct package *p, void *d, size_t l) {
-    if (!args.compress_flag)
-        return loop_write(p->write_fd, d, l);
+    ssize_t r = -1;
+    
+    if (!args.compress_flag) {
+
+        if ((r = loop_write(p->write_fd, d, l)) < 0)
+            fprintf(stderr, "write(): %s\n", strerror(errno));
+
+        goto finish;
+    }
 
-    assert(0);
+    if (!p->write_zbuf && alloc_write_zbuf(p) < 0)
+        return -1;    
     
-    return -1;
+    p->write_z.next_in = d;
+    p->write_z.avail_in = l;
+
+    while (p->write_z.avail_in > 0) {
+        size_t t;
+
+        p->write_z.next_out = p->write_zbuf;
+        p->write_z.avail_out = ZBUFSIZE;
+
+
+        if (deflate(&p->write_z, Z_NO_FLUSH) != Z_OK) {
+            fprintf(stderr, "deflate() failed\n");
+            goto finish;
+        }
+
+        t = (void*) p->write_z.next_out - p->write_zbuf;
+
+        if (t) {
+            ssize_t n;
+            
+            if ((n = loop_write(p->write_fd, p->write_zbuf, t)) < 0) {
+                fprintf(stderr, "loop_write(): %s\n", strerror(errno));
+                goto finish;
+            }
+
+            if (n != t) {
+                if ((r = (void*) p->write_z.next_in - d) > 0)
+                    r --;
+
+                goto finish;
+            }
+        }
+    }
+
+    r = l;
+    
+finish:
+    
+    return r;
+
 }
 
+#define CBUFSIZE ZBUFSIZE
 static int copy_deflate(struct package *p, int sfd, off_t l) {
+    void *buf = NULL;
+    int r = -1;
 
-    assert(0);
+    if (!(buf = malloc(CBUFSIZE))) {
+        fprintf(stderr, "malloc(): %s\n", strerror(errno));
+        goto finish;
+    }
     
-    return 0;
+    while (l > 0) {
+        size_t t = MIN(l, CBUFSIZE);
+        ssize_t n;
+        
+        if ((n = loop_read(sfd, buf, t)) != t) {
+            fprintf(stderr, "read() : %s\n", n < 0 ? strerror(errno) : "EOF");
+            goto finish;
+        }
+
+        if (package_write(p, buf, n) != n)
+            goto finish;
+
+        l -= n;
+    }
+
+    r = 0;
+
+finish:
+    if (buf)
+        free(buf);
+    
+    return r;
 }
 
 static int copy_inflate(struct package *p, int dfd, off_t l) {
+    void *buf = NULL;
+    int r = -1;
 
-    assert(0);
+    if (!(buf = malloc(CBUFSIZE))) {
+        fprintf(stderr, "malloc(): %s\n", strerror(errno));
+        goto finish;
+    }
     
-    return 0;
+    while (l > 0) {
+        size_t t = MIN(l, CBUFSIZE);
+        ssize_t n;
+
+        if (package_read(p, buf, t) != t)
+            goto finish;
+        
+        if ((n = loop_write(dfd, buf, t)) != t) {
+            fprintf(stderr, "write() : %s\n", n < 0 ? strerror(errno) : "EOF");
+            goto finish;
+        }
+
+        l -= n;
+    }
+
+    r = 0;
+
+finish:
+    if (buf)
+        free(buf);
+    
+    return r;
 }
 
 static char *tmp(char *fn, int l) {
@@ -147,6 +321,67 @@ static void append_item(struct package *p, struct package_item *i) {
         p->items = p->last = i;
 }
 
+static void close_read_fd(struct package *p) {
+    assert(p);
+
+    if (p->read_fd >= 0) {
+        if (p->compressed)
+            inflateEnd(&p->read_z);
+        
+        if (p->read_fd != STDIN_FILENO)
+            close(p->read_fd);
+        p->read_fd = -1;
+    }
+}
+
+static int close_write_fd(struct package *p) {
+    int r = 0;
+    assert(p);
+
+    if (p->write_fd >= 0) {
+        if (args.compress_flag) {
+            int z;
+
+            p->write_z.next_in = NULL;
+            p->write_z.avail_in = 0;
+            
+            do {
+                size_t t;
+                ssize_t n;
+
+                p->write_z.next_out = p->write_zbuf;
+                p->write_z.avail_out = ZBUFSIZE;
+
+                z = deflate(&p->write_z, Z_FINISH);
+
+                t = (void*) p->write_z.next_out - p->write_zbuf;
+                if (t) {
+                    if ((n = loop_write(p->write_fd, p->write_zbuf, t)) != t) {
+                        fprintf(stderr, "loop_write(): %s\n", n < 0 ? strerror(errno) : "EOF");
+                        r = -1;
+                        break;
+                    }
+                }
+                
+            } while (z == Z_OK);
+
+            if (!r && z != Z_STREAM_END) {
+                fprintf(stderr, "Final deflate() failure\n");
+                r = -1;
+            }
+            
+            deflateEnd(&p->write_z);
+        }
+
+        if (p->write_fd != STDOUT_FILENO)
+            close(p->write_fd);
+        
+        p->write_fd = -1;
+    }
+
+    return r;
+}
+
 #define X32(i) (((i)<<24)|(((i)<<8)&0x00FF0000)|(((i)>>8)&0x0000FF00)|((i)>>24))
 #define X64(i) (X32((i)>>32)|X32((i)>>32))
 
@@ -164,22 +399,17 @@ static int read_item(struct package *p) {
 
     if ((r = package_read(p, name, PACKAGE_ITEMNAMELEN)) != PACKAGE_ITEMNAMELEN) {
         if (r == 0) {
-            close(p->read_fd);
-            p->read_fd = -1;
+            close_read_fd(p);
             return 0;
         } else if (r > 0)
             fprintf(stderr, "Short read\n");
-        else
-            fprintf(stderr, "read(): %s\n", strerror(errno));
         
         goto fail;
     }
 
-    if ((r = package_read(p, &l, 8) != 8)) {
+    if ((r = package_read(p, &l, 8)) != 8) {
         if (r >= 0)
             fprintf(stderr, "Short read\n");
-        else
-            fprintf(stderr, "read(): %s\n", strerror(errno));
 
         goto fail;
     }
@@ -247,21 +477,17 @@ static int write_item(struct package *p, struct package_item *i) {
     memset(name, 0, sizeof(name));
     strncpy(name, i->name, PACKAGE_ITEMNAMELEN);
     
-    if ((r = package_write(p, name, PACKAGE_ITEMNAMELEN) != PACKAGE_ITEMNAMELEN)) {
+    if ((r = package_write(p, name, PACKAGE_ITEMNAMELEN)) != PACKAGE_ITEMNAMELEN) {
         if (r >= 0)
             fprintf(stderr, "Short write\n");
-        else
-            fprintf(stderr, "write(): %s\n", strerror(errno));
         
         goto fail;
     }
 
     l = (uint64_t) size;
-    if ((r = package_write(p, &l, 8) != 8)) {
+    if ((r = package_write(p, &l, 8)) != 8) {
         if (r >= 0)
             fprintf(stderr, "Short write\n");
-        else
-            fprintf(stderr, "write(): %s\n", strerror(errno));
 
         goto fail;
     }
@@ -312,30 +538,31 @@ struct package* package_open(const char *fn, int force) {
     
     if (fn) {
 
-        if ((p->read_fd = open(fn, O_RDONLY)) < 0) {
+
+        if (!strcmp(fn, "-"))
+            p->read_fd = STDIN_FILENO;
+        else if ((p->read_fd = open(fn, O_RDONLY)) < 0) {
             
             if (errno != ENOENT || !force) {
                 fprintf(stderr, "Failed to open <%s>: %s\n", fn, strerror(errno));
                 goto finish;
             }
-            
-        } else if (p->read_fd >= 0) {
+        }
+
+        if (p->read_fd >= 0) {
             uint32_t id;
             ssize_t r;
             
             if ((r = loop_read(p->read_fd, &id, 4)) != 4) {
-                if (r < 0)
-                    fprintf(stderr, "read(): %s\n", strerror(errno));
-                else
-                    fprintf(stderr, "Short read\n");
-
+                fprintf(stderr, "read(): %s\n", r < 0 ? strerror(errno) : "EOF");
+                
                 goto finish;
             }
 
             if (id == PACKAGE_FILEID)
-                p->x_endianess = 0;
+                p->x_endianess = p->compressed = 0;
             else if (id == X32(PACKAGE_FILEID))
-                p->x_endianess = 1;
+                p->x_endianess = !(p->compressed = 0);
             else if (id == PACKAGE_FILEIDCOMPRESSED)
                 p->x_endianess = !(p->compressed = 1);
             else if (id == X32(PACKAGE_FILEIDCOMPRESSED))
@@ -348,6 +575,19 @@ struct package* package_open(const char *fn, int force) {
         
     }
 
+    if (p->compressed) {
+
+        memset(&p->read_z, 0, sizeof(p->read_z));
+        
+        if (inflateInit(&p->read_z) != Z_OK) {
+            fprintf(stderr, "zlib initialisation failure\n");
+            goto finish;
+        }
+
+        
+    }
+    
+
     return p;
 
 finish:
@@ -385,25 +625,32 @@ int package_save(struct package *p, const char *fn) {
     if (load_complete(p) < 0)
         goto finish;
 
-    if (!fn)
+    if (!fn || !strcmp(fn, "-"))
         p->write_fd = STDOUT_FILENO;
     else if ((p->write_fd = open(fn, O_RDWR|O_TRUNC|O_CREAT, 0666)) < 0) {  /* RDWR for mmap compatibility */
         fprintf(stderr, "open(\"%s\") XX: %s\n", fn, strerror(errno));
         goto finish;
     }
 
-    id = p->compressed ? PACKAGE_FILEIDCOMPRESSED : PACKAGE_FILEID;
+    id = args.compress_flag ? PACKAGE_FILEIDCOMPRESSED : PACKAGE_FILEID;
 
-    if ((n = loop_write(p->write_fd, &id, 4) != 4)) {
+    if ((n = loop_write(p->write_fd, &id, 4)) != 4) {
         if (n < 0)
-            fprintf(stderr, "write(): %s\n", strerror(errno));
-        else
-            fprintf(stderr, "Short write\n");
+            fprintf(stderr, "write(): %s\n", r < 0 ? strerror(errno) : "EOF");
 
         goto finish;
     }
-        
 
+    if (args.compress_flag) {
+
+        memset(&p->write_z, 0, sizeof(p->write_z));
+        
+        if (deflateInit(&p->write_z, Z_DEFAULT_COMPRESSION) != Z_OK) {
+            fprintf(stderr, "zlib initialisation failure\n");
+            goto finish;
+        }
+    }
+    
     for (i = p->items; i; i = i->next) {
         
         if (!i->name || !i->path)
@@ -419,10 +666,8 @@ int package_save(struct package *p, const char *fn) {
 
 finish:
 
-    if (fn && p->write_fd >= 0)
-        close(p->write_fd);
-
-    p->write_fd = -1;
+    if (close_write_fd(p) < 0)
+        r = -1;
 
     if (r < 0 && fn)
         unlink(fn);
@@ -507,11 +752,14 @@ void package_remove(struct package *p) {
         free(p->base);
     }
 
-    if (p->read_fd >= 0)
-        close(p->read_fd);
+    close_read_fd(p);
+    close_write_fd(p);
 
-    if (p->write_fd >= 0)
-        close(p->write_fd);
+    if (p->read_zbuf)
+        free(p->read_zbuf);
+    
+    if (p->write_zbuf)
+        free(p->write_zbuf);
     
     free(p);
 }
diff --git a/test/Makefile b/test/Makefile
index fc92e0e..39c51ae 100644
--- a/test/Makefile
+++ b/test/Makefile
@@ -4,7 +4,7 @@ ifdef VERBOSE
 VERBOSE=-v
 endif
 
-ARGS=-T --output-directory=extract -opatch --show-deleted --show-times --prune-empty
+ARGS+=-z -T --output-directory=extract -opatch --show-deleted --show-times --prune-empty
 
 %:
 	@mkdir -p rep1 rep2