package.c rewrite

git-svn-id: file:///home/lennart/svn/public/syrep/trunk@14 07ea20a6-d2c5-0310-9e02-9ef735347d72

8 files changed, 608 insertions, 261 deletions

diff --git a/doc/package.txt b/doc/package.txt
index e20274c..a7aae34 100644
--- a/doc/package.txt
+++ b/doc/package.txt
@@ -3,11 +3,10 @@
 File format for patches and snapshots:
 
 
-8 bytes: id SYREPLP0
+4 bytes: id SREP or CREP
 
 Multiple items of:
 
-         4 bytes: flags
          32 bytes: name as asciiz string
          4 bytes: length of item
 
diff --git a/doc/todo.txt b/doc/todo.txt
index 78e828e..2097ed9 100644
--- a/doc/todo.txt
+++ b/doc/todo.txt
@@ -10,11 +10,13 @@ pre-0.1:
 - warn if hostname != origin                           DONE
 - signal handling                                      DONE
 - add --cleanup                                        DONE
-- package.c rewrite: add progressive, mmap, sendfile, gzip support, atomicity
+- package.c: progressive, sendfile, mmap               DONE
+- package.c: gzip support, atomicity
 - better usage info on --help
 - documentation/man pages
 - autoconf
 - testing
+- replace -z by -Z
 
 post-0.1:
 - continue merge on copy failure
@@ -25,5 +27,6 @@ post-0.1:
 - PATH_MAX or PATH_MAX+1?
 - add --always-copy
 - add --forget 
+- valgrind
 
 $Id$
diff --git a/src/package.c b/src/package.c
index 2c396be..9eb3aad 100644
--- a/src/package.c
+++ b/src/package.c
@@ -28,8 +28,11 @@
 #include <assert.h>
 #include <sys/types.h>
 #include <sys/stat.h>
+#include <fcntl.h>
 
 #include "package.h"
+#include "util.h"
+#include "syrep.h"
 
 struct package_item;
 
@@ -43,30 +46,44 @@ struct package_item {
 struct package {
     char *base;
     int count;
+    int read_fd, write_fd;
+    int x_endianess;
+    int compressed;
     struct package_item *items;
     struct package_item *last;
 };
 
-static int copy(FILE *f, FILE *t, uint32_t len) {
-    static char buf[2048*4];
+
+static ssize_t package_read(struct package *p, void *d, size_t l) {
+    if (!p->compressed)
+        return loop_read(p->read_fd, d, l);
+
+    assert(0);
     
-    while (len > 0) {
-        uint32_t n;
-        
-        if (feof(f))
-            return -1;
+    return -1;
+}
 
-        n = len > sizeof(buf) ? sizeof(buf) : len;
-        
-        if (fread(buf, n, 1, f) != 1)
-            return -1;
-        
-        if (fwrite(buf, n, 1, t) != 1)
-            return -1;
 
-        len -= n;
-    }
+static ssize_t package_write(struct package *p, void *d, size_t l) {
+    if (!args.compress_flag)
+        return loop_write(p->write_fd, d, l);
 
+    assert(0);
+    
+    return -1;
+}
+
+static int copy_deflate(struct package *p, int sfd, size_t l) {
+
+    assert(0);
+    
+    return 0;
+}
+
+static int copy_inflate(struct package *p, int dfd, size_t l) {
+
+    assert(0);
+    
     return 0;
 }
 
@@ -118,9 +135,153 @@ fail:
     return NULL;
 }
 
+static void append_item(struct package *p, struct package_item *i) {
+    assert(p && i && !!p->last == !!p->items);
+
+    i->next = NULL;
+    
+    if (p->last) {
+        p->last->next = i;
+        p->last = i;
+    } else
+        p->items = p->last = i;
+}
+
+#define X(i) (((i)<<24)|(((i)<<8)&0x0F00)|(((i)>>8)&0x00F0)|((i)>>24))
+
+static int read_item(struct package *p) {
+    char name[PACKAGE_ITEMNAMELEN+1];
+    char path[PATH_MAX] = "";
+    uint32_t size;
+    ssize_t r;
+    int fd = -1;
+    struct package_item *i;
+    
+    if (p->read_fd < 0)
+        return 0;
+
+    if ((r = package_read(p, name, PACKAGE_ITEMNAMELEN)) != PACKAGE_ITEMNAMELEN) {
+        if (r == 0) {
+            close(p->read_fd);
+            p->read_fd = -1;
+            return 0;
+        } else if (r > 0)
+            fprintf(stderr, "Short read\n");
+        else
+            fprintf(stderr, "read(): %s\n", strerror(errno));
+        
+        goto fail;
+    }
+
+    if ((r = package_read(p, &size, 4) != 4)) {
+        if (r >= 0)
+            fprintf(stderr, "Short read\n");
+        else
+            fprintf(stderr, "read(): %s\n", strerror(errno));
+
+        goto fail;
+    }
+    
+    if (p->x_endianess)
+        size = X(size);
+
+    snprintf(path, sizeof(path), "%s/%i", p->base, p->count++);
+
+    if ((fd = open(path, O_RDWR|O_CREAT|O_EXCL, 0666)) < 0) {  /* RDWR for mmap compatibility */
+        fprintf(stderr, "open(\"%s\"): %s\n", path, strerror(errno));
+        goto fail;
+    }
+    
+    if (!p->compressed) {
+        if (copy_fd(p->read_fd, fd, size) < 0)
+            goto fail;
+    } else {
+        if (copy_fd_inflate(p->read_fd, fd, size) < 0)
+            goto fail;
+    }
+
+    if (!(i = item_new(name, path, 1)))
+        goto fail;
+
+    append_item(p, i);
+    
+    close(fd);
+    
+    return 1;
+
+fail:
+
+    if (fd >= 0)
+        close(fd);
+
+    if (path[0])
+        unlink(path);
+    
+    return -1;
+}
+
+
+static int write_item(struct package *p, struct package_item *i) {
+    char name[PACKAGE_ITEMNAMELEN+1];
+    ssize_t r;
+    int fd = -1;
+    uint32_t size;
+
+    if ((fd = open(i->path, O_RDONLY)) < 0) {
+        if (errno == ENOENT)
+            return 0;
+
+        fprintf(stderr, "open(\"%s\"): %s\n", i->path, strerror(errno));
+        goto fail;
+    }
+        
+    if ((size = filesize(fd)) < 0)
+        return -1;
+    
+    memset(name, 0, sizeof(name));
+    strncpy(name, i->name, PACKAGE_ITEMNAMELEN);
+    
+    if ((r = package_write(p, name, PACKAGE_ITEMNAMELEN) != PACKAGE_ITEMNAMELEN)) {
+        if (r >= 0)
+            fprintf(stderr, "Short write\n");
+        else
+            fprintf(stderr, "write(): %s\n", strerror(errno));
+        
+        goto fail;
+    }
+
+    if ((r = package_write(p, &size, 4) != 4)) {
+        if (r >= 0)
+            fprintf(stderr, "Short write\n");
+        else
+            fprintf(stderr, "write(): %s\n", strerror(errno));
+
+        goto fail;
+    }
+
+    if (size) {
+        if (!args.compress_flag) {
+            if (copy_fd(fd, p->write_fd, size) < 0)
+                goto fail;
+        } else {
+            if (copy_fd_deflate(fd, p->write_fd, size) < 0)
+                goto fail;
+        }
+    }
+
+    close(fd);
+    return 0;
+    
+fail:
+
+    if (fd > 0)
+        close(fd);
+
+    return -1;
+}
+
 struct package* package_open(const char *fn, int force) {
     struct package *p;
-    FILE *f = NULL;
     char path[PATH_MAX];
 
     if (!(p = malloc(sizeof(struct package)))) {
@@ -129,6 +290,7 @@ struct package* package_open(const char *fn, int force) {
     }
 
     memset(p, 0, sizeof(struct package));
+    p->read_fd = p->write_fd = -1;
 
     tmp(path, sizeof(path));
     if (!mkdtemp(path)) {
@@ -142,71 +304,46 @@ struct package* package_open(const char *fn, int force) {
     }
     
     if (fn) {
-        if (!(f = fopen(fn, "r"))) {
 
+        if ((p->read_fd = open(fn, O_RDONLY)) < 0) {
+            
             if (errno != ENOENT || !force) {
                 fprintf(stderr, "Failed to open <%s>: %s\n", fn, strerror(errno));
                 goto finish;
             }
-        }
-
-        if (f) {
-            
-            for (;;) {
-                uint32_t len;
-                char name[PACKAGE_ITEM_NAME_LEN+1];
-                FILE *d = NULL;
-                struct package_item *pi;
-                
-                if (!fread(name, sizeof(name)-1, 1, f))
-                    break;
-                
-                name[sizeof(name)-1] = 0;
-                
-                if (fread(&len, sizeof(len), 1, f) != 1) {
-                    fprintf(stderr, "Short read while reading length: %s\n", strerror(errno));
-                    goto finish;
-                }
-            
-                snprintf(path, sizeof(path), "%s/%i", p->base, p->count++);
-                if (!(d = fopen(path, "w+"))) {
-                    fprintf(stderr, "Failed to open target file: %s\n", strerror(errno));
-                    goto finish;
-                }
             
-                if (copy(f, d, len) < 0) {
-                    fprintf(stderr, "Copy failed: %s\n", strerror(errno));
-                    fclose(d);
-                    unlink(path);
-                    goto finish;
-                }
+        } else if (p->read_fd >= 0) {
+            uint32_t id;
+            ssize_t r;
             
-                fclose(d);
-                if (!(pi = item_new(name, path, 1))) {
-                    unlink(path);
-                    fprintf(stderr, "Failed to allocate memory.\n");
-                    goto finish;
-                }
-
-                assert(!!p->last == !!p->items);
-            
-                if (p->last) {
-                    p->last->next = pi;
-                    p->last = pi;
-                } else
-                    p->items = p->last = pi;
-            }
+            if ((r = loop_read(p->read_fd, &id, 4)) != 4) {
+                if (r < 0)
+                    fprintf(stderr, "read(): %s\n", strerror(errno));
+                else
+                    fprintf(stderr, "Short read\n");
 
+                goto finish;
+            }
 
-            fclose(f);
+            if (id == PACKAGE_FILEID)
+                p->x_endianess = 0;
+            else if (id == X(PACKAGE_FILEID))
+                p->x_endianess = 1;
+            else if (id == PACKAGE_FILEIDCOMPRESSED)
+                p->x_endianess = !(p->compressed = 1);
+            else if (id == X(PACKAGE_FILEIDCOMPRESSED))
+                p->x_endianess = p->compressed = 1;
+            else {
+                fprintf(stderr, "%s is not a syrep snapshot or patch\n", fn);
+                goto finish;
+            }
         }
+        
     }
 
     return p;
 
 finish:
-    if (f)
-        fclose(f);
 
     if (p)
         package_remove(p);
@@ -214,69 +351,68 @@ finish:
     return NULL;
 }
 
+static int load_complete(struct package *p) {
+    int r;
+    
+    assert(p);
+    
+    if (p->read_fd < 0)
+        return 0;
+
+    while ((r = read_item(p)) > 0);
+
+    return r;
+}
+
 int package_save(struct package *p, const char *fn) {
-    FILE *f, *s = NULL;
-    int r = 0;
+    int r = -1;
     struct package_item *i;
+    uint32_t id;
+    ssize_t n;
     
-    assert(p);
+    assert(p && p->write_fd < 0);
 
-    if (!fn)
-        f = stdout;
-    else if (!(f = fopen(fn, "w+")))
-        return -1;
+    if (load_complete(p) < 0)
+        goto finish;
 
-    for (i = p->items; i; i = i->next) {
-        char name[PACKAGE_ITEM_NAME_LEN+1];
-        uint32_t len;
-        struct stat st;
+    if (!fn)
+        p->write_fd = STDOUT_FILENO;
+    else if ((p->write_fd = open(fn, O_RDWR|O_TRUNC|O_CREAT, 0666)) < 0) {  /* RDWR for mmap compatibility */
+        fprintf(stderr, "open(\"%s\") XX: %s\n", fn, strerror(errno));
+        goto finish;
+    }
 
-        if (!i->name || !i->path)
-            continue;
+    id = p->compressed ? PACKAGE_FILEIDCOMPRESSED : PACKAGE_FILEID;
 
-        if (!(s = fopen(i->path, "r")))
-            continue;
+    if ((n = loop_write(p->write_fd, &id, 4) != 4)) {
+        if (n < 0)
+            fprintf(stderr, "write(): %s\n", strerror(errno));
+        else
+            fprintf(stderr, "Short write\n");
 
-        //fprintf(stderr, "Saving %s (%s)...\n", i->name, i->path);
-        
-        memset(name, 0, sizeof(name));
-        strncpy(name, i->name, sizeof(name)-1);
-        
-        if (fwrite(name, sizeof(name)-1, 1, f) != 1) {
-            r = 1;
-            break;
-        }
-        
-        if (fstat(fileno(s), &st) < 0) {
-            fprintf(stderr, "Could not get file size: %s\n", strerror(errno));
-            r = 1;
-            break;
-        }
-        
-        len = st.st_size;
-        
-        if (fwrite(&len, sizeof(len), 1, f) != 1) {
-            r = 1;
-            break;
-        }
+        goto finish;
+    }
         
-        if (copy(s, f, len) < 0) {
-            fprintf(stderr, "Could not copy file to package: %s\n", strerror(errno));
-            r = 1;
-            break;
-        }
+
+    for (i = p->items; i; i = i->next) {
         
-        fclose(s);
-        s = NULL;
+        if (!i->name || !i->path)
+            continue;
+
+        if (write_item(p, i) < 0)
+            goto finish;
     }
+    
+    r = 0;
 
-    if (s)
-        fclose(s);
+finish:
+
+    if (fn && p->write_fd >= 0)
+        close(p->write_fd);
 
-    if (fn)
-        fclose(f);
+    p->write_fd = -1;
 
-    if (r != 0)
+    if (r < 0 && fn)
         unlink(fn);
 
     return r;
@@ -288,9 +424,23 @@ const char *package_get_item(struct package* p, const char *name, int c) {
     assert(p && name);
 
     for (i = p->items; i; i = i->next)
-        if (strncmp(name, i->name, PACKAGE_ITEM_NAME_LEN) == 0)
+        if (!strncmp(name, i->name, PACKAGE_ITEMNAMELEN))
             return i->path;
 
+    for (;;) {
+        int r;
+        
+        if ((r = read_item(p)) < 0)
+            return NULL;
+
+        if (r == 0)
+            break;
+
+        assert(p->last);
+        if (!strncmp(name, p->last->name, PACKAGE_ITEMNAMELEN))
+            return p->last->path;
+    }
+
     if (!c)
         return NULL;
     
@@ -301,13 +451,7 @@ const char *package_get_item(struct package* p, const char *name, int c) {
         return NULL;
     }
 
-    assert(!!p->last == !!p->items);
-    
-    if (p->last) {
-        p->last->next = i;
-        p->last = i;
-    } else
-        p->items = p->last = i;
+    append_item(p, i);
     
     return i->path;
 }
@@ -317,17 +461,12 @@ int package_add_file(struct package *p, const char *name, const char *fn) {
 
     if (!(i = item_new(name, fn, 0)))
         return -1;
-        
-    if (p->last) {
-        p->last->next = i;
-        p->last = i;
-    } else
-        p->items = p->last = i;
+    
+    append_item(p, i);
     
     return 0;
 }
 
-
 void package_remove(struct package *p) {
     struct package_item *i, *n;
     assert(p);
@@ -353,17 +492,46 @@ void package_remove(struct package *p) {
         
         free(p->base);
     }
+
+    if (p->read_fd >= 0)
+        close(p->read_fd);
+
+    if (p->write_fd >= 0)
+        close(p->write_fd);
+    
     free(p);
 }
 
 
 int package_foreach(struct package *p, int (*cb) (struct package *p, const char *name, const char *path, void *u), void *u) {
     struct package_item *i;
+    char rname[PACKAGE_ITEMNAMELEN+1];
     assert(p);
     
-    for (i = p->items; i; i = i->next)
-        if (cb(p, i->name, i->path, u) < 0)
+    for (i = p->items; i; i = i->next) {
+        memset(rname, 0, sizeof(rname));
+        strncpy(rname, i->name, PACKAGE_ITEMNAMELEN);
+        if (cb(p, rname, i->path, u) < 0)
             return -1;
+    }
+    
+    for (;;) {
+        int r;
+        
+        if ((r = read_item(p)) < 0)
+            return -1;
+
+        if (r == 0)
+            break;
 
+        assert(p->last);
+
+        memset(rname, 0, sizeof(rname));
+        strncpy(rname, p->last->name, PACKAGE_ITEMNAMELEN);
+
+        if (cb(p, rname, p->last->path, u) < 0)
+            return -1;
+    }
+    
     return 0;
 }
diff --git a/src/package.h b/src/package.h
index 9a7bfd2..722ad54 100644
--- a/src/package.h
+++ b/src/package.h
@@ -21,7 +21,12 @@
   Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ***/
 
-#define PACKAGE_ITEM_NAME_LEN 32
+#include <stdint.h>
+
+#define PACKAGE_FILEID (*((uint32_t*) "SREP"))
+#define PACKAGE_FILEIDCOMPRESSED (*((uint32_t*) "CREP"))
+
+#define PACKAGE_ITEMNAMELEN 32
 
 struct package;
 
diff --git a/src/syrep.c b/src/syrep.c
index f95f086..03cf5d8 100644
--- a/src/syrep.c
+++ b/src/syrep.c
@@ -67,7 +67,7 @@ static int do_diff(void) {
     }
 
     if (args.local_temp_flag && isdirectory(args.inputs[0]) >= 1) {
-        const char *p = get_snapshot_filename(args.inputs[0], SYREP_TEMPDIR);
+        const char *p = get_attached_filename(args.inputs[0], SYREP_TEMPDIR);
         mkdir(p, 0777);
         setenv("TMPDIR", p, 1);
     }
@@ -132,7 +132,7 @@ static int do_merge(void) {
     }
 
     if (args.local_temp_flag) {
-        const char *p = get_snapshot_filename(args.inputs[1], SYREP_TEMPDIR);
+        const char *p = get_attached_filename(args.inputs[1], SYREP_TEMPDIR);
         mkdir(p, 0777);
         setenv("TMPDIR", p, 1);
     }
@@ -194,7 +194,7 @@ static int do_makepatch(void) {
     }
     
     if (args.local_temp_flag) {
-        const char *p = get_snapshot_filename(args.inputs[0], SYREP_TEMPDIR);
+        const char *p = get_attached_filename(args.inputs[0], SYREP_TEMPDIR);
         mkdir(p, 0777);
         setenv("TMPDIR", p, 1);
     }
@@ -258,7 +258,7 @@ static int do_foreach(int (*func) (struct syrep_db_context *c), int m) {
         static char saved_cwd[PATH_MAX];
 
         if (args.local_temp_flag && isdirectory(args.inputs[i]) >= 1) {
-            const char *p = get_snapshot_filename(args.inputs[i], SYREP_TEMPDIR);
+            const char *p = get_attached_filename(args.inputs[i], SYREP_TEMPDIR);
             mkdir(p, 0777);
             setenv("TMPDIR", p, 1);
         }
@@ -362,7 +362,7 @@ static int do_update(void) {
             
             if (args.cache_given)
                 cache = md_cache_open(args.cache_arg, args.ro_cache_flag);
-            else if ((p = get_snapshot_filename(args.inputs[i], SYREP_MDCACHEFILENAME)))
+            else if ((p = get_attached_filename(args.inputs[i], SYREP_MDCACHEFILENAME)))
                 cache = md_cache_open(p, args.ro_cache_flag);
         }
 
@@ -452,18 +452,24 @@ static void sigint(int s) {
     interrupted = 1;
 }
 
+static void free_args(void) {
+    int i;
+    
+    for (i = 0; i < args.inputs_num; i++)
+        free(args.inputs[i]);
+
+    free(args.inputs);
+}
+
 int main(int argc, char *argv[]) {
-    char *b;
     struct sigaction sa;
 
     memset(&sa, 0, sizeof(sa));
     sa.sa_handler = sigint;
     sigaction(SIGINT, &sa, NULL);
 
-    if ((b = strrchr(argv[0], '/')))
-        argv[0] = b+1;
-
     cmdline_parser(argc, argv, &args);
+    atexit(free_args);
     
     if (args.list_flag) 
         return do_foreach(list, 0);
diff --git a/src/syrep.ggo b/src/syrep.ggo
index b3e16ef..d14f5ea 100644
--- a/src/syrep.ggo
+++ b/src/syrep.ggo
@@ -22,7 +22,8 @@ version "0.1"
 
 option "verbose" v "Enable verbose operation" flag off
 option "local-temp" T "Use temporary directory inside repository" flag off
-option "ignore-origin" - "update: Don't warn if snapshot not local in update, merge, makepatch" flag off
+option "ignore-origin" - "Don't warn if snapshot not local in update, merge, makepatch" flag off
+option "compress" z "Compress snapshots or patches" flag off
 
 option "list" - "List a repository snapshot" flag off
  option "show-deleted" - "list: Show deleted entries of repository snapshot" flag off
@@ -55,4 +56,4 @@ option "extract" - "Extract the context of a snapshot or patch" flag off
  option "output-directory" D "extract: Write output to specified directory" string no
 
 option "cleanup" - "Remove syrep info from repository" flag off
- option "cleanup-level" - "cleanup: 1 - just remove temporary data and trash (default); 2 - remove MD cache as well; 3 - remove all syrep data" int default="1" no
+ option "cleanup-level" l "cleanup: 1 - just remove temporary data and trash (default); 2 - remove MD cache as well; 3 - remove all syrep data" int default="1" no
diff --git a/src/util.c b/src/util.c
index f0d66d1..7067428 100644
--- a/src/util.c
+++ b/src/util.c
@@ -18,7 +18,7 @@
   Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
 ***/
 
-#define USE_SENDFILE
+//#define USE_SENDFILE
 
 #include <stdio.h>
 #include <assert.h>
@@ -115,29 +115,32 @@ void rotdash_hide(void) {
         fputs(" \b", stderr);
 }
 
-const char* get_snapshot_filename(const char *path, const char *fn) {
+const char* get_attached_filename(const char *root, const char *fn) {
     static char npath[PATH_MAX];
+    snprintf(npath, sizeof(npath), "%s/.syrep", root);
+    mkdir(npath, 0777);
+    snprintf(npath, sizeof(npath), "%s/.syrep/%s", root, fn);
+    return npath;
+}
+
+const char* get_snapshot_filename(const char *root, const char *fn) {
     struct stat st;
 
-    if (stat(path, &st) < 0) {
+    if (stat(root, &st) < 0) {
         if (errno == ENOENT)
-            return path;
+            return root;
         
-        fprintf(stderr, "stat(%s) failed: %s\n", path, strerror(errno));
+        fprintf(stderr, "stat(%s) failed: %s\n", root, strerror(errno));
         return NULL;
     }
 
     if (S_ISREG(st.st_mode))
-        return path;
+        return root;
 
-    if (S_ISDIR(st.st_mode)) {
-        snprintf(npath, sizeof(npath), "%s/.syrep", path);
-        mkdir(npath, 0777);
-        snprintf(npath, sizeof(npath), "%s/.syrep/%s", path, fn);
-        return npath;
-    }
+    if (S_ISDIR(st.st_mode)) 
+        return get_attached_filename(root, fn);
 
-    fprintf(stderr, "<%s> is not a valid syrep snapshot\n", path);
+    fprintf(stderr, "<%s> is not a valid syrep snapshot\n", root);
     return NULL;
 }
 
@@ -151,149 +154,271 @@ int isdirectory(const char *path) {
 }
 
 
-#define MMAPSIZE (100*1024*1024)
-#define BUFSIZE (1024*1024)
-
-int copy_fd(int sfd, int dfd, uint32_t l) {
-    off_t src_o, dst_o;
-    void *src_p, *dst_p;
-    void *buf = NULL;
-    int r = -1;
-    struct stat st;
 #ifdef USE_SENDFILE    
-    ssize_t k;
-    off_t sfo;
-#endif
 
-    if ((src_o = lseek(sfd, 0, SEEK_CUR)) == (off_t) -1) {
-        fprintf(stderr, "lseek(): %s\n", strerror(errno));
-        goto finish;
-    }
+static int copy_fd_sendfile(int sfd, int dfd, uint32_t l) {
+    off_t sfo, o;
+    ssize_t r;
 
-    if (fstat(sfd, &st) < 0) {
-        fprintf(stderr, "fstat(): %s\n", strerror(errno));
-        goto finish;
-    }
+    if ((sfo = lseek(sfd, 0, SEEK_CUR)) == (off_t) -1)
+        o = 0;
+    else
+        o = sfo;
+    
+    if ((r = sendfile(dfd, sfd, &o, l)) != l)
+        return -1;
 
-    if (l == (uint32_t) -1 || l >= st.st_size - src_o)
-        l = st.st_size - src_o;
+    if (sfo != (off_t) -1) {
+        sfo += r;
 
-    if ((dst_o = lseek(dfd, 0, SEEK_CUR)) == (off_t) -1) {
-        fprintf(stderr, "lseek(): %s\n", strerror(errno));
-        goto finish;
+        lseek(sfd, sfo, SEEK_SET);
     }
+
+    return 0;
+}
+
+#endif
     
-    if (fstat(dfd, &st) < 0) {
-        fprintf(stderr, "fstat(): %s\n", strerror(errno));
-        goto finish;
+
+int filesize(int fd) {
+    struct stat st;
+
+    if (fstat(fd, &st) < 0) {
+        fprintf(stderr, "stat(): %s\n", strerror(errno));
+        return -1;
     }
+
+    return st.st_size;
+}
+
+int expand_file(int fd, size_t l) {
+    size_t s;
+
+    if ((s = filesize(fd)) < 0)
+        return -1;
     
-    if (dst_o+l > st.st_size)
-        if (ftruncate(dfd, dst_o+l) < 0) {
+    if (s < l)
+        if (ftruncate(fd, l) < 0) {
             fprintf(stderr, "ftruncate(): %s\n", strerror(errno));
-            goto finish;
+            return -1;
         }
 
-    
-#ifdef USE_SENDFILE    
+    return 0;
+}
 
-    sfo = src_o;
-    if ((k = sendfile(dfd, sfd, &sfo, l)) > 0) {
+#define MMAPSIZE (100*1024*1024)
+#define BUFSIZE (32*1024)
 
-        l -= k;
-        dst_o += k;
-        src_o = sfo;
-        
-        if (lseek(sfd, src_o, SEEK_SET) == (off_t) -1) {
-            fprintf(stderr, "lseek(): %s\n", strerror(errno));
-            goto finish;
-        }
-    } else if (k < 0)
-        fprintf(stderr, "sendfile() failed for file copy, trying mmap(): %s\n", strerror(errno));
+int copy_fd(int sfd, int dfd, size_t l) {
+    off_t sfo, dfo, msfo, mdfo;
+    void *sp, *dp;
+    size_t m, sm, dm;
+    static size_t psize = 0;
 
+#ifdef USE_SENDFILE
+    if (!copy_fd_sendfile(sfd, dfd, l))
+        return 0;
 #endif
 
+    //fprintf(stderr, "copy_fd(%u)\n", l);
+    
+    if (psize == 0)
+        psize = sysconf(_SC_PAGESIZE);
+
+    sp = dp = MAP_FAILED;
+
     if (l > BUFSIZE) {
-        uint32_t m = l < MMAPSIZE ? l : MMAPSIZE;
 
-        while (l > 0) {
-            if ((src_p = mmap(NULL, m, PROT_READ, MAP_SHARED, sfd, src_o)) == MAP_FAILED)
-                break;
+        m = l < MMAPSIZE ? l : MMAPSIZE;
+
+        if ((sfo = lseek(sfd, 0, SEEK_CUR)) != (off_t) -1) {
+            size_t s;
             
-            if ((dst_p = mmap(NULL, m, PROT_READ|PROT_WRITE, MAP_SHARED, dfd, dst_o)) == MAP_FAILED) {
-                munmap(src_p, m);
-                break;
+            if ((s = filesize(sfd)) < 0)
+                return -1;
+
+            if (s < sfo+l) {
+                fprintf(stderr, "File too short (%u vs. %lu %i)\n", s, sfo+l, l);
+                return -1;
             }
+
+            msfo = (sfo/psize)*psize;
+            sm = m+(sfo-msfo);
+            sp = mmap(NULL, sm, PROT_READ, MAP_SHARED, sfd, msfo);
+        }
+        
+        if ((dfo = lseek(dfd, 0, SEEK_CUR)) != (off_t) -1) {
+            if (expand_file(dfd, dfo+l) < 0)
+                return -1;
+
+            mdfo = (dfo/psize)*psize;
+            dm = m+(dfo-mdfo);
+            dp = mmap(NULL, dm, PROT_READ|PROT_WRITE, MAP_SHARED, dfd, mdfo);
+        }
+        
+    }
+
+    if (sp == MAP_FAILED && dp == MAP_FAILED) { /* copy fd to fd */
+        void *buf = NULL;
+
+        if (!(buf = malloc(BUFSIZE))) {
+            fprintf(stderr, "malloc(): %s\n", strerror(errno));
+            return -1;
+        }
+
+        while (l > 0) {
+            ssize_t n;
             
-            memcpy(dst_p, src_p, m);
+            m = l > BUFSIZE ? BUFSIZE : l;
             
-            munmap(src_p, m);
-            munmap(dst_p, m);
+            if ((n = loop_read(sfd, buf, m)) != m) {
+                
+                if (n < 0)
+                    fprintf(stderr, "read(): %s\n", strerror(errno));
+                else
+                    fprintf(stderr, "Short read\n");
 
-            if (interrupted) {
-                fprintf(stderr, "Canceled.\n");
-                goto finish;
+                free(buf);
+                return -1;
+            }
+
+            if ((n = loop_write(dfd, buf, m)) != m) {
+
+                if (n < 0)
+                    fprintf(stderr, "write(): %s\n", strerror(errno));
+                else
+                    fprintf(stderr, "Short write\n");
+
+                free(buf);
+                return -1;
+            }
+
+            l -= m;
+        }
+
+        free(buf);
+        return 0;
+    } else if (sp == MAP_FAILED) { /* copy fd to mmap */
+
+        for (;;) {
+            ssize_t n;
+
+            n = loop_read(sfd, dp+(dfo-mdfo), m);
+            munmap(dp, dm);
+
+            if (n != m) {
+
+                if (n < 0)
+                    fprintf(stderr, "read(): %s\n", strerror(errno));
+                else
+                    fprintf(stderr, "Short read\n");
+
+                return -1;
             }
             
-            src_o += m;
-            dst_o += m;
+
             l -= m;
+            dfo += m;
+
+            if (l <= 0) {
+                if (lseek(dfd, dfo, SEEK_SET) == (off_t) -1)
+                    return -1;
+                
+                return 0;
+            }
             
-            m = l < MMAPSIZE ? l : MMAPSIZE;
+            mdfo = (dfo/psize)*psize;
+            dm = m+(dfo-mdfo);
+            if ((dp = mmap(NULL, dm, PROT_READ|PROT_WRITE, MAP_SHARED, dfd, mdfo)) == MAP_FAILED) {
+                fprintf(stderr, "mmap(): %s\n", strerror(errno));
+                return -1;
+            }
+                
         }
+
         
-    
-        if (lseek(sfd, src_o, SEEK_SET) == (off_t) -1) {
-            fprintf(stderr, "lseek(): %s\n", strerror(errno));
-            goto finish;
+    } else if (dp == MAP_FAILED) { /* copy mmap to fd */
+
+        for (;;) {
+            ssize_t n;
+
+            n = loop_write(dfd, sp+(sfo-msfo), m);
+            munmap(sp, sm);
+
+            if (n != m) {
+
+                if (n < 0)
+                    fprintf(stderr, "write(): %s\n", strerror(errno));
+                else
+                    fprintf(stderr, "Short write\n");
+
+                return -1;
+            }
+
+            l -= m;
+            sfo += m;
+
+            if (l <= 0) {
+                if (lseek(sfd, sfo, SEEK_SET) == (off_t) -1)
+                    return -1;
+                
+                return 0;
+            }
+            
+            msfo = (sfo/psize)*psize;
+            sm = m+(sfo-msfo);
+            if ((sp = mmap(NULL, sm, PROT_READ, MAP_SHARED, sfd, msfo)) == MAP_FAILED) {
+                fprintf(stderr, "mmap(): %s\n", strerror(errno));
+                return -1;
+            }
         }
         
-        if (lseek(dfd, dst_o, SEEK_SET) == (off_t) -1) {
-            fprintf(stderr, "lseek(): %s\n", strerror(errno));
-            goto finish;
-        }
-
-        if (l > 0)
-            fprintf(stderr, "mmap() failed for file copy, trying read()/write(): %s\n", strerror(errno));
-    }
-    
-    if (l > 0) {
-        if (!(buf = malloc(BUFSIZE)))
-            goto finish;
+ 
+    } else {  /* copy mmap to mmap */
 
         for (;;) {
-            ssize_t m;
 
-            if (interrupted) {
-                fprintf(stderr, "Canceled.\n");
-                goto finish;
-            }
+            memcpy(dp+(dfo-mdfo), sp+(sfo-msfo), m);
+
+            munmap(sp, sm);
+            munmap(dp, dm);
+
+            l -= m;
+            sfo += m;
+            dfo += m;
             
-            if ((m = read(sfd, buf, BUFSIZE)) < 0) {
-                fprintf(stderr, "read(): %s\n", strerror(errno));
-                goto finish;
+            if (l <= 0) {
+
+                if (lseek(sfd, sfo, SEEK_SET) == (off_t) -1)
+                    return -1;
+
+                if (lseek(dfd, dfo, SEEK_SET) == (off_t) -1)
+                    return -1;
+
+                return 0;
             }
 
-            if (!m)
-                break;
-                
-            if (write(dfd, buf, m) < 0) {
-                fprintf(stderr, "write(): %s\n", strerror(errno));
-                goto finish;
+            msfo = (sfo/psize)*psize;
+            sm = m+(sfo-msfo);
+            if ((sp = mmap(NULL, sm, PROT_READ, MAP_SHARED, sfd, msfo)) == MAP_FAILED) {
+                fprintf(stderr, "mmap(): %s\n", strerror(errno));
+                return -1;
+            }
+
+            mdfo = (dfo/psize)*psize;
+            dm = m+(dfo-mdfo);
+            if ((dp = mmap(NULL, dm, PROT_READ|PROT_WRITE, MAP_SHARED, dfd, mdfo)) == MAP_FAILED) {
+                fprintf(stderr, "mmap(): %s\n", strerror(errno));
+                return -1;
             }
         }
     }
-
-    r = 0;
-    
-finish:
-
-    return r;
 }
 
 int copy_file(const char *src, const char *dst, int c) {
     int sfd = -1, dfd = -1, r = -1;
-
+    size_t size;
+    
     if ((sfd = open(src, O_RDONLY)) < 0) {
         fprintf(stderr, "open(\"%s\", O_RDONLY): %s\n", src, strerror(errno));
         goto finish;
@@ -304,7 +429,10 @@ int copy_file(const char *src, const char *dst, int c) {
         goto finish;
     }
 
-    if (copy_fd(sfd, dfd, (uint32_t) -1) < 0)
+    if ((size = filesize(sfd)) < 0)
+        goto finish;
+    
+    if (copy_fd(sfd, dfd, size) < 0)
         goto finish;
 
     r = 0;
@@ -570,3 +698,34 @@ finish:
     return r;
 }
 
+ssize_t loop_read(int fd, void *d, size_t l) {
+    void *p = d;
+    
+    while (l > 0) {
+        ssize_t r;
+        
+        if ((r = read(fd, p, l)) <= 0)
+            return p-d > 0 ? p-d : r;
+
+        p += r;
+        l -= r;
+    }
+
+    return p-d;
+}
+
+ssize_t loop_write(int fd, void *d, size_t l) {
+    void *p = d;
+    
+    while (l > 0) {
+        ssize_t r;
+        
+        if ((r = write(fd, p, l)) <= 0)
+            return p-d > 0 ? p-d : r;
+
+        p += r;
+        l -= r;
+    }
+
+    return p-d;
+}
diff --git a/src/util.h b/src/util.h
index a44c598..d16f0a6 100644
--- a/src/util.h
+++ b/src/util.h
@@ -33,8 +33,9 @@ char* normalize_path(char *s);
 void rotdash(void);
 void rotdash_hide(void);
 const char* get_snapshot_filename(const char *path, const char *fn);
+const char* get_attached_filename(const char *root, const char *fn);
 int isdirectory(const char *path);
-int copy_fd(int sfd, int dfd, uint32_t l);
+int copy_fd(int sfd, int dfd, size_t l);
 int copy_file(const char *src, const char *dst, int c);
 int move_file(const char *src, const char *dst, int c);
 int copy_or_link_file(const char *src, const char *dst, int c);
@@ -53,5 +54,10 @@ int question(const char *q, const char *resp);
 /* Same as /bin/rm -rf in the shell */
 int rm_rf(const char *root, int rec);
 
+ssize_t loop_read(int fd, void *d, size_t l);
+ssize_t loop_write(int fd, void *d, size_t l);
+
+int expand_file(int fd, size_t l);
+int filesize(int fd);
 
 #endif