add proper locking for archives

git-svn-id: file:///home/lennart/svn/public/sse/trunk@44 5fbabb74-0606-0410-a5e4-b5cc6a42724e

4 files changed, 79 insertions, 18 deletions

diff --git a/feed/Makefile b/feed/Makefile
index b23efe3..3bf3493 100644
--- a/feed/Makefile
+++ b/feed/Makefile
@@ -2,7 +2,8 @@ CFLAGS=-Wall -W -O3 -pipe
 #CFLAGS=-Wextra -g -O0 -pipe
 LIBS=-lfl
 
-PYO=sse_config.pyo sse_debian.pyo sse_feed.pyo sse_grab.pyo sse_db.pyo sse_defs.pyo sse_fm.pyo sse_tar.pyo
+PYTHON_SOURCES=$(wildcard *.py)
+PYTHON_OBJECTS=$(PYTHON_SOURCES:.py=.pyo)
 
 %.yy.c: %.l
 	flex -o $@ $^
@@ -10,7 +11,7 @@ PYO=sse_config.pyo sse_debian.pyo sse_feed.pyo sse_grab.pyo sse_db.pyo sse_defs.
 %.pyo: %.py
 	./compile $<
 
-all: sse_lex_c $(PYO)
+all: sse_lex_c $(PYTHON_OBJECTS)
 
 sse_lex_c: sse_lex_c.yy.o
 	$(CC) -o $@ $^ $(LIBS)
diff --git a/feed/sse_fm.py b/feed/sse_fm.py
index d46a8f2..befdf11 100755
--- a/feed/sse_fm.py
+++ b/feed/sse_fm.py
@@ -17,7 +17,8 @@ license_blacklist = [
     "Other/Proprietary License",
     "Aladdin Free Public License (AFPL)",
     "Free For Home Use",
-    "Unknown"
+    "Unknown",
+    "Free For Educational Use"
     ]
 
 license_whitelist = [
@@ -39,7 +40,9 @@ license_whitelist = [
     "The Apache License 2.0",
     "Perl License",
     "The PHP License",
-    "Python License"
+    "Python License",
+    "Eiffel Forum License (EFL)",
+    "QT Public License (QPL)"
     ]
 
 def process_record(meta):
diff --git a/feed/sse_grab.py b/feed/sse_grab.py
index 302c5a7..b1d11db 100644
--- a/feed/sse_grab.py
+++ b/feed/sse_grab.py
@@ -2,36 +2,42 @@
 
 import sys, urllib2, os, socket
 
-import sse_tar
+import sse_tar, sse_lock
 from sse_defs import *
 from sse_config import *
 
 socket.setdefaulttimeout(SSE_SOCKET_TIMEOUT)
 
 def grab_archive(meta, recid, provider_id = SSE_PROVIDER_NONE):
-
+    
     try:
         os.mkdir("%s/download" % HOME)
     except:
         pass
     
     fn = os.path.join(HOME, "download", recid)
+    lock_fn = fn + ".lock"
 
-    download = False
+    if not sse_lock.try_lock_file(lock_fn):
+        print "Skipping locked archive '%s'." % recid
+        return
 
     try:
-        f = open(fn+".release", "r")
-    except:
-        download = True
-    else:
-        download = f.read(SSE_BLOCK_SIZE) != meta["version"]
-        f.close()
+        download = False
+
+        try:
+            f = open(fn+".release", "r")
+        except IOError:
+            download = True
+        else:
+            download = f.read(SSE_BLOCK_SIZE) != meta["version"]
+            f.close()
 
-    archive_url = meta["archive-url"]
+        archive_url = meta["archive-url"]
 
-    if not download:
-        print "File %s up-to-date." % archive_url
-    else:
+        if not download:
+            print "File %s up-to-date." % archive_url
+            return
 
         print "Downloading %s..." % archive_url
 
@@ -73,7 +79,10 @@ def grab_archive(meta, recid, provider_id = SSE_PROVIDER_NONE):
             f = open(fn+".release", "w")
         except:
             os.unlink(fn)
-
+        
         f.write(meta["version"])
         f.close()
+
+    finally:
+        sse_lock.unlock_file(lock_fn)
                             
diff --git a/feed/sse_lock.py b/feed/sse_lock.py
new file mode 100755
index 0000000..8fd0459
--- /dev/null
+++ b/feed/sse_lock.py
@@ -0,0 +1,48 @@
+#!/usr/bin/python
+import os, errno, time
+
+def try_lock_file(fn):
+
+    lp = "%s.%i" % (fn, os.getpid())
+    
+    fd = os.open(lp, os.O_RDWR|os.O_CREAT, 0666)
+
+    try:
+        os.link(lp, fn)
+    except OSError, e:
+        os.close(fd)
+        os.unlink(lp)
+
+        if e.errno != errno.EEXIST:
+            raise e
+
+        return False
+
+    t = os.fstat(fd)
+    os.close(fd)
+    os.unlink(lp)
+
+    return t.st_nlink >= 2
+
+def unlock_file(fn):
+
+    os.unlink(fn)
+
+
+if __name__ == "__main__":
+    
+    if not try_lock_file("test.lock"):
+        print "FAILED"
+    else:
+        try:
+            print "LOCKED"
+            time.sleep(60)
+        finally:
+            unlock_file("test.lock")
+            
+        print "UNLOCKED"
+    
+
+    
+    
+