diff options
author | Lennart Poettering <lennart@poettering.net> | 2005-11-23 18:35:19 +0000 |
---|---|---|
committer | Lennart Poettering <lennart@poettering.net> | 2005-11-23 18:35:19 +0000 |
commit | 3971bc230008a576b9afa9eda47147d88a584776 (patch) | |
tree | 81ee529d872606b7279c9631b66eb133e0134ab5 /feed | |
parent | 3a24de286d8bb6a6fa5c802ad9587e15550f0801 (diff) |
add proper locking for archives
git-svn-id: file:///home/lennart/svn/public/sse/trunk@44 5fbabb74-0606-0410-a5e4-b5cc6a42724e
Diffstat (limited to 'feed')
-rw-r--r-- | feed/Makefile | 5 | ||||
-rwxr-xr-x | feed/sse_fm.py | 7 | ||||
-rw-r--r-- | feed/sse_grab.py | 37 | ||||
-rwxr-xr-x | feed/sse_lock.py | 48 |
4 files changed, 79 insertions, 18 deletions
diff --git a/feed/Makefile b/feed/Makefile index b23efe3..3bf3493 100644 --- a/feed/Makefile +++ b/feed/Makefile @@ -2,7 +2,8 @@ CFLAGS=-Wall -W -O3 -pipe #CFLAGS=-Wextra -g -O0 -pipe LIBS=-lfl -PYO=sse_config.pyo sse_debian.pyo sse_feed.pyo sse_grab.pyo sse_db.pyo sse_defs.pyo sse_fm.pyo sse_tar.pyo +PYTHON_SOURCES=$(wildcard *.py) +PYTHON_OBJECTS=$(PYTHON_SOURCES:.py=.pyo) %.yy.c: %.l flex -o $@ $^ @@ -10,7 +11,7 @@ PYO=sse_config.pyo sse_debian.pyo sse_feed.pyo sse_grab.pyo sse_db.pyo sse_defs. %.pyo: %.py ./compile $< -all: sse_lex_c $(PYO) +all: sse_lex_c $(PYTHON_OBJECTS) sse_lex_c: sse_lex_c.yy.o $(CC) -o $@ $^ $(LIBS) diff --git a/feed/sse_fm.py b/feed/sse_fm.py index d46a8f2..befdf11 100755 --- a/feed/sse_fm.py +++ b/feed/sse_fm.py @@ -17,7 +17,8 @@ license_blacklist = [ "Other/Proprietary License", "Aladdin Free Public License (AFPL)", "Free For Home Use", - "Unknown" + "Unknown", + "Free For Educational Use" ] license_whitelist = [ @@ -39,7 +40,9 @@ license_whitelist = [ "The Apache License 2.0", "Perl License", "The PHP License", - "Python License" + "Python License", + "Eiffel Forum License (EFL)", + "QT Public License (QPL)" ] def process_record(meta): diff --git a/feed/sse_grab.py b/feed/sse_grab.py index 302c5a7..b1d11db 100644 --- a/feed/sse_grab.py +++ b/feed/sse_grab.py @@ -2,36 +2,42 @@ import sys, urllib2, os, socket -import sse_tar +import sse_tar, sse_lock from sse_defs import * from sse_config import * socket.setdefaulttimeout(SSE_SOCKET_TIMEOUT) def grab_archive(meta, recid, provider_id = SSE_PROVIDER_NONE): - + try: os.mkdir("%s/download" % HOME) except: pass fn = os.path.join(HOME, "download", recid) + lock_fn = fn + ".lock" - download = False + if not sse_lock.try_lock_file(lock_fn): + print "Skipping locked archive '%s'." % recid + return try: - f = open(fn+".release", "r") - except: - download = True - else: - download = f.read(SSE_BLOCK_SIZE) != meta["version"] - f.close() + download = False + + try: + f = open(fn+".release", "r") + except IOError: + download = True + else: + download = f.read(SSE_BLOCK_SIZE) != meta["version"] + f.close() - archive_url = meta["archive-url"] + archive_url = meta["archive-url"] - if not download: - print "File %s up-to-date." % archive_url - else: + if not download: + print "File %s up-to-date." % archive_url + return print "Downloading %s..." % archive_url @@ -73,7 +79,10 @@ def grab_archive(meta, recid, provider_id = SSE_PROVIDER_NONE): f = open(fn+".release", "w") except: os.unlink(fn) - + f.write(meta["version"]) f.close() + + finally: + sse_lock.unlock_file(lock_fn) diff --git a/feed/sse_lock.py b/feed/sse_lock.py new file mode 100755 index 0000000..8fd0459 --- /dev/null +++ b/feed/sse_lock.py @@ -0,0 +1,48 @@ +#!/usr/bin/python +import os, errno, time + +def try_lock_file(fn): + + lp = "%s.%i" % (fn, os.getpid()) + + fd = os.open(lp, os.O_RDWR|os.O_CREAT, 0666) + + try: + os.link(lp, fn) + except OSError, e: + os.close(fd) + os.unlink(lp) + + if e.errno != errno.EEXIST: + raise e + + return False + + t = os.fstat(fd) + os.close(fd) + os.unlink(lp) + + return t.st_nlink >= 2 + +def unlock_file(fn): + + os.unlink(fn) + + +if __name__ == "__main__": + + if not try_lock_file("test.lock"): + print "FAILED" + else: + try: + print "LOCKED" + time.sleep(60) + finally: + unlock_file("test.lock") + + print "UNLOCKED" + + + + + |