add debian feeder

git-svn-id: file:///home/lennart/svn/public/sse/trunk@20 5fbabb74-0606-0410-a5e4-b5cc6a42724e

5 files changed, 98 insertions, 37 deletions

diff --git a/feed/sse_debian.py b/feed/sse_debian.py
new file mode 100755
index 0000000..ec0cabe
--- /dev/null
+++ b/feed/sse_debian.py
@@ -0,0 +1,57 @@
+#!/usr/bin/python
+
+#SSE_DEBIAN_URL = "http://ftp.us.debian.org/debian/"
+SSE_DEBIAN_URL = "http://update.alturo-server.de/debian/"
+SSE_DISTRIBUTION = "unstable"
+SSE_RELEASE = "main"
+
+import sys
+
+def process_entry(entry):
+
+    for k, v in entry.items():
+        entry[k] = v.strip()
+
+    meta = {}
+
+    for l in entry["Files"].split("\n"):
+
+        a = l.split()[2]
+
+        if a.endswith(".tar.gz"):
+            meta["archive"] = a
+            break
+
+    meta["name"] = entry["Package"]
+    meta["version"] = entry["Version"]
+    meta["archive-url"] = SSE_DEBIAN_URL + entry["Directory"] + "/" + meta["archive"]
+    meta["license"] = "DFSG approved"
+    meta["project-url"] = "http://packages.debian.org/cgi-bin/search_packages.pl?searchon=sourcenames&version="+ SSE_DISTRIBUTION + "&exact=1&keywords=" + entry["Package"] + "&release=" + SSE_RELEASE
+    print meta
+
+def process_list(f):
+
+    while True:
+
+        entry = {}
+        key = None
+
+        while True:
+
+            ln = f.readline().strip("\n\r")
+
+            if ln == "":
+                break
+
+            elif ln[0].isspace():
+                entry[key] += "\n" + ln[1:]
+
+            else:
+                key, data = ln.split(":", 1)
+                entry[key] = data
+
+        process_entry(entry)
+
+
+if __name__ == "__main__":
+    process_list(sys.stdin)
diff --git a/feed/sse_defs.py b/feed/sse_defs.py
index 9c2ce53..bbb3f14 100644
--- a/feed/sse_defs.py
+++ b/feed/sse_defs.py
@@ -8,3 +8,4 @@ SSE_LANGUAGE_C = 0
 
 SSE_MAX_DOWNLOAD = 1024*1024*40
 SSE_BLOCK_SIZE = 10*1024
+SSE_SOCKET_TIMEOUT = 20
diff --git a/feed/sse_fm.py b/feed/sse_fm.py
index 7161d7a..0edb92d 100755
--- a/feed/sse_fm.py
+++ b/feed/sse_fm.py
@@ -28,7 +28,8 @@ license_whitelist = [
     "Artistic License",
     "MIT/X Consortium License",
     "The Clarified Artistic License",
-    "BSD License (revised)"
+    "BSD License (revised)",
+    "DFSG approved"
     ]
 
 def process_record(meta):
diff --git a/feed/sse_grab.py b/feed/sse_grab.py
index 937aad4..302c5a7 100644
--- a/feed/sse_grab.py
+++ b/feed/sse_grab.py
@@ -6,7 +6,7 @@ import sse_tar
 from sse_defs import *
 from sse_config import *
 
-socket.setdefaulttimeout(20)
+socket.setdefaulttimeout(SSE_SOCKET_TIMEOUT)
 
 def grab_archive(meta, recid, provider_id = SSE_PROVIDER_NONE):
 
@@ -24,7 +24,7 @@ def grab_archive(meta, recid, provider_id = SSE_PROVIDER_NONE):
     except:
         download = True
     else:
-        download = f.read() != meta["version"]
+        download = f.read(SSE_BLOCK_SIZE) != meta["version"]
         f.close()
 
     archive_url = meta["archive-url"]
diff --git a/feed/sse_tar.py b/feed/sse_tar.py
index bb01987..ea94ca4 100755
--- a/feed/sse_tar.py
+++ b/feed/sse_tar.py
@@ -112,57 +112,59 @@ def rm_rf(root):
 
     os.rmdir(root)
 
-
 def process_archive(archive, meta = {}, recid = None, provider_id = SSE_PROVIDER_NONE):
 
     if recid is None:
         recid = archive
 
     md = calc_md(archive)
-    root = os.path.join(HOME, "sources", md)
-
-    try:
-        rm_rf(root)
-    except:
-        pass
-    os.makedirs(root)
-
     meta["md"] = md
 
     sse_db.start_transaction()
 
-    package_id = sse_db.find_package(md)
+    try:
+        package_id = sse_db.find_package(md)
+
+        if not package_id is None:
+            print "Package '%s' already in database." % recid
 
-    if package_id is None:
-        package_id = sse_db.new_package(archive, root, meta)
+            if not recid is None:
+                # Update provider record
+                sse_db.new_provider_record(recid, package_id, provider_id, meta)
+                
+        else:
+            package_id = sse_db.new_package(archive, root, meta)
         
-        print "Package '%s' is new in database." % recid
+            print "Package '%s' is new in database." % recid
         
-        if not recid is None:
-            sse_db.new_provider_record(recid, package_id, provider_id, meta)
-            
-        try:    
-            n = uncompress_tar(archive, root, package_id, meta)
-        except tarfile.TarError:
-            try:    
-                n = uncompress_zip(archive, root, package_id, meta)
-            except zipfile.error:
-                n = None
-                print "Unknown file format."
+            if not recid is None:
+                sse_db.new_provider_record(recid, package_id, provider_id, meta)
 
-        if not n is None:
-            print "Successfully processed %i files." % n
-            
-    else:
+            root = os.path.join(HOME, "sources", md)
+
+            try:
+                rm_rf(root)
+            except:
+                pass
+            os.makedirs(root)
 
-        print "Package '%s' already in database." % recid
+            try:    
+                n = uncompress_tar(archive, root, package_id, meta)
+            except tarfile.TarError:
+                try:    
+                    n = uncompress_zip(archive, root, package_id, meta)
+                except zipfile.error:
+                    n = -1
+                    print "Unknown file format."
+
+            if n >= 0:
+                print "Successfully processed %i files." % n
 
-        if not recid is None:
-            sse_db.new_provider_record(recid, package_id, provider_id, meta)
+        sse_db.commit()
 
-    sse_db.commit()
+    except Exception, e:
+        sse_db.rollback()
 
 if __name__ == "__main__":
     process_archive(sys.argv[1])
-
-    
+