From 118623476f8609c40425d751f56e44d720e3d6ff Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Mon, 21 Nov 2005 16:38:21 +0000 Subject: add new sse-tar2 utility git-svn-id: file:///home/lennart/svn/public/sse/trunk@11 5fbabb74-0606-0410-a5e4-b5cc6a42724e --- feed/sse-config.py | 5 ++++ feed/sse-feed | 70 ---------------------------------------------- feed/sse-fm | 5 ++-- feed/sse-tar2 | 66 ++++++++++++++++++++++++++++++++++++++++++++ feed/sse_feed | 81 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 155 insertions(+), 72 deletions(-) create mode 100644 feed/sse-config.py delete mode 100755 feed/sse-feed create mode 100755 feed/sse-tar2 create mode 100755 feed/sse_feed (limited to 'feed') diff --git a/feed/sse-config.py b/feed/sse-config.py new file mode 100644 index 0000000..40d631f --- /dev/null +++ b/feed/sse-config.py @@ -0,0 +1,5 @@ + +import os + +HOME = os.environ["HOME"] +SSE_DIR = "/home/lennart/sse/feed" diff --git a/feed/sse-feed b/feed/sse-feed deleted file mode 100755 index 29718ca..0000000 --- a/feed/sse-feed +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/python - -import sys, os, MySQLdb, stat -from popen2 import Popen3 - -def last_insert_id(cursor): - cursor.execute("SELECT LAST_INSERT_ID()"); - return cursor.fetchone()[0] - -def process_file(package_id, root, path): - global cursor - print "Processing %s" % path - - cursor.execute("INSERT INTO file (package_id, path, language_id) VALUES (%i, '%s', '0')" % (package_id, path)); - - file_id = last_insert_id(cursor); - - p = Popen3("lex-c %s" % (os.path.join(root, path))) - - for identifier in p.fromchild: - text = identifier.strip() - - cursor.execute("INSERT IGNORE INTO word (text, type, file_id) VALUES ('%s', 'word', '%i')" % (text, file_id)) - cursor.execute("UPDATE word SET cnt=cnt+1 WHERE text='%s' AND type='word' AND file_id=%i" % (text, file_id)) - - if p.wait() != 0: - print "WARNING: Subprocess failed!" - - del p - -def handle_file(package_id, root, path, filename): - - t = sys.lstat(os.path.join(path, filename)) - - if stat.F_ISREG(t.st_mode): - - extension = filename.split(".")[-1] - - if extension in ("c", "h"): - process_file(package_id, root, os.path.join(path, filename)) - return - - os.unlink(os.path.join(root, path, filename)) - -def handle_tree(path, name, url, md): - global cursor - - cursor.execute("INSERT INTO package (path, name, url, timestamp, md) VALUES ('%s', '%s', '%s', NOW(), '%s')" % (path + "/%s", name, url, md)); - package_id = last_insert_id(cursor); - - path = os.path.realpath(path) - - for dirpath, dirs, files in os.walk(path): - for f in files: - assert path + "/" == (dirpath + "/") [:len(path)+1] - - handle_file(package_id, path, dirpath[len(path)+1:], f) - -db = MySQLdb.connect(host = "localhost", user = "sse_web", passwd = "ece6Yoli", db = "sse") -cursor = db.cursor(); -cursor.execute("SET AUTOCOMMIT=0") -cursor.execute("START TRANSACTION") - -assert len(sys.argv) == 5 - -handle_tree(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]) - -cursor.execute("COMMIT") -cursor.close() -db.close() diff --git a/feed/sse-fm b/feed/sse-fm index f9fae44..dd45d58 100755 --- a/feed/sse-fm +++ b/feed/sse-fm @@ -1,10 +1,11 @@ #!/usr/bin/python + import sys, urllib2, os, socket from xml.sax import ContentHandler, make_parser +from sse-config import * -HOME = os.environ["HOME"] -SSE_TAR = HOME + "/bin/sse-tar" +SSE_TAR = SSE_DIR + "sse-tar" def process_tar(project, tar): print "New tar %s" % tar diff --git a/feed/sse-tar2 b/feed/sse-tar2 new file mode 100755 index 0000000..79671f6 --- /dev/null +++ b/feed/sse-tar2 @@ -0,0 +1,66 @@ +#!/usr/bin/python + +import tarfile, zipfile, sys, os +import sse_feed + +def archive_uncompress(archive, root, meta = {}): + + f = tarfile.open(archive, "r") + + while True: + + i = f.next() + + if i is None: + break + + if not i.isreg(): + continue + + if not sse_feed.supported_source(i.name): + continue + + dst = os.path.join(root, i.name) + f.extract(i, dst) + os.utime(dst, (i.mtime, i.mtime)) + + sse_feed.process_source(archive, root, i.name, meta) + + del f + +def calc_md(fn): + m = md5.new() + f = fopen(fn) + + while True: + data = m.read(1024) + + if len(data) <= 0: + break + + m.update(data) + + del f + + return m.hexdigest() + + +def process_archive(archive, meta = {}): + + md = calc_md(archive) + root = os.path.join(HOME, "sources", md) + + try: + os.mkdir(root) + except: + pass + + meta["md"] = md + + archive_uncompress(archive, root, meta) + +if __name__ == "__main__": + + archive_uncompress(sys.argv[1]) + + diff --git a/feed/sse_feed b/feed/sse_feed new file mode 100755 index 0000000..c6f0de9 --- /dev/null +++ b/feed/sse_feed @@ -0,0 +1,81 @@ +#!/usr/bin/python + +import sys, os, MySQLdb, stat +from popen2 import Popen3 + +supported = [".c", ".h"] + +def supported_source(fn): + + for e in supported: + if fn.endswith(e): + return True + + return False + +def last_insert_id(cursor): + cursor.execute("SELECT LAST_INSERT_ID()"); + return cursor.fetchone()[0] + +def process_file(package_id, root, path): + global cursor + print "Processing %s" % path + + cursor.execute("INSERT INTO file (package_id, path, language_id) VALUES (%i, '%s', '0')" % (package_id, path)); + + file_id = last_insert_id(cursor); + + p = Popen3("lex-c %s" % (os.path.join(root, path))) + + for identifier in p.fromchild: + text = identifier.strip() + + cursor.execute("INSERT IGNORE INTO word (text, type, file_id) VALUES ('%s', 'word', '%i')" % (text, file_id)) + cursor.execute("UPDATE word SET cnt=cnt+1 WHERE text='%s' AND type='word' AND file_id=%i" % (text, file_id)) + + if p.wait() != 0: + print "WARNING: Subprocess failed!" + + del p + +def handle_file(package_id, root, path, filename): + + t = sys.lstat(os.path.join(path, filename)) + + if stat.F_ISREG(t.st_mode): + + extension = filename.split(".")[-1] + + if extension in ("c", "h"): + process_file(package_id, root, os.path.join(path, filename)) + return + + os.unlink(os.path.join(root, path, filename)) + +def handle_tree(path, name, url, md): + global cursor + + cursor.execute("INSERT INTO package (path, name, url, timestamp, md) VALUES ('%s', '%s', '%s', NOW(), '%s')" % (path + "/%s", name, url, md)); + package_id = last_insert_id(cursor); + + path = os.path.realpath(path) + + for dirpath, dirs, files in os.walk(path): + for f in files: + assert path + "/" == (dirpath + "/") [:len(path)+1] + + handle_file(package_id, path, dirpath[len(path)+1:], f) + +if __name__ == "__main__": + db = MySQLdb.connect(host = "localhost", user = "sse_web", passwd = "ece6Yoli", db = "sse") + cursor = db.cursor(); + cursor.execute("SET AUTOCOMMIT=0") + cursor.execute("START TRANSACTION") + + assert len(sys.argv) == 5 + + handle_tree(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]) + + cursor.execute("COMMIT") + cursor.close() + db.close() -- cgit