From 86b5e7777f80fdd8f2556105e6b9c3126316dfd3 Mon Sep 17 00:00:00 2001 From: Lennart Poettering Date: Mon, 21 Nov 2005 16:39:26 +0000 Subject: change dashes to underscores git-svn-id: file:///home/lennart/svn/public/sse/trunk@12 5fbabb74-0606-0410-a5e4-b5cc6a42724e --- feed/sse-config.py | 5 -- feed/sse-fm | 134 ----------------------------------------------------- feed/sse-tar2 | 66 -------------------------- feed/sse_config.py | 5 ++ feed/sse_feed | 81 -------------------------------- feed/sse_feed.py | 81 ++++++++++++++++++++++++++++++++ feed/sse_fm.py | 134 +++++++++++++++++++++++++++++++++++++++++++++++++++++ feed/sse_tar.py | 66 ++++++++++++++++++++++++++ 8 files changed, 286 insertions(+), 286 deletions(-) delete mode 100644 feed/sse-config.py delete mode 100755 feed/sse-fm delete mode 100755 feed/sse-tar2 create mode 100644 feed/sse_config.py delete mode 100755 feed/sse_feed create mode 100755 feed/sse_feed.py create mode 100755 feed/sse_fm.py create mode 100755 feed/sse_tar.py (limited to 'feed') diff --git a/feed/sse-config.py b/feed/sse-config.py deleted file mode 100644 index 40d631f..0000000 --- a/feed/sse-config.py +++ /dev/null @@ -1,5 +0,0 @@ - -import os - -HOME = os.environ["HOME"] -SSE_DIR = "/home/lennart/sse/feed" diff --git a/feed/sse-fm b/feed/sse-fm deleted file mode 100755 index dd45d58..0000000 --- a/feed/sse-fm +++ /dev/null @@ -1,134 +0,0 @@ -#!/usr/bin/python - - -import sys, urllib2, os, socket -from xml.sax import ContentHandler, make_parser -from sse-config import * - -SSE_TAR = SSE_DIR + "sse-tar" - -def process_tar(project, tar): - print "New tar %s" % tar - - ret = os.system("%s '%s' '%s' '%s'" % (SSE_TAR, tar, project["name"], project["project-url"])) - - if ret != 0: - print "WARNING: Process returned %i" % ret - -def process_project(project): - - archive_url = None - - for a in ("archive-tgz-url", "archive-bz2-url", "archive-zip-url"): - - if project.has_key(a) and project[a] != "": - archive_url = project[a] - break - - if archive_url is None: - print "WARNING: Ignoring project '%s' without archive URL!" % project["name"] - return - - fn = "%s/download/freshmeat-%i" % (HOME, int(project["id"])) - - download = False - - try: - f = open(fn+".release", "r") - except: - download = True - else: - download = f.read() != project["date"].strip() - - if not download: - print "File %s up-to-date." % archive_url - else: - - print "Downloading %s..." % archive_url - - try: - dst = file(fn, "w") - src = urllib2.urlopen(archive_url) - - while True: - data = src.read(1024) - - if len(data) <= 0: - break - - dst.write(data) - - del dst - del src - - except IOError, e: - os.unlink(fn) - print "WARNING: Failed to download %s!" % archive_url - return - - try: - f = open(fn+".release", "w") - except: - os.unlink(fn) - - f.write(project["date"].strip()) - del f - - process_tar(project, fn) - -class docHandler(ContentHandler): - - project_data = {} - field = None - - def startElement(self, name, attrs): - if name == "project": - self.project_data = {} - self.field = None - elif name == "project_id": - self.field = "id" - elif name == "projectname_full": - self.field = "name" - elif name == "url_project_page": - self.field = "project-url" - elif name == "url_tgz": - self.field = "archive-tgz-url" - elif name == "url_bz2": - self.field = "archive-bz2-url" - elif name == "url_zip": - self.field = "archive-zip-url" - elif name == "license": - self.field = "license" - elif name == "latest_release_date": - self.field = "date" - else: - self.field = None - - def characters(self, data): - - if not self.field is None: - if self.project_data.has_key(self.field): - self.project_data[self.field] += data - else: - self.project_data[self.field] = data - - def endElement(self, name): - if name == "project": - process_project(self.project_data) - self.project_data = None - - self.field = None - -try: - os.mkdir("%s/download" % HOME) -except: - pass - -socket.setdefaulttimeout(20) - -dh = docHandler() - -parser = make_parser() - -parser.setContentHandler(dh) -parser.parse(sys.stdin) diff --git a/feed/sse-tar2 b/feed/sse-tar2 deleted file mode 100755 index 79671f6..0000000 --- a/feed/sse-tar2 +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/python - -import tarfile, zipfile, sys, os -import sse_feed - -def archive_uncompress(archive, root, meta = {}): - - f = tarfile.open(archive, "r") - - while True: - - i = f.next() - - if i is None: - break - - if not i.isreg(): - continue - - if not sse_feed.supported_source(i.name): - continue - - dst = os.path.join(root, i.name) - f.extract(i, dst) - os.utime(dst, (i.mtime, i.mtime)) - - sse_feed.process_source(archive, root, i.name, meta) - - del f - -def calc_md(fn): - m = md5.new() - f = fopen(fn) - - while True: - data = m.read(1024) - - if len(data) <= 0: - break - - m.update(data) - - del f - - return m.hexdigest() - - -def process_archive(archive, meta = {}): - - md = calc_md(archive) - root = os.path.join(HOME, "sources", md) - - try: - os.mkdir(root) - except: - pass - - meta["md"] = md - - archive_uncompress(archive, root, meta) - -if __name__ == "__main__": - - archive_uncompress(sys.argv[1]) - - diff --git a/feed/sse_config.py b/feed/sse_config.py new file mode 100644 index 0000000..40d631f --- /dev/null +++ b/feed/sse_config.py @@ -0,0 +1,5 @@ + +import os + +HOME = os.environ["HOME"] +SSE_DIR = "/home/lennart/sse/feed" diff --git a/feed/sse_feed b/feed/sse_feed deleted file mode 100755 index c6f0de9..0000000 --- a/feed/sse_feed +++ /dev/null @@ -1,81 +0,0 @@ -#!/usr/bin/python - -import sys, os, MySQLdb, stat -from popen2 import Popen3 - -supported = [".c", ".h"] - -def supported_source(fn): - - for e in supported: - if fn.endswith(e): - return True - - return False - -def last_insert_id(cursor): - cursor.execute("SELECT LAST_INSERT_ID()"); - return cursor.fetchone()[0] - -def process_file(package_id, root, path): - global cursor - print "Processing %s" % path - - cursor.execute("INSERT INTO file (package_id, path, language_id) VALUES (%i, '%s', '0')" % (package_id, path)); - - file_id = last_insert_id(cursor); - - p = Popen3("lex-c %s" % (os.path.join(root, path))) - - for identifier in p.fromchild: - text = identifier.strip() - - cursor.execute("INSERT IGNORE INTO word (text, type, file_id) VALUES ('%s', 'word', '%i')" % (text, file_id)) - cursor.execute("UPDATE word SET cnt=cnt+1 WHERE text='%s' AND type='word' AND file_id=%i" % (text, file_id)) - - if p.wait() != 0: - print "WARNING: Subprocess failed!" - - del p - -def handle_file(package_id, root, path, filename): - - t = sys.lstat(os.path.join(path, filename)) - - if stat.F_ISREG(t.st_mode): - - extension = filename.split(".")[-1] - - if extension in ("c", "h"): - process_file(package_id, root, os.path.join(path, filename)) - return - - os.unlink(os.path.join(root, path, filename)) - -def handle_tree(path, name, url, md): - global cursor - - cursor.execute("INSERT INTO package (path, name, url, timestamp, md) VALUES ('%s', '%s', '%s', NOW(), '%s')" % (path + "/%s", name, url, md)); - package_id = last_insert_id(cursor); - - path = os.path.realpath(path) - - for dirpath, dirs, files in os.walk(path): - for f in files: - assert path + "/" == (dirpath + "/") [:len(path)+1] - - handle_file(package_id, path, dirpath[len(path)+1:], f) - -if __name__ == "__main__": - db = MySQLdb.connect(host = "localhost", user = "sse_web", passwd = "ece6Yoli", db = "sse") - cursor = db.cursor(); - cursor.execute("SET AUTOCOMMIT=0") - cursor.execute("START TRANSACTION") - - assert len(sys.argv) == 5 - - handle_tree(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]) - - cursor.execute("COMMIT") - cursor.close() - db.close() diff --git a/feed/sse_feed.py b/feed/sse_feed.py new file mode 100755 index 0000000..c6f0de9 --- /dev/null +++ b/feed/sse_feed.py @@ -0,0 +1,81 @@ +#!/usr/bin/python + +import sys, os, MySQLdb, stat +from popen2 import Popen3 + +supported = [".c", ".h"] + +def supported_source(fn): + + for e in supported: + if fn.endswith(e): + return True + + return False + +def last_insert_id(cursor): + cursor.execute("SELECT LAST_INSERT_ID()"); + return cursor.fetchone()[0] + +def process_file(package_id, root, path): + global cursor + print "Processing %s" % path + + cursor.execute("INSERT INTO file (package_id, path, language_id) VALUES (%i, '%s', '0')" % (package_id, path)); + + file_id = last_insert_id(cursor); + + p = Popen3("lex-c %s" % (os.path.join(root, path))) + + for identifier in p.fromchild: + text = identifier.strip() + + cursor.execute("INSERT IGNORE INTO word (text, type, file_id) VALUES ('%s', 'word', '%i')" % (text, file_id)) + cursor.execute("UPDATE word SET cnt=cnt+1 WHERE text='%s' AND type='word' AND file_id=%i" % (text, file_id)) + + if p.wait() != 0: + print "WARNING: Subprocess failed!" + + del p + +def handle_file(package_id, root, path, filename): + + t = sys.lstat(os.path.join(path, filename)) + + if stat.F_ISREG(t.st_mode): + + extension = filename.split(".")[-1] + + if extension in ("c", "h"): + process_file(package_id, root, os.path.join(path, filename)) + return + + os.unlink(os.path.join(root, path, filename)) + +def handle_tree(path, name, url, md): + global cursor + + cursor.execute("INSERT INTO package (path, name, url, timestamp, md) VALUES ('%s', '%s', '%s', NOW(), '%s')" % (path + "/%s", name, url, md)); + package_id = last_insert_id(cursor); + + path = os.path.realpath(path) + + for dirpath, dirs, files in os.walk(path): + for f in files: + assert path + "/" == (dirpath + "/") [:len(path)+1] + + handle_file(package_id, path, dirpath[len(path)+1:], f) + +if __name__ == "__main__": + db = MySQLdb.connect(host = "localhost", user = "sse_web", passwd = "ece6Yoli", db = "sse") + cursor = db.cursor(); + cursor.execute("SET AUTOCOMMIT=0") + cursor.execute("START TRANSACTION") + + assert len(sys.argv) == 5 + + handle_tree(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]) + + cursor.execute("COMMIT") + cursor.close() + db.close() diff --git a/feed/sse_fm.py b/feed/sse_fm.py new file mode 100755 index 0000000..dd45d58 --- /dev/null +++ b/feed/sse_fm.py @@ -0,0 +1,134 @@ +#!/usr/bin/python + + +import sys, urllib2, os, socket +from xml.sax import ContentHandler, make_parser +from sse-config import * + +SSE_TAR = SSE_DIR + "sse-tar" + +def process_tar(project, tar): + print "New tar %s" % tar + + ret = os.system("%s '%s' '%s' '%s'" % (SSE_TAR, tar, project["name"], project["project-url"])) + + if ret != 0: + print "WARNING: Process returned %i" % ret + +def process_project(project): + + archive_url = None + + for a in ("archive-tgz-url", "archive-bz2-url", "archive-zip-url"): + + if project.has_key(a) and project[a] != "": + archive_url = project[a] + break + + if archive_url is None: + print "WARNING: Ignoring project '%s' without archive URL!" % project["name"] + return + + fn = "%s/download/freshmeat-%i" % (HOME, int(project["id"])) + + download = False + + try: + f = open(fn+".release", "r") + except: + download = True + else: + download = f.read() != project["date"].strip() + + if not download: + print "File %s up-to-date." % archive_url + else: + + print "Downloading %s..." % archive_url + + try: + dst = file(fn, "w") + src = urllib2.urlopen(archive_url) + + while True: + data = src.read(1024) + + if len(data) <= 0: + break + + dst.write(data) + + del dst + del src + + except IOError, e: + os.unlink(fn) + print "WARNING: Failed to download %s!" % archive_url + return + + try: + f = open(fn+".release", "w") + except: + os.unlink(fn) + + f.write(project["date"].strip()) + del f + + process_tar(project, fn) + +class docHandler(ContentHandler): + + project_data = {} + field = None + + def startElement(self, name, attrs): + if name == "project": + self.project_data = {} + self.field = None + elif name == "project_id": + self.field = "id" + elif name == "projectname_full": + self.field = "name" + elif name == "url_project_page": + self.field = "project-url" + elif name == "url_tgz": + self.field = "archive-tgz-url" + elif name == "url_bz2": + self.field = "archive-bz2-url" + elif name == "url_zip": + self.field = "archive-zip-url" + elif name == "license": + self.field = "license" + elif name == "latest_release_date": + self.field = "date" + else: + self.field = None + + def characters(self, data): + + if not self.field is None: + if self.project_data.has_key(self.field): + self.project_data[self.field] += data + else: + self.project_data[self.field] = data + + def endElement(self, name): + if name == "project": + process_project(self.project_data) + self.project_data = None + + self.field = None + +try: + os.mkdir("%s/download" % HOME) +except: + pass + +socket.setdefaulttimeout(20) + +dh = docHandler() + +parser = make_parser() + +parser.setContentHandler(dh) +parser.parse(sys.stdin) diff --git a/feed/sse_tar.py b/feed/sse_tar.py new file mode 100755 index 0000000..79671f6 --- /dev/null +++ b/feed/sse_tar.py @@ -0,0 +1,66 @@ +#!/usr/bin/python + +import tarfile, zipfile, sys, os +import sse_feed + +def archive_uncompress(archive, root, meta = {}): + + f = tarfile.open(archive, "r") + + while True: + + i = f.next() + + if i is None: + break + + if not i.isreg(): + continue + + if not sse_feed.supported_source(i.name): + continue + + dst = os.path.join(root, i.name) + f.extract(i, dst) + os.utime(dst, (i.mtime, i.mtime)) + + sse_feed.process_source(archive, root, i.name, meta) + + del f + +def calc_md(fn): + m = md5.new() + f = fopen(fn) + + while True: + data = m.read(1024) + + if len(data) <= 0: + break + + m.update(data) + + del f + + return m.hexdigest() + + +def process_archive(archive, meta = {}): + + md = calc_md(archive) + root = os.path.join(HOME, "sources", md) + + try: + os.mkdir(root) + except: + pass + + meta["md"] = md + + archive_uncompress(archive, root, meta) + +if __name__ == "__main__": + + archive_uncompress(sys.argv[1]) + + -- cgit