diff --git a/pyferea.py b/pyferea.py index 7ffafc3..3bf1ef0 100644 --- a/pyferea.py +++ b/pyferea.py @@ -29,7 +29,7 @@ from urlparse import urlparse, urlunparse, urljoin import feedparser from lxml import etree from cStringIO import StringIO -import shelve +import sqlite_db import time import datetime import os, re @@ -97,7 +97,7 @@ def markup_escape_text(text): """ if not text: return "" - return GLib.markup_escape_text(text.encode('utf-8')) + return GLib.markup_escape_text(text) class TabLabel(Gtk.HBox): """A class for Tab labels""" @@ -489,16 +489,13 @@ class EntryTree(Gtk.TreeView): _, it = selection.get_selected() if not it: return item = self.get_model().get_value(it, 0) - entry = self.feeddb[self.feedurl] - if entry['items'][item]['unread']: - title = markup_escape_text(entry['items'][item]['title']) - date = get_time_pretty(entry['items'][item]['date']) + entry = self.feeddb.get_entry(self.feedurl, item) + if entry['unread']: + title = markup_escape_text(entry['title']) + date = get_time_pretty(entry['date']) self.get_model().set_value(it, 1, title) self.get_model().set_value(it, 2, date) - entry['items'][item]['unread'] = False - entry['unread'] -= 1 - self.feeddb[self.feedurl] = entry - #self.feeddb.sync() # dont sync on every unread item - makes stuff extremely slow over time + self.feeddb.mark_read(self.feedurl, item) self.emit("item-selected", self.feedurl, item) self.connect("cursor-changed", on_cursor_changed_cb) @@ -517,11 +514,11 @@ class EntryTree(Gtk.TreeView): self.feedurl = None for feedurl in config: - if self.feeddb.get(feedurl): + if self.feeddb.feed_exists(feedurl): self.update(feedurl) def display(self, feedurl): - if not feedurl or feedurl not in self.feeddb: + if not feedurl or not self.feeddb.feed_exists(feedurl): self.set_model(self.empty_model) self.feedurl = None else: @@ -532,14 +529,13 @@ class EntryTree(Gtk.TreeView): model = Gtk.ListStore(GObject.TYPE_STRING, GObject.TYPE_STRING, GObject.TYPE_STRING) # using model.set_sort_column_id is horribly slow, so append them # sorted instead - items = sorted(self.feeddb[feedurl]['items'].iteritems(), key=lambda x: x[1]['date'], reverse=True) - for guid, value in items: + for value in self.feeddb.get_entries_all(feedurl): title = markup_escape_text(value.get('title', "")) date = get_time_pretty(value['date']) if value['unread']: title = ""+title+"" date = ""+date+"" - model.append([guid, title, date]) + model.append([value['entry'], title, date]) def compare_date(model, a, b, data): item1 = model.get_value(a, 0) item2 = model.get_value(b, 0) @@ -609,15 +605,16 @@ class FeedTree(Gtk.TreeView): for category, feeds in categories.items(): it = self.model.append(None, [None, category, folder_icon, None]) for feedurl in feeds: - if self.feeddb.get(feedurl): - feed_icon = self.feeddb[feedurl].get('favicon') + if self.feeddb.feed_exists(feedurl): + feed = self.feeddb.get_feed(feedurl) + feed_icon = feed.get('favicon') if feed_icon: feed_icon = pixbuf_new_from_file_in_memory(feed_icon, (16, 16)) else: feed_icon = self.render_icon(Gtk.STOCK_FILE, Gtk.IconSize.MENU, None) - label = markup_escape_text(self.feeddb[feedurl].get('title', feedurl)) - unread = self.feeddb[feedurl].get('unread') + label = markup_escape_text(feed.get('title', feedurl)) + unread = feed.get('unread') if unread > 0: label = ""+label+" (%d)"%unread+"" else: @@ -656,13 +653,9 @@ class FeedTree(Gtk.TreeView): def mark_read(self, it, sync=True): feedurl = self.model.get_value(it, 0) - entry = self.feeddb.get(feedurl) - if not entry: return - entry['unread'] = 0 - for item in entry['items'].values(): - item['unread'] = False - self.model.set_value(it, 1, markup_escape_text(entry['title'])) - self.feeddb[feedurl] = entry + feed = self.feeddb.get_feed(feedurl) + self.feeddb.mark_read_feed(feedurl) + self.model.set_value(it, 1, markup_escape_text(feed['title'])) self.emit("update-feed", feedurl) if sync: self.feeddb.sync() @@ -709,8 +702,9 @@ class FeedTree(Gtk.TreeView): itc = self.model.iter_children(it) while (itc): feedurl = self.model.get_value(itc, 0) - title = markup_escape_text(self.feeddb[feedurl]['title']) - unread = self.feeddb[feedurl]['unread'] + feed = self.feeddb.get_feed(feedurl) + title = markup_escape_text(feed['title']) + unread = feed['unread'] if unread > 0: title = ""+title+" (%d)"%unread+"" self.model.set_value(itc, 1, title) @@ -758,10 +752,11 @@ class FeedTree(Gtk.TreeView): def update_feed(self, it): feedurl = self.model.get_value(it, 0) msg = Soup.Message.new("GET", feedurl) - if self.feeddb.get(feedurl) and self.feeddb[feedurl].get('etag'): - msg.request_headers.append('If-None-Match', self.feeddb[feedurl]['etag']) - if self.feeddb.get(feedurl) and self.feeddb[feedurl].get('lastmodified'): - msg.request_headers.append('If-Modified-Since', self.feeddb[feedurl]['lastmodified']) + feed = self.feeddb.get_feed(feedurl) + if feed.get('etag'): + msg.request_headers.append('If-None-Match', feed['etag']) + if feed.get('lastmodified'): + msg.request_headers.append('If-Modified-Since', feed['lastmodified']) def complete_cb(session, msg, it): if msg.status_code not in [200, 304]: @@ -771,7 +766,7 @@ class FeedTree(Gtk.TreeView): return # get existing feedentry or create new one - entry = self.feeddb.get(feedurl, dict()) + entry = self.feeddb.get_feed(feedurl) if entry.get('favicon'): icon = pixbuf_new_from_file_in_memory(entry['favicon'], (16, 16)) @@ -796,7 +791,7 @@ class FeedTree(Gtk.TreeView): entry['lastmodified'] = msg.response_headers.get_one('Last-Modified') try: - feed = feedparser.parse(msg.response_body.flatten().get_data()) + feedparse = feedparser.parse(msg.response_body.flatten().get_data()) except: print "error parsing feed:" print msg.response_body.flatten().get_data() @@ -805,14 +800,14 @@ class FeedTree(Gtk.TreeView): self.update_feed_done(feedurl) return - if feed.bozo != 0: + if feedparse.bozo != 0: # retrieved data was no valid feed error_icon = self.render_icon(Gtk.STOCK_DIALOG_ERROR, Gtk.IconSize.MENU, None) self.model.set_value(it, 2, error_icon) self.update_feed_done(feedurl) return - entry['title'] = feed.feed.get('title') + entry['title'] = feedparse.feed.get('title') self.model.set_value(it, 1, markup_escape_text(entry['title'])) # assumption: favicon never changes @@ -820,14 +815,14 @@ class FeedTree(Gtk.TreeView): self.updating.add(feedurl+"_icon") self.update_icon(it, feedurl) - for item in feed.entries: + for item in feedparse.entries: # use guid with fallback to link as identifier itemid = item.get("id", item.get("link")) if not itemid: # TODO: display error "cannot identify feeditems" break - if entry['items'].has_key(itemid): + if self.feeddb.entry_exists(feedurl, itemid): # already exists continue @@ -836,7 +831,7 @@ class FeedTree(Gtk.TreeView): 'title': item.get('title'), 'date': item.get('published_parsed'), 'content': item.get('content'), - 'categories': [cat for _, cat in item.get('categories', [])] or None, + 'categories': ','.join([cat for _, cat in item.get('categories', [])]) or "", 'unread': True } @@ -858,14 +853,14 @@ class FeedTree(Gtk.TreeView): else: new_item['content'] = "" - entry['items'][itemid] = new_item + self.feeddb.add_entry(feedurl, itemid, new_item) entry['unread'] += 1 if entry['unread'] > 0: self.model.set_value(it, 1, ''+markup_escape_text(entry['title'])+" (%d)"%entry['unread']+'') - self.feeddb[feedurl] = entry + self.feeddb.update_feed(feedurl, entry) self.emit("update-feed", feedurl) @@ -894,9 +889,7 @@ class FeedTree(Gtk.TreeView): data = msg.response_body.flatten().get_data() if len(data): icon = pixbuf_new_from_file_in_memory(data, (16, 16)) - entry = self.feeddb[feedurl] - entry['favicon'] = data - self.feeddb[feedurl] = entry + self.feeddb.set_favicon(feedurl, data) self.model.set_value(it, 2, icon) self.update_feed_done(feedurl) else: @@ -920,9 +913,7 @@ class FeedTree(Gtk.TreeView): icon = self.render_icon(Gtk.STOCK_FILE, Gtk.IconSize.MENU, None) else: icon = self.render_icon(Gtk.STOCK_FILE, Gtk.IconSize.MENU, None) - entry = self.feeddb[feedurl] - entry['favicon'] = data - self.feeddb[feedurl] = entry + self.feeddb.set_favicon(feedurl, data) self.model.set_value(it, 2, icon) self.update_feed_done(feedurl+"_icon") self.session.queue_message(msg, complete_cb, it) @@ -932,23 +923,23 @@ class FeedReaderWindow(Gtk.Window): def __init__(self): Gtk.Window.__init__(self) - # try the following paths for pyferea.db in this order + # try the following paths for pyferea.sqlite in this order xdg_data_home = os.environ.get('XDG_DATA_HOME') or os.path.join(os.path.expanduser('~'), '.local', 'share') feeddb_paths = [ - "./pyferea.db", - os.path.join(xdg_data_home, "pyferea", "pyferea.db"), + "./pyferea.sqlite", + os.path.join(xdg_data_home, "pyferea", "pyferea.sqlite"), ] feeddb = None for path in feeddb_paths: if os.path.exists(path): - feeddb = shelve.open(path) + feeddb = sqlite_db.SQLStorage(path) break if not feeddb: - print "cannot find pyferea.db in any of the following locations:" + print "cannot find pyferea.sqlite in any of the following locations:" for path in feeddb_paths: print path print "creating new db at %s"%feeddb_paths[0] - feeddb = shelve.open(feeddb_paths[0]) + feeddb = sqlite_db.SQLStorage(feeddb_paths[0]) # try the following paths for feeds.yaml in this order xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or os.path.join(os.path.expanduser('~'), '.config') @@ -1030,7 +1021,7 @@ class FeedReaderWindow(Gtk.Window): entries = EntryTree(config, feeddb) def item_selected_cb(entry, feedurl, item): - item = feeddb[feedurl]['items'][item] + item = feeddb.get_entry(feedurl, item) if config[feedurl]['loadlink']: content_pane.load_uri(item['link']) else: diff --git a/sqlite_db.py b/sqlite_db.py new file mode 100644 index 0000000..a86bd60 --- /dev/null +++ b/sqlite_db.py @@ -0,0 +1,107 @@ +import sqlite3 + + +def dbcreate(conn): + conn.execute(""" + CREATE TABLE IF NOT EXISTS feeds ( + feed TEXT NOT NULL, + title TEXT, + favicon BLOB, + etag TEXT, + lastmodified TEXT, + unread INTEGER + ) + """) + conn.execute(""" + CREATE UNIQUE INDEX IF NOT EXISTS feedidx ON feeds (feed) + """) + conn.execute(""" + CREATE TABLE IF NOT EXISTS entries ( + feed TEXT NOT NULL, + entry TEXT NOT NULL, + title TEXT, + content TEXT, + link TEXT, + date INTEGER, + unread INTEGER, + categories TEXT + )""") + conn.execute(""" + CREATE UNIQUE INDEX IF NOT EXISTS entridx ON entries (feed,entry) + """) + conn.commit() + +def convert(filename): + conn = sqlite3.connect(filename) + conn.text_factory = str + dbcreate(conn) + import shelve + feeddb = shelve.open("pyferea.db") + for feed, fvalues in feeddb.items(): + conn.execute("""REPLACE INTO feeds (feed, title, favicon, etag, lastmodified, unread) VALUES (?,?,?,?,?,?)""", + (feed, fvalues.get('title'), fvalues.get('favicon'), fvalues.get('etag'), fvalues.get('lastmodified'), fvalues.get('unread'))) + for entry, evalues in fvalues['items'].items(): + conn.execute("""REPLACE INTO entries (feed, entry, title, content, link, date, unread, categories) VALUES (?, ?, ?, ?, ?, ?, ?, ?)""", + (feed, entry, evalues.get('title'), evalues.get('content'), evalues.get('link'), evalues.get('date'), evalues.get('unread'), ', '.join(evalues.get('categories') or []))) + conn.commit() + conn.close() + feeddb.close() + +class SQLStorage(): + def __init__(self, filename=':memory:'): + self.conn = sqlite3.connect(filename) + self.conn.text_factory = str + dbcreate(self.conn) + + def get_feed(self, feed): + return dict(zip(('title', 'favicon', 'etag', 'lastmodified', 'unread'), + self.conn.execute("""SELECT title, favicon, etag, lastmodified, unread FROM feeds WHERE feed=?""", (feed,)).fetchone())) + + def get_entry(self, feed, entry): + return dict(zip(('title', 'content', 'link', 'date', 'unread', 'categories'), + self.conn.execute("""SELECT title, content, link, date, unread, categories FROM entries WHERE feed=? AND entry=?""", (feed, entry)).fetchone())) + + def get_entries_all(self, feed): + return [dict(zip(('entry', 'title', 'date', 'unread'), content)) + for content in self.conn.execute(""" + SELECT entry, title, date, unread FROM entries WHERE feed=? ORDER BY date DESC + """, (feed,)).fetchall()] + + def add_entry(self, feed, entry, values): + self.conn.execute("""REPLACE INTO entries (feed, entry, title, content, link, date, unread, categories) VALUES (?, ?, ?, ?, ?, ?, ?, ?)""", + (feed, entry, values['title'], values['content'], values['link'], values['date'], values['unread'], values['categories'])) + self.conn.commit() + + def update_feed(self, feed, values): + self.conn.execute("""UPDATE feeds SET title=?, favicon=?, etag=?, lastmodified=?, unread=? WHERE feed=?""", + (values['title'], values['favicon'], values['etag'], values['lastmodified'], values['unread'], feed)) + self.conn.commit() + + def set_favicon(self, feed, favicon): + self.conn.execute("""UPDATE feeds SET favicon=? WHERE feed=?""", (favicon, feed)) + self.conn.commit() + + def mark_read(self, feed, entry): + self.conn.execute("""UPDATE entries SET unread=0 WHERE feed=? AND entry=?""", (feed, entry)) + self.conn.execute("""UPDATE feeds SET unread=unread-1 WHERE feed=?""", (feed,)) + self.conn.commit() + + def mark_read_feed(self, feed): + self.conn.execute("""UPDATE entries SET unread=0 WHERE unread=1""") + self.conn.execute("""UPDATE feeds set unread=0 WHERE feed=?""", (feed,)) + self.conn.commit() + + def feed_exists(self, feed): + return self.conn.execute("""SELECT feed FROM feeds WHERE feed=?""", (feed,)).fetchone() is not None + + def entry_exists(self, feed, entry): + return self.conn.execute("""SELECT feed FROM entries WHERE feed=? AND entry=?""", (feed,entry)).fetchone() is not None + + def close(self): + self.conn.close() + + def sync(self): + pass + +if __name__ == "__main__": + convert("pyferea.sqlite")