use sqlite as database backend

pyferea.py

@@ -29,7 +29,7 @@ from urlparse import urlparse, urlunparse, urljoin
 import feedparser
 from lxml import etree
 from cStringIO import StringIO
-import shelve
+import sqlite_db
 import time
 import datetime
 import os, re
@@ -97,7 +97,7 @@ def markup_escape_text(text):
     """
     if not text:
         return ""
-    return GLib.markup_escape_text(text.encode('utf-8'))
+    return GLib.markup_escape_text(text)
 
 class TabLabel(Gtk.HBox):
     """A class for Tab labels"""
@@ -489,16 +489,13 @@ class EntryTree(Gtk.TreeView):
             _, it = selection.get_selected()
             if not it: return
             item = self.get_model().get_value(it, 0)
-            entry = self.feeddb[self.feedurl]
+            entry = self.feeddb.get_entry(self.feedurl, item)
-            if entry['items'][item]['unread']:
+            if entry['unread']:
-                title = markup_escape_text(entry['items'][item]['title'])
+                title = markup_escape_text(entry['title'])
-                date = get_time_pretty(entry['items'][item]['date'])
+                date = get_time_pretty(entry['date'])
                 self.get_model().set_value(it, 1, title)
                 self.get_model().set_value(it, 2, date)
-                entry['items'][item]['unread'] = False
+                self.feeddb.mark_read(self.feedurl, item)
-                entry['unread'] -= 1
-                self.feeddb[self.feedurl] = entry
-                #self.feeddb.sync() # dont sync on every unread item - makes stuff extremely slow over time
             self.emit("item-selected", self.feedurl, item)
         self.connect("cursor-changed", on_cursor_changed_cb)
 
@@ -517,11 +514,11 @@ class EntryTree(Gtk.TreeView):
         self.feedurl = None
 
         for feedurl in config:
-            if self.feeddb.get(feedurl):
+            if self.feeddb.feed_exists(feedurl):
                 self.update(feedurl)
 
     def display(self, feedurl):
-        if not feedurl or feedurl not in self.feeddb:
+        if not feedurl or not self.feeddb.feed_exists(feedurl):
             self.set_model(self.empty_model)
             self.feedurl = None
         else:
@@ -532,14 +529,13 @@ class EntryTree(Gtk.TreeView):
         model = Gtk.ListStore(GObject.TYPE_STRING, GObject.TYPE_STRING, GObject.TYPE_STRING)
         # using model.set_sort_column_id is horribly slow, so append them
         # sorted instead
-        items = sorted(self.feeddb[feedurl]['items'].iteritems(), key=lambda x: x[1]['date'], reverse=True)
+        for value in self.feeddb.get_entries_all(feedurl):
-        for guid, value in items:
             title = markup_escape_text(value.get('title', ""))
             date = get_time_pretty(value['date'])
             if value['unread']:
                 title = "<b>"+title+"</b>"
                 date = "<b>"+date+"</b>"
-            model.append([guid, title, date])
+            model.append([value['entry'], title, date])
         def compare_date(model, a, b, data):
             item1 = model.get_value(a, 0)
             item2 = model.get_value(b, 0)
@@ -609,15 +605,16 @@ class FeedTree(Gtk.TreeView):
         for category, feeds in categories.items():
             it = self.model.append(None, [None, category, folder_icon, None])
             for feedurl in feeds:
-                if self.feeddb.get(feedurl):
+                if self.feeddb.feed_exists(feedurl):
-                    feed_icon = self.feeddb[feedurl].get('favicon')
+                    feed = self.feeddb.get_feed(feedurl)
+                    feed_icon = feed.get('favicon')
                     if feed_icon:
                         feed_icon = pixbuf_new_from_file_in_memory(feed_icon, (16, 16))
                     else:
                         feed_icon = self.render_icon(Gtk.STOCK_FILE, Gtk.IconSize.MENU, None)
 
-                    label = markup_escape_text(self.feeddb[feedurl].get('title', feedurl))
+                    label = markup_escape_text(feed.get('title', feedurl))
-                    unread = self.feeddb[feedurl].get('unread')
+                    unread = feed.get('unread')
                     if unread > 0:
                         label = "<b>"+label+" (%d)"%unread+"</b>"
                 else:
@@ -656,13 +653,9 @@ class FeedTree(Gtk.TreeView):
 
     def mark_read(self, it, sync=True):
         feedurl = self.model.get_value(it, 0)
-        entry = self.feeddb.get(feedurl)
+        feed = self.feeddb.get_feed(feedurl)
-        if not entry: return
+        self.feeddb.mark_read_feed(feedurl)
-        entry['unread'] = 0
+        self.model.set_value(it, 1, markup_escape_text(feed['title']))
-        for item in entry['items'].values():
-            item['unread'] = False
-        self.model.set_value(it, 1, markup_escape_text(entry['title']))
-        self.feeddb[feedurl] = entry
         self.emit("update-feed", feedurl)
         if sync:
             self.feeddb.sync()
@@ -709,8 +702,9 @@ class FeedTree(Gtk.TreeView):
             itc = self.model.iter_children(it)
             while (itc):
                 feedurl = self.model.get_value(itc, 0)
-                title = markup_escape_text(self.feeddb[feedurl]['title'])
+                feed = self.feeddb.get_feed(feedurl)
-                unread = self.feeddb[feedurl]['unread']
+                title = markup_escape_text(feed['title'])
+                unread = feed['unread']
                 if unread > 0:
                     title = "<b>"+title+" (%d)"%unread+"</b>"
                 self.model.set_value(itc, 1, title)
@@ -758,10 +752,11 @@ class FeedTree(Gtk.TreeView):
     def update_feed(self, it):
         feedurl = self.model.get_value(it, 0)
         msg = Soup.Message.new("GET", feedurl)
-        if self.feeddb.get(feedurl) and self.feeddb[feedurl].get('etag'):
+        feed = self.feeddb.get_feed(feedurl)
-            msg.request_headers.append('If-None-Match', self.feeddb[feedurl]['etag'])
+        if feed.get('etag'):
-        if self.feeddb.get(feedurl) and self.feeddb[feedurl].get('lastmodified'):
+            msg.request_headers.append('If-None-Match', feed['etag'])
-            msg.request_headers.append('If-Modified-Since', self.feeddb[feedurl]['lastmodified'])
+        if feed.get('lastmodified'):
+            msg.request_headers.append('If-Modified-Since', feed['lastmodified'])
 
         def complete_cb(session, msg, it):
             if msg.status_code not in [200, 304]:
@@ -771,7 +766,7 @@ class FeedTree(Gtk.TreeView):
                 return
 
             # get existing feedentry or create new one
-            entry = self.feeddb.get(feedurl, dict())
+            entry = self.feeddb.get_feed(feedurl)
 
             if entry.get('favicon'):
                 icon = pixbuf_new_from_file_in_memory(entry['favicon'], (16, 16))
@@ -796,7 +791,7 @@ class FeedTree(Gtk.TreeView):
                 entry['lastmodified'] = msg.response_headers.get_one('Last-Modified')
 
             try:
-                feed = feedparser.parse(msg.response_body.flatten().get_data())
+                feedparse = feedparser.parse(msg.response_body.flatten().get_data())
             except:
                 print "error parsing feed:"
                 print msg.response_body.flatten().get_data()
@@ -805,14 +800,14 @@ class FeedTree(Gtk.TreeView):
                 self.update_feed_done(feedurl)
                 return
 
-            if feed.bozo != 0:
+            if feedparse.bozo != 0:
                 # retrieved data was no valid feed
                 error_icon = self.render_icon(Gtk.STOCK_DIALOG_ERROR, Gtk.IconSize.MENU, None)
                 self.model.set_value(it, 2, error_icon)
                 self.update_feed_done(feedurl)
                 return
 
-            entry['title'] = feed.feed.get('title')
+            entry['title'] = feedparse.feed.get('title')
             self.model.set_value(it, 1, markup_escape_text(entry['title']))
 
             # assumption: favicon never changes
@@ -820,14 +815,14 @@ class FeedTree(Gtk.TreeView):
                 self.updating.add(feedurl+"_icon")
                 self.update_icon(it, feedurl)
 
-            for item in feed.entries:
+            for item in feedparse.entries:
                 # use guid with fallback to link as identifier
                 itemid = item.get("id", item.get("link"))
                 if not itemid:
                     # TODO: display error "cannot identify feeditems"
                     break
 
-                if entry['items'].has_key(itemid):
+                if self.feeddb.entry_exists(feedurl, itemid):
                     # already exists
                     continue
 
@@ -836,7 +831,7 @@ class FeedTree(Gtk.TreeView):
                     'title': item.get('title'),
                     'date': item.get('published_parsed'),
                     'content': item.get('content'),
-                    'categories': [cat for _, cat in item.get('categories', [])] or None,
+                    'categories': ','.join([cat for _, cat in item.get('categories', [])]) or "",
                     'unread': True
                 }
 
@@ -858,14 +853,14 @@ class FeedTree(Gtk.TreeView):
                 else:
                     new_item['content'] = ""
 
-                entry['items'][itemid] = new_item
+                self.feeddb.add_entry(feedurl, itemid, new_item)
 
                 entry['unread'] += 1
 
             if entry['unread'] > 0:
                 self.model.set_value(it, 1, '<b>'+markup_escape_text(entry['title'])+" (%d)"%entry['unread']+'</b>')
 
-            self.feeddb[feedurl] = entry
+            self.feeddb.update_feed(feedurl, entry)
 
             self.emit("update-feed", feedurl)
 
@@ -894,9 +889,7 @@ class FeedTree(Gtk.TreeView):
                 data = msg.response_body.flatten().get_data()
                 if len(data):
                     icon = pixbuf_new_from_file_in_memory(data, (16, 16))
-                    entry = self.feeddb[feedurl]
+                    self.feeddb.set_favicon(feedurl, data)
-                    entry['favicon'] = data
-                    self.feeddb[feedurl] = entry
                     self.model.set_value(it, 2, icon)
                     self.update_feed_done(feedurl)
                 else:
@@ -920,9 +913,7 @@ class FeedTree(Gtk.TreeView):
                     icon = self.render_icon(Gtk.STOCK_FILE, Gtk.IconSize.MENU, None)
             else:
                 icon = self.render_icon(Gtk.STOCK_FILE, Gtk.IconSize.MENU, None)
-            entry = self.feeddb[feedurl]
+            self.feeddb.set_favicon(feedurl, data)
-            entry['favicon'] = data
-            self.feeddb[feedurl] = entry
             self.model.set_value(it, 2, icon)
             self.update_feed_done(feedurl+"_icon")
         self.session.queue_message(msg, complete_cb, it)
@@ -932,23 +923,23 @@ class FeedReaderWindow(Gtk.Window):
     def __init__(self):
         Gtk.Window.__init__(self)
 
-        # try the following paths for pyferea.db in this order
+        # try the following paths for pyferea.sqlite in this order
         xdg_data_home = os.environ.get('XDG_DATA_HOME') or os.path.join(os.path.expanduser('~'), '.local', 'share')
         feeddb_paths = [
-            "./pyferea.db",
+            "./pyferea.sqlite",
-            os.path.join(xdg_data_home, "pyferea", "pyferea.db"),
+            os.path.join(xdg_data_home, "pyferea", "pyferea.sqlite"),
         ]
         feeddb = None
         for path in feeddb_paths:
             if os.path.exists(path):
-                feeddb = shelve.open(path)
+                feeddb = sqlite_db.SQLStorage(path)
                 break
         if not feeddb:
-            print "cannot find pyferea.db in any of the following locations:"
+            print "cannot find pyferea.sqlite in any of the following locations:"
             for path in feeddb_paths:
                 print path
             print "creating new db at %s"%feeddb_paths[0]
-            feeddb = shelve.open(feeddb_paths[0])
+            feeddb = sqlite_db.SQLStorage(feeddb_paths[0])
 
         # try the following paths for feeds.yaml in this order
         xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or os.path.join(os.path.expanduser('~'), '.config')
@@ -1030,7 +1021,7 @@ class FeedReaderWindow(Gtk.Window):
         entries = EntryTree(config, feeddb)
 
         def item_selected_cb(entry, feedurl, item):
-            item = feeddb[feedurl]['items'][item]
+            item = feeddb.get_entry(feedurl, item)
             if config[feedurl]['loadlink']:
                 content_pane.load_uri(item['link'])
             else:

sqlite_db.py

@@ -0,0 +1,107 @@
+import sqlite3
+
+
+def dbcreate(conn):
+    conn.execute("""
+    CREATE TABLE IF NOT EXISTS feeds (
+    feed TEXT NOT NULL,
+    title TEXT,
+    favicon BLOB,
+    etag TEXT,
+    lastmodified TEXT,
+    unread INTEGER
+    )
+    """)
+    conn.execute("""
+    CREATE UNIQUE INDEX IF NOT EXISTS feedidx ON feeds (feed)
+    """)
+    conn.execute("""
+    CREATE TABLE IF NOT EXISTS entries (
+    feed TEXT NOT NULL,
+    entry TEXT NOT NULL,
+    title TEXT,
+    content TEXT,
+    link TEXT,
+    date INTEGER,
+    unread INTEGER,
+    categories TEXT
+    )""")
+    conn.execute("""
+    CREATE UNIQUE INDEX IF NOT EXISTS entridx ON entries (feed,entry)
+    """)
+    conn.commit()
+
+def convert(filename):
+    conn = sqlite3.connect(filename)
+    conn.text_factory = str
+    dbcreate(conn)
+    import shelve
+    feeddb = shelve.open("pyferea.db")
+    for feed, fvalues in feeddb.items():
+        conn.execute("""REPLACE INTO feeds (feed, title, favicon, etag, lastmodified, unread) VALUES (?,?,?,?,?,?)""",
+        (feed, fvalues.get('title'), fvalues.get('favicon'), fvalues.get('etag'), fvalues.get('lastmodified'), fvalues.get('unread')))
+        for entry, evalues in fvalues['items'].items():
+            conn.execute("""REPLACE INTO entries (feed, entry, title, content, link, date, unread, categories) VALUES (?, ?, ?, ?, ?, ?, ?, ?)""",
+            (feed, entry, evalues.get('title'), evalues.get('content'), evalues.get('link'), evalues.get('date'), evalues.get('unread'), ', '.join(evalues.get('categories') or [])))
+    conn.commit()
+    conn.close()
+    feeddb.close()
+
+class SQLStorage():
+    def __init__(self, filename=':memory:'):
+        self.conn = sqlite3.connect(filename)
+        self.conn.text_factory = str
+        dbcreate(self.conn)
+
+    def get_feed(self, feed):
+        return dict(zip(('title', 'favicon', 'etag', 'lastmodified', 'unread'),
+            self.conn.execute("""SELECT title, favicon, etag, lastmodified, unread FROM feeds WHERE feed=?""", (feed,)).fetchone()))
+
+    def get_entry(self, feed, entry):
+        return dict(zip(('title', 'content', 'link', 'date', 'unread', 'categories'),
+                  self.conn.execute("""SELECT title, content, link, date, unread, categories FROM entries WHERE feed=? AND entry=?""", (feed, entry)).fetchone()))
+
+    def get_entries_all(self, feed):
+        return [dict(zip(('entry', 'title', 'date', 'unread'), content))
+                for content in self.conn.execute("""
+                    SELECT entry, title, date, unread FROM entries WHERE feed=? ORDER BY date DESC
+                """, (feed,)).fetchall()]
+
+    def add_entry(self, feed, entry, values):
+        self.conn.execute("""REPLACE INTO entries (feed, entry, title, content, link, date, unread, categories) VALUES (?, ?, ?, ?, ?, ?, ?, ?)""",
+            (feed, entry, values['title'], values['content'], values['link'], values['date'], values['unread'], values['categories']))
+        self.conn.commit()
+
+    def update_feed(self, feed, values):
+        self.conn.execute("""UPDATE feeds SET title=?, favicon=?, etag=?, lastmodified=?, unread=? WHERE feed=?""",
+            (values['title'], values['favicon'], values['etag'], values['lastmodified'], values['unread'], feed))
+        self.conn.commit()
+
+    def set_favicon(self, feed, favicon):
+        self.conn.execute("""UPDATE feeds SET favicon=? WHERE feed=?""", (favicon, feed))
+        self.conn.commit()
+
+    def mark_read(self, feed, entry):
+        self.conn.execute("""UPDATE entries SET unread=0 WHERE feed=? AND entry=?""", (feed, entry))
+        self.conn.execute("""UPDATE feeds SET unread=unread-1 WHERE feed=?""", (feed,))
+        self.conn.commit()
+
+    def mark_read_feed(self, feed):
+        self.conn.execute("""UPDATE entries SET unread=0 WHERE unread=1""")
+        self.conn.execute("""UPDATE feeds set unread=0 WHERE feed=?""", (feed,))
+        self.conn.commit()
+
+    def feed_exists(self, feed):
+        return self.conn.execute("""SELECT feed FROM feeds WHERE feed=?""", (feed,)).fetchone() is not None
+
+    def entry_exists(self, feed, entry):
+        return self.conn.execute("""SELECT feed FROM entries WHERE feed=? AND entry=?""", (feed,entry)).fetchone() is not None
+
+    def close(self):
+        self.conn.close()
+
+    def sync(self):
+        pass
+
+if __name__ == "__main__":
+    convert("pyferea.sqlite")