diff --git a/pyferea.py b/pyferea.py
index 7ffafc3..3bf1ef0 100644
--- a/pyferea.py
+++ b/pyferea.py
@@ -29,7 +29,7 @@ from urlparse import urlparse, urlunparse, urljoin
import feedparser
from lxml import etree
from cStringIO import StringIO
-import shelve
+import sqlite_db
import time
import datetime
import os, re
@@ -97,7 +97,7 @@ def markup_escape_text(text):
"""
if not text:
return ""
- return GLib.markup_escape_text(text.encode('utf-8'))
+ return GLib.markup_escape_text(text)
class TabLabel(Gtk.HBox):
"""A class for Tab labels"""
@@ -489,16 +489,13 @@ class EntryTree(Gtk.TreeView):
_, it = selection.get_selected()
if not it: return
item = self.get_model().get_value(it, 0)
- entry = self.feeddb[self.feedurl]
- if entry['items'][item]['unread']:
- title = markup_escape_text(entry['items'][item]['title'])
- date = get_time_pretty(entry['items'][item]['date'])
+ entry = self.feeddb.get_entry(self.feedurl, item)
+ if entry['unread']:
+ title = markup_escape_text(entry['title'])
+ date = get_time_pretty(entry['date'])
self.get_model().set_value(it, 1, title)
self.get_model().set_value(it, 2, date)
- entry['items'][item]['unread'] = False
- entry['unread'] -= 1
- self.feeddb[self.feedurl] = entry
- #self.feeddb.sync() # dont sync on every unread item - makes stuff extremely slow over time
+ self.feeddb.mark_read(self.feedurl, item)
self.emit("item-selected", self.feedurl, item)
self.connect("cursor-changed", on_cursor_changed_cb)
@@ -517,11 +514,11 @@ class EntryTree(Gtk.TreeView):
self.feedurl = None
for feedurl in config:
- if self.feeddb.get(feedurl):
+ if self.feeddb.feed_exists(feedurl):
self.update(feedurl)
def display(self, feedurl):
- if not feedurl or feedurl not in self.feeddb:
+ if not feedurl or not self.feeddb.feed_exists(feedurl):
self.set_model(self.empty_model)
self.feedurl = None
else:
@@ -532,14 +529,13 @@ class EntryTree(Gtk.TreeView):
model = Gtk.ListStore(GObject.TYPE_STRING, GObject.TYPE_STRING, GObject.TYPE_STRING)
# using model.set_sort_column_id is horribly slow, so append them
# sorted instead
- items = sorted(self.feeddb[feedurl]['items'].iteritems(), key=lambda x: x[1]['date'], reverse=True)
- for guid, value in items:
+ for value in self.feeddb.get_entries_all(feedurl):
title = markup_escape_text(value.get('title', ""))
date = get_time_pretty(value['date'])
if value['unread']:
title = ""+title+""
date = ""+date+""
- model.append([guid, title, date])
+ model.append([value['entry'], title, date])
def compare_date(model, a, b, data):
item1 = model.get_value(a, 0)
item2 = model.get_value(b, 0)
@@ -609,15 +605,16 @@ class FeedTree(Gtk.TreeView):
for category, feeds in categories.items():
it = self.model.append(None, [None, category, folder_icon, None])
for feedurl in feeds:
- if self.feeddb.get(feedurl):
- feed_icon = self.feeddb[feedurl].get('favicon')
+ if self.feeddb.feed_exists(feedurl):
+ feed = self.feeddb.get_feed(feedurl)
+ feed_icon = feed.get('favicon')
if feed_icon:
feed_icon = pixbuf_new_from_file_in_memory(feed_icon, (16, 16))
else:
feed_icon = self.render_icon(Gtk.STOCK_FILE, Gtk.IconSize.MENU, None)
- label = markup_escape_text(self.feeddb[feedurl].get('title', feedurl))
- unread = self.feeddb[feedurl].get('unread')
+ label = markup_escape_text(feed.get('title', feedurl))
+ unread = feed.get('unread')
if unread > 0:
label = ""+label+" (%d)"%unread+""
else:
@@ -656,13 +653,9 @@ class FeedTree(Gtk.TreeView):
def mark_read(self, it, sync=True):
feedurl = self.model.get_value(it, 0)
- entry = self.feeddb.get(feedurl)
- if not entry: return
- entry['unread'] = 0
- for item in entry['items'].values():
- item['unread'] = False
- self.model.set_value(it, 1, markup_escape_text(entry['title']))
- self.feeddb[feedurl] = entry
+ feed = self.feeddb.get_feed(feedurl)
+ self.feeddb.mark_read_feed(feedurl)
+ self.model.set_value(it, 1, markup_escape_text(feed['title']))
self.emit("update-feed", feedurl)
if sync:
self.feeddb.sync()
@@ -709,8 +702,9 @@ class FeedTree(Gtk.TreeView):
itc = self.model.iter_children(it)
while (itc):
feedurl = self.model.get_value(itc, 0)
- title = markup_escape_text(self.feeddb[feedurl]['title'])
- unread = self.feeddb[feedurl]['unread']
+ feed = self.feeddb.get_feed(feedurl)
+ title = markup_escape_text(feed['title'])
+ unread = feed['unread']
if unread > 0:
title = ""+title+" (%d)"%unread+""
self.model.set_value(itc, 1, title)
@@ -758,10 +752,11 @@ class FeedTree(Gtk.TreeView):
def update_feed(self, it):
feedurl = self.model.get_value(it, 0)
msg = Soup.Message.new("GET", feedurl)
- if self.feeddb.get(feedurl) and self.feeddb[feedurl].get('etag'):
- msg.request_headers.append('If-None-Match', self.feeddb[feedurl]['etag'])
- if self.feeddb.get(feedurl) and self.feeddb[feedurl].get('lastmodified'):
- msg.request_headers.append('If-Modified-Since', self.feeddb[feedurl]['lastmodified'])
+ feed = self.feeddb.get_feed(feedurl)
+ if feed.get('etag'):
+ msg.request_headers.append('If-None-Match', feed['etag'])
+ if feed.get('lastmodified'):
+ msg.request_headers.append('If-Modified-Since', feed['lastmodified'])
def complete_cb(session, msg, it):
if msg.status_code not in [200, 304]:
@@ -771,7 +766,7 @@ class FeedTree(Gtk.TreeView):
return
# get existing feedentry or create new one
- entry = self.feeddb.get(feedurl, dict())
+ entry = self.feeddb.get_feed(feedurl)
if entry.get('favicon'):
icon = pixbuf_new_from_file_in_memory(entry['favicon'], (16, 16))
@@ -796,7 +791,7 @@ class FeedTree(Gtk.TreeView):
entry['lastmodified'] = msg.response_headers.get_one('Last-Modified')
try:
- feed = feedparser.parse(msg.response_body.flatten().get_data())
+ feedparse = feedparser.parse(msg.response_body.flatten().get_data())
except:
print "error parsing feed:"
print msg.response_body.flatten().get_data()
@@ -805,14 +800,14 @@ class FeedTree(Gtk.TreeView):
self.update_feed_done(feedurl)
return
- if feed.bozo != 0:
+ if feedparse.bozo != 0:
# retrieved data was no valid feed
error_icon = self.render_icon(Gtk.STOCK_DIALOG_ERROR, Gtk.IconSize.MENU, None)
self.model.set_value(it, 2, error_icon)
self.update_feed_done(feedurl)
return
- entry['title'] = feed.feed.get('title')
+ entry['title'] = feedparse.feed.get('title')
self.model.set_value(it, 1, markup_escape_text(entry['title']))
# assumption: favicon never changes
@@ -820,14 +815,14 @@ class FeedTree(Gtk.TreeView):
self.updating.add(feedurl+"_icon")
self.update_icon(it, feedurl)
- for item in feed.entries:
+ for item in feedparse.entries:
# use guid with fallback to link as identifier
itemid = item.get("id", item.get("link"))
if not itemid:
# TODO: display error "cannot identify feeditems"
break
- if entry['items'].has_key(itemid):
+ if self.feeddb.entry_exists(feedurl, itemid):
# already exists
continue
@@ -836,7 +831,7 @@ class FeedTree(Gtk.TreeView):
'title': item.get('title'),
'date': item.get('published_parsed'),
'content': item.get('content'),
- 'categories': [cat for _, cat in item.get('categories', [])] or None,
+ 'categories': ','.join([cat for _, cat in item.get('categories', [])]) or "",
'unread': True
}
@@ -858,14 +853,14 @@ class FeedTree(Gtk.TreeView):
else:
new_item['content'] = ""
- entry['items'][itemid] = new_item
+ self.feeddb.add_entry(feedurl, itemid, new_item)
entry['unread'] += 1
if entry['unread'] > 0:
self.model.set_value(it, 1, ''+markup_escape_text(entry['title'])+" (%d)"%entry['unread']+'')
- self.feeddb[feedurl] = entry
+ self.feeddb.update_feed(feedurl, entry)
self.emit("update-feed", feedurl)
@@ -894,9 +889,7 @@ class FeedTree(Gtk.TreeView):
data = msg.response_body.flatten().get_data()
if len(data):
icon = pixbuf_new_from_file_in_memory(data, (16, 16))
- entry = self.feeddb[feedurl]
- entry['favicon'] = data
- self.feeddb[feedurl] = entry
+ self.feeddb.set_favicon(feedurl, data)
self.model.set_value(it, 2, icon)
self.update_feed_done(feedurl)
else:
@@ -920,9 +913,7 @@ class FeedTree(Gtk.TreeView):
icon = self.render_icon(Gtk.STOCK_FILE, Gtk.IconSize.MENU, None)
else:
icon = self.render_icon(Gtk.STOCK_FILE, Gtk.IconSize.MENU, None)
- entry = self.feeddb[feedurl]
- entry['favicon'] = data
- self.feeddb[feedurl] = entry
+ self.feeddb.set_favicon(feedurl, data)
self.model.set_value(it, 2, icon)
self.update_feed_done(feedurl+"_icon")
self.session.queue_message(msg, complete_cb, it)
@@ -932,23 +923,23 @@ class FeedReaderWindow(Gtk.Window):
def __init__(self):
Gtk.Window.__init__(self)
- # try the following paths for pyferea.db in this order
+ # try the following paths for pyferea.sqlite in this order
xdg_data_home = os.environ.get('XDG_DATA_HOME') or os.path.join(os.path.expanduser('~'), '.local', 'share')
feeddb_paths = [
- "./pyferea.db",
- os.path.join(xdg_data_home, "pyferea", "pyferea.db"),
+ "./pyferea.sqlite",
+ os.path.join(xdg_data_home, "pyferea", "pyferea.sqlite"),
]
feeddb = None
for path in feeddb_paths:
if os.path.exists(path):
- feeddb = shelve.open(path)
+ feeddb = sqlite_db.SQLStorage(path)
break
if not feeddb:
- print "cannot find pyferea.db in any of the following locations:"
+ print "cannot find pyferea.sqlite in any of the following locations:"
for path in feeddb_paths:
print path
print "creating new db at %s"%feeddb_paths[0]
- feeddb = shelve.open(feeddb_paths[0])
+ feeddb = sqlite_db.SQLStorage(feeddb_paths[0])
# try the following paths for feeds.yaml in this order
xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or os.path.join(os.path.expanduser('~'), '.config')
@@ -1030,7 +1021,7 @@ class FeedReaderWindow(Gtk.Window):
entries = EntryTree(config, feeddb)
def item_selected_cb(entry, feedurl, item):
- item = feeddb[feedurl]['items'][item]
+ item = feeddb.get_entry(feedurl, item)
if config[feedurl]['loadlink']:
content_pane.load_uri(item['link'])
else:
diff --git a/sqlite_db.py b/sqlite_db.py
new file mode 100644
index 0000000..a86bd60
--- /dev/null
+++ b/sqlite_db.py
@@ -0,0 +1,107 @@
+import sqlite3
+
+
+def dbcreate(conn):
+ conn.execute("""
+ CREATE TABLE IF NOT EXISTS feeds (
+ feed TEXT NOT NULL,
+ title TEXT,
+ favicon BLOB,
+ etag TEXT,
+ lastmodified TEXT,
+ unread INTEGER
+ )
+ """)
+ conn.execute("""
+ CREATE UNIQUE INDEX IF NOT EXISTS feedidx ON feeds (feed)
+ """)
+ conn.execute("""
+ CREATE TABLE IF NOT EXISTS entries (
+ feed TEXT NOT NULL,
+ entry TEXT NOT NULL,
+ title TEXT,
+ content TEXT,
+ link TEXT,
+ date INTEGER,
+ unread INTEGER,
+ categories TEXT
+ )""")
+ conn.execute("""
+ CREATE UNIQUE INDEX IF NOT EXISTS entridx ON entries (feed,entry)
+ """)
+ conn.commit()
+
+def convert(filename):
+ conn = sqlite3.connect(filename)
+ conn.text_factory = str
+ dbcreate(conn)
+ import shelve
+ feeddb = shelve.open("pyferea.db")
+ for feed, fvalues in feeddb.items():
+ conn.execute("""REPLACE INTO feeds (feed, title, favicon, etag, lastmodified, unread) VALUES (?,?,?,?,?,?)""",
+ (feed, fvalues.get('title'), fvalues.get('favicon'), fvalues.get('etag'), fvalues.get('lastmodified'), fvalues.get('unread')))
+ for entry, evalues in fvalues['items'].items():
+ conn.execute("""REPLACE INTO entries (feed, entry, title, content, link, date, unread, categories) VALUES (?, ?, ?, ?, ?, ?, ?, ?)""",
+ (feed, entry, evalues.get('title'), evalues.get('content'), evalues.get('link'), evalues.get('date'), evalues.get('unread'), ', '.join(evalues.get('categories') or [])))
+ conn.commit()
+ conn.close()
+ feeddb.close()
+
+class SQLStorage():
+ def __init__(self, filename=':memory:'):
+ self.conn = sqlite3.connect(filename)
+ self.conn.text_factory = str
+ dbcreate(self.conn)
+
+ def get_feed(self, feed):
+ return dict(zip(('title', 'favicon', 'etag', 'lastmodified', 'unread'),
+ self.conn.execute("""SELECT title, favicon, etag, lastmodified, unread FROM feeds WHERE feed=?""", (feed,)).fetchone()))
+
+ def get_entry(self, feed, entry):
+ return dict(zip(('title', 'content', 'link', 'date', 'unread', 'categories'),
+ self.conn.execute("""SELECT title, content, link, date, unread, categories FROM entries WHERE feed=? AND entry=?""", (feed, entry)).fetchone()))
+
+ def get_entries_all(self, feed):
+ return [dict(zip(('entry', 'title', 'date', 'unread'), content))
+ for content in self.conn.execute("""
+ SELECT entry, title, date, unread FROM entries WHERE feed=? ORDER BY date DESC
+ """, (feed,)).fetchall()]
+
+ def add_entry(self, feed, entry, values):
+ self.conn.execute("""REPLACE INTO entries (feed, entry, title, content, link, date, unread, categories) VALUES (?, ?, ?, ?, ?, ?, ?, ?)""",
+ (feed, entry, values['title'], values['content'], values['link'], values['date'], values['unread'], values['categories']))
+ self.conn.commit()
+
+ def update_feed(self, feed, values):
+ self.conn.execute("""UPDATE feeds SET title=?, favicon=?, etag=?, lastmodified=?, unread=? WHERE feed=?""",
+ (values['title'], values['favicon'], values['etag'], values['lastmodified'], values['unread'], feed))
+ self.conn.commit()
+
+ def set_favicon(self, feed, favicon):
+ self.conn.execute("""UPDATE feeds SET favicon=? WHERE feed=?""", (favicon, feed))
+ self.conn.commit()
+
+ def mark_read(self, feed, entry):
+ self.conn.execute("""UPDATE entries SET unread=0 WHERE feed=? AND entry=?""", (feed, entry))
+ self.conn.execute("""UPDATE feeds SET unread=unread-1 WHERE feed=?""", (feed,))
+ self.conn.commit()
+
+ def mark_read_feed(self, feed):
+ self.conn.execute("""UPDATE entries SET unread=0 WHERE unread=1""")
+ self.conn.execute("""UPDATE feeds set unread=0 WHERE feed=?""", (feed,))
+ self.conn.commit()
+
+ def feed_exists(self, feed):
+ return self.conn.execute("""SELECT feed FROM feeds WHERE feed=?""", (feed,)).fetchone() is not None
+
+ def entry_exists(self, feed, entry):
+ return self.conn.execute("""SELECT feed FROM entries WHERE feed=? AND entry=?""", (feed,entry)).fetchone() is not None
+
+ def close(self):
+ self.conn.close()
+
+ def sync(self):
+ pass
+
+if __name__ == "__main__":
+ convert("pyferea.sqlite")