josch
/
pyferea
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

use sqlite as database backend

Browse Source
main
josch 12 years ago
parent 776c30770f
commit b55097e8c3
2 changed files with 152 additions and 54 deletions
Show Stats Download Patch File Download Diff File

99
pyferea.py
Unescape Escape View File

@ -29,7 +29,7 @@ from urlparse import urlparse, urlunparse, urljoin
import feedparser import feedparser
from lxml import etree from lxml import etree
from cStringIO import StringIO from cStringIO import StringIO
import shelve import sqlite_db
import time import time
import datetime import datetime
import os, re import os, re
@ -97,7 +97,7 @@ def markup_escape_text(text):
""" """
if not text: if not text:
return "" return ""
return GLib.markup_escape_text(text.encode('utf-8')) return GLib.markup_escape_text(text)
class TabLabel(Gtk.HBox): class TabLabel(Gtk.HBox):
"""A class for Tab labels""" """A class for Tab labels"""
@ -489,16 +489,13 @@ class EntryTree(Gtk.TreeView):
_, it = selection.get_selected() _, it = selection.get_selected()
if not it: return if not it: return
item = self.get_model().get_value(it, 0) item = self.get_model().get_value(it, 0)
entry = self.feeddb[self.feedurl] entry = self.feeddb.get_entry(self.feedurl, item)
if entry['items'][item]['unread']: if entry['unread']:
title = markup_escape_text(entry['items'][item]['title']) title = markup_escape_text(entry['title'])
date = get_time_pretty(entry['items'][item]['date']) date = get_time_pretty(entry['date'])
self.get_model().set_value(it, 1, title) self.get_model().set_value(it, 1, title)
self.get_model().set_value(it, 2, date) self.get_model().set_value(it, 2, date)
entry['items'][item]['unread'] = False self.feeddb.mark_read(self.feedurl, item)
entry['unread'] -= 1
self.feeddb[self.feedurl] = entry
#self.feeddb.sync() # dont sync on every unread item - makes stuff extremely slow over time
self.emit("item-selected", self.feedurl, item) self.emit("item-selected", self.feedurl, item)
self.connect("cursor-changed", on_cursor_changed_cb) self.connect("cursor-changed", on_cursor_changed_cb)
@ -517,11 +514,11 @@ class EntryTree(Gtk.TreeView):
self.feedurl = None self.feedurl = None
for feedurl in config: for feedurl in config:
if self.feeddb.get(feedurl): if self.feeddb.feed_exists(feedurl):
self.update(feedurl) self.update(feedurl)
def display(self, feedurl): def display(self, feedurl):
if not feedurl or feedurl not in self.feeddb: if not feedurl or not self.feeddb.feed_exists(feedurl):
self.set_model(self.empty_model) self.set_model(self.empty_model)
self.feedurl = None self.feedurl = None
else: else:
@ -532,14 +529,13 @@ class EntryTree(Gtk.TreeView):
model = Gtk.ListStore(GObject.TYPE_STRING, GObject.TYPE_STRING, GObject.TYPE_STRING) model = Gtk.ListStore(GObject.TYPE_STRING, GObject.TYPE_STRING, GObject.TYPE_STRING)
# using model.set_sort_column_id is horribly slow, so append them # using model.set_sort_column_id is horribly slow, so append them
# sorted instead # sorted instead
items = sorted(self.feeddb[feedurl]['items'].iteritems(), key=lambda x: x[1]['date'], reverse=True) for value in self.feeddb.get_entries_all(feedurl):
for guid, value in items:
title = markup_escape_text(value.get('title', "")) title = markup_escape_text(value.get('title', ""))
date = get_time_pretty(value['date']) date = get_time_pretty(value['date'])
if value['unread']: if value['unread']:
title = "<b>"+title+"</b>" title = "<b>"+title+"</b>"
date = "<b>"+date+"</b>" date = "<b>"+date+"</b>"
model.append([guid, title, date]) model.append([value['entry'], title, date])
def compare_date(model, a, b, data): def compare_date(model, a, b, data):
item1 = model.get_value(a, 0) item1 = model.get_value(a, 0)
item2 = model.get_value(b, 0) item2 = model.get_value(b, 0)
@ -609,15 +605,16 @@ class FeedTree(Gtk.TreeView):
for category, feeds in categories.items(): for category, feeds in categories.items():
it = self.model.append(None, [None, category, folder_icon, None]) it = self.model.append(None, [None, category, folder_icon, None])
for feedurl in feeds: for feedurl in feeds:
if self.feeddb.get(feedurl): if self.feeddb.feed_exists(feedurl):
feed_icon = self.feeddb[feedurl].get('favicon') feed = self.feeddb.get_feed(feedurl)
feed_icon = feed.get('favicon')
if feed_icon: if feed_icon:
feed_icon = pixbuf_new_from_file_in_memory(feed_icon, (16, 16)) feed_icon = pixbuf_new_from_file_in_memory(feed_icon, (16, 16))
else: else:
feed_icon = self.render_icon(Gtk.STOCK_FILE, Gtk.IconSize.MENU, None) feed_icon = self.render_icon(Gtk.STOCK_FILE, Gtk.IconSize.MENU, None)
label = markup_escape_text(self.feeddb[feedurl].get('title', feedurl)) label = markup_escape_text(feed.get('title', feedurl))
unread = self.feeddb[feedurl].get('unread') unread = feed.get('unread')
if unread > 0: if unread > 0:
label = "<b>"+label+" (%d)"%unread+"</b>" label = "<b>"+label+" (%d)"%unread+"</b>"
else: else:
@ -656,13 +653,9 @@ class FeedTree(Gtk.TreeView):
def mark_read(self, it, sync=True): def mark_read(self, it, sync=True):
feedurl = self.model.get_value(it, 0) feedurl = self.model.get_value(it, 0)
entry = self.feeddb.get(feedurl) feed = self.feeddb.get_feed(feedurl)
if not entry: return self.feeddb.mark_read_feed(feedurl)
entry['unread'] = 0 self.model.set_value(it, 1, markup_escape_text(feed['title']))
for item in entry['items'].values():
item['unread'] = False
self.model.set_value(it, 1, markup_escape_text(entry['title']))
self.feeddb[feedurl] = entry
self.emit("update-feed", feedurl) self.emit("update-feed", feedurl)
if sync: if sync:
self.feeddb.sync() self.feeddb.sync()
@ -709,8 +702,9 @@ class FeedTree(Gtk.TreeView):
itc = self.model.iter_children(it) itc = self.model.iter_children(it)
while (itc): while (itc):
feedurl = self.model.get_value(itc, 0) feedurl = self.model.get_value(itc, 0)
title = markup_escape_text(self.feeddb[feedurl]['title']) feed = self.feeddb.get_feed(feedurl)
unread = self.feeddb[feedurl]['unread'] title = markup_escape_text(feed['title'])
unread = feed['unread']
if unread > 0: if unread > 0:
title = "<b>"+title+" (%d)"%unread+"</b>" title = "<b>"+title+" (%d)"%unread+"</b>"
self.model.set_value(itc, 1, title) self.model.set_value(itc, 1, title)
@ -758,10 +752,11 @@ class FeedTree(Gtk.TreeView):
def update_feed(self, it): def update_feed(self, it):
feedurl = self.model.get_value(it, 0) feedurl = self.model.get_value(it, 0)
msg = Soup.Message.new("GET", feedurl) msg = Soup.Message.new("GET", feedurl)
if self.feeddb.get(feedurl) and self.feeddb[feedurl].get('etag'): feed = self.feeddb.get_feed(feedurl)
msg.request_headers.append('If-None-Match', self.feeddb[feedurl]['etag']) if feed.get('etag'):
if self.feeddb.get(feedurl) and self.feeddb[feedurl].get('lastmodified'): msg.request_headers.append('If-None-Match', feed['etag'])
msg.request_headers.append('If-Modified-Since', self.feeddb[feedurl]['lastmodified']) if feed.get('lastmodified'):
msg.request_headers.append('If-Modified-Since', feed['lastmodified'])
def complete_cb(session, msg, it): def complete_cb(session, msg, it):
if msg.status_code not in [200, 304]: if msg.status_code not in [200, 304]:
@ -771,7 +766,7 @@ class FeedTree(Gtk.TreeView):
return return
# get existing feedentry or create new one # get existing feedentry or create new one
entry = self.feeddb.get(feedurl, dict()) entry = self.feeddb.get_feed(feedurl)
if entry.get('favicon'): if entry.get('favicon'):
icon = pixbuf_new_from_file_in_memory(entry['favicon'], (16, 16)) icon = pixbuf_new_from_file_in_memory(entry['favicon'], (16, 16))
@ -796,7 +791,7 @@ class FeedTree(Gtk.TreeView):
entry['lastmodified'] = msg.response_headers.get_one('Last-Modified') entry['lastmodified'] = msg.response_headers.get_one('Last-Modified')
try: try:
feed = feedparser.parse(msg.response_body.flatten().get_data()) feedparse = feedparser.parse(msg.response_body.flatten().get_data())
except: except:
print "error parsing feed:" print "error parsing feed:"
print msg.response_body.flatten().get_data() print msg.response_body.flatten().get_data()
@ -805,14 +800,14 @@ class FeedTree(Gtk.TreeView):
self.update_feed_done(feedurl) self.update_feed_done(feedurl)
return return
if feed.bozo != 0: if feedparse.bozo != 0:
# retrieved data was no valid feed # retrieved data was no valid feed
error_icon = self.render_icon(Gtk.STOCK_DIALOG_ERROR, Gtk.IconSize.MENU, None) error_icon = self.render_icon(Gtk.STOCK_DIALOG_ERROR, Gtk.IconSize.MENU, None)
self.model.set_value(it, 2, error_icon) self.model.set_value(it, 2, error_icon)
self.update_feed_done(feedurl) self.update_feed_done(feedurl)
return return
entry['title'] = feed.feed.get('title') entry['title'] = feedparse.feed.get('title')
self.model.set_value(it, 1, markup_escape_text(entry['title'])) self.model.set_value(it, 1, markup_escape_text(entry['title']))
# assumption: favicon never changes # assumption: favicon never changes
@ -820,14 +815,14 @@ class FeedTree(Gtk.TreeView):
self.updating.add(feedurl+"_icon") self.updating.add(feedurl+"_icon")
self.update_icon(it, feedurl) self.update_icon(it, feedurl)
for item in feed.entries: for item in feedparse.entries:
# use guid with fallback to link as identifier # use guid with fallback to link as identifier
itemid = item.get("id", item.get("link")) itemid = item.get("id", item.get("link"))
if not itemid: if not itemid:
# TODO: display error "cannot identify feeditems" # TODO: display error "cannot identify feeditems"
break break
if entry['items'].has_key(itemid): if self.feeddb.entry_exists(feedurl, itemid):
# already exists # already exists
continue continue
@ -836,7 +831,7 @@ class FeedTree(Gtk.TreeView):
'title': item.get('title'), 'title': item.get('title'),
'date': item.get('published_parsed'), 'date': item.get('published_parsed'),
'content': item.get('content'), 'content': item.get('content'),
'categories': [cat for _, cat in item.get('categories', [])] or None, 'categories': ','.join([cat for _, cat in item.get('categories', [])]) or "",
'unread': True 'unread': True
} }
@ -858,14 +853,14 @@ class FeedTree(Gtk.TreeView):
else: else:
new_item['content'] = "" new_item['content'] = ""
entry['items'][itemid] = new_item self.feeddb.add_entry(feedurl, itemid, new_item)
entry['unread'] += 1 entry['unread'] += 1
if entry['unread'] > 0: if entry['unread'] > 0:
self.model.set_value(it, 1, '<b>'+markup_escape_text(entry['title'])+" (%d)"%entry['unread']+'</b>') self.model.set_value(it, 1, '<b>'+markup_escape_text(entry['title'])+" (%d)"%entry['unread']+'</b>')
self.feeddb[feedurl] = entry self.feeddb.update_feed(feedurl, entry)
self.emit("update-feed", feedurl) self.emit("update-feed", feedurl)
@ -894,9 +889,7 @@ class FeedTree(Gtk.TreeView):
data = msg.response_body.flatten().get_data() data = msg.response_body.flatten().get_data()
if len(data): if len(data):
icon = pixbuf_new_from_file_in_memory(data, (16, 16)) icon = pixbuf_new_from_file_in_memory(data, (16, 16))
entry = self.feeddb[feedurl] self.feeddb.set_favicon(feedurl, data)
entry['favicon'] = data
self.feeddb[feedurl] = entry
self.model.set_value(it, 2, icon) self.model.set_value(it, 2, icon)
self.update_feed_done(feedurl) self.update_feed_done(feedurl)
else: else:
@ -920,9 +913,7 @@ class FeedTree(Gtk.TreeView):
icon = self.render_icon(Gtk.STOCK_FILE, Gtk.IconSize.MENU, None) icon = self.render_icon(Gtk.STOCK_FILE, Gtk.IconSize.MENU, None)
else: else:
icon = self.render_icon(Gtk.STOCK_FILE, Gtk.IconSize.MENU, None) icon = self.render_icon(Gtk.STOCK_FILE, Gtk.IconSize.MENU, None)
entry = self.feeddb[feedurl] self.feeddb.set_favicon(feedurl, data)
entry['favicon'] = data
self.feeddb[feedurl] = entry
self.model.set_value(it, 2, icon) self.model.set_value(it, 2, icon)
self.update_feed_done(feedurl+"_icon") self.update_feed_done(feedurl+"_icon")
self.session.queue_message(msg, complete_cb, it) self.session.queue_message(msg, complete_cb, it)
@ -932,23 +923,23 @@ class FeedReaderWindow(Gtk.Window):
def __init__(self): def __init__(self):
Gtk.Window.__init__(self) Gtk.Window.__init__(self)
# try the following paths for pyferea.db in this order # try the following paths for pyferea.sqlite in this order
xdg_data_home = os.environ.get('XDG_DATA_HOME') or os.path.join(os.path.expanduser('~'), '.local', 'share') xdg_data_home = os.environ.get('XDG_DATA_HOME') or os.path.join(os.path.expanduser('~'), '.local', 'share')
feeddb_paths = [ feeddb_paths = [
"./pyferea.db", "./pyferea.sqlite",
os.path.join(xdg_data_home, "pyferea", "pyferea.db"), os.path.join(xdg_data_home, "pyferea", "pyferea.sqlite"),
] ]
feeddb = None feeddb = None
for path in feeddb_paths: for path in feeddb_paths:
if os.path.exists(path): if os.path.exists(path):
feeddb = shelve.open(path) feeddb = sqlite_db.SQLStorage(path)
break break
if not feeddb: if not feeddb:
print "cannot find pyferea.db in any of the following locations:" print "cannot find pyferea.sqlite in any of the following locations:"
for path in feeddb_paths: for path in feeddb_paths:
print path print path
print "creating new db at %s"%feeddb_paths[0] print "creating new db at %s"%feeddb_paths[0]
feeddb = shelve.open(feeddb_paths[0]) feeddb = sqlite_db.SQLStorage(feeddb_paths[0])
# try the following paths for feeds.yaml in this order # try the following paths for feeds.yaml in this order
xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or os.path.join(os.path.expanduser('~'), '.config') xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or os.path.join(os.path.expanduser('~'), '.config')
@ -1030,7 +1021,7 @@ class FeedReaderWindow(Gtk.Window):
entries = EntryTree(config, feeddb) entries = EntryTree(config, feeddb)
def item_selected_cb(entry, feedurl, item): def item_selected_cb(entry, feedurl, item):
item = feeddb[feedurl]['items'][item] item = feeddb.get_entry(feedurl, item)
if config[feedurl]['loadlink']: if config[feedurl]['loadlink']:
content_pane.load_uri(item['link']) content_pane.load_uri(item['link'])
else: else:

107
sqlite_db.py
Unescape Escape View File

@ -0,0 +1,107 @@
import sqlite3
def dbcreate(conn):
conn.execute("""
CREATE TABLE IF NOT EXISTS feeds (
feed TEXT NOT NULL,
title TEXT,
favicon BLOB,
etag TEXT,
lastmodified TEXT,
unread INTEGER
)
""")
conn.execute("""
CREATE UNIQUE INDEX IF NOT EXISTS feedidx ON feeds (feed)
""")
conn.execute("""
CREATE TABLE IF NOT EXISTS entries (
feed TEXT NOT NULL,
entry TEXT NOT NULL,
title TEXT,
content TEXT,
link TEXT,
date INTEGER,
unread INTEGER,
categories TEXT
)""")
conn.execute("""
CREATE UNIQUE INDEX IF NOT EXISTS entridx ON entries (feed,entry)
""")
conn.commit()
def convert(filename):
conn = sqlite3.connect(filename)
conn.text_factory = str
dbcreate(conn)
import shelve
feeddb = shelve.open("pyferea.db")
for feed, fvalues in feeddb.items():
conn.execute("""REPLACE INTO feeds (feed, title, favicon, etag, lastmodified, unread) VALUES (?,?,?,?,?,?)""",
(feed, fvalues.get('title'), fvalues.get('favicon'), fvalues.get('etag'), fvalues.get('lastmodified'), fvalues.get('unread')))
for entry, evalues in fvalues['items'].items():
conn.execute("""REPLACE INTO entries (feed, entry, title, content, link, date, unread, categories) VALUES (?, ?, ?, ?, ?, ?, ?, ?)""",
(feed, entry, evalues.get('title'), evalues.get('content'), evalues.get('link'), evalues.get('date'), evalues.get('unread'), ', '.join(evalues.get('categories') or [])))
conn.commit()
conn.close()
feeddb.close()
class SQLStorage():
def __init__(self, filename=':memory:'):
self.conn = sqlite3.connect(filename)
self.conn.text_factory = str
dbcreate(self.conn)
def get_feed(self, feed):
return dict(zip(('title', 'favicon', 'etag', 'lastmodified', 'unread'),
self.conn.execute("""SELECT title, favicon, etag, lastmodified, unread FROM feeds WHERE feed=?""", (feed,)).fetchone()))
def get_entry(self, feed, entry):
return dict(zip(('title', 'content', 'link', 'date', 'unread', 'categories'),
self.conn.execute("""SELECT title, content, link, date, unread, categories FROM entries WHERE feed=? AND entry=?""", (feed, entry)).fetchone()))
def get_entries_all(self, feed):
return [dict(zip(('entry', 'title', 'date', 'unread'), content))
for content in self.conn.execute("""
SELECT entry, title, date, unread FROM entries WHERE feed=? ORDER BY date DESC
""", (feed,)).fetchall()]
def add_entry(self, feed, entry, values):
self.conn.execute("""REPLACE INTO entries (feed, entry, title, content, link, date, unread, categories) VALUES (?, ?, ?, ?, ?, ?, ?, ?)""",
(feed, entry, values['title'], values['content'], values['link'], values['date'], values['unread'], values['categories']))
self.conn.commit()
def update_feed(self, feed, values):
self.conn.execute("""UPDATE feeds SET title=?, favicon=?, etag=?, lastmodified=?, unread=? WHERE feed=?""",
(values['title'], values['favicon'], values['etag'], values['lastmodified'], values['unread'], feed))
self.conn.commit()
def set_favicon(self, feed, favicon):
self.conn.execute("""UPDATE feeds SET favicon=? WHERE feed=?""", (favicon, feed))
self.conn.commit()
def mark_read(self, feed, entry):
self.conn.execute("""UPDATE entries SET unread=0 WHERE feed=? AND entry=?""", (feed, entry))
self.conn.execute("""UPDATE feeds SET unread=unread-1 WHERE feed=?""", (feed,))
self.conn.commit()
def mark_read_feed(self, feed):
self.conn.execute("""UPDATE entries SET unread=0 WHERE unread=1""")
self.conn.execute("""UPDATE feeds set unread=0 WHERE feed=?""", (feed,))
self.conn.commit()
def feed_exists(self, feed):
return self.conn.execute("""SELECT feed FROM feeds WHERE feed=?""", (feed,)).fetchone() is not None
def entry_exists(self, feed, entry):
return self.conn.execute("""SELECT feed FROM entries WHERE feed=? AND entry=?""", (feed,entry)).fetchone() is not None
def close(self):
self.conn.close()
def sync(self):
pass
if __name__ == "__main__":
convert("pyferea.sqlite")
Write Preview
Loading…
Cancel
Save