use sqlite as database backend
This commit is contained in:
josch
parent
776c30770f
commit
b55097e8c3
2 changed files with 152 additions and 54 deletions
99
pyferea.py
99
pyferea.py
|
|
@ -29,7 +29,7 @@ from urlparse import urlparse, urlunparse, urljoin
|
|||
import feedparser
|
||||
from lxml import etree
|
||||
from cStringIO import StringIO
|
||||
import shelve
|
||||
import sqlite_db
|
||||
import time
|
||||
import datetime
|
||||
import os, re
|
||||
|
|
@ -97,7 +97,7 @@ def markup_escape_text(text):
|
|||
"""
|
||||
if not text:
|
||||
return ""
|
||||
return GLib.markup_escape_text(text.encode('utf-8'))
|
||||
return GLib.markup_escape_text(text)
|
||||
|
||||
class TabLabel(Gtk.HBox):
|
||||
"""A class for Tab labels"""
|
||||
|
|
@ -489,16 +489,13 @@ class EntryTree(Gtk.TreeView):
|
|||
_, it = selection.get_selected()
|
||||
if not it: return
|
||||
item = self.get_model().get_value(it, 0)
|
||||
entry = self.feeddb[self.feedurl]
|
||||
if entry['items'][item]['unread']:
|
||||
title = markup_escape_text(entry['items'][item]['title'])
|
||||
date = get_time_pretty(entry['items'][item]['date'])
|
||||
entry = self.feeddb.get_entry(self.feedurl, item)
|
||||
if entry['unread']:
|
||||
title = markup_escape_text(entry['title'])
|
||||
date = get_time_pretty(entry['date'])
|
||||
self.get_model().set_value(it, 1, title)
|
||||
self.get_model().set_value(it, 2, date)
|
||||
entry['items'][item]['unread'] = False
|
||||
entry['unread'] -= 1
|
||||
self.feeddb[self.feedurl] = entry
|
||||
#self.feeddb.sync() # dont sync on every unread item - makes stuff extremely slow over time
|
||||
self.feeddb.mark_read(self.feedurl, item)
|
||||
self.emit("item-selected", self.feedurl, item)
|
||||
self.connect("cursor-changed", on_cursor_changed_cb)
|
||||
|
||||
|
|
@ -517,11 +514,11 @@ class EntryTree(Gtk.TreeView):
|
|||
self.feedurl = None
|
||||
|
||||
for feedurl in config:
|
||||
if self.feeddb.get(feedurl):
|
||||
if self.feeddb.feed_exists(feedurl):
|
||||
self.update(feedurl)
|
||||
|
||||
def display(self, feedurl):
|
||||
if not feedurl or feedurl not in self.feeddb:
|
||||
if not feedurl or not self.feeddb.feed_exists(feedurl):
|
||||
self.set_model(self.empty_model)
|
||||
self.feedurl = None
|
||||
else:
|
||||
|
|
@ -532,14 +529,13 @@ class EntryTree(Gtk.TreeView):
|
|||
model = Gtk.ListStore(GObject.TYPE_STRING, GObject.TYPE_STRING, GObject.TYPE_STRING)
|
||||
# using model.set_sort_column_id is horribly slow, so append them
|
||||
# sorted instead
|
||||
items = sorted(self.feeddb[feedurl]['items'].iteritems(), key=lambda x: x[1]['date'], reverse=True)
|
||||
for guid, value in items:
|
||||
for value in self.feeddb.get_entries_all(feedurl):
|
||||
title = markup_escape_text(value.get('title', ""))
|
||||
date = get_time_pretty(value['date'])
|
||||
if value['unread']:
|
||||
title = "<b>"+title+"</b>"
|
||||
date = "<b>"+date+"</b>"
|
||||
model.append([guid, title, date])
|
||||
model.append([value['entry'], title, date])
|
||||
def compare_date(model, a, b, data):
|
||||
item1 = model.get_value(a, 0)
|
||||
item2 = model.get_value(b, 0)
|
||||
|
|
@ -609,15 +605,16 @@ class FeedTree(Gtk.TreeView):
|
|||
for category, feeds in categories.items():
|
||||
it = self.model.append(None, [None, category, folder_icon, None])
|
||||
for feedurl in feeds:
|
||||
if self.feeddb.get(feedurl):
|
||||
feed_icon = self.feeddb[feedurl].get('favicon')
|
||||
if self.feeddb.feed_exists(feedurl):
|
||||
feed = self.feeddb.get_feed(feedurl)
|
||||
feed_icon = feed.get('favicon')
|
||||
if feed_icon:
|
||||
feed_icon = pixbuf_new_from_file_in_memory(feed_icon, (16, 16))
|
||||
else:
|
||||
feed_icon = self.render_icon(Gtk.STOCK_FILE, Gtk.IconSize.MENU, None)
|
||||
|
||||
label = markup_escape_text(self.feeddb[feedurl].get('title', feedurl))
|
||||
unread = self.feeddb[feedurl].get('unread')
|
||||
label = markup_escape_text(feed.get('title', feedurl))
|
||||
unread = feed.get('unread')
|
||||
if unread > 0:
|
||||
label = "<b>"+label+" (%d)"%unread+"</b>"
|
||||
else:
|
||||
|
|
@ -656,13 +653,9 @@ class FeedTree(Gtk.TreeView):
|
|||
|
||||
def mark_read(self, it, sync=True):
|
||||
feedurl = self.model.get_value(it, 0)
|
||||
entry = self.feeddb.get(feedurl)
|
||||
if not entry: return
|
||||
entry['unread'] = 0
|
||||
for item in entry['items'].values():
|
||||
item['unread'] = False
|
||||
self.model.set_value(it, 1, markup_escape_text(entry['title']))
|
||||
self.feeddb[feedurl] = entry
|
||||
feed = self.feeddb.get_feed(feedurl)
|
||||
self.feeddb.mark_read_feed(feedurl)
|
||||
self.model.set_value(it, 1, markup_escape_text(feed['title']))
|
||||
self.emit("update-feed", feedurl)
|
||||
if sync:
|
||||
self.feeddb.sync()
|
||||
|
|
@ -709,8 +702,9 @@ class FeedTree(Gtk.TreeView):
|
|||
itc = self.model.iter_children(it)
|
||||
while (itc):
|
||||
feedurl = self.model.get_value(itc, 0)
|
||||
title = markup_escape_text(self.feeddb[feedurl]['title'])
|
||||
unread = self.feeddb[feedurl]['unread']
|
||||
feed = self.feeddb.get_feed(feedurl)
|
||||
title = markup_escape_text(feed['title'])
|
||||
unread = feed['unread']
|
||||
if unread > 0:
|
||||
title = "<b>"+title+" (%d)"%unread+"</b>"
|
||||
self.model.set_value(itc, 1, title)
|
||||
|
|
@ -758,10 +752,11 @@ class FeedTree(Gtk.TreeView):
|
|||
def update_feed(self, it):
|
||||
feedurl = self.model.get_value(it, 0)
|
||||
msg = Soup.Message.new("GET", feedurl)
|
||||
if self.feeddb.get(feedurl) and self.feeddb[feedurl].get('etag'):
|
||||
msg.request_headers.append('If-None-Match', self.feeddb[feedurl]['etag'])
|
||||
if self.feeddb.get(feedurl) and self.feeddb[feedurl].get('lastmodified'):
|
||||
msg.request_headers.append('If-Modified-Since', self.feeddb[feedurl]['lastmodified'])
|
||||
feed = self.feeddb.get_feed(feedurl)
|
||||
if feed.get('etag'):
|
||||
msg.request_headers.append('If-None-Match', feed['etag'])
|
||||
if feed.get('lastmodified'):
|
||||
msg.request_headers.append('If-Modified-Since', feed['lastmodified'])
|
||||
|
||||
def complete_cb(session, msg, it):
|
||||
if msg.status_code not in [200, 304]:
|
||||
|
|
@ -771,7 +766,7 @@ class FeedTree(Gtk.TreeView):
|
|||
return
|
||||
|
||||
# get existing feedentry or create new one
|
||||
entry = self.feeddb.get(feedurl, dict())
|
||||
entry = self.feeddb.get_feed(feedurl)
|
||||
|
||||
if entry.get('favicon'):
|
||||
icon = pixbuf_new_from_file_in_memory(entry['favicon'], (16, 16))
|
||||
|
|
@ -796,7 +791,7 @@ class FeedTree(Gtk.TreeView):
|
|||
entry['lastmodified'] = msg.response_headers.get_one('Last-Modified')
|
||||
|
||||
try:
|
||||
feed = feedparser.parse(msg.response_body.flatten().get_data())
|
||||
feedparse = feedparser.parse(msg.response_body.flatten().get_data())
|
||||
except:
|
||||
print "error parsing feed:"
|
||||
print msg.response_body.flatten().get_data()
|
||||
|
|
@ -805,14 +800,14 @@ class FeedTree(Gtk.TreeView):
|
|||
self.update_feed_done(feedurl)
|
||||
return
|
||||
|
||||
if feed.bozo != 0:
|
||||
if feedparse.bozo != 0:
|
||||
# retrieved data was no valid feed
|
||||
error_icon = self.render_icon(Gtk.STOCK_DIALOG_ERROR, Gtk.IconSize.MENU, None)
|
||||
self.model.set_value(it, 2, error_icon)
|
||||
self.update_feed_done(feedurl)
|
||||
return
|
||||
|
||||
entry['title'] = feed.feed.get('title')
|
||||
entry['title'] = feedparse.feed.get('title')
|
||||
self.model.set_value(it, 1, markup_escape_text(entry['title']))
|
||||
|
||||
# assumption: favicon never changes
|
||||
|
|
@ -820,14 +815,14 @@ class FeedTree(Gtk.TreeView):
|
|||
self.updating.add(feedurl+"_icon")
|
||||
self.update_icon(it, feedurl)
|
||||
|
||||
for item in feed.entries:
|
||||
for item in feedparse.entries:
|
||||
# use guid with fallback to link as identifier
|
||||
itemid = item.get("id", item.get("link"))
|
||||
if not itemid:
|
||||
# TODO: display error "cannot identify feeditems"
|
||||
break
|
||||
|
||||
if entry['items'].has_key(itemid):
|
||||
if self.feeddb.entry_exists(feedurl, itemid):
|
||||
# already exists
|
||||
continue
|
||||
|
||||
|
|
@ -836,7 +831,7 @@ class FeedTree(Gtk.TreeView):
|
|||
'title': item.get('title'),
|
||||
'date': item.get('published_parsed'),
|
||||
'content': item.get('content'),
|
||||
'categories': [cat for _, cat in item.get('categories', [])] or None,
|
||||
'categories': ','.join([cat for _, cat in item.get('categories', [])]) or "",
|
||||
'unread': True
|
||||
}
|
||||
|
||||
|
|
@ -858,14 +853,14 @@ class FeedTree(Gtk.TreeView):
|
|||
else:
|
||||
new_item['content'] = ""
|
||||
|
||||
entry['items'][itemid] = new_item
|
||||
self.feeddb.add_entry(feedurl, itemid, new_item)
|
||||
|
||||
entry['unread'] += 1
|
||||
|
||||
if entry['unread'] > 0:
|
||||
self.model.set_value(it, 1, '<b>'+markup_escape_text(entry['title'])+" (%d)"%entry['unread']+'</b>')
|
||||
|
||||
self.feeddb[feedurl] = entry
|
||||
self.feeddb.update_feed(feedurl, entry)
|
||||
|
||||
self.emit("update-feed", feedurl)
|
||||
|
||||
|
|
@ -894,9 +889,7 @@ class FeedTree(Gtk.TreeView):
|
|||
data = msg.response_body.flatten().get_data()
|
||||
if len(data):
|
||||
icon = pixbuf_new_from_file_in_memory(data, (16, 16))
|
||||
entry = self.feeddb[feedurl]
|
||||
entry['favicon'] = data
|
||||
self.feeddb[feedurl] = entry
|
||||
self.feeddb.set_favicon(feedurl, data)
|
||||
self.model.set_value(it, 2, icon)
|
||||
self.update_feed_done(feedurl)
|
||||
else:
|
||||
|
|
@ -920,9 +913,7 @@ class FeedTree(Gtk.TreeView):
|
|||
icon = self.render_icon(Gtk.STOCK_FILE, Gtk.IconSize.MENU, None)
|
||||
else:
|
||||
icon = self.render_icon(Gtk.STOCK_FILE, Gtk.IconSize.MENU, None)
|
||||
entry = self.feeddb[feedurl]
|
||||
entry['favicon'] = data
|
||||
self.feeddb[feedurl] = entry
|
||||
self.feeddb.set_favicon(feedurl, data)
|
||||
self.model.set_value(it, 2, icon)
|
||||
self.update_feed_done(feedurl+"_icon")
|
||||
self.session.queue_message(msg, complete_cb, it)
|
||||
|
|
@ -932,23 +923,23 @@ class FeedReaderWindow(Gtk.Window):
|
|||
def __init__(self):
|
||||
Gtk.Window.__init__(self)
|
||||
|
||||
# try the following paths for pyferea.db in this order
|
||||
# try the following paths for pyferea.sqlite in this order
|
||||
xdg_data_home = os.environ.get('XDG_DATA_HOME') or os.path.join(os.path.expanduser('~'), '.local', 'share')
|
||||
feeddb_paths = [
|
||||
"./pyferea.db",
|
||||
os.path.join(xdg_data_home, "pyferea", "pyferea.db"),
|
||||
"./pyferea.sqlite",
|
||||
os.path.join(xdg_data_home, "pyferea", "pyferea.sqlite"),
|
||||
]
|
||||
feeddb = None
|
||||
for path in feeddb_paths:
|
||||
if os.path.exists(path):
|
||||
feeddb = shelve.open(path)
|
||||
feeddb = sqlite_db.SQLStorage(path)
|
||||
break
|
||||
if not feeddb:
|
||||
print "cannot find pyferea.db in any of the following locations:"
|
||||
print "cannot find pyferea.sqlite in any of the following locations:"
|
||||
for path in feeddb_paths:
|
||||
print path
|
||||
print "creating new db at %s"%feeddb_paths[0]
|
||||
feeddb = shelve.open(feeddb_paths[0])
|
||||
feeddb = sqlite_db.SQLStorage(feeddb_paths[0])
|
||||
|
||||
# try the following paths for feeds.yaml in this order
|
||||
xdg_config_home = os.environ.get('XDG_CONFIG_HOME') or os.path.join(os.path.expanduser('~'), '.config')
|
||||
|
|
@ -1030,7 +1021,7 @@ class FeedReaderWindow(Gtk.Window):
|
|||
entries = EntryTree(config, feeddb)
|
||||
|
||||
def item_selected_cb(entry, feedurl, item):
|
||||
item = feeddb[feedurl]['items'][item]
|
||||
item = feeddb.get_entry(feedurl, item)
|
||||
if config[feedurl]['loadlink']:
|
||||
content_pane.load_uri(item['link'])
|
||||
else:
|
||||
|
|
|
|||
107
sqlite_db.py
Normal file
107
sqlite_db.py
Normal file
|
|
@ -0,0 +1,107 @@
|
|||
import sqlite3
|
||||
|
||||
|
||||
def dbcreate(conn):
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS feeds (
|
||||
feed TEXT NOT NULL,
|
||||
title TEXT,
|
||||
favicon BLOB,
|
||||
etag TEXT,
|
||||
lastmodified TEXT,
|
||||
unread INTEGER
|
||||
)
|
||||
""")
|
||||
conn.execute("""
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS feedidx ON feeds (feed)
|
||||
""")
|
||||
conn.execute("""
|
||||
CREATE TABLE IF NOT EXISTS entries (
|
||||
feed TEXT NOT NULL,
|
||||
entry TEXT NOT NULL,
|
||||
title TEXT,
|
||||
content TEXT,
|
||||
link TEXT,
|
||||
date INTEGER,
|
||||
unread INTEGER,
|
||||
categories TEXT
|
||||
)""")
|
||||
conn.execute("""
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS entridx ON entries (feed,entry)
|
||||
""")
|
||||
conn.commit()
|
||||
|
||||
def convert(filename):
|
||||
conn = sqlite3.connect(filename)
|
||||
conn.text_factory = str
|
||||
dbcreate(conn)
|
||||
import shelve
|
||||
feeddb = shelve.open("pyferea.db")
|
||||
for feed, fvalues in feeddb.items():
|
||||
conn.execute("""REPLACE INTO feeds (feed, title, favicon, etag, lastmodified, unread) VALUES (?,?,?,?,?,?)""",
|
||||
(feed, fvalues.get('title'), fvalues.get('favicon'), fvalues.get('etag'), fvalues.get('lastmodified'), fvalues.get('unread')))
|
||||
for entry, evalues in fvalues['items'].items():
|
||||
conn.execute("""REPLACE INTO entries (feed, entry, title, content, link, date, unread, categories) VALUES (?, ?, ?, ?, ?, ?, ?, ?)""",
|
||||
(feed, entry, evalues.get('title'), evalues.get('content'), evalues.get('link'), evalues.get('date'), evalues.get('unread'), ', '.join(evalues.get('categories') or [])))
|
||||
conn.commit()
|
||||
conn.close()
|
||||
feeddb.close()
|
||||
|
||||
class SQLStorage():
|
||||
def __init__(self, filename=':memory:'):
|
||||
self.conn = sqlite3.connect(filename)
|
||||
self.conn.text_factory = str
|
||||
dbcreate(self.conn)
|
||||
|
||||
def get_feed(self, feed):
|
||||
return dict(zip(('title', 'favicon', 'etag', 'lastmodified', 'unread'),
|
||||
self.conn.execute("""SELECT title, favicon, etag, lastmodified, unread FROM feeds WHERE feed=?""", (feed,)).fetchone()))
|
||||
|
||||
def get_entry(self, feed, entry):
|
||||
return dict(zip(('title', 'content', 'link', 'date', 'unread', 'categories'),
|
||||
self.conn.execute("""SELECT title, content, link, date, unread, categories FROM entries WHERE feed=? AND entry=?""", (feed, entry)).fetchone()))
|
||||
|
||||
def get_entries_all(self, feed):
|
||||
return [dict(zip(('entry', 'title', 'date', 'unread'), content))
|
||||
for content in self.conn.execute("""
|
||||
SELECT entry, title, date, unread FROM entries WHERE feed=? ORDER BY date DESC
|
||||
""", (feed,)).fetchall()]
|
||||
|
||||
def add_entry(self, feed, entry, values):
|
||||
self.conn.execute("""REPLACE INTO entries (feed, entry, title, content, link, date, unread, categories) VALUES (?, ?, ?, ?, ?, ?, ?, ?)""",
|
||||
(feed, entry, values['title'], values['content'], values['link'], values['date'], values['unread'], values['categories']))
|
||||
self.conn.commit()
|
||||
|
||||
def update_feed(self, feed, values):
|
||||
self.conn.execute("""UPDATE feeds SET title=?, favicon=?, etag=?, lastmodified=?, unread=? WHERE feed=?""",
|
||||
(values['title'], values['favicon'], values['etag'], values['lastmodified'], values['unread'], feed))
|
||||
self.conn.commit()
|
||||
|
||||
def set_favicon(self, feed, favicon):
|
||||
self.conn.execute("""UPDATE feeds SET favicon=? WHERE feed=?""", (favicon, feed))
|
||||
self.conn.commit()
|
||||
|
||||
def mark_read(self, feed, entry):
|
||||
self.conn.execute("""UPDATE entries SET unread=0 WHERE feed=? AND entry=?""", (feed, entry))
|
||||
self.conn.execute("""UPDATE feeds SET unread=unread-1 WHERE feed=?""", (feed,))
|
||||
self.conn.commit()
|
||||
|
||||
def mark_read_feed(self, feed):
|
||||
self.conn.execute("""UPDATE entries SET unread=0 WHERE unread=1""")
|
||||
self.conn.execute("""UPDATE feeds set unread=0 WHERE feed=?""", (feed,))
|
||||
self.conn.commit()
|
||||
|
||||
def feed_exists(self, feed):
|
||||
return self.conn.execute("""SELECT feed FROM feeds WHERE feed=?""", (feed,)).fetchone() is not None
|
||||
|
||||
def entry_exists(self, feed, entry):
|
||||
return self.conn.execute("""SELECT feed FROM entries WHERE feed=? AND entry=?""", (feed,entry)).fetchone() is not None
|
||||
|
||||
def close(self):
|
||||
self.conn.close()
|
||||
|
||||
def sync(self):
|
||||
pass
|
||||
|
||||
if __name__ == "__main__":
|
||||
convert("pyferea.sqlite")
|
||||
Loading…
Reference in a new issue