From be5c5a2018a39f2a356885e4817238f6d0d61a8a Mon Sep 17 00:00:00 2001 From: Damien Elmes Date: Sun, 28 Aug 2011 13:44:29 +0900 Subject: [PATCH] move tags into deck; code into separate file - moved tags into json like previous changes, and dropped the unnecessary id - added tags.py for a tag manager - moved the tag utilities from utils into tags.py --- anki/deck.py | 116 ++++--------------------- anki/facts.py | 11 ++- anki/groups.py | 2 +- anki/importing/__init__.py | 3 +- anki/importing/anki10.py | 2 +- anki/media.py | 2 +- anki/models.py | 2 +- anki/sched.py | 2 +- anki/storage.py | 39 ++++----- anki/tags.py | 172 +++++++++++++++++++++++++++++++++++++ anki/utils.py | 44 ---------- tests/test_deck.py | 24 +++--- tests/test_find.py | 4 +- 13 files changed, 231 insertions(+), 192 deletions(-) create mode 100644 anki/tags.py diff --git a/anki/deck.py b/anki/deck.py index a6b651b72..24f6526ab 100644 --- a/anki/deck.py +++ b/anki/deck.py @@ -4,14 +4,14 @@ import time, os, random, re, stat, simplejson, datetime, copy, shutil from anki.lang import _, ngettext -from anki.utils import parseTags, ids2str, hexifyID, \ - checksum, fieldChecksum, addTags, delTags, stripHTML, intTime, \ - splitFields +from anki.utils import ids2str, hexifyID, checksum, fieldChecksum, stripHTML, \ + intTime, splitFields from anki.hooks import runHook, runFilter from anki.sched import Scheduler -from anki.models import ModelRegistry -from anki.media import MediaRegistry -from anki.groups import GroupRegistry +from anki.models import ModelManager +from anki.media import MediaManager +from anki.groups import GroupManager +from anki.tags import TagManager from anki.consts import * from anki.errors import AnkiError @@ -52,9 +52,10 @@ class _Deck(object): self.path = db._path self._lastSave = time.time() self.clearUndo() - self.media = MediaRegistry(self) - self.models = ModelRegistry(self) - self.groups = GroupRegistry(self) + self.media = MediaManager(self) + self.models = ModelManager(self) + self.groups = GroupManager(self) + self.tags = TagManager(self) self.load() if not self.crt: d = datetime.datetime.today() @@ -90,13 +91,15 @@ class _Deck(object): self.conf, models, groups, - gconf) = self.db.first(""" + gconf, + tags) = self.db.first(""" select crt, mod, scm, dty, syncName, lastSync, -qconf, conf, models, groups, gconf from deck""") +qconf, conf, models, groups, gconf, tags from deck""") self.qconf = simplejson.loads(self.qconf) self.conf = simplejson.loads(self.conf) self.models.load(models) self.groups.load(groups, gconf) + self.tags.load(tags) def flush(self, mod=None): "Flush state to DB, updating mod time." @@ -111,6 +114,7 @@ qconf=?, conf=?""", simplejson.dumps(self.conf)) self.models.flush() self.groups.flush() + self.tags.flush() def save(self, name=None, mod=None): "Flush, commit DB, and take out another write lock." @@ -447,93 +451,6 @@ from cards c, facts f where c.fid == f.id %s""" % where) - # Tags - ########################################################################## - - def tagList(self): - return self.db.list("select name from tags order by name") - - def updateFactTags(self, fids=None): - "Add any missing tags to the tags list." - if fids: - lim = " where id in " + ids2str(fids) - else: - lim = "" - self.registerTags(set(parseTags( - " ".join(self.db.list("select distinct tags from facts"+lim))))) - - def registerTags(self, tags): - r = [] - for t in tags: - r.append({'t': t}) - self.db.executemany(""" -insert or ignore into tags (mod, name) values (%d, :t)""" % intTime(), - r) - - def addTags(self, ids, tags, add=True): - "Add tags in bulk. TAGS is space-separated." - newTags = parseTags(tags) - if not newTags: - return - # cache tag names - self.registerTags(newTags) - # find facts missing the tags - if add: - l = "tags not " - fn = addTags - else: - l = "tags " - fn = delTags - lim = " or ".join( - [l+"like :_%d" % c for c, t in enumerate(newTags)]) - res = self.db.all( - "select id, tags from facts where id in %s and %s" % ( - ids2str(ids), lim), - **dict([("_%d" % x, '%% %s %%' % y) for x, y in enumerate(newTags)])) - # update tags - fids = [] - def fix(row): - fids.append(row[0]) - return {'id': row[0], 't': fn(tags, row[1]), 'n':intTime()} - self.db.executemany(""" -update facts set tags = :t, mod = :n where id = :id""", [fix(row) for row in res]) - # update q/a cache - self.registerTags(parseTags(tags)) - - def delTags(self, ids, tags): - self.addTags(ids, tags, False) - - # Tag-based selective study - ########################################################################## - - def selTagFids(self, yes, no): - l = [] - # find facts that match yes - lim = "" - args = [] - query = "select id from facts" - if not yes and not no: - pass - else: - if yes: - lim += " or ".join(["tags like ?" for t in yes]) - args += ['%% %s %%' % t for t in yes] - if no: - lim2 = " and ".join(["tags not like ?" for t in no]) - if lim: - lim = "(%s) and %s" % (lim, lim2) - else: - lim = lim2 - args += ['%% %s %%' % t for t in no] - query += " where " + lim - return self.db.list(query, *args) - - def setGroupForTags(self, yes, no, gid): - fids = self.selTagFids(yes, no) - self.db.execute( - "update cards set gid = ? where fid in "+ids2str(fids), - gid) - # Finding cards ########################################################################## @@ -685,8 +602,7 @@ update facts set tags = :t, mod = :n where id = :id""", [fix(row) for row in res select id from facts where id not in (select distinct fid from cards)""") self._delFacts(ids) # tags - self.db.execute("delete from tags") - self.updateFactTags() + self.tags.registerFacts() # field cache for m in self.models.all(): self.updateFieldCache(self.models.fids(m)) diff --git a/anki/facts.py b/anki/facts.py index 361f84eb3..ad746945b 100644 --- a/anki/facts.py +++ b/anki/facts.py @@ -5,8 +5,7 @@ import time from anki.errors import AnkiError from anki.utils import fieldChecksum, intTime, \ - joinFields, splitFields, ids2str, parseTags, canonifyTags, hasTag, \ - stripHTML, timestampID + joinFields, splitFields, ids2str, stripHTML, timestampID class Fact(object): @@ -35,7 +34,7 @@ class Fact(object): self.data) = self.deck.db.first(""" select mid, gid, mod, tags, flds, data from facts where id = ?""", self.id) self.fields = splitFields(self.fields) - self.tags = parseTags(self.tags) + self.tags = self.deck.tags.split(self.tags) self._model = self.deck.models.get(self.mid) self._fmap = self.deck.models.fieldMap(self._model) @@ -50,7 +49,7 @@ insert or replace into facts values (?, ?, ?, ?, ?, ?, ?, ?)""", sfld, self.data) self.id = res.lastrowid self.updateFieldChecksums() - self.deck.registerTags(self.tags) + self.deck.tags.register(self.tags) def joinedFields(self): return joinFields(self.fields) @@ -109,10 +108,10 @@ insert or replace into facts values (?, ?, ?, ?, ?, ?, ?, ?)""", ################################################## def hasTag(self, tag): - return hasTag(tag, self.tags) + return self.deck.tags.inStr(tag, self.tags) def stringTags(self): - return canonifyTags(self.tags) + return self.deck.tags.canonify(self.tags) def delTag(self, tag): rem = [] diff --git a/anki/groups.py b/anki/groups.py index 02f88b0f8..a0f117178 100644 --- a/anki/groups.py +++ b/anki/groups.py @@ -40,7 +40,7 @@ defaultData = { 'inactiveTags': None, } -class GroupRegistry(object): +class GroupManager(object): # Registry save/load ############################################################# diff --git a/anki/importing/__init__.py b/anki/importing/__init__.py index 5156a194d..fc601ca6e 100644 --- a/anki/importing/__init__.py +++ b/anki/importing/__init__.py @@ -15,8 +15,7 @@ import time #from anki.cards import cardsTable #from anki.facts import factsTable, fieldsTable from anki.lang import _ -from anki.utils import canonifyTags, fieldChecksum -from anki.utils import canonifyTags, ids2str +from anki.utils import fieldChecksum, ids2str from anki.errors import * #from anki.deck import NEW_CARDS_RANDOM diff --git a/anki/importing/anki10.py b/anki/importing/anki10.py index ba89111b8..2f179cf6a 100644 --- a/anki/importing/anki10.py +++ b/anki/importing/anki10.py @@ -50,7 +50,7 @@ class Anki10Importer(Importer): copyLocalMedia(server.deck, client.deck) # add tags fids = [f[0] for f in res['added-facts']['facts']] - self.deck.addTags(fids, self.tagsToAdd) + self.deck.tags.add(fids, self.tagsToAdd) # mark import material as newly added self.deck.db.execute( "update cards set modified = :t where id in %s" % diff --git a/anki/media.py b/anki/media.py index d612bbec7..3f5dc8182 100644 --- a/anki/media.py +++ b/anki/media.py @@ -7,7 +7,7 @@ import os, shutil, re, urllib, urllib2, time, unicodedata, \ from anki.utils import checksum, intTime, namedtmp, isWin from anki.lang import _ -class MediaRegistry(object): +class MediaManager(object): # can be altered at the class level for dropbox, etc mediaPrefix = "" diff --git a/anki/models.py b/anki/models.py index aeb1bfa64..dddca74ef 100644 --- a/anki/models.py +++ b/anki/models.py @@ -56,7 +56,7 @@ defaultTemplate = { 'gid': None, } -class ModelRegistry(object): +class ModelManager(object): # Saving/loading registry ############################################################# diff --git a/anki/sched.py b/anki/sched.py index 389947e84..f3758a8e5 100644 --- a/anki/sched.py +++ b/anki/sched.py @@ -6,7 +6,7 @@ import time, datetime, simplejson, random, itertools from operator import itemgetter from heapq import * #from anki.cards import Card -from anki.utils import parseTags, ids2str, intTime, fmtTimeSpan +from anki.utils import ids2str, intTime, fmtTimeSpan from anki.lang import _, ngettext from anki.consts import * from anki.hooks import runHook diff --git a/anki/storage.py b/anki/storage.py index 600d42931..da15eee58 100644 --- a/anki/storage.py +++ b/anki/storage.py @@ -69,7 +69,8 @@ create table if not exists deck ( conf text not null, models text not null, groups text not null, - gconf text not null + gconf text not null, + tags text not null ); create table if not exists cards ( @@ -125,28 +126,20 @@ create table if not exists revlog ( type integer not null ); -create table if not exists tags ( - id integer primary key, - mod integer not null, - name text not null collate nocase unique -); - insert or ignore into deck -values(1,0,0,0,%(v)s,0,'',0,'','','','',''); +values(1,0,0,0,%(v)s,0,'',0,'','','{}','','','{}'); """ % ({'v':CURRENT_VERSION})) import anki.deck import anki.groups if setDeckConf: db.execute(""" -update deck set qconf = ?, conf = ?, models = ?, groups = ?, gconf = ?""", +update deck set qconf = ?, conf = ?, groups = ?, gconf = ?""", simplejson.dumps(anki.deck.defaultQconf), simplejson.dumps(anki.deck.defaultConf), - "{}", simplejson.dumps({'1': {'name': _("Default"), 'conf': 1, 'mod': intTime()}}), simplejson.dumps({'1': anki.groups.defaultConf})) - def _updateIndices(db): "Add indices to the DB." db.executescript(""" @@ -192,14 +185,6 @@ def _upgradeSchema(db): return ver runHook("1.x upgrade", db) - # tags - ########### - _moveTable(db, "tags") - db.execute("insert or ignore into tags select id, ?, tag from tags2", - intTime()) - db.execute("drop table tags2") - db.execute("drop table cardTags") - # facts ########### # tags should have a leading and trailing space if not empty, and not @@ -328,10 +313,22 @@ yesCount from reviewHistory"""): "insert or ignore into revlog values (?,?,?,?,?,?,?,?)", r) db.execute("drop table reviewHistory") + # deck + ########### + _migrateDeckTbl(db) + + # tags + ########### + tags = {} + for t in db.list("select tag from tags"): + tags[t] = intTime() + db.execute("update deck set tags = ?", simplejson.dumps(tags)) + db.execute("drop table tags") + db.execute("drop table cardTags") + # the rest ########### db.execute("drop table media") - _migrateDeckTbl(db) _migrateModels(db) _updateIndices(db) return ver @@ -342,7 +339,7 @@ def _migrateDeckTbl(db): db.execute(""" insert or replace into deck select id, cast(created as int), :t, :t, 99, 0, ifnull(syncName, ""), cast(lastSync as int), -"", "", "", "", "" from decks""", t=intTime()) +"", "", "", "", "", "" from decks""", t=intTime()) # update selective study qconf = anki.deck.defaultQconf.copy() # delete old selective study settings, which we can't auto-upgrade easily diff --git a/anki/tags.py b/anki/tags.py new file mode 100644 index 000000000..d506beebf --- /dev/null +++ b/anki/tags.py @@ -0,0 +1,172 @@ +# -*- coding: utf-8 -*- +# Copyright: Damien Elmes +# License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html + +import simplejson +from anki.utils import intTime, ids2str + +""" +Anki maintains a cache of used tags so it can quickly present a list of tags +for autocomplete and in the browser. For efficiency, deletions are not +tracked, so unused tags can only be removed from the list with a DB check. + +This module manages the tag cache and tags for facts. +""" + +class TagManager(object): + + # Registry save/load + ############################################################# + + def __init__(self, deck): + self.deck = deck + + def load(self, json): + self.tags = simplejson.loads(json) + self.changed = False + + def flush(self): + if self.changed: + self.deck.db.execute("update deck set tags=?", + simplejson.dumps(self.tags)) + + # Registering and fetching tags + ############################################################# + + def register(self, tags): + "Given a list of tags, add any missing ones to tag registry." + # case is stored as received, so user can create different case + # versions of the same tag if they ignore the qt autocomplete. + for t in tags: + if t not in self.tags: + self.tags[t] = intTime() + self.changed = True + + def all(self): + return self.tags.keys() + + def registerFacts(self, fids=None): + "Add any missing tags from facts to the tags list." + # when called without an argument, the old list is cleared first. + if fids: + lim = " where id in " + ids2str(fids) + else: + lim = "" + self.tags = {} + self.changed = True + self.register(set(self.split( + " ".join(self.deck.db.list("select distinct tags from facts"+lim))))) + + # Bulk addition/removal from facts + ############################################################# + + def bulkAdd(self, ids, tags, add=True): + "Add tags in bulk. TAGS is space-separated." + newTags = self.split(tags) + if not newTags: + return + # cache tag names + self.register(newTags) + # find facts missing the tags + if add: + l = "tags not " + fn = self.addToStr + else: + l = "tags " + fn = self.remFromStr + lim = " or ".join( + [l+"like :_%d" % c for c, t in enumerate(newTags)]) + res = self.deck.db.all( + "select id, tags from facts where id in %s and %s" % ( + ids2str(ids), lim), + **dict([("_%d" % x, '%% %s %%' % y) + for x, y in enumerate(newTags)])) + # update tags + fids = [] + def fix(row): + fids.append(row[0]) + return {'id': row[0], 't': fn(tags, row[1]), 'n':intTime()} + self.deck.db.executemany( + "update facts set tags = :t, mod = :n where id = :id", + [fix(row) for row in res]) + + def bulkRem(self, ids, tags): + self.bulkAdd(ids, tags, False) + + # String-based utilities + ########################################################################## + + def split(self, tags): + "Parse a string and return a list of tags." + return [t for t in tags.split(" ") if t] + + def join(self, tags): + "Join tags into a single string, with leading and trailing spaces." + if not tags: + return u"" + return u" %s " % u" ".join(tags) + + def addToStr(self, addtags, tags): + "Add tags if they don't exist." + currentTags = self.split(tags) + for tag in self.split(addtags): + if not self.inList(tag, currentTags): + currentTags.append(tag) + return self.canonify(currentTags) + + def remFromStr(self, deltags, tags): + "Delete tags if they don't exists." + currentTags = self.split(tags) + for tag in self.split(deltags): + # find tags, ignoring case + remove = [] + for tx in currentTags: + if tag.lower() == tx.lower(): + remove.append(tx) + # remove them + for r in remove: + currentTags.remove(r) + return self.canonify(currentTags) + + # List-based utilities + ########################################################################## + + def canonify(self, tags): + "Strip leading/trailing/superfluous spaces and duplicates." + tags = [t.lstrip(":") for t in set(tags)] + return self.join(sorted(tags)) + + def inList(self, tag, tags): + "True if TAG is in TAGS. Ignore case." + return tag.lower() in [t.lower() for t in tags] + + # Tag-based selective study + ########################################################################## + + def selTagFids(self, yes, no): + l = [] + # find facts that match yes + lim = "" + args = [] + query = "select id from facts" + if not yes and not no: + pass + else: + if yes: + lim += " or ".join(["tags like ?" for t in yes]) + args += ['%% %s %%' % t for t in yes] + if no: + lim2 = " and ".join(["tags not like ?" for t in no]) + if lim: + lim = "(%s) and %s" % (lim, lim2) + else: + lim = lim2 + args += ['%% %s %%' % t for t in no] + query += " where " + lim + return self.deck.db.list(query, *args) + + def setGroupForTags(self, yes, no, gid): + fids = self.selTagFids(yes, no) + self.deck.db.execute( + "update cards set gid = ? where fid in "+ids2str(fids), + gid) diff --git a/anki/utils.py b/anki/utils.py index 18405de0b..c74f1f8c3 100644 --- a/anki/utils.py +++ b/anki/utils.py @@ -189,50 +189,6 @@ def timestampID(db, table): t += 1 return t -# Tags -############################################################################## - -def parseTags(tags): - "Parse a string and return a list of tags." - return [t for t in tags.split(" ") if t] - -def joinTags(tags): - "Join tags into a single string, with leading and trailing spaces." - if not tags: - return u"" - return u" %s " % u" ".join(tags) - -def canonifyTags(tags): - "Strip leading/trailing/superfluous spaces and duplicates." - tags = [t.lstrip(":") for t in set(tags)] - return joinTags(sorted(tags)) - -def hasTag(tag, tags): - "True if TAG is in TAGS. Ignore case." - return tag.lower() in [t.lower() for t in tags] - -def addTags(addtags, tags): - "Add tags if they don't exist." - currentTags = parseTags(tags) - for tag in parseTags(addtags): - if not hasTag(tag, currentTags): - currentTags.append(tag) - return canonifyTags(currentTags) - -def delTags(deltags, tags): - "Delete tags if they don't exists." - currentTags = parseTags(tags) - for tag in parseTags(deltags): - # find tags, ignoring case - remove = [] - for tx in currentTags: - if tag.lower() == tx.lower(): - remove.append(tx) - # remove them - for r in remove: - currentTags.remove(r) - return canonifyTags(currentTags) - # Fields ############################################################################## diff --git a/tests/test_deck.py b/tests/test_deck.py index 4db191f90..ff306656f 100644 --- a/tests/test_deck.py +++ b/tests/test_deck.py @@ -156,17 +156,17 @@ def test_selective(): f = deck.newFact() f['Front'] = u"3"; f.tags = ["one", "two", "three", "four"] deck.addFact(f) - assert len(deck.selTagFids(["one"], [])) == 2 - assert len(deck.selTagFids(["three"], [])) == 3 - assert len(deck.selTagFids([], ["three"])) == 0 - assert len(deck.selTagFids(["one"], ["three"])) == 0 - assert len(deck.selTagFids(["one"], ["two"])) == 1 - assert len(deck.selTagFids(["two", "three"], [])) == 3 - assert len(deck.selTagFids(["two", "three"], ["one"])) == 1 - assert len(deck.selTagFids(["one", "three"], ["two", "four"])) == 1 - deck.setGroupForTags(["three"], [], 3) + assert len(deck.tags.selTagFids(["one"], [])) == 2 + assert len(deck.tags.selTagFids(["three"], [])) == 3 + assert len(deck.tags.selTagFids([], ["three"])) == 0 + assert len(deck.tags.selTagFids(["one"], ["three"])) == 0 + assert len(deck.tags.selTagFids(["one"], ["two"])) == 1 + assert len(deck.tags.selTagFids(["two", "three"], [])) == 3 + assert len(deck.tags.selTagFids(["two", "three"], ["one"])) == 1 + assert len(deck.tags.selTagFids(["one", "three"], ["two", "four"])) == 1 + deck.tags.setGroupForTags(["three"], [], 3) assert deck.db.scalar("select count() from cards where gid = 3") == 3 - deck.setGroupForTags(["one"], [], 2) + deck.tags.setGroupForTags(["one"], [], 2) assert deck.db.scalar("select count() from cards where gid = 2") == 2 def test_addDelTags(): @@ -178,12 +178,12 @@ def test_addDelTags(): f2['Front'] = u"2" deck.addFact(f2) # adding for a given id - deck.addTags([f.id], "foo") + deck.tags.bulkAdd([f.id], "foo") f.load(); f2.load() assert "foo" in f.tags assert "foo" not in f2.tags # should be canonified - deck.addTags([f.id], "foo aaa") + deck.tags.bulkAdd([f.id], "foo aaa") f.load() assert f.tags[0] == "aaa" assert len(f.tags) == 2 diff --git a/tests/test_find.py b/tests/test_find.py index 93bb0bbb8..08f8aede4 100644 --- a/tests/test_find.py +++ b/tests/test_find.py @@ -36,11 +36,11 @@ def test_findCards(): assert len(deck.findCards("tag:monkey")) == 1 assert len(deck.findCards("tag:sheep -tag:monkey")) == 1 assert len(deck.findCards("-tag:sheep")) == 4 - deck.addTags(deck.db.list("select id from facts"), "foo bar") + deck.tags.bulkAdd(deck.db.list("select id from facts"), "foo bar") assert (len(deck.findCards("tag:foo")) == len(deck.findCards("tag:bar")) == 5) - deck.delTags(deck.db.list("select id from facts"), "foo") + deck.tags.bulkRem(deck.db.list("select id from facts"), "foo") assert len(deck.findCards("tag:foo")) == 0 assert len(deck.findCards("tag:bar")) == 5 # text searches