move tags into deck; code into separate file

- moved tags into json like previous changes, and dropped the unnecessary id
- added tags.py for a tag manager
- moved the tag utilities from utils into tags.py

anki/deck.py

@@ -4,14 +4,14 @@
 
 import time, os, random, re, stat, simplejson, datetime, copy, shutil
 from anki.lang import _, ngettext
-from anki.utils import parseTags, ids2str, hexifyID, \
+from anki.utils import ids2str, hexifyID, checksum, fieldChecksum, stripHTML, \
-     checksum, fieldChecksum, addTags, delTags, stripHTML, intTime, \
+    intTime, splitFields
-     splitFields
 from anki.hooks import runHook, runFilter
 from anki.sched import Scheduler
-from anki.models import ModelRegistry
+from anki.models import ModelManager
-from anki.media import MediaRegistry
+from anki.media import MediaManager
-from anki.groups import GroupRegistry
+from anki.groups import GroupManager
+from anki.tags import TagManager
 from anki.consts import *
 from anki.errors import AnkiError
 
@@ -52,9 +52,10 @@ class _Deck(object):
         self.path = db._path
         self._lastSave = time.time()
         self.clearUndo()
-        self.media = MediaRegistry(self)
+        self.media = MediaManager(self)
-        self.models = ModelRegistry(self)
+        self.models = ModelManager(self)
-        self.groups = GroupRegistry(self)
+        self.groups = GroupManager(self)
+        self.tags = TagManager(self)
         self.load()
         if not self.crt:
             d = datetime.datetime.today()
@@ -90,13 +91,15 @@ class _Deck(object):
          self.conf,
          models,
          groups,
-         gconf) = self.db.first("""
+         gconf,
+         tags) = self.db.first("""
 select crt, mod, scm, dty, syncName, lastSync,
-qconf, conf, models, groups, gconf from deck""")
+qconf, conf, models, groups, gconf, tags from deck""")
         self.qconf = simplejson.loads(self.qconf)
         self.conf = simplejson.loads(self.conf)
         self.models.load(models)
         self.groups.load(groups, gconf)
+        self.tags.load(tags)
 
     def flush(self, mod=None):
         "Flush state to DB, updating mod time."
@@ -111,6 +114,7 @@ qconf=?, conf=?""",
             simplejson.dumps(self.conf))
         self.models.flush()
         self.groups.flush()
+        self.tags.flush()
 
     def save(self, name=None, mod=None):
         "Flush, commit DB, and take out another write lock."
@@ -447,93 +451,6 @@ from cards c, facts f
 where c.fid == f.id
 %s""" % where)
 
-    # Tags
-    ##########################################################################
-
-    def tagList(self):
-        return self.db.list("select name from tags order by name")
-
-    def updateFactTags(self, fids=None):
-        "Add any missing tags to the tags list."
-        if fids:
-            lim = " where id in " + ids2str(fids)
-        else:
-            lim = ""
-        self.registerTags(set(parseTags(
-            " ".join(self.db.list("select distinct tags from facts"+lim)))))
-
-    def registerTags(self, tags):
-        r = []
-        for t in tags:
-            r.append({'t': t})
-        self.db.executemany("""
-insert or ignore into tags (mod, name) values (%d, :t)""" % intTime(),
-                            r)
-
-    def addTags(self, ids, tags, add=True):
-        "Add tags in bulk. TAGS is space-separated."
-        newTags = parseTags(tags)
-        if not newTags:
-            return
-        # cache tag names
-        self.registerTags(newTags)
-        # find facts missing the tags
-        if add:
-            l = "tags not "
-            fn = addTags
-        else:
-            l = "tags "
-            fn = delTags
-        lim = " or ".join(
-            [l+"like :_%d" % c for c, t in enumerate(newTags)])
-        res = self.db.all(
-            "select id, tags from facts where id in %s and %s" % (
-                ids2str(ids), lim),
-            **dict([("_%d" % x, '%% %s %%' % y) for x, y in enumerate(newTags)]))
-        # update tags
-        fids = []
-        def fix(row):
-            fids.append(row[0])
-            return {'id': row[0], 't': fn(tags, row[1]), 'n':intTime()}
-        self.db.executemany("""
-update facts set tags = :t, mod = :n where id = :id""", [fix(row) for row in res])
-        # update q/a cache
-        self.registerTags(parseTags(tags))
-
-    def delTags(self, ids, tags):
-        self.addTags(ids, tags, False)
-
-    # Tag-based selective study
-    ##########################################################################
-
-    def selTagFids(self, yes, no):
-        l = []
-        # find facts that match yes
-        lim = ""
-        args = []
-        query = "select id from facts"
-        if not yes and not no:
-            pass
-        else:
-            if yes:
-                lim += " or ".join(["tags like ?" for t in yes])
-                args += ['%% %s %%' % t for t in yes]
-            if no:
-                lim2 = " and ".join(["tags not like ?" for t in no])
-                if lim:
-                    lim = "(%s) and %s" % (lim, lim2)
-                else:
-                    lim = lim2
-                args += ['%% %s %%' % t for t in no]
-            query += " where " + lim
-        return self.db.list(query, *args)
-
-    def setGroupForTags(self, yes, no, gid):
-        fids = self.selTagFids(yes, no)
-        self.db.execute(
-            "update cards set gid = ? where fid in "+ids2str(fids),
-            gid)
-
     # Finding cards
     ##########################################################################
 
@@ -685,8 +602,7 @@ update facts set tags = :t, mod = :n where id = :id""", [fix(row) for row in res
 select id from facts where id not in (select distinct fid from cards)""")
         self._delFacts(ids)
         # tags
-        self.db.execute("delete from tags")
+        self.tags.registerFacts()
-        self.updateFactTags()
         # field cache
         for m in self.models.all():
             self.updateFieldCache(self.models.fids(m))

anki/facts.py

@@ -5,8 +5,7 @@
 import time
 from anki.errors import AnkiError
 from anki.utils import fieldChecksum, intTime, \
-    joinFields, splitFields, ids2str, parseTags, canonifyTags, hasTag, \
+    joinFields, splitFields, ids2str, stripHTML, timestampID
-    stripHTML, timestampID
 
 class Fact(object):
 
@@ -35,7 +34,7 @@ class Fact(object):
          self.data) = self.deck.db.first("""
 select mid, gid, mod, tags, flds, data from facts where id = ?""", self.id)
         self.fields = splitFields(self.fields)
-        self.tags = parseTags(self.tags)
+        self.tags = self.deck.tags.split(self.tags)
         self._model = self.deck.models.get(self.mid)
         self._fmap = self.deck.models.fieldMap(self._model)
 
@@ -50,7 +49,7 @@ insert or replace into facts values (?, ?, ?, ?, ?, ?, ?, ?)""",
                             sfld, self.data)
         self.id = res.lastrowid
         self.updateFieldChecksums()
-        self.deck.registerTags(self.tags)
+        self.deck.tags.register(self.tags)
 
     def joinedFields(self):
         return joinFields(self.fields)
@@ -109,10 +108,10 @@ insert or replace into facts values (?, ?, ?, ?, ?, ?, ?, ?)""",
     ##################################################
 
     def hasTag(self, tag):
-        return hasTag(tag, self.tags)
+        return self.deck.tags.inStr(tag, self.tags)
 
     def stringTags(self):
-        return canonifyTags(self.tags)
+        return self.deck.tags.canonify(self.tags)
 
     def delTag(self, tag):
         rem = []

anki/groups.py

@@ -40,7 +40,7 @@ defaultData = {
     'inactiveTags': None,
 }
 
-class GroupRegistry(object):
+class GroupManager(object):
 
     # Registry save/load
     #############################################################

anki/importing/__init__.py

@@ -15,8 +15,7 @@ import time
 #from anki.cards import cardsTable
 #from anki.facts import factsTable, fieldsTable
 from anki.lang import _
-from anki.utils import canonifyTags, fieldChecksum
+from anki.utils import fieldChecksum, ids2str
-from anki.utils import canonifyTags, ids2str
 from anki.errors import *
 #from anki.deck import NEW_CARDS_RANDOM
 

anki/importing/anki10.py

@@ -50,7 +50,7 @@ class Anki10Importer(Importer):
         copyLocalMedia(server.deck, client.deck)
         # add tags
         fids = [f[0] for f in res['added-facts']['facts']]
-        self.deck.addTags(fids, self.tagsToAdd)
+        self.deck.tags.add(fids, self.tagsToAdd)
         # mark import material as newly added
         self.deck.db.execute(
             "update cards set modified = :t where id in %s" %

anki/media.py

@@ -7,7 +7,7 @@ import os, shutil, re, urllib, urllib2, time, unicodedata, \
 from anki.utils import checksum, intTime, namedtmp, isWin
 from anki.lang import _
 
-class MediaRegistry(object):
+class MediaManager(object):
 
     # can be altered at the class level for dropbox, etc
     mediaPrefix = ""

anki/models.py

@@ -56,7 +56,7 @@ defaultTemplate = {
     'gid': None,
 }
 
-class ModelRegistry(object):
+class ModelManager(object):
 
     # Saving/loading registry
     #############################################################

anki/sched.py

@@ -6,7 +6,7 @@ import time, datetime, simplejson, random, itertools
 from operator import itemgetter
 from heapq import *
 #from anki.cards import Card
-from anki.utils import parseTags, ids2str, intTime, fmtTimeSpan
+from anki.utils import ids2str, intTime, fmtTimeSpan
 from anki.lang import _, ngettext
 from anki.consts import *
 from anki.hooks import runHook

anki/storage.py

@@ -69,7 +69,8 @@ create table if not exists deck (
     conf            text not null,
     models          text not null,
     groups          text not null,
-    gconf           text not null
+    gconf           text not null,
+    tags            text not null
 );
 
 create table if not exists cards (
@@ -125,28 +126,20 @@ create table if not exists revlog (
     type            integer not null
 );
 
-create table if not exists tags (
-    id              integer primary key,
-    mod             integer not null,
-    name            text not null collate nocase unique
-);
-
 insert or ignore into deck
-values(1,0,0,0,%(v)s,0,'',0,'','','','','');
+values(1,0,0,0,%(v)s,0,'',0,'','','{}','','','{}');
 """ % ({'v':CURRENT_VERSION}))
     import anki.deck
     import anki.groups
     if setDeckConf:
         db.execute("""
-update deck set qconf = ?, conf = ?, models = ?, groups = ?, gconf = ?""",
+update deck set qconf = ?, conf = ?, groups = ?, gconf = ?""",
                    simplejson.dumps(anki.deck.defaultQconf),
                    simplejson.dumps(anki.deck.defaultConf),
-                   "{}",
                    simplejson.dumps({'1': {'name': _("Default"), 'conf': 1,
                                        'mod': intTime()}}),
                    simplejson.dumps({'1': anki.groups.defaultConf}))
 
-
 def _updateIndices(db):
     "Add indices to the DB."
     db.executescript("""
@@ -192,14 +185,6 @@ def _upgradeSchema(db):
         return ver
     runHook("1.x upgrade", db)
 
-    # tags
-    ###########
-    _moveTable(db, "tags")
-    db.execute("insert or ignore into tags select id, ?, tag from tags2",
-               intTime())
-    db.execute("drop table tags2")
-    db.execute("drop table cardTags")
-
     # facts
     ###########
     # tags should have a leading and trailing space if not empty, and not
@@ -328,10 +313,22 @@ yesCount from reviewHistory"""):
         "insert or ignore into revlog values (?,?,?,?,?,?,?,?)", r)
     db.execute("drop table reviewHistory")
 
+    # deck
+    ###########
+    _migrateDeckTbl(db)
+
+    # tags
+    ###########
+    tags = {}
+    for t in db.list("select tag from tags"):
+        tags[t] = intTime()
+    db.execute("update deck set tags = ?", simplejson.dumps(tags))
+    db.execute("drop table tags")
+    db.execute("drop table cardTags")
+
     # the rest
     ###########
     db.execute("drop table media")
-    _migrateDeckTbl(db)
     _migrateModels(db)
     _updateIndices(db)
     return ver
@@ -342,7 +339,7 @@ def _migrateDeckTbl(db):
     db.execute("""
 insert or replace into deck select id, cast(created as int), :t,
 :t, 99, 0, ifnull(syncName, ""), cast(lastSync as int),
-"", "", "", "", "" from decks""", t=intTime())
+"", "", "", "", "", "" from decks""", t=intTime())
     # update selective study
     qconf = anki.deck.defaultQconf.copy()
     # delete old selective study settings, which we can't auto-upgrade easily

anki/tags.py

@@ -0,0 +1,172 @@
+# -*- coding: utf-8 -*-
+# Copyright: Damien Elmes <anki@ichi2.net>
+# License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
+
+import simplejson
+from anki.utils import intTime, ids2str
+
+"""
+Anki maintains a cache of used tags so it can quickly present a list of tags
+for autocomplete and in the browser. For efficiency, deletions are not
+tracked, so unused tags can only be removed from the list with a DB check.
+
+This module manages the tag cache and tags for facts.
+"""
+
+class TagManager(object):
+
+    # Registry save/load
+    #############################################################
+
+    def __init__(self, deck):
+        self.deck = deck
+
+    def load(self, json):
+        self.tags = simplejson.loads(json)
+        self.changed = False
+
+    def flush(self):
+        if self.changed:
+            self.deck.db.execute("update deck set tags=?",
+                                 simplejson.dumps(self.tags))
+
+    # Registering and fetching tags
+    #############################################################
+
+    def register(self, tags):
+        "Given a list of tags, add any missing ones to tag registry."
+        # case is stored as received, so user can create different case
+        # versions of the same tag if they ignore the qt autocomplete.
+        for t in tags:
+            if t not in self.tags:
+                self.tags[t] = intTime()
+                self.changed = True
+
+    def all(self):
+        return self.tags.keys()
+
+    def registerFacts(self, fids=None):
+        "Add any missing tags from facts to the tags list."
+        # when called without an argument, the old list is cleared first.
+        if fids:
+            lim = " where id in " + ids2str(fids)
+        else:
+            lim = ""
+            self.tags = {}
+            self.changed = True
+        self.register(set(self.split(
+            " ".join(self.deck.db.list("select distinct tags from facts"+lim)))))
+
+    # Bulk addition/removal from facts
+    #############################################################
+
+    def bulkAdd(self, ids, tags, add=True):
+        "Add tags in bulk. TAGS is space-separated."
+        newTags = self.split(tags)
+        if not newTags:
+            return
+        # cache tag names
+        self.register(newTags)
+        # find facts missing the tags
+        if add:
+            l = "tags not "
+            fn = self.addToStr
+        else:
+            l = "tags "
+            fn = self.remFromStr
+        lim = " or ".join(
+            [l+"like :_%d" % c for c, t in enumerate(newTags)])
+        res = self.deck.db.all(
+            "select id, tags from facts where id in %s and %s" % (
+                ids2str(ids), lim),
+            **dict([("_%d" % x, '%% %s %%' % y)
+                    for x, y in enumerate(newTags)]))
+        # update tags
+        fids = []
+        def fix(row):
+            fids.append(row[0])
+            return {'id': row[0], 't': fn(tags, row[1]), 'n':intTime()}
+        self.deck.db.executemany(
+            "update facts set tags = :t, mod = :n where id = :id",
+            [fix(row) for row in res])
+
+    def bulkRem(self, ids, tags):
+        self.bulkAdd(ids, tags, False)
+
+    # String-based utilities
+    ##########################################################################
+
+    def split(self, tags):
+        "Parse a string and return a list of tags."
+        return [t for t in tags.split(" ") if t]
+
+    def join(self, tags):
+        "Join tags into a single string, with leading and trailing spaces."
+        if not tags:
+            return u""
+        return u" %s " % u" ".join(tags)
+
+    def addToStr(self, addtags, tags):
+        "Add tags if they don't exist."
+        currentTags = self.split(tags)
+        for tag in self.split(addtags):
+            if not self.inList(tag, currentTags):
+                currentTags.append(tag)
+        return self.canonify(currentTags)
+
+    def remFromStr(self, deltags, tags):
+        "Delete tags if they don't exists."
+        currentTags = self.split(tags)
+        for tag in self.split(deltags):
+            # find tags, ignoring case
+            remove = []
+            for tx in currentTags:
+                if tag.lower() == tx.lower():
+                    remove.append(tx)
+            # remove them
+            for r in remove:
+                currentTags.remove(r)
+        return self.canonify(currentTags)
+
+    # List-based utilities
+    ##########################################################################
+
+    def canonify(self, tags):
+        "Strip leading/trailing/superfluous spaces and duplicates."
+        tags = [t.lstrip(":") for t in set(tags)]
+        return self.join(sorted(tags))
+
+    def inList(self, tag, tags):
+        "True if TAG is in TAGS. Ignore case."
+        return tag.lower() in [t.lower() for t in tags]
+
+    # Tag-based selective study
+    ##########################################################################
+
+    def selTagFids(self, yes, no):
+        l = []
+        # find facts that match yes
+        lim = ""
+        args = []
+        query = "select id from facts"
+        if not yes and not no:
+            pass
+        else:
+            if yes:
+                lim += " or ".join(["tags like ?" for t in yes])
+                args += ['%% %s %%' % t for t in yes]
+            if no:
+                lim2 = " and ".join(["tags not like ?" for t in no])
+                if lim:
+                    lim = "(%s) and %s" % (lim, lim2)
+                else:
+                    lim = lim2
+                args += ['%% %s %%' % t for t in no]
+            query += " where " + lim
+        return self.deck.db.list(query, *args)
+
+    def setGroupForTags(self, yes, no, gid):
+        fids = self.selTagFids(yes, no)
+        self.deck.db.execute(
+            "update cards set gid = ? where fid in "+ids2str(fids),
+            gid)

anki/utils.py

@@ -189,50 +189,6 @@ def timestampID(db, table):
         t += 1
     return t
 
-# Tags
-##############################################################################
-
-def parseTags(tags):
-    "Parse a string and return a list of tags."
-    return [t for t in tags.split(" ") if t]
-
-def joinTags(tags):
-    "Join tags into a single string, with leading and trailing spaces."
-    if not tags:
-        return u""
-    return u" %s " % u" ".join(tags)
-
-def canonifyTags(tags):
-    "Strip leading/trailing/superfluous spaces and duplicates."
-    tags = [t.lstrip(":") for t in set(tags)]
-    return joinTags(sorted(tags))
-
-def hasTag(tag, tags):
-    "True if TAG is in TAGS. Ignore case."
-    return tag.lower() in [t.lower() for t in tags]
-
-def addTags(addtags, tags):
-    "Add tags if they don't exist."
-    currentTags = parseTags(tags)
-    for tag in parseTags(addtags):
-        if not hasTag(tag, currentTags):
-            currentTags.append(tag)
-    return canonifyTags(currentTags)
-
-def delTags(deltags, tags):
-    "Delete tags if they don't exists."
-    currentTags = parseTags(tags)
-    for tag in parseTags(deltags):
-        # find tags, ignoring case
-        remove = []
-        for tx in currentTags:
-            if tag.lower() == tx.lower():
-                remove.append(tx)
-        # remove them
-        for r in remove:
-            currentTags.remove(r)
-    return canonifyTags(currentTags)
-
 # Fields
 ##############################################################################
 

tests/test_deck.py

@@ -156,17 +156,17 @@ def test_selective():
     f = deck.newFact()
     f['Front'] = u"3"; f.tags = ["one", "two", "three", "four"]
     deck.addFact(f)
-    assert len(deck.selTagFids(["one"], [])) == 2
+    assert len(deck.tags.selTagFids(["one"], [])) == 2
-    assert len(deck.selTagFids(["three"], [])) == 3
+    assert len(deck.tags.selTagFids(["three"], [])) == 3
-    assert len(deck.selTagFids([], ["three"])) == 0
+    assert len(deck.tags.selTagFids([], ["three"])) == 0
-    assert len(deck.selTagFids(["one"], ["three"])) == 0
+    assert len(deck.tags.selTagFids(["one"], ["three"])) == 0
-    assert len(deck.selTagFids(["one"], ["two"])) == 1
+    assert len(deck.tags.selTagFids(["one"], ["two"])) == 1
-    assert len(deck.selTagFids(["two", "three"], [])) == 3
+    assert len(deck.tags.selTagFids(["two", "three"], [])) == 3
-    assert len(deck.selTagFids(["two", "three"], ["one"])) == 1
+    assert len(deck.tags.selTagFids(["two", "three"], ["one"])) == 1
-    assert len(deck.selTagFids(["one", "three"], ["two", "four"])) == 1
+    assert len(deck.tags.selTagFids(["one", "three"], ["two", "four"])) == 1
-    deck.setGroupForTags(["three"], [], 3)
+    deck.tags.setGroupForTags(["three"], [], 3)
     assert deck.db.scalar("select count() from cards where gid = 3") == 3
-    deck.setGroupForTags(["one"], [], 2)
+    deck.tags.setGroupForTags(["one"], [], 2)
     assert deck.db.scalar("select count() from cards where gid = 2") == 2
 
 def test_addDelTags():
@@ -178,12 +178,12 @@ def test_addDelTags():
     f2['Front'] = u"2"
     deck.addFact(f2)
     # adding for a given id
-    deck.addTags([f.id], "foo")
+    deck.tags.bulkAdd([f.id], "foo")
     f.load(); f2.load()
     assert "foo" in f.tags
     assert "foo" not in f2.tags
     # should be canonified
-    deck.addTags([f.id], "foo aaa")
+    deck.tags.bulkAdd([f.id], "foo aaa")
     f.load()
     assert f.tags[0] == "aaa"
     assert len(f.tags) == 2

tests/test_find.py

@@ -36,11 +36,11 @@ def test_findCards():
     assert len(deck.findCards("tag:monkey")) == 1
     assert len(deck.findCards("tag:sheep -tag:monkey")) == 1
     assert len(deck.findCards("-tag:sheep")) == 4
-    deck.addTags(deck.db.list("select id from facts"), "foo bar")
+    deck.tags.bulkAdd(deck.db.list("select id from facts"), "foo bar")
     assert (len(deck.findCards("tag:foo")) ==
             len(deck.findCards("tag:bar")) ==
             5)
-    deck.delTags(deck.db.list("select id from facts"), "foo")
+    deck.tags.bulkRem(deck.db.list("select id from facts"), "foo")
     assert len(deck.findCards("tag:foo")) == 0
     assert len(deck.findCards("tag:bar")) == 5
     # text searches