diff --git a/anki/cards.py b/anki/cards.py index 4b33bdd60..842cbc65e 100644 --- a/anki/cards.py +++ b/anki/cards.py @@ -138,7 +138,7 @@ class Card(object): return self.htmlQuestion(type="answer", align=align) def _splitTags(self): - return (self.fact.tags, self.fact.model.name, self.cardModel.name) + return (self.fact._tags, self.fact.model.name, self.cardModel.name) # Non-ORM ########################################################################## @@ -192,8 +192,3 @@ mapper(Card, cardsTable, properties={ 'fact': relation(Fact, backref="cards", primaryjoin= cardsTable.c.factId == factsTable.c.id), }) - -mapper(Fact, factsTable, properties={ - 'model': relation(Model), - 'fields': relation(Field, backref="fact", order_by=Field.ordinal), - }) diff --git a/anki/deck.py b/anki/deck.py index bbf937c31..280a60421 100644 --- a/anki/deck.py +++ b/anki/deck.py @@ -10,11 +10,10 @@ from anki.lang import _, ngettext from anki.errors import DeckAccessError from anki.stdmodels import BasicModel from anki.utils import parseTags, tidyHTML, genID, ids2str, hexifyID, \ - canonifyTags, joinTags, addTags, checksum, fieldChecksum, intTime + canonifyTags, joinTags, addTags, deleteTags, checksum, fieldChecksum, intTime from anki.revlog import logReview from anki.models import Model, CardModel, formatQA from anki.fonts import toPlatformFont -from anki.tags import tagIds, tagId from operator import itemgetter from itertools import groupby from anki.hooks import runHook, hookEmpty @@ -444,7 +443,7 @@ due > :now and due < :now""", now=time.time()) cards.append(card) # update card q/a fact.setModified(True, self) - self.updateFactTags([fact.id]) + self.registerTags(fact.tags()) self.flushMod() if reset: self.reset() @@ -476,7 +475,7 @@ due > :now and due < :now""", now=time.time()) empty["text:"+k] = u"" local["text:"+k] = local[k] empty['tags'] = "" - local['tags'] = fact.tags + local['tags'] = fact._tags try: if (render(format, local) == render(format, empty)): @@ -503,7 +502,6 @@ where factId = :fid and cardModelId = :cmid""", card = anki.cards.Card( fact, cardModel, fact.created+0.0001*cardModel.ordinal) - self.updateCardTags([card.id]) raise Exception("incorrect; not checking selective study") self.newAvail += 1 ids.append(card.id) @@ -581,7 +579,7 @@ where facts.id not in (select distinct factId from cards)""") fact = self.newFact(model) for field in fact.fields: fact[field.name] = oldFact[field.name] - fact.tags = oldFact.tags + fact._tags = oldFact._tags return fact # Cards @@ -609,21 +607,6 @@ where facts.id not in (select distinct factId from cards)""") self.db.statement("delete from cards where id in %s" % strids) # note deleted anki.graves.registerMany(self.db, anki.graves.CARD, ids) - # gather affected tags - tags = self.db.column0( - "select tagId from cardTags where cardId in %s" % - strids) - # delete - self.db.statement("delete from cardTags where cardId in %s" % strids) - # find out if they're used by anything else - unused = [] - for tag in tags: - if not self.db.scalar( - "select 1 from cardTags where tagId = :d limit 1", d=tag): - unused.append(tag) - # delete unused - self.db.statement("delete from tags where id in %s" % - ids2str(unused)) # remove any dangling facts self.deleteDanglingFacts() self.refreshSession() @@ -819,7 +802,6 @@ where id in %s""" % ids2str(ids), new=new.id, ord=new.ordinal) cardIds = self.db.column0( "select id from cards where factId in %s" % ids2str(factIds)) - self.updateCardTags(cardIds) self.refreshSession() self.finishProgress() @@ -1095,9 +1077,12 @@ modified = :now where cardModelId in %s""" % strids, now=time.time()) self.flushMod() - # Tags: querying + # Tags ########################################################################## + def tagList(self): + return self.db.column0("select name from tags order by name") + def splitTagsList(self, where=""): return self.db.all(""" select cards.id, facts.tags, models.name, cardModels.name @@ -1112,221 +1097,62 @@ select cards.id from cards, facts where facts.tags = "" and cards.factId = facts.id""") - def cardsWithTags(self, tagStr, search="and"): - tagIds = [] - # get ids - for tag in tagStr.split(" "): - tag = tag.replace("*", "%") - if "%" in tag: - ids = self.db.column0( - "select id from tags where name like :tag", tag=tag) - if search == "and" and not ids: - return [] - tagIds.append(ids) - else: - id = self.db.scalar( - "select id from tags where name = :tag", tag=tag) - if search == "and" and not id: - return [] - tagIds.append(id) - # search for any - if search == "or": - return self.db.column0( - "select cardId from cardTags where tagId in %s" % - ids2str(tagIds)) - else: - # search for all - l = [] - for ids in tagIds: - if isinstance(ids, types.ListType): - l.append("select cardId from cardTags where tagId in %s" % - ids2str(ids)) - else: - l.append("select cardId from cardTags where tagId = %d" % - ids) - q = " intersect ".join(l) - return self.db.column0(q) - - def allTags(self): - return self.db.column0("select name from tags order by name") - - def allTags_(self, where=""): - t = self.db.column0("select tags from facts %s" % where) - t += self.db.column0("select name from models") - t += self.db.column0("select name from cardModels") - return sorted(list(set(parseTags(joinTags(t))))) - - def allUserTags(self): - return sorted(list(set(parseTags(joinTags(self.db.column0( - "select tags from facts")))))) - - def factTags(self, ids): - return self.db.all(""" -select id, tags from facts -where id in %s""" % ids2str(ids)) - def cardHasTag(self, card, tag): - id = tagId(self.db, tag, create=False) - if id: - return self.db.scalar( - "select 1 from cardTags where cardId = :c and tagId = :t", - c=card.id, t=id) + tags = self.db.scalar("select tags from fact where id = :fid", + fid=card.factId) + return tag.lower() in parseTags(tags.lower()) - # Tags: caching - ########################################################################## - - def updateFactTags(self, factIds): - self.updateCardTags(self.db.column0( - "select id from cards where factId in %s" % - ids2str(factIds))) - - def updateModelTags(self, modelId): - self.updateCardTags(self.db.column0(""" -select cards.id from cards, facts where -cards.factId = facts.id and -facts.modelId = :id""", id=modelId)) - - def updateCardTags(self, cardIds=None): - self.db.flush() - if cardIds is None: - self.db.statement("delete from cardTags") - self.db.statement("delete from tags") - tids = tagIds(self.db, self.allTags_()) - rows = self.splitTagsList() + def updateFactTags(self, factIds=None): + "Add any missing tags to the tags list." + if factIds: + lim = " where id in " + ids2str(factIds) else: - self.db.statement("delete from cardTags where cardId in %s" % - ids2str(cardIds)) - fids = ids2str(self.db.column0( - "select factId from cards where id in %s" % - ids2str(cardIds))) - tids = tagIds(self.db, self.allTags_( - where="where id in %s" % fids)) - rows = self.splitTagsList( - where="and facts.id in %s" % fids) - d = [] - for (id, fact, model, templ) in rows: - for tag in parseTags(fact): - d.append({"cardId": id, - "tagId": tids[tag.lower()], - "src": 0}) - for tag in parseTags(model): - d.append({"cardId": id, - "tagId": tids[tag.lower()], - "src": 1}) - for tag in parseTags(templ): - d.append({"cardId": id, - "tagId": tids[tag.lower()], - "src": 2}) - if d: - self.db.statements(""" -insert into cardTags -(cardId, tagId, type) values -(:cardId, :tagId, :src)""", d) - self.deleteUnusedTags() + lim = "" + self.registerTags(set(parseTags( + " ".join(self.db.column0("select distinct tags from facts"+lim))))) - def updateTagsForModel(self, model): - cards = self.db.all(""" -select cards.id, cards.cardModelId from cards, facts where -facts.modelId = :m and cards.factId = facts.id""", m=model.id) - cardIds = [x[0] for x in cards] - factIds = self.db.column0(""" -select facts.id from facts where -facts.modelId = :m""", m=model.id) - cmtags = " ".join([cm.name for cm in model.cardModels]) - tids = tagIds(self.db, parseTags(model.tags) + - parseTags(cmtags)) - self.db.statement(""" -delete from cardTags where cardId in %s -and src in (1, 2)""" % ids2str(cardIds)) - d = [] - for tag in parseTags(model.tags): - for id in cardIds: - d.append({"cardId": id, - "tagId": tids[tag.lower()], - "src": 1}) - cmtags = {} - for cm in model.cardModels: - cmtags[cm.id] = parseTags(cm.name) - for c in cards: - for tag in cmtags[c[1]]: - d.append({"cardId": c[0], - "tagId": tids[tag.lower()], - "src": 2}) - if d: - self.db.statements(""" -insert into cardTags -(cardId, tagId, src) values -(:cardId, :tagId, :src)""", d) - self.deleteUnusedTags() - - # Tags: adding/removing in bulk - ########################################################################## - # these could be optimized to use the tag cache in the future - - def deleteUnusedTags(self): - self.db.statement(""" -delete from tags where id not in (select distinct tagId from cardTags)""") - - def addTags(self, ids, tags): - "Add tags in bulk. Caller must .reset()" - self.startProgress() - tlist = self.factTags(ids) - newTags = parseTags(tags) - now = time.time() - pending = [] - for (id, tags) in tlist: - oldTags = parseTags(tags) - tmpTags = list(set(oldTags + newTags)) - if tmpTags != oldTags: - pending.append( - {'id': id, 'now': now, 'tags': " ".join(tmpTags)}) + def registerTags(self, tags): + r = [] + for t in tags: + r.append({'t': t}) self.db.statements(""" -update facts set -tags = :tags, -modified = :now -where id = :id""", pending) - factIds = [c['id'] for c in pending] - cardIds = self.db.column0( - "select id from cards where factId in %s" % - ids2str(factIds)) - self.updateCardQACacheFromIds(factIds, type="facts") - self.updateCardTags(cardIds) +insert or ignore into tags (modified, name) values (%d, :t)""" % intTime(), + r) + + def addTags(self, ids, tags, add=True): + "Add tags in bulk. TAGS is space-separated." + self.startProgress() + newTags = parseTags(tags) + # cache tag names + self.registerTags(newTags) + # find facts missing the tags + if add: + l = "tags not " + fn = addTags + else: + l = "tags " + fn = deleteTags + lim = " or ".join( + [l+"like :_%d" % c for c, t in enumerate(newTags)]) + res = self.db.all( + "select id, tags from facts where " + lim, + **dict([("_%d" % x, '%% %s %%' % y) for x, y in enumerate(newTags)])) + # update tags + fids = [] + def fix(row): + fids.append(row[0]) + return {'id': row[0], 't': fn(tags, row[1])} + self.db.statements(""" +update facts set tags = :t, modified = %d +where id = :id""" % intTime(), [fix(row) for row in res]) + # update q/a cache + self.updateCardQACacheFromIds(fids, type="facts") self.flushMod() self.finishProgress() self.refreshSession() def deleteTags(self, ids, tags): - "Delete tags in bulk. Caller must .reset()" - self.startProgress() - tlist = self.factTags(ids) - newTags = parseTags(tags) - now = time.time() - pending = [] - for (id, tags) in tlist: - oldTags = parseTags(tags) - tmpTags = oldTags[:] - for tag in newTags: - try: - tmpTags.remove(tag) - except ValueError: - pass - if tmpTags != oldTags: - pending.append( - {'id': id, 'now': now, 'tags': " ".join(tmpTags)}) - self.db.statements(""" -update facts set -tags = :tags, -modified = :now -where id = :id""", pending) - factIds = [c['id'] for c in pending] - cardIds = self.db.column0( - "select id from cards where factId in %s" % - ids2str(factIds)) - self.updateCardQACacheFromIds(factIds, type="facts") - self.updateCardTags(cardIds) - self.flushMod() - self.finishProgress() - self.refreshSession() + self.addTags(ids, tags, False) # Finding cards ########################################################################## @@ -1742,6 +1568,8 @@ where id in %s""" % ids2str(ids)): f.tags = self.db.scalar(""" select group_concat(name, " ") from tags t, cardTags ct where cardId = :cid and ct.tagId = t.id""", cid=id) or u"" + if f.tags: + f.tags = " " + f.tags + " " except: raise Exception("Your sqlite is too old.") cards = self.addFact(f) @@ -1860,7 +1688,9 @@ select id from fields where factId not in (select id from facts)""") "where allowEmptyAnswer is null or typeAnswer is null") # fix tags self.updateProgress() - self.updateCardTags() + self.db.statement("delete from tags") + self.updateFactTags() + print "should ensure tags having leading/trailing space" # make sure ordinals are correct self.updateProgress() self.db.statement(""" @@ -2241,16 +2071,10 @@ insert into groups values (1, :t, "Default", 1)""", """ create table tags ( id integer not null, +modified integer not null, name text not null collate nocase unique, -priority integer not null default 0, primary key(id))""", """ -create table cardTags ( -cardId integer not null, -tagId integer not null, -type integer not null, -primary key(tagId, cardId))""", - """ create table groups ( id integer primary key autoincrement, modified integer not null, diff --git a/anki/facts.py b/anki/facts.py index 008d24a96..a04b93577 100644 --- a/anki/facts.py +++ b/anki/facts.py @@ -6,7 +6,8 @@ import time from anki.db import * from anki.errors import * from anki.models import Model, FieldModel, fieldModelsTable -from anki.utils import genID, stripHTMLMedia, fieldChecksum, intTime +from anki.utils import genID, stripHTMLMedia, fieldChecksum, intTime, \ + addTags, deleteTags, parseTags from anki.hooks import runHook # Fields in a fact @@ -63,7 +64,9 @@ class Fact(object): def __init__(self, model=None, pos=None): self.model = model self.id = genID() + self._tags = u"" if model: + # creating for fm in model.fieldModels: self.fields.append(Field(fm)) self.pos = pos @@ -101,6 +104,15 @@ class Fact(object): except (IndexError, KeyError): return default + def addTags(self, tags): + self._tags = addTags(tags, self._tags) + + def deleteTags(self, tags): + self._tags = deleteTags(tags, self._tags) + + def tags(self): + return parseTags(self._tags) + def assertValid(self): "Raise an error if required fields are empty." for field in self.fields: @@ -148,3 +160,9 @@ class Fact(object): self.values())) for card in self.cards: card.rebuildQA(deck) + +mapper(Fact, factsTable, properties={ + 'model': relation(Model), + 'fields': relation(Field, backref="fact", order_by=Field.ordinal), + '_tags': factsTable.c.tags + }) diff --git a/anki/find.py b/anki/find.py index c98703932..45a1f7e89 100644 --- a/anki/find.py +++ b/anki/find.py @@ -65,7 +65,7 @@ def findCardsWhere(deck, query): q = "" x = [] if tquery: - x.append(" id in (%s)" % tquery) + x.append(" factId in (%s)" % tquery) if fquery: x.append(" factId in (%s)" % fquery) if qquery: @@ -397,16 +397,19 @@ def _findCards(deck, query): else: tquery += " intersect " elif isNeg: - tquery += "select id from cards except " + tquery += "select id from facts except " if token == "none": tquery += """ select cards.id from cards, facts where facts.tags = '' and cards.factId = facts.id """ else: token = token.replace("*", "%") - ids = deck.db.column0(""" -select id from tags where name like :tag escape '\\'""", tag=token) + if not token.startswith("%"): + token = "% " + token + if not token.endswith("%"): + token += " %" + args["_tag_%d" % c] = token tquery += """ -select cardId from cardTags where cardTags.tagId in %s""" % ids2str(ids) +select id from facts where tags like :_tag_%d""" % c elif type == SEARCH_TYPE: if qquery: if isNeg: @@ -449,6 +452,7 @@ select cardId from cardTags where cardTags.tagId in %s""" % ids2str(ids) fidquery += "select id from cards except " fidquery += "select id from cards where factId in (%s)" % token elif type == SEARCH_CARD: + print "search_card broken" token = token.replace("*", "%") ids = deck.db.column0(""" select id from tags where name like :tag escape '\\'""", tag=token) diff --git a/anki/sched.py b/anki/sched.py index f048620a6..8119a54c3 100644 --- a/anki/sched.py +++ b/anki/sched.py @@ -8,7 +8,6 @@ from heapq import * from anki.db import * from anki.cards import Card from anki.utils import parseTags, ids2str -from anki.tags import tagIds from anki.lang import _ from anki.consts import * diff --git a/anki/stdmodels.py b/anki/stdmodels.py index a27219b38..d3e7c7188 100644 --- a/anki/stdmodels.py +++ b/anki/stdmodels.py @@ -34,7 +34,6 @@ def BasicModel(): m.addCardModel(CardModel(u'Forward', u'%(Front)s', u'%(Back)s')) m.addCardModel(CardModel(u'Reverse', u'%(Back)s', u'%(Front)s', active=False)) - m.tags = u"Basic" return m models['Basic'] = BasicModel @@ -47,5 +46,4 @@ def RecoveryModel(): m.addFieldModel(FieldModel(u'Question', False, False)) m.addFieldModel(FieldModel(u'Answer', False, False)) m.addCardModel(CardModel(u'Single', u'{{{Question}}}', u'{{{Answer}}}')) - m.tags = u"Recovery" return m diff --git a/anki/sync.py b/anki/sync.py index 71ebafaee..1c8fe372a 100644 --- a/anki/sync.py +++ b/anki/sync.py @@ -26,6 +26,8 @@ SYNC_PORT = int(os.environ.get("SYNC_PORT") or 80) SYNC_URL = "http://%s:%d/sync/" % (SYNC_HOST, SYNC_PORT) KEYS = ("models", "facts", "cards", "media") +# - need to add tags table syncing + ########################################################################## # Monkey-patch httplib to incrementally send instead of chewing up large # amounts of memory, and track progress. diff --git a/anki/tags.py b/anki/tags.py deleted file mode 100644 index 178de85c9..000000000 --- a/anki/tags.py +++ /dev/null @@ -1,29 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright: Damien Elmes -# License: GNU GPL, version 3 or later; http://www.gnu.org/copyleft/gpl.html - -from anki.db import * - -# Type: 0=fact, 1=model, 2=template - -def tagId(s, tag, create=True): - "Return ID for tag, creating if necessary." - id = s.scalar("select id from tags where name = :tag", tag=tag) - if id or not create: - return id - s.statement(""" -insert or ignore into tags -(name) values (:tag)""", tag=tag) - return s.scalar("select id from tags where name = :tag", tag=tag) - -def tagIds(s, tags, create=True): - "Return an ID for all tags, creating if necessary." - ids = {} - if create: - s.statements("insert or ignore into tags (name) values (:tag)", - [{'tag': t} for t in tags]) - tagsD = dict([(x.lower(), y) for (x, y) in s.all(""" -select name, id from tags -where name in (%s)""" % ",".join([ - "'%s'" % t.replace("'", "''") for t in tags]))]) - return tagsD diff --git a/anki/upgrade.py b/anki/upgrade.py index 9a06c1a77..b59a7abce 100644 --- a/anki/upgrade.py +++ b/anki/upgrade.py @@ -44,14 +44,20 @@ cast(factor*1000 as int), reps, successive, noCount, 0, 0 from cards2""") # tags ########### moveTable(s, "tags") - moveTable(s, "cardTags") import deck deck.DeckStorage._addTables(engine) - s.execute("insert or ignore into tags select id, tag, 0 from tags2") + s.execute("insert or ignore into tags select id, :t, tag from tags2", + {'t':intTime()}) + # tags should have a leading and trailing space if not empty, and not + # use commas s.execute(""" -insert or ignore into cardTags select cardId, tagId, src from cardTags2""") +update facts set tags = (case +when trim(tags) == "" then "" +else " " || replace(replace(trim(tags), ",", " "), " ", " ") || " " +end) +""") s.execute("drop table tags2") - s.execute("drop table cardTags2") + s.execute("drop table cardTags") # facts ########### s.execute(""" @@ -158,9 +164,6 @@ create index if not exists ix_media_chksum on media (chksum)""") # deletion tracking db.execute(""" create index if not exists ix_gravestones_delTime on gravestones (delTime)""") - # tags - db.execute(""" -create index if not exists ix_cardTags_cardId on cardTags (cardId)""") def upgradeDeck(deck): "Upgrade deck to the latest version." diff --git a/anki/utils.py b/anki/utils.py index f1e59c8d3..269e211e1 100644 --- a/anki/utils.py +++ b/anki/utils.py @@ -233,11 +233,13 @@ to be integers.""" def parseTags(tags): "Parse a string and return a list of tags." - tags = re.split(" |, ?", tags) - return [t.strip() for t in tags if t.strip()] + return [t for t in tags.split(" ") if t] def joinTags(tags): - return u" ".join(tags) + "Join tags into a single string, with leading and trailing spaces." + if not tags: + return u"" + return u" %s " % u" ".join(tags) def canonifyTags(tags): "Strip leading/trailing/superfluous commas and duplicates." @@ -246,26 +248,28 @@ def canonifyTags(tags): def findTag(tag, tags): "True if TAG is in TAGS. Ignore case." - if not isinstance(tags, types.ListType): - tags = parseTags(tags) return tag.lower() in [t.lower() for t in tags] -def addTags(tagstr, tags): +def addTags(addtags, tags): "Add tags if they don't exist." currentTags = parseTags(tags) - for tag in parseTags(tagstr): + for tag in parseTags(addtags): if not findTag(tag, currentTags): currentTags.append(tag) return joinTags(currentTags) -def deleteTags(tagstr, tags): +def deleteTags(deltags, tags): "Delete tags if they don't exists." currentTags = parseTags(tags) - for tag in parseTags(tagstr): - try: - currentTags.remove(tag) - except ValueError: - pass + for tag in parseTags(deltags): + # find tags, ignoring case + remove = [] + for tx in currentTags: + if tag.lower() == tx.lower(): + remove.append(tx) + # remove them + for r in remove: + currentTags.remove(r) return joinTags(currentTags) # Misc diff --git a/tests/test_deck.py b/tests/test_deck.py index 2eb9b4201..8f36948d6 100644 --- a/tests/test_deck.py +++ b/tests/test_deck.py @@ -268,12 +268,12 @@ def test_findCards(): f = deck.newFact() f['Front'] = u'dog' f['Back'] = u'cat' - f.tags = u"monkey" + f.addTags(u"monkey") deck.addFact(f) f = deck.newFact() f['Front'] = u'goats are fun' f['Back'] = u'sheep' - f.tags = u"sheep goat horse" + f.addTags(u"sheep goat horse") deck.addFact(f) f = deck.newFact() f['Front'] = u'cat' @@ -292,21 +292,13 @@ def test_findCards(): assert len(deck.findCards("are goats")) == 1 assert len(deck.findCards('"are goats"')) == 0 assert len(deck.findCards('"goats are"')) == 1 - # make sure card templates and models match too - assert len(deck.findCards('tag:basic')) == 3 - assert len(deck.findCards('tag:forward')) == 3 - deck.addModel(BasicModel()) - f = deck.newFact() - f['Front'] = u'foo' - f['Back'] = u'bar' - deck.addFact(f) - deck.currentModel.cardModels[1].active = True - f = deck.newFact() - f['Front'] = u'baz' - f['Back'] = u'qux' - c = deck.addFact(f) - assert len(deck.findCards('tag:forward')) == 5 - assert len(deck.findCards('tag:reverse')) == 1 + deck.addTags(deck.db.column0("select id from cards"), "foo bar") + assert (len(deck.findCards("tag:foo")) == + len(deck.findCards("tag:bar")) == + 3) + deck.deleteTags(deck.db.column0("select id from cards"), "foo") + assert len(deck.findCards("tag:foo")) == 0 + assert len(deck.findCards("tag:bar")) == 3 def test_upgrade(): src = os.path.expanduser("~/Scratch/upgrade.anki")