GitHub/Anki
1
0
Fork 0
mirror of https://github.com/ankitects/anki.git synced 2025-09-25 01:06:35 -04:00
Code Issues Projects Releases Packages 1 Wiki Activity Actions

initial changes to models, deck, etc

Browse source 
- move most scheduling parameters from deck to models
- remove obsolete fields in deck and models
- decks->deck
- remove deck id reference in models
- move some deckVars into the deck table
- simplify deckstorage
- lock sessionhelper by default
- add models/currentModel as properties instead of ORM mappings
- remove models.tags
- remove remaining support for memory-backed databases
- use a blank string for syncName instead of null
- remove backup code; will handle in gui
- bump version to 100
- update unit tests
This commit is contained in:
Damien Elmes 2011-02-25 11:32:42 +09:00
parent 2d1d946504
commit 8a60ee0794
7 changed files with 294 additions and 411 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
anki/cards.py
View file

@ -132,7 +132,7 @@ class Card(object):
return self.htmlQuestion(type="answer", align=align) return self.htmlQuestion(type="answer", align=align)
def _splitTags(self): def _splitTags(self):
return (self.fact.tags, self.fact.model.tags, self.cardModel.name) return (self.fact.tags, self.fact.model.name, self.cardModel.name)
# Non-ORM # Non-ORM
########################################################################## ##########################################################################

9
anki/db.py
View file

@ -54,7 +54,7 @@ metadata = MetaData()
class SessionHelper(object): class SessionHelper(object):
"Add some convenience routines to a session." "Add some convenience routines to a session."
def __init__(self, session, lock=False, transaction=True): def __init__(self, session, lock=True, transaction=True):
self._session = session self._session = session
self._lock = lock self._lock = lock
self._transaction = transaction self._transaction = transaction
@ -71,7 +71,7 @@ class SessionHelper(object):
else: else:
self._session.add(obj) self._session.add(obj)
def clear(self): def expunge_all(self):
# compat # compat
if sqlalchemy.__version__.startswith("0.4."): if sqlalchemy.__version__.startswith("0.4."):
self._session.clear() self._session.clear()
@ -128,12 +128,13 @@ class SessionHelper(object):
def _lockDB(self): def _lockDB(self):
"Take out a write lock." "Take out a write lock."
self._session.execute(text("update decks set modified=modified")) self._session.execute("pragma locking_mode = exclusive")
self._session.execute(text("update deck set modified=modified"))
def object_session(*args): def object_session(*args):
s = _object_session(*args) s = _object_session(*args)
if s: if s:
return SessionHelper(s, transaction=False) return SessionHelper(s, lock=False, transaction=False)
return None return None
def sessionmaker(*args, **kwargs): def sessionmaker(*args, **kwargs):

473
anki/deck.py
View file

@ -55,80 +55,53 @@ deckVarsTable = Table(
Column('key', UnicodeText, nullable=False, primary_key=True), Column('key', UnicodeText, nullable=False, primary_key=True),
Column('value', UnicodeText)) Column('value', UnicodeText))
# syncName: md5sum of current deck location, to detect if deck was moved or
# renamed. mobile clients can treat this as a simple boolean
# utcOffset: store independent of tz?
# parts of the code assume we only have one deck # parts of the code assume we only have one deck
decksTable = Table( deckTable = Table(
'decks', metadata, 'deck', metadata,
Column('id', Integer, primary_key=True), Column('id', Integer, nullable=False, primary_key=True),
Column('created', Float, nullable=False, default=time.time), Column('created', Float, nullable=False, default=time.time),
Column('modified', Float, nullable=False, default=time.time), Column('modified', Float, nullable=False, default=time.time),
Column('description', UnicodeText, nullable=False, default=u""), Column('schemaMod', Float, nullable=False, default=0),
Column('version', Integer, nullable=False, default=DECK_VERSION), Column('version', Integer, nullable=False, default=DECK_VERSION),
Column('currentModelId', Integer, ForeignKey("models.id")), Column('currentModelId', Integer, ForeignKey("models.id")),
# syncName stores an md5sum of the deck path when syncing is enabled. If Column('syncName', UnicodeText, nullable=False, default=u""),
# it doesn't match the current deck path, the deck has been moved,
# and syncing is disabled on load.
Column('syncName', UnicodeText),
Column('lastSync', Float, nullable=False, default=0), Column('lastSync', Float, nullable=False, default=0),
# scheduling # scheduling
############## Column('utcOffset', Integer, nullable=False, default=-2),
# initial intervals
Column('hardIntervalMin', Float, nullable=False, default=1.0),
Column('hardIntervalMax', Float, nullable=False, default=1.1),
Column('midIntervalMin', Float, nullable=False, default=3.0),
Column('midIntervalMax', Float, nullable=False, default=5.0),
Column('easyIntervalMin', Float, nullable=False, default=7.0),
Column('easyIntervalMax', Float, nullable=False, default=9.0),
# delays on failure
Column('delay0', Integer, nullable=False, default=600),
# days to delay mature fails
Column('delay1', Integer, nullable=False, default=0),
Column('delay2', Float, nullable=False, default=0.0),
# collapsing future cards
Column('collapseTime', Integer, nullable=False, default=1),
# priorities & postponing - all obsolete
Column('highPriority', UnicodeText, nullable=False, default=u"PriorityVeryHigh"),
Column('medPriority', UnicodeText, nullable=False, default=u"PriorityHigh"),
Column('lowPriority', UnicodeText, nullable=False, default=u"PriorityLow"),
Column('suspended', UnicodeText, nullable=False, default=u""),
# 0 is random, 1 is by input date
Column('newCardOrder', Integer, nullable=False, default=1), Column('newCardOrder', Integer, nullable=False, default=1),
# when to show new cards
Column('newCardSpacing', Integer, nullable=False, default=NEW_CARDS_DISTRIBUTE), Column('newCardSpacing', Integer, nullable=False, default=NEW_CARDS_DISTRIBUTE),
# limit the number of failed cards in play
Column('failedCardMax', Integer, nullable=False, default=20),
# number of new cards to show per day
Column('newCardsPerDay', Integer, nullable=False, default=20), Column('newCardsPerDay', Integer, nullable=False, default=20),
# currently unused Column('revCardOrder', Integer, nullable=False, default=0),
Column('collapseTime', Integer, nullable=False, default=600),
# timeboxing
Column('sessionRepLimit', Integer, nullable=False, default=0), Column('sessionRepLimit', Integer, nullable=False, default=0),
Column('sessionTimeLimit', Integer, nullable=False, default=600), Column('sessionTimeLimit', Integer, nullable=False, default=600),
# stats offset # leeches
Column('utcOffset', Float, nullable=False, default=-1), Column('suspendLeeches', Boolean, nullable=False, default=True),
# count cache Column('leechFails', Integer, nullable=False, default=16),
Column('cardCount', Integer, nullable=False, default=0), # selective study
Column('factCount', Integer, nullable=False, default=0), Column('newActive', UnicodeText, nullable=False, default=u""),
Column('failedNowCount', Integer, nullable=False, default=0), # obsolete Column('newInactive', UnicodeText, nullable=False, default=u""),
Column('failedSoonCount', Integer, nullable=False, default=0), Column('revActive', UnicodeText, nullable=False, default=u""),
Column('revCount', Integer, nullable=False, default=0), Column('revInactive', UnicodeText, nullable=False, default=u""),
Column('newCount', Integer, nullable=False, default=0), )
# rev order
Column('revCardOrder', Integer, nullable=False, default=0))
class Deck(object): class Deck(object):
"Top-level object. Manages facts, cards and scheduling information." "Top-level object. Manages facts, cards and scheduling information."
factorFour = 1.3 factorFour = 1.3
initialFactor = 2.5
maxScheduleTime = 36500
def __init__(self, path=None):
"Create a new deck."
# a limit of 1 deck in the table
self.id = 1
def _initVars(self): def _initVars(self):
self.tmpMediaDir = None if self.utcOffset == -2:
# shared deck; reset timezone and creation date
self.utcOffset = time.timezone + 60*60*4
self.created = time.time()
self.mediaPrefix = "" self.mediaPrefix = ""
self.lastTags = u""
self.lastLoaded = time.time() self.lastLoaded = time.time()
self.undoEnabled = False self.undoEnabled = False
self.sessionStartReps = 0 self.sessionStartReps = 0
@ -136,15 +109,7 @@ class Deck(object):
self.lastSessionStart = 0 self.lastSessionStart = 0
self.queueLimit = 200 self.queueLimit = 200
# if most recent deck var not defined, make sure defaults are set # if most recent deck var not defined, make sure defaults are set
if not self.db.scalar("select 1 from deckVars where key = 'schemaMod'"): if not self.db.scalar("select 1 from deckVars where key = 'latexPost'"):
self.setVarDefault("suspendLeeches", True)
self.setVarDefault("leechFails", 16)
self.setVarDefault("perDay", True)
self.setVarDefault("newActive", "")
self.setVarDefault("revActive", "")
self.setVarDefault("newInactive", self.suspended)
self.setVarDefault("revInactive", self.suspended)
self.setVarDefault("newSpacing", 60)
self.setVarDefault("mediaURL", "") self.setVarDefault("mediaURL", "")
self.setVarDefault("latexPre", """\ self.setVarDefault("latexPre", """\
\\documentclass[12pt]{article} \\documentclass[12pt]{article}
@ -156,8 +121,6 @@ class Deck(object):
\\begin{document} \\begin{document}
""") """)
self.setVarDefault("latexPost", "\\end{document}") self.setVarDefault("latexPost", "\\end{document}")
self.setVarDefault("revSpacing", 0.1)
self.setVarDefault("schemaMod", 0)
self.updateCutoff() self.updateCutoff()
self.setupStandardScheduler() self.setupStandardScheduler()
@ -214,8 +177,8 @@ where time > :t""", t=self.failedCutoff-86400)
self.rebuildNewCount() self.rebuildNewCount()
def _cardLimit(self, active, inactive, sql): def _cardLimit(self, active, inactive, sql):
yes = parseTags(self.getVar(active)) yes = parseTags(getattr(self, active))
no = parseTags(self.getVar(inactive)) no = parseTags(getattr(self, inactive))
if yes: if yes:
yids = tagIds(self.db, yes).values() yids = tagIds(self.db, yes).values()
nids = tagIds(self.db, no).values() nids = tagIds(self.db, no).values()
@ -465,8 +428,9 @@ when queue != -1""")
self.rebuildCSS() self.rebuildCSS()
# spacing for delayed cards - not to be confused with newCardSpacing # spacing for delayed cards - not to be confused with newCardSpacing
# above # above
self.newSpacing = self.getFloat('newSpacing') print "newSpacing/revSpacing"
self.revSpacing = self.getFloat('revSpacing') self.newSpacing = 0
self.revSpacing = 0
def checkDay(self): def checkDay(self):
# check if the day has rolled over # check if the day has rolled over
@ -924,8 +888,6 @@ and queue between 1 and 2""",
interval = (interval + delay) * factor * self.factorFour interval = (interval + delay) * factor * self.factorFour
fuzz = random.uniform(0.95, 1.05) fuzz = random.uniform(0.95, 1.05)
interval *= fuzz interval *= fuzz
if self.maxScheduleTime:
interval = min(interval, self.maxScheduleTime)
return interval return interval
def nextIntervalStr(self, card, ease, short=False): def nextIntervalStr(self, card, ease, short=False):
@ -1293,8 +1255,6 @@ due > :now and due < :now""", now=time.time())
# update card q/a # update card q/a
fact.setModified(True, self) fact.setModified(True, self)
self.updateFactTags([fact.id]) self.updateFactTags([fact.id])
# keep track of last used tags for convenience
self.lastTags = fact.tags
self.flushMod() self.flushMod()
if reset: if reset:
self.reset() self.reset()
@ -1478,10 +1438,19 @@ where facts.id not in (select distinct factId from cards)""")
# Models # Models
########################################################################## ##########################################################################
def getCurrentModel(self):
return self.db.query(anki.models.Model).get(self.currentModelId)
def setCurrentModel(self, model):
self.currentModelId = model.id
currentModel = property(getCurrentModel, setCurrentModel)
def getModels(self):
return self.db.query(anki.models.Model).all()
models = property(getModels)
def addModel(self, model): def addModel(self, model):
if model not in self.models: self.db.add(model)
self.setSchemaModified() self.setSchemaModified()
self.models.append(model)
self.currentModel = model self.currentModel = model
self.flushMod() self.flushMod()
@ -1573,8 +1542,6 @@ select id from models""")
for c in oldModel.cardModels: for c in oldModel.cardModels:
c = c.copy() c = c.copy()
m.addCardModel(c) m.addCardModel(c)
for attr in ("tags", "spacing", "initialSpacing"):
setattr(m, attr, getattr(oldModel, attr))
self.addModel(m) self.addModel(m)
return m return m
@ -1913,7 +1880,7 @@ where cardModelId in %s""" % strids, now=time.time())
def splitTagsList(self, where=""): def splitTagsList(self, where=""):
return self.db.all(""" return self.db.all("""
select cards.id, facts.tags, models.tags, cardModels.name select cards.id, facts.tags, models.name, cardModels.name
from cards, facts, models, cardModels where from cards, facts, models, cardModels where
cards.factId == facts.id and facts.modelId == models.id cards.factId == facts.id and facts.modelId == models.id
and cards.cardModelId = cardModels.id and cards.cardModelId = cardModels.id
@ -1965,7 +1932,7 @@ and cards.factId = facts.id""")
def allTags_(self, where=""): def allTags_(self, where=""):
t = self.db.column0("select tags from facts %s" % where) t = self.db.column0("select tags from facts %s" % where)
t += self.db.column0("select tags from models") t += self.db.column0("select name from models")
t += self.db.column0("select name from cardModels") t += self.db.column0("select name from cardModels")
return sorted(list(set(parseTags(joinTags(t))))) return sorted(list(set(parseTags(joinTags(t)))))
@ -2749,6 +2716,15 @@ select id from facts where cache like :_ff_%d escape '\\'""" % c
if self.progressHandlerEnabled: if self.progressHandlerEnabled:
runHook("dbProgress") runHook("dbProgress")
def setupProgressHandler(self):
self.progressHandlerCalled = 0
self.progressHandlerEnabled = False
try:
self.engine.raw_connection().set_progress_handler(
deck.progressHandler, 100)
except:
pass
def enableProgressHandler(self): def enableProgressHandler(self):
self.progressHandlerEnabled = True self.progressHandlerEnabled = True
@ -2901,7 +2877,6 @@ where key = :key""", key=key, value=value):
def mediaDir(self, create=False): def mediaDir(self, create=False):
"Return the media directory if exists. None if couldn't create." "Return the media directory if exists. None if couldn't create."
if self.path:
if self.mediaPrefix: if self.mediaPrefix:
dir = os.path.join( dir = os.path.join(
self.mediaPrefix, os.path.basename(self.path)) self.mediaPrefix, os.path.basename(self.path))
@ -2917,11 +2892,6 @@ where key = :key""", key=key, value=value):
except OSError: except OSError:
# permission denied # permission denied
return None return None
else:
# memory-backed; need temp store
if not self.tmpMediaDir and create:
self.tmpMediaDir = tempfile.mkdtemp(prefix="anki")
dir = self.tmpMediaDir
if not dir or not os.path.exists(dir): if not dir or not os.path.exists(dir):
return None return None
# change to the current dir # change to the current dir
@ -2958,7 +2928,6 @@ Return new path, relative to media dir."""
def close(self): def close(self):
if self.db: if self.db:
self.db.rollback() self.db.rollback()
self.db.clear()
self.db.close() self.db.close()
self.db = None self.db = None
self.s = None self.s = None
@ -2968,7 +2937,7 @@ Return new path, relative to media dir."""
def rollback(self): def rollback(self):
"Roll back the current transaction and reset session state." "Roll back the current transaction and reset session state."
self.db.rollback() self.db.rollback()
self.db.clear() self.db.expunge_all()
self.db.update(self) self.db.update(self)
self.db.refresh(self) self.db.refresh(self)
@ -2977,9 +2946,10 @@ Return new path, relative to media dir."""
self.db.flush() self.db.flush()
self.db.expire_all() self.db.expire_all()
def openSession(self): def openSession(self, first=False):
"Open a new session. Assumes old session is already closed." "Open a new session. Assumes old session is already closed."
self.db = SessionHelper(self.Session(), lock=self.needLock) self.db = SessionHelper(self.Session())
self.s = self.db
self.db.update(self) self.db.update(self)
self.refreshSession() self.refreshSession()
@ -3001,8 +2971,7 @@ Return new path, relative to media dir."""
self.modified = newTime or time.time() self.modified = newTime or time.time()
def setSchemaModified(self): def setSchemaModified(self):
# we might be called during an upgrade, so avoid bumping modtime self.schemaMod = time.time()
self.setVar("schemaMod", time.time(), mod=False)
# since we guarantee a full sync to all clients, this is a good time # since we guarantee a full sync to all clients, this is a good time
# to forget old gravestones # to forget old gravestones
for k in ("cards", "facts", "models", "media"): for k in ("cards", "facts", "models", "media"):
@ -3053,7 +3022,7 @@ Return new path, relative to media dir."""
if oldMediaDir: if oldMediaDir:
newDeck.renameMediaDir(oldMediaDir) newDeck.renameMediaDir(oldMediaDir)
# forget sync name # forget sync name
newDeck.syncName = None newDeck.syncName = u""
newDeck.db.commit() newDeck.db.commit()
# and return the new deck # and return the new deck
self.finishProgress() self.finishProgress()
@ -3069,7 +3038,7 @@ Return new path, relative to media dir."""
self.db.commit() self.db.commit()
def disableSyncing(self): def disableSyncing(self):
self.syncName = None self.syncName = u""
self.db.commit() self.db.commit()
def syncingEnabled(self): def syncingEnabled(self):
@ -3083,7 +3052,7 @@ has been disabled (ERR-0100).
You can disable this check in Settings>Preferences>Network.""") % self.name()) You can disable this check in Settings>Preferences>Network.""") % self.name())
self.disableSyncing() self.disableSyncing()
self.syncName = None self.syncName = u""
# DB maintenance # DB maintenance
########################################################################## ##########################################################################
@ -3481,6 +3450,8 @@ seq > :s and seq <= :e order by seq desc""", s=start, e=end)
if analyze: if analyze:
self.db.statement("analyze") self.db.statement("analyze")
mapper(Deck, deckTable)
# Shared decks # Shared decks
########################################################################## ##########################################################################
@ -3494,219 +3465,9 @@ sourcesTable = Table(
# not currently exposed in the GUI # not currently exposed in the GUI
Column('syncPeriod', Integer, nullable=False, default=0)) Column('syncPeriod', Integer, nullable=False, default=0))
# Maps # Labels
########################################################################## ##########################################################################
mapper(Deck, decksTable, properties={
'currentModel': relation(anki.models.Model, primaryjoin=
decksTable.c.currentModelId ==
anki.models.modelsTable.c.id),
'models': relation(anki.models.Model, post_update=True,
primaryjoin=
decksTable.c.id ==
anki.models.modelsTable.c.deckId),
})
# Deck storage
##########################################################################
numBackups = 30
backupDir = os.path.expanduser("~/.anki/backups")
class DeckStorage(object):
def Deck(path=None, backup=True, lock=True, pool=True, rebuild=True):
"Create a new deck or attach to an existing one."
create = True
if path is None:
sqlpath = None
else:
path = os.path.abspath(path)
# check if we need to init
if os.path.exists(path):
create = False
# sqlite needs utf8
sqlpath = path.encode("utf-8")
try:
(engine, session) = DeckStorage._attach(sqlpath, create, pool)
s = session()
if create:
ver = 999
metadata.create_all(engine)
deck = DeckStorage._init(s)
else:
ver = upgradeSchema(engine, s)
# add any possibly new tables if we're upgrading
if ver < DECK_VERSION:
metadata.create_all(engine)
deck = s.query(Deck).get(1)
if not deck:
raise DeckAccessError(_("Deck missing core table"),
type="nocore")
# attach db vars
deck.path = path
deck.engine = engine
deck.Session = session
deck.needLock = lock
deck.progressHandlerCalled = 0
deck.progressHandlerEnabled = False
if pool:
try:
deck.engine.raw_connection().set_progress_handler(
deck.progressHandler, 100)
except:
print "please install pysqlite 2.4 for better progress dialogs"
deck.engine.execute("pragma locking_mode = exclusive")
# db is new style; s old style
deck.db = SessionHelper(s, lock=lock)
deck.s = deck.db
# force a write lock
deck.db.execute("update decks set modified = modified")
if deck.utcOffset == -2:
# make sure we do this before initVars
DeckStorage._setUTCOffset(deck)
deck.created = time.time()
if create:
# new-style file format
deck.db.commit()
deck.db.execute("pragma legacy_file_format = off")
deck.db.execute("pragma default_cache_size= 20000")
deck.db.execute("vacuum")
# add tags and indices
initTagTables(deck.db)
updateIndices(deck)
deck.db.statement("analyze")
deck._initVars()
else:
if backup:
DeckStorage.backup(deck, path)
deck._initVars()
upgradeDeck(deck)
except OperationalError, e:
engine.dispose()
if (str(e.orig).startswith("database table is locked") or
str(e.orig).startswith("database is locked")):
raise DeckAccessError(_("File is in use by another process"),
type="inuse")
else:
raise e
if not rebuild:
# minimal startup
return deck
oldMod = deck.modified
# fix a bug with current model being unset
if not deck.currentModel and deck.models:
deck.currentModel = deck.models[0]
# ensure the necessary indices are available
deck.updateDynamicIndices()
# - new delay1 handling
if deck.delay1 > 7:
deck.delay1 = 0
# unsuspend buried/rev early
deck.db.statement(
"update cards set queue = type where queue between -3 and -2")
deck.db.commit()
# check if deck has been moved, and disable syncing
deck.checkSyncHash()
# determine starting factor for new cards
deck.averageFactor = 2.5
# rebuild queue
deck.reset()
# make sure we haven't accidentally bumped the modification time
assert deck.modified == oldMod
return deck
Deck = staticmethod(Deck)
def _attach(path, create, pool=True):
"Attach to a file, initializing DB"
if path is None:
path = "sqlite://"
else:
path = "sqlite:///" + path
if pool:
# open and lock connection for single use
engine = create_engine(path, connect_args={'timeout': 0},
strategy="threadlocal")
else:
# no pool & concurrent access w/ timeout
engine = create_engine(path,
poolclass=NullPool,
connect_args={'timeout': 60})
session = sessionmaker(bind=engine,
autoflush=False,
autocommit=True)
return (engine, session)
_attach = staticmethod(_attach)
def _init(s):
"Add a new deck to the database. Return saved deck."
deck = Deck()
if sqlalchemy.__version__.startswith("0.4."):
s.save(deck)
else:
s.add(deck)
s.flush()
return deck
_init = staticmethod(_init)
def _setUTCOffset(deck):
# 4am
deck.utcOffset = time.timezone + 60*60*4
_setUTCOffset = staticmethod(_setUTCOffset)
def backup(deck, path):
"""Path must not be unicode."""
if not numBackups:
return
def escape(path):
path = os.path.abspath(path)
path = path.replace("\\", "!")
path = path.replace("/", "!")
path = path.replace(":", "")
return path
escp = escape(path)
# make sure backup dir exists
try:
os.makedirs(backupDir)
except (OSError, IOError):
pass
# find existing backups
gen = re.sub("\.anki$", ".backup-(\d+).anki", re.escape(escp))
backups = []
for file in os.listdir(backupDir):
m = re.match(gen, file)
if m:
backups.append((int(m.group(1)), file))
backups.sort()
# check if last backup is the same
if backups:
latest = os.path.join(backupDir, backups[-1][1])
if int(deck.modified) == int(
os.stat(latest)[stat.ST_MTIME]):
return
# check integrity
if not deck.db.scalar("pragma integrity_check") == "ok":
raise DeckAccessError(_("Deck is corrupt."), type="corrupt")
# get next num
if not backups:
n = 1
else:
n = backups[-1][0] + 1
# do backup
newpath = os.path.join(backupDir, os.path.basename(
re.sub("\.anki$", ".backup-%s.anki" % n, escp)))
shutil.copy2(path, newpath)
# set mtimes to be identical
if deck.modified:
os.utime(newpath, (deck.modified, deck.modified))
# remove if over
if len(backups) + 1 > numBackups:
delete = len(backups) + 1 - numBackups
delete = backups[:delete]
for file in delete:
os.unlink(os.path.join(backupDir, file[1]))
backup = staticmethod(backup)
def newCardOrderLabels(): def newCardOrderLabels():
return { return {
0: _("Show new cards in random order"), 0: _("Show new cards in random order"),
@ -3721,6 +3482,7 @@ def newCardSchedulingLabels():
2: _("Show new cards before reviews"), 2: _("Show new cards before reviews"),
} }
# FIXME: order due is not very useful anymore
def revCardOrderLabels(): def revCardOrderLabels():
return { return {
0: _("Review cards from largest interval"), 0: _("Review cards from largest interval"),
@ -3738,3 +3500,104 @@ def failedCardOptionLabels():
4: _("Show failed cards in 3 days"), 4: _("Show failed cards in 3 days"),
5: _("Custom failed cards handling"), 5: _("Custom failed cards handling"),
} }
# Deck storage
##########################################################################
class DeckStorage(object):
def _getDeck(path, create, pool):
engine = None
try:
(engine, session) = DeckStorage._attach(path, create, pool)
s = session()
if create:
metadata.create_all(engine)
initTagTables(engine)
deck = DeckStorage._init(s)
updateIndices(engine)
engine.execute("analyze")
else:
ver = upgradeSchema(engine, s)
# add any possibly new tables if we're upgrading
if ver < DECK_VERSION:
metadata.create_all(engine)
deck = s.query(Deck).get(1)
if not deck:
raise DeckAccessError(_("Deck missing core table"),
type="nocore")
# attach db vars
deck.path = path
deck.Session = session
deck.engine = engine
# db is new style; s old style
deck.db = SessionHelper(s)
deck.s = deck.db
deck._initVars()
if not create:
upgradeDeck(deck)
return deck
except OperationalError, e:
if engine:
engine.dispose()
if (str(e.orig).startswith("database table is locked") or
str(e.orig).startswith("database is locked")):
raise DeckAccessError(_("File is in use by another process"),
type="inuse")
else:
raise e
_getDeck = staticmethod(_getDeck)
def _attach(path, create, pool=True):
"Attach to a file, maybe initializing DB"
path = "sqlite:///" + path.encode("utf-8")
if pool:
# open and lock connection for single use
engine = create_engine(
path, connect_args={'timeout': 0}, strategy="threadlocal")
else:
# no pool & concurrent access w/ timeout
engine = create_engine(
path, poolclass=NullPool, connect_args={'timeout': 60})
session = sessionmaker(bind=engine, autoflush=False, autocommit=True)
if create:
engine.execute("pragma page_size = 4096")
engine.execute("pragma legacy_file_format = 0")
engine.execute("vacuum")
engine.execute("pragma cache_size = 20000")
return (engine, session)
_attach = staticmethod(_attach)
def _init(s):
"Add a new deck to the database. Return saved deck."
deck = Deck()
if sqlalchemy.__version__.startswith("0.4."):
s.save(deck)
else:
s.add(deck)
s.flush()
return deck
_init = staticmethod(_init)
def Deck(path, backup=True, pool=True, rebuild=True):
"Create a new deck or attach to an existing one. Path should be unicode."
path = os.path.abspath(path)
create = not os.path.exists(path)
deck = DeckStorage._getDeck(path, create, pool)
if not rebuild:
# minimal startup
return deck
oldMod = deck.modified
# unsuspend buried/rev early
deck.db.statement(
"update cards set queue = type where queue between -3 and -2")
deck.db.commit()
# check if deck has been moved, and disable syncing
deck.checkSyncHash()
# rebuild queue
deck.reset()
# make sure we haven't accidentally bumped the modification time
assert deck.modified == oldMod
return deck
Deck = staticmethod(Deck)

34
anki/models.py
View file

@ -153,19 +153,39 @@ def formatQA(cid, mid, fact, tags, cm, deck):
# Model table # Model table
########################################################################## ##########################################################################
# ...Schedule: fail, pass1, pass2, etc in minutes
# ...Intervals: graduation, first remove, later remove
# maybe define a random cutoff at say +/-30% which controls exit interval
# variation - 30% of 1 day is 0.7 or 1.3 so always 1 day; 30% of 4 days is
# 2.8-5.2, so any time from 3-5 days is acceptable
# collapse time should be bigger than default failSchedule
# need to think about failed interval handling - if the final drill is
# optional, what intervals should the default be? 3 days or more if cards are
# over that interval range? and what about failed mature bonus?
modelsTable = Table( modelsTable = Table(
'models', metadata, 'models', metadata,
Column('id', Integer, primary_key=True), Column('id', Integer, primary_key=True),
Column('deckId', Integer, ForeignKey("decks.id", use_alter=True, name="deckIdfk")),
Column('created', Float, nullable=False, default=time.time), Column('created', Float, nullable=False, default=time.time),
Column('modified', Float, nullable=False, default=time.time), Column('modified', Float, nullable=False, default=time.time),
Column('tags', UnicodeText, nullable=False, default=u""),
Column('name', UnicodeText, nullable=False), Column('name', UnicodeText, nullable=False),
Column('description', UnicodeText, nullable=False, default=u""), # obsolete # new cards
Column('features', UnicodeText, nullable=False, default=u""), # used as mediaURL Column('newSched', UnicodeText, nullable=False, default=u"[0.5, 3, 10]"),
Column('spacing', Float, nullable=False, default=0.1), # obsolete Column('newInts', UnicodeText, nullable=False, default=u"[1, 7, 4]"),
Column('initialSpacing', Float, nullable=False, default=60), # obsolete # failed cards
Column('source', Integer, nullable=False, default=0)) Column('failSched', UnicodeText, nullable=False, default=u"[0.5, 3, 10]"),
Column('failInts', UnicodeText, nullable=False, default=u"[1, 7, 4]"),
Column('failMult', Float, nullable=False, default=0),
# other scheduling
Column('initialFactor', Float, nullable=False, default=2.5),
)
class Model(object): class Model(object):
"Defines the way a fact behaves, what fields it can contain, etc." "Defines the way a fact behaves, what fields it can contain, etc."

114
anki/upgrade.py
View file

@ -2,8 +2,9 @@
# Copyright: Damien Elmes <anki@ichi2.net> # Copyright: Damien Elmes <anki@ichi2.net>
# License: GNU GPL, version 3 or later; http://www.gnu.org/copyleft/gpl.html # License: GNU GPL, version 3 or later; http://www.gnu.org/copyleft/gpl.html
DECK_VERSION = 75 DECK_VERSION = 100
import time
from anki.db import * from anki.db import *
from anki.lang import _ from anki.lang import _
from anki.media import rebuildMediaDir from anki.media import rebuildMediaDir
@ -19,16 +20,17 @@ def moveTable(s, table):
def upgradeSchema(engine, s): def upgradeSchema(engine, s):
"Alter tables prior to ORM initialization." "Alter tables prior to ORM initialization."
ver = s.scalar("select version from decks limit 1")
# add a checksum to fields
if ver < 71:
try: try:
s.execute( ver = s.scalar("select version from deck limit 1")
"alter table fields add column chksum text "+
"not null default ''")
except: except:
pass ver = s.scalar("select version from decks limit 1")
if ver < 75: if ver < 65:
raise Exception("oldDeckVersion")
if ver < 99:
# fields
###########
s.execute(
"alter table fields add column chksum text not null default ''")
# cards # cards
########### ###########
moveTable(s, "cards") moveTable(s, "cards")
@ -67,48 +69,70 @@ spaceUntil from facts2""")
insert or ignore into media select id, filename, size, created, insert or ignore into media select id, filename, size, created,
originalPath from media2""") originalPath from media2""")
s.execute("drop table media2") s.execute("drop table media2")
# deck
###########
import deck
metadata.create_all(engine, tables=[deck.deckTable])
s.execute("""
insert into deck select id, created, modified, 0, 99, currentModelId,
ifnull(syncName, ""), lastSync, utcOffset, newCardOrder,
newCardSpacing, newCardsPerDay, revCardOrder, 600, sessionRepLimit,
sessionTimeLimit, 1, 16, '', '', '', '' from decks
""")
s.execute("drop table decks")
# models
###########
moveTable(s, "models")
import models
metadata.create_all(engine, tables=[models.modelsTable])
s.execute("""
insert or ignore into models select id, created, modified, name,
'[0.5, 3, 10]', '[1, 7, 4]',
'[0.5, 3, 10]', '[1, 7, 4]',
0, 2.5 from models2""")
s.execute("drop table models2")
return ver return ver
def updateIndices(deck): def updateIndices(db):
"Add indices to the DB." "Add indices to the DB."
# due counts, failed card queue # due counts, failed card queue
deck.db.statement(""" db.execute("""
create index if not exists ix_cards_queueDue on cards create index if not exists ix_cards_queueDue on cards
(queue, due, factId)""") (queue, due, factId)""")
# counting cards of a given type # counting cards of a given type
deck.db.statement(""" db.execute("""
create index if not exists ix_cards_type on cards create index if not exists ix_cards_type on cards
(type)""") (type)""")
# sync summaries # sync summaries
deck.db.statement(""" db.execute("""
create index if not exists ix_cards_modified on cards create index if not exists ix_cards_modified on cards
(modified)""") (modified)""")
deck.db.statement(""" db.execute("""
create index if not exists ix_facts_modified on facts create index if not exists ix_facts_modified on facts
(modified)""") (modified)""")
# card spacing # card spacing
deck.db.statement(""" db.execute("""
create index if not exists ix_cards_factId on cards (factId)""") create index if not exists ix_cards_factId on cards (factId)""")
# fields # fields
deck.db.statement(""" db.execute("""
create index if not exists ix_fields_factId on fields (factId)""") create index if not exists ix_fields_factId on fields (factId)""")
deck.db.statement(""" db.execute("""
create index if not exists ix_fields_chksum on fields (chksum)""") create index if not exists ix_fields_chksum on fields (chksum)""")
# media # media
deck.db.statement(""" db.execute("""
create index if not exists ix_media_chksum on media (chksum)""") create index if not exists ix_media_chksum on media (chksum)""")
# deletion tracking # deletion tracking
deck.db.statement(""" db.execute("""
create index if not exists ix_cardsDeleted_cardId on cardsDeleted (cardId)""") create index if not exists ix_cardsDeleted_cardId on cardsDeleted (cardId)""")
deck.db.statement(""" db.execute("""
create index if not exists ix_modelsDeleted_modelId on modelsDeleted (modelId)""") create index if not exists ix_modelsDeleted_modelId on modelsDeleted (modelId)""")
deck.db.statement(""" db.execute("""
create index if not exists ix_factsDeleted_factId on factsDeleted (factId)""") create index if not exists ix_factsDeleted_factId on factsDeleted (factId)""")
deck.db.statement(""" db.execute("""
create index if not exists ix_mediaDeleted_factId on mediaDeleted (mediaId)""") create index if not exists ix_mediaDeleted_factId on mediaDeleted (mediaId)""")
# tags # tags
deck.db.statement(""" db.execute("""
create index if not exists ix_cardTags_cardId on cardTags (cardId)""") create index if not exists ix_cardTags_cardId on cardTags (cardId)""")
def upgradeDeck(deck): def upgradeDeck(deck):
@ -117,17 +141,10 @@ def upgradeDeck(deck):
prog = True prog = True
deck.startProgress() deck.startProgress()
deck.updateProgress(_("Upgrading Deck...")) deck.updateProgress(_("Upgrading Deck..."))
if deck.utcOffset == -1:
# we're opening a shared deck with no indices - we'll need
# them if we want to rebuild the queue
updateIndices(deck)
oldmod = deck.modified oldmod = deck.modified
else: else:
prog = False prog = False
if deck.version < 65: if deck.version < 100:
raise Exception("oldDeckVersion")
# skip a few to allow for updates to stable tree
if deck.version < 70:
# update dynamic indices given we don't use priority anymore # update dynamic indices given we don't use priority anymore
for d in ("intervalDesc", "intervalAsc", "randomOrder", for d in ("intervalDesc", "intervalAsc", "randomOrder",
"dueAsc", "dueDesc"): "dueAsc", "dueDesc"):
@ -139,29 +156,14 @@ def upgradeDeck(deck):
"revCardsDue", "revCardsRandom", "acqCardsRandom", "revCardsDue", "revCardsRandom", "acqCardsRandom",
"acqCardsOld", "acqCardsNew"): "acqCardsOld", "acqCardsNew"):
deck.db.statement("drop view if exists %s" % v) deck.db.statement("drop view if exists %s" % v)
deck.version = 70
deck.db.commit()
if deck.version < 71:
# remove the expensive value cache # remove the expensive value cache
deck.db.statement("drop index if exists ix_fields_value") deck.db.statement("drop index if exists ix_fields_value")
# add checksums and index # add checksums and index
deck.updateAllFieldChecksums() deck.updateAllFieldChecksums()
updateIndices(deck)
deck.db.execute("vacuum")
deck.db.execute("analyze")
deck.version = 71
deck.db.commit()
if deck.version < 72:
# this was only used for calculating average factor # this was only used for calculating average factor
deck.db.statement("drop index if exists ix_cards_factor") deck.db.statement("drop index if exists ix_cards_factor")
deck.version = 72
deck.db.commit()
if deck.version < 73:
# remove stats, as it's all in the revlog now # remove stats, as it's all in the revlog now
deck.db.statement("drop table if exists stats") deck.db.statement("drop table if exists stats")
deck.version = 73
deck.db.commit()
if deck.version < 74:
# migrate revlog data to new table # migrate revlog data to new table
deck.db.statement(""" deck.db.statement("""
insert or ignore into revlog select insert or ignore into revlog select
@ -172,28 +174,20 @@ cast(min(thinkingTime, 60)*1000 as int), 0 from reviewHistory""")
deck.db.statement("update revlog set ease = 1 where ease = 0") deck.db.statement("update revlog set ease = 1 where ease = 0")
# remove priority index # remove priority index
deck.db.statement("drop index if exists ix_cards_priority") deck.db.statement("drop index if exists ix_cards_priority")
deck.version = 74
deck.db.commit()
if deck.version < 75:
# suspended cards don't use ranges anymore # suspended cards don't use ranges anymore
deck.db.execute("update cards set queue=-1 where queue between -3 and -1") deck.db.execute("update cards set queue=-1 where queue between -3 and -1")
deck.db.execute("update cards set queue=-2 where queue between 3 and 5") deck.db.execute("update cards set queue=-2 where queue between 3 and 5")
deck.db.execute("update cards set queue=-3 where queue between 6 and 8") deck.db.execute("update cards set queue=-3 where queue between 6 and 8")
# don't need an index on fieldModelId # don't need an index on fieldModelId
deck.db.statement("drop index if exists ix_fields_fieldModelId") deck.db.statement("drop index if exists ix_fields_fieldModelId")
# new indices for new cards table # update schema time
updateIndices(deck) deck.db.statement("update deck set schemaMod = :t", t=time.time())
deck.version = 75
deck.db.commit()
# executing a pragma here is very slow on large decks, so we store # finally, update indices & optimize
# our own record updateIndices(deck.db)
if not deck.getInt("pageSize") == 4096:
deck.db.commit()
deck.db.execute("pragma page_size = 4096")
deck.db.execute("pragma legacy_file_format = 0")
deck.db.execute("vacuum") deck.db.execute("vacuum")
deck.setVar("pageSize", 4096, mod=False) deck.db.execute("analyze")
deck.version = 100
deck.db.commit() deck.db.commit()
if prog: if prog:
assert deck.modified == oldmod assert deck.modified == oldmod

29
tests/test_deck.py
View file

@ -15,13 +15,13 @@ newModified = None
testDir = os.path.dirname(__file__) testDir = os.path.dirname(__file__)
## opening/closing def getDeck():
import tempfile
(fd, nam) = tempfile.mkstemp(suffix=".anki")
os.unlink(nam)
return DeckStorage.Deck(nam)
def test_new(): ## opening/closing
deck = DeckStorage.Deck()
assert not deck.path
assert deck.engine
assert deck.modified
def test_attachNew(): def test_attachNew():
global newPath, newModified global newPath, newModified
@ -33,7 +33,6 @@ def test_attachNew():
deck = DeckStorage.Deck(path) deck = DeckStorage.Deck(path)
# for attachOld() # for attachOld()
newPath = deck.path newPath = deck.path
deck.setVar("pageSize", 4096)
deck.save() deck.save()
newModified = deck.modified newModified = deck.modified
deck.close() deck.close()
@ -67,7 +66,7 @@ def test_saveAs():
except OSError: except OSError:
pass pass
# start with an in-memory deck # start with an in-memory deck
deck = DeckStorage.Deck() deck = getDeck()
deck.addModel(BasicModel()) deck.addModel(BasicModel())
# add a card # add a card
f = deck.newFact() f = deck.newFact()
@ -90,7 +89,7 @@ def test_saveAs():
newDeck.close() newDeck.close()
def test_factAddDelete(): def test_factAddDelete():
deck = DeckStorage.Deck() deck = getDeck()
deck.addModel(BasicModel()) deck.addModel(BasicModel())
# set rollback point # set rollback point
deck.db.commit() deck.db.commit()
@ -130,7 +129,7 @@ def test_factAddDelete():
deck.deleteCard(id2) deck.deleteCard(id2)
def test_fieldChecksum(): def test_fieldChecksum():
deck = DeckStorage.Deck() deck = getDeck()
deck.addModel(BasicModel()) deck.addModel(BasicModel())
f = deck.newFact() f = deck.newFact()
f['Front'] = u"new"; f['Back'] = u"new2" f['Front'] = u"new"; f['Back'] = u"new2"
@ -165,7 +164,7 @@ def test_fieldChecksum():
"select chksum from fields where id = :id", id=id) == "" "select chksum from fields where id = :id", id=id) == ""
def test_modelAddDelete(): def test_modelAddDelete():
deck = DeckStorage.Deck() deck = getDeck()
deck.addModel(BasicModel()) deck.addModel(BasicModel())
deck.addModel(BasicModel()) deck.addModel(BasicModel())
f = deck.newFact() f = deck.newFact()
@ -179,7 +178,7 @@ def test_modelAddDelete():
deck.db.refresh(deck) deck.db.refresh(deck)
def test_modelCopy(): def test_modelCopy():
deck = DeckStorage.Deck() deck = getDeck()
m = BasicModel() m = BasicModel()
assert len(m.fieldModels) == 2 assert len(m.fieldModels) == 2
assert len(m.cardModels) == 2 assert len(m.cardModels) == 2
@ -199,7 +198,7 @@ def test_modelCopy():
assert len(m2.cardModels) == 2 assert len(m2.cardModels) == 2
def test_media(): def test_media():
deck = DeckStorage.Deck() deck = getDeck()
# create a media dir # create a media dir
deck.mediaDir(create=True) deck.mediaDir(create=True)
# put a file into it # put a file into it
@ -216,7 +215,7 @@ def test_media():
assert os.path.exists("/tmp/saveAs2.media/%s" % sum) assert os.path.exists("/tmp/saveAs2.media/%s" % sum)
def test_modelChange(): def test_modelChange():
deck = DeckStorage.Deck() deck = getDeck()
m = Model(u"Japanese") m = Model(u"Japanese")
m1 = m m1 = m
f = FieldModel(u'Expression', True, True) f = FieldModel(u'Expression', True, True)
@ -270,7 +269,7 @@ select question, answer from cards where factId = :id""",
assert stripHTML(a) == u"r" assert stripHTML(a) == u"r"
def test_findCards(): def test_findCards():
deck = DeckStorage.Deck() deck = getDeck()
deck.addModel(BasicModel()) deck.addModel(BasicModel())
f = deck.newFact() f = deck.newFact()
f['Front'] = u'dog' f['Front'] = u'dog'

10
tests/test_media.py
View file

@ -6,6 +6,12 @@ from anki import DeckStorage
from anki.stdmodels import BasicModel from anki.stdmodels import BasicModel
from anki.utils import checksum from anki.utils import checksum
def getDeck():
import tempfile
(fd, nam) = tempfile.mkstemp(suffix=".anki")
os.unlink(nam)
return DeckStorage.Deck(nam)
# uniqueness check # uniqueness check
def test_unique(): def test_unique():
dir = tempfile.mkdtemp(prefix="anki") dir = tempfile.mkdtemp(prefix="anki")
@ -26,7 +32,7 @@ def test_unique():
# copying files to media folder # copying files to media folder
def test_copy(): def test_copy():
deck = DeckStorage.Deck() deck = getDeck()
dir = tempfile.mkdtemp(prefix="anki") dir = tempfile.mkdtemp(prefix="anki")
path = os.path.join(dir, "foo.jpg") path = os.path.join(dir, "foo.jpg")
open(path, "w").write("hello") open(path, "w").write("hello")
@ -42,7 +48,7 @@ insert into media values (null, 'foo.jpg', 0, 0, :sum)""",
# media db # media db
def test_db(): def test_db():
deck = DeckStorage.Deck() deck = getDeck()
deck.addModel(BasicModel()) deck.addModel(BasicModel())
dir = tempfile.mkdtemp(prefix="anki") dir = tempfile.mkdtemp(prefix="anki")
path = os.path.join(dir, "foo.jpg") path = os.path.join(dir, "foo.jpg")