mirror of
https://github.com/ankitects/anki.git
synced 2025-09-19 22:42:25 -04:00
1078285f0f
Since Anki first moved to an SQL backend, it has stored fields in a fields table, with one field per line. This is a natural layout in a relational database, and it had some nice properties. It meant we could retrieve an individual field of a fact, which we used for limiting searches to a particular field, for sorting, and for determining if a field was unique, by adding an index on the field value. The index was very expensive, so as part of the early work towards 2.0 I added a checksum field instead, and added an index to that. This was a lot cheaper than storing the entire value twice for the purpose of fast searches, but it only partly solved the problem. We still needed an index on factId so that we could retrieve a given fact's fields quickly. For simple models this was fairly cheap, but as the number of fields grows the table grows very big. 25k facts with 30 fields each and the fields table has grown to 750k entries. This makes the factId index and checksum index really expensive - with the q/a cache removed, about 30% of the deck in such a situation. Equally problematic was sorting on those fields. Short of adding another expensive index, a sort involves a table scan of the entire table. We solve these problems by moving all fields into the facts table. For this to work, we need to address some issues: Sorting: we'll add an option to the model to specify the sort field. When facts are modified, that field is written to a separate sort column. It can be HTML stripped, and possibly truncated to a maximum number of letters. This means that switching sort to a different field involves an expensive rewrite of the sort column, but people tend to leave their sort field set to the same value, and we don't need to clear the field if the user switches temporarily to a non-field sort like due order. And it has the nice properties of allowing different models to be sorted on different columns at the same time, and makes it impossible for models to be hidden because the user has sorted on a field which doesn't appear in some models. Searching for words with embedded HTML: 1.2 introduced a HTML-stripped cache of the fields content, which both sped up searches (since we didn't have to search the possibly large fields table), and meant we could find "bob" in "b<b>ob</b>" quickly. The ability to quickly search for words peppered with HTML was nice, but it meant doubling the cost of storing text in many cases, and meant after any edit more data has to be written to the DB. Instead, we'll do it on the fly. On this i7 computer, stripping HTML from all fields takes 1-2.6 seconds on 25-50k decks. We could possibly skip the stripping for people who don't require it - the number of people who bold parts of words is actually pretty small. Duplicate detection: one option would be to fetch all fields when the add cards dialog or editor are opened. But this will be expensive on mobile devices. Instead, we'll create a separate table of (fid, csum), with an index on both columns. When we edit a fact, we delete all the existing checksums for that fact, and add checksums for any fields that must be checked as unique. We could optionally skip the index on csum - some benchmarking is required. As for the new table layout, creating separate columns for each field won't scale. Instead, we store the fields in a single column, separated by an ascii record separator. We split on that character when extracting from the database, and join on it when writing to the DB. Searching on a particular field in the browser will be accomplished by finding all facts that match, and then unpacking to see if the relevant field matched. Tags have been moved back to a separate column. Now that fields are on the facts table, there is no need to pack them in as a field simply to avoid another table hit.
687 lines
26 KiB
Python
687 lines
26 KiB
Python
# -*- coding: utf-8 -*-
|
|
# Copyright: Damien Elmes <anki@ichi2.net>
|
|
# License: GNU GPL, version 3 or later; http://www.gnu.org/copyleft/gpl.html
|
|
|
|
# A lot of findCards() and related functions was contributed by
|
|
# Marcus.
|
|
|
|
import re
|
|
from anki.utils import ids2str
|
|
|
|
SEARCH_TAG = 0
|
|
SEARCH_TYPE = 1
|
|
SEARCH_PHRASE = 2
|
|
SEARCH_FID = 3
|
|
SEARCH_CARD = 4
|
|
SEARCH_DISTINCT = 5
|
|
SEARCH_FIELD = 6
|
|
SEARCH_FIELD_EXISTS = 7
|
|
SEARCH_QA = 8
|
|
SEARCH_PHRASE_WB = 9
|
|
|
|
# Find
|
|
##########################################################################
|
|
|
|
def findCards(deck, query):
|
|
(q, cmquery, showdistinct, filters, args) = findCardsWhere(deck, query)
|
|
(fidList, cardIdList) = findCardsMatchingFilters(deck, filters)
|
|
query = "select id from cards"
|
|
hasWhere = False
|
|
if q:
|
|
query += " where " + q
|
|
hasWhere = True
|
|
if cmquery['pos'] or cmquery['neg']:
|
|
if hasWhere is False:
|
|
query += " where "
|
|
hasWhere = True
|
|
else: query += " and "
|
|
if cmquery['pos']:
|
|
query += (" fid in(select distinct fid from cards "+
|
|
"where id in (" + cmquery['pos'] + ")) ")
|
|
query += " and id in(" + cmquery['pos'] + ") "
|
|
if cmquery['neg']:
|
|
query += (" fid not in(select distinct fid from "+
|
|
"cards where id in (" + cmquery['neg'] + ")) ")
|
|
if fidList is not None:
|
|
if hasWhere is False:
|
|
query += " where "
|
|
hasWhere = True
|
|
else: query += " and "
|
|
query += " fid IN %s" % ids2str(fidList)
|
|
if cardIdList is not None:
|
|
if hasWhere is False:
|
|
query += " where "
|
|
hasWhere = True
|
|
else: query += " and "
|
|
query += " id IN %s" % ids2str(cardIdList)
|
|
if showdistinct:
|
|
query += " group by fid"
|
|
#print query, args
|
|
return deck.db.list(query, **args)
|
|
|
|
def findCardsWhere(deck, query):
|
|
(tquery, fquery, qquery, fidquery, cmquery, sfquery, qaquery,
|
|
showdistinct, filters, args) = _findCards(deck, query)
|
|
q = ""
|
|
x = []
|
|
if tquery:
|
|
x.append(" fid in (%s)" % tquery)
|
|
if fquery:
|
|
x.append(" fid in (%s)" % fquery)
|
|
if qquery:
|
|
x.append(" id in (%s)" % qquery)
|
|
if fidquery:
|
|
x.append(" id in (%s)" % fidquery)
|
|
if sfquery:
|
|
x.append(" fid in (%s)" % sfquery)
|
|
if qaquery:
|
|
x.append(" id in (%s)" % qaquery)
|
|
if x:
|
|
q += " and ".join(x)
|
|
return q, cmquery, showdistinct, filters, args
|
|
|
|
def allFMFields(deck, tolower=False):
|
|
fields = []
|
|
try:
|
|
fields = deck.db.list(
|
|
"select distinct name from fieldmodels order by name")
|
|
except:
|
|
fields = []
|
|
if tolower is True:
|
|
for i, v in enumerate(fields):
|
|
fields[i] = v.lower()
|
|
return fields
|
|
|
|
def _parseQuery(deck, query):
|
|
tokens = []
|
|
res = []
|
|
|
|
allowedfields = allFMFields(deck, True)
|
|
def addSearchFieldToken(field, value, isNeg, filter):
|
|
if field.lower() in allowedfields:
|
|
res.append((field + ':' + value, isNeg, SEARCH_FIELD, filter))
|
|
elif field in ['question', 'answer']:
|
|
res.append((field + ':' + value, isNeg, SEARCH_QA, filter))
|
|
else:
|
|
for p in phraselog:
|
|
res.append((p['value'], p['is_neg'], p['type'], p['filter']))
|
|
# break query into words or phraselog
|
|
# an extra space is added so the loop never ends in the middle
|
|
# completing a token
|
|
for match in re.findall(
|
|
r'(-)?\'(([^\'\\]|\\.)*)\'|(-)?"(([^"\\]|\\.)*)"|(-)?([^ ]+)|([ ]+)',
|
|
query + ' '):
|
|
type = ' '
|
|
if match[1]: type = "'"
|
|
elif match[4]: type = '"'
|
|
|
|
value = (match[1] or match[4] or match[7])
|
|
isNeg = (match[0] == '-' or match[3] == '-' or match[6] == '-')
|
|
|
|
tokens.append({'type': type, 'value': value, 'is_neg': isNeg,
|
|
'filter': ('wb' if type == "'" else 'none')})
|
|
intoken = isNeg = False
|
|
field = '' #name of the field for field related commands
|
|
phraselog = [] #log of phrases in case potential command is not a commad
|
|
for c, token in enumerate(tokens):
|
|
doprocess = True # only look for commands when this is true
|
|
#prevent cases such as "field" : value as being processed as a command
|
|
if len(token['value']) == 0:
|
|
if intoken is True and type == SEARCH_FIELD and field:
|
|
#case: fieldname: any thing here check for existance of fieldname
|
|
addSearchFieldToken(field, '*', isNeg, 'none')
|
|
phraselog = [] # reset phrases since command is completed
|
|
intoken = doprocess = False
|
|
if intoken is True:
|
|
if type == SEARCH_FIELD_EXISTS:
|
|
#case: field:"value"
|
|
res.append((token['value'], isNeg, type, 'none'))
|
|
intoken = doprocess = False
|
|
elif type == SEARCH_FIELD and field:
|
|
#case: fieldname:"value"
|
|
addSearchFieldToken(
|
|
field, token['value'], isNeg, token['filter'])
|
|
intoken = doprocess = False
|
|
|
|
elif type == SEARCH_FIELD and not field:
|
|
#case: "fieldname":"name" or "field" anything
|
|
if token['value'].startswith(":") and len(phraselog) == 1:
|
|
#we now know a colon is next, so mark it as field
|
|
# and keep looking for the value
|
|
field = phraselog[0]['value']
|
|
parts = token['value'].split(':', 1)
|
|
phraselog.append(
|
|
{'value': token['value'], 'is_neg': False,
|
|
'type': SEARCH_PHRASE, 'filter': token['filter']})
|
|
if parts[1]:
|
|
#value is included with the :, so wrap it up
|
|
addSearchFieldToken(field, parts[1], isNeg, 'none')
|
|
intoken = doprocess = False
|
|
doprocess = False
|
|
else:
|
|
#case: "fieldname"string/"fieldname"tag:name
|
|
intoken = False
|
|
if intoken is False and doprocess is False:
|
|
#command has been fully processed
|
|
phraselog = [] # reset phraselog, since we used it for a command
|
|
if intoken is False:
|
|
#include any non-command related phrases in the query
|
|
for p in phraselog: res.append(
|
|
(p['value'], p['is_neg'], p['type'], p['filter']))
|
|
phraselog = []
|
|
if intoken is False and doprocess is True:
|
|
field = ''
|
|
isNeg = token['is_neg']
|
|
if token['value'].startswith("tag:"):
|
|
token['value'] = token['value'][4:]
|
|
type = SEARCH_TAG
|
|
elif token['value'].startswith("is:"):
|
|
token['value'] = token['value'][3:].lower()
|
|
type = SEARCH_TYPE
|
|
elif token['value'].startswith("fid:") and len(token['value']) > 4:
|
|
dec = token['value'][4:]
|
|
try:
|
|
int(dec)
|
|
token['value'] = token['value'][4:]
|
|
except:
|
|
try:
|
|
for d in dec.split(","):
|
|
int(d)
|
|
token['value'] = token['value'][4:]
|
|
except:
|
|
token['value'] = "0"
|
|
type = SEARCH_FID
|
|
elif token['value'].startswith("card:"):
|
|
token['value'] = token['value'][5:]
|
|
type = SEARCH_CARD
|
|
elif token['value'].startswith("show:"):
|
|
token['value'] = token['value'][5:].lower()
|
|
type = SEARCH_DISTINCT
|
|
elif token['value'].startswith("field:"):
|
|
type = SEARCH_FIELD_EXISTS
|
|
parts = token['value'][6:].split(':', 1)
|
|
field = parts[0]
|
|
if len(parts) == 1 and parts[0]:
|
|
token['value'] = parts[0]
|
|
elif len(parts) == 1 and not parts[0]:
|
|
intoken = True
|
|
else:
|
|
type = SEARCH_FIELD
|
|
intoken = True
|
|
parts = token['value'].split(':', 1)
|
|
|
|
phraselog.append(
|
|
{'value': token['value'], 'is_neg': isNeg,
|
|
'type': SEARCH_PHRASE, 'filter': token['filter']})
|
|
if len(parts) == 2 and parts[0]:
|
|
field = parts[0]
|
|
if parts[1]:
|
|
#simple fieldname:value case - no need to look for more data
|
|
addSearchFieldToken(field, parts[1], isNeg, 'none')
|
|
intoken = doprocess = False
|
|
|
|
if intoken is False: phraselog = []
|
|
if intoken is False and doprocess is True:
|
|
res.append((token['value'], isNeg, type, token['filter']))
|
|
return res
|
|
|
|
def findCardsMatchingFilters(deck, filters):
|
|
factFilters = []
|
|
fieldFilters = {}
|
|
cardFilters = {}
|
|
|
|
factFilterMatches = []
|
|
fieldFilterMatches = []
|
|
cardFilterMatches = []
|
|
|
|
if (len(filters) > 0):
|
|
for filter in filters:
|
|
if filter['scope'] == 'fact':
|
|
regexp = re.compile(
|
|
r'\b' + re.escape(filter['value']) + r'\b', flags=re.I)
|
|
factFilters.append(
|
|
{'value': filter['value'], 'regexp': regexp,
|
|
'is_neg': filter['is_neg']})
|
|
if filter['scope'] == 'field':
|
|
fieldName = filter['field'].lower()
|
|
if (fieldName in fieldFilters) is False:
|
|
fieldFilters[fieldName] = []
|
|
regexp = re.compile(
|
|
r'\b' + re.escape(filter['value']) + r'\b', flags=re.I)
|
|
fieldFilters[fieldName].append(
|
|
{'value': filter['value'], 'regexp': regexp,
|
|
'is_neg': filter['is_neg']})
|
|
if filter['scope'] == 'card':
|
|
fieldName = filter['field'].lower()
|
|
if (fieldName in cardFilters) is False:
|
|
cardFilters[fieldName] = []
|
|
regexp = re.compile(r'\b' + re.escape(filter['value']) +
|
|
r'\b', flags=re.I)
|
|
cardFilters[fieldName].append(
|
|
{'value': filter['value'], 'regexp': regexp,
|
|
'is_neg': filter['is_neg']})
|
|
|
|
if len(factFilters) > 0:
|
|
fquery = ''
|
|
args = {}
|
|
for filter in factFilters:
|
|
c = len(args)
|
|
if fquery:
|
|
if filter['is_neg']: fquery += " except "
|
|
else: fquery += " intersect "
|
|
elif filter['is_neg']: fquery += "select id from fdata except "
|
|
|
|
value = filter['value'].replace("*", "%")
|
|
args["_ff_%d" % c] = "%"+value+"%"
|
|
|
|
fquery += (
|
|
"select id from fdata where value like "+
|
|
":_ff_%d escape '\\'" % c)
|
|
|
|
rows = deck.db.execute(
|
|
'select fid, value from fdata where id in (' +
|
|
fquery + ')', args)
|
|
while (1):
|
|
row = rows.fetchone()
|
|
if row is None: break
|
|
doesMatch = False
|
|
for filter in factFilters:
|
|
res = filter['regexp'].search(row[1])
|
|
if ((filter['is_neg'] is False and res) or
|
|
(filter['is_neg'] is True and res is None)):
|
|
factFilterMatches.append(row[0])
|
|
|
|
if len(fieldFilters) > 0:
|
|
sfquery = ''
|
|
args = {}
|
|
for field, filters in fieldFilters.iteritems():
|
|
for filter in filters:
|
|
c = len(args)
|
|
if sfquery:
|
|
if filter['is_neg']: sfquery += " except "
|
|
else: sfquery += " intersect "
|
|
elif filter['is_neg']: sfquery += "select id from fdata except "
|
|
field = field.replace("*", "%")
|
|
value = filter['value'].replace("*", "%")
|
|
args["_ff_%d" % c] = "%"+value+"%"
|
|
|
|
ids = deck.db.list(
|
|
"select id from fieldmodels where name like "+
|
|
":field escape '\\'", field=field)
|
|
sfquery += ("select id from fdata where "+
|
|
"fmid in %s and value like "+
|
|
":_ff_%d escape '\\'") % (ids2str(ids), c)
|
|
|
|
rows = deck.db.execute(
|
|
'select f.fid, f.value, fm.name from fdata as f '+
|
|
'left join fieldmodels as fm ON (f.fmid = '+
|
|
'fm.id) where f.id in (' + sfquery + ')', args)
|
|
while (1):
|
|
row = rows.fetchone()
|
|
if row is None: break
|
|
field = row[2].lower()
|
|
doesMatch = False
|
|
if field in fieldFilters:
|
|
for filter in fieldFilters[field]:
|
|
res = filter['regexp'].search(row[1])
|
|
if ((filter['is_neg'] is False and res) or
|
|
(filter['is_neg'] is True and res is None)):
|
|
fieldFilterMatches.append(row[0])
|
|
|
|
|
|
if len(cardFilters) > 0:
|
|
qaquery = ''
|
|
args = {}
|
|
for field, filters in cardFilters.iteritems():
|
|
for filter in filters:
|
|
c = len(args)
|
|
if qaquery:
|
|
if filter['is_neg']: qaquery += " except "
|
|
else: qaquery += " intersect "
|
|
elif filter['is_neg']: qaquery += "select id from cards except "
|
|
value = value.replace("*", "%")
|
|
args["_ff_%d" % c] = "%"+value+"%"
|
|
|
|
if field == 'question':
|
|
qaquery += "select id from cards where question "
|
|
qaquery += "like :_ff_%d escape '\\'" % c
|
|
else:
|
|
qaquery += "select id from cards where answer "
|
|
qaquery += "like :_ff_%d escape '\\'" % c
|
|
|
|
rows = deck.db.execute(
|
|
'select id, question, answer from cards where id IN (' +
|
|
qaquery + ')', args)
|
|
while (1):
|
|
row = rows.fetchone()
|
|
if row is None: break
|
|
doesMatch = False
|
|
if field in cardFilters:
|
|
rowValue = row[1] if field == 'question' else row[2]
|
|
for filter in cardFilters[field]:
|
|
res = filter['regexp'].search(rowValue)
|
|
if ((filter['is_neg'] is False and res) or
|
|
(filter['is_neg'] is True and res is None)):
|
|
cardFilterMatches.append(row[0])
|
|
|
|
fids = None
|
|
if len(factFilters) > 0 or len(fieldFilters) > 0:
|
|
fids = []
|
|
fids.extend(factFilterMatches)
|
|
fids.extend(fieldFilterMatches)
|
|
|
|
cardIds = None
|
|
if len(cardFilters) > 0:
|
|
cardIds = []
|
|
cardIds.extend(cardFilterMatches)
|
|
|
|
return (fids, cardIds)
|
|
|
|
def _findCards(deck, query):
|
|
"Find facts matching QUERY."
|
|
tquery = ""
|
|
fquery = ""
|
|
qquery = ""
|
|
fidquery = ""
|
|
cmquery = { 'pos': '', 'neg': '' }
|
|
sfquery = qaquery = ""
|
|
showdistinct = False
|
|
filters = []
|
|
args = {}
|
|
for c, (token, isNeg, type, filter) in enumerate(_parseQuery(deck, query)):
|
|
if type == SEARCH_TAG:
|
|
# a tag
|
|
if tquery:
|
|
if isNeg:
|
|
tquery += " except "
|
|
else:
|
|
tquery += " intersect "
|
|
elif isNeg:
|
|
tquery += "select id from facts except "
|
|
if token == "none":
|
|
tquery += """
|
|
select id from cards where fid in (select id from facts where tags = '')"""
|
|
else:
|
|
token = token.replace("*", "%")
|
|
if not token.startswith("%"):
|
|
token = "% " + token
|
|
if not token.endswith("%"):
|
|
token += " %"
|
|
args["_tag_%d" % c] = token
|
|
tquery += """
|
|
select id from facts where tags like :_tag_%d""" % c
|
|
elif type == SEARCH_TYPE:
|
|
if qquery:
|
|
if isNeg:
|
|
qquery += " except "
|
|
else:
|
|
qquery += " intersect "
|
|
elif isNeg:
|
|
qquery += "select id from cards except "
|
|
if token in ("rev", "new", "lrn"):
|
|
if token == "rev":
|
|
n = 1
|
|
elif token == "new":
|
|
n = 2
|
|
else:
|
|
n = 0
|
|
qquery += "select id from cards where type = %d" % n
|
|
elif token == "delayed":
|
|
print "delayed"
|
|
qquery += ("select id from cards where "
|
|
"due < %d and due > %d and "
|
|
"type in (0,1,2)") % (
|
|
deck.dayCutoff, deck.dayCutoff)
|
|
elif token == "suspended":
|
|
qquery += ("select id from cards where "
|
|
"queue = -1")
|
|
elif token == "leech":
|
|
qquery += (
|
|
"select id from cards where noCount >= (select value "
|
|
"from deckvars where key = 'leechFails')")
|
|
else: # due
|
|
qquery += ("select id from cards where "
|
|
"queue between 0 and 1 and due < %d") % deck.dayCutoff
|
|
elif type == SEARCH_FID:
|
|
if fidquery:
|
|
if isNeg:
|
|
fidquery += " except "
|
|
else:
|
|
fidquery += " intersect "
|
|
elif isNeg:
|
|
fidquery += "select id from cards except "
|
|
fidquery += "select id from cards where fid in (%s)" % token
|
|
elif type == SEARCH_CARD:
|
|
print "search_card broken"
|
|
token = token.replace("*", "%")
|
|
ids = deck.db.list("""
|
|
select id from tags where name like :tag escape '\\'""", tag=token)
|
|
if isNeg:
|
|
if cmquery['neg']:
|
|
cmquery['neg'] += " intersect "
|
|
cmquery['neg'] += """
|
|
select cardId from cardTags where src = 2 and cardTags.tagId in %s""" % ids2str(ids)
|
|
else:
|
|
if cmquery['pos']:
|
|
cmquery['pos'] += " intersect "
|
|
cmquery['pos'] += """
|
|
select cardId from cardTags where src = 2 and cardTags.tagId in %s""" % ids2str(ids)
|
|
elif type == SEARCH_FIELD or type == SEARCH_FIELD_EXISTS:
|
|
field = value = ''
|
|
if type == SEARCH_FIELD:
|
|
parts = token.split(':', 1);
|
|
if len(parts) == 2:
|
|
field = parts[0]
|
|
value = parts[1]
|
|
elif type == SEARCH_FIELD_EXISTS:
|
|
field = token
|
|
value = '*'
|
|
if (type == SEARCH_FIELD and filter != 'none'):
|
|
if field and value:
|
|
filters.append(
|
|
{'scope': 'field', 'type': filter,
|
|
'field': field, 'value': value, 'is_neg': isNeg})
|
|
else:
|
|
if field and value:
|
|
if sfquery:
|
|
if isNeg:
|
|
sfquery += " except "
|
|
else:
|
|
sfquery += " intersect "
|
|
elif isNeg:
|
|
sfquery += "select id from facts except "
|
|
field = field.replace("*", "%")
|
|
value = value.replace("*", "%")
|
|
args["_ff_%d" % c] = "%"+value+"%"
|
|
ids = deck.db.list("""
|
|
select id from fieldmodels where name like :field escape '\\'""", field=field)
|
|
sfquery += """
|
|
select fid from fdata where fmid in %s and
|
|
value like :_ff_%d escape '\\'""" % (ids2str(ids), c)
|
|
elif type == SEARCH_QA:
|
|
field = value = ''
|
|
parts = token.split(':', 1);
|
|
if len(parts) == 2:
|
|
field = parts[0]
|
|
value = parts[1]
|
|
if (filter != 'none'):
|
|
if field and value:
|
|
filters.append(
|
|
{'scope': 'card', 'type': filter, 'field': field,
|
|
'value': value, 'is_neg': isNeg})
|
|
else:
|
|
if field and value:
|
|
if qaquery:
|
|
if isNeg:
|
|
qaquery += " except "
|
|
else:
|
|
qaquery += " intersect "
|
|
elif isNeg:
|
|
qaquery += "select id from cards except "
|
|
value = value.replace("*", "%")
|
|
args["_ff_%d" % c] = "%"+value+"%"
|
|
|
|
if field == 'question':
|
|
qaquery += """
|
|
select id from cards where question like :_ff_%d escape '\\'""" % c
|
|
else:
|
|
qaquery += """
|
|
select id from cards where answer like :_ff_%d escape '\\'""" % c
|
|
elif type == SEARCH_DISTINCT:
|
|
if isNeg is False:
|
|
showdistinct = True if token == "one" else False
|
|
else:
|
|
showdistinct = False if token == "one" else True
|
|
else:
|
|
if (filter != 'none'):
|
|
filters.append(
|
|
{'scope': 'fact', 'type': filter,
|
|
'value': token, 'is_neg': isNeg})
|
|
else:
|
|
if fquery:
|
|
if isNeg:
|
|
fquery += " except "
|
|
else:
|
|
fquery += " intersect "
|
|
elif isNeg:
|
|
fquery += "select id from facts except "
|
|
token = token.replace("*", "%")
|
|
args["_ff_%d" % c] = "%"+token+"%"
|
|
fquery += """
|
|
select id from facts where flds like :_ff_%d escape '\\'""" % c
|
|
return (tquery, fquery, qquery, fidquery, cmquery, sfquery,
|
|
qaquery, showdistinct, filters, args)
|
|
|
|
# Find and replace
|
|
##########################################################################
|
|
|
|
def findReplace(deck, fids, src, dst, isRe=False, field=None):
|
|
"Find and replace fields in a fact."
|
|
# find
|
|
s = "select id, fid, value from fdata where fid in %s"
|
|
if isRe:
|
|
isRe = re.compile(src)
|
|
else:
|
|
s += " and value like :v"
|
|
if field:
|
|
s += " and fmid = :fmid"
|
|
rows = deck.db.all(s % ids2str(fids),
|
|
v="%"+src.replace("%", "%%")+"%",
|
|
fmid=field)
|
|
modded = []
|
|
if isRe:
|
|
modded = [
|
|
{'id': id, 'fid': fid, 'val': re.sub(isRe, dst, val)}
|
|
for (id, fid, val) in rows
|
|
if isRe.search(val)]
|
|
else:
|
|
modded = [
|
|
{'id': id, 'fid': fid, 'val': val.replace(src, dst)}
|
|
for (id, fid, val) in rows
|
|
if val.find(src) != -1]
|
|
# update
|
|
if modded:
|
|
deck.db.executemany(
|
|
'update fdata set value = :val where id = :id', modded)
|
|
deck.updateCardQACacheFromIds([f['fid'] for f in modded],
|
|
type="facts")
|
|
if field:
|
|
deck.updateFieldChecksums(field)
|
|
else:
|
|
deck.updateAllFieldChecksums()
|
|
return len(set([f['fid'] for f in modded]))
|
|
|
|
# Find duplicates
|
|
##########################################################################
|
|
|
|
def findDuplicates(deck, fmids):
|
|
data = deck.db.all(
|
|
"select fid, value from fdata where fmid in %s" %
|
|
ids2str(fmids))
|
|
vals = {}
|
|
for (fid, val) in data:
|
|
if not val.strip():
|
|
continue
|
|
if val not in vals:
|
|
vals[val] = [fid]
|
|
else:
|
|
vals[val].append(fid)
|
|
return [(k,v) for (k,v) in vals.items() if len(v) > 1]
|
|
|
|
# Find & sort
|
|
##########################################################################
|
|
|
|
# copied from ankiqt and trivially changed; will not work at the moment
|
|
|
|
# if idx == 0:
|
|
# self.sortKey = "question"
|
|
# elif idx == 1:
|
|
# self.sortKey = "answer"
|
|
# elif idx == 2:
|
|
# self.sortKey = "created"
|
|
# elif idx == 3:
|
|
# self.sortKey = "modified"
|
|
# elif idx == 4:
|
|
# self.sortKey = "combinedDue"
|
|
# elif idx == 5:
|
|
# self.sortKey = "interval"
|
|
# elif idx == 6:
|
|
# self.sortKey = "reps"
|
|
# elif idx == 7:
|
|
# self.sortKey = "factor"
|
|
# elif idx == 8:
|
|
# self.sortKey = "fact"
|
|
# elif idx == 9:
|
|
# self.sortKey = "noCount"
|
|
# elif idx == 10:
|
|
# self.sortKey = "firstAnswered"
|
|
# else:
|
|
# self.sortKey = ("field", self.sortFields[idx-11])
|
|
|
|
def findSorted(deck, query, sortKey):
|
|
# sorting
|
|
if not query.strip():
|
|
ads = ""
|
|
else:
|
|
ids = self.deck.findCards(query)
|
|
ads = "cards.id in %s" % ids2str(ids)
|
|
sort = ""
|
|
if isinstance(sortKey, types.StringType):
|
|
# card property
|
|
if sortKey == "fact":
|
|
sort = "order by facts.created, cards.created"
|
|
else:
|
|
sort = "order by cards." + sortKey
|
|
if sortKey in ("question", "answer"):
|
|
sort += " collate nocase"
|
|
if sortKey == "fact":
|
|
query = """
|
|
select cards.id from cards, facts
|
|
where cards.fid = facts.id """
|
|
if ads:
|
|
query += "and " + ads + " "
|
|
else:
|
|
query = "select id from cards "
|
|
if ads:
|
|
query += "where %s " % ads
|
|
query += sort
|
|
else:
|
|
# field value
|
|
ret = self.deck.db.all(
|
|
"select id, numeric from fields where name = :name",
|
|
name=sortKey[1])
|
|
fields = ",".join([str(x[0]) for x in ret])
|
|
# if multiple models have the same field, use the first numeric bool
|
|
numeric = ret[0][1]
|
|
if numeric:
|
|
order = "cast(fdata.value as real)"
|
|
else:
|
|
order = "fdata.value collate nocase"
|
|
if ads:
|
|
ads = " and " + ads
|
|
query = ("select cards.id "
|
|
"from fdata, cards where fdata.fmid in (%s) "
|
|
"and fdata.fid = cards.fid" + ads +
|
|
" order by cards.ordinal, %s") % (fields, order)
|
|
# run the query
|
|
self.cards = self.deck.db.all(query)
|