mirror of
https://github.com/ankitects/anki.git
synced 2025-09-25 01:06:35 -04:00
move finding code into separate file
This commit is contained in:
Damien Elmes
parent
55f4b9b7d0
commit
a902ad0b5f
3 changed files with 613 additions and 590 deletions
|
|
@ -21,15 +21,3 @@ NEW_TODAY_DUE = 1
|
||||||
REV_CARDS_OLD_FIRST = 0
|
REV_CARDS_OLD_FIRST = 0
|
||||||
REV_CARDS_NEW_FIRST = 1
|
REV_CARDS_NEW_FIRST = 1
|
||||||
REV_CARDS_RANDOM = 2
|
REV_CARDS_RANDOM = 2
|
||||||
|
|
||||||
# searching
|
|
||||||
SEARCH_TAG = 0
|
|
||||||
SEARCH_TYPE = 1
|
|
||||||
SEARCH_PHRASE = 2
|
|
||||||
SEARCH_FID = 3
|
|
||||||
SEARCH_CARD = 4
|
|
||||||
SEARCH_DISTINCT = 5
|
|
||||||
SEARCH_FIELD = 6
|
|
||||||
SEARCH_FIELD_EXISTS = 7
|
|
||||||
SEARCH_QA = 8
|
|
||||||
SEARCH_PHRASE_WB = 9
|
|
||||||
|
|
|
||||||
586
anki/deck.py
586
anki/deck.py
|
|
@ -1328,590 +1328,20 @@ where id = :id""", pending)
|
||||||
self.finishProgress()
|
self.finishProgress()
|
||||||
self.refreshSession()
|
self.refreshSession()
|
||||||
|
|
||||||
# Find
|
# Finding cards
|
||||||
##########################################################################
|
##########################################################################
|
||||||
|
|
||||||
def allFMFields(self, tolower=False):
|
|
||||||
fields = []
|
|
||||||
try:
|
|
||||||
fields = self.db.column0(
|
|
||||||
"select distinct name from fieldmodels order by name")
|
|
||||||
except:
|
|
||||||
fields = []
|
|
||||||
if tolower is True:
|
|
||||||
for i, v in enumerate(fields):
|
|
||||||
fields[i] = v.lower()
|
|
||||||
return fields
|
|
||||||
|
|
||||||
def _parseQuery(self, query):
|
|
||||||
tokens = []
|
|
||||||
res = []
|
|
||||||
|
|
||||||
allowedfields = self.allFMFields(True)
|
|
||||||
def addSearchFieldToken(field, value, isNeg, filter):
|
|
||||||
if field.lower() in allowedfields:
|
|
||||||
res.append((field + ':' + value, isNeg, SEARCH_FIELD, filter))
|
|
||||||
elif field in ['question', 'answer']:
|
|
||||||
res.append((field + ':' + value, isNeg, SEARCH_QA, filter))
|
|
||||||
else:
|
|
||||||
for p in phraselog:
|
|
||||||
res.append((p['value'], p['is_neg'], p['type'], p['filter']))
|
|
||||||
# break query into words or phraselog
|
|
||||||
# an extra space is added so the loop never ends in the middle
|
|
||||||
# completing a token
|
|
||||||
for match in re.findall(
|
|
||||||
r'(-)?\'(([^\'\\]|\\.)*)\'|(-)?"(([^"\\]|\\.)*)"|(-)?([^ ]+)|([ ]+)',
|
|
||||||
query + ' '):
|
|
||||||
type = ' '
|
|
||||||
if match[1]: type = "'"
|
|
||||||
elif match[4]: type = '"'
|
|
||||||
|
|
||||||
value = (match[1] or match[4] or match[7])
|
|
||||||
isNeg = (match[0] == '-' or match[3] == '-' or match[6] == '-')
|
|
||||||
|
|
||||||
tokens.append({'type': type, 'value': value, 'is_neg': isNeg,
|
|
||||||
'filter': ('wb' if type == "'" else 'none')})
|
|
||||||
intoken = isNeg = False
|
|
||||||
field = '' #name of the field for field related commands
|
|
||||||
phraselog = [] #log of phrases in case potential command is not a commad
|
|
||||||
for c, token in enumerate(tokens):
|
|
||||||
doprocess = True # only look for commands when this is true
|
|
||||||
#prevent cases such as "field" : value as being processed as a command
|
|
||||||
if len(token['value']) == 0:
|
|
||||||
if intoken is True and type == SEARCH_FIELD and field:
|
|
||||||
#case: fieldname: any thing here check for existance of fieldname
|
|
||||||
addSearchFieldToken(field, '*', isNeg, 'none')
|
|
||||||
phraselog = [] # reset phrases since command is completed
|
|
||||||
intoken = doprocess = False
|
|
||||||
if intoken is True:
|
|
||||||
if type == SEARCH_FIELD_EXISTS:
|
|
||||||
#case: field:"value"
|
|
||||||
res.append((token['value'], isNeg, type, 'none'))
|
|
||||||
intoken = doprocess = False
|
|
||||||
elif type == SEARCH_FIELD and field:
|
|
||||||
#case: fieldname:"value"
|
|
||||||
addSearchFieldToken(
|
|
||||||
field, token['value'], isNeg, token['filter'])
|
|
||||||
intoken = doprocess = False
|
|
||||||
|
|
||||||
elif type == SEARCH_FIELD and not field:
|
|
||||||
#case: "fieldname":"name" or "field" anything
|
|
||||||
if token['value'].startswith(":") and len(phraselog) == 1:
|
|
||||||
#we now know a colon is next, so mark it as field
|
|
||||||
# and keep looking for the value
|
|
||||||
field = phraselog[0]['value']
|
|
||||||
parts = token['value'].split(':', 1)
|
|
||||||
phraselog.append(
|
|
||||||
{'value': token['value'], 'is_neg': False,
|
|
||||||
'type': SEARCH_PHRASE, 'filter': token['filter']})
|
|
||||||
if parts[1]:
|
|
||||||
#value is included with the :, so wrap it up
|
|
||||||
addSearchFieldToken(field, parts[1], isNeg, 'none')
|
|
||||||
intoken = doprocess = False
|
|
||||||
doprocess = False
|
|
||||||
else:
|
|
||||||
#case: "fieldname"string/"fieldname"tag:name
|
|
||||||
intoken = False
|
|
||||||
if intoken is False and doprocess is False:
|
|
||||||
#command has been fully processed
|
|
||||||
phraselog = [] # reset phraselog, since we used it for a command
|
|
||||||
if intoken is False:
|
|
||||||
#include any non-command related phrases in the query
|
|
||||||
for p in phraselog: res.append(
|
|
||||||
(p['value'], p['is_neg'], p['type'], p['filter']))
|
|
||||||
phraselog = []
|
|
||||||
if intoken is False and doprocess is True:
|
|
||||||
field = ''
|
|
||||||
isNeg = token['is_neg']
|
|
||||||
if token['value'].startswith("tag:"):
|
|
||||||
token['value'] = token['value'][4:]
|
|
||||||
type = SEARCH_TAG
|
|
||||||
elif token['value'].startswith("is:"):
|
|
||||||
token['value'] = token['value'][3:].lower()
|
|
||||||
type = SEARCH_TYPE
|
|
||||||
elif token['value'].startswith("fid:") and len(token['value']) > 4:
|
|
||||||
dec = token['value'][4:]
|
|
||||||
try:
|
|
||||||
int(dec)
|
|
||||||
token['value'] = token['value'][4:]
|
|
||||||
except:
|
|
||||||
try:
|
|
||||||
for d in dec.split(","):
|
|
||||||
int(d)
|
|
||||||
token['value'] = token['value'][4:]
|
|
||||||
except:
|
|
||||||
token['value'] = "0"
|
|
||||||
type = SEARCH_FID
|
|
||||||
elif token['value'].startswith("card:"):
|
|
||||||
token['value'] = token['value'][5:]
|
|
||||||
type = SEARCH_CARD
|
|
||||||
elif token['value'].startswith("show:"):
|
|
||||||
token['value'] = token['value'][5:].lower()
|
|
||||||
type = SEARCH_DISTINCT
|
|
||||||
elif token['value'].startswith("field:"):
|
|
||||||
type = SEARCH_FIELD_EXISTS
|
|
||||||
parts = token['value'][6:].split(':', 1)
|
|
||||||
field = parts[0]
|
|
||||||
if len(parts) == 1 and parts[0]:
|
|
||||||
token['value'] = parts[0]
|
|
||||||
elif len(parts) == 1 and not parts[0]:
|
|
||||||
intoken = True
|
|
||||||
else:
|
|
||||||
type = SEARCH_FIELD
|
|
||||||
intoken = True
|
|
||||||
parts = token['value'].split(':', 1)
|
|
||||||
|
|
||||||
phraselog.append(
|
|
||||||
{'value': token['value'], 'is_neg': isNeg,
|
|
||||||
'type': SEARCH_PHRASE, 'filter': token['filter']})
|
|
||||||
if len(parts) == 2 and parts[0]:
|
|
||||||
field = parts[0]
|
|
||||||
if parts[1]:
|
|
||||||
#simple fieldname:value case - no need to look for more data
|
|
||||||
addSearchFieldToken(field, parts[1], isNeg, 'none')
|
|
||||||
intoken = doprocess = False
|
|
||||||
|
|
||||||
if intoken is False: phraselog = []
|
|
||||||
if intoken is False and doprocess is True:
|
|
||||||
res.append((token['value'], isNeg, type, token['filter']))
|
|
||||||
return res
|
|
||||||
|
|
||||||
def findCards(self, query):
|
def findCards(self, query):
|
||||||
(q, cmquery, showdistinct, filters, args) = self.findCardsWhere(query)
|
import anki.find
|
||||||
(factIdList, cardIdList) = self.findCardsMatchingFilters(filters)
|
return anki.find.findCards(self, query)
|
||||||
query = "select id from cards"
|
|
||||||
hasWhere = False
|
|
||||||
if q:
|
|
||||||
query += " where " + q
|
|
||||||
hasWhere = True
|
|
||||||
if cmquery['pos'] or cmquery['neg']:
|
|
||||||
if hasWhere is False:
|
|
||||||
query += " where "
|
|
||||||
hasWhere = True
|
|
||||||
else: query += " and "
|
|
||||||
if cmquery['pos']:
|
|
||||||
query += (" factId in(select distinct factId from cards "+
|
|
||||||
"where id in (" + cmquery['pos'] + ")) ")
|
|
||||||
query += " and id in(" + cmquery['pos'] + ") "
|
|
||||||
if cmquery['neg']:
|
|
||||||
query += (" factId not in(select distinct factId from "+
|
|
||||||
"cards where id in (" + cmquery['neg'] + ")) ")
|
|
||||||
if factIdList is not None:
|
|
||||||
if hasWhere is False:
|
|
||||||
query += " where "
|
|
||||||
hasWhere = True
|
|
||||||
else: query += " and "
|
|
||||||
query += " factId IN %s" % ids2str(factIdList)
|
|
||||||
if cardIdList is not None:
|
|
||||||
if hasWhere is False:
|
|
||||||
query += " where "
|
|
||||||
hasWhere = True
|
|
||||||
else: query += " and "
|
|
||||||
query += " id IN %s" % ids2str(cardIdList)
|
|
||||||
if showdistinct:
|
|
||||||
query += " group by factId"
|
|
||||||
#print query, args
|
|
||||||
return self.db.column0(query, **args)
|
|
||||||
|
|
||||||
def findCardsWhere(self, query):
|
def findReplace(self, *args, **kwargs):
|
||||||
(tquery, fquery, qquery, fidquery, cmquery, sfquery, qaquery,
|
import anki.find
|
||||||
showdistinct, filters, args) = self._findCards(query)
|
return anki.find.findReplace(self, *args, **kwargs)
|
||||||
q = ""
|
|
||||||
x = []
|
|
||||||
if tquery:
|
|
||||||
x.append(" id in (%s)" % tquery)
|
|
||||||
if fquery:
|
|
||||||
x.append(" factId in (%s)" % fquery)
|
|
||||||
if qquery:
|
|
||||||
x.append(" id in (%s)" % qquery)
|
|
||||||
if fidquery:
|
|
||||||
x.append(" id in (%s)" % fidquery)
|
|
||||||
if sfquery:
|
|
||||||
x.append(" factId in (%s)" % sfquery)
|
|
||||||
if qaquery:
|
|
||||||
x.append(" id in (%s)" % qaquery)
|
|
||||||
if x:
|
|
||||||
q += " and ".join(x)
|
|
||||||
return q, cmquery, showdistinct, filters, args
|
|
||||||
|
|
||||||
def findCardsMatchingFilters(self, filters):
|
|
||||||
factFilters = []
|
|
||||||
fieldFilters = {}
|
|
||||||
cardFilters = {}
|
|
||||||
|
|
||||||
factFilterMatches = []
|
|
||||||
fieldFilterMatches = []
|
|
||||||
cardFilterMatches = []
|
|
||||||
|
|
||||||
if (len(filters) > 0):
|
|
||||||
for filter in filters:
|
|
||||||
if filter['scope'] == 'fact':
|
|
||||||
regexp = re.compile(
|
|
||||||
r'\b' + re.escape(filter['value']) + r'\b', flags=re.I)
|
|
||||||
factFilters.append(
|
|
||||||
{'value': filter['value'], 'regexp': regexp,
|
|
||||||
'is_neg': filter['is_neg']})
|
|
||||||
if filter['scope'] == 'field':
|
|
||||||
fieldName = filter['field'].lower()
|
|
||||||
if (fieldName in fieldFilters) is False:
|
|
||||||
fieldFilters[fieldName] = []
|
|
||||||
regexp = re.compile(
|
|
||||||
r'\b' + re.escape(filter['value']) + r'\b', flags=re.I)
|
|
||||||
fieldFilters[fieldName].append(
|
|
||||||
{'value': filter['value'], 'regexp': regexp,
|
|
||||||
'is_neg': filter['is_neg']})
|
|
||||||
if filter['scope'] == 'card':
|
|
||||||
fieldName = filter['field'].lower()
|
|
||||||
if (fieldName in cardFilters) is False:
|
|
||||||
cardFilters[fieldName] = []
|
|
||||||
regexp = re.compile(r'\b' + re.escape(filter['value']) +
|
|
||||||
r'\b', flags=re.I)
|
|
||||||
cardFilters[fieldName].append(
|
|
||||||
{'value': filter['value'], 'regexp': regexp,
|
|
||||||
'is_neg': filter['is_neg']})
|
|
||||||
|
|
||||||
if len(factFilters) > 0:
|
|
||||||
fquery = ''
|
|
||||||
args = {}
|
|
||||||
for filter in factFilters:
|
|
||||||
c = len(args)
|
|
||||||
if fquery:
|
|
||||||
if filter['is_neg']: fquery += " except "
|
|
||||||
else: fquery += " intersect "
|
|
||||||
elif filter['is_neg']: fquery += "select id from fields except "
|
|
||||||
|
|
||||||
value = filter['value'].replace("*", "%")
|
|
||||||
args["_ff_%d" % c] = "%"+value+"%"
|
|
||||||
|
|
||||||
fquery += (
|
|
||||||
"select id from fields where value like "+
|
|
||||||
":_ff_%d escape '\\'" % c)
|
|
||||||
|
|
||||||
rows = self.db.execute(
|
|
||||||
'select factId, value from fields where id in (' +
|
|
||||||
fquery + ')', args)
|
|
||||||
while (1):
|
|
||||||
row = rows.fetchone()
|
|
||||||
if row is None: break
|
|
||||||
doesMatch = False
|
|
||||||
for filter in factFilters:
|
|
||||||
res = filter['regexp'].search(row[1])
|
|
||||||
if ((filter['is_neg'] is False and res) or
|
|
||||||
(filter['is_neg'] is True and res is None)):
|
|
||||||
factFilterMatches.append(row[0])
|
|
||||||
|
|
||||||
if len(fieldFilters) > 0:
|
|
||||||
sfquery = ''
|
|
||||||
args = {}
|
|
||||||
for field, filters in fieldFilters.iteritems():
|
|
||||||
for filter in filters:
|
|
||||||
c = len(args)
|
|
||||||
if sfquery:
|
|
||||||
if filter['is_neg']: sfquery += " except "
|
|
||||||
else: sfquery += " intersect "
|
|
||||||
elif filter['is_neg']: sfquery += "select id from fields except "
|
|
||||||
field = field.replace("*", "%")
|
|
||||||
value = filter['value'].replace("*", "%")
|
|
||||||
args["_ff_%d" % c] = "%"+value+"%"
|
|
||||||
|
|
||||||
ids = self.db.column0(
|
|
||||||
"select id from fieldmodels where name like "+
|
|
||||||
":field escape '\\'", field=field)
|
|
||||||
sfquery += ("select id from fields where "+
|
|
||||||
"fieldModelId in %s and value like "+
|
|
||||||
":_ff_%d escape '\\'") % (ids2str(ids), c)
|
|
||||||
|
|
||||||
rows = self.db.execute(
|
|
||||||
'select f.factId, f.value, fm.name from fields as f '+
|
|
||||||
'left join fieldmodels as fm ON (f.fieldModelId = '+
|
|
||||||
'fm.id) where f.id in (' + sfquery + ')', args)
|
|
||||||
while (1):
|
|
||||||
row = rows.fetchone()
|
|
||||||
if row is None: break
|
|
||||||
field = row[2].lower()
|
|
||||||
doesMatch = False
|
|
||||||
if field in fieldFilters:
|
|
||||||
for filter in fieldFilters[field]:
|
|
||||||
res = filter['regexp'].search(row[1])
|
|
||||||
if ((filter['is_neg'] is False and res) or
|
|
||||||
(filter['is_neg'] is True and res is None)):
|
|
||||||
fieldFilterMatches.append(row[0])
|
|
||||||
|
|
||||||
|
|
||||||
if len(cardFilters) > 0:
|
|
||||||
qaquery = ''
|
|
||||||
args = {}
|
|
||||||
for field, filters in cardFilters.iteritems():
|
|
||||||
for filter in filters:
|
|
||||||
c = len(args)
|
|
||||||
if qaquery:
|
|
||||||
if filter['is_neg']: qaquery += " except "
|
|
||||||
else: qaquery += " intersect "
|
|
||||||
elif filter['is_neg']: qaquery += "select id from cards except "
|
|
||||||
value = value.replace("*", "%")
|
|
||||||
args["_ff_%d" % c] = "%"+value+"%"
|
|
||||||
|
|
||||||
if field == 'question':
|
|
||||||
qaquery += "select id from cards where question "
|
|
||||||
qaquery += "like :_ff_%d escape '\\'" % c
|
|
||||||
else:
|
|
||||||
qaquery += "select id from cards where answer "
|
|
||||||
qaquery += "like :_ff_%d escape '\\'" % c
|
|
||||||
|
|
||||||
rows = self.db.execute(
|
|
||||||
'select id, question, answer from cards where id IN (' +
|
|
||||||
qaquery + ')', args)
|
|
||||||
while (1):
|
|
||||||
row = rows.fetchone()
|
|
||||||
if row is None: break
|
|
||||||
doesMatch = False
|
|
||||||
if field in cardFilters:
|
|
||||||
rowValue = row[1] if field == 'question' else row[2]
|
|
||||||
for filter in cardFilters[field]:
|
|
||||||
res = filter['regexp'].search(rowValue)
|
|
||||||
if ((filter['is_neg'] is False and res) or
|
|
||||||
(filter['is_neg'] is True and res is None)):
|
|
||||||
cardFilterMatches.append(row[0])
|
|
||||||
|
|
||||||
factIds = None
|
|
||||||
if len(factFilters) > 0 or len(fieldFilters) > 0:
|
|
||||||
factIds = []
|
|
||||||
factIds.extend(factFilterMatches)
|
|
||||||
factIds.extend(fieldFilterMatches)
|
|
||||||
|
|
||||||
cardIds = None
|
|
||||||
if len(cardFilters) > 0:
|
|
||||||
cardIds = []
|
|
||||||
cardIds.extend(cardFilterMatches)
|
|
||||||
|
|
||||||
return (factIds, cardIds)
|
|
||||||
|
|
||||||
def _findCards(self, query):
|
|
||||||
"Find facts matching QUERY."
|
|
||||||
tquery = ""
|
|
||||||
fquery = ""
|
|
||||||
qquery = ""
|
|
||||||
fidquery = ""
|
|
||||||
cmquery = { 'pos': '', 'neg': '' }
|
|
||||||
sfquery = qaquery = ""
|
|
||||||
showdistinct = False
|
|
||||||
filters = []
|
|
||||||
args = {}
|
|
||||||
for c, (token, isNeg, type, filter) in enumerate(self._parseQuery(query)):
|
|
||||||
if type == SEARCH_TAG:
|
|
||||||
# a tag
|
|
||||||
if tquery:
|
|
||||||
if isNeg:
|
|
||||||
tquery += " except "
|
|
||||||
else:
|
|
||||||
tquery += " intersect "
|
|
||||||
elif isNeg:
|
|
||||||
tquery += "select id from cards except "
|
|
||||||
if token == "none":
|
|
||||||
tquery += """
|
|
||||||
select cards.id from cards, facts where facts.tags = '' and cards.factId = facts.id """
|
|
||||||
else:
|
|
||||||
token = token.replace("*", "%")
|
|
||||||
ids = self.db.column0("""
|
|
||||||
select id from tags where name like :tag escape '\\'""", tag=token)
|
|
||||||
tquery += """
|
|
||||||
select cardId from cardTags where cardTags.tagId in %s""" % ids2str(ids)
|
|
||||||
elif type == SEARCH_TYPE:
|
|
||||||
if qquery:
|
|
||||||
if isNeg:
|
|
||||||
qquery += " except "
|
|
||||||
else:
|
|
||||||
qquery += " intersect "
|
|
||||||
elif isNeg:
|
|
||||||
qquery += "select id from cards except "
|
|
||||||
if token in ("rev", "new", "lrn"):
|
|
||||||
if token == "rev":
|
|
||||||
n = 1
|
|
||||||
elif token == "new":
|
|
||||||
n = 2
|
|
||||||
else:
|
|
||||||
n = 0
|
|
||||||
qquery += "select id from cards where type = %d" % n
|
|
||||||
elif token == "delayed":
|
|
||||||
print "delayed"
|
|
||||||
qquery += ("select id from cards where "
|
|
||||||
"due < %d and due > %d and "
|
|
||||||
"type in (0,1,2)") % (
|
|
||||||
self.dayCutoff, self.dayCutoff)
|
|
||||||
elif token == "suspended":
|
|
||||||
qquery += ("select id from cards where "
|
|
||||||
"queue = -1")
|
|
||||||
elif token == "leech":
|
|
||||||
qquery += (
|
|
||||||
"select id from cards where noCount >= (select value "
|
|
||||||
"from deckvars where key = 'leechFails')")
|
|
||||||
else: # due
|
|
||||||
qquery += ("select id from cards where "
|
|
||||||
"queue between 0 and 1 and due < %d") % self.dayCutoff
|
|
||||||
elif type == SEARCH_FID:
|
|
||||||
if fidquery:
|
|
||||||
if isNeg:
|
|
||||||
fidquery += " except "
|
|
||||||
else:
|
|
||||||
fidquery += " intersect "
|
|
||||||
elif isNeg:
|
|
||||||
fidquery += "select id from cards except "
|
|
||||||
fidquery += "select id from cards where factId in (%s)" % token
|
|
||||||
elif type == SEARCH_CARD:
|
|
||||||
token = token.replace("*", "%")
|
|
||||||
ids = self.db.column0("""
|
|
||||||
select id from tags where name like :tag escape '\\'""", tag=token)
|
|
||||||
if isNeg:
|
|
||||||
if cmquery['neg']:
|
|
||||||
cmquery['neg'] += " intersect "
|
|
||||||
cmquery['neg'] += """
|
|
||||||
select cardId from cardTags where src = 2 and cardTags.tagId in %s""" % ids2str(ids)
|
|
||||||
else:
|
|
||||||
if cmquery['pos']:
|
|
||||||
cmquery['pos'] += " intersect "
|
|
||||||
cmquery['pos'] += """
|
|
||||||
select cardId from cardTags where src = 2 and cardTags.tagId in %s""" % ids2str(ids)
|
|
||||||
elif type == SEARCH_FIELD or type == SEARCH_FIELD_EXISTS:
|
|
||||||
field = value = ''
|
|
||||||
if type == SEARCH_FIELD:
|
|
||||||
parts = token.split(':', 1);
|
|
||||||
if len(parts) == 2:
|
|
||||||
field = parts[0]
|
|
||||||
value = parts[1]
|
|
||||||
elif type == SEARCH_FIELD_EXISTS:
|
|
||||||
field = token
|
|
||||||
value = '*'
|
|
||||||
if (type == SEARCH_FIELD and filter != 'none'):
|
|
||||||
if field and value:
|
|
||||||
filters.append(
|
|
||||||
{'scope': 'field', 'type': filter,
|
|
||||||
'field': field, 'value': value, 'is_neg': isNeg})
|
|
||||||
else:
|
|
||||||
if field and value:
|
|
||||||
if sfquery:
|
|
||||||
if isNeg:
|
|
||||||
sfquery += " except "
|
|
||||||
else:
|
|
||||||
sfquery += " intersect "
|
|
||||||
elif isNeg:
|
|
||||||
sfquery += "select id from facts except "
|
|
||||||
field = field.replace("*", "%")
|
|
||||||
value = value.replace("*", "%")
|
|
||||||
args["_ff_%d" % c] = "%"+value+"%"
|
|
||||||
ids = self.db.column0("""
|
|
||||||
select id from fieldmodels where name like :field escape '\\'""", field=field)
|
|
||||||
sfquery += """
|
|
||||||
select factId from fields where fieldModelId in %s and
|
|
||||||
value like :_ff_%d escape '\\'""" % (ids2str(ids), c)
|
|
||||||
elif type == SEARCH_QA:
|
|
||||||
field = value = ''
|
|
||||||
parts = token.split(':', 1);
|
|
||||||
if len(parts) == 2:
|
|
||||||
field = parts[0]
|
|
||||||
value = parts[1]
|
|
||||||
if (filter != 'none'):
|
|
||||||
if field and value:
|
|
||||||
filters.append(
|
|
||||||
{'scope': 'card', 'type': filter, 'field': field,
|
|
||||||
'value': value, 'is_neg': isNeg})
|
|
||||||
else:
|
|
||||||
if field and value:
|
|
||||||
if qaquery:
|
|
||||||
if isNeg:
|
|
||||||
qaquery += " except "
|
|
||||||
else:
|
|
||||||
qaquery += " intersect "
|
|
||||||
elif isNeg:
|
|
||||||
qaquery += "select id from cards except "
|
|
||||||
value = value.replace("*", "%")
|
|
||||||
args["_ff_%d" % c] = "%"+value+"%"
|
|
||||||
|
|
||||||
if field == 'question':
|
|
||||||
qaquery += """
|
|
||||||
select id from cards where question like :_ff_%d escape '\\'""" % c
|
|
||||||
else:
|
|
||||||
qaquery += """
|
|
||||||
select id from cards where answer like :_ff_%d escape '\\'""" % c
|
|
||||||
elif type == SEARCH_DISTINCT:
|
|
||||||
if isNeg is False:
|
|
||||||
showdistinct = True if token == "one" else False
|
|
||||||
else:
|
|
||||||
showdistinct = False if token == "one" else True
|
|
||||||
else:
|
|
||||||
if (filter != 'none'):
|
|
||||||
filters.append(
|
|
||||||
{'scope': 'fact', 'type': filter,
|
|
||||||
'value': token, 'is_neg': isNeg})
|
|
||||||
else:
|
|
||||||
if fquery:
|
|
||||||
if isNeg:
|
|
||||||
fquery += " except "
|
|
||||||
else:
|
|
||||||
fquery += " intersect "
|
|
||||||
elif isNeg:
|
|
||||||
fquery += "select id from facts except "
|
|
||||||
token = token.replace("*", "%")
|
|
||||||
args["_ff_%d" % c] = "%"+token+"%"
|
|
||||||
fquery += """
|
|
||||||
select id from facts where cache like :_ff_%d escape '\\'""" % c
|
|
||||||
return (tquery, fquery, qquery, fidquery, cmquery, sfquery,
|
|
||||||
qaquery, showdistinct, filters, args)
|
|
||||||
|
|
||||||
# Find and replace
|
|
||||||
##########################################################################
|
|
||||||
|
|
||||||
def findReplace(self, factIds, src, dst, isRe=False, field=None):
|
|
||||||
"Find and replace fields in a fact."
|
|
||||||
# find
|
|
||||||
s = "select id, factId, value from fields where factId in %s"
|
|
||||||
if isRe:
|
|
||||||
isRe = re.compile(src)
|
|
||||||
else:
|
|
||||||
s += " and value like :v"
|
|
||||||
if field:
|
|
||||||
s += " and fieldModelId = :fmid"
|
|
||||||
rows = self.db.all(s % ids2str(factIds),
|
|
||||||
v="%"+src.replace("%", "%%")+"%",
|
|
||||||
fmid=field)
|
|
||||||
modded = []
|
|
||||||
if isRe:
|
|
||||||
modded = [
|
|
||||||
{'id': id, 'fid': fid, 'val': re.sub(isRe, dst, val)}
|
|
||||||
for (id, fid, val) in rows
|
|
||||||
if isRe.search(val)]
|
|
||||||
else:
|
|
||||||
modded = [
|
|
||||||
{'id': id, 'fid': fid, 'val': val.replace(src, dst)}
|
|
||||||
for (id, fid, val) in rows
|
|
||||||
if val.find(src) != -1]
|
|
||||||
# update
|
|
||||||
if modded:
|
|
||||||
self.db.statements(
|
|
||||||
'update fields set value = :val where id = :id', modded)
|
|
||||||
self.updateCardQACacheFromIds([f['fid'] for f in modded],
|
|
||||||
type="facts")
|
|
||||||
if field:
|
|
||||||
self.updateFieldChecksums(field)
|
|
||||||
else:
|
|
||||||
self.updateAllFieldChecksums()
|
|
||||||
return len(set([f['fid'] for f in modded]))
|
|
||||||
|
|
||||||
# Find duplicates
|
|
||||||
##########################################################################
|
|
||||||
|
|
||||||
def findDuplicates(self, fmids):
|
def findDuplicates(self, fmids):
|
||||||
data = self.db.all(
|
import anki.find
|
||||||
"select factId, value from fields where fieldModelId in %s" %
|
return anki.find.findDuplicates(self, fmids)
|
||||||
ids2str(fmids))
|
|
||||||
vals = {}
|
|
||||||
for (fid, val) in data:
|
|
||||||
if not val.strip():
|
|
||||||
continue
|
|
||||||
if val not in vals:
|
|
||||||
vals[val] = [fid]
|
|
||||||
else:
|
|
||||||
vals[val].append(fid)
|
|
||||||
return [(k,v) for (k,v) in vals.items() if len(v) > 1]
|
|
||||||
|
|
||||||
# Progress info
|
# Progress info
|
||||||
##########################################################################
|
##########################################################################
|
||||||
|
|
|
||||||
605
anki/find.py
Normal file
605
anki/find.py
Normal file
|
|
@ -0,0 +1,605 @@
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
# Copyright: Damien Elmes <anki@ichi2.net>
|
||||||
|
# License: GNU GPL, version 3 or later; http://www.gnu.org/copyleft/gpl.html
|
||||||
|
|
||||||
|
# A lot of findCards() and related functions was contributed by
|
||||||
|
# Marcus.
|
||||||
|
|
||||||
|
import re
|
||||||
|
from anki.utils import ids2str
|
||||||
|
|
||||||
|
SEARCH_TAG = 0
|
||||||
|
SEARCH_TYPE = 1
|
||||||
|
SEARCH_PHRASE = 2
|
||||||
|
SEARCH_FID = 3
|
||||||
|
SEARCH_CARD = 4
|
||||||
|
SEARCH_DISTINCT = 5
|
||||||
|
SEARCH_FIELD = 6
|
||||||
|
SEARCH_FIELD_EXISTS = 7
|
||||||
|
SEARCH_QA = 8
|
||||||
|
SEARCH_PHRASE_WB = 9
|
||||||
|
|
||||||
|
# Find
|
||||||
|
##########################################################################
|
||||||
|
|
||||||
|
def allFMFields(deck, tolower=False):
|
||||||
|
fields = []
|
||||||
|
try:
|
||||||
|
fields = deck.db.column0(
|
||||||
|
"select distinct name from fieldmodels order by name")
|
||||||
|
except:
|
||||||
|
fields = []
|
||||||
|
if tolower is True:
|
||||||
|
for i, v in enumerate(fields):
|
||||||
|
fields[i] = v.lower()
|
||||||
|
return fields
|
||||||
|
|
||||||
|
def _parseQuery(deck, query):
|
||||||
|
tokens = []
|
||||||
|
res = []
|
||||||
|
|
||||||
|
allowedfields = allFMFields(deck, True)
|
||||||
|
def addSearchFieldToken(field, value, isNeg, filter):
|
||||||
|
if field.lower() in allowedfields:
|
||||||
|
res.append((field + ':' + value, isNeg, SEARCH_FIELD, filter))
|
||||||
|
elif field in ['question', 'answer']:
|
||||||
|
res.append((field + ':' + value, isNeg, SEARCH_QA, filter))
|
||||||
|
else:
|
||||||
|
for p in phraselog:
|
||||||
|
res.append((p['value'], p['is_neg'], p['type'], p['filter']))
|
||||||
|
# break query into words or phraselog
|
||||||
|
# an extra space is added so the loop never ends in the middle
|
||||||
|
# completing a token
|
||||||
|
for match in re.findall(
|
||||||
|
r'(-)?\'(([^\'\\]|\\.)*)\'|(-)?"(([^"\\]|\\.)*)"|(-)?([^ ]+)|([ ]+)',
|
||||||
|
query + ' '):
|
||||||
|
type = ' '
|
||||||
|
if match[1]: type = "'"
|
||||||
|
elif match[4]: type = '"'
|
||||||
|
|
||||||
|
value = (match[1] or match[4] or match[7])
|
||||||
|
isNeg = (match[0] == '-' or match[3] == '-' or match[6] == '-')
|
||||||
|
|
||||||
|
tokens.append({'type': type, 'value': value, 'is_neg': isNeg,
|
||||||
|
'filter': ('wb' if type == "'" else 'none')})
|
||||||
|
intoken = isNeg = False
|
||||||
|
field = '' #name of the field for field related commands
|
||||||
|
phraselog = [] #log of phrases in case potential command is not a commad
|
||||||
|
for c, token in enumerate(tokens):
|
||||||
|
doprocess = True # only look for commands when this is true
|
||||||
|
#prevent cases such as "field" : value as being processed as a command
|
||||||
|
if len(token['value']) == 0:
|
||||||
|
if intoken is True and type == SEARCH_FIELD and field:
|
||||||
|
#case: fieldname: any thing here check for existance of fieldname
|
||||||
|
addSearchFieldToken(field, '*', isNeg, 'none')
|
||||||
|
phraselog = [] # reset phrases since command is completed
|
||||||
|
intoken = doprocess = False
|
||||||
|
if intoken is True:
|
||||||
|
if type == SEARCH_FIELD_EXISTS:
|
||||||
|
#case: field:"value"
|
||||||
|
res.append((token['value'], isNeg, type, 'none'))
|
||||||
|
intoken = doprocess = False
|
||||||
|
elif type == SEARCH_FIELD and field:
|
||||||
|
#case: fieldname:"value"
|
||||||
|
addSearchFieldToken(
|
||||||
|
field, token['value'], isNeg, token['filter'])
|
||||||
|
intoken = doprocess = False
|
||||||
|
|
||||||
|
elif type == SEARCH_FIELD and not field:
|
||||||
|
#case: "fieldname":"name" or "field" anything
|
||||||
|
if token['value'].startswith(":") and len(phraselog) == 1:
|
||||||
|
#we now know a colon is next, so mark it as field
|
||||||
|
# and keep looking for the value
|
||||||
|
field = phraselog[0]['value']
|
||||||
|
parts = token['value'].split(':', 1)
|
||||||
|
phraselog.append(
|
||||||
|
{'value': token['value'], 'is_neg': False,
|
||||||
|
'type': SEARCH_PHRASE, 'filter': token['filter']})
|
||||||
|
if parts[1]:
|
||||||
|
#value is included with the :, so wrap it up
|
||||||
|
addSearchFieldToken(field, parts[1], isNeg, 'none')
|
||||||
|
intoken = doprocess = False
|
||||||
|
doprocess = False
|
||||||
|
else:
|
||||||
|
#case: "fieldname"string/"fieldname"tag:name
|
||||||
|
intoken = False
|
||||||
|
if intoken is False and doprocess is False:
|
||||||
|
#command has been fully processed
|
||||||
|
phraselog = [] # reset phraselog, since we used it for a command
|
||||||
|
if intoken is False:
|
||||||
|
#include any non-command related phrases in the query
|
||||||
|
for p in phraselog: res.append(
|
||||||
|
(p['value'], p['is_neg'], p['type'], p['filter']))
|
||||||
|
phraselog = []
|
||||||
|
if intoken is False and doprocess is True:
|
||||||
|
field = ''
|
||||||
|
isNeg = token['is_neg']
|
||||||
|
if token['value'].startswith("tag:"):
|
||||||
|
token['value'] = token['value'][4:]
|
||||||
|
type = SEARCH_TAG
|
||||||
|
elif token['value'].startswith("is:"):
|
||||||
|
token['value'] = token['value'][3:].lower()
|
||||||
|
type = SEARCH_TYPE
|
||||||
|
elif token['value'].startswith("fid:") and len(token['value']) > 4:
|
||||||
|
dec = token['value'][4:]
|
||||||
|
try:
|
||||||
|
int(dec)
|
||||||
|
token['value'] = token['value'][4:]
|
||||||
|
except:
|
||||||
|
try:
|
||||||
|
for d in dec.split(","):
|
||||||
|
int(d)
|
||||||
|
token['value'] = token['value'][4:]
|
||||||
|
except:
|
||||||
|
token['value'] = "0"
|
||||||
|
type = SEARCH_FID
|
||||||
|
elif token['value'].startswith("card:"):
|
||||||
|
token['value'] = token['value'][5:]
|
||||||
|
type = SEARCH_CARD
|
||||||
|
elif token['value'].startswith("show:"):
|
||||||
|
token['value'] = token['value'][5:].lower()
|
||||||
|
type = SEARCH_DISTINCT
|
||||||
|
elif token['value'].startswith("field:"):
|
||||||
|
type = SEARCH_FIELD_EXISTS
|
||||||
|
parts = token['value'][6:].split(':', 1)
|
||||||
|
field = parts[0]
|
||||||
|
if len(parts) == 1 and parts[0]:
|
||||||
|
token['value'] = parts[0]
|
||||||
|
elif len(parts) == 1 and not parts[0]:
|
||||||
|
intoken = True
|
||||||
|
else:
|
||||||
|
type = SEARCH_FIELD
|
||||||
|
intoken = True
|
||||||
|
parts = token['value'].split(':', 1)
|
||||||
|
|
||||||
|
phraselog.append(
|
||||||
|
{'value': token['value'], 'is_neg': isNeg,
|
||||||
|
'type': SEARCH_PHRASE, 'filter': token['filter']})
|
||||||
|
if len(parts) == 2 and parts[0]:
|
||||||
|
field = parts[0]
|
||||||
|
if parts[1]:
|
||||||
|
#simple fieldname:value case - no need to look for more data
|
||||||
|
addSearchFieldToken(field, parts[1], isNeg, 'none')
|
||||||
|
intoken = doprocess = False
|
||||||
|
|
||||||
|
if intoken is False: phraselog = []
|
||||||
|
if intoken is False and doprocess is True:
|
||||||
|
res.append((token['value'], isNeg, type, token['filter']))
|
||||||
|
return res
|
||||||
|
|
||||||
|
def findCards(deck, query):
|
||||||
|
(q, cmquery, showdistinct, filters, args) = findCardsWhere(deck, query)
|
||||||
|
(factIdList, cardIdList) = findCardsMatchingFilters(deck, filters)
|
||||||
|
query = "select id from cards"
|
||||||
|
hasWhere = False
|
||||||
|
if q:
|
||||||
|
query += " where " + q
|
||||||
|
hasWhere = True
|
||||||
|
if cmquery['pos'] or cmquery['neg']:
|
||||||
|
if hasWhere is False:
|
||||||
|
query += " where "
|
||||||
|
hasWhere = True
|
||||||
|
else: query += " and "
|
||||||
|
if cmquery['pos']:
|
||||||
|
query += (" factId in(select distinct factId from cards "+
|
||||||
|
"where id in (" + cmquery['pos'] + ")) ")
|
||||||
|
query += " and id in(" + cmquery['pos'] + ") "
|
||||||
|
if cmquery['neg']:
|
||||||
|
query += (" factId not in(select distinct factId from "+
|
||||||
|
"cards where id in (" + cmquery['neg'] + ")) ")
|
||||||
|
if factIdList is not None:
|
||||||
|
if hasWhere is False:
|
||||||
|
query += " where "
|
||||||
|
hasWhere = True
|
||||||
|
else: query += " and "
|
||||||
|
query += " factId IN %s" % ids2str(factIdList)
|
||||||
|
if cardIdList is not None:
|
||||||
|
if hasWhere is False:
|
||||||
|
query += " where "
|
||||||
|
hasWhere = True
|
||||||
|
else: query += " and "
|
||||||
|
query += " id IN %s" % ids2str(cardIdList)
|
||||||
|
if showdistinct:
|
||||||
|
query += " group by factId"
|
||||||
|
#print query, args
|
||||||
|
return deck.db.column0(query, **args)
|
||||||
|
|
||||||
|
def findCardsWhere(deck, query):
|
||||||
|
(tquery, fquery, qquery, fidquery, cmquery, sfquery, qaquery,
|
||||||
|
showdistinct, filters, args) = _findCards(deck, query)
|
||||||
|
q = ""
|
||||||
|
x = []
|
||||||
|
if tquery:
|
||||||
|
x.append(" id in (%s)" % tquery)
|
||||||
|
if fquery:
|
||||||
|
x.append(" factId in (%s)" % fquery)
|
||||||
|
if qquery:
|
||||||
|
x.append(" id in (%s)" % qquery)
|
||||||
|
if fidquery:
|
||||||
|
x.append(" id in (%s)" % fidquery)
|
||||||
|
if sfquery:
|
||||||
|
x.append(" factId in (%s)" % sfquery)
|
||||||
|
if qaquery:
|
||||||
|
x.append(" id in (%s)" % qaquery)
|
||||||
|
if x:
|
||||||
|
q += " and ".join(x)
|
||||||
|
return q, cmquery, showdistinct, filters, args
|
||||||
|
|
||||||
|
def findCardsMatchingFilters(deck, filters):
|
||||||
|
factFilters = []
|
||||||
|
fieldFilters = {}
|
||||||
|
cardFilters = {}
|
||||||
|
|
||||||
|
factFilterMatches = []
|
||||||
|
fieldFilterMatches = []
|
||||||
|
cardFilterMatches = []
|
||||||
|
|
||||||
|
if (len(filters) > 0):
|
||||||
|
for filter in filters:
|
||||||
|
if filter['scope'] == 'fact':
|
||||||
|
regexp = re.compile(
|
||||||
|
r'\b' + re.escape(filter['value']) + r'\b', flags=re.I)
|
||||||
|
factFilters.append(
|
||||||
|
{'value': filter['value'], 'regexp': regexp,
|
||||||
|
'is_neg': filter['is_neg']})
|
||||||
|
if filter['scope'] == 'field':
|
||||||
|
fieldName = filter['field'].lower()
|
||||||
|
if (fieldName in fieldFilters) is False:
|
||||||
|
fieldFilters[fieldName] = []
|
||||||
|
regexp = re.compile(
|
||||||
|
r'\b' + re.escape(filter['value']) + r'\b', flags=re.I)
|
||||||
|
fieldFilters[fieldName].append(
|
||||||
|
{'value': filter['value'], 'regexp': regexp,
|
||||||
|
'is_neg': filter['is_neg']})
|
||||||
|
if filter['scope'] == 'card':
|
||||||
|
fieldName = filter['field'].lower()
|
||||||
|
if (fieldName in cardFilters) is False:
|
||||||
|
cardFilters[fieldName] = []
|
||||||
|
regexp = re.compile(r'\b' + re.escape(filter['value']) +
|
||||||
|
r'\b', flags=re.I)
|
||||||
|
cardFilters[fieldName].append(
|
||||||
|
{'value': filter['value'], 'regexp': regexp,
|
||||||
|
'is_neg': filter['is_neg']})
|
||||||
|
|
||||||
|
if len(factFilters) > 0:
|
||||||
|
fquery = ''
|
||||||
|
args = {}
|
||||||
|
for filter in factFilters:
|
||||||
|
c = len(args)
|
||||||
|
if fquery:
|
||||||
|
if filter['is_neg']: fquery += " except "
|
||||||
|
else: fquery += " intersect "
|
||||||
|
elif filter['is_neg']: fquery += "select id from fields except "
|
||||||
|
|
||||||
|
value = filter['value'].replace("*", "%")
|
||||||
|
args["_ff_%d" % c] = "%"+value+"%"
|
||||||
|
|
||||||
|
fquery += (
|
||||||
|
"select id from fields where value like "+
|
||||||
|
":_ff_%d escape '\\'" % c)
|
||||||
|
|
||||||
|
rows = deck.db.execute(
|
||||||
|
'select factId, value from fields where id in (' +
|
||||||
|
fquery + ')', args)
|
||||||
|
while (1):
|
||||||
|
row = rows.fetchone()
|
||||||
|
if row is None: break
|
||||||
|
doesMatch = False
|
||||||
|
for filter in factFilters:
|
||||||
|
res = filter['regexp'].search(row[1])
|
||||||
|
if ((filter['is_neg'] is False and res) or
|
||||||
|
(filter['is_neg'] is True and res is None)):
|
||||||
|
factFilterMatches.append(row[0])
|
||||||
|
|
||||||
|
if len(fieldFilters) > 0:
|
||||||
|
sfquery = ''
|
||||||
|
args = {}
|
||||||
|
for field, filters in fieldFilters.iteritems():
|
||||||
|
for filter in filters:
|
||||||
|
c = len(args)
|
||||||
|
if sfquery:
|
||||||
|
if filter['is_neg']: sfquery += " except "
|
||||||
|
else: sfquery += " intersect "
|
||||||
|
elif filter['is_neg']: sfquery += "select id from fields except "
|
||||||
|
field = field.replace("*", "%")
|
||||||
|
value = filter['value'].replace("*", "%")
|
||||||
|
args["_ff_%d" % c] = "%"+value+"%"
|
||||||
|
|
||||||
|
ids = deck.db.column0(
|
||||||
|
"select id from fieldmodels where name like "+
|
||||||
|
":field escape '\\'", field=field)
|
||||||
|
sfquery += ("select id from fields where "+
|
||||||
|
"fieldModelId in %s and value like "+
|
||||||
|
":_ff_%d escape '\\'") % (ids2str(ids), c)
|
||||||
|
|
||||||
|
rows = deck.db.execute(
|
||||||
|
'select f.factId, f.value, fm.name from fields as f '+
|
||||||
|
'left join fieldmodels as fm ON (f.fieldModelId = '+
|
||||||
|
'fm.id) where f.id in (' + sfquery + ')', args)
|
||||||
|
while (1):
|
||||||
|
row = rows.fetchone()
|
||||||
|
if row is None: break
|
||||||
|
field = row[2].lower()
|
||||||
|
doesMatch = False
|
||||||
|
if field in fieldFilters:
|
||||||
|
for filter in fieldFilters[field]:
|
||||||
|
res = filter['regexp'].search(row[1])
|
||||||
|
if ((filter['is_neg'] is False and res) or
|
||||||
|
(filter['is_neg'] is True and res is None)):
|
||||||
|
fieldFilterMatches.append(row[0])
|
||||||
|
|
||||||
|
|
||||||
|
if len(cardFilters) > 0:
|
||||||
|
qaquery = ''
|
||||||
|
args = {}
|
||||||
|
for field, filters in cardFilters.iteritems():
|
||||||
|
for filter in filters:
|
||||||
|
c = len(args)
|
||||||
|
if qaquery:
|
||||||
|
if filter['is_neg']: qaquery += " except "
|
||||||
|
else: qaquery += " intersect "
|
||||||
|
elif filter['is_neg']: qaquery += "select id from cards except "
|
||||||
|
value = value.replace("*", "%")
|
||||||
|
args["_ff_%d" % c] = "%"+value+"%"
|
||||||
|
|
||||||
|
if field == 'question':
|
||||||
|
qaquery += "select id from cards where question "
|
||||||
|
qaquery += "like :_ff_%d escape '\\'" % c
|
||||||
|
else:
|
||||||
|
qaquery += "select id from cards where answer "
|
||||||
|
qaquery += "like :_ff_%d escape '\\'" % c
|
||||||
|
|
||||||
|
rows = deck.db.execute(
|
||||||
|
'select id, question, answer from cards where id IN (' +
|
||||||
|
qaquery + ')', args)
|
||||||
|
while (1):
|
||||||
|
row = rows.fetchone()
|
||||||
|
if row is None: break
|
||||||
|
doesMatch = False
|
||||||
|
if field in cardFilters:
|
||||||
|
rowValue = row[1] if field == 'question' else row[2]
|
||||||
|
for filter in cardFilters[field]:
|
||||||
|
res = filter['regexp'].search(rowValue)
|
||||||
|
if ((filter['is_neg'] is False and res) or
|
||||||
|
(filter['is_neg'] is True and res is None)):
|
||||||
|
cardFilterMatches.append(row[0])
|
||||||
|
|
||||||
|
factIds = None
|
||||||
|
if len(factFilters) > 0 or len(fieldFilters) > 0:
|
||||||
|
factIds = []
|
||||||
|
factIds.extend(factFilterMatches)
|
||||||
|
factIds.extend(fieldFilterMatches)
|
||||||
|
|
||||||
|
cardIds = None
|
||||||
|
if len(cardFilters) > 0:
|
||||||
|
cardIds = []
|
||||||
|
cardIds.extend(cardFilterMatches)
|
||||||
|
|
||||||
|
return (factIds, cardIds)
|
||||||
|
|
||||||
|
def _findCards(deck, query):
|
||||||
|
"Find facts matching QUERY."
|
||||||
|
tquery = ""
|
||||||
|
fquery = ""
|
||||||
|
qquery = ""
|
||||||
|
fidquery = ""
|
||||||
|
cmquery = { 'pos': '', 'neg': '' }
|
||||||
|
sfquery = qaquery = ""
|
||||||
|
showdistinct = False
|
||||||
|
filters = []
|
||||||
|
args = {}
|
||||||
|
for c, (token, isNeg, type, filter) in enumerate(_parseQuery(deck, query)):
|
||||||
|
if type == SEARCH_TAG:
|
||||||
|
# a tag
|
||||||
|
if tquery:
|
||||||
|
if isNeg:
|
||||||
|
tquery += " except "
|
||||||
|
else:
|
||||||
|
tquery += " intersect "
|
||||||
|
elif isNeg:
|
||||||
|
tquery += "select id from cards except "
|
||||||
|
if token == "none":
|
||||||
|
tquery += """
|
||||||
|
select cards.id from cards, facts where facts.tags = '' and cards.factId = facts.id """
|
||||||
|
else:
|
||||||
|
token = token.replace("*", "%")
|
||||||
|
ids = deck.db.column0("""
|
||||||
|
select id from tags where name like :tag escape '\\'""", tag=token)
|
||||||
|
tquery += """
|
||||||
|
select cardId from cardTags where cardTags.tagId in %s""" % ids2str(ids)
|
||||||
|
elif type == SEARCH_TYPE:
|
||||||
|
if qquery:
|
||||||
|
if isNeg:
|
||||||
|
qquery += " except "
|
||||||
|
else:
|
||||||
|
qquery += " intersect "
|
||||||
|
elif isNeg:
|
||||||
|
qquery += "select id from cards except "
|
||||||
|
if token in ("rev", "new", "lrn"):
|
||||||
|
if token == "rev":
|
||||||
|
n = 1
|
||||||
|
elif token == "new":
|
||||||
|
n = 2
|
||||||
|
else:
|
||||||
|
n = 0
|
||||||
|
qquery += "select id from cards where type = %d" % n
|
||||||
|
elif token == "delayed":
|
||||||
|
print "delayed"
|
||||||
|
qquery += ("select id from cards where "
|
||||||
|
"due < %d and due > %d and "
|
||||||
|
"type in (0,1,2)") % (
|
||||||
|
deck.dayCutoff, deck.dayCutoff)
|
||||||
|
elif token == "suspended":
|
||||||
|
qquery += ("select id from cards where "
|
||||||
|
"queue = -1")
|
||||||
|
elif token == "leech":
|
||||||
|
qquery += (
|
||||||
|
"select id from cards where noCount >= (select value "
|
||||||
|
"from deckvars where key = 'leechFails')")
|
||||||
|
else: # due
|
||||||
|
qquery += ("select id from cards where "
|
||||||
|
"queue between 0 and 1 and due < %d") % deck.dayCutoff
|
||||||
|
elif type == SEARCH_FID:
|
||||||
|
if fidquery:
|
||||||
|
if isNeg:
|
||||||
|
fidquery += " except "
|
||||||
|
else:
|
||||||
|
fidquery += " intersect "
|
||||||
|
elif isNeg:
|
||||||
|
fidquery += "select id from cards except "
|
||||||
|
fidquery += "select id from cards where factId in (%s)" % token
|
||||||
|
elif type == SEARCH_CARD:
|
||||||
|
token = token.replace("*", "%")
|
||||||
|
ids = deck.db.column0("""
|
||||||
|
select id from tags where name like :tag escape '\\'""", tag=token)
|
||||||
|
if isNeg:
|
||||||
|
if cmquery['neg']:
|
||||||
|
cmquery['neg'] += " intersect "
|
||||||
|
cmquery['neg'] += """
|
||||||
|
select cardId from cardTags where src = 2 and cardTags.tagId in %s""" % ids2str(ids)
|
||||||
|
else:
|
||||||
|
if cmquery['pos']:
|
||||||
|
cmquery['pos'] += " intersect "
|
||||||
|
cmquery['pos'] += """
|
||||||
|
select cardId from cardTags where src = 2 and cardTags.tagId in %s""" % ids2str(ids)
|
||||||
|
elif type == SEARCH_FIELD or type == SEARCH_FIELD_EXISTS:
|
||||||
|
field = value = ''
|
||||||
|
if type == SEARCH_FIELD:
|
||||||
|
parts = token.split(':', 1);
|
||||||
|
if len(parts) == 2:
|
||||||
|
field = parts[0]
|
||||||
|
value = parts[1]
|
||||||
|
elif type == SEARCH_FIELD_EXISTS:
|
||||||
|
field = token
|
||||||
|
value = '*'
|
||||||
|
if (type == SEARCH_FIELD and filter != 'none'):
|
||||||
|
if field and value:
|
||||||
|
filters.append(
|
||||||
|
{'scope': 'field', 'type': filter,
|
||||||
|
'field': field, 'value': value, 'is_neg': isNeg})
|
||||||
|
else:
|
||||||
|
if field and value:
|
||||||
|
if sfquery:
|
||||||
|
if isNeg:
|
||||||
|
sfquery += " except "
|
||||||
|
else:
|
||||||
|
sfquery += " intersect "
|
||||||
|
elif isNeg:
|
||||||
|
sfquery += "select id from facts except "
|
||||||
|
field = field.replace("*", "%")
|
||||||
|
value = value.replace("*", "%")
|
||||||
|
args["_ff_%d" % c] = "%"+value+"%"
|
||||||
|
ids = deck.db.column0("""
|
||||||
|
select id from fieldmodels where name like :field escape '\\'""", field=field)
|
||||||
|
sfquery += """
|
||||||
|
select factId from fields where fieldModelId in %s and
|
||||||
|
value like :_ff_%d escape '\\'""" % (ids2str(ids), c)
|
||||||
|
elif type == SEARCH_QA:
|
||||||
|
field = value = ''
|
||||||
|
parts = token.split(':', 1);
|
||||||
|
if len(parts) == 2:
|
||||||
|
field = parts[0]
|
||||||
|
value = parts[1]
|
||||||
|
if (filter != 'none'):
|
||||||
|
if field and value:
|
||||||
|
filters.append(
|
||||||
|
{'scope': 'card', 'type': filter, 'field': field,
|
||||||
|
'value': value, 'is_neg': isNeg})
|
||||||
|
else:
|
||||||
|
if field and value:
|
||||||
|
if qaquery:
|
||||||
|
if isNeg:
|
||||||
|
qaquery += " except "
|
||||||
|
else:
|
||||||
|
qaquery += " intersect "
|
||||||
|
elif isNeg:
|
||||||
|
qaquery += "select id from cards except "
|
||||||
|
value = value.replace("*", "%")
|
||||||
|
args["_ff_%d" % c] = "%"+value+"%"
|
||||||
|
|
||||||
|
if field == 'question':
|
||||||
|
qaquery += """
|
||||||
|
select id from cards where question like :_ff_%d escape '\\'""" % c
|
||||||
|
else:
|
||||||
|
qaquery += """
|
||||||
|
select id from cards where answer like :_ff_%d escape '\\'""" % c
|
||||||
|
elif type == SEARCH_DISTINCT:
|
||||||
|
if isNeg is False:
|
||||||
|
showdistinct = True if token == "one" else False
|
||||||
|
else:
|
||||||
|
showdistinct = False if token == "one" else True
|
||||||
|
else:
|
||||||
|
if (filter != 'none'):
|
||||||
|
filters.append(
|
||||||
|
{'scope': 'fact', 'type': filter,
|
||||||
|
'value': token, 'is_neg': isNeg})
|
||||||
|
else:
|
||||||
|
if fquery:
|
||||||
|
if isNeg:
|
||||||
|
fquery += " except "
|
||||||
|
else:
|
||||||
|
fquery += " intersect "
|
||||||
|
elif isNeg:
|
||||||
|
fquery += "select id from facts except "
|
||||||
|
token = token.replace("*", "%")
|
||||||
|
args["_ff_%d" % c] = "%"+token+"%"
|
||||||
|
fquery += """
|
||||||
|
select id from facts where cache like :_ff_%d escape '\\'""" % c
|
||||||
|
return (tquery, fquery, qquery, fidquery, cmquery, sfquery,
|
||||||
|
qaquery, showdistinct, filters, args)
|
||||||
|
|
||||||
|
# Find and replace
|
||||||
|
##########################################################################
|
||||||
|
|
||||||
|
def findReplace(deck, factIds, src, dst, isRe=False, field=None):
|
||||||
|
"Find and replace fields in a fact."
|
||||||
|
# find
|
||||||
|
s = "select id, factId, value from fields where factId in %s"
|
||||||
|
if isRe:
|
||||||
|
isRe = re.compile(src)
|
||||||
|
else:
|
||||||
|
s += " and value like :v"
|
||||||
|
if field:
|
||||||
|
s += " and fieldModelId = :fmid"
|
||||||
|
rows = deck.db.all(s % ids2str(factIds),
|
||||||
|
v="%"+src.replace("%", "%%")+"%",
|
||||||
|
fmid=field)
|
||||||
|
modded = []
|
||||||
|
if isRe:
|
||||||
|
modded = [
|
||||||
|
{'id': id, 'fid': fid, 'val': re.sub(isRe, dst, val)}
|
||||||
|
for (id, fid, val) in rows
|
||||||
|
if isRe.search(val)]
|
||||||
|
else:
|
||||||
|
modded = [
|
||||||
|
{'id': id, 'fid': fid, 'val': val.replace(src, dst)}
|
||||||
|
for (id, fid, val) in rows
|
||||||
|
if val.find(src) != -1]
|
||||||
|
# update
|
||||||
|
if modded:
|
||||||
|
deck.db.statements(
|
||||||
|
'update fields set value = :val where id = :id', modded)
|
||||||
|
deck.updateCardQACacheFromIds([f['fid'] for f in modded],
|
||||||
|
type="facts")
|
||||||
|
if field:
|
||||||
|
deck.updateFieldChecksums(field)
|
||||||
|
else:
|
||||||
|
deck.updateAllFieldChecksums()
|
||||||
|
return len(set([f['fid'] for f in modded]))
|
||||||
|
|
||||||
|
# Find duplicates
|
||||||
|
##########################################################################
|
||||||
|
|
||||||
|
def findDuplicates(deck, fmids):
|
||||||
|
data = deck.db.all(
|
||||||
|
"select factId, value from fields where fieldModelId in %s" %
|
||||||
|
ids2str(fmids))
|
||||||
|
vals = {}
|
||||||
|
for (fid, val) in data:
|
||||||
|
if not val.strip():
|
||||||
|
continue
|
||||||
|
if val not in vals:
|
||||||
|
vals[val] = [fid]
|
||||||
|
else:
|
||||||
|
vals[val].append(fid)
|
||||||
|
return [(k,v) for (k,v) in vals.items() if len(v) > 1]
|
||||||
Loading…
Reference in a new issue