add guid and flags to facts

Syncing and shared decks have conflicting priorities:

- For syncing, we need to ensure that the deck remains in a consistent state.
  In the past, Anki allowed deletions to be overriden by a more recently
  modified object, but this could lead to a broken deck in certain
  circumstances. For example, if a user deletes a fact (and its cards) on one
  side, but does something to bump a card's mod time on another side, then
  when syncing the card would be brought back to life without its fact. Short
  of complex code to check all the relations, we're limited to two options:
  forcing a full sync when things are deleted, or ensuring objects can't come
  back to life.

- When facts are shared between people, we need a way to identify if two facts
  arose from the same source. We can't compare based on content, as the
  content may have changed partially or completely. And we can't use the
  timestamp ids because of the above restriction on bringing objects back to
  life. If we did that, people could download a shared deck, decide they don't
  want it, and delete it. When they later decide to add it again, it wouldn't
  be possible: either nothing would be imported because of the old graves, or
  the ids would have to be rewritten. If we do the latter, the facts are no
  longer associated with each other, and we lose the ability to update the
  deck.

So we need to give facts two IDs: one used as the primary key and for syncing,
and another 'global id' for importing/sharing. I used a 64 bit random number,
because a) it's what Anki's used in the past, so by reusing the old IDs we
don't break existing associations on upgrade, and b) it's a decent compromise
between the possibility of conflicts and performance.

Also re-added a flags column to the facts. The 'data' column is intended to
store JSON in the future for extra features without changing the schema, but
that's slow for simple state checks. Flags will be used as a bitmask.

anki/facts.py

@@ -5,7 +5,7 @@
 import time
 from anki.errors import AnkiError
 from anki.utils import fieldChecksum, intTime, \
-    joinFields, splitFields, ids2str, stripHTML, timestampID
+    joinFields, splitFields, ids2str, stripHTML, timestampID, guid64
 
 class Fact(object):
 
@@ -17,23 +17,28 @@ class Fact(object):
             self.load()
         else:
             self.id = timestampID(deck.db, "facts")
+            self.guid = guid64()
             self._model = model
             self.gid = model['gid']
             self.mid = model['id']
             self.tags = []
             self.fields = [""] * len(self._model['flds'])
+            self.flags = 0
             self.data = ""
             self._fmap = self.deck.models.fieldMap(self._model)
 
     def load(self):
-        (self.mid,
+        (self.guid,
+         self.mid,
          self.gid,
          self.mod,
          self.usn,
          self.tags,
          self.fields,
+         self.flags,
          self.data) = self.deck.db.first("""
-select mid, gid, mod, usn, tags, flds, data from facts where id = ?""", self.id)
+select guid, mid, gid, mod, usn, tags, flds, flags, data
+from facts where id = ?""", self.id)
         self.fields = splitFields(self.fields)
         self.tags = self.deck.tags.split(self.tags)
         self._model = self.deck.models.get(self.mid)
@@ -45,10 +50,10 @@ select mid, gid, mod, usn, tags, flds, data from facts where id = ?""", self.id)
         sfld = stripHTML(self.fields[self.deck.models.sortIdx(self._model)])
         tags = self.stringTags()
         res = self.deck.db.execute("""
-insert or replace into facts values (?, ?, ?, ?, ?, ?, ?, ?, ?)""",
-                            self.id, self.mid, self.gid,
+insert or replace into facts values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)""",
+                            self.id, self.guid, self.mid, self.gid,
                             self.mod, self.usn, tags,
-                            self.joinedFields(), sfld, self.data)
+                            self.joinedFields(), sfld, self.flags, self.data)
         self.id = res.lastrowid
         self.updateFieldChecksums()
         self.deck.tags.register(self.tags)

anki/storage.py

@@ -74,6 +74,7 @@ create table if not exists deck (
 
 create table if not exists facts (
     id              integer primary key,
+    guid            integer not null,
     mid             integer not null,
     gid             integer not null,
     mod             integer not null,
@@ -81,6 +82,7 @@ create table if not exists facts (
     tags            text not null,
     flds            text not null,
     sfld            integer not null,
+    flags           integer not null,
     data            text not null
 );
 
@@ -205,7 +207,7 @@ end)
 """)
     # pull facts into memory, so we can merge them with fields efficiently
     facts = db.all("""
-select id, modelId, 1, cast(created*1000 as int), cast(modified as int), 0, tags
+select id, id, modelId, 1, cast(created*1000 as int), cast(modified as int), 0, tags
 from facts order by created""")
     # build field hash
     fields = {}
@@ -225,19 +227,19 @@ from facts order by created""")
         oldid = row[0]
         row = list(row)
         # get rid of old created column and update id
-        while row[3] in times:
-            row[3] += 1
-        times[row[3]] = True
-        factidmap[row[0]] = row[3]
-        row[0] = row[3]
-        del row[3]
+        while row[4] in times:
+            row[4] += 1
+        times[row[4]] = True
+        factidmap[row[0]] = row[4]
+        row[0] = row[4]
+        del row[4]
         map[oldid] = row[0]
         row.append(minimizeHTML("\x1f".join([x[1] for x in sorted(fields[oldid])])))
         data.append(row)
     # and put the facts into the new table
     db.execute("drop table facts")
     _addSchema(db, False)
-    db.executemany("insert into facts values (?,?,?,?,?,?,?,'','')", data)
+    db.executemany("insert into facts values (?,?,?,?,?,?,?,?,'',0,'')", data)
     db.execute("drop table fields")
 
     # cards

anki/utils.py

@@ -182,6 +182,9 @@ def timestampID(db, table):
         t += 1
     return t
 
+def guid64():
+    return random.randint(-sys.maxint-1, sys.maxint)
+
 # Fields
 ##############################################################################