convert guid to base 91 string

Because JSON doesn't support 64 bit numbers, we need to either convert the 64
bit numbers to a string during transport, or store the ids as a string. At
base 91 a 64 bit number only takes an extra two bytes, and it means we can
dump DB results directly into JSON without having to apply any transformation.

anki/storage.py

@@ -83,7 +83,7 @@ create table if not exists col (
 
 create table if not exists notes (
     id              integer primary key,
-    guid            integer not null,
+    guid            text not null,
     mid             integer not null,
     did             integer not null,
     mod             integer not null,

anki/sync.py

@@ -15,13 +15,10 @@ from hooks import runHook
 if simplejson.__version__ < "1.7.3":
     raise Exception("SimpleJSON must be 1.7.3 or later.")
 
-# - 64 bit guid will be munged in js; need to escape or rethink
-
 # - make sure /sync/download is compressed
 # - status() should be using the hooks instead
 
 # todo:
-# - ensure all urllib references are converted to urllib2 for proxies
 # - ability to cancel
 # - need to make sure syncing doesn't bump the col modified time if nothing was
 #    changed, since by default closing the col bumps the mod time

anki/upgrade.py

@@ -4,7 +4,7 @@
 
 import os, time, simplejson, re, datetime, shutil
 from anki.lang import _
-from anki.utils import intTime, tmpfile, ids2str, splitFields
+from anki.utils import intTime, tmpfile, ids2str, splitFields, base91
 from anki.db import DB
 from anki.collection import _Collection
 from anki.consts import *
@@ -175,6 +175,8 @@ select id, id, modelId, 1, cast(created*1000 as int), cast(modified as int),
             row[0] = row[4]
             del row[4]
             map[oldid] = row[0]
+            # convert old 64bit id into a string, discarding sign bit
+            row[1] = base91(abs(row[1]))
             row.append(minimizeHTML("\x1f".join([x[1] for x in sorted(fields[oldid])])))
             data.append(row)
         # and put the facts into the new table

anki/utils.py

@@ -3,7 +3,7 @@
 # License: GNU AGPL, version 3 or later; http://www.gnu.org/licenses/agpl.html
 
 import re, os, random, time, types, math, htmlentitydefs, subprocess, \
-    tempfile, shutil
+    tempfile, shutil, string
 from hashlib import sha1
 from anki.lang import _, ngettext
 import locale, sys
@@ -190,12 +190,19 @@ def maxID(db):
                 "select max(id) from %s" % tbl))
     return now + 1
 
-def guid64():
+def base91(num):
-    return random.randint(-sys.maxint-1, sys.maxint)
+    s = string
+    # all printable characters minus quotes, backslash and separators
+    table = s.letters + s.digits + "!#$%&()*+,-./:;<=>?@[]^_`{|}~"
+    buf = ""
+    while num:
+        num, i = divmod(num, len(table))
+        buf = table[i] + buf
+    return buf
 
-def guid32():
+def guid64():
-    max = 2**32
+    "Return a base91-encoded 64bit random number."
-    return random.randint(0, max-1)
+    return base91(random.randint(0, 2**64-1))
 
 # Fields
 ##############################################################################