0001"""
0002Implementation of JSONDecoder
0003"""
0004import re
0005
0006from simplejson.scanner import Scanner, pattern
0007
0008FLAGS = re.VERBOSE | re.MULTILINE | re.DOTALL
0009
0010def _floatconstants():
0011 import struct
0012 import sys
0013 _BYTES = '7FF80000000000007FF0000000000000'.decode('hex')
0014 if sys.byteorder != 'big':
0015 _BYTES = _BYTES[:8][::-1] + _BYTES[8:][::-1]
0016 nan, inf = struct.unpack('dd', _BYTES)
0017 return nan, inf, -inf
0018
0019NaN, PosInf, NegInf = _floatconstants()
0020
0021def linecol(doc, pos):
0022 lineno = doc.count('\n', 0, pos) + 1
0023 if lineno == 1:
0024 colno = pos
0025 else:
0026 colno = pos - doc.rindex('\n', 0, pos)
0027 return lineno, colno
0028
0029def errmsg(msg, doc, pos, end=None):
0030 lineno, colno = linecol(doc, pos)
0031 if end is None:
0032 return '%s: line %d column %d (char %d)' % (msg, lineno, colno, pos)
0033 endlineno, endcolno = linecol(doc, end)
0034 return '%s: line %d column %d - line %d column %d (char %d - %d)' % (
0035 msg, lineno, colno, endlineno, endcolno, pos, end)
0036
0037_CONSTANTS = {
0038 '-Infinity': NegInf,
0039 'Infinity': PosInf,
0040 'NaN': NaN,
0041 'true': True,
0042 'false': False,
0043 'null': None,
0044}
0045
0046def JSONConstant(match, context, c=_CONSTANTS):
0047 return c[match.group(0)], None
0048pattern('(-?Infinity|NaN|true|false|null)')(JSONConstant)
0049
0050def JSONNumber(match, context):
0051 match = JSONNumber.regex.match(match.string, *match.span())
0052 integer, frac, exp = match.groups()
0053 if frac or exp:
0054 res = float(integer + (frac or '') + (exp or ''))
0055 else:
0056 res = int(integer)
0057 return res, None
0058pattern(r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?')(JSONNumber)
0059
0060STRINGCHUNK = re.compile(r'(.*?)(["\\])', FLAGS)
0061BACKSLASH = {
0062 '"': u'"', '\\': u'\\', '/': u'/',
0063 'b': u'\b', 'f': u'\f', 'n': u'\n', 'r': u'\r', 't': u'\t',
0064}
0065
0066DEFAULT_ENCODING = "utf-8"
0067
0068def scanstring(s, end, encoding=None, _b=BACKSLASH, _m=STRINGCHUNK.match):
0069 if encoding is None:
0070 encoding = DEFAULT_ENCODING
0071 chunks = []
0072 _append = chunks.append
0073 begin = end - 1
0074 while 1:
0075 chunk = _m(s, end)
0076 if chunk is None:
0077 raise ValueError(
0078 errmsg("Unterminated string starting at", s, begin))
0079 end = chunk.end()
0080 content, terminator = chunk.groups()
0081 if content:
0082 if not isinstance(content, unicode):
0083 content = unicode(content, encoding)
0084 _append(content)
0085 if terminator == '"':
0086 break
0087 try:
0088 esc = s[end]
0089 except IndexError:
0090 raise ValueError(
0091 errmsg("Unterminated string starting at", s, begin))
0092 if esc != 'u':
0093 try:
0094 m = _b[esc]
0095 except KeyError:
0096 raise ValueError(
0097 errmsg("Invalid \\escape: %r" % (esc,), s, end))
0098 end += 1
0099 else:
0100 esc = s[end + 1:end + 5]
0101 try:
0102 m = unichr(int(esc, 16))
0103 if len(esc) != 4 or not esc.isalnum():
0104 raise ValueError
0105 except ValueError:
0106 raise ValueError(errmsg("Invalid \\uXXXX escape", s, end))
0107 end += 5
0108 _append(m)
0109 return u''.join(chunks), end
0110
0111def JSONString(match, context):
0112 encoding = getattr(context, 'encoding', None)
0113 return scanstring(match.string, match.end(), encoding)
0114pattern(r'"')(JSONString)
0115
0116WHITESPACE = re.compile(r'\s*', FLAGS)
0117
0118def JSONObject(match, context, _w=WHITESPACE.match):
0119 pairs = {}
0120 s = match.string
0121 end = _w(s, match.end()).end()
0122 nextchar = s[end:end + 1]
0123
0124 if nextchar == '}':
0125 return pairs, end + 1
0126 if nextchar != '"':
0127 raise ValueError(errmsg("Expecting property name", s, end))
0128 end += 1
0129 encoding = getattr(context, 'encoding', None)
0130 while True:
0131 key, end = scanstring(s, end, encoding)
0132 end = _w(s, end).end()
0133 if s[end:end + 1] != ':':
0134 raise ValueError(errmsg("Expecting : delimiter", s, end))
0135 end = _w(s, end + 1).end()
0136 try:
0137 value, end = JSONScanner.iterscan(s, idx=end).next()
0138 except StopIteration:
0139 raise ValueError(errmsg("Expecting object", s, end))
0140 pairs[key] = value
0141 end = _w(s, end).end()
0142 nextchar = s[end:end + 1]
0143 end += 1
0144 if nextchar == '}':
0145 break
0146 if nextchar != ',':
0147 raise ValueError(errmsg("Expecting , delimiter", s, end - 1))
0148 end = _w(s, end).end()
0149 nextchar = s[end:end + 1]
0150 end += 1
0151 if nextchar != '"':
0152 raise ValueError(errmsg("Expecting property name", s, end - 1))
0153 object_hook = getattr(context, 'object_hook', None)
0154 if object_hook is not None:
0155 pairs = object_hook(pairs)
0156 return pairs, end
0157pattern(r'{')(JSONObject)
0158
0159def JSONArray(match, context, _w=WHITESPACE.match):
0160 values = []
0161 s = match.string
0162 end = _w(s, match.end()).end()
0163
0164 nextchar = s[end:end + 1]
0165 if nextchar == ']':
0166 return values, end + 1
0167 while True:
0168 try:
0169 value, end = JSONScanner.iterscan(s, idx=end).next()
0170 except StopIteration:
0171 raise ValueError(errmsg("Expecting object", s, end))
0172 values.append(value)
0173 end = _w(s, end).end()
0174 nextchar = s[end:end + 1]
0175 end += 1
0176 if nextchar == ']':
0177 break
0178 if nextchar != ',':
0179 raise ValueError(errmsg("Expecting , delimiter", s, end))
0180 end = _w(s, end).end()
0181 return values, end
0182pattern(r'\[')(JSONArray)
0183
0184ANYTHING = [
0185 JSONObject,
0186 JSONArray,
0187 JSONString,
0188 JSONConstant,
0189 JSONNumber,
0190]
0191
0192JSONScanner = Scanner(ANYTHING)
0193
0194class JSONDecoder(object):
0195 """
0196 Simple JSON <http://json.org> decoder
0197
0198 Performs the following translations in decoding:
0199
0200 +---------------+-------------------+
0201 | JSON | Python |
0202 +===============+===================+
0203 | object | dict |
0204 +---------------+-------------------+
0205 | array | list |
0206 +---------------+-------------------+
0207 | string | unicode |
0208 +---------------+-------------------+
0209 | number (int) | int, long |
0210 +---------------+-------------------+
0211 | number (real) | float |
0212 +---------------+-------------------+
0213 | true | True |
0214 +---------------+-------------------+
0215 | false | False |
0216 +---------------+-------------------+
0217 | null | None |
0218 +---------------+-------------------+
0219
0220 It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as
0221 their corresponding ``float`` values, which is outside the JSON spec.
0222 """
0223
0224 _scanner = Scanner(ANYTHING)
0225 __all__ = ['__init__', 'decode', 'raw_decode']
0226
0227 def __init__(self, encoding=None, object_hook=None):
0228 """
0229 ``encoding`` determines the encoding used to interpret any ``str``
0230 objects decoded by this instance (utf-8 by default). It has no
0231 effect when decoding ``unicode`` objects.
0232
0233 Note that currently only encodings that are a superset of ASCII work,
0234 strings of other encodings should be passed in as ``unicode``.
0235
0236 ``object_hook``, if specified, will be called with the result
0237 of every JSON object decoded and its return value will be used in
0238 place of the given ``dict``. This can be used to provide custom
0239 deserializations (e.g. to support JSON-RPC class hinting).
0240 """
0241 self.encoding = encoding
0242 self.object_hook = object_hook
0243
0244 def decode(self, s, _w=WHITESPACE.match):
0245 """
0246 Return the Python representation of ``s`` (a ``str`` or ``unicode``
0247 instance containing a JSON document)
0248 """
0249 obj, end = self.raw_decode(s, idx=_w(s, 0).end())
0250 end = _w(s, end).end()
0251 if end != len(s):
0252 raise ValueError(errmsg("Extra data", s, end, len(s)))
0253 return obj
0254
0255 def raw_decode(self, s, **kw):
0256 """
0257 Decode a JSON document from ``s`` (a ``str`` or ``unicode`` beginning
0258 with a JSON document) and return a 2-tuple of the Python
0259 representation and the index in ``s`` where the document ended.
0260
0261 This can be used to decode a JSON document from a string that may
0262 have extraneous data at the end.
0263 """
0264 kw.setdefault('context', self)
0265 try:
0266 obj, end = self._scanner.iterscan(s, **kw).next()
0267 except StopIteration:
0268 raise ValueError("No JSON object could be decoded")
0269 return obj, end
0270
0271__all__ = ['JSONDecoder']