@@ -13,18 +13,29 @@ from ordereddict import OrderedDict
def tokenize(data):
while len(data):
- if data[0] in ['{', '}', ':', ',', '[', ']']:
- yield data[0]
- data = data[1:]
- elif data[0] in ' \n':
- data = data[1:]
- elif data[0] == "'":
- data = data[1:]
+ ch = data[0]
+ data = data[1:]
+ if ch in ['{', '}', ':', ',', '[', ']']:
+ yield ch
+ elif ch in ' \n':
+ None
+ elif ch == "'":
string = ''
- while data[0] != "'":
- string += data[0]
+ esc = False
+ while True:
+ if (data == ''):
+ raise Exception("Mismatched quotes")
+ ch = data[0]
data = data[1:]
- data = data[1:]
+ if esc:
+ string += ch
+ esc = False
+ elif ch == "\\":
+ esc = True
+ elif ch == "'":
+ break
+ else:
+ string += ch
yield string
def parse(tokens):