vapi_json_parser.py revision 1b1ccadc
1#!/usr/bin/env python3
2
3import json
4
5
6class ParseError (Exception):
7    pass
8
9
10magic_prefix = "vl_api_"
11magic_suffix = "_t"
12
13
14def remove_magic(what):
15    if what.startswith(magic_prefix) and what.endswith(magic_suffix):
16        return what[len(magic_prefix): - len(magic_suffix)]
17    return what
18
19
20class Field(object):
21
22    def __init__(self, field_name, field_type, array_len=None,
23                 nelem_field=None):
24        self.name = field_name
25        self.type = field_type
26        self.len = array_len
27        self.nelem_field = nelem_field
28
29    def __str__(self):
30        if self.len is None:
31            return "Field(name: %s, type: %s)" % (self.name, self.type)
32        elif self.len > 0:
33            return "Field(name: %s, type: %s, length: %s)" % (self.name,
34                                                              self.type,
35                                                              self.len)
36        else:
37            return (
38                "Field(name: %s, type: %s, variable length stored in: %s)" %
39                (self.name, self.type, self.nelem_field))
40
41    def is_vla(self):
42        return self.nelem_field is not None
43
44    def has_vla(self):
45        return self.is_vla() or self.type.has_vla()
46
47
48class Alias(Field):
49    pass
50
51
52class Type(object):
53    def __init__(self, name):
54        self.name = name
55
56    def __str__(self):
57        return self.name
58
59
60class SimpleType (Type):
61
62    def has_vla(self):
63        return False
64
65
66def get_msg_header_defs(struct_type_class, field_class, json_parser, logger):
67    return [
68        struct_type_class(['msg_header1_t',
69                           ['u16', '_vl_msg_id'],
70                           ['u32', 'context'],
71                           ],
72                          json_parser, field_class, logger
73                          ),
74        struct_type_class(['msg_header2_t',
75                           ['u16', '_vl_msg_id'],
76                           ['u32', 'client_index'],
77                           ['u32', 'context'],
78                           ],
79                          json_parser, field_class, logger
80                          ),
81    ]
82
83
84class Struct(object):
85
86    def __init__(self, name, fields):
87        self.name = name
88        self.fields = fields
89        self.field_names = [n.name for n in self.fields]
90        self.depends = [f.type for f in self.fields]
91
92    def __str__(self):
93        return "[%s]" % "], [".join([str(f) for f in self.fields])
94
95    def has_vla(self):
96        for f in self.fields:
97            if f.has_vla():
98                return True
99        return False
100
101
102class Enum(SimpleType):
103    def __init__(self, name, value_pairs, enumtype):
104        super(Enum, self).__init__(name)
105        self.type = enumtype
106        self.value_pairs = value_pairs
107
108    def __str__(self):
109        return "Enum(%s, [%s])" % (
110            self.name,
111            "], [" .join(["%s => %s" % (i, j) for i, j in self.value_pairs])
112        )
113
114
115class Union(Type):
116    def __init__(self, name, type_pairs, crc):
117        Type.__init__(self, name)
118        self.crc = crc
119        self.type_pairs = type_pairs
120        self.depends = [t for t, _ in self.type_pairs]
121
122    def __str__(self):
123        return "Union(%s, [%s])" % (
124            self.name,
125            "], [" .join(["%s %s" % (i, j) for i, j in self.type_pairs])
126        )
127
128    def has_vla(self):
129        return False
130
131
132class Message(object):
133
134    def __init__(self, logger, definition, json_parser):
135        struct_type_class = json_parser.struct_type_class
136        field_class = json_parser.field_class
137        self.request = None
138        self.logger = logger
139        m = definition
140        logger.debug("Parsing message definition `%s'" % m)
141        name = m[0]
142        self.name = name
143        logger.debug("Message name is `%s'" % name)
144        ignore = True
145        self.header = None
146        self.is_reply = json_parser.is_reply(self.name)
147        self.is_event = json_parser.is_event(self.name)
148        fields = []
149        for header in get_msg_header_defs(struct_type_class, field_class,
150                                          json_parser, logger):
151            logger.debug("Probing header `%s'" % header.name)
152            if header.is_part_of_def(m[1:]):
153                self.header = header
154                logger.debug("Found header `%s'" % header.name)
155                fields.append(field_class(field_name='header',
156                                          field_type=self.header))
157                ignore = False
158                break
159        if ignore and not self.is_event and not self.is_reply:
160            raise ParseError("While parsing message `%s': could not find all "
161                             "common header fields" % name)
162        for field in m[1:]:
163            if len(field) == 1 and 'crc' in field:
164                self.crc = field['crc']
165                logger.debug("Found CRC `%s'" % self.crc)
166                continue
167            else:
168                field_type = json_parser.lookup_type_like_id(field[0])
169                logger.debug("Parsing message field `%s'" % field)
170                l = len(field)
171                if any(type(n) is dict for n in field):
172                    l -= 1
173                if l == 2:
174                    if self.header is not None and\
175                            self.header.has_field(field[1]):
176                        continue
177                    p = field_class(field_name=field[1],
178                                    field_type=field_type)
179                elif l == 3:
180                    if field[2] == 0 and field[0] != 'string':
181                        raise ParseError(
182                            "While parsing message `%s': variable length "
183                            "array `%s' doesn't have reference to member "
184                            "containing the actual length" % (
185                                name, field[1]))
186                    if field[0] == 'string' and field[2] > 0:
187                        field_type = json_parser.lookup_type_like_id('u8')
188
189                    p = field_class(
190                        field_name=field[1],
191                        field_type=field_type,
192                        array_len=field[2])
193                elif l == 4:
194                    nelem_field = None
195                    for f in fields:
196                        if f.name == field[3]:
197                            nelem_field = f
198                    if nelem_field is None:
199                        raise ParseError(
200                            "While parsing message `%s': couldn't find "
201                            "variable length array `%s' member containing "
202                            "the actual length `%s'" % (
203                                name, field[1], field[3]))
204                    p = field_class(
205                        field_name=field[1],
206                        field_type=field_type,
207                        array_len=field[2],
208                        nelem_field=nelem_field)
209                else:
210                    raise Exception("Don't know how to parse message "
211                                    "definition for message `%s': `%s'" %
212                                    (m, m[1:]))
213                logger.debug("Parsed field `%s'" % p)
214                fields.append(p)
215        self.fields = fields
216        self.depends = [f.type for f in self.fields]
217        logger.debug("Parsed message: %s" % self)
218
219    def __str__(self):
220        return "Message(%s, [%s], {crc: %s}" % \
221            (self.name,
222             "], [".join([str(f) for f in self.fields]),
223             self.crc)
224
225
226class StructType (Type, Struct):
227
228    def __init__(self, definition, json_parser, field_class, logger):
229        t = definition
230        logger.debug("Parsing struct definition `%s'" % t)
231        name = t[0]
232        fields = []
233        for field in t[1:]:
234            if len(field) == 1 and 'crc' in field:
235                self.crc = field['crc']
236                continue
237            field_type = json_parser.lookup_type_like_id(field[0])
238            logger.debug("Parsing type field `%s'" % field)
239            if len(field) == 2:
240                p = field_class(field_name=field[1],
241                                field_type=field_type)
242            elif len(field) == 3:
243                if field[2] == 0:
244                    raise ParseError("While parsing type `%s': array `%s' has "
245                                     "variable length" % (name, field[1]))
246                p = field_class(field_name=field[1],
247                                field_type=field_type,
248                                array_len=field[2])
249            elif len(field) == 4:
250                nelem_field = None
251                for f in fields:
252                    if f.name == field[3]:
253                        nelem_field = f
254                if nelem_field is None:
255                    raise ParseError(
256                        "While parsing message `%s': couldn't find "
257                        "variable length array `%s' member containing "
258                        "the actual length `%s'" % (
259                            name, field[1], field[3]))
260                p = field_class(field_name=field[1],
261                                field_type=field_type,
262                                array_len=field[2],
263                                nelem_field=nelem_field)
264            else:
265                raise ParseError(
266                    "Don't know how to parse field `%s' of type definition "
267                    "for type `%s'" % (field, t))
268            fields.append(p)
269        Type.__init__(self, name)
270        Struct.__init__(self, name, fields)
271
272    def __str__(self):
273        return "StructType(%s, %s)" % (Type.__str__(self),
274                                       Struct.__str__(self))
275
276    def has_field(self, name):
277        return name in self.field_names
278
279    def is_part_of_def(self, definition):
280        for idx in range(len(self.fields)):
281            field = definition[idx]
282            p = self.fields[idx]
283            if field[1] != p.name:
284                return False
285            if field[0] != p.type.name:
286                raise ParseError(
287                    "Unexpected field type `%s' (should be `%s'), "
288                    "while parsing msg/def/field `%s/%s/%s'" %
289                    (field[0], p.type, p.name, definition, field))
290        return True
291
292
293class JsonParser(object):
294    def __init__(self, logger, files, simple_type_class=SimpleType,
295                 enum_class=Enum, union_class=Union,
296                 struct_type_class=StructType, field_class=Field,
297                 message_class=Message, alias_class=Alias):
298        self.services = {}
299        self.messages = {}
300        self.enums = {}
301        self.unions = {}
302        self.aliases = {}
303        self.types = {
304            x: simple_type_class(x) for x in [
305                'i8', 'i16', 'i32', 'i64',
306                'u8', 'u16', 'u32', 'u64',
307                'f64', 'bool'
308            ]
309        }
310
311        self.types['string'] = simple_type_class('vl_api_string_t')
312        self.replies = set()
313        self.events = set()
314        self.simple_type_class = simple_type_class
315        self.enum_class = enum_class
316        self.union_class = union_class
317        self.struct_type_class = struct_type_class
318        self.field_class = field_class
319        self.alias_class = alias_class
320        self.message_class = message_class
321
322        self.exceptions = []
323        self.json_files = []
324        self.types_by_json = {}
325        self.enums_by_json = {}
326        self.unions_by_json = {}
327        self.aliases_by_json = {}
328        self.messages_by_json = {}
329        self.logger = logger
330        for f in files:
331            self.parse_json_file(f)
332        self.finalize_parsing()
333
334    def parse_json_file(self, path):
335        self.logger.info("Parsing json api file: `%s'" % path)
336        self.json_files.append(path)
337        self.types_by_json[path] = []
338        self.enums_by_json[path] = []
339        self.unions_by_json[path] = []
340        self.aliases_by_json[path] = []
341        self.messages_by_json[path] = {}
342        with open(path) as f:
343            j = json.load(f)
344            for k in j['services']:
345                if k in self.services:
346                    raise ParseError("Duplicate service `%s'" % k)
347                self.services[k] = j['services'][k]
348                self.replies.add(self.services[k]["reply"])
349                if "events" in self.services[k]:
350                    for x in self.services[k]["events"]:
351                        self.events.add(x)
352            for e in j['enums']:
353                name = e[0]
354                value_pairs = e[1:-1]
355                enumtype = self.types[e[-1]["enumtype"]]
356                enum = self.enum_class(name, value_pairs, enumtype)
357                self.enums[enum.name] = enum
358                self.logger.debug("Parsed enum: %s" % enum)
359                self.enums_by_json[path].append(enum)
360            exceptions = []
361            progress = 0
362            last_progress = 0
363            while True:
364                for u in j['unions']:
365                    name = u[0]
366                    if name in self.unions:
367                        progress = progress + 1
368                        continue
369                    try:
370                        type_pairs = [[self.lookup_type_like_id(t), n]
371                                      for t, n in u[1:]]
372                        union = self.union_class(name, type_pairs, 0)
373                        progress = progress + 1
374                    except ParseError as e:
375                        exceptions.append(e)
376                        continue
377                    self.unions[union.name] = union
378                    self.logger.debug("Parsed union: %s" % union)
379                    self.unions_by_json[path].append(union)
380                for t in j['types']:
381                    if t[0] in self.types:
382                        progress = progress + 1
383                        continue
384                    try:
385                        type_ = self.struct_type_class(t, self,
386                                                       self.field_class,
387                                                       self.logger)
388                        if type_.name in self.types:
389                            raise ParseError(
390                                "Duplicate type `%s'" % type_.name)
391                        progress = progress + 1
392                    except ParseError as e:
393                        exceptions.append(e)
394                        continue
395                    self.types[type_.name] = type_
396                    self.types_by_json[path].append(type_)
397                    self.logger.debug("Parsed type: %s" % type_)
398                for name, body in j['aliases'].items():
399                    if name in self.aliases:
400                        progress = progress + 1
401                        continue
402                    if 'length' in body:
403                        array_len = body['length']
404                    else:
405                        array_len = None
406                    try:
407                        t = self.lookup_type_like_id(body['type'])
408                    except ParseError as e:
409                        exceptions.append(e)
410                        continue
411                    alias = self.alias_class(name, t, array_len)
412                    self.aliases[name] = alias
413                    self.logger.debug("Parsed alias: %s" % alias)
414                    self.aliases_by_json[path].append(alias)
415                if not exceptions:
416                    # finished parsing
417                    break
418                if progress <= last_progress:
419                    # cannot make forward progress
420                    self.exceptions.extend(exceptions)
421                    break
422                exceptions = []
423                last_progress = progress
424                progress = 0
425            prev_length = len(self.messages)
426            processed = []
427            while True:
428                exceptions = []
429                for m in j['messages']:
430                    if m in processed:
431                        continue
432                    try:
433                        msg = self.message_class(self.logger, m, self)
434                        if msg.name in self.messages:
435                            raise ParseError(
436                                "Duplicate message `%s'" % msg.name)
437                    except ParseError as e:
438                        exceptions.append(e)
439                        continue
440                    self.messages[msg.name] = msg
441                    self.messages_by_json[path][msg.name] = msg
442                    processed.append(m)
443                if prev_length == len(self.messages):
444                    # cannot make forward progress ...
445                    self.exceptions.extend(exceptions)
446                    break
447                prev_length = len(self.messages)
448
449    def lookup_type_like_id(self, name):
450        mundane_name = remove_magic(name)
451        if name in self.types:
452            return self.types[name]
453        elif name in self.enums:
454            return self.enums[name]
455        elif name in self.unions:
456            return self.unions[name]
457        elif name in self.aliases:
458            return self.aliases[name]
459        elif mundane_name in self.types:
460            return self.types[mundane_name]
461        elif mundane_name in self.enums:
462            return self.enums[mundane_name]
463        elif mundane_name in self.unions:
464            return self.unions[mundane_name]
465        elif mundane_name in self.aliases:
466            return self.aliases[mundane_name]
467        raise ParseError(
468            "Could not find type, enum or union by magic name `%s' nor by "
469            "mundane name `%s'" % (name, mundane_name))
470
471    def is_reply(self, message):
472        return message in self.replies
473
474    def is_event(self, message):
475        return message in self.events
476
477    def get_reply(self, message):
478        return self.messages[self.services[message]['reply']]
479
480    def finalize_parsing(self):
481        if len(self.messages) == 0:
482            for e in self.exceptions:
483                self.logger.warning(e)
484        for jn, j in self.messages_by_json.items():
485            remove = []
486            for n, m in j.items():
487                try:
488                    if not m.is_reply and not m.is_event:
489                        try:
490                            m.reply = self.get_reply(n)
491                            if "stream" in self.services[m.name]:
492                                m.reply_is_stream = \
493                                    self.services[m.name]["stream"]
494                            else:
495                                m.reply_is_stream = False
496                            m.reply.request = m
497                        except:
498                            raise ParseError(
499                                "Cannot find reply to message `%s'" % n)
500                except ParseError as e:
501                    self.exceptions.append(e)
502                    remove.append(n)
503
504            self.messages_by_json[jn] = {
505                k: v for k, v in j.items() if k not in remove}
506