siphon_process.py revision 440cde68
1#!/usr/bin/env python
2# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at:
7#
8#     http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16# Filter for .siphon files that are generated by other filters.
17# The idea is to siphon off certain initializers so that we can better
18# auto-document the contents of that initializer.
19
20import os, sys, re, argparse, cgi, json
21import pyparsing as pp
22
23import pprint
24
25DEFAULT_SIPHON ="clicmd"
26DEFAULT_OUTPUT = None
27DEFAULT_PREFIX = os.getcwd()
28
29siphon_map = {
30    'clicmd': "VLIB_CLI_COMMAND",
31}
32
33ap = argparse.ArgumentParser()
34ap.add_argument("--type", '-t', metavar="siphon_type", default=DEFAULT_SIPHON,
35        choices=siphon_map.keys(),
36        help="Siphon type to process [%s]" % DEFAULT_SIPHON)
37ap.add_argument("--output", '-o', metavar="directory", default=DEFAULT_OUTPUT,
38        help="Output directory for .md files [%s]" % DEFAULT_OUTPUT)
39ap.add_argument("--input-prefix", metavar="path", default=DEFAULT_PREFIX,
40        help="Prefix to strip from input pathnames [%s]" % DEFAULT_PREFIX)
41ap.add_argument("input", nargs='+', metavar="input_file",
42        help="Input .siphon files")
43args = ap.parse_args()
44
45if args.output is None:
46    sys.stderr.write("Error: Siphon processor requires --output to be set.")
47    sys.exit(1)
48
49
50def clicmd_index_sort(cfg, group, dec):
51    if group in dec and 'group_label' in dec[group]:
52        return dec[group]['group_label']
53    return group
54
55def clicmd_index_header(cfg):
56    s = "# CLI command index\n"
57    s += "\n[TOC]\n"
58    return s
59
60def clicmd_index_section(cfg, group, md):
61    return "\n@subpage %s\n\n" % md
62
63def clicmd_index_entry(cfg, meta, item):
64    v = item["value"]
65    return "* [%s](@ref %s)\n" % (v["path"], meta["label"])
66
67def clicmd_sort(cfg, meta, item):
68    return item['value']['path']
69
70def clicmd_header(cfg, group, md, dec):
71    if group in dec and 'group_label' in dec[group]:
72        label = dec[group]['group_label']
73    else:
74        label = group
75    return "\n@page %s %s\n" % (md, label)
76
77def clicmd_format(cfg, meta, item):
78    v = item["value"]
79    s = "\n@section %s %s\n" % (meta['label'], v['path'])
80
81    # The text from '.short_help = '.
82    # Later we should split this into short_help and usage_help
83    # since the latter is how it is primarily used but the former
84    # is also needed.
85    if "short_help" in v:
86        tmp = v["short_help"].strip()
87
88        # Bit hacky. Add a trailing period if it doesn't have one.
89        if tmp[-1] != ".":
90            tmp += "."
91
92        s += "### Summary/usage\n    %s\n\n" % tmp
93
94    # This is seldom used and will likely be deprecated
95    if "long_help" in v:
96        tmp = v["long_help"]
97
98        s += "### Long help\n    %s\n\n" % tmp
99
100    # Extracted from the code in /*? ... ?*/ blocks
101    if "siphon_block" in item["meta"]:
102        sb = item["meta"]["siphon_block"]
103
104        if sb != "":
105            # hack. still needed?
106            sb = sb.replace("\n", "\\n")
107            try:
108                sb = json.loads('"'+sb+'"')
109                s += "### Description\n%s\n\n" % sb
110            except:
111                pass
112
113    # Gives some developer-useful linking
114    if "item" in meta or "function" in v:
115        s += "### Declaration and implementation\n\n"
116
117        if "item" in meta:
118            s += "Declaration: @ref %s (%s:%d)\n\n" % \
119                (meta['item'], meta["file"], int(item["meta"]["line_start"]))
120
121        if "function" in v:
122            s += "Implementation: @ref %s.\n\n" % v["function"]
123
124    return s
125
126
127siphons = {
128    "VLIB_CLI_COMMAND": {
129        "index_sort_key": clicmd_index_sort,
130        "index_header": clicmd_index_header,
131        "index_section": clicmd_index_section,
132        "index_entry": clicmd_index_entry,
133        'sort_key': clicmd_sort,
134        "header": clicmd_header,
135        "format": clicmd_format,
136    }
137}
138
139
140# PyParsing definition for our struct initializers which look like this:
141# VLIB_CLI_COMMAND (show_sr_tunnel_command, static) = {
142#    .path = "show sr tunnel",
143#    .short_help = "show sr tunnel [name <sr-tunnel-name>]",
144#    .function = show_sr_tunnel_fn,
145#};
146def getMacroInitializerBNF():
147    cs = pp.Forward()
148    ident = pp.Word(pp.alphas + "_", pp.alphas + pp.nums + "_")
149    intNum = pp.Word(pp.nums)
150    hexNum = pp.Literal("0x") + pp.Word(pp.hexnums)
151    octalNum = pp.Literal("0") + pp.Word("01234567")
152    integer = (hexNum | octalNum | intNum) + \
153        pp.Optional(pp.Literal("ULL") | pp.Literal("LL") | pp.Literal("L"))
154    floatNum = pp.Regex(r'\d+(\.\d*)?([eE]\d+)?') + pp.Optional(pp.Literal("f"))
155    char = pp.Literal("'") + pp.Word(pp.printables, exact=1) + pp.Literal("'")
156    arrayIndex = integer | ident
157
158    lbracket = pp.Literal("(").suppress()
159    rbracket = pp.Literal(")").suppress()
160    lbrace = pp.Literal("{").suppress()
161    rbrace = pp.Literal("}").suppress()
162    comma = pp.Literal(",").suppress()
163    equals = pp.Literal("=").suppress()
164    dot = pp.Literal(".").suppress()
165    semicolon = pp.Literal(";").suppress()
166
167    # initializer := { [member = ] (variable | expression | { initializer } ) }
168    typeName = ident
169    varName = ident
170
171    typeSpec = pp.Optional("unsigned") + \
172               pp.oneOf("int long short float double char u8 i8 void") + \
173               pp.Optional(pp.Word("*"), default="")
174    typeCast = pp.Combine( "(" + ( typeSpec | typeName ) + ")" ).suppress()
175
176    string = pp.Combine(pp.OneOrMore(pp.QuotedString(quoteChar='"',
177        escChar='\\', multiline=True)), adjacent=False)
178    literal = pp.Optional(typeCast) + (integer | floatNum | char | string)
179    var = pp.Combine(pp.Optional(typeCast) + varName + pp.Optional("[" + arrayIndex + "]"))
180
181    expr = (literal | var) # TODO
182
183
184    member = pp.Combine(dot + varName + pp.Optional("[" + arrayIndex + "]"), adjacent=False)
185    value = (expr | cs)
186
187    entry = pp.Group(pp.Optional(member + equals, default="") + value)
188    entries = (pp.ZeroOrMore(entry + comma) + entry + pp.Optional(comma)) | \
189              (pp.ZeroOrMore(entry + comma))
190
191    cs << (lbrace + entries + rbrace)
192
193    macroName = ident
194    params = pp.Group(pp.ZeroOrMore(expr + comma) + expr)
195    macroParams = lbracket + params + rbracket
196
197    mi = macroName + pp.Optional(macroParams) + equals + pp.Group(cs) + semicolon
198    mi.ignore(pp.cppStyleComment)
199    return mi
200
201
202mi = getMacroInitializerBNF()
203
204# Parse the input file into a more usable dictionary structure
205cmds = {}
206line_num = 0
207line_start = 0
208for filename in args.input:
209    sys.stderr.write("Parsing items in file \"%s\"...\n" % filename)
210    data = None
211    with open(filename, "r") as fd:
212        data = json.load(fd)
213
214    cmds['_global'] = data['global']
215
216    # iterate the items loaded and regroup it
217    for item in data["items"]:
218        try:
219            o = mi.parseString(item['block']).asList()
220        except:
221            sys.stderr.write("Exception parsing item: %s\n%s\n" \
222                    % (json.dumps(item, separators=(',', ': '), indent=4),
223                        item['block']))
224            raise
225
226        group = item['group']
227        file = item['file']
228        macro = o[0]
229        param = o[1][0]
230
231        if group not in cmds:
232            cmds[group] = {}
233
234        if file not in cmds[group]:
235            cmds[group][file] = {}
236
237        if macro not in cmds[group][file]:
238            cmds[group][file][macro] = {}
239
240        c = {
241            'params': o[2],
242            'meta': {},
243            'value': {},
244        }
245
246        for key in item:
247            if key == 'block':
248                continue
249            c['meta'][key] = item[key]
250
251        for i in c['params']:
252            c['value'][i[0]] = cgi.escape(i[1])
253
254        cmds[group][file][macro][param] = c
255
256
257# Write the header for this siphon type
258cfg = siphons[siphon_map[args.type]]
259sys.stdout.write(cfg["index_header"](cfg))
260contents = ""
261
262def group_sort_key(item):
263    if "index_sort_key" in cfg:
264        return cfg["index_sort_key"](cfg, item, cmds['_global'])
265    return item
266
267# Iterate the dictionary and process it
268for group in sorted(cmds.keys(), key=group_sort_key):
269    if group.startswith('_'):
270        continue
271
272    sys.stderr.write("Processing items in group \"%s\"...\n" % group)
273
274    cfg = siphons[siphon_map[args.type]]
275    md = group.replace("/", "_").replace(".", "_")
276    sys.stdout.write(cfg["index_section"](cfg, group, md))
277
278    if "header" in cfg:
279        dec = cmds['_global']
280        contents += cfg["header"](cfg, group, md, dec)
281
282    for file in sorted(cmds[group].keys()):
283        if group.startswith('_'):
284            continue
285
286        sys.stderr.write("- Processing items in file \"%s\"...\n" % file)
287
288        for macro in sorted(cmds[group][file].keys()):
289            if macro != siphon_map[args.type]:
290                continue
291            sys.stderr.write("-- Processing items in macro \"%s\"...\n" % macro)
292            cfg = siphons[macro]
293
294            meta = {
295                "group": group,
296                "file": file,
297                "macro": macro,
298                "md": md,
299            }
300
301            def item_sort_key(item):
302                if "sort_key" in cfg:
303                    return cfg["sort_key"](cfg, meta, cmds[group][file][macro][item])
304                return item
305
306            for param in sorted(cmds[group][file][macro].keys(), key=item_sort_key):
307                sys.stderr.write("--- Processing item \"%s\"...\n" % param)
308
309                meta["item"] = param
310
311                # mangle "md" and the item to make a reference label
312                meta["label"] = "%s___%s" % (meta["md"], param)
313
314                if "index_entry" in cfg:
315                    s = cfg["index_entry"](cfg, meta, cmds[group][file][macro][param])
316                    sys.stdout.write(s)
317
318                if "format" in cfg:
319                    contents += cfg["format"](cfg, meta, cmds[group][file][macro][param])
320
321sys.stdout.write(contents)
322
323# All done
324