siphon_generate.py revision 440cde68
1#!/usr/bin/env python
2# Copyright (c) 2016 Comcast Cable Communications Management, LLC.
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at:
7#
8#     http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16# Looks for preprocessor macros with struct initializers and siphons them
17# off into another file for later parsing; ostensibly to generate
18# documentation from struct initializer data.
19
20import os, sys, re, argparse, json
21
22DEFAULT_OUTPUT = "build-root/docs/siphons"
23DEFAULT_PREFIX = os.getcwd()
24
25ap = argparse.ArgumentParser()
26ap.add_argument("--output", '-o', metavar="directory", default=DEFAULT_OUTPUT,
27        help="Output directory for .siphon files [%s]" % DEFAULT_OUTPUT)
28ap.add_argument("--input-prefix", metavar="path", default=DEFAULT_PREFIX,
29        help="Prefix to strip from input pathnames [%s]" % DEFAULT_PREFIX)
30ap.add_argument("input", nargs='+', metavar="input_file",
31        help="Input C source files")
32args = ap.parse_args()
33
34"""Patterns that match the start of code blocks we want to siphon"""
35siphon_patterns = [
36    ( re.compile("(?P<m>VLIB_CLI_COMMAND)\s*[(](?P<name>[a-zA-Z0-9_]+)(,[^)]*)?[)]"), "clicmd" ),
37]
38
39"""Matches a siphon comment block start"""
40siphon_block_start = re.compile("^\s*/\*\?\s*(.*)$")
41
42"""Matches a siphon comment block stop"""
43siphon_block_stop = re.compile("^(.*)\s*\?\*/\s*$")
44
45"""Siphon block directive delimiter"""
46siphon_block_delimiter = "%%"
47
48"""Matches a siphon block directive such as '%clicmd:group_label Debug CLI%'"""
49siphon_block_directive = re.compile("(%s)\s*([a-zA-Z0-9_:]+)\s+(.*)\s*(%s)" % \
50        (siphon_block_delimiter, siphon_block_delimiter))
51
52"""Matches the start of an initializer block"""
53siphon_initializer = re.compile("\s*=")
54
55"""
56count open and close braces in str
57return (0, index) when braces were found and count becomes 0.
58index indicates the position at which the last closing brace was
59found.
60return (-1, -1) if a closing brace is found before any opening one.
61return (count, -1) if not all opening braces are closed, count is the
62current depth
63"""
64def count_braces(str, count=0, found=False):
65    for index in range(0, len(str)):
66        if str[index] == '{':
67            count += 1;
68            found = True
69        elif str[index] == '}':
70            if count == 0:
71                # means we never found an open brace
72                return (-1, -1)
73            count -= 1;
74
75        if count == 0 and found:
76            return (count, index)
77
78    return (count, -1)
79
80# Collated output for each siphon
81output = {}
82
83# Build a list of known siphons
84known_siphons = []
85for item in siphon_patterns:
86	siphon = item[1]
87	if siphon not in known_siphons:
88		known_siphons.append(siphon)
89
90# Setup information for siphons we know about
91for siphon in known_siphons:
92	output[siphon] = {
93            "file": "%s/%s.siphon" % (args.output, siphon),
94            "global": {},
95            "items": [],
96        }
97
98# Pre-process file names in case they indicate a file with
99# a list of files
100files = []
101for filename in args.input:
102    if filename.startswith('@'):
103        with open(filename[1:], 'r') as fp:
104            lines = fp.readlines()
105            for line in lines:
106                files.append(line.strip())
107            lines = None
108    else:
109        files.append(filename)
110
111# Iterate all the input files we've been given
112for filename in files:
113    # Strip the current directory off the start of the
114    # filename for brevity
115    if filename[0:len(args.input_prefix)] == args.input_prefix:
116        filename = filename[len(args.input_prefix):]
117        if filename[0] == "/":
118            filename = filename[1:]
119
120    # Work out the abbreviated directory name
121    directory = os.path.dirname(filename)
122    if directory[0:2] == "./":
123        directory = directory[2:]
124    elif directory[0:len(args.input_prefix)] == args.input_prefix:
125        directory = directory[len(args.input_prefix):]
126    if directory[0] == "/":
127    	directory = directory[1:]
128
129    # Open the file and explore its contents...
130    sys.stderr.write("Siphoning from %s...\n" % filename)
131    directives = {}
132    with open(filename) as fd:
133        siphon = None
134        close_siphon = None
135        siphon_block = ""
136        in_block = False
137        line_num = 0
138        siphon_line = 0
139
140        for line in fd:
141            line_num += 1
142            str = line[:-1] # filter \n
143
144            """See if there is a block directive and if so extract it"""
145            def process_block_directive(str, directives):
146                m = siphon_block_directive.search(str)
147                if m is not None:
148                    k = m.group(2)
149                    v = m.group(3).strip()
150                    directives[k] = v
151                    # Return only the parts we did not match
152                    return str[0:m.start(1)] + str[m.end(4):]
153
154                return str
155
156            def process_block_prefix(str):
157                if str.startswith(" * "):
158                    str = str[3:]
159                elif str == " *":
160                    str = ""
161                return str
162
163            if not in_block:
164                # See if the line contains the start of a siphon doc block
165                m = siphon_block_start.search(str)
166                if m is not None:
167                    in_block = True
168                    t = m.group(1)
169
170                    # Now check if the block closes on the same line
171                    m = siphon_block_stop.search(t)
172                    if m is not None:
173                        t = m.group(1)
174                        in_block = False
175
176                    # Check for directives
177                    t = process_block_directive(t, directives)
178
179                    # Filter for normal comment prefixes
180                    t = process_block_prefix(t)
181
182                    # Add what is left
183                    siphon_block += t
184
185                    # Skip to next line
186                    continue
187
188            else:
189                # Check to see if we have an end block marker
190                m = siphon_block_stop.search(str)
191                if m is not None:
192                    in_block = False
193                    t = m.group(1)
194                else:
195                    t = str
196
197                # Check for directives
198                t = process_block_directive(t, directives)
199
200                # Filter for normal comment prefixes
201                t = process_block_prefix(t)
202
203                # Add what is left
204                siphon_block += t + "\n"
205
206                # Skip to next line
207                continue
208
209
210            if siphon is None:
211                # Look for blocks we need to siphon
212                for p in siphon_patterns:
213                    if p[0].match(str):
214                        siphon = [ p[1], str + "\n", 0 ]
215                        siphon_line = line_num
216
217                        # see if we have an initializer
218                        m = siphon_initializer.search(str)
219                        if m is not None:
220                            # count the braces on this line
221                            (count, index) = count_braces(str[m.start():])
222                            siphon[2] = count
223                            # TODO - it's possible we have the initializer all on the first line
224                            # we should check for it, but also account for the possibility that
225                            # the open brace is on the next line
226                            #if count == 0:
227                            #    # braces balanced
228                            #    close_siphon = siphon
229                            #    siphon = None
230                        else:
231                            # no initializer: close the siphon right now
232                            close_siphon = siphon
233                            siphon = None
234            else:
235                # See if we should end the siphon here - do we have balanced
236                # braces?
237                (count, index) = count_braces(str, count=siphon[2], found=True)
238                if count == 0:
239                    # braces balanced - add the substring and close the siphon
240                    siphon[1] += str[:index+1] + ";\n"
241                    close_siphon = siphon
242                    siphon = None
243                else:
244                    # add the whole string, move on
245                    siphon[2] = count
246                    siphon[1] += str + "\n"
247
248            if close_siphon is not None:
249                # Write the siphoned contents to the right place
250                siphon_name = close_siphon[0]
251
252                # Copy directives for the file
253                details = {}
254                for key in directives:
255                    if ":" in key:
256                        (sn, label) = key.split(":")
257                        if sn == siphon_name:
258                            details[label] = directives[key]
259                    else:
260                        details[key] = directives[key]
261
262                # Copy details for this block
263                details['file'] = filename
264                details['line_start'] = siphon_line
265                details['line_end'] = line_num
266                details['siphon_block'] = siphon_block.strip()
267
268                # Some defaults
269                if "group" not in details:
270                    if "group_label" in details:
271                        # use the filename since group labels are mostly of file scope
272                        details['group'] = details['file']
273                    else:
274			details['group'] = directory
275
276                if "group_label" not in details:
277                    details['group_label'] = details['group']
278
279                details["block"] = close_siphon[1]
280
281                # Store the item
282                output[siphon_name]['items'].append(details)
283
284                # All done
285                close_siphon = None
286                siphon_block = ""
287
288        # Update globals
289        for key in directives.keys():
290            if ':' not in key:
291                continue
292
293            if filename.endswith("/dir.dox"):
294                # very special! use the parent directory name
295                l = directory
296            else:
297                l = filename
298
299            (sn, label) = key.split(":")
300
301            if sn not in output:
302                output[sn] = {}
303            if 'global' not in output[sn]:
304                output[sn]['global'] = {}
305            if l not in output[sn]['global']:
306                output[sn]['global'][l] = {}
307            if 'file' not in output[sn]:
308                output[sn]['file'] = "%s/%s.siphon" % (args.output, sn)
309            if 'items' not in output[sn]:
310                output[sn]['items'] = []
311
312            output[sn]['global'][l][label] = directives[key]
313
314
315# Write out the data
316for siphon in output.keys():
317    sys.stderr.write("Saving siphon %s...\n" % siphon)
318    s = output[siphon]
319    with open(s['file'], "a") as fp:
320        json.dump(s, fp, separators=(',', ': '), indent=4, sort_keys=True)
321
322# All done
323