scapy_service.py revision cdbdece3
1
2import os
3import sys
4stl_pathname = os.path.abspath(os.path.join(os.pardir, os.pardir))
5sys.path.append(stl_pathname)
6
7from trex_stl_lib.api import *
8import tempfile
9import hashlib
10import base64
11import numbers
12import random
13import inspect
14import json
15from pprint import pprint
16
17# add some layers as an example
18# need to test more
19from scapy.layers.dns import *
20from scapy.layers.dhcp import *
21from scapy.layers.ipsec import *
22from scapy.layers.netflow import *
23from scapy.layers.sctp import *
24from scapy.layers.tftp import *
25
26from scapy.contrib.mpls import *
27from scapy.contrib.igmp import *
28from scapy.contrib.igmpv3 import *
29
30
31
32
33#additional_stl_udp_pkts = os.path.abspath(os.path.join(os.pardir,os.pardir,os.pardir,os.pardir, os.pardir,'stl'))
34#sys.path.append(additional_stl_udp_pkts)
35#from udp_1pkt_vxlan import VXLAN
36#sys.path.remove(additional_stl_udp_pkts)
37
38try:
39    from cStringIO import StringIO
40except ImportError:
41    from io import StringIO
42
43
44
45
46class Scapy_service_api():
47
48    def get_version_handler(self,client_v_major,client_v_minor):
49        """ get_version_handler(self,client_v_major,client_v_minor)
50
51            Gives a handler to client to connect and use server api
52
53            Parameters
54            ----------
55            client_v_major - major number of api version on the client side
56
57            Returns
58            -------
59            Handler(string) to provide when using server api
60        """
61        pass
62    def get_all(self,client_v_handler):
63        """ get_all(self,client_v_handler)
64
65        Sends all the protocols and fields that Scapy Service supports.
66        also sends the md5 of the Protocol DB and Fields DB used to check if the DB's are up to date
67
68        Parameters
69        ----------
70        None
71
72        Returns
73        -------
74        Dictionary (of protocol DB and scapy fields DB)
75
76        Raises
77        ------
78        Raises an exception when a DB error occurs (i.e a layer is not loaded properly and has missing components)
79        """
80        pass
81
82    def check_update_of_dbs(self,client_v_handler,db_md5,field_md5):
83        """ check_update_of_dbs(self,client_v_handler,db_md5,field_md5)
84        Checks if the Scapy Service running on the server has a newer version of the databases that the client has
85
86        Parameters
87        ----------
88        db_md5 - The md5 that was delivered with the protocol database that the client owns, when first received at the client
89        field_md5 - The md5 that was delivered with the fields database that the client owns, when first received at the client
90
91        Returns
92        -------
93        True/False according the Databases version(determined by their md5)
94
95        Raises
96        ------
97        Raises an exception (ScapyException) when protocol DB/Fields DB is not up to date
98        """
99        pass
100
101
102    def build_pkt(self,client_v_handler,pkt_model_descriptor):
103        """ build_pkt(self,client_v_handler,pkt_model_descriptor) -> Dictionary (of Offsets,Show2 and Buffer)
104
105        Performs calculations on the given packet and returns results for that packet.
106
107        Parameters
108        ----------
109        pkt_descriptor - An array of dictionaries describing a network packet
110
111        Returns
112        -------
113        - The packets offsets: each field in every layer is mapped inside the Offsets Dictionary
114        - The Show2: A description of each field and its value in every layer of the packet
115        - The Buffer: The Hexdump of packet encoded in base64
116
117        Raises
118        ------
119        will raise an exception when the Scapy string format is illegal, contains syntax error, contains non-supported
120        protocl, etc.
121        """
122        pass
123
124
125    def get_tree(self,client_v_handler):
126        """ get_tree(self) -> Dictionary describing an example of hierarchy in layers
127
128        Scapy service holds a tree of layers that can be stacked to a recommended packet
129        according to the hierarchy
130
131        Parameters
132        ----------
133        None
134
135        Returns
136        -------
137        Returns an example hierarchy tree of layers that can be stacked to a packet
138
139        Raises
140        ------
141        None
142        """
143        pass
144
145    def reconstruct_pkt(self,client_v_handler,binary_pkt,model_descriptor):
146        """ reconstruct_pkt(self,client_v_handler,binary_pkt)
147
148        Makes a Scapy valid packet by applying changes to binary packet and returns all information returned in build_pkt
149
150        Parameters
151        ----------
152        Source packet in binary_pkt, formatted in "base64" encoding
153        List of changes in model_descriptor
154
155        Returns
156        -------
157        All data provided in build_pkt:
158        show2 - detailed description of the packet
159        buffer - the packet presented in binary
160        offsets - the offset[in bytes] of each field in the packet
161
162        """
163        pass
164
165    def read_pcap(self,client_v_handler,pcap_base64):
166        """ read_pcap(self,client_v_handler,pcap_base64)
167
168        Parses pcap file contents and returns an array with build_pkt information for each packet
169
170        Parameters
171        ----------
172        binary pcap file in base64 encoding
173
174        Returns
175        -------
176        Array of build_pkt(packet)
177        """
178        pass
179
180    def write_pcap(self,client_v_handler,packets_base64):
181        """ write_pcap(self,client_v_handler,packets_base64)
182
183        Writes binary packets to pcap file
184
185        Parameters
186        ----------
187        array of binary packets in base64 encoding
188
189        Returns
190        -------
191        binary pcap file in base64 encoding
192        """
193        pass
194
195    def get_definitions(self,client_v_handler, def_filter):
196        """ get_definitions(self,client_v_handler, def_filter)
197
198        Returns protocols and fields metadata of scapy service
199
200        Parameters
201        ----------
202        def_filter - array of protocol names
203
204        Returns
205        -------
206        definitions for protocols
207        """
208        pass
209
210    def get_payload_classes(self,client_v_handler, pkt_model_descriptor):
211        """ get_payload_classes(self,client_v_handler, pkt_model_descriptor)
212
213        Returns an array of protocol classes, which normally can be used as a payload
214
215        Parameters
216        ----------
217        pkt_model_descriptor - see build_pkt
218
219        Returns
220        -------
221        array of supported protocol classes
222        """
223        pass
224
225def is_python(version):
226    return version == sys.version_info[0]
227
228def is_number(obj):
229    return isinstance(obj, numbers.Number)
230
231def is_string(obj):
232    return type(obj) == str or type(obj).__name__ == 'unicode' # python3 doesn't have unicode type
233
234def is_ascii_str(strval):
235    return strval and all(ord(ch) < 128 for ch in strval)
236
237def is_ascii_bytes(buf):
238    return buf and all(byte < 128 for byte in buf)
239
240def is_ascii(obj):
241    if is_bytes3(obj):
242        return is_ascii_bytes(obj)
243    else:
244        return is_ascii_str(obj)
245
246def is_bytes3(obj):
247    # checks if obj is exactly bytes(always false for python2)
248    return is_python(3) and type(obj) == bytes
249
250def str_to_bytes(strval):
251    return strval.encode("utf8")
252
253def bytes_to_str(buf):
254    return buf.decode("utf8")
255
256def b64_to_bytes(payload_base64):
257    # get bytes from base64 string(unicode)
258    return base64.b64decode(payload_base64)
259
260def bytes_to_b64(buf):
261    # bytes to base64 string(unicode)
262    return base64.b64encode(buf).decode('ascii')
263
264def get_sample_field_val(scapy_layer, fieldId):
265    # get some sample value for the field, to determine the value type
266    # use random or serialized value if default value is None
267    field_desc, current_val = scapy_layer.getfield_and_val(fieldId)
268    if current_val is not None:
269        return current_val
270    try:
271        # try to get some random value to determine type
272        return field_desc.randval()._fix()
273    except:
274        pass
275    try:
276        # try to serialize/deserialize
277        ltype = type(scapy_layer)
278        pkt = ltype(bytes(ltype()))
279        return pkt.getfieldval(fieldId)
280    except:
281        pass
282
283def generate_random_bytes(sz, seed, start, end):
284    # generate bytes of specified range with a fixed seed and size
285    rnd = random.Random(seed)
286    end = end + 1 # to include end value
287    res = [rnd.randrange(start, end) for _i in range(sz)]
288    if is_python(2):
289        return ''.join(chr(x) for x in res)
290    else:
291        return bytes(res)
292
293def generate_bytes_from_template(sz, template):
294    # generate bytes by repeating a template
295    res = str_to_bytes('') # new bytes array
296    if len(template) == 0:
297        return res
298    while len(res) < sz:
299        res = res + template
300    return res[:sz]
301
302def parse_template_code(template_code):
303    template_code = re.sub("0[xX]", '', template_code) # remove 0x
304    template_code = re.sub("[\s]", '', template_code) # remove spaces
305    return bytearray.fromhex(template_code)
306
307def generate_bytes(bytes_definition):
308    # accepts a bytes definition object
309    # {generate: random_bytes or random_ascii, seed: <seed_number>, size: <size_bytes>}
310    # {generate: template, template_base64: '<base64str>',  size: <size_bytes>}
311    # {generate: template_code, template_text_code: '<template_code_str>',  size: <size_bytes>}
312    gen_type = bytes_definition.get('generate')
313    bytes_size = bytes_definition.get('size')
314    seed = bytes_definition.get('seed') or 12345
315    if gen_type == 'random_bytes':
316        return generate_random_bytes(bytes_size, seed, 0, 0xFF)
317    elif gen_type == 'random_ascii':
318        return generate_random_bytes(bytes_size, seed, 0x20, 0x7E)
319    elif gen_type == 'template':
320        return generate_bytes_from_template(bytes_size, b64_to_bytes(bytes_definition["template_base64"]))
321    elif gen_type == 'template_code':
322        return generate_bytes_from_template(bytes_size, bytes_definition["template_code"])
323    elif gen_type == None:
324        return b64_to_bytes(bytes_definition['base64'])
325
326
327class ScapyException(Exception): pass
328class Scapy_service(Scapy_service_api):
329
330#----------------------------------------------------------------------------------------------------
331    class ScapyFieldDesc:
332        def __init__(self,FieldName,regex='empty'):
333            self.FieldName = FieldName
334            self.regex = regex
335            #defualt values - should be changed when needed, or added to constructor
336            self.string_input =""
337            self.string_input_mex_len = 1
338            self.integer_input = 0
339            self.integer_input_min = 0
340            self.integer_input_max = 1
341            self.input_array = []
342            self.input_list_max_len = 1
343
344        def stringRegex(self):
345            return self.regex
346#----------------------------------------------------------------------------------------------------
347    def __init__(self):
348        self.Raw = {'Raw':''}
349        self.high_level_protocols = ['Raw']
350        self.transport_protocols = {'TCP':self.Raw,'UDP':self.Raw}
351        self.network_protocols = {'IP':self.transport_protocols ,'ARP':''}
352        self.low_level_protocols = { 'Ether': self.network_protocols }
353        self.regexDB= {'MACField' : self.ScapyFieldDesc('MACField','^([0-9a-fA-F][0-9a-fA-F]:){5}([0-9a-fA-F][0-9a-fA-F])$'),
354              'IPField' : self.ScapyFieldDesc('IPField','^(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])$')}
355        self.all_protocols = self._build_lib()
356        self.protocol_tree = {'ALL':{'Ether':{'ARP':{},'IP':{'TCP':{'RAW':'payload'},'UDP':{'RAW':'payload'}}}}}
357        self.version_major = '1'
358        self.version_minor = '01'
359        self.server_v_hashed = self._generate_version_hash(self.version_major,self.version_minor)
360        self.protocol_definitions = {} # protocolId -> prococol definition overrides data
361        self._load_definitions_from_json()
362
363    def _load_definitions_from_json(self):
364        # load protocol definitions from a json file
365        self.protocol_definitions = {}
366        with file('protocols.json') as f:
367            protocols = json.load(f)
368            for protocol in protocols:
369                self.protocol_definitions[ protocol['id'] ] = protocol
370
371    def _all_protocol_structs(self):
372        old_stdout = sys.stdout
373        sys.stdout = mystdout = StringIO()
374        ls()
375        sys.stdout = old_stdout
376        all_protocol_data= mystdout.getvalue()
377        return all_protocol_data
378
379    def _protocol_struct(self,protocol):
380        if '_' in protocol:
381            return []
382        if not protocol=='':
383            if protocol not in self.all_protocols:
384                return 'protocol not supported'
385        protocol = eval(protocol)
386        old_stdout = sys.stdout
387        sys.stdout = mystdout = StringIO()
388        ls(protocol)
389        sys.stdout = old_stdout
390        protocol_data= mystdout.getvalue()
391        return protocol_data
392
393    def _build_lib(self):
394        lib = self._all_protocol_structs()
395        lib = lib.splitlines()
396        all_protocols=[]
397        for entry in lib:
398            entry = entry.split(':')
399            all_protocols.append(entry[0].strip())
400        del all_protocols[len(all_protocols)-1]
401        return all_protocols
402
403    def _parse_description_line(self,line):
404        line_arr = [x.strip() for x in re.split(': | = ',line)]
405        return tuple(line_arr)
406
407    def _parse_entire_description(self,description):
408        description = description.split('\n')
409        description_list = [self._parse_description_line(x) for x in description]
410        del description_list[len(description_list)-1]
411        return description_list
412
413    def _get_protocol_details(self,p_name):
414        protocol_str = self._protocol_struct(p_name)
415        if protocol_str=='protocol not supported':
416            return 'protocol not supported'
417        if len(protocol_str) is 0:
418            return []
419        tupled_protocol = self._parse_entire_description(protocol_str)
420        return tupled_protocol
421
422    def _value_from_dict(self, val):
423        # allows building python objects from json
424        if type(val) == type({}):
425            value_type = val['vtype']
426            if value_type == 'EXPRESSION':
427                return eval(val['expr'], {})
428            elif value_type == 'BYTES':   # bytes payload(ex Raw.load)
429                return generate_bytes(val)
430            elif value_type == 'OBJECT':
431                return val['value']
432            else:
433                return val # it's better to specify type explicitly
434        elif type(val) == type([]):
435            return [self._value_from_dict(v) for v in val]
436        else:
437            return val
438
439    def _field_value_from_def(self, scapy_pkt, layer, fieldId, val):
440        field_desc = layer.get_field(fieldId)
441        sample_val = get_sample_field_val(layer, fieldId)
442        # extensions for field values
443        if type(val) == type({}):
444            value_type = val['vtype']
445            if value_type == 'UNDEFINED': # clear field value
446                return None
447            elif value_type == 'RANDOM': # random field value
448                return field_desc.randval()
449            elif value_type == 'MACHINE': # internal machine field repr
450                return field_desc.m2i(layer, b64_to_bytes(val['base64']))
451            elif value_type == 'BYTES':
452                if 'total_size' in val: # custom case for total pkt size
453                    gen = {}
454                    gen.update(val)
455                    total_sz = gen['total_size']
456                    del gen['total_size']
457                    gen['size'] = total_sz - len(scapy_pkt)
458                    return generate_bytes(gen)
459                else:
460                    return generate_bytes(val)
461        if is_number(sample_val) and is_string(val):
462            # human-value. guess the type and convert to internal value
463            # seems setfieldval already does this for some fields,
464            # but does not convert strings/hex(0x123) to integers and long
465            val = str(val) # unicode -> str(ascii)
466            # parse str to int/long as a decimal or hex
467            val_constructor = type(sample_val)
468            if len(val) == 0:
469                return None
470            elif re.match(r"^0x[\da-f]+$", val, flags=re.IGNORECASE): # hex
471                return val_constructor(val, 16)
472            elif re.match(r"^\d+L?$", val): # base10
473                return val_constructor(val)
474        # generate recursive field-independent values
475        return self._value_from_dict(val)
476
477    def _print_tree(self):
478        pprint(self.protocol_tree)
479
480    def _get_all_db(self):
481        db = {}
482        for pro in self.all_protocols:
483            details = self._get_protocol_details(pro)
484            db[pro] = details
485        return db
486
487    def _get_all_fields(self):
488        fields = []
489        for pro in self.all_protocols:
490            details = self._get_protocol_details(pro)
491            for i in range(0,len(details),1):
492                if len(details[i]) == 3:
493                    fields.append(details[i][1])
494        uniqueFields = list(set(fields))
495        fieldDict = {}
496        for f in uniqueFields:
497            if f in self.regexDB:
498                fieldDict[f] = self.regexDB[f].stringRegex()
499            else:
500                fieldDict[f] = self.ScapyFieldDesc(f).stringRegex()
501        return fieldDict
502
503    def _fully_define(self,pkt):
504        # returns scapy object with all fields initialized
505        rootClass = type(pkt)
506        full_pkt = rootClass(bytes(pkt))
507        full_pkt.build() # this trick initializes offset
508        return full_pkt
509
510    def _bytes_to_value(self, payload_bytes):
511        # generates struct with a value
512        return { "vtype": "BYTES", "base64": bytes_to_b64(payload_bytes) }
513
514    def _pkt_to_field_tree(self,pkt):
515        pkt.build()
516        result = []
517        pcap_struct = self._fully_define(pkt) # structure, which will appear in pcap binary
518        while pkt:
519            layer_id = type(pkt).__name__ # Scapy classname
520            layer_full = self._fully_define(pkt) # current layer recreated from binary to get auto-calculated vals
521            real_layer_id = type(pcap_struct).__name__ if pcap_struct else None
522            valid_struct = True # shows if packet is mapped correctly to the binary representation
523            if not pcap_struct:
524                valid_struct = False
525            elif not issubclass(type(pkt), type(pcap_struct)) and not issubclass(type(pcap_struct), type(pkt)):
526                # structure mismatch. no need to go deeper in pcap_struct
527                valid_struct = False
528                pcap_struct = None
529            fields = []
530            for field_desc in pkt.fields_desc:
531                field_id = field_desc.name
532                ignored = field_id not in layer_full.fields
533                offset = field_desc.offset
534                protocol_offset = pkt.offset
535                field_sz = field_desc.get_size_bytes()
536                # some values are unavailable in pkt(original model)
537                # at the same time,
538                fieldval = pkt.getfieldval(field_id)
539                pkt_fieldval_defined = is_string(fieldval) or is_number(fieldval) or is_bytes3(fieldval)
540                if not pkt_fieldval_defined:
541                    fieldval = layer_full.getfieldval(field_id)
542                value = None
543                hvalue = None
544                value_base64 = None
545                if is_python(3) and is_bytes3(fieldval):
546                    value = self._bytes_to_value(fieldval)
547                    if is_ascii_bytes(fieldval):
548                        hvalue = bytes_to_str(fieldval)
549                    else:
550                        # can't be shown as ascii.
551                        # also this buffer may not be unicode-compatible(still can try to convert)
552                        value = self._bytes_to_value(fieldval)
553                        hvalue = '<binary>'
554                elif not is_string(fieldval):
555                    # value as is. this can be int,long, or custom object(list/dict)
556                    # "nice" human value, i2repr(string) will have quotes, so we have special handling for them
557                    hvalue = field_desc.i2repr(pkt, fieldval)
558
559                    if is_number(fieldval):
560                        value = fieldval
561                        if is_string(hvalue) and re.match(r"^\d+L$", hvalue):
562                            hvalue =  hvalue[:-1] # chop trailing L for long decimal number(python2)
563                    else:
564                        # fieldval is an object( class / list / dict )
565                        # generic serialization/deserialization needed for proper packet rebuilding from packet tree,
566                        # some classes can not be mapped to json, but we can pass them serialize them
567                        # as a python eval expr, value bytes base64, or field machine internal val(m2i)
568                        value = {"vtype": "EXPRESSION", "expr": hvalue}
569                if is_python(3) and is_string(fieldval):
570                    hvalue = value = fieldval
571                if is_python(2) and is_string(fieldval):
572                    if is_ascii(fieldval):
573                        hvalue = value = fieldval
574                    else:
575                        # python2 non-ascii byte buffers
576                        # payload contains non-ascii chars, which
577                        # sometimes can not be passed as unicode strings
578                        value = self._bytes_to_value(fieldval)
579                        hvalue = '<binary>'
580                if field_desc.name == 'load':
581                    # show Padding(and possible similar classes) as Raw
582                    layer_id = 'Raw'
583                    field_sz = len(pkt)
584                    value = self._bytes_to_value(fieldval)
585                field_data = {
586                        "id": field_id,
587                        "value": value,
588                        "hvalue": hvalue,
589                        "offset": offset,
590                        "length": field_sz
591                        }
592                if ignored:
593                    field_data["ignored"] = ignored
594                fields.append(field_data)
595            layer_data = {
596                    "id": layer_id,
597                    "offset": pkt.offset,
598                    "fields": fields,
599                    "real_id": real_layer_id,
600                    "valid_structure": valid_struct,
601                    }
602            result.append(layer_data)
603            pkt = pkt.payload
604            if pcap_struct:
605                pcap_struct = pcap_struct.payload or None
606        return result
607
608#input: container
609#output: md5 encoded in base64
610    def _get_md5(self,container):
611        container = json.dumps(container)
612        m = hashlib.md5()
613        m.update(str_to_bytes(container))
614        res_md5 = bytes_to_b64(m.digest())
615        return res_md5
616
617    def get_version(self):
618        return {'built_by':'itraviv','version':self.version_major+'.'+self.version_minor}
619
620    def supported_methods(self,method_name='all'):
621        if method_name=='all':
622            methods = {}
623            for f in dir(Scapy_service):
624                if f[0]=='_':
625                    continue
626                if inspect.ismethod(eval('Scapy_service.'+f)):
627                    param_list = inspect.getargspec(eval('Scapy_service.'+f))[0]
628                    del param_list[0] #deleting the parameter "self" that appears in every method
629                                      #because the server automatically operates on an instance,
630                                      #and this can cause confusion
631                    methods[f] = (len(param_list), param_list)
632            return methods
633        if method_name in dir(Scapy_service):
634            return True
635        return False
636
637    def _generate_version_hash(self,v_major,v_minor):
638        v_for_hash = v_major+v_minor+v_major+v_minor
639        m = hashlib.md5()
640        m.update(str_to_bytes(v_for_hash))
641        return bytes_to_b64(m.digest())
642
643    def _generate_invalid_version_error(self):
644        error_desc1 = "Provided version handler does not correspond to the server's version.\nUpdate client to latest version.\nServer version:"+self.version_major+"."+self.version_minor
645        return error_desc1
646
647    def _verify_version_handler(self,client_v_handler):
648        return (self.server_v_hashed == client_v_handler)
649
650    def _parse_packet_dict(self, layer, layer_classes, base_layer):
651        class_p = layer_classes[layer['id']] # class id -> class dict
652        scapy_layer = class_p()
653        if isinstance(scapy_layer, Raw):
654            scapy_layer.load = str_to_bytes("dummy")
655        if base_layer == None:
656            base_layer = scapy_layer
657        if 'fields' in layer:
658            self._modify_layer(base_layer, scapy_layer, layer['fields'])
659        return scapy_layer
660
661    def _packet_model_to_scapy_packet(self,data):
662        layer_classes = {}
663        for layer_class in Packet.__subclasses__():
664            layer_classes[layer_class.__name__] = layer_class
665        base_layer = self._parse_packet_dict(data[0], layer_classes, None)
666        for i in range(1,len(data),1):
667            packet_layer = self._parse_packet_dict(data[i], layer_classes, base_layer)
668            base_layer = base_layer/packet_layer
669        return base_layer
670
671    def _pkt_data(self,pkt):
672        if pkt == None:
673            return {'data': [], 'binary': None}
674        data = self._pkt_to_field_tree(pkt)
675        binary = bytes_to_b64(bytes(pkt))
676        res = {'data': data, 'binary': binary}
677        return res
678
679#--------------------------------------------API implementation-------------
680    def get_tree(self,client_v_handler):
681        if not (self._verify_version_handler(client_v_handler)):
682            raise ScapyException(self._generate_invalid_version_error())
683        return self.protocol_tree
684
685    def get_version_handler(self,client_v_major,client_v_minor):
686        v_handle = self._generate_version_hash(client_v_major,client_v_minor)
687        return v_handle
688
689# pkt_descriptor in packet model format (dictionary)
690    def build_pkt(self,client_v_handler,pkt_model_descriptor):
691        if not (self._verify_version_handler(client_v_handler)):
692            raise ScapyException(self._generate_invalid_version_error())
693        pkt = self._packet_model_to_scapy_packet(pkt_model_descriptor)
694        return self._pkt_data(pkt)
695
696    # @deprecated. to be removed
697    def get_all(self,client_v_handler):
698        if not (self._verify_version_handler(client_v_handler)):
699            raise ScapyException(self._generate_invalid_version_error())
700        fields=self._get_all_fields()
701        db=self._get_all_db()
702        fields_md5 = self._get_md5(fields)
703        db_md5 = self._get_md5(db)
704        res = {}
705        res['db'] = db
706        res['fields'] = fields
707        res['db_md5'] = db_md5
708        res['fields_md5'] = fields_md5
709        return res
710
711    def _is_packet_class(self, pkt_class):
712        # returns true for final Packet classes. skips aliases and metaclasses
713        return issubclass(pkt_class, Packet) and pkt_class.name and pkt_class.fields_desc
714
715    def _getDummyPacket(self, pkt_class):
716        if issubclass(pkt_class, Raw):
717            # need to have some payload. otherwise won't appear in the binary chunk
718            return pkt_class(load=str_to_bytes("dummy"))
719        else:
720            return pkt_class()
721
722
723    def _get_payload_classes(self, pkt_class):
724        # tries to find, which subclasses allowed.
725        # this can take long time, since it tries to build packets with all subclasses(O(N))
726        allowed_subclasses = []
727        for pkt_subclass in conf.layers:
728            if self._is_packet_class(pkt_subclass):
729                try:
730                    pkt_w_payload = pkt_class() / self._getDummyPacket(pkt_subclass)
731                    recreated_pkt = pkt_class(bytes(pkt_w_payload))
732                    if type(recreated_pkt.lastlayer()) is pkt_subclass:
733                        allowed_subclasses.append(pkt_subclass)
734                except Exception as e:
735                    # no actions needed on fail, just sliently skip
736                    pass
737        return allowed_subclasses
738
739    def _get_fields_definition(self, pkt_class, fieldsDef):
740        # fieldsDef - array of field definitions(or empty array)
741        fields = []
742        for field_desc in pkt_class.fields_desc:
743            fieldId = field_desc.name
744            field_data = {
745                    "id": fieldId,
746                    "name": field_desc.name
747            }
748            for fieldDef in fieldsDef:
749                if fieldDef['id'] == fieldId:
750                    field_data.update(fieldDef)
751            if isinstance(field_desc, EnumField):
752                try:
753                    field_data["values_dict"] = field_desc.s2i
754                    if field_data.get("type") == None:
755                        if len(field_data["values_dict"] > 0):
756                            field_data["type"] = "ENUM"
757                        elif fieldId == 'load':
758                            field_data["type"] = "BYTES"
759                        else:
760                            field_data["type"] = "STRING"
761                    field_data["values_dict"] = field_desc.s2i
762                except:
763                    # MultiEnumField doesn't have s2i. need better handling
764                    pass
765            fields.append(field_data)
766        return fields
767
768    def get_definitions(self,client_v_handler, def_filter):
769        # def_filter is an array of classnames or None
770        all_classes = Packet.__subclasses__() # as an alternative to conf.layers
771        if def_filter:
772            all_classes = [c for c in all_classes if c.__name__ in def_filter]
773        protocols = []
774        for pkt_class in all_classes:
775            if self._is_packet_class(pkt_class):
776                # enumerate all non-abstract Packet classes
777                protocolId = pkt_class.__name__
778                protoDef = self.protocol_definitions.get(protocolId) or {}
779                protocols.append({
780                    "id": protocolId,
781                    "name": protoDef.get('name') or pkt_class.name,
782                    "fields": self._get_fields_definition(pkt_class, protoDef.get('fields') or [])
783                    })
784        res = {"protocols": protocols}
785        return res
786
787    def get_payload_classes(self,client_v_handler, pkt_model_descriptor):
788        pkt = self._packet_model_to_scapy_packet(pkt_model_descriptor)
789        pkt_class = type(pkt.lastlayer())
790        protocolDef = self.protocol_definitions.get(pkt_class.__name__)
791        if protocolDef and protocolDef.get('payload'):
792            return protocolDef['payload']
793        return [c.__name__ for c in self._get_payload_classes(pkt_class)]
794
795#input in string encoded base64
796    def check_update_of_dbs(self,client_v_handler,db_md5,field_md5):
797        if not (self._verify_version_handler(client_v_handler)):
798            raise ScapyException(self._generate_invalid_version_error())
799        fields=self._get_all_fields()
800        db=self._get_all_db()
801        current_db_md5 = self._get_md5(db)
802        current_field_md5 = self._get_md5(fields)
803        res = []
804        if (field_md5 == current_field_md5):
805            if (db_md5 == current_db_md5):
806                return True
807            else:
808                raise ScapyException("Protocol DB is not up to date")
809        else:
810            raise ScapyException("Fields DB is not up to date")
811
812    def _modify_layer(self, scapy_pkt, scapy_layer, fields):
813        for field in fields:
814            fieldId = str(field['id'])
815            fieldval = self._field_value_from_def(scapy_pkt, scapy_layer, fieldId, field['value'])
816            if fieldval is not None:
817                scapy_layer.setfieldval(fieldId, fieldval)
818            else:
819                scapy_layer.delfieldval(fieldId)
820
821    def _is_last_layer(self, layer):
822        # can be used, that layer has no payload
823        # if true, the layer.payload is likely NoPayload()
824        return layer is layer.lastlayer()
825
826#input of binary_pkt must be encoded in base64
827    def reconstruct_pkt(self,client_v_handler,binary_pkt,model_descriptor):
828        pkt_bin = b64_to_bytes(binary_pkt)
829        scapy_pkt = Ether(pkt_bin)
830        if not model_descriptor:
831            model_descriptor = []
832        for depth in range(len(model_descriptor)):
833            model_layer = model_descriptor[depth]
834            if model_layer.get('delete') is True:
835                # slice packet from the current item
836                if depth == 0:
837                    scapy_pkt = None
838                    break
839                else:
840                    scapy_pkt[depth-1].payload = None
841                    break
842            if depth > 0 and self._is_last_layer(scapy_pkt[depth-1]):
843                # insert new layer(s) from json definition
844                remaining_definitions = model_descriptor[depth:]
845                pkt_to_append = self._packet_model_to_scapy_packet(remaining_definitions)
846                scapy_pkt = scapy_pkt / pkt_to_append
847                break
848            # modify fields of existing stack items
849            scapy_layer = scapy_pkt[depth]
850            if model_layer['id'] != type(scapy_layer).__name__:
851                # TODO: support replacing payload, instead of breaking
852                raise ScapyException("Protocol id inconsistent")
853            if 'fields' in model_layer:
854                self._modify_layer(scapy_pkt, scapy_layer, model_layer['fields'])
855        return self._pkt_data(scapy_pkt)
856
857    def read_pcap(self,client_v_handler,pcap_base64):
858        pcap_bin = b64_to_bytes(pcap_base64)
859        pcap = []
860        res_packets = []
861        with tempfile.NamedTemporaryFile(mode='w+b') as tmpPcap:
862            tmpPcap.write(pcap_bin)
863            tmpPcap.flush()
864            pcap = rdpcap(tmpPcap.name)
865        for scapy_packet in pcap:
866            res_packets.append(self._pkt_data(scapy_packet))
867        return res_packets
868
869    def write_pcap(self,client_v_handler,packets_base64):
870        packets = [Ether(b64_to_bytes(pkt_b64)) for pkt_b64 in packets_base64]
871        pcap_bin = None
872        with tempfile.NamedTemporaryFile(mode='r+b') as tmpPcap:
873            wrpcap(tmpPcap.name, packets)
874            pcap_bin = tmpPcap.read()
875        return bytes_to_b64(pcap_bin)
876
877
878
879
880#---------------------------------------------------------------------------
881
882
883