scapy_service.py revision bd63922f
1
2import os
3import sys
4
5stl_pathname = os.path.abspath(os.path.join(os.pardir, os.pardir))
6sys.path.append(stl_pathname)
7
8from trex_stl_lib.api import *
9import trex_stl_lib.trex_stl_packet_builder_scapy
10import tempfile
11import hashlib
12import base64
13import numbers
14import random
15from inspect import getdoc
16import json
17from pprint import pprint
18
19# add some layers as an example
20# need to test more
21from scapy.layers.dns import *
22from scapy.layers.dhcp import *
23from scapy.layers.ipsec import *
24from scapy.layers.netflow import *
25from scapy.layers.sctp import *
26from scapy.layers.tftp import *
27
28from scapy.contrib.mpls import *
29from scapy.contrib.igmp import *
30from scapy.contrib.igmpv3 import *
31
32
33
34
35#additional_stl_udp_pkts = os.path.abspath(os.path.join(os.pardir,os.pardir,os.pardir,os.pardir, os.pardir,'stl'))
36#sys.path.append(additional_stl_udp_pkts)
37#from udp_1pkt_vxlan import VXLAN
38#sys.path.remove(additional_stl_udp_pkts)
39
40try:
41    from cStringIO import StringIO
42except ImportError:
43    from io import StringIO
44
45
46
47
48class Scapy_service_api():
49
50    def get_version_handler(self,client_v_major,client_v_minor):
51        """ get_version_handler(self,client_v_major,client_v_minor)
52
53            Gives a handler to client to connect and use server api
54
55            Parameters
56            ----------
57            client_v_major - major number of api version on the client side
58
59            Returns
60            -------
61            Handler(string) to provide when using server api
62        """
63        pass
64    def get_all(self,client_v_handler):
65        """ get_all(self,client_v_handler)
66
67        Sends all the protocols and fields that Scapy Service supports.
68        also sends the md5 of the Protocol DB and Fields DB used to check if the DB's are up to date
69
70        Parameters
71        ----------
72        None
73
74        Returns
75        -------
76        Dictionary (of protocol DB and scapy fields DB)
77
78        Raises
79        ------
80        Raises an exception when a DB error occurs (i.e a layer is not loaded properly and has missing components)
81        """
82        pass
83
84    def check_update_of_dbs(self,client_v_handler,db_md5,field_md5):
85        """ check_update_of_dbs(self,client_v_handler,db_md5,field_md5)
86        Checks if the Scapy Service running on the server has a newer version of the databases that the client has
87
88        Parameters
89        ----------
90        db_md5 - The md5 that was delivered with the protocol database that the client owns, when first received at the client
91        field_md5 - The md5 that was delivered with the fields database that the client owns, when first received at the client
92
93        Returns
94        -------
95        True/False according the Databases version(determined by their md5)
96
97        Raises
98        ------
99        Raises an exception (ScapyException) when protocol DB/Fields DB is not up to date
100        """
101        pass
102
103
104    def build_pkt(self,client_v_handler,pkt_model_descriptor):
105        """ build_pkt(self,client_v_handler,pkt_model_descriptor) -> Dictionary (of Offsets,Show2 and Buffer)
106
107        Performs calculations on the given packet and returns results for that packet.
108
109        Parameters
110        ----------
111        pkt_descriptor - An array of dictionaries describing a network packet
112
113        Returns
114        -------
115        - The packets offsets: each field in every layer is mapped inside the Offsets Dictionary
116        - The Show2: A description of each field and its value in every layer of the packet
117        - The Buffer: The Hexdump of packet encoded in base64
118
119        Raises
120        ------
121        will raise an exception when the Scapy string format is illegal, contains syntax error, contains non-supported
122        protocl, etc.
123        """
124        pass
125
126    def build_pkt_ex(self, client_v_handler, pkt_model_descriptor, extra_options):
127        """ build_pkt_ex(self,client_v_handler,pkt_model_descriptor, extra_options) -> Dictionary (of Offsets,Show2 and Buffer)
128        Performs calculations on the given packet and returns results for that packet.
129
130        Parameters
131        ----------
132        pkt_descriptor - An array of dictionaries describing a network packet
133        extra_options - A dictionary of extra options required for building packet
134
135        Returns
136        -------
137        - The packets offsets: each field in every layer is mapped inside the Offsets Dictionary
138        - The Show2: A description of each field and its value in every layer of the packet
139        - The Buffer: The Hexdump of packet encoded in base64
140
141        Raises
142        ------
143        will raise an exception when the Scapy string format is illegal, contains syntax error, contains non-supported
144        protocl, etc.
145        """
146        pass
147
148    def get_tree(self,client_v_handler):
149        """ get_tree(self) -> Dictionary describing an example of hierarchy in layers
150
151        Scapy service holds a tree of layers that can be stacked to a recommended packet
152        according to the hierarchy
153
154        Parameters
155        ----------
156        None
157
158        Returns
159        -------
160        Returns an example hierarchy tree of layers that can be stacked to a packet
161
162        Raises
163        ------
164        None
165        """
166        pass
167
168    def reconstruct_pkt(self,client_v_handler,binary_pkt,model_descriptor):
169        """ reconstruct_pkt(self,client_v_handler,binary_pkt)
170
171        Makes a Scapy valid packet by applying changes to binary packet and returns all information returned in build_pkt
172
173        Parameters
174        ----------
175        Source packet in binary_pkt, formatted in "base64" encoding
176        List of changes in model_descriptor
177
178        Returns
179        -------
180        All data provided in build_pkt:
181        show2 - detailed description of the packet
182        buffer - the packet presented in binary
183        offsets - the offset[in bytes] of each field in the packet
184
185        """
186        pass
187
188    def read_pcap(self,client_v_handler,pcap_base64):
189        """ read_pcap(self,client_v_handler,pcap_base64)
190
191        Parses pcap file contents and returns an array with build_pkt information for each packet
192
193        Parameters
194        ----------
195        binary pcap file in base64 encoding
196
197        Returns
198        -------
199        Array of build_pkt(packet)
200        """
201        pass
202
203    def write_pcap(self,client_v_handler,packets_base64):
204        """ write_pcap(self,client_v_handler,packets_base64)
205
206        Writes binary packets to pcap file
207
208        Parameters
209        ----------
210        array of binary packets in base64 encoding
211
212        Returns
213        -------
214        binary pcap file in base64 encoding
215        """
216        pass
217
218    def get_definitions(self,client_v_handler, def_filter):
219        """ get_definitions(self,client_v_handler, def_filter)
220
221        Returns protocols and fields metadata of scapy service
222
223        Parameters
224        ----------
225        def_filter - array of protocol names
226
227        Returns
228        -------
229        definitions for protocols
230        """
231        pass
232
233    def get_payload_classes(self,client_v_handler, pkt_model_descriptor):
234        """ get_payload_classes(self,client_v_handler, pkt_model_descriptor)
235
236        Returns an array of protocol classes, which normally can be used as a payload
237
238        Parameters
239        ----------
240        pkt_model_descriptor - see build_pkt
241
242        Returns
243        -------
244        array of supported protocol classes
245        """
246        pass
247
248def is_python(version):
249    return version == sys.version_info[0]
250
251def is_number(obj):
252    return isinstance(obj, numbers.Number)
253
254def is_string(obj):
255    return type(obj) == str or type(obj).__name__ == 'unicode' # python3 doesn't have unicode type
256
257def is_ascii_str(strval):
258    return strval and all(ord(ch) < 128 for ch in strval)
259
260def is_ascii_bytes(buf):
261    return buf and all(byte < 128 for byte in buf)
262
263def is_ascii(obj):
264    if is_bytes3(obj):
265        return is_ascii_bytes(obj)
266    else:
267        return is_ascii_str(obj)
268
269def is_bytes3(obj):
270    # checks if obj is exactly bytes(always false for python2)
271    return is_python(3) and type(obj) == bytes
272
273def str_to_bytes(strval):
274    return strval.encode("utf8")
275
276def bytes_to_str(buf):
277    return buf.decode("utf8")
278
279def b64_to_bytes(payload_base64):
280    # get bytes from base64 string(unicode)
281    return base64.b64decode(payload_base64)
282
283def bytes_to_b64(buf):
284    # bytes to base64 string(unicode)
285    return base64.b64encode(buf).decode('ascii')
286
287def get_sample_field_val(scapy_layer, fieldId):
288    # get some sample value for the field, to determine the value type
289    # use random or serialized value if default value is None
290    field_desc, current_val = scapy_layer.getfield_and_val(fieldId)
291    if current_val is not None:
292        return current_val
293    try:
294        # try to get some random value to determine type
295        return field_desc.randval()._fix()
296    except:
297        pass
298    try:
299        # try to serialize/deserialize
300        ltype = type(scapy_layer)
301        pkt = ltype(bytes(ltype()))
302        return pkt.getfieldval(fieldId)
303    except:
304        pass
305
306def generate_random_bytes(sz, seed, start, end):
307    # generate bytes of specified range with a fixed seed and size
308    rnd = random.Random()
309    n = end - start + 1
310    if is_python(2):
311        rnd = random.Random(seed)
312        res = [start + int(rnd.random()*n) for _i in range(sz)]
313        return ''.join(chr(x) for x in res)
314    else:
315        rnd = random.Random()
316        # to generate same random sequence as 2.x
317        rnd.seed(seed, version=1)
318        res = [start + int(rnd.random()*n) for _i in range(sz)]
319        return bytes(res)
320
321def generate_bytes_from_template(sz, template):
322    # generate bytes by repeating a template
323    res = str_to_bytes('') # new bytes array
324    if len(template) == 0:
325        return res
326    while len(res) < sz:
327        res = res + template
328    return res[:sz]
329
330def parse_template_code(template_code):
331    template_code = re.sub("0[xX]", '', template_code) # remove 0x
332    template_code = re.sub("[\s]", '', template_code) # remove spaces
333    return bytearray.fromhex(template_code)
334
335def verify_payload_size(size):
336    assert(size != None)
337    if (size > (1<<20)): # 1Mb ought to be enough for anybody
338        raise ValueError('size is too large')
339
340def generate_bytes(bytes_definition):
341    # accepts a bytes definition object
342    # {generate: random_bytes or random_ascii, seed: <seed_number>, size: <size_bytes>}
343    # {generate: template, template_base64: '<base64str>',  size: <size_bytes>}
344    # {generate: template_code, template_text_code: '<template_code_str>',  size: <size_bytes>}
345    gen_type = bytes_definition.get('generate')
346    if gen_type == None:
347        return b64_to_bytes(bytes_definition['base64'])
348    elif gen_type == 'template_code':
349        code = parse_template_code(bytes_definition["template_code"])
350        bytes_size = int(bytes_definition.get('size') or len(code))
351        verify_payload_size(bytes_size)
352        return generate_bytes_from_template(bytes_size, code)
353    else:
354        bytes_size = int(bytes_definition['size']) # required
355        seed = int(bytes_definition.get('seed') or 12345) # optional
356        verify_payload_size(bytes_size)
357        if gen_type == 'random_bytes':
358            return generate_random_bytes(bytes_size, seed, 0, 0xFF)
359        elif gen_type == 'random_ascii':
360            return generate_random_bytes(bytes_size, seed, 0x20, 0x7E)
361        elif gen_type == 'template':
362            return generate_bytes_from_template(bytes_size, b64_to_bytes(bytes_definition["template_base64"]))
363
364class ScapyException(Exception): pass
365class Scapy_service(Scapy_service_api):
366
367#----------------------------------------------------------------------------------------------------
368    class ScapyFieldDesc:
369        def __init__(self,FieldName,regex='empty'):
370            self.FieldName = FieldName
371            self.regex = regex
372            #defualt values - should be changed when needed, or added to constructor
373            self.string_input =""
374            self.string_input_mex_len = 1
375            self.integer_input = 0
376            self.integer_input_min = 0
377            self.integer_input_max = 1
378            self.input_array = []
379            self.input_list_max_len = 1
380
381        def stringRegex(self):
382            return self.regex
383#----------------------------------------------------------------------------------------------------
384    def __init__(self):
385        self.Raw = {'Raw':''}
386        self.high_level_protocols = ['Raw']
387        self.transport_protocols = {'TCP':self.Raw,'UDP':self.Raw}
388        self.network_protocols = {'IP':self.transport_protocols ,'ARP':''}
389        self.low_level_protocols = { 'Ether': self.network_protocols }
390        self.regexDB= {'MACField' : self.ScapyFieldDesc('MACField','^([0-9a-fA-F][0-9a-fA-F]:){5}([0-9a-fA-F][0-9a-fA-F])$'),
391              'IPField' : self.ScapyFieldDesc('IPField','^(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])$')}
392        self.all_protocols = self._build_lib()
393        self.protocol_tree = {'ALL':{'Ether':{'ARP':{},'IP':{'TCP':{'RAW':'payload'},'UDP':{'RAW':'payload'}}}}}
394        self.version_major = '1'
395        self.version_minor = '01'
396        self.server_v_hashed = self._generate_version_hash(self.version_major,self.version_minor)
397        self.protocol_definitions = {} # protocolId -> prococol definition overrides data
398        self.protocol_fields_fe_aware = {}
399        self.instruction_parameter_meta_definitions = []
400        self.field_engine_parameter_meta_definitions = []
401        self.field_engine_instructions_meta = []
402        self.field_engine_instruction_expressions = []
403        self._load_definitions_from_json()
404        self._load_field_engine_meta_from_json()
405        self._vm_instructions = dict([m for m in inspect.getmembers(trex_stl_lib.trex_stl_packet_builder_scapy, inspect.isclass) if m[1].__module__ == 'trex_stl_lib.trex_stl_packet_builder_scapy'])
406
407    def _load_definitions_from_json(self):
408        # load protocol definitions from a json file
409        self.protocol_definitions = {}
410        with open('protocols.json', 'r') as f:
411            protocols = json.load(f)
412            for protocol in protocols:
413                self.protocol_definitions[ protocol['id'] ] = protocol
414
415    def _load_field_engine_meta_from_json(self):
416        # load protocol definitions from a json file
417        self.instruction_parameter_meta_definitions = []
418        self.protocol_fields_fe_aware = {}
419        self.field_engine_parameter_meta_definitions = []
420        with open('field_engine.json', 'r') as f:
421            metas = json.load(f)
422            self.instruction_parameter_meta_definitions = metas["instruction_params_meta"]
423            self.field_engine_instructions_meta = metas["instructions"]
424            self._append_intructions_help()
425            self.protocol_fields_fe_aware = metas["protocol_fields"]
426            self.field_engine_parameter_meta_definitions = metas["global_params_meta"]
427
428
429    def _append_intructions_help(self):
430        for instruction_meta in self.field_engine_instructions_meta:
431            clazz = eval(instruction_meta['id'])
432            instruction_meta['help'] = base64.b64encode(getdoc(clazz.__init__)).decode('ascii')
433
434    def _all_protocol_structs(self):
435        old_stdout = sys.stdout
436        sys.stdout = mystdout = StringIO()
437        ls()
438        sys.stdout = old_stdout
439        all_protocol_data= mystdout.getvalue()
440        return all_protocol_data
441
442    def _protocol_struct(self,protocol):
443        if '_' in protocol:
444            return []
445        if not protocol=='':
446            if protocol not in self.all_protocols:
447                return 'protocol not supported'
448        protocol = eval(protocol)
449        old_stdout = sys.stdout
450        sys.stdout = mystdout = StringIO()
451        ls(protocol)
452        sys.stdout = old_stdout
453        protocol_data= mystdout.getvalue()
454        return protocol_data
455
456    def _build_lib(self):
457        lib = self._all_protocol_structs()
458        lib = lib.splitlines()
459        all_protocols=[]
460        for entry in lib:
461            entry = entry.split(':')
462            all_protocols.append(entry[0].strip())
463        del all_protocols[len(all_protocols)-1]
464        return all_protocols
465
466    def _parse_description_line(self,line):
467        line_arr = [x.strip() for x in re.split(': | = ',line)]
468        return tuple(line_arr)
469
470    def _parse_entire_description(self,description):
471        description = description.split('\n')
472        description_list = [self._parse_description_line(x) for x in description]
473        del description_list[len(description_list)-1]
474        return description_list
475
476    def _get_protocol_details(self,p_name):
477        protocol_str = self._protocol_struct(p_name)
478        if protocol_str=='protocol not supported':
479            return 'protocol not supported'
480        if len(protocol_str) is 0:
481            return []
482        tupled_protocol = self._parse_entire_description(protocol_str)
483        return tupled_protocol
484
485    def _value_from_dict(self, val):
486        # allows building python objects from json
487        if type(val) == type({}):
488            value_type = val['vtype']
489            if value_type == 'EXPRESSION':
490                return eval(val['expr'], scapy.all.__dict__)
491            elif value_type == 'BYTES':   # bytes payload(ex Raw.load)
492                return generate_bytes(val)
493            elif value_type == 'OBJECT':
494                return val['value']
495            else:
496                return val # it's better to specify type explicitly
497        elif type(val) == type([]):
498            return [self._value_from_dict(v) for v in val]
499        else:
500            return val
501
502    def _field_value_from_def(self, scapy_pkt, layer, fieldId, val):
503        field_desc = layer.get_field(fieldId)
504        sample_val = get_sample_field_val(layer, fieldId)
505        # extensions for field values
506        if type(val) == type({}):
507            value_type = val['vtype']
508            if value_type == 'UNDEFINED': # clear field value
509                return None
510            elif value_type == 'RANDOM': # random field value
511                return field_desc.randval()
512            elif value_type == 'MACHINE': # internal machine field repr
513                return field_desc.m2i(layer, b64_to_bytes(val['base64']))
514            elif value_type == 'BYTES':
515                if 'total_size' in val: # custom case for total pkt size
516                    gen = {}
517                    gen.update(val)
518                    total_sz = gen['total_size']
519                    del gen['total_size']
520                    gen['size'] = total_sz - len(scapy_pkt)
521                    return generate_bytes(gen)
522                else:
523                    return generate_bytes(val)
524        if is_number(sample_val) and is_string(val):
525            # human-value. guess the type and convert to internal value
526            # seems setfieldval already does this for some fields,
527            # but does not convert strings/hex(0x123) to integers and long
528            val = str(val) # unicode -> str(ascii)
529            # parse str to int/long as a decimal or hex
530            val_constructor = type(sample_val)
531            if len(val) == 0:
532                return None
533            elif re.match(r"^0x[\da-f]+$", val, flags=re.IGNORECASE): # hex
534                return val_constructor(val, 16)
535            elif re.match(r"^\d+L?$", val): # base10
536                return val_constructor(val)
537        # generate recursive field-independent values
538        return self._value_from_dict(val)
539
540    def _print_tree(self):
541        pprint(self.protocol_tree)
542
543    def _get_all_db(self):
544        db = {}
545        for pro in self.all_protocols:
546            details = self._get_protocol_details(pro)
547            db[pro] = details
548        return db
549
550    def _get_all_fields(self):
551        fields = []
552        for pro in self.all_protocols:
553            details = self._get_protocol_details(pro)
554            for i in range(0,len(details),1):
555                if len(details[i]) == 3:
556                    fields.append(details[i][1])
557        uniqueFields = list(set(fields))
558        fieldDict = {}
559        for f in uniqueFields:
560            if f in self.regexDB:
561                fieldDict[f] = self.regexDB[f].stringRegex()
562            else:
563                fieldDict[f] = self.ScapyFieldDesc(f).stringRegex()
564        return fieldDict
565
566    def _fully_define(self,pkt):
567        # returns scapy object with all fields initialized
568        rootClass = type(pkt)
569        full_pkt = rootClass(bytes(pkt))
570        full_pkt.build() # this trick initializes offset
571        return full_pkt
572
573    def _bytes_to_value(self, payload_bytes):
574        # generates struct with a value
575        return { "vtype": "BYTES", "base64": bytes_to_b64(payload_bytes) }
576
577    def _pkt_to_field_tree(self,pkt):
578        pkt.build()
579        result = []
580        pcap_struct = self._fully_define(pkt) # structure, which will appear in pcap binary
581        while pkt:
582            layer_id = type(pkt).__name__ # Scapy classname
583            layer_full = self._fully_define(pkt) # current layer recreated from binary to get auto-calculated vals
584            real_layer_id = type(pcap_struct).__name__ if pcap_struct else None
585            valid_struct = True # shows if packet is mapped correctly to the binary representation
586            if not pcap_struct:
587                valid_struct = False
588            elif not issubclass(type(pkt), type(pcap_struct)) and not issubclass(type(pcap_struct), type(pkt)):
589                # structure mismatch. no need to go deeper in pcap_struct
590                valid_struct = False
591                pcap_struct = None
592            fields = []
593            for field_desc in pkt.fields_desc:
594                field_id = field_desc.name
595                ignored = field_id not in layer_full.fields
596                offset = field_desc.offset
597                protocol_offset = pkt.offset
598                field_sz = field_desc.get_size_bytes()
599                # some values are unavailable in pkt(original model)
600                # at the same time,
601                fieldval = pkt.getfieldval(field_id)
602                pkt_fieldval_defined = is_string(fieldval) or is_number(fieldval) or is_bytes3(fieldval)
603                if not pkt_fieldval_defined:
604                    fieldval = layer_full.getfieldval(field_id)
605                value = None
606                hvalue = None
607                value_base64 = None
608                if is_python(3) and is_bytes3(fieldval):
609                    value = self._bytes_to_value(fieldval)
610                    if is_ascii_bytes(fieldval):
611                        hvalue = bytes_to_str(fieldval)
612                    else:
613                        # can't be shown as ascii.
614                        # also this buffer may not be unicode-compatible(still can try to convert)
615                        value = self._bytes_to_value(fieldval)
616                        hvalue = '<binary>'
617                elif not is_string(fieldval):
618                    # value as is. this can be int,long, or custom object(list/dict)
619                    # "nice" human value, i2repr(string) will have quotes, so we have special handling for them
620                    hvalue = field_desc.i2repr(pkt, fieldval)
621
622                    if is_number(fieldval):
623                        value = fieldval
624                        if is_string(hvalue) and re.match(r"^\d+L$", hvalue):
625                            hvalue =  hvalue[:-1] # chop trailing L for long decimal number(python2)
626                    else:
627                        # fieldval is an object( class / list / dict )
628                        # generic serialization/deserialization needed for proper packet rebuilding from packet tree,
629                        # some classes can not be mapped to json, but we can pass them serialize them
630                        # as a python eval expr, value bytes base64, or field machine internal val(m2i)
631                        value = {"vtype": "EXPRESSION", "expr": hvalue}
632                if is_python(3) and is_string(fieldval):
633                    hvalue = value = fieldval
634                if is_python(2) and is_string(fieldval):
635                    if is_ascii(fieldval):
636                        hvalue = value = fieldval
637                    else:
638                        # python2 non-ascii byte buffers
639                        # payload contains non-ascii chars, which
640                        # sometimes can not be passed as unicode strings
641                        value = self._bytes_to_value(fieldval)
642                        hvalue = '<binary>'
643                if field_desc.name == 'load':
644                    # show Padding(and possible similar classes) as Raw
645                    layer_id = 'Raw'
646                    field_sz = len(pkt)
647                    value = self._bytes_to_value(fieldval)
648                field_data = {
649                        "id": field_id,
650                        "value": value,
651                        "hvalue": hvalue,
652                        "offset": offset,
653                        "length": field_sz
654                        }
655                if ignored:
656                    field_data["ignored"] = ignored
657                fields.append(field_data)
658            layer_data = {
659                    "id": layer_id,
660                    "offset": pkt.offset,
661                    "fields": fields,
662                    "real_id": real_layer_id,
663                    "valid_structure": valid_struct,
664                    }
665            result.append(layer_data)
666            pkt = pkt.payload
667            if pcap_struct:
668                pcap_struct = pcap_struct.payload or None
669        return result
670
671#input: container
672#output: md5 encoded in base64
673    def _get_md5(self,container):
674        container = json.dumps(container)
675        m = hashlib.md5()
676        m.update(str_to_bytes(container))
677        res_md5 = bytes_to_b64(m.digest())
678        return res_md5
679
680    def get_version(self):
681        return {'built_by':'itraviv','version':self.version_major+'.'+self.version_minor}
682
683    def supported_methods(self,method_name='all'):
684        if method_name=='all':
685            methods = {}
686            for f in dir(Scapy_service):
687                if f[0]=='_':
688                    continue
689                if inspect.ismethod(eval('Scapy_service.'+f)):
690                    param_list = inspect.getargspec(eval('Scapy_service.'+f))[0]
691                    del param_list[0] #deleting the parameter "self" that appears in every method
692                                      #because the server automatically operates on an instance,
693                                      #and this can cause confusion
694                    methods[f] = (len(param_list), param_list)
695            return methods
696        if method_name in dir(Scapy_service):
697            return True
698        return False
699
700    def _generate_version_hash(self,v_major,v_minor):
701        v_for_hash = v_major+v_minor+v_major+v_minor
702        m = hashlib.md5()
703        m.update(str_to_bytes(v_for_hash))
704        return bytes_to_b64(m.digest())
705
706    def _generate_invalid_version_error(self):
707        error_desc1 = "Provided version handler does not correspond to the server's version.\nUpdate client to latest version.\nServer version:"+self.version_major+"."+self.version_minor
708        return error_desc1
709
710    def _verify_version_handler(self,client_v_handler):
711        return (self.server_v_hashed == client_v_handler)
712
713    def _parse_packet_dict(self, layer, layer_classes, base_layer):
714        class_p = layer_classes[layer['id']] # class id -> class dict
715        scapy_layer = class_p()
716        if isinstance(scapy_layer, Raw):
717            scapy_layer.load = str_to_bytes("dummy")
718        if base_layer == None:
719            base_layer = scapy_layer
720        if 'fields' in layer:
721            self._modify_layer(base_layer, scapy_layer, layer['fields'])
722        return scapy_layer
723
724    def _packet_model_to_scapy_packet(self,data):
725        layer_classes = {}
726        for layer_class in Packet.__subclasses__():
727            layer_classes[layer_class.__name__] = layer_class
728        base_layer = self._parse_packet_dict(data[0], layer_classes, None)
729        for i in range(1,len(data),1):
730            packet_layer = self._parse_packet_dict(data[i], layer_classes, base_layer)
731            base_layer = base_layer/packet_layer
732        return base_layer
733
734    def _pkt_data(self,pkt):
735        if pkt == None:
736            return {'data': [], 'binary': None}
737        data = self._pkt_to_field_tree(pkt)
738        binary = bytes_to_b64(bytes(pkt))
739        res = {'data': data, 'binary': binary}
740        return res
741
742#--------------------------------------------API implementation-------------
743    def get_tree(self,client_v_handler):
744        if not (self._verify_version_handler(client_v_handler)):
745            raise ScapyException(self._generate_invalid_version_error())
746        return self.protocol_tree
747
748    def get_version_handler(self,client_v_major,client_v_minor):
749        v_handle = self._generate_version_hash(client_v_major,client_v_minor)
750        return v_handle
751
752# pkt_descriptor in packet model format (dictionary)
753    def build_pkt(self,client_v_handler,pkt_model_descriptor):
754        if not (self._verify_version_handler(client_v_handler)):
755            raise ScapyException(self._generate_invalid_version_error())
756        pkt = self._packet_model_to_scapy_packet(pkt_model_descriptor)
757        return self._pkt_data(pkt)
758
759
760    def build_pkt_ex(self, client_v_handler, pkt_model_descriptor, extra_options):
761        res = self.build_pkt(client_v_handler, pkt_model_descriptor)
762        pkt = self._packet_model_to_scapy_packet(pkt_model_descriptor)
763        res['vm_instructions'] = self._generate_vm_instructions(pkt, extra_options['field_engine'])
764        res['vm_instructions_expressions'] = self.field_engine_instruction_expressions
765        return res
766
767    def _generate_vm_instructions(self, pkt, field_engine_model_descriptor):
768        self.field_engine_instruction_expressions = []
769        instructions = []
770        instructions_def = field_engine_model_descriptor['instructions']
771        for instruction_def in instructions_def:
772            instruction_id = instruction_def['id']
773            instruction_class = self._vm_instructions[instruction_id]
774            parameters = {k: self._sanitize_value(v) for (k, v) in instruction_def['parameters'].iteritems()}
775            instructions.append(instruction_class(**parameters))
776
777        fe_parameters = field_engine_model_descriptor['global_parameters']
778
779        cache_size = None
780        if "cache_size" in fe_parameters:
781            cache_size = int(fe_parameters['cache_size'])
782
783        pkt_builder = STLPktBuilder(pkt=pkt, vm=STLScVmRaw(instructions, cache_size=cache_size))
784        pkt_builder.compile()
785        return pkt_builder.get_vm_data()
786
787    def _sanitize_value(self, val):
788        if val == "true":
789            return True
790        elif val == "false":
791            return False
792        else:
793            return int(val) if val.isdigit() else str(val)
794
795    def _get_instruction_parameter_meta(self, param_id):
796        for meta in self.instruction_parameter_meta_definitions:
797            if meta['id'] == param_id:
798                return meta
799        raise Scapy_Exception("Unable to get meta for {0}" % param_id)
800
801    def _is_int(self, val):
802        try:
803            int(val)
804            return True
805        except ValueError:
806            return False
807
808    # @deprecated. to be removed
809    def get_all(self,client_v_handler):
810        if not (self._verify_version_handler(client_v_handler)):
811            raise ScapyException(self._generate_invalid_version_error())
812        fields=self._get_all_fields()
813        db=self._get_all_db()
814        fields_md5 = self._get_md5(fields)
815        db_md5 = self._get_md5(db)
816        res = {}
817        res['db'] = db
818        res['fields'] = fields
819        res['db_md5'] = db_md5
820        res['fields_md5'] = fields_md5
821        return res
822
823    def _is_packet_class(self, pkt_class):
824        # returns true for final Packet classes. skips aliases and metaclasses
825        return issubclass(pkt_class, Packet) and pkt_class.name and pkt_class.fields_desc
826
827    def _getDummyPacket(self, pkt_class):
828        if issubclass(pkt_class, Raw):
829            # need to have some payload. otherwise won't appear in the binary chunk
830            return pkt_class(load=str_to_bytes("dummy"))
831        else:
832            return pkt_class()
833
834
835    def _get_payload_classes(self, pkt_class):
836        # tries to find, which subclasses allowed.
837        # this can take long time, since it tries to build packets with all subclasses(O(N))
838        allowed_subclasses = []
839        for pkt_subclass in conf.layers:
840            if self._is_packet_class(pkt_subclass):
841                try:
842                    pkt_w_payload = pkt_class() / self._getDummyPacket(pkt_subclass)
843                    recreated_pkt = pkt_class(bytes(pkt_w_payload))
844                    if type(recreated_pkt.lastlayer()) is pkt_subclass:
845                        allowed_subclasses.append(pkt_subclass)
846                except Exception as e:
847                    # no actions needed on fail, just sliently skip
848                    pass
849        return allowed_subclasses
850
851    def _get_fields_definition(self, pkt_class, fieldsDef):
852        # fieldsDef - array of field definitions(or empty array)
853        fields = []
854        for field_desc in pkt_class.fields_desc:
855            fieldId = field_desc.name
856            field_data = {
857                    "id": fieldId,
858                    "name": field_desc.name
859            }
860            for fieldDef in fieldsDef:
861                if fieldDef['id'] == fieldId:
862                    field_data.update(fieldDef)
863            if isinstance(field_desc, EnumField):
864                try:
865                    field_data["values_dict"] = field_desc.s2i
866                    if field_data.get("type") == None:
867                        if len(field_data["values_dict"] > 0):
868                            field_data["type"] = "ENUM"
869                        elif fieldId == 'load':
870                            field_data["type"] = "BYTES"
871                        else:
872                            field_data["type"] = "STRING"
873                    field_data["values_dict"] = field_desc.s2i
874                except:
875                    # MultiEnumField doesn't have s2i. need better handling
876                    pass
877            fields.append(field_data)
878        return fields
879
880    def get_definitions(self,client_v_handler, def_filter):
881        # def_filter is an array of classnames or None
882        all_classes = Packet.__subclasses__() # as an alternative to conf.layers
883        if def_filter:
884            all_classes = [c for c in all_classes if c.__name__ in def_filter]
885        protocols = []
886        for pkt_class in all_classes:
887            if self._is_packet_class(pkt_class):
888                # enumerate all non-abstract Packet classes
889                protocolId = pkt_class.__name__
890                protoDef = self.protocol_definitions.get(protocolId) or {}
891                protocols.append({
892                    "id": protocolId,
893                    "name": protoDef.get('name') or pkt_class.name,
894                    "fields": self._get_fields_definition(pkt_class, protoDef.get('fields') or []),
895                    "fieldEngineAwareFields": self.protocol_fields_fe_aware.get(protocolId) or []
896                    })
897        res = {"protocols": protocols,
898               "feInstructionParameters": self.instruction_parameter_meta_definitions,
899               "feInstructions": self.field_engine_instructions_meta,
900               "feParameters": self.field_engine_parameter_meta_definitions}
901        return res
902
903    def get_payload_classes(self,client_v_handler, pkt_model_descriptor):
904        pkt = self._packet_model_to_scapy_packet(pkt_model_descriptor)
905        pkt_class = type(pkt.lastlayer())
906        protocolDef = self.protocol_definitions.get(pkt_class.__name__)
907        if protocolDef and protocolDef.get('payload'):
908            return protocolDef['payload']
909        return [c.__name__ for c in self._get_payload_classes(pkt_class)]
910
911#input in string encoded base64
912    def check_update_of_dbs(self,client_v_handler,db_md5,field_md5):
913        if not (self._verify_version_handler(client_v_handler)):
914            raise ScapyException(self._generate_invalid_version_error())
915        fields=self._get_all_fields()
916        db=self._get_all_db()
917        current_db_md5 = self._get_md5(db)
918        current_field_md5 = self._get_md5(fields)
919        res = []
920        if (field_md5 == current_field_md5):
921            if (db_md5 == current_db_md5):
922                return True
923            else:
924                raise ScapyException("Protocol DB is not up to date")
925        else:
926            raise ScapyException("Fields DB is not up to date")
927
928    def _modify_layer(self, scapy_pkt, scapy_layer, fields):
929        for field in fields:
930            fieldId = str(field['id'])
931            fieldval = self._field_value_from_def(scapy_pkt, scapy_layer, fieldId, field['value'])
932            if fieldval is not None:
933                scapy_layer.setfieldval(fieldId, fieldval)
934            else:
935                scapy_layer.delfieldval(fieldId)
936
937    def _is_last_layer(self, layer):
938        # can be used, that layer has no payload
939        # if true, the layer.payload is likely NoPayload()
940        return layer is layer.lastlayer()
941
942#input of binary_pkt must be encoded in base64
943    def reconstruct_pkt(self,client_v_handler,binary_pkt,model_descriptor):
944        pkt_bin = b64_to_bytes(binary_pkt)
945        scapy_pkt = Ether(pkt_bin)
946        if not model_descriptor:
947            model_descriptor = []
948        for depth in range(len(model_descriptor)):
949            model_layer = model_descriptor[depth]
950            if model_layer.get('delete') is True:
951                # slice packet from the current item
952                if depth == 0:
953                    scapy_pkt = None
954                    break
955                else:
956                    scapy_pkt[depth-1].payload = None
957                    break
958            if depth > 0 and self._is_last_layer(scapy_pkt[depth-1]):
959                # insert new layer(s) from json definition
960                remaining_definitions = model_descriptor[depth:]
961                pkt_to_append = self._packet_model_to_scapy_packet(remaining_definitions)
962                scapy_pkt = scapy_pkt / pkt_to_append
963                break
964            # modify fields of existing stack items
965            scapy_layer = scapy_pkt[depth]
966            if model_layer['id'] != type(scapy_layer).__name__:
967                # TODO: support replacing payload, instead of breaking
968                raise ScapyException("Protocol id inconsistent")
969            if 'fields' in model_layer:
970                self._modify_layer(scapy_pkt, scapy_layer, model_layer['fields'])
971        return self._pkt_data(scapy_pkt)
972
973    def read_pcap(self,client_v_handler,pcap_base64):
974        pcap_bin = b64_to_bytes(pcap_base64)
975        pcap = []
976        res_packets = []
977        with tempfile.NamedTemporaryFile(mode='w+b') as tmpPcap:
978            tmpPcap.write(pcap_bin)
979            tmpPcap.flush()
980            pcap = rdpcap(tmpPcap.name)
981        for scapy_packet in pcap:
982            res_packets.append(self._pkt_data(scapy_packet))
983        return res_packets
984
985    def write_pcap(self,client_v_handler,packets_base64):
986        packets = [Ether(b64_to_bytes(pkt_b64)) for pkt_b64 in packets_base64]
987        pcap_bin = None
988        with tempfile.NamedTemporaryFile(mode='r+b') as tmpPcap:
989            wrpcap(tmpPcap.name, packets)
990            pcap_bin = tmpPcap.read()
991        return bytes_to_b64(pcap_bin)
992
993
994#---------------------------------------------------------------------------
995
996
997