scapy_service.py revision 3b576aaf
1
2import os
3import sys
4
5stl_pathname = os.path.abspath(os.path.join(os.pardir, os.pardir))
6sys.path.append(stl_pathname)
7
8from trex_stl_lib.api import *
9import trex_stl_lib.trex_stl_packet_builder_scapy
10import tempfile
11import hashlib
12import base64
13import numbers
14import random
15from inspect import getdoc
16import json
17from pprint import pprint
18
19# add some layers as an example
20# need to test more
21from scapy.layers.dns import *
22from scapy.layers.dhcp import *
23from scapy.layers.ipsec import *
24from scapy.layers.netflow import *
25from scapy.layers.sctp import *
26from scapy.layers.tftp import *
27
28from scapy.contrib.mpls import *
29from scapy.contrib.igmp import *
30from scapy.contrib.igmpv3 import *
31
32
33
34
35#additional_stl_udp_pkts = os.path.abspath(os.path.join(os.pardir,os.pardir,os.pardir,os.pardir, os.pardir,'stl'))
36#sys.path.append(additional_stl_udp_pkts)
37#from udp_1pkt_vxlan import VXLAN
38#sys.path.remove(additional_stl_udp_pkts)
39
40try:
41    from cStringIO import StringIO
42except ImportError:
43    from io import StringIO
44
45
46
47
48class Scapy_service_api():
49
50    def get_version_handler(self,client_v_major,client_v_minor):
51        """ get_version_handler(self,client_v_major,client_v_minor)
52
53            Gives a handler to client to connect and use server api
54
55            Parameters
56            ----------
57            client_v_major - major number of api version on the client side
58
59            Returns
60            -------
61            Handler(string) to provide when using server api
62        """
63        pass
64    def get_all(self,client_v_handler):
65        """ get_all(self,client_v_handler)
66
67        Sends all the protocols and fields that Scapy Service supports.
68        also sends the md5 of the Protocol DB and Fields DB used to check if the DB's are up to date
69
70        Parameters
71        ----------
72        None
73
74        Returns
75        -------
76        Dictionary (of protocol DB and scapy fields DB)
77
78        Raises
79        ------
80        Raises an exception when a DB error occurs (i.e a layer is not loaded properly and has missing components)
81        """
82        pass
83
84    def check_update_of_dbs(self,client_v_handler,db_md5,field_md5):
85        """ check_update_of_dbs(self,client_v_handler,db_md5,field_md5)
86        Checks if the Scapy Service running on the server has a newer version of the databases that the client has
87
88        Parameters
89        ----------
90        db_md5 - The md5 that was delivered with the protocol database that the client owns, when first received at the client
91        field_md5 - The md5 that was delivered with the fields database that the client owns, when first received at the client
92
93        Returns
94        -------
95        True/False according the Databases version(determined by their md5)
96
97        Raises
98        ------
99        Raises an exception (ScapyException) when protocol DB/Fields DB is not up to date
100        """
101        pass
102
103
104    def build_pkt(self,client_v_handler,pkt_model_descriptor):
105        """ build_pkt(self,client_v_handler,pkt_model_descriptor) -> Dictionary (of Offsets,Show2 and Buffer)
106
107        Performs calculations on the given packet and returns results for that packet.
108
109        Parameters
110        ----------
111        pkt_descriptor - An array of dictionaries describing a network packet
112
113        Returns
114        -------
115        - The packets offsets: each field in every layer is mapped inside the Offsets Dictionary
116        - The Show2: A description of each field and its value in every layer of the packet
117        - The Buffer: The Hexdump of packet encoded in base64
118
119        Raises
120        ------
121        will raise an exception when the Scapy string format is illegal, contains syntax error, contains non-supported
122        protocl, etc.
123        """
124        pass
125
126    def build_pkt_ex(self, client_v_handler, pkt_model_descriptor, extra_options):
127        """ build_pkt_ex(self,client_v_handler,pkt_model_descriptor, extra_options) -> Dictionary (of Offsets,Show2 and Buffer)
128        Performs calculations on the given packet and returns results for that packet.
129
130        Parameters
131        ----------
132        pkt_descriptor - An array of dictionaries describing a network packet
133        extra_options - A dictionary of extra options required for building packet
134
135        Returns
136        -------
137        - The packets offsets: each field in every layer is mapped inside the Offsets Dictionary
138        - The Show2: A description of each field and its value in every layer of the packet
139        - The Buffer: The Hexdump of packet encoded in base64
140
141        Raises
142        ------
143        will raise an exception when the Scapy string format is illegal, contains syntax error, contains non-supported
144        protocl, etc.
145        """
146        pass
147
148    def load_instruction_parameter_values(self, client_v_handler, pkt_model_descriptor, vm_instructions_model, parameter_id):
149        """ load_instruction_parameter_values(self,client_v_handler,pkt_model_descriptor, vm_instructions_model, parameter_id) -> Dictionary (of possible parameter values)
150        Returns possible valies for given pararameter id depends on current pkt structure and vm_instructions
151        model.
152
153        Parameters
154        ----------
155        pkt_descriptor - An array of dictionaries describing a network packet
156        vm_instructions_model - A dictionary of extra options required for building packet
157        parameter_id - A string of parameter id
158
159        Returns
160        -------
161        Possible parameter values map.
162
163        """
164        pass
165
166    def get_tree(self,client_v_handler):
167        """ get_tree(self) -> Dictionary describing an example of hierarchy in layers
168
169        Scapy service holds a tree of layers that can be stacked to a recommended packet
170        according to the hierarchy
171
172        Parameters
173        ----------
174        None
175
176        Returns
177        -------
178        Returns an example hierarchy tree of layers that can be stacked to a packet
179
180        Raises
181        ------
182        None
183        """
184        pass
185
186    def reconstruct_pkt(self,client_v_handler,binary_pkt,model_descriptor):
187        """ reconstruct_pkt(self,client_v_handler,binary_pkt)
188
189        Makes a Scapy valid packet by applying changes to binary packet and returns all information returned in build_pkt
190
191        Parameters
192        ----------
193        Source packet in binary_pkt, formatted in "base64" encoding
194        List of changes in model_descriptor
195
196        Returns
197        -------
198        All data provided in build_pkt:
199        show2 - detailed description of the packet
200        buffer - the packet presented in binary
201        offsets - the offset[in bytes] of each field in the packet
202
203        """
204        pass
205
206    def read_pcap(self,client_v_handler,pcap_base64):
207        """ read_pcap(self,client_v_handler,pcap_base64)
208
209        Parses pcap file contents and returns an array with build_pkt information for each packet
210
211        Parameters
212        ----------
213        binary pcap file in base64 encoding
214
215        Returns
216        -------
217        Array of build_pkt(packet)
218        """
219        pass
220
221    def write_pcap(self,client_v_handler,packets_base64):
222        """ write_pcap(self,client_v_handler,packets_base64)
223
224        Writes binary packets to pcap file
225
226        Parameters
227        ----------
228        array of binary packets in base64 encoding
229
230        Returns
231        -------
232        binary pcap file in base64 encoding
233        """
234        pass
235
236    def get_definitions(self,client_v_handler, def_filter):
237        """ get_definitions(self,client_v_handler, def_filter)
238
239        Returns protocols and fields metadata of scapy service
240
241        Parameters
242        ----------
243        def_filter - array of protocol names
244
245        Returns
246        -------
247        definitions for protocols
248        """
249        pass
250
251    def get_payload_classes(self,client_v_handler, pkt_model_descriptor):
252        """ get_payload_classes(self,client_v_handler, pkt_model_descriptor)
253
254        Returns an array of protocol classes, which normally can be used as a payload
255
256        Parameters
257        ----------
258        pkt_model_descriptor - see build_pkt
259
260        Returns
261        -------
262        array of supported protocol classes
263        """
264        pass
265
266def is_python(version):
267    return version == sys.version_info[0]
268
269def is_number(obj):
270    return isinstance(obj, numbers.Number)
271
272def is_string(obj):
273    return type(obj) == str or type(obj).__name__ == 'unicode' # python3 doesn't have unicode type
274
275def is_ascii_str(strval):
276    return strval and all(ord(ch) < 128 for ch in strval)
277
278def is_ascii_bytes(buf):
279    return buf and all(byte < 128 for byte in buf)
280
281def is_ascii(obj):
282    if is_bytes3(obj):
283        return is_ascii_bytes(obj)
284    else:
285        return is_ascii_str(obj)
286
287def is_bytes3(obj):
288    # checks if obj is exactly bytes(always false for python2)
289    return is_python(3) and type(obj) == bytes
290
291def str_to_bytes(strval):
292    return strval.encode("utf8")
293
294def bytes_to_str(buf):
295    return buf.decode("utf8")
296
297def b64_to_bytes(payload_base64):
298    # get bytes from base64 string(unicode)
299    return base64.b64decode(payload_base64)
300
301def bytes_to_b64(buf):
302    # bytes to base64 string(unicode)
303    return base64.b64encode(buf).decode('ascii')
304
305def get_sample_field_val(scapy_layer, fieldId):
306    # get some sample value for the field, to determine the value type
307    # use random or serialized value if default value is None
308    field_desc, current_val = scapy_layer.getfield_and_val(fieldId)
309    if current_val is not None:
310        return current_val
311    try:
312        # try to get some random value to determine type
313        return field_desc.randval()._fix()
314    except:
315        pass
316    try:
317        # try to serialize/deserialize
318        ltype = type(scapy_layer)
319        pkt = ltype(bytes(ltype()))
320        return pkt.getfieldval(fieldId)
321    except:
322        pass
323
324def generate_random_bytes(sz, seed, start, end):
325    # generate bytes of specified range with a fixed seed and size
326    rnd = random.Random()
327    n = end - start + 1
328    if is_python(2):
329        rnd = random.Random(seed)
330        res = [start + int(rnd.random()*n) for _i in range(sz)]
331        return ''.join(chr(x) for x in res)
332    else:
333        rnd = random.Random()
334        # to generate same random sequence as 2.x
335        rnd.seed(seed, version=1)
336        res = [start + int(rnd.random()*n) for _i in range(sz)]
337        return bytes(res)
338
339def generate_bytes_from_template(sz, template):
340    # generate bytes by repeating a template
341    res = str_to_bytes('') # new bytes array
342    if len(template) == 0:
343        return res
344    while len(res) < sz:
345        res = res + template
346    return res[:sz]
347
348def parse_template_code(template_code):
349    template_code = re.sub("0[xX]", '', template_code) # remove 0x
350    template_code = re.sub("[\s]", '', template_code) # remove spaces
351    return bytearray.fromhex(template_code)
352
353def verify_payload_size(size):
354    assert(size != None)
355    if (size > (1<<20)): # 1Mb ought to be enough for anybody
356        raise ValueError('size is too large')
357
358def generate_bytes(bytes_definition):
359    # accepts a bytes definition object
360    # {generate: random_bytes or random_ascii, seed: <seed_number>, size: <size_bytes>}
361    # {generate: template, template_base64: '<base64str>',  size: <size_bytes>}
362    # {generate: template_code, template_text_code: '<template_code_str>',  size: <size_bytes>}
363    gen_type = bytes_definition.get('generate')
364    if gen_type == None:
365        return b64_to_bytes(bytes_definition['base64'])
366    elif gen_type == 'template_code':
367        code = parse_template_code(bytes_definition["template_code"])
368        bytes_size = int(bytes_definition.get('size') or len(code))
369        verify_payload_size(bytes_size)
370        return generate_bytes_from_template(bytes_size, code)
371    else:
372        bytes_size = int(bytes_definition['size']) # required
373        seed = int(bytes_definition.get('seed') or 12345) # optional
374        verify_payload_size(bytes_size)
375        if gen_type == 'random_bytes':
376            return generate_random_bytes(bytes_size, seed, 0, 0xFF)
377        elif gen_type == 'random_ascii':
378            return generate_random_bytes(bytes_size, seed, 0x20, 0x7E)
379        elif gen_type == 'template':
380            return generate_bytes_from_template(bytes_size, b64_to_bytes(bytes_definition["template_base64"]))
381
382class ScapyException(Exception): pass
383class Scapy_service(Scapy_service_api):
384
385#----------------------------------------------------------------------------------------------------
386    class ScapyFieldDesc:
387        def __init__(self,FieldName,regex='empty'):
388            self.FieldName = FieldName
389            self.regex = regex
390            #defualt values - should be changed when needed, or added to constructor
391            self.string_input =""
392            self.string_input_mex_len = 1
393            self.integer_input = 0
394            self.integer_input_min = 0
395            self.integer_input_max = 1
396            self.input_array = []
397            self.input_list_max_len = 1
398
399        def stringRegex(self):
400            return self.regex
401#----------------------------------------------------------------------------------------------------
402    def __init__(self):
403        self.Raw = {'Raw':''}
404        self.high_level_protocols = ['Raw']
405        self.transport_protocols = {'TCP':self.Raw,'UDP':self.Raw}
406        self.network_protocols = {'IP':self.transport_protocols ,'ARP':''}
407        self.low_level_protocols = { 'Ether': self.network_protocols }
408        self.regexDB= {'MACField' : self.ScapyFieldDesc('MACField','^([0-9a-fA-F][0-9a-fA-F]:){5}([0-9a-fA-F][0-9a-fA-F])$'),
409              'IPField' : self.ScapyFieldDesc('IPField','^(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])$')}
410        self.all_protocols = self._build_lib()
411        self.protocol_tree = {'ALL':{'Ether':{'ARP':{},'IP':{'TCP':{'RAW':'payload'},'UDP':{'RAW':'payload'}}}}}
412        self.version_major = '1'
413        self.version_minor = '01'
414        self.server_v_hashed = self._generate_version_hash(self.version_major,self.version_minor)
415        self.protocol_definitions = {} # protocolId -> prococol definition overrides data
416        self.field_engine_supported_protocols = {}
417        self.instruction_parameter_meta_definitions = []
418        self.field_engine_parameter_meta_definitions = []
419        self.field_engine_instructions_meta = []
420        self.field_engine_instruction_expressions = []
421        self._load_definitions_from_json()
422        self._load_field_engine_meta_from_json()
423        self._vm_instructions = dict([m for m in inspect.getmembers(trex_stl_lib.trex_stl_packet_builder_scapy, inspect.isclass) if m[1].__module__ == 'trex_stl_lib.trex_stl_packet_builder_scapy'])
424
425    def _load_definitions_from_json(self):
426        # load protocol definitions from a json file
427        self.protocol_definitions = {}
428        with open('protocols.json', 'r') as f:
429            protocols = json.load(f)
430            for protocol in protocols:
431                self.protocol_definitions[ protocol['id'] ] = protocol
432
433    def _load_field_engine_meta_from_json(self):
434        # load protocol definitions from a json file
435        self.instruction_parameter_meta_definitions = []
436        self.field_engine_supported_protocols = {}
437        self.field_engine_parameter_meta_definitions = []
438        with open('field_engine.json', 'r') as f:
439            metas = json.load(f)
440            self.instruction_parameter_meta_definitions = metas["instruction_params_meta"]
441            self.field_engine_instructions_meta = metas["instructions"]
442            self._append_intructions_help()
443            self.field_engine_supported_protocols = metas["supported_protocols"]
444            self.field_engine_parameter_meta_definitions = metas["global_params_meta"]
445
446
447    def _append_intructions_help(self):
448        for instruction_meta in self.field_engine_instructions_meta:
449            clazz = eval(instruction_meta['id'])
450            instruction_meta['help'] = base64.b64encode(getdoc(clazz.__init__)).decode('ascii')
451
452    def _all_protocol_structs(self):
453        old_stdout = sys.stdout
454        sys.stdout = mystdout = StringIO()
455        ls()
456        sys.stdout = old_stdout
457        all_protocol_data= mystdout.getvalue()
458        return all_protocol_data
459
460    def _protocol_struct(self,protocol):
461        if '_' in protocol:
462            return []
463        if not protocol=='':
464            if protocol not in self.all_protocols:
465                return 'protocol not supported'
466        protocol = eval(protocol)
467        old_stdout = sys.stdout
468        sys.stdout = mystdout = StringIO()
469        ls(protocol)
470        sys.stdout = old_stdout
471        protocol_data= mystdout.getvalue()
472        return protocol_data
473
474    def _build_lib(self):
475        lib = self._all_protocol_structs()
476        lib = lib.splitlines()
477        all_protocols=[]
478        for entry in lib:
479            entry = entry.split(':')
480            all_protocols.append(entry[0].strip())
481        del all_protocols[len(all_protocols)-1]
482        return all_protocols
483
484    def _parse_description_line(self,line):
485        line_arr = [x.strip() for x in re.split(': | = ',line)]
486        return tuple(line_arr)
487
488    def _parse_entire_description(self,description):
489        description = description.split('\n')
490        description_list = [self._parse_description_line(x) for x in description]
491        del description_list[len(description_list)-1]
492        return description_list
493
494    def _get_protocol_details(self,p_name):
495        protocol_str = self._protocol_struct(p_name)
496        if protocol_str=='protocol not supported':
497            return 'protocol not supported'
498        if len(protocol_str) is 0:
499            return []
500        tupled_protocol = self._parse_entire_description(protocol_str)
501        return tupled_protocol
502
503    def _value_from_dict(self, val):
504        # allows building python objects from json
505        if type(val) == type({}):
506            value_type = val['vtype']
507            if value_type == 'EXPRESSION':
508                return eval(val['expr'], scapy.all.__dict__)
509            elif value_type == 'BYTES':   # bytes payload(ex Raw.load)
510                return generate_bytes(val)
511            elif value_type == 'OBJECT':
512                return val['value']
513            else:
514                return val # it's better to specify type explicitly
515        elif type(val) == type([]):
516            return [self._value_from_dict(v) for v in val]
517        else:
518            return val
519
520    def _field_value_from_def(self, scapy_pkt, layer, fieldId, val):
521        field_desc = layer.get_field(fieldId)
522        sample_val = get_sample_field_val(layer, fieldId)
523        # extensions for field values
524        if type(val) == type({}):
525            value_type = val['vtype']
526            if value_type == 'UNDEFINED': # clear field value
527                return None
528            elif value_type == 'RANDOM': # random field value
529                return field_desc.randval()
530            elif value_type == 'MACHINE': # internal machine field repr
531                return field_desc.m2i(layer, b64_to_bytes(val['base64']))
532            elif value_type == 'BYTES':
533                if 'total_size' in val: # custom case for total pkt size
534                    gen = {}
535                    gen.update(val)
536                    total_sz = gen['total_size']
537                    del gen['total_size']
538                    gen['size'] = total_sz - len(scapy_pkt)
539                    return generate_bytes(gen)
540                else:
541                    return generate_bytes(val)
542        if is_number(sample_val) and is_string(val):
543            # human-value. guess the type and convert to internal value
544            # seems setfieldval already does this for some fields,
545            # but does not convert strings/hex(0x123) to integers and long
546            val = str(val) # unicode -> str(ascii)
547            # parse str to int/long as a decimal or hex
548            val_constructor = type(sample_val)
549            if len(val) == 0:
550                return None
551            elif re.match(r"^0x[\da-f]+$", val, flags=re.IGNORECASE): # hex
552                return val_constructor(val, 16)
553            elif re.match(r"^\d+L?$", val): # base10
554                return val_constructor(val)
555        # generate recursive field-independent values
556        return self._value_from_dict(val)
557
558    def _print_tree(self):
559        pprint(self.protocol_tree)
560
561    def _get_all_db(self):
562        db = {}
563        for pro in self.all_protocols:
564            details = self._get_protocol_details(pro)
565            db[pro] = details
566        return db
567
568    def _get_all_fields(self):
569        fields = []
570        for pro in self.all_protocols:
571            details = self._get_protocol_details(pro)
572            for i in range(0,len(details),1):
573                if len(details[i]) == 3:
574                    fields.append(details[i][1])
575        uniqueFields = list(set(fields))
576        fieldDict = {}
577        for f in uniqueFields:
578            if f in self.regexDB:
579                fieldDict[f] = self.regexDB[f].stringRegex()
580            else:
581                fieldDict[f] = self.ScapyFieldDesc(f).stringRegex()
582        return fieldDict
583
584    def _fully_define(self,pkt):
585        # returns scapy object with all fields initialized
586        rootClass = type(pkt)
587        full_pkt = rootClass(bytes(pkt))
588        full_pkt.build() # this trick initializes offset
589        return full_pkt
590
591    def _bytes_to_value(self, payload_bytes):
592        # generates struct with a value
593        return { "vtype": "BYTES", "base64": bytes_to_b64(payload_bytes) }
594
595    def _pkt_to_field_tree(self,pkt):
596        pkt.build()
597        result = []
598        pcap_struct = self._fully_define(pkt) # structure, which will appear in pcap binary
599        while pkt:
600            layer_id = type(pkt).__name__ # Scapy classname
601            layer_full = self._fully_define(pkt) # current layer recreated from binary to get auto-calculated vals
602            real_layer_id = type(pcap_struct).__name__ if pcap_struct else None
603            valid_struct = True # shows if packet is mapped correctly to the binary representation
604            if not pcap_struct:
605                valid_struct = False
606            elif not issubclass(type(pkt), type(pcap_struct)) and not issubclass(type(pcap_struct), type(pkt)):
607                # structure mismatch. no need to go deeper in pcap_struct
608                valid_struct = False
609                pcap_struct = None
610            fields = []
611            for field_desc in pkt.fields_desc:
612                field_id = field_desc.name
613                ignored = field_id not in layer_full.fields
614                offset = field_desc.offset
615                protocol_offset = pkt.offset
616                field_sz = field_desc.get_size_bytes()
617                # some values are unavailable in pkt(original model)
618                # at the same time,
619                fieldval = pkt.getfieldval(field_id)
620                pkt_fieldval_defined = is_string(fieldval) or is_number(fieldval) or is_bytes3(fieldval)
621                if not pkt_fieldval_defined:
622                    fieldval = layer_full.getfieldval(field_id)
623                value = None
624                hvalue = None
625                value_base64 = None
626                if is_python(3) and is_bytes3(fieldval):
627                    value = self._bytes_to_value(fieldval)
628                    if is_ascii_bytes(fieldval):
629                        hvalue = bytes_to_str(fieldval)
630                    else:
631                        # can't be shown as ascii.
632                        # also this buffer may not be unicode-compatible(still can try to convert)
633                        value = self._bytes_to_value(fieldval)
634                        hvalue = '<binary>'
635                elif not is_string(fieldval):
636                    # value as is. this can be int,long, or custom object(list/dict)
637                    # "nice" human value, i2repr(string) will have quotes, so we have special handling for them
638                    hvalue = field_desc.i2repr(pkt, fieldval)
639
640                    if is_number(fieldval):
641                        value = fieldval
642                        if is_string(hvalue) and re.match(r"^\d+L$", hvalue):
643                            hvalue =  hvalue[:-1] # chop trailing L for long decimal number(python2)
644                    else:
645                        # fieldval is an object( class / list / dict )
646                        # generic serialization/deserialization needed for proper packet rebuilding from packet tree,
647                        # some classes can not be mapped to json, but we can pass them serialize them
648                        # as a python eval expr, value bytes base64, or field machine internal val(m2i)
649                        value = {"vtype": "EXPRESSION", "expr": hvalue}
650                if is_python(3) and is_string(fieldval):
651                    hvalue = value = fieldval
652                if is_python(2) and is_string(fieldval):
653                    if is_ascii(fieldval):
654                        hvalue = value = fieldval
655                    else:
656                        # python2 non-ascii byte buffers
657                        # payload contains non-ascii chars, which
658                        # sometimes can not be passed as unicode strings
659                        value = self._bytes_to_value(fieldval)
660                        hvalue = '<binary>'
661                if field_desc.name == 'load':
662                    # show Padding(and possible similar classes) as Raw
663                    layer_id = 'Raw'
664                    field_sz = len(pkt)
665                    value = self._bytes_to_value(fieldval)
666                field_data = {
667                        "id": field_id,
668                        "value": value,
669                        "hvalue": hvalue,
670                        "offset": offset,
671                        "length": field_sz
672                        }
673                if ignored:
674                    field_data["ignored"] = ignored
675                fields.append(field_data)
676            layer_data = {
677                    "id": layer_id,
678                    "offset": pkt.offset,
679                    "fields": fields,
680                    "real_id": real_layer_id,
681                    "valid_structure": valid_struct,
682                    }
683            result.append(layer_data)
684            pkt = pkt.payload
685            if pcap_struct:
686                pcap_struct = pcap_struct.payload or None
687        return result
688
689#input: container
690#output: md5 encoded in base64
691    def _get_md5(self,container):
692        container = json.dumps(container)
693        m = hashlib.md5()
694        m.update(str_to_bytes(container))
695        res_md5 = bytes_to_b64(m.digest())
696        return res_md5
697
698    def get_version(self):
699        return {'built_by':'itraviv','version':self.version_major+'.'+self.version_minor}
700
701    def supported_methods(self,method_name='all'):
702        if method_name=='all':
703            methods = {}
704            for f in dir(Scapy_service):
705                if f[0]=='_':
706                    continue
707                if inspect.ismethod(eval('Scapy_service.'+f)):
708                    param_list = inspect.getargspec(eval('Scapy_service.'+f))[0]
709                    del param_list[0] #deleting the parameter "self" that appears in every method
710                                      #because the server automatically operates on an instance,
711                                      #and this can cause confusion
712                    methods[f] = (len(param_list), param_list)
713            return methods
714        if method_name in dir(Scapy_service):
715            return True
716        return False
717
718    def _generate_version_hash(self,v_major,v_minor):
719        v_for_hash = v_major+v_minor+v_major+v_minor
720        m = hashlib.md5()
721        m.update(str_to_bytes(v_for_hash))
722        return bytes_to_b64(m.digest())
723
724    def _generate_invalid_version_error(self):
725        error_desc1 = "Provided version handler does not correspond to the server's version.\nUpdate client to latest version.\nServer version:"+self.version_major+"."+self.version_minor
726        return error_desc1
727
728    def _verify_version_handler(self,client_v_handler):
729        return (self.server_v_hashed == client_v_handler)
730
731    def _parse_packet_dict(self, layer, layer_classes, base_layer):
732        class_p = layer_classes[layer['id']] # class id -> class dict
733        scapy_layer = class_p()
734        if isinstance(scapy_layer, Raw):
735            scapy_layer.load = str_to_bytes("dummy")
736        if base_layer == None:
737            base_layer = scapy_layer
738        if 'fields' in layer:
739            self._modify_layer(base_layer, scapy_layer, layer['fields'])
740        return scapy_layer
741
742    def _packet_model_to_scapy_packet(self,data):
743        layer_classes = {}
744        for layer_class in Packet.__subclasses__():
745            layer_classes[layer_class.__name__] = layer_class
746        base_layer = self._parse_packet_dict(data[0], layer_classes, None)
747        for i in range(1,len(data),1):
748            packet_layer = self._parse_packet_dict(data[i], layer_classes, base_layer)
749            base_layer = base_layer/packet_layer
750        return base_layer
751
752    def _pkt_data(self,pkt):
753        if pkt == None:
754            return {'data': [], 'binary': None}
755        data = self._pkt_to_field_tree(pkt)
756        binary = bytes_to_b64(bytes(pkt))
757        res = {'data': data, 'binary': binary}
758        return res
759
760#--------------------------------------------API implementation-------------
761    def get_tree(self,client_v_handler):
762        if not (self._verify_version_handler(client_v_handler)):
763            raise ScapyException(self._generate_invalid_version_error())
764        return self.protocol_tree
765
766    def get_version_handler(self,client_v_major,client_v_minor):
767        v_handle = self._generate_version_hash(client_v_major,client_v_minor)
768        return v_handle
769
770# pkt_descriptor in packet model format (dictionary)
771    def build_pkt(self,client_v_handler,pkt_model_descriptor):
772        if not (self._verify_version_handler(client_v_handler)):
773            raise ScapyException(self._generate_invalid_version_error())
774        pkt = self._packet_model_to_scapy_packet(pkt_model_descriptor)
775        return self._pkt_data(pkt)
776
777
778    def build_pkt_ex(self, client_v_handler, pkt_model_descriptor, extra_options):
779        res = self.build_pkt(client_v_handler, pkt_model_descriptor)
780        pkt = self._packet_model_to_scapy_packet(pkt_model_descriptor)
781
782        field_engine = {}
783        field_engine['instructions'] = []
784        field_engine['error'] = None
785        try:
786            field_engine['instructions'] = self._generate_vm_instructions(pkt, extra_options['field_engine'])
787        except AssertionError as e:
788            field_engine['error'] = e.message
789        except CTRexPacketBuildException as e:
790            field_engine['error'] = e.message
791
792        field_engine['vm_instructions_expressions'] = self.field_engine_instruction_expressions
793        res['field_engine'] = field_engine
794        return res
795
796    def load_instruction_parameter_values(self, client_v_handler, pkt_model_descriptor, vm_instructions_model, parameter_id):
797
798        given_protocol_ids = [str(proto['id']) for proto in pkt_model_descriptor]
799
800        values = {}
801        if parameter_id == "name":
802            values = self._curent_pkt_protocol_fields(given_protocol_ids, "_")
803
804        if parameter_id == "fv_name":
805            values = self._existed_flow_var_names(vm_instructions_model['field_engine']['instructions'])
806
807        if parameter_id == "pkt_offset":
808            values = self._curent_pkt_protocol_fields(given_protocol_ids, ".")
809
810        if parameter_id == "offset":
811            for ip_idx in range(given_protocol_ids.count("IP")):
812                value = "IP:{0}".format(ip_idx)
813                values[value] = value
814
815        return {"map": values}
816
817    def _existed_flow_var_names(self, instructions):
818        return dict((instruction['parameters']['name'], instruction['parameters']['name']) for instruction in instructions if self._nameParamterExist(instruction))
819
820    def _nameParamterExist(self, instruction):
821        try:
822            instruction['parameters']['name']
823            return True
824        except KeyError:
825            return False
826
827    def _curent_pkt_protocol_fields(self, given_protocol_ids, delimiter):
828        given_protocol_classes = [c for c in Packet.__subclasses__() if c.__name__ in given_protocol_ids and c.__name__ != "Ether"]
829        protocol_fields = {}
830        for protocol_class in given_protocol_classes:
831            protocol_name = protocol_class.__name__
832            protocol_count = given_protocol_ids.count(protocol_name)
833            for field_desc in protocol_class.fields_desc:
834                if delimiter == '.' and protocol_count > 1:
835                    for idx in range(protocol_count):
836                        formatted_name = "{0}:{1}{2}{3}".format(protocol_name, idx, delimiter, field_desc.name)
837                        protocol_fields[formatted_name] = formatted_name
838                else:
839                    formatted_name = "{0}{1}{2}".format(protocol_name, delimiter, field_desc.name)
840                protocol_fields[formatted_name] = formatted_name
841
842        return protocol_fields
843
844    def _generate_vm_instructions(self, pkt, field_engine_model_descriptor):
845        self.field_engine_instruction_expressions = []
846        instructions = []
847        instructions_def = field_engine_model_descriptor['instructions']
848        for instruction_def in instructions_def:
849            instruction_id = instruction_def['id']
850            instruction_class = self._vm_instructions[instruction_id]
851            parameters = {k: self._sanitize_value(v) for (k, v) in instruction_def['parameters'].iteritems()}
852            instructions.append(instruction_class(**parameters))
853
854        fe_parameters = field_engine_model_descriptor['global_parameters']
855
856        cache_size = None
857        if "cache_size" in fe_parameters:
858            cache_size = int(fe_parameters['cache_size'])
859
860        pkt_builder = STLPktBuilder(pkt=pkt, vm=STLScVmRaw(instructions, cache_size=cache_size))
861        pkt_builder.compile()
862        return pkt_builder.get_vm_data()
863
864    def _sanitize_value(self, val):
865        if val == "None" or val == "none":
866            return None
867        if val == "true":
868            return True
869        elif val == "false":
870            return False
871        elif self._is_int(val):
872            return int(val)
873        else:
874            str_val = str(val)
875            return int(str_val, 16) if str_val.startswith("0x") else str_val
876
877    def _get_instruction_parameter_meta(self, param_id):
878        for meta in self.instruction_parameter_meta_definitions:
879            if meta['id'] == param_id:
880                return meta
881        raise Scapy_Exception("Unable to get meta for {0}" % param_id)
882
883    def _is_int(self, val):
884        try:
885            int(val)
886            return True
887        except ValueError:
888            return False
889
890    # @deprecated. to be removed
891    def get_all(self,client_v_handler):
892        if not (self._verify_version_handler(client_v_handler)):
893            raise ScapyException(self._generate_invalid_version_error())
894        fields=self._get_all_fields()
895        db=self._get_all_db()
896        fields_md5 = self._get_md5(fields)
897        db_md5 = self._get_md5(db)
898        res = {}
899        res['db'] = db
900        res['fields'] = fields
901        res['db_md5'] = db_md5
902        res['fields_md5'] = fields_md5
903        return res
904
905    def _is_packet_class(self, pkt_class):
906        # returns true for final Packet classes. skips aliases and metaclasses
907        return issubclass(pkt_class, Packet) and pkt_class.name and pkt_class.fields_desc
908
909    def _getDummyPacket(self, pkt_class):
910        if issubclass(pkt_class, Raw):
911            # need to have some payload. otherwise won't appear in the binary chunk
912            return pkt_class(load=str_to_bytes("dummy"))
913        else:
914            return pkt_class()
915
916
917    def _get_payload_classes(self, pkt_class):
918        # tries to find, which subclasses allowed.
919        # this can take long time, since it tries to build packets with all subclasses(O(N))
920        allowed_subclasses = []
921        for pkt_subclass in conf.layers:
922            if self._is_packet_class(pkt_subclass):
923                try:
924                    pkt_w_payload = pkt_class() / self._getDummyPacket(pkt_subclass)
925                    recreated_pkt = pkt_class(bytes(pkt_w_payload))
926                    if type(recreated_pkt.lastlayer()) is pkt_subclass:
927                        allowed_subclasses.append(pkt_subclass)
928                except Exception as e:
929                    # no actions needed on fail, just sliently skip
930                    pass
931        return allowed_subclasses
932
933    def _get_fields_definition(self, pkt_class, fieldsDef):
934        # fieldsDef - array of field definitions(or empty array)
935        fields = []
936        for field_desc in pkt_class.fields_desc:
937            fieldId = field_desc.name
938            field_data = {
939                    "id": fieldId,
940                    "name": field_desc.name
941            }
942            for fieldDef in fieldsDef:
943                if fieldDef['id'] == fieldId:
944                    field_data.update(fieldDef)
945            if isinstance(field_desc, EnumField):
946                try:
947                    field_data["values_dict"] = field_desc.s2i
948                    if field_data.get("type") == None:
949                        if len(field_data["values_dict"] > 0):
950                            field_data["type"] = "ENUM"
951                        elif fieldId == 'load':
952                            field_data["type"] = "BYTES"
953                        else:
954                            field_data["type"] = "STRING"
955                    field_data["values_dict"] = field_desc.s2i
956                except:
957                    # MultiEnumField doesn't have s2i. need better handling
958                    pass
959            fields.append(field_data)
960        return fields
961
962    def get_definitions(self,client_v_handler, def_filter):
963        # def_filter is an array of classnames or None
964        all_classes = Packet.__subclasses__() # as an alternative to conf.layers
965        if def_filter:
966            all_classes = [c for c in all_classes if c.__name__ in def_filter]
967        protocols = []
968        for pkt_class in all_classes:
969            if self._is_packet_class(pkt_class):
970                # enumerate all non-abstract Packet classes
971                protocolId = pkt_class.__name__
972                protoDef = self.protocol_definitions.get(protocolId) or {}
973                protocols.append({
974                    "id": protocolId,
975                    "name": protoDef.get('name') or pkt_class.name,
976                    "fields": self._get_fields_definition(pkt_class, protoDef.get('fields') or [])
977                    })
978        res = {"protocols": protocols,
979               "feInstructionParameters": self.instruction_parameter_meta_definitions,
980               "feInstructions": self.field_engine_instructions_meta,
981               "feParameters": self.field_engine_parameter_meta_definitions}
982        return res
983
984    def get_payload_classes(self,client_v_handler, pkt_model_descriptor):
985        pkt = self._packet_model_to_scapy_packet(pkt_model_descriptor)
986        pkt_class = type(pkt.lastlayer())
987        protocolDef = self.protocol_definitions.get(pkt_class.__name__)
988        if protocolDef and protocolDef.get('payload'):
989            return protocolDef['payload']
990        return [c.__name__ for c in self._get_payload_classes(pkt_class)]
991
992#input in string encoded base64
993    def check_update_of_dbs(self,client_v_handler,db_md5,field_md5):
994        if not (self._verify_version_handler(client_v_handler)):
995            raise ScapyException(self._generate_invalid_version_error())
996        fields=self._get_all_fields()
997        db=self._get_all_db()
998        current_db_md5 = self._get_md5(db)
999        current_field_md5 = self._get_md5(fields)
1000        res = []
1001        if (field_md5 == current_field_md5):
1002            if (db_md5 == current_db_md5):
1003                return True
1004            else:
1005                raise ScapyException("Protocol DB is not up to date")
1006        else:
1007            raise ScapyException("Fields DB is not up to date")
1008
1009    def _modify_layer(self, scapy_pkt, scapy_layer, fields):
1010        for field in fields:
1011            fieldId = str(field['id'])
1012            fieldval = self._field_value_from_def(scapy_pkt, scapy_layer, fieldId, field['value'])
1013            if fieldval is not None:
1014                scapy_layer.setfieldval(fieldId, fieldval)
1015            else:
1016                scapy_layer.delfieldval(fieldId)
1017
1018    def _is_last_layer(self, layer):
1019        # can be used, that layer has no payload
1020        # if true, the layer.payload is likely NoPayload()
1021        return layer is layer.lastlayer()
1022
1023#input of binary_pkt must be encoded in base64
1024    def reconstruct_pkt(self,client_v_handler,binary_pkt,model_descriptor):
1025        pkt_bin = b64_to_bytes(binary_pkt)
1026        scapy_pkt = Ether(pkt_bin)
1027        if not model_descriptor:
1028            model_descriptor = []
1029        for depth in range(len(model_descriptor)):
1030            model_layer = model_descriptor[depth]
1031            if model_layer.get('delete') is True:
1032                # slice packet from the current item
1033                if depth == 0:
1034                    scapy_pkt = None
1035                    break
1036                else:
1037                    scapy_pkt[depth-1].payload = None
1038                    break
1039            if depth > 0 and self._is_last_layer(scapy_pkt[depth-1]):
1040                # insert new layer(s) from json definition
1041                remaining_definitions = model_descriptor[depth:]
1042                pkt_to_append = self._packet_model_to_scapy_packet(remaining_definitions)
1043                scapy_pkt = scapy_pkt / pkt_to_append
1044                break
1045            # modify fields of existing stack items
1046            scapy_layer = scapy_pkt[depth]
1047            if model_layer['id'] != type(scapy_layer).__name__:
1048                # TODO: support replacing payload, instead of breaking
1049                raise ScapyException("Protocol id inconsistent")
1050            if 'fields' in model_layer:
1051                self._modify_layer(scapy_pkt, scapy_layer, model_layer['fields'])
1052        return self._pkt_data(scapy_pkt)
1053
1054    def read_pcap(self,client_v_handler,pcap_base64):
1055        pcap_bin = b64_to_bytes(pcap_base64)
1056        pcap = []
1057        res_packets = []
1058        with tempfile.NamedTemporaryFile(mode='w+b') as tmpPcap:
1059            tmpPcap.write(pcap_bin)
1060            tmpPcap.flush()
1061            pcap = rdpcap(tmpPcap.name)
1062        for scapy_packet in pcap:
1063            res_packets.append(self._pkt_data(scapy_packet))
1064        return res_packets
1065
1066    def write_pcap(self,client_v_handler,packets_base64):
1067        packets = [Ether(b64_to_bytes(pkt_b64)) for pkt_b64 in packets_base64]
1068        pcap_bin = None
1069        with tempfile.NamedTemporaryFile(mode='r+b') as tmpPcap:
1070            wrpcap(tmpPcap.name, packets)
1071            pcap_bin = tmpPcap.read()
1072        return bytes_to_b64(pcap_bin)
1073
1074
1075#---------------------------------------------------------------------------
1076
1077
1078