scapy_service.py revision ca4ceb64
1
2import os
3import sys
4
5stl_pathname = os.path.abspath(os.path.join(os.pardir, os.pardir))
6sys.path.append(stl_pathname)
7
8from trex_stl_lib.api import *
9import trex_stl_lib.trex_stl_packet_builder_scapy
10import tempfile
11import hashlib
12import base64
13import numbers
14import random
15from inspect import getdoc
16import json
17import re
18from pprint import pprint
19
20# add some layers as an example
21# need to test more
22from scapy.layers.dns import *
23from scapy.layers.dhcp import *
24from scapy.layers.ipsec import *
25from scapy.layers.netflow import *
26from scapy.layers.sctp import *
27from scapy.layers.tftp import *
28
29from scapy.contrib.mpls import *
30from scapy.contrib.igmp import *
31from scapy.contrib.igmpv3 import *
32
33
34
35
36#additional_stl_udp_pkts = os.path.abspath(os.path.join(os.pardir,os.pardir,os.pardir,os.pardir, os.pardir,'stl'))
37#sys.path.append(additional_stl_udp_pkts)
38#from udp_1pkt_vxlan import VXLAN
39#sys.path.remove(additional_stl_udp_pkts)
40
41try:
42    from cStringIO import StringIO
43except ImportError:
44    from io import StringIO
45
46
47
48
49class Scapy_service_api():
50
51    def get_version_handler(self,client_v_major,client_v_minor):
52        """ get_version_handler(self,client_v_major,client_v_minor)
53
54            Gives a handler to client to connect and use server api
55
56            Parameters
57            ----------
58            client_v_major - major number of api version on the client side
59
60            Returns
61            -------
62            Handler(string) to provide when using server api
63        """
64        pass
65    def get_all(self,client_v_handler):
66        """ get_all(self,client_v_handler)
67
68        Sends all the protocols and fields that Scapy Service supports.
69        also sends the md5 of the Protocol DB and Fields DB used to check if the DB's are up to date
70
71        Parameters
72        ----------
73        None
74
75        Returns
76        -------
77        Dictionary (of protocol DB and scapy fields DB)
78
79        Raises
80        ------
81        Raises an exception when a DB error occurs (i.e a layer is not loaded properly and has missing components)
82        """
83        pass
84
85    def check_update_of_dbs(self,client_v_handler,db_md5,field_md5):
86        """ check_update_of_dbs(self,client_v_handler,db_md5,field_md5)
87        Checks if the Scapy Service running on the server has a newer version of the databases that the client has
88
89        Parameters
90        ----------
91        db_md5 - The md5 that was delivered with the protocol database that the client owns, when first received at the client
92        field_md5 - The md5 that was delivered with the fields database that the client owns, when first received at the client
93
94        Returns
95        -------
96        True/False according the Databases version(determined by their md5)
97
98        Raises
99        ------
100        Raises an exception (ScapyException) when protocol DB/Fields DB is not up to date
101        """
102        pass
103
104
105    def build_pkt(self,client_v_handler,pkt_model_descriptor):
106        """ build_pkt(self,client_v_handler,pkt_model_descriptor) -> Dictionary (of Offsets,Show2 and Buffer)
107
108        Performs calculations on the given packet and returns results for that packet.
109
110        Parameters
111        ----------
112        pkt_descriptor - An array of dictionaries describing a network packet
113
114        Returns
115        -------
116        - The packets offsets: each field in every layer is mapped inside the Offsets Dictionary
117        - The Show2: A description of each field and its value in every layer of the packet
118        - The Buffer: The Hexdump of packet encoded in base64
119
120        Raises
121        ------
122        will raise an exception when the Scapy string format is illegal, contains syntax error, contains non-supported
123        protocl, etc.
124        """
125        pass
126
127    def build_pkt_ex(self, client_v_handler, pkt_model_descriptor, extra_options):
128        """ build_pkt_ex(self,client_v_handler,pkt_model_descriptor, extra_options) -> Dictionary (of Offsets,Show2 and Buffer)
129        Performs calculations on the given packet and returns results for that packet.
130
131        Parameters
132        ----------
133        pkt_descriptor - An array of dictionaries describing a network packet
134        extra_options - A dictionary of extra options required for building packet
135
136        Returns
137        -------
138        - The packets offsets: each field in every layer is mapped inside the Offsets Dictionary
139        - The Show2: A description of each field and its value in every layer of the packet
140        - The Buffer: The Hexdump of packet encoded in base64
141
142        Raises
143        ------
144        will raise an exception when the Scapy string format is illegal, contains syntax error, contains non-supported
145        protocl, etc.
146        """
147        pass
148
149    def load_instruction_parameter_values(self, client_v_handler, pkt_model_descriptor, vm_instructions_model, parameter_id):
150        """ load_instruction_parameter_values(self,client_v_handler,pkt_model_descriptor, vm_instructions_model, parameter_id) -> Dictionary (of possible parameter values)
151        Returns possible valies for given pararameter id depends on current pkt structure and vm_instructions
152        model.
153
154        Parameters
155        ----------
156        pkt_descriptor - An array of dictionaries describing a network packet
157        vm_instructions_model - A dictionary of extra options required for building packet
158        parameter_id - A string of parameter id
159
160        Returns
161        -------
162        Possible parameter values map.
163
164        """
165        pass
166
167    def get_tree(self,client_v_handler):
168        """ get_tree(self) -> Dictionary describing an example of hierarchy in layers
169
170        Scapy service holds a tree of layers that can be stacked to a recommended packet
171        according to the hierarchy
172
173        Parameters
174        ----------
175        None
176
177        Returns
178        -------
179        Returns an example hierarchy tree of layers that can be stacked to a packet
180
181        Raises
182        ------
183        None
184        """
185        pass
186
187    def reconstruct_pkt(self,client_v_handler,binary_pkt,model_descriptor):
188        """ reconstruct_pkt(self,client_v_handler,binary_pkt)
189
190        Makes a Scapy valid packet by applying changes to binary packet and returns all information returned in build_pkt
191
192        Parameters
193        ----------
194        Source packet in binary_pkt, formatted in "base64" encoding
195        List of changes in model_descriptor
196
197        Returns
198        -------
199        All data provided in build_pkt:
200        show2 - detailed description of the packet
201        buffer - the packet presented in binary
202        offsets - the offset[in bytes] of each field in the packet
203
204        """
205        pass
206
207    def read_pcap(self,client_v_handler,pcap_base64):
208        """ read_pcap(self,client_v_handler,pcap_base64)
209
210        Parses pcap file contents and returns an array with build_pkt information for each packet
211
212        Parameters
213        ----------
214        binary pcap file in base64 encoding
215
216        Returns
217        -------
218        Array of build_pkt(packet)
219        """
220        pass
221
222    def write_pcap(self,client_v_handler,packets_base64):
223        """ write_pcap(self,client_v_handler,packets_base64)
224
225        Writes binary packets to pcap file
226
227        Parameters
228        ----------
229        array of binary packets in base64 encoding
230
231        Returns
232        -------
233        binary pcap file in base64 encoding
234        """
235        pass
236
237    def get_definitions(self,client_v_handler, def_filter):
238        """ get_definitions(self,client_v_handler, def_filter)
239
240        Returns protocols and fields metadata of scapy service
241
242        Parameters
243        ----------
244        def_filter - array of protocol names
245
246        Returns
247        -------
248        definitions for protocols
249        """
250        pass
251
252    def get_payload_classes(self,client_v_handler, pkt_model_descriptor):
253        """ get_payload_classes(self,client_v_handler, pkt_model_descriptor)
254
255        Returns an array of protocol classes, which normally can be used as a payload
256
257        Parameters
258        ----------
259        pkt_model_descriptor - see build_pkt
260
261        Returns
262        -------
263        array of supported protocol classes
264        """
265        pass
266
267    def get_templates(self,client_v_handler):
268        """ get_templates(self,client_v_handler)
269
270        Returns an array of templates, which normally can be used for creating packet
271
272        Parameters
273        ----------
274
275        Returns
276        -------
277        array of templates
278        """
279        pass
280
281    def get_template(self,client_v_handler,template):
282        """ get_template(self,client_v_handler,template)
283
284        Returns a template, which normally can be used for creating packet
285
286        Parameters
287        ----------
288
289        Returns
290        -------
291        base64 of template content
292        """
293        pass
294
295def is_python(version):
296    return version == sys.version_info[0]
297
298def is_number(obj):
299    return isinstance(obj, numbers.Number)
300
301def is_string(obj):
302    return type(obj) == str or type(obj).__name__ == 'unicode' # python3 doesn't have unicode type
303
304def is_ascii_str(strval):
305    return strval and all(ord(ch) < 128 for ch in strval)
306
307def is_ascii_bytes(buf):
308    return buf and all(byte < 128 for byte in buf)
309
310def is_ascii(obj):
311    if is_bytes3(obj):
312        return is_ascii_bytes(obj)
313    else:
314        return is_ascii_str(obj)
315
316def is_bytes3(obj):
317    # checks if obj is exactly bytes(always false for python2)
318    return is_python(3) and type(obj) == bytes
319
320def str_to_bytes(strval):
321    return strval.encode("utf8")
322
323def bytes_to_str(buf):
324    return buf.decode("utf8")
325
326def b64_to_bytes(payload_base64):
327    # get bytes from base64 string(unicode)
328    return base64.b64decode(payload_base64)
329
330def bytes_to_b64(buf):
331    # bytes to base64 string(unicode)
332    return base64.b64encode(buf).decode('ascii')
333
334def get_sample_field_val(scapy_layer, fieldId):
335    # get some sample value for the field, to determine the value type
336    # use random or serialized value if default value is None
337    field_desc, current_val = scapy_layer.getfield_and_val(fieldId)
338    if current_val is not None:
339        return current_val
340    try:
341        # try to get some random value to determine type
342        return field_desc.randval()._fix()
343    except:
344        pass
345    try:
346        # try to serialize/deserialize
347        ltype = type(scapy_layer)
348        pkt = ltype(bytes(ltype()))
349        return pkt.getfieldval(fieldId)
350    except:
351        pass
352
353def generate_random_bytes(sz, seed, start, end):
354    # generate bytes of specified range with a fixed seed and size
355    rnd = random.Random()
356    n = end - start + 1
357    if is_python(2):
358        rnd = random.Random(seed)
359        res = [start + int(rnd.random()*n) for _i in range(sz)]
360        return ''.join(chr(x) for x in res)
361    else:
362        rnd = random.Random()
363        # to generate same random sequence as 2.x
364        rnd.seed(seed, version=1)
365        res = [start + int(rnd.random()*n) for _i in range(sz)]
366        return bytes(res)
367
368def generate_bytes_from_template(sz, template):
369    # generate bytes by repeating a template
370    res = str_to_bytes('') # new bytes array
371    if len(template) == 0:
372        return res
373    while len(res) < sz:
374        res = res + template
375    return res[:sz]
376
377def parse_template_code(template_code):
378    template_code = re.sub("0[xX]", '', template_code) # remove 0x
379    template_code = re.sub("[\s]", '', template_code) # remove spaces
380    return bytearray.fromhex(template_code)
381
382def verify_payload_size(size):
383    assert(size != None)
384    if (size > (1<<20)): # 1Mb ought to be enough for anybody
385        raise ValueError('size is too large')
386
387def generate_bytes(bytes_definition):
388    # accepts a bytes definition object
389    # {generate: random_bytes or random_ascii, seed: <seed_number>, size: <size_bytes>}
390    # {generate: template, template_base64: '<base64str>',  size: <size_bytes>}
391    # {generate: template_code, template_text_code: '<template_code_str>',  size: <size_bytes>}
392    gen_type = bytes_definition.get('generate')
393    if gen_type == None:
394        return b64_to_bytes(bytes_definition['base64'])
395    elif gen_type == 'template_code':
396        code = parse_template_code(bytes_definition["template_code"])
397        bytes_size = int(bytes_definition.get('size') or len(code))
398        verify_payload_size(bytes_size)
399        return generate_bytes_from_template(bytes_size, code)
400    else:
401        bytes_size = int(bytes_definition['size']) # required
402        seed = int(bytes_definition.get('seed') or 12345) # optional
403        verify_payload_size(bytes_size)
404        if gen_type == 'random_bytes':
405            return generate_random_bytes(bytes_size, seed, 0, 0xFF)
406        elif gen_type == 'random_ascii':
407            return generate_random_bytes(bytes_size, seed, 0x20, 0x7E)
408        elif gen_type == 'template':
409            return generate_bytes_from_template(bytes_size, b64_to_bytes(bytes_definition["template_base64"]))
410
411class ScapyException(Exception): pass
412class Scapy_service(Scapy_service_api):
413
414#----------------------------------------------------------------------------------------------------
415    class ScapyFieldDesc:
416        def __init__(self,FieldName,regex='empty'):
417            self.FieldName = FieldName
418            self.regex = regex
419            #defualt values - should be changed when needed, or added to constructor
420            self.string_input =""
421            self.string_input_mex_len = 1
422            self.integer_input = 0
423            self.integer_input_min = 0
424            self.integer_input_max = 1
425            self.input_array = []
426            self.input_list_max_len = 1
427
428        def stringRegex(self):
429            return self.regex
430#----------------------------------------------------------------------------------------------------
431    def __init__(self):
432        self.Raw = {'Raw':''}
433        self.high_level_protocols = ['Raw']
434        self.transport_protocols = {'TCP':self.Raw,'UDP':self.Raw}
435        self.network_protocols = {'IP':self.transport_protocols ,'ARP':''}
436        self.low_level_protocols = { 'Ether': self.network_protocols }
437        self.regexDB= {'MACField' : self.ScapyFieldDesc('MACField','^([0-9a-fA-F][0-9a-fA-F]:){5}([0-9a-fA-F][0-9a-fA-F])$'),
438              'IPField' : self.ScapyFieldDesc('IPField','^(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])$')}
439        self.all_protocols = self._build_lib()
440        self.protocol_tree = {'ALL':{'Ether':{'ARP':{},'IP':{'TCP':{'RAW':'payload'},'UDP':{'RAW':'payload'}}}}}
441        self.version_major = '1'
442        self.version_minor = '01'
443        self.server_v_hashed = self._generate_version_hash(self.version_major,self.version_minor)
444        self.protocol_definitions = {} # protocolId -> prococol definition overrides data
445        self.field_engine_supported_protocols = {}
446        self.instruction_parameter_meta_definitions = []
447        self.field_engine_parameter_meta_definitions = []
448        self.field_engine_templates_definitions = []
449        self.field_engine_instructions_meta = []
450        self.field_engine_instruction_expressions = []
451        self._load_definitions_from_json()
452        self._load_field_engine_meta_from_json()
453        self._vm_instructions = dict([m for m in inspect.getmembers(trex_stl_lib.trex_stl_packet_builder_scapy, inspect.isclass) if m[1].__module__ == 'trex_stl_lib.trex_stl_packet_builder_scapy'])
454
455    def _load_definitions_from_json(self):
456        # load protocol definitions from a json file
457        self.protocol_definitions = {}
458        with open('protocols.json', 'r') as f:
459            protocols = json.load(f)
460            for protocol in protocols:
461                self.protocol_definitions[ protocol['id'] ] = protocol
462
463    def _load_field_engine_meta_from_json(self):
464        # load protocol definitions from a json file
465        self.instruction_parameter_meta_definitions = []
466        self.field_engine_supported_protocols = {}
467        self.field_engine_parameter_meta_definitions = []
468        self.field_engine_templates_definitions = []
469        with open('field_engine.json', 'r') as f:
470            metas = json.load(f)
471            self.instruction_parameter_meta_definitions = metas["instruction_params_meta"]
472            self.field_engine_instructions_meta = metas["instructions"]
473            self._append_intructions_help()
474            self.field_engine_supported_protocols = metas["supported_protocols"]
475            self.field_engine_parameter_meta_definitions = metas["global_params_meta"]
476            self.field_engine_templates_definitions = metas["templates"]
477
478
479    def _append_intructions_help(self):
480        for instruction_meta in self.field_engine_instructions_meta:
481            clazz = eval(instruction_meta['id'])
482            instruction_meta['help'] = base64.b64encode(getdoc(clazz.__init__).encode()).decode('ascii')
483
484    def _all_protocol_structs(self):
485        old_stdout = sys.stdout
486        sys.stdout = mystdout = StringIO()
487        ls()
488        sys.stdout = old_stdout
489        all_protocol_data= mystdout.getvalue()
490        return all_protocol_data
491
492    def _protocol_struct(self,protocol):
493        if '_' in protocol:
494            return []
495        if not protocol=='':
496            if protocol not in self.all_protocols:
497                return 'protocol not supported'
498        protocol = eval(protocol)
499        old_stdout = sys.stdout
500        sys.stdout = mystdout = StringIO()
501        ls(protocol)
502        sys.stdout = old_stdout
503        protocol_data= mystdout.getvalue()
504        return protocol_data
505
506    def _build_lib(self):
507        lib = self._all_protocol_structs()
508        lib = lib.splitlines()
509        all_protocols=[]
510        for entry in lib:
511            entry = entry.split(':')
512            all_protocols.append(entry[0].strip())
513        del all_protocols[len(all_protocols)-1]
514        return all_protocols
515
516    def _parse_description_line(self,line):
517        line_arr = [x.strip() for x in re.split(': | = ',line)]
518        return tuple(line_arr)
519
520    def _parse_entire_description(self,description):
521        description = description.split('\n')
522        description_list = [self._parse_description_line(x) for x in description]
523        del description_list[len(description_list)-1]
524        return description_list
525
526    def _get_protocol_details(self,p_name):
527        protocol_str = self._protocol_struct(p_name)
528        if protocol_str=='protocol not supported':
529            return 'protocol not supported'
530        if len(protocol_str) is 0:
531            return []
532        tupled_protocol = self._parse_entire_description(protocol_str)
533        return tupled_protocol
534
535    def _value_from_dict(self, val):
536        # allows building python objects from json
537        if type(val) == type({}):
538            value_type = val['vtype']
539            if value_type == 'EXPRESSION':
540                return eval(val['expr'], scapy.all.__dict__)
541            elif value_type == 'BYTES':   # bytes payload(ex Raw.load)
542                return generate_bytes(val)
543            elif value_type == 'OBJECT':
544                return val['value']
545            else:
546                return val # it's better to specify type explicitly
547        elif type(val) == type([]):
548            return [self._value_from_dict(v) for v in val]
549        else:
550            return val
551
552    def _field_value_from_def(self, scapy_pkt, layer, fieldId, val):
553        field_desc = layer.get_field(fieldId)
554        sample_val = get_sample_field_val(layer, fieldId)
555        # extensions for field values
556        if type(val) == type({}):
557            value_type = val['vtype']
558            if value_type == 'UNDEFINED': # clear field value
559                return None
560            elif value_type == 'RANDOM': # random field value
561                return field_desc.randval()
562            elif value_type == 'MACHINE': # internal machine field repr
563                return field_desc.m2i(layer, b64_to_bytes(val['base64']))
564            elif value_type == 'BYTES':
565                if 'total_size' in val: # custom case for total pkt size
566                    gen = {}
567                    gen.update(val)
568                    total_sz = gen['total_size']
569                    del gen['total_size']
570                    ether_chksum_size_bytes = 4 # will be added outside of Scapy. needs to be excluded here
571                    gen['size'] = total_sz - len(scapy_pkt) - ether_chksum_size_bytes
572                    return generate_bytes(gen)
573                else:
574                    return generate_bytes(val)
575        if is_number(sample_val) and is_string(val):
576            # human-value. guess the type and convert to internal value
577            # seems setfieldval already does this for some fields,
578            # but does not convert strings/hex(0x123) to integers and long
579            val = str(val) # unicode -> str(ascii)
580            # parse str to int/long as a decimal or hex
581            val_constructor = type(sample_val)
582            if len(val) == 0:
583                return None
584            elif re.match(r"^0x[\da-f]+$", val, flags=re.IGNORECASE): # hex
585                return val_constructor(val, 16)
586            elif re.match(r"^\d+L?$", val): # base10
587                return val_constructor(val)
588        # generate recursive field-independent values
589        return self._value_from_dict(val)
590
591    def _print_tree(self):
592        pprint(self.protocol_tree)
593
594    def _get_all_db(self):
595        db = {}
596        for pro in self.all_protocols:
597            details = self._get_protocol_details(pro)
598            db[pro] = details
599        return db
600
601    def _get_all_fields(self):
602        fields = []
603        for pro in self.all_protocols:
604            details = self._get_protocol_details(pro)
605            for i in range(0,len(details),1):
606                if len(details[i]) == 3:
607                    fields.append(details[i][1])
608        uniqueFields = list(set(fields))
609        fieldDict = {}
610        for f in uniqueFields:
611            if f in self.regexDB:
612                fieldDict[f] = self.regexDB[f].stringRegex()
613            else:
614                fieldDict[f] = self.ScapyFieldDesc(f).stringRegex()
615        return fieldDict
616
617    def _fully_define(self,pkt):
618        # returns scapy object with all fields initialized
619        rootClass = type(pkt)
620        full_pkt = rootClass(bytes(pkt))
621        full_pkt.build() # this trick initializes offset
622        return full_pkt
623
624    def _bytes_to_value(self, payload_bytes):
625        # generates struct with a value
626        return { "vtype": "BYTES", "base64": bytes_to_b64(payload_bytes) }
627
628    def _pkt_to_field_tree(self,pkt):
629        pkt.build()
630        result = []
631        pcap_struct = self._fully_define(pkt) # structure, which will appear in pcap binary
632        while pkt:
633            layer_id = type(pkt).__name__ # Scapy classname
634            layer_full = self._fully_define(pkt) # current layer recreated from binary to get auto-calculated vals
635            real_layer_id = type(pcap_struct).__name__ if pcap_struct else None
636            valid_struct = True # shows if packet is mapped correctly to the binary representation
637            if not pcap_struct:
638                valid_struct = False
639            elif not issubclass(type(pkt), type(pcap_struct)) and not issubclass(type(pcap_struct), type(pkt)):
640                # structure mismatch. no need to go deeper in pcap_struct
641                valid_struct = False
642                pcap_struct = None
643            fields = []
644            for field_desc in pkt.fields_desc:
645                field_id = field_desc.name
646                ignored = field_id not in layer_full.fields
647                # scapy offset/length calculation doesn't support dynamic size structures
648                # since PktClass.fields_desc is a singletone,
649                # _offset/size can be missing, uninitialized or contain values from the previous runs
650                offset = getattr(field_desc, '_offset', None)
651                protocol_offset = getattr(pkt, '_offset', None)
652                field_sz = field_desc.get_size_bytes()
653                # some values are unavailable in pkt(original model)
654                # at the same time,
655                fieldval = pkt.getfieldval(field_id)
656                pkt_fieldval_defined = is_string(fieldval) or is_number(fieldval) or is_bytes3(fieldval)
657                if not pkt_fieldval_defined:
658                    fieldval = layer_full.getfieldval(field_id)
659                value = None
660                hvalue = None
661                value_base64 = None
662                if is_python(3) and is_bytes3(fieldval):
663                    value = self._bytes_to_value(fieldval)
664                    if is_ascii_bytes(fieldval):
665                        hvalue = bytes_to_str(fieldval)
666                    else:
667                        # can't be shown as ascii.
668                        # also this buffer may not be unicode-compatible(still can try to convert)
669                        value = self._bytes_to_value(fieldval)
670                        hvalue = '<binary>'
671                elif not is_string(fieldval):
672                    # value as is. this can be int,long, or custom object(list/dict)
673                    # "nice" human value, i2repr(string) will have quotes, so we have special handling for them
674                    hvalue = field_desc.i2repr(pkt, fieldval)
675
676                    if is_number(fieldval):
677                        value = fieldval
678                        if is_string(hvalue) and re.match(r"^\d+L$", hvalue):
679                            hvalue =  hvalue[:-1] # chop trailing L for long decimal number(python2)
680                    else:
681                        # fieldval is an object( class / list / dict )
682                        # generic serialization/deserialization needed for proper packet rebuilding from packet tree,
683                        # some classes can not be mapped to json, but we can pass them serialize them
684                        # as a python eval expr, value bytes base64, or field machine internal val(m2i)
685                        value = {"vtype": "EXPRESSION", "expr": hvalue}
686                if is_python(3) and is_string(fieldval):
687                    hvalue = value = fieldval
688                if is_python(2) and is_string(fieldval):
689                    if is_ascii(fieldval):
690                        hvalue = value = fieldval
691                    else:
692                        # python2 non-ascii byte buffers
693                        # payload contains non-ascii chars, which
694                        # sometimes can not be passed as unicode strings
695                        value = self._bytes_to_value(fieldval)
696                        hvalue = '<binary>'
697                if field_desc.name == 'load':
698                    # show Padding(and possible similar classes) as Raw
699                    layer_id = 'Raw'
700                    field_sz = len(pkt)
701                    value = self._bytes_to_value(fieldval)
702                field_data = {
703                        "id": field_id,
704                        "value": value,
705                        "hvalue": hvalue,
706                        "offset": offset,
707                        "length": field_sz
708                        }
709                if ignored:
710                    field_data["ignored"] = ignored
711                fields.append(field_data)
712            layer_data = {
713                    "id": layer_id,
714                    "offset": protocol_offset,
715                    "fields": fields,
716                    "real_id": real_layer_id,
717                    "valid_structure": valid_struct,
718                    }
719            result.append(layer_data)
720            pkt = pkt.payload
721            if pcap_struct:
722                pcap_struct = pcap_struct.payload or None
723        return result
724
725#input: container
726#output: md5 encoded in base64
727    def _get_md5(self,container):
728        container = json.dumps(container)
729        m = hashlib.md5()
730        m.update(str_to_bytes(container))
731        res_md5 = bytes_to_b64(m.digest())
732        return res_md5
733
734    def get_version(self):
735        return {'built_by':'itraviv','version':self.version_major+'.'+self.version_minor}
736
737    def supported_methods(self,method_name='all'):
738        if method_name=='all':
739            methods = {}
740            for f in dir(Scapy_service):
741                if f[0]=='_':
742                    continue
743                if inspect.ismethod(eval('Scapy_service.'+f)):
744                    param_list = inspect.getargspec(eval('Scapy_service.'+f))[0]
745                    del param_list[0] #deleting the parameter "self" that appears in every method
746                                      #because the server automatically operates on an instance,
747                                      #and this can cause confusion
748                    methods[f] = (len(param_list), param_list)
749            return methods
750        if method_name in dir(Scapy_service):
751            return True
752        return False
753
754    def _generate_version_hash(self,v_major,v_minor):
755        v_for_hash = v_major+v_minor+v_major+v_minor
756        m = hashlib.md5()
757        m.update(str_to_bytes(v_for_hash))
758        return bytes_to_b64(m.digest())
759
760    def _generate_invalid_version_error(self):
761        error_desc1 = "Provided version handler does not correspond to the server's version.\nUpdate client to latest version.\nServer version:"+self.version_major+"."+self.version_minor
762        return error_desc1
763
764    def _verify_version_handler(self,client_v_handler):
765        return (self.server_v_hashed == client_v_handler)
766
767    def _parse_packet_dict(self, layer, layer_classes, base_layer):
768        class_p = layer_classes[layer['id']] # class id -> class dict
769        scapy_layer = class_p()
770        if isinstance(scapy_layer, Raw):
771            scapy_layer.load = str_to_bytes("dummy")
772        if base_layer == None:
773            base_layer = scapy_layer
774        if 'fields' in layer:
775            self._modify_layer(base_layer, scapy_layer, layer['fields'])
776        return scapy_layer
777
778    def _packet_model_to_scapy_packet(self,data):
779        layer_classes = {}
780        for layer_class in Packet.__subclasses__():
781            layer_classes[layer_class.__name__] = layer_class
782        base_layer = self._parse_packet_dict(data[0], layer_classes, None)
783        for i in range(1,len(data),1):
784            packet_layer = self._parse_packet_dict(data[i], layer_classes, base_layer)
785            base_layer = base_layer/packet_layer
786        return base_layer
787
788    def _pkt_data(self,pkt):
789        if pkt == None:
790            return {'data': [], 'binary': None}
791        data = self._pkt_to_field_tree(pkt)
792        binary = bytes_to_b64(bytes(pkt))
793        res = {'data': data, 'binary': binary}
794        return res
795
796#--------------------------------------------API implementation-------------
797    def get_tree(self,client_v_handler):
798        if not (self._verify_version_handler(client_v_handler)):
799            raise ScapyException(self._generate_invalid_version_error())
800        return self.protocol_tree
801
802    def get_version_handler(self,client_v_major,client_v_minor):
803        v_handle = self._generate_version_hash(client_v_major,client_v_minor)
804        return v_handle
805
806# pkt_descriptor in packet model format (dictionary)
807    def build_pkt(self,client_v_handler,pkt_model_descriptor):
808        if not (self._verify_version_handler(client_v_handler)):
809            raise ScapyException(self._generate_invalid_version_error())
810        pkt = self._packet_model_to_scapy_packet(pkt_model_descriptor)
811        return self._pkt_data(pkt)
812
813
814    def build_pkt_ex(self, client_v_handler, pkt_model_descriptor, extra_options):
815        res = self.build_pkt(client_v_handler, pkt_model_descriptor)
816        pkt = self._packet_model_to_scapy_packet(pkt_model_descriptor)
817
818        field_engine = {}
819        field_engine['instructions'] = []
820        field_engine['error'] = None
821        try:
822            field_engine['instructions'] = self._generate_vm_instructions(pkt, extra_options['field_engine'])
823        except AssertionError as e:
824            field_engine['error'] = e.message
825        except CTRexPacketBuildException as e:
826            field_engine['error'] = e.message
827
828        field_engine['vm_instructions_expressions'] = self.field_engine_instruction_expressions
829        res['field_engine'] = field_engine
830        return res
831
832    def load_instruction_parameter_values(self, client_v_handler, pkt_model_descriptor, vm_instructions_model, parameter_id):
833
834        given_protocol_ids = [str(proto['id']) for proto in pkt_model_descriptor]
835
836        values = {}
837        if parameter_id == "name":
838            values = self._curent_pkt_protocol_fields(given_protocol_ids, "_")
839
840        if parameter_id == "fv_name":
841            values = self._existed_flow_var_names(vm_instructions_model['field_engine']['instructions'])
842
843        if parameter_id == "pkt_offset":
844            values = self._curent_pkt_protocol_fields(given_protocol_ids, ".")
845
846        if parameter_id == "offset":
847            for ip_idx in range(given_protocol_ids.count("IP")):
848                value = "IP:{0}".format(ip_idx)
849                values[value] = value
850
851        return {"map": values}
852
853    def _existed_flow_var_names(self, instructions):
854        return dict((instruction['parameters']['name'], instruction['parameters']['name']) for instruction in instructions if self._nameParamterExist(instruction))
855
856    def _nameParamterExist(self, instruction):
857        try:
858            instruction['parameters']['name']
859            return True
860        except KeyError:
861            return False
862
863    def _curent_pkt_protocol_fields(self, given_protocol_ids, delimiter):
864        given_protocol_classes = [c for c in Packet.__subclasses__() if c.__name__ in given_protocol_ids]
865        protocol_fields = {}
866        for protocol_class in given_protocol_classes:
867            protocol_name = protocol_class.__name__
868            protocol_count = given_protocol_ids.count(protocol_name)
869            for field_desc in protocol_class.fields_desc:
870                if delimiter == '.' and protocol_count > 1:
871                    for idx in range(protocol_count):
872                        formatted_name = "{0}:{1}{2}{3}".format(protocol_name, idx, delimiter, field_desc.name)
873                        protocol_fields[formatted_name] = formatted_name
874                else:
875                    formatted_name = "{0}{1}{2}".format(protocol_name, delimiter, field_desc.name)
876                protocol_fields[formatted_name] = formatted_name
877
878        return protocol_fields
879
880    def _generate_vm_instructions(self, pkt, field_engine_model_descriptor):
881        self.field_engine_instruction_expressions = []
882        instructions = []
883        instructions_def = field_engine_model_descriptor['instructions']
884        for instruction_def in instructions_def:
885            instruction_id = instruction_def['id']
886            instruction_class = self._vm_instructions[instruction_id]
887            parameters = {k: self._sanitize_value(k, v) for (k, v) in instruction_def['parameters'].items()}
888            instructions.append(instruction_class(**parameters))
889
890        fe_parameters = field_engine_model_descriptor['global_parameters']
891
892        cache_size = None
893        if "cache_size" in fe_parameters:
894            assert self._is_int(fe_parameters['cache_size']), 'Cache size must be a number'
895            cache_size = int(fe_parameters['cache_size'])
896
897
898        pkt_builder = STLPktBuilder(pkt=pkt, vm=STLScVmRaw(instructions, cache_size=cache_size))
899        pkt_builder.compile()
900        return pkt_builder.get_vm_data()
901
902    def _sanitize_value(self, param_id, val):
903        if param_id == "pkt_offset":
904            if self._is_int(val):
905                return int(val)
906            elif val == "Ether.src":
907                return 0
908            elif val == "Ether.dst":
909                return 6
910            elif val == "Ether.type":
911                return 12
912        else:
913            if val == "None" or val == "none":
914                return None
915            if val == "true":
916                return True
917            elif val == "false":
918                return False
919            elif re.match("[0-9a-f]{2}([-:])[0-9a-f]{2}(\\1[0-9a-f]{2}){4}$", str(val.lower())):
920                return int(str(val).replace(":", ""), 16)
921
922        if self._is_int(val):
923            return int(val)
924
925        str_val = str(val)
926        return int(str_val, 16) if str_val.startswith("0x") else str_val
927
928    def _get_instruction_parameter_meta(self, param_id):
929        for meta in self.instruction_parameter_meta_definitions:
930            if meta['id'] == param_id:
931                return meta
932        raise Scapy_Exception("Unable to get meta for {0}" % param_id)
933
934    def _is_int(self, val):
935        try:
936            int(val)
937            return True
938        except ValueError:
939            return False
940
941    # @deprecated. to be removed
942    def get_all(self,client_v_handler):
943        if not (self._verify_version_handler(client_v_handler)):
944            raise ScapyException(self._generate_invalid_version_error())
945        fields=self._get_all_fields()
946        db=self._get_all_db()
947        fields_md5 = self._get_md5(fields)
948        db_md5 = self._get_md5(db)
949        res = {}
950        res['db'] = db
951        res['fields'] = fields
952        res['db_md5'] = db_md5
953        res['fields_md5'] = fields_md5
954        return res
955
956    def _is_packet_class(self, pkt_class):
957        # returns true for final Packet classes. skips aliases and metaclasses
958        return issubclass(pkt_class, Packet) and pkt_class.name and pkt_class.fields_desc
959
960    def _getDummyPacket(self, pkt_class):
961        if issubclass(pkt_class, Raw):
962            # need to have some payload. otherwise won't appear in the binary chunk
963            return pkt_class(load=str_to_bytes("dummy"))
964        else:
965            return pkt_class()
966
967
968    def _get_payload_classes(self, pkt_class):
969        # tries to find, which subclasses allowed.
970        # this can take long time, since it tries to build packets with all subclasses(O(N))
971        allowed_subclasses = []
972        for pkt_subclass in conf.layers:
973            if self._is_packet_class(pkt_subclass):
974                try:
975                    pkt_w_payload = pkt_class() / self._getDummyPacket(pkt_subclass)
976                    recreated_pkt = pkt_class(bytes(pkt_w_payload))
977                    if type(recreated_pkt.lastlayer()) is pkt_subclass:
978                        allowed_subclasses.append(pkt_subclass)
979                except Exception as e:
980                    # no actions needed on fail, just sliently skip
981                    pass
982        return allowed_subclasses
983
984
985
986    def _get_templates(self):
987        templates = []
988        for root, subdirs, files in os.walk("templates"):
989            for file in files:
990                if not file.endswith('.trp'):
991                    continue
992                try:
993                    f = os.path.join(root, file)
994                    c = None
995                    with open(f, 'r') as templatefile:
996                        c = json.loads(templatefile.read())
997                    id = f.replace("templates" + os.path.sep, "", 1)
998                    id = id.split(os.path.sep)
999                    id[-1] = id[-1].replace(".trp", "", 1)
1000                    id = "/".join(id)
1001                    t = {
1002                            "id": id,
1003                             "meta": {
1004                                 "name": c["metadata"]["caption"],
1005                                 "description": ""
1006                             }
1007                        }
1008                    templates.append(t)
1009                except:
1010                    pass
1011        return templates
1012
1013    def _get_template(self,template):
1014        id = template["id"]
1015        f2 = "templates" + os.path.sep + os.path.sep.join(id.split("/")) + ".trp"
1016        for c in r'[]\;,><&*:%=+@!#^()|?^':
1017            id = id.replace(c,'')
1018        id = id.replace("..", "")
1019        id = id.split("/")
1020        f = "templates" + os.path.sep + os.path.sep.join(id) + ".trp"
1021        if f != f2:
1022            return ""
1023        with open(f, 'r') as content_file:
1024            content = base64.b64encode(content_file.read())
1025        return content
1026
1027
1028    def _get_fields_definition(self, pkt_class, fieldsDef):
1029        # fieldsDef - array of field definitions(or empty array)
1030        fields = []
1031        for field_desc in pkt_class.fields_desc:
1032            fieldId = field_desc.name
1033            field_data = {
1034                    "id": fieldId,
1035                    "name": field_desc.name
1036            }
1037            for fieldDef in fieldsDef:
1038                if fieldDef['id'] == fieldId:
1039                    field_data.update(fieldDef)
1040            if isinstance(field_desc, EnumField):
1041                try:
1042                    field_data["values_dict"] = field_desc.s2i
1043                    if field_data.get("type") == None:
1044                        if len(field_data["values_dict"] > 0):
1045                            field_data["type"] = "ENUM"
1046                        elif fieldId == 'load':
1047                            field_data["type"] = "BYTES"
1048                        else:
1049                            field_data["type"] = "STRING"
1050                    field_data["values_dict"] = field_desc.s2i
1051                except:
1052                    # MultiEnumField doesn't have s2i. need better handling
1053                    pass
1054            fields.append(field_data)
1055        return fields
1056
1057    def get_definitions(self,client_v_handler, def_filter):
1058        # def_filter is an array of classnames or None
1059        all_classes = Packet.__subclasses__() # as an alternative to conf.layers
1060        if def_filter:
1061            all_classes = [c for c in all_classes if c.__name__ in def_filter]
1062        protocols = []
1063        for pkt_class in all_classes:
1064            if self._is_packet_class(pkt_class):
1065                # enumerate all non-abstract Packet classes
1066                protocolId = pkt_class.__name__
1067                protoDef = self.protocol_definitions.get(protocolId) or {}
1068                protocols.append({
1069                    "id": protocolId,
1070                    "name": protoDef.get('name') or pkt_class.name,
1071                    "fields": self._get_fields_definition(pkt_class, protoDef.get('fields') or [])
1072                    })
1073        res = {"protocols": protocols,
1074               "feInstructionParameters": self.instruction_parameter_meta_definitions,
1075               "feInstructions": self.field_engine_instructions_meta,
1076               "feParameters": self.field_engine_parameter_meta_definitions,
1077               "feTemplates": self.field_engine_templates_definitions}
1078        return res
1079
1080    def get_payload_classes(self,client_v_handler, pkt_model_descriptor):
1081        pkt = self._packet_model_to_scapy_packet(pkt_model_descriptor)
1082        pkt_class = type(pkt.lastlayer())
1083        protocolDef = self.protocol_definitions.get(pkt_class.__name__)
1084        if protocolDef and protocolDef.get('payload'):
1085            return protocolDef['payload']
1086        return [c.__name__ for c in self._get_payload_classes(pkt_class)]
1087
1088    def get_templates(self,client_v_handler):
1089        return self._get_templates()
1090
1091    def get_template(self,client_v_handler,template):
1092        return self._get_template(template)
1093
1094#input in string encoded base64
1095    def check_update_of_dbs(self,client_v_handler,db_md5,field_md5):
1096        if not (self._verify_version_handler(client_v_handler)):
1097            raise ScapyException(self._generate_invalid_version_error())
1098        fields=self._get_all_fields()
1099        db=self._get_all_db()
1100        current_db_md5 = self._get_md5(db)
1101        current_field_md5 = self._get_md5(fields)
1102        res = []
1103        if (field_md5 == current_field_md5):
1104            if (db_md5 == current_db_md5):
1105                return True
1106            else:
1107                raise ScapyException("Protocol DB is not up to date")
1108        else:
1109            raise ScapyException("Fields DB is not up to date")
1110
1111    def _modify_layer(self, scapy_pkt, scapy_layer, fields):
1112        for field in fields:
1113            fieldId = str(field['id'])
1114            fieldval = self._field_value_from_def(scapy_pkt, scapy_layer, fieldId, field['value'])
1115            if fieldval is not None:
1116                scapy_layer.setfieldval(fieldId, fieldval)
1117            else:
1118                scapy_layer.delfieldval(fieldId)
1119
1120    def _is_last_layer(self, layer):
1121        # can be used, that layer has no payload
1122        # if true, the layer.payload is likely NoPayload()
1123        return layer is layer.lastlayer()
1124
1125#input of binary_pkt must be encoded in base64
1126    def reconstruct_pkt(self,client_v_handler,binary_pkt,model_descriptor):
1127        pkt_bin = b64_to_bytes(binary_pkt)
1128        scapy_pkt = Ether(pkt_bin)
1129        if not model_descriptor:
1130            model_descriptor = []
1131        for depth in range(len(model_descriptor)):
1132            model_layer = model_descriptor[depth]
1133            if model_layer.get('delete') is True:
1134                # slice packet from the current item
1135                if depth == 0:
1136                    scapy_pkt = None
1137                    break
1138                else:
1139                    scapy_pkt[depth-1].payload = None
1140                    break
1141            if depth > 0 and self._is_last_layer(scapy_pkt[depth-1]):
1142                # insert new layer(s) from json definition
1143                remaining_definitions = model_descriptor[depth:]
1144                pkt_to_append = self._packet_model_to_scapy_packet(remaining_definitions)
1145                scapy_pkt = scapy_pkt / pkt_to_append
1146                break
1147            # modify fields of existing stack items
1148            scapy_layer = scapy_pkt[depth]
1149            if model_layer['id'] != type(scapy_layer).__name__:
1150                # TODO: support replacing payload, instead of breaking
1151                raise ScapyException("Protocol id inconsistent")
1152            if 'fields' in model_layer:
1153                self._modify_layer(scapy_pkt, scapy_layer, model_layer['fields'])
1154        return self._pkt_data(scapy_pkt)
1155
1156    def read_pcap(self,client_v_handler,pcap_base64):
1157        pcap_bin = b64_to_bytes(pcap_base64)
1158        pcap = []
1159        res_packets = []
1160        with tempfile.NamedTemporaryFile(mode='w+b') as tmpPcap:
1161            tmpPcap.write(pcap_bin)
1162            tmpPcap.flush()
1163            pcap = rdpcap(tmpPcap.name)
1164        for scapy_packet in pcap:
1165            res_packets.append(self._pkt_data(scapy_packet))
1166        return res_packets
1167
1168    def write_pcap(self,client_v_handler,packets_base64):
1169        packets = [Ether(b64_to_bytes(pkt_b64)) for pkt_b64 in packets_base64]
1170        pcap_bin = None
1171        with tempfile.NamedTemporaryFile(mode='r+b') as tmpPcap:
1172            wrpcap(tmpPcap.name, packets)
1173            pcap_bin = tmpPcap.read()
1174        return bytes_to_b64(pcap_bin)
1175
1176
1177#---------------------------------------------------------------------------
1178
1179
1180