STRLCPY/protobuf-decoder

Lot of changes: - Lot of fixes - Added URL encoding/decoding after/before base64 encodind/decoding - Added recursive support for nested messages - Added recursive import if proto depends on other protos - Added python compilation for big protos - Added a filter to search for proto - Added an option in the contextual menu to decode with protoc - Last proto used in a tab is saved and set as default for that tab - Migrated from protobuf ParseMessage (deprecated) to MessageFactory method - Added a different default behaviour if no proto is selected, executing all the loaded protos (commented in the code)
federicodotta committed 3 years ago

def7d078

1 parent 7252cabc

Revision indexing in progress... (symbol navigation in revisions will be accurate after indexed)

■ ■ ■ ■ ■ ■

Lib/ui.py

		skipped 11 lines
12	12		from binascii import hexlify, unhexlify
13	13		from zlib import compress, decompress
14	14
	15	+	from urllib import unquote, quote_plus
	16	+
	17	+	def decode_url_and_base64(to_decode):
	18	+	return b64decode(unquote(to_decode).decode('utf8'))
	19	+
	20	+	def encode_base64_and_url(to_encode):
	21	+	return quote_plus(b64encode(to_encode))
15	22
16	23		PARAMETER_TYPES = {
17	24		'PARAM_BODY': IParameter.PARAM_BODY,
		skipped 12 lines
30	37		'zlib decompress': decompress,
31	38		'base64 encode': b64encode,
32	39		'base64 decode': b64decode,
	40	+	'base64 encode + URL encoding': encode_base64_and_url,
	41	+	'URL decode + base64 decode': decode_url_and_base64,
33	42		'url-base64 encode': urlsafe_b64encode,
34	43		'url-base64 decode': urlsafe_b64decode,
35	44		'hex encode': hexlify,
36	45		'hex decode': unhexlify,
37	46
38	47		}
39		-
40	48
41	49		class ParameterProcessingRulesTable(JPanel):
42	50		def __init__(self, extender=None, *rows):
		skipped 200 lines

■ ■ ■ ■ ■ ■

protoburp.py

		skipped 10 lines
11	11		import tempfile
12	12		import traceback
13	13		import StringIO
	14	+	import re
14	15		import gzip
15	16		import array
16	17		import platform
	18	+	from urllib import unquote, quote_plus
17	19
18	20		# Patch dir this file was loaded from into the path
19	21		# (Burp doesn't do it automatically)
		skipped 3 lines
23	25		from burp import IBurpExtender, IMessageEditorTab, IMessageEditorTabFactory, ITab, \
24	26		IExtensionStateListener
25	27
26		-	from google.protobuf.reflection import ParseMessage as parse_message
	28	+	# Deprecated, replaced by an implementation based on message_factory
	29	+	#from google.protobuf.reflection import ParseMessage as parse_message
	30	+
27	31		from google.protobuf.text_format import Merge as merge_message
	32	+	from google.protobuf import message_factory
28	33
29	34		from java.awt.event import ActionListener, MouseAdapter
30	35		from java.lang import Boolean, RuntimeException
31		-	from java.io import FileFilter
	36	+	from java.io import FileFilter, File
32	37		from javax.swing import JButton, JFileChooser, JMenu, JMenuItem, JOptionPane, JPanel, JPopupMenu
33	38		from javax.swing.filechooser import FileNameExtensionFilter
34	39		from java.lang import System
35	40
36	41		from ui import ParameterProcessingRulesTable
37		-
	42	+	from ui import decode_url_and_base64, encode_base64_and_url
38	43
39	44		CONTENT_PROTOBUF = ['application/protobuf', 'application/x-protobuf', 'application/x-protobuffer', 'application/x-protobuffer; charset=utf-8', 'application/octet-stream', 'application/grpc-web+proto']
40	45
41	46		PROTO_FILENAME_EXTENSION_FILTER = FileNameExtensionFilter(".proto, .py",
42	47		["proto", "py"])
43	48		CONTENT_GZIP = ('gzip')
	49	+
	50	+	PYTHON2_BINARY = 'python2'
44	51
45	52		def detectProtocBinaryLocation():
46	53		system = System.getProperty('os.name')
		skipped 134 lines
181	188		class ProtobufEditorTab(IMessageEditorTab):
182	189		TAB_CAPTION = "Protobuf Decoder"
183	190
	191	+
184	192		def __init__(self, extender, controller, editable):
185	193		self.extender = extender
186	194		self.callbacks = extender.callbacks
		skipped 11 lines
198	206		self.editor = extender.callbacks.createTextEditor()
199	207		self.editor.setEditable(editable)
200	208
	209	+	self.filter_search = None
	210	+
201	211		mouseListener = LoadProtoMenuMouseListener(self)
202	212		self.getUiComponent().addMouseListener(mouseListener)
	213	+
	214	+	self.last_proto = None
203	215
204	216		def getTabCaption(self):
205	217		return self.TAB_CAPTION
		skipped 6 lines
212	224		return False
213	225
214	226		if isRequest:
	227	+
	228	+	# Necessary sometimes when content-type is not set
	229	+	#return True
	230	+
215	231		info = self.helpers.analyzeRequest(content)
216	232
217	233		# check if request contains a specific parameter
		skipped 3 lines
221	237		return True
222	238
223	239		headers = info.getHeaders()
	240	+
224	241		else:
	242	+
	243	+	# Necessary sometimes when content-type is not set
	244	+	#return True
	245	+
225	246		headers = self.helpers.analyzeResponse(content).getHeaders()
226	247
227	248		# first header is the request/response line
		skipped 54 lines
282	303
283	304		break
284	305
285		-	#set message
286		-	rawBytes = (content[info.getBodyOffset():])
287		-	global oldPadding
288		-	global hasPadding
289		-	hasPadding = False
	306	+	if parameter is None:
	307	+	#set message
	308	+	rawBytes = (content[info.getBodyOffset():])
	309	+	global oldPadding
	310	+	global hasPadding
	311	+	hasPadding = False
	312	+
	313	+	oldPadding= rawBytes[0:4]
	314	+	if rawBytes[0] == 0 and rawBytes[1] == 0 and rawBytes[2] == 0 and rawBytes[3] == 0:
	315	+	rawBytes = rawBytes[5:rawBytes[4]+5]
	316	+	hasPadding = True
	317	+	body = rawBytes.tostring()
	318	+
	319	+
	320	+	# If we already selected a proto for this specific tab, continue to use that very proto
	321	+
	322	+	if(self.last_proto is not None):
	323	+
	324	+	factory = message_factory.MessageFactory()
	325	+	klass = factory.GetPrototype(self.descriptor)
	326	+	klass_instance = klass()
	327	+	klass_instance.ParseFromString(body)
290	328
291		-	oldPadding= rawBytes[0:4]
292		-	if rawBytes[0] == 0 and rawBytes[1] == 0 and rawBytes[2] == 0 and rawBytes[3] == 0:
293		-	rawBytes = rawBytes[5:rawBytes[4]+5]
294		-	hasPadding = True
295		-	body = rawBytes.tostring()
	329	+	message = klass_instance
296	330
297		-	# Loop through all proto descriptors loaded
	331	+	self.tab.editor.setText(str(klass_instance))
	332	+	self.tab.editor.setEditable(True)
	333	+	self.tab._current = (content, message, info, parameter)
	334	+	return
	335	+
	336	+	# 1 - Loop through all proto descriptors loaded and use the first that matches
	337	+	'''
298	338		for package, descriptors in self.descriptors.iteritems():
299	339		for name, descriptor in descriptors.iteritems():
300	340		try:
		skipped 22 lines
323	363		self.editor.setEditable(True)
324	364		self._current = (content, message, info, parameter)
325	365		return
	366	+	'''
326	367
327		-	# If we get to this point, then no loaded protos could deserialize
328		-	# the message. Shelling out to protoc should be a last resort.
	368	+	# 2 - This implementation prints the results of all the protos that matches
	369	+	'''
	370	+	content_pane = ""
	371	+
	372	+	# Loop through all proto descriptors loaded
	373	+	for package, descriptors in self.descriptors.iteritems():
	374	+	for name, descriptor in descriptors.iteritems():
	375	+	try:
	376	+	print "Parsing message with proto descriptor %s (auto)." % (name)
	377	+	message = parse_message(descriptor.Request, body)
	378	+	except Exception:
	379	+	print "(exception parsing message... - continue)"
	380	+	continue
	381	+
	382	+	# Stop parsing on the first valid message we encounter
	383	+	# this may result in a false positive, so we should still
	384	+	# allow users to specify a proto manually (select from a
	385	+	# context menu).
	386	+
	387	+	if message.IsInitialized():
	388	+	# The message is initialized if all of its
	389	+	# required fields are set.
	390	+	#print "Message: [%s]" % (message)
	391	+
	392	+	if str(message) == "":
	393	+	# parse_message() returned an empty message, but no
	394	+	# error or exception: continue to the next proto descriptor
	395	+	print "(message is empty, trying other proto descriptors...)"
	396	+	else:
329	397
	398	+	content_pane = content_pane + "***** " + str(name) + " - " + str(package) + "\n"
	399	+	content_pane = content_pane + str(message) + "\n"
	400	+
	401	+	if(content_pane != ""):
	402	+	self.editor.setText(str(content_pane))
	403	+	self.editor.setEditable(False)
	404	+	self._current = (content, content_pane, info, parameter)
	405	+	return
	406	+	'''
	407	+
	408	+	# 3 - This implementation (the one that I prefer) decodes without protos with protoc if no proto is selected
330	409		process = subprocess.Popen([PROTOC_BINARY_LOCATION, '--decode_raw'],
331	410		stdin=subprocess.PIPE,
332	411		stdout=subprocess.PIPE,
		skipped 31 lines
364	443
365	444		try:
366	445		merge_message(self.editor.getText().tostring(), message)
	446	+
367	447		headers = info.getHeaders()
368	448		serialized = message.SerializeToString()
369	449
370		-	if hasPadding:
	450	+	if parameter is None and hasPadding:
371	451		oldPadding.append(len(serialized))
372	452		serialized = oldPadding.tostring() + serialized
373	453
		skipped 12 lines
386	466
387	467		except Exception as error:
388	468		JOptionPane.showMessageDialog(self.getUiComponent(),
389		-	error.message, 'Error parsing message!',
	469	+	error.message + str(traceback.format_exc()), 'Error parsing message!',
390	470		JOptionPane.ERROR_MESSAGE)
391	471
392	472		# an error occurred while re-serializing the message,
		skipped 21 lines
414	494		def mouseReleased(self, event):
415	495		return self.handleMouseEvent(event)
416	496
	497	+	# Recursive method necessary to add also subtimes (a message can have inside other message definitions...)
	498	+	def populate_menu_recursive(self, descriptors, fatherMenu, father_match):
	499	+
	500	+	iter_found = False
	501	+
	502	+	single_father_match = father_match
	503	+	total_father_match = father_match
	504	+
	505	+	# if self.tab.filter_search is None:
	506	+
	507	+	for name, descriptor in descriptors.iteritems():
	508	+
	509	+	current_nested_dict = descriptor.nested_types_by_name
	510	+
	511	+	if self.tab.filter_search is not None:
	512	+	single_father_match = father_match or re.search(self.tab.filter_search, name, re.IGNORECASE)
	513	+	total_father_match = total_father_match or re.search(self.tab.filter_search, name, re.IGNORECASE)
	514	+
	515	+	if(len(current_nested_dict) > 0):
	516	+
	517	+	protoMenu = JMenu(name)
	518	+
	519	+	# The father is the messsage that encloses other messages
	520	+	enclosureOBject = JMenuItem("* Father")
	521	+	enclosureOBject.addActionListener(DeserializeProtoActionListener(self.tab, descriptor))
	522	+	protoMenu.add(enclosureOBject)
	523	+
	524	+	if self.populate_menu_recursive(current_nested_dict, protoMenu, single_father_match) or self.tab.filter_search is None:
	525	+	fatherMenu.add(protoMenu)
	526	+	iter_found = True
	527	+
	528	+	else:
	529	+
	530	+	if self.tab.filter_search is None or re.search(self.tab.filter_search, name, re.IGNORECASE) or single_father_match:
	531	+
	532	+	protoMenu = JMenuItem(name)
	533	+	protoMenu.addActionListener(DeserializeProtoActionListener(self.tab, descriptor))
	534	+	fatherMenu.add(protoMenu)
	535	+
	536	+	iter_found = True
	537	+
	538	+	return iter_found or total_father_match
	539	+
	540	+
417	541		def handleMouseEvent(self, event):
418	542		if event.isPopupTrigger():
419	543		loadMenu = JMenuItem("Load .proto")
		skipped 1 lines
421	545
422	546		popup = JPopupMenu()
423	547		popup.add(loadMenu)
	548	+
	549	+	filterMenu = JMenuItem("Filter .proto")
	550	+	filterMenu.addActionListener(SearchProtoActionListener(self.tab, event.getComponent()))
	551	+	popup.add(filterMenu)
424	552
425	553		if self.tab.descriptors:
426	554
		skipped 1 lines
428	556
429	557		popup.addSeparator()
430	558		popup.add(deserializeAsMenu)
	559	+
	560	+	# Raw deserialize using protoc without proto (it cannot be serialized if modified)
	561	+	rawMenu = JMenuItem("Raw")
	562	+	deserializeAsMenu.add(rawMenu)
	563	+	rawMenu.addActionListener(DeserializeProtoActionListener(self.tab, "raw"))
431	564
432	565		for pb2, descriptors in self.tab.descriptors.iteritems():
433		-	subMenu = JMenu(pb2)
434		-	deserializeAsMenu.add(subMenu)
435	566
436		-	for name, descriptor in descriptors.iteritems():
437		-	protoMenu = JMenuItem(name)
438		-	protoMenu.addActionListener(
439		-	DeserializeProtoActionListener(self.tab, descriptor))
	567	+	subMenu = JMenu(pb2)
440	568
441		-	subMenu.add(protoMenu)
	569	+	if self.populate_menu_recursive(descriptors, subMenu, self.tab.filter_search is None):
	570	+	deserializeAsMenu.add(subMenu)
442	571
443	572		popup.show(event.getComponent(), event.getX(), event.getY())
444	573
		skipped 24 lines
469	598		self.updateDescriptors(name, module_)
470	599
471	600		return
	601	+
	602	+	# Method created to handle the situation in which a proto depends on another proto. In this situation the dependencies are imported recursively.
	603	+	def importProtoFileRecusive(self, proto):
	604	+
	605	+	try:
	606	+	module = compile_and_import_proto(proto)
	607	+	if module:
	608	+	return module
	609	+
	610	+	except (Exception, RuntimeException) as error:
	611	+
	612	+	missing_module_regex = re.search('No module named (.*)_pb2', str(error))
	613	+	compilation_regex = re.search('.Module or method too large in `(.)`.*', str(error)) #
	614	+
	615	+	if(missing_module_regex):
	616	+	missing_module = missing_module_regex.group(1) + ".proto"
	617	+	self.tab.callbacks.getStdout().write('*** %s depends on %s. Trying to import it...!\n' % (proto, missing_module, ))
	618	+	if self.importProtoFileRecusive(File(proto.getParent(),missing_module)):
	619	+	return self.importProtoFileRecusive(proto)
	620	+	else:
	621	+	self.tab.callbacks.getStderr().write('*** ERROR, infinite recursion with proto %s!\n' % (str(proto.getAbsolutePath()), ))
	622	+
	623	+	# This compile with python pb2 too large
	624	+	elif(compilation_regex):
	625	+	subprocess.check_call([PYTHON2_BINARY, '-m', 'py_compile', compilation_regex.group(1)])
	626	+	return self.importProtoFileRecusive(proto)
	627	+
	628	+	else:
	629	+	self.tab.callbacks.getStderr().write('*** ERROR in recursive import: %s!\n' % (str(error), ))
	630	+	tb = traceback.format_exc()
	631	+	self.tab.callbacks.getStderr().write('Traceback: %s!\n' % (str(tb), ))
	632	+	return None
472	633
473	634		def importProtoFiles(self, selectedFiles):
474	635		for selectedFile in selectedFiles:
		skipped 2 lines
477	638		self.importProtoFiles(selectedFile.listFiles(ListProtoFileFilter()))
478	639		else:
479	640		self.chooser.setCurrentDirectory(selectedFile.getParentFile())
480		-
481		-	try:
482		-	module = compile_and_import_proto(selectedFile)
483		-	if module:
484		-	yield module
	641	+	yield self.importProtoFileRecusive(selectedFile)
485	642
486		-	except (Exception, RuntimeException) as error:
487		-	self.tab.callbacks.getStderr().write(
488		-	'Error importing proto %s!\n' % (selectedFile, ))
489		-
490		-	traceback.print_exc(file=self.tab.callbacks.getStderr())
491		-
492		-	JOptionPane.showMessageDialog(None,
493		-	'%s: %s' % (error.message, selectedFile),
494		-	'Error importing proto!', JOptionPane.ERROR_MESSAGE)
495	643
496	644		def actionPerformed(self, event):
497	645		if self.chooser.showOpenDialog(None) == JFileChooser.APPROVE_OPTION:
		skipped 3 lines
501	649		return
502	650
503	651
	652	+	# The purpose is being able to search for protos, if we have tons of proto
	653	+	class SearchProtoActionListener(ActionListener):
	654	+	def __init__(self, tab, component):
	655	+	self.tab = tab
	656	+	self.component = component
	657	+
	658	+	def actionPerformed(self, event):
	659	+	self.tab.filter_search = JOptionPane.showInputDialog(self.component, "Search: ", "Search", 1);
	660	+
	661	+
504	662		class DeserializeProtoActionListener(ActionListener):
505	663		def __init__(self, tab, descriptor):
506	664		self.tab = tab
		skipped 11 lines
518	676		body = gUnzip(content[info.getBodyOffset():].tostring())
519	677		else:
520	678		body = content[info.getBodyOffset():].tostring()
521		-
	679	+	#body = content[info.getBodyOffset():]
522	680
523	681		if parameter is not None:
524	682		param = self.tab.helpers.getRequestParameter(
		skipped 6 lines
531	689		for rule in rules.get('before', []):
532	690		body = rule(body)
533	691
	692	+	if parameter is None:
534	693
	694	+	# cut 5 bytes for grpc web
	695	+	rawBytes = (content[info.getBodyOffset():])
	696	+	global oldPadding
	697	+	global hasPadding
	698	+	hasPadding = False
535	699
536		-	# cut 5 bytes for grpc web
537		-	rawBytes = (content[info.getBodyOffset():])
538		-	global oldPadding
539		-	global hasPadding
540		-	hasPadding = False
	700	+	oldPadding= rawBytes[0:4]
	701	+	if rawBytes[0] == 0 and rawBytes[1] == 0 and rawBytes[2] == 0 and rawBytes[3] == 0:
	702	+	rawBytes = rawBytes[5:]
	703	+	hasPadding = True
541	704
542		-	oldPadding= rawBytes[0:4]
543		-	if rawBytes[0] == 0 and rawBytes[1] == 0 and rawBytes[2] == 0 and rawBytes[3] == 0:
544		-	rawBytes = rawBytes[5:]
545		-	hasPadding = True
	705	+	body = rawBytes.tostring()
	706	+	#body = rawBytes
546	707
547		-	body = rawBytes.tostring()
	708	+	if self.descriptor != "raw":
548	709
	710	+	print "Parsing message with proto descriptor %s (by user)." % (self.descriptor.name)
549	711
550		-	print "Parsing message with proto descriptor %s (by user)." % (self.descriptor.name)
551		-	message = parse_message(self.descriptor, body)
	712	+	# Deprecated method
	713	+	#message = parse_message(self.descriptor, body)
552	714
553		-	self.tab.editor.setText(str(message))
554		-	self.tab.editor.setEditable(True)
555		-	self.tab._current = (content, message, info, parameter)
	715	+	factory = message_factory.MessageFactory()
	716	+	klass = factory.GetPrototype(self.descriptor)
	717	+	klass_instance = klass()
	718	+	klass_instance.ParseFromString(body)
	719	+
	720	+	message = klass_instance
	721	+
	722	+	self.tab.editor.setText(str(klass_instance))
	723	+	self.tab.editor.setEditable(True)
	724	+	self.tab._current = (content, message, info, parameter)
	725	+	self.tab.last_proto = self.descriptor
	726	+
	727	+	else:
	728	+
	729	+	print "Parsing message without any proto"
	730	+
	731	+	process = subprocess.Popen([PROTOC_BINARY_LOCATION, '--decode_raw'],
	732	+	stdin=subprocess.PIPE,
	733	+	stdout=subprocess.PIPE,
	734	+	stderr=subprocess.PIPE)
	735	+
	736	+
	737	+	output = error = None
	738	+	try:
	739	+	output, error = process.communicate(body)
	740	+	except OSError:
	741	+	pass
	742	+	finally:
	743	+	if process.poll() != 0:
	744	+	process.wait()
	745	+
	746	+	if error:
	747	+	#print "protoc displaying message - error..."
	748	+	self.tab.editor.setText(error)
	749	+	else:
	750	+	#print "protoc displaying message - output..."
	751	+	self.tab.editor.setText(output)
	752	+
	753	+	self.tab.editor.setEditable(False)
	754	+	self.tab._current = (content, None, info, parameter)
556	755
557	756		except Exception as error:
558		-	title = "Error parsing message as %s!" % (self.descriptor.name, )
	757	+
	758	+	if self.descriptor != "raw":
	759	+	title = "Error parsing message as %s!" % (self.descriptor.name, )
	760	+	else:
	761	+	title = "Error parsing message as without any proto"
559	762		JOptionPane.showMessageDialog(self.tab.getUiComponent(),
560		-	error.message, title, JOptionPane.ERROR_MESSAGE)
	763	+	error.message + str(traceback.format_exc()), title, JOptionPane.ERROR_MESSAGE)
	764	+
	765	+	#print(str(error))
	766	+	#print(str(error.message))
	767	+	#print(str(traceback.format_exc()))
561	768
562	769		return
563	770
		skipped 2 lines
566	773		curdir = os.path.abspath(os.curdir)
567	774		tempdir = tempfile.mkdtemp()
568	775
	776	+	print(str(tempdir))
	777	+
569	778		is_proto = os.path.splitext(proto.getName())[-1] == '.proto'
570	779
571	780		if is_proto:
		skipped 3 lines
575	784		tempdir, proto.getName()])
576	785		module = proto.getName().replace('.proto', '_pb2')
577	786
578		-	except subprocess.CalledProcessError:
	787	+	except subprocess.CalledProcessError as e:
	788	+	print("*** ERROR COMPILING")
	789	+	print(e)
579	790		shutil.rmtree(tempdir)
580	791		return None
581	792
		skipped 10 lines
592	803		os.chdir(proto.getParent())
593	804
594	805		sys.path.append(os.path.abspath(os.curdir))
	806	+
	807	+	# Added compilation in order to avoid error with big proto files
	808	+	subprocess.check_call([PYTHON2_BINARY, '-m', 'py_compile', str(module) + ".py"])
	809	+
595	810		return importlib.import_module(module)
	811	+
	812	+	except subprocess.CalledProcessError as e:
	813	+	#except Exception as e:
	814	+	print("*** ERROR COMPILING")
	815	+	print(e)
	816	+	print(str(traceback.format_exc()))
596	817
597	818		finally:
598	819		sys.path.pop()
		skipped 4 lines