STRLCPY/metabigor

Improve nmap parser
j3ssie committed 4 years ago

b9f77b26

1 parent 1f878e19

■ ■ ■ ■ ■ ■ ■

cmd/root.go

		skipped 32 lines
33	33
34	34		func init() {
35	35		cobra.OnInitialize(initConfig)
36		-	RootCmd.PersistentFlags().StringVar(&options.Scan.TmpOutput, "tmp", "", "Temp Output folder")
	36	+	RootCmd.PersistentFlags().StringVarP(&options.Scan.TmpOutput, "tmp", "T", "", "Temp Output folder")
37	37		RootCmd.PersistentFlags().StringVar(&options.Proxy, "proxy", "", "Proxy for doing request")
38	38		RootCmd.PersistentFlags().IntVarP(&options.Concurrency, "concurrency", "c", 5, "concurrency")
39	39		RootCmd.PersistentFlags().IntVar(&options.Timeout, "timeout", 40, "timeout")
40	40		RootCmd.PersistentFlags().StringVarP(&options.Input, "input", "i", "-", "input as a string, file or from stdin")
41	41		RootCmd.PersistentFlags().StringVarP(&options.Output, "output", "o", "out.txt", "output name")
42	42		RootCmd.PersistentFlags().BoolVar(&options.Debug, "debug", false, "Debug")
	43	+	RootCmd.PersistentFlags().BoolVarP(&options.JsonOutput, "json", "j", false, "Output as JSON")
43	44		RootCmd.PersistentFlags().BoolVarP(&options.Verbose, "verbose", "v", false, "Verbose")
	45	+	RootCmd.SetHelpFunc(RootMessage)
44	46		}
45	47
46	48		// initConfig reads in config file and ENV variables if set.
		skipped 33 lines
80	82		core.InforF(fmt.Sprintf("Store log file to: %v", options.LogFile))
81	83		}
82	84
	85	+	// RootMessage print help message
	86	+	func RootMessage(cmd *cobra.Command, _ []string) {
	87	+	fmt.Printf(cmd.UsageString())
	88	+	h := `
	89	+	Examples Commands:
	90	+	# discovery IP of a company/organization
	91	+	echo "company" \| metabigor net --org -o /tmp/result.txt
	92	+
	93	+	# discovery IP of an ASN
	94	+	echo "ASN1111" \| metabigor net --asn -o /tmp/result.txt
	95	+	cat list_of_ASNs \| metabigor net --asn -o /tmp/result.txt
	96	+
	97	+	# Only run masscan full ports
	98	+	echo '1.2.3.4/24' \| metabigor scan -o result.txt
	99	+
	100	+	# Only run nmap detail scan
	101	+	echo '1.2.3.4:21' \| metabigor scan -s -c 10
	102	+	echo '1.2.3.4:21' \| metabigor scan --tmp /tmp/raw-result/ -s -o result.txt
	103	+
	104	+	# Only run scan with zmap
	105	+	cat ranges.txt \| metabigor scan -p '443,80' -z
	106	+
	107	+	# search result on fofa
	108	+	echo 'title="RabbitMQ Management"' \| metabigor search -x -v -o /tmp/result.txt
	109	+
	110	+	# search IP on shodan
	111	+	echo '1.2.3.4' \| metabigor ip -s 'shodan' -v
	112	+	`
	113	+	fmt.Printf(h)
	114	+	}
	115	+

■ ■ ■ ■ ■ ■

cmd/scan.go

		skipped 1 lines
2	2
3	3		import (
4	4		"fmt"
	5	+	"github.com/j3ssie/osmedeus/utils"
5	6		"io/ioutil"
6	7		"os"
7	8		"strings"
		skipped 16 lines
24	25		scanCmd.Flags().StringP("rate", "r", "5000", "rate limit for masscan command")
25	26		scanCmd.Flags().BoolP("detail", "D", false, "Do Nmap scan based on previous output")
26	27
27		-	scanCmd.Flags().BoolP("flat", "f", false, "format output like this: 1.2.3.4:443")
	28	+	scanCmd.Flags().BoolP("flat", "f", true, "format output like this: 1.2.3.4:443")
	29	+	scanCmd.Flags().BoolP("nmap", "n", false, "Use nmap instead of masscan for overview scan")
	30	+	scanCmd.Flags().BoolP("zmap", "z", false, "Only scan range with zmap")
	31	+
28	32		scanCmd.Flags().BoolP("skip-masscan", "s", false, "run nmap from input format like this: 1.2.3.4:443")
29	33		scanCmd.Flags().StringP("script", "S", "", "nmap scripts")
30	34		scanCmd.Flags().StringP("grep", "g", "", "match string to confirm script success")
31	35		// only parse scan
32	36		scanCmd.Flags().StringP("result-folder", "R", "", "Result folder")
33		-
	37	+	scanCmd.SetHelpFunc(ScanHelp)
34	38		RootCmd.AddCommand(scanCmd)
35	39
36	40		}
		skipped 5 lines
42	46		options.Scan.Rate, _ = cmd.Flags().GetString("rate")
43	47		options.Scan.Detail, _ = cmd.Flags().GetBool("detail")
44	48		options.Scan.Flat, _ = cmd.Flags().GetBool("flat")
	49	+	options.Scan.NmapOverview, _ = cmd.Flags().GetBool("nmap")
	50	+	options.Scan.ZmapOverview, _ = cmd.Flags().GetBool("zmap")
45	51		options.Scan.SkipOverview, _ = cmd.Flags().GetBool("skip-masscan")
46	52		// only parse result
47	53		resultFolder, _ := cmd.Flags().GetString("result-folder")
		skipped 19 lines
67	73		var wg sync.WaitGroup
68	74		jobs := make(chan string)
69	75
	76	+	if options.Scan.ZmapOverview {
	77	+	inputFile := StoreTmpInput(inputs, options)
	78	+	ports := core.GenPorts(options.Scan.Ports)
	79	+	utils.DebugF("Store temp input in: %v", inputFile)
	80	+	utils.DebugF("Run port scan with: %v", ports)
	81	+	if inputFile == "" \|\| len(ports) == 0 {
	82	+	core.ErrorF("Error gen input or ports")
	83	+	return nil
	84	+	}
	85	+	for i := 0; i < options.Concurrency; i++ {
	86	+	wg.Add(1)
	87	+	go func() {
	88	+	defer wg.Done()
	89	+	for job := range jobs {
	90	+	// do real stuff here
	91	+	core.BannerF("Run zmap scan on port ", job)
	92	+	result = modules.RunZmap(inputFile, job, options)
	93	+	StoreData(result, options)
	94	+	}
	95	+	}()
	96	+	}
	97	+	for _, port := range ports {
	98	+	jobs <- port
	99	+	}
	100	+	close(jobs)
	101	+	wg.Wait()
	102	+	return nil
	103	+	}
	104	+
70	105		for i := 0; i < options.Concurrency; i++ {
71	106		wg.Add(1)
72	107		go func() {
		skipped 21 lines
94	129		}
95	130
96	131		func runRoutine(input string, options core.Options) []string {
97		-	core.BannerF("Run quick scan on: ", input)
	132	+	core.BannerF("Run overview scan on: ", input)
98	133		var data []string
99		-	data = append(data, modules.RunMasscan(input, options)...)
	134	+
	135	+	if options.Scan.NmapOverview {
	136	+	data = append(data, modules.RunNmap(input, "", options)...)
	137	+	} else {
	138	+	data = append(data, modules.RunMasscan(input, options)...)
	139	+	}
	140	+
100	141		if !options.Scan.Detail {
101	142		return data
102	143		}
		skipped 24 lines
127	168
128	169		host := strings.Split(input, " - ")[0]
129	170		ports := strings.Split(input, " - ")[1]
130		-	core.BannerF("Run detail scan on: ", fmt.Sprintf("%v %v", host, ports))
	171	+	core.BannerF("Run detail scan on: ", fmt.Sprintf("%v:%v", host, ports))
131	172		return modules.RunNmap(host, ports, options)
132	173		}
133	174
		skipped 6 lines
140	181		}
141	182		host := strings.Split(input, ":")[0]
142	183		ports := strings.Split(input, ":")[1]
143		-	core.BannerF("Run detail scan on: ", fmt.Sprintf("%v %v", host, ports))
	184	+	core.BannerF("Run detail scan on: ", fmt.Sprintf("%v:%v", host, ports))
144	185		return modules.RunNmap(host, ports, options)
145	186		}
146	187
		skipped 16 lines
163	204		core.DebugF("Reading: %v", filename)
164	205		if strings.HasSuffix(file.Name(), "xml") && strings.HasPrefix(filename, "nmap") {
165	206		data := core.GetFileContent(filename)
166		-	rawResult := modules.ParsingNmap(data, options)
167		-	for k, v := range rawResult {
168		-	fmt.Printf("%v - %v\n", k, strings.Join(v, ","))
	207	+	result := modules.ParseNmap(data, options)
	208	+	if len(result) > 0 {
	209	+	fmt.Printf(strings.Join(result, "\n"))
169	210		}
170	211		}
171	212		}
		skipped 18 lines
190	231		}
191	232		}
192	233
	234	+	// StoreTmpInput store list of string to tmp file
	235	+	func StoreTmpInput(raw []string, options core.Options) string {
	236	+	tmpDest := options.Scan.TmpOutput
	237	+	tmpFile, _ := ioutil.TempFile(options.Scan.TmpOutput, "zmap-*.txt")
	238	+	if tmpDest != "" {
	239	+	tmpFile, _ = ioutil.TempFile(tmpDest, "zmap-input-*.txt")
	240	+	}
	241	+	tmpDest = tmpFile.Name()
	242	+	core.WriteToFile(tmpDest, strings.Join(raw, "\n"))
	243	+	return tmpDest
	244	+	}
	245	+
	246	+	// ScanHelp print help message
	247	+	func ScanHelp(cmd *cobra.Command, _ []string) {
	248	+	fmt.Println(cmd.UsageString())
	249	+	h := "\nExample Commands:\n"
	250	+	h += " # Only run masscan full ports\n"
	251	+	h += " echo '1.2.3.4/24' \| metabigor scan -o result.txt\n\n"
	252	+	h += " # Only run nmap detail scan\n"
	253	+	h += " echo '1.2.3.4:21' \| metabigor scan -s -c 10\n"
	254	+	h += " echo '1.2.3.4:21' \| metabigor scan --tmp /tmp/raw-result/ -s -o result.txt\n\n"
	255	+	h += " # Only run scan with zmap \n"
	256	+	h += " cat ranges.txt \| metabigor scan -p '443,80' -z\n"
	257	+	h += "\n"
	258	+	fmt.Printf(h)
	259	+	}
	260	+

■ ■ ■ ■ ■ ■

core/common.go

		skipped 28 lines
29	29		logger.Error("error opening file: %v", err)
30	30		}
31	31
32		-	mwr := io.MultiWriter(os.Stdout, f)
	32	+	mwr := io.MultiWriter(os.Stderr, f)
33	33
34	34		logger.SetLevel(logrus.InfoLevel)
35		-
36	35		logger = &logrus.Logger{
37	36		Out: mwr,
38	37		Level: logrus.InfoLevel,
		skipped 17 lines
56	55		// GoodF print good message
57	56		func GoodF(format string, args ...interface{}) {
58	57		good := color.HiGreenString("[+]")
59		-	fmt.Printf("%s %s\n", good, fmt.Sprintf(format, args...))
	58	+	fmt.Fprintf(os.Stderr, "%s %s\n", good, fmt.Sprintf(format, args...))
60	59		}
61	60
62	61		// BannerF print info message
		skipped 10 lines
73	72		// WarningF print good message
74	73		func WarningF(format string, args ...interface{}) {
75	74		good := color.YellowString("[!]")
76		-	fmt.Printf("%s %s\n", good, fmt.Sprintf(format, args...))
	75	+	fmt.Fprintf(os.Stderr, "%s %s\n", good, fmt.Sprintf(format, args...))
77	76		}
78	77
79	78		// DebugF print debug message
		skipped 4 lines
84	83		// ErrorF print good message
85	84		func ErrorF(format string, args ...interface{}) {
86	85		good := color.RedString("[-]")
87		-	fmt.Printf("%s %s\n", good, fmt.Sprintf(format, args...))
	86	+	fmt.Fprintf(os.Stderr, "%s %s\n", good, fmt.Sprintf(format, args...))
88	87		}
89	88

■ ■ ■ ■ ■ ■

core/helper.go

		skipped 303 lines
304	304		return decodedValue
305	305		}
306	306
	307	+	// GenPorts gen list of ports based on input
	308	+	func GenPorts(raw string) []string {
	309	+	var ports []string
	310	+	if strings.Contains(raw, ",") {
	311	+	items := strings.Split(raw, ",")
	312	+	for _, item := range items {
	313	+	if strings.Contains(item, "-") {
	314	+	min, err := strconv.Atoi(strings.Split(item, "-")[0])
	315	+	if err != nil {
	316	+	continue
	317	+	}
	318	+	max, err := strconv.Atoi(strings.Split(item, "-")[1])
	319	+	if err != nil {
	320	+	continue
	321	+	}
	322	+	for i := min; i <= max; i++ {
	323	+	ports = append(ports, fmt.Sprintf("%v", i))
	324	+	}
	325	+	} else {
	326	+	ports = append(ports, item)
	327	+	}
	328	+	}
	329	+	} else {
	330	+	if strings.Contains(raw, "-") {
	331	+	min, err := strconv.Atoi(strings.Split(raw, "-")[0])
	332	+	if err != nil {
	333	+	return ports
	334	+	}
	335	+	max, err := strconv.Atoi(strings.Split(raw, "-")[1])
	336	+	if err != nil {
	337	+	return ports
	338	+	}
	339	+	for i := min; i <= max; i++ {
	340	+	ports = append(ports, fmt.Sprintf("%v", i))
	341	+	}
	342	+	} else {
	343	+	ports = append(ports, raw)
	344	+	}
	345	+	}
	346	+
	347	+	return ports
	348	+	}
	349	+

■ ■ ■ ■ ■ ■

core/options.go

		skipped 12 lines
13	13		Delay int
14	14		SaveRaw bool
15	15		Timeout int
	16	+	JsonOutput bool
16	17		Verbose bool
17	18		Debug bool
18	19		Scan ScanOptions
		skipped 5 lines
24	25		type ScanOptions struct {
25	26		Ports string
26	27		Rate string
	28	+	NmapOverview bool
	29	+	ZmapOverview bool
27	30		Detail bool
28	31		Flat bool
29	32		SkipOverview bool
		skipped 50 lines

■ ■ ■ ■ ■ ■

core/version.go

		skipped 1 lines
2	2
3	3		const (
4	4		// VERSION current Metabigor version
5		-	VERSION = "beta v1.5"
	5	+	VERSION = "beta v1.6"
6	6		// AUTHOR author of this
7	7		AUTHOR = "@j3ssiejjj"
8	8		)
		skipped 1 lines

■ ■ ■ ■ ■ ■

modules/nmap.go

1	+	package modules
2	+
3	+	import (
4	+	"encoding/xml"
5	+	"github.com/PuerkitoBio/goquery"
6	+	"github.com/j3ssie/metabigor/core"
7	+	"regexp"
8	+	"strings"
9	+	)
10	+
11	+	// NmapRuns nmap multiple scan XML to struct
12	+	type NmapRuns struct {
13	+	XMLName xml.Name `xml:"nmaprun"`
14	+	Text string `xml:",chardata"`
15	+	Scanner string `xml:"scanner,attr"`
16	+	Args string `xml:"args,attr"`
17	+	Start string `xml:"start,attr"`
18	+	Startstr string `xml:"startstr,attr"`
19	+	Version string `xml:"version,attr"`
20	+	Xmloutputversion string `xml:"xmloutputversion,attr"`
21	+	Scaninfo struct {
22	+	Text string `xml:",chardata"`
23	+	Type string `xml:"type,attr"`
24	+	Protocol string `xml:"protocol,attr"`
25	+	Numservices string `xml:"numservices,attr"`
26	+	Services string `xml:"services,attr"`
27	+	} `xml:"scaninfo"`
28	+	Verbose struct {
29	+	Text string `xml:",chardata"`
30	+	Level string `xml:"level,attr"`
31	+	} `xml:"verbose"`
32	+	Debugging struct {
33	+	Text string `xml:",chardata"`
34	+	Level string `xml:"level,attr"`
35	+	} `xml:"debugging"`
36	+	Taskbegin []struct {
37	+	Text string `xml:",chardata"`
38	+	Task string `xml:"task,attr"`
39	+	Time string `xml:"time,attr"`
40	+	} `xml:"taskbegin"`
41	+	Taskend []struct {
42	+	Text string `xml:",chardata"`
43	+	Task string `xml:"task,attr"`
44	+	Time string `xml:"time,attr"`
45	+	Extrainfo string `xml:"extrainfo,attr"`
46	+	} `xml:"taskend"`
47	+	Host []struct {
48	+	Text string `xml:",chardata"`
49	+	Starttime string `xml:"starttime,attr"`
50	+	Endtime string `xml:"endtime,attr"`
51	+	Status struct {
52	+	Text string `xml:",chardata"`
53	+	State string `xml:"state,attr"`
54	+	Reason string `xml:"reason,attr"`
55	+	ReasonTtl string `xml:"reason_ttl,attr"`
56	+	} `xml:"status"`
57	+	Address struct {
58	+	Text string `xml:",chardata"`
59	+	Addr string `xml:"addr,attr"`
60	+	Addrtype string `xml:"addrtype,attr"`
61	+	} `xml:"address"`
62	+	Hostnames struct {
63	+	Text string `xml:",chardata"`
64	+	Hostname struct {
65	+	Text string `xml:",chardata"`
66	+	Name string `xml:"name,attr"`
67	+	Type string `xml:"type,attr"`
68	+	} `xml:"hostname"`
69	+	} `xml:"hostnames"`
70	+	Ports struct {
71	+	Text string `xml:",chardata"`
72	+	Extraports struct {
73	+	Text string `xml:",chardata"`
74	+	State string `xml:"state,attr"`
75	+	Count string `xml:"count,attr"`
76	+	Extrareasons []struct {
77	+	Text string `xml:",chardata"`
78	+	Reason string `xml:"reason,attr"`
79	+	Count string `xml:"count,attr"`
80	+	} `xml:"extrareasons"`
81	+	} `xml:"extraports"`
82	+	Port []struct {
83	+	Text string `xml:",chardata"`
84	+	Protocol string `xml:"protocol,attr"`
85	+	Portid string `xml:"portid,attr"`
86	+	State struct {
87	+	Text string `xml:",chardata"`
88	+	State string `xml:"state,attr"`
89	+	Reason string `xml:"reason,attr"`
90	+	ReasonTtl string `xml:"reason_ttl,attr"`
91	+	} `xml:"state"`
92	+	Service struct {
93	+	Text string `xml:",chardata"`
94	+	Name string `xml:"name,attr"`
95	+	Tunnel string `xml:"tunnel,attr"`
96	+	Method string `xml:"method,attr"`
97	+	Conf string `xml:"conf,attr"`
98	+	Product string `xml:"product,attr"`
99	+	Devicetype string `xml:"devicetype,attr"`
100	+	Servicefp string `xml:"servicefp,attr"`
101	+	Cpe string `xml:"cpe"`
102	+	} `xml:"service"`
103	+	Script struct {
104	+	Text string `xml:",chardata"`
105	+	ID string `xml:"id,attr"`
106	+	Output string `xml:"output,attr"`
107	+	Elem []struct {
108	+	Text string `xml:",chardata"`
109	+	Key string `xml:"key,attr"`
110	+	} `xml:"elem"`
111	+	} `xml:"script"`
112	+	} `xml:"port"`
113	+	} `xml:"ports"`
114	+	Times struct {
115	+	Text string `xml:",chardata"`
116	+	Srtt string `xml:"srtt,attr"`
117	+	Rttvar string `xml:"rttvar,attr"`
118	+	To string `xml:"to,attr"`
119	+	} `xml:"times"`
120	+	} `xml:"host"`
121	+	Taskprogress []struct {
122	+	Text string `xml:",chardata"`
123	+	Task string `xml:"task,attr"`
124	+	Time string `xml:"time,attr"`
125	+	Percent string `xml:"percent,attr"`
126	+	Remaining string `xml:"remaining,attr"`
127	+	Etc string `xml:"etc,attr"`
128	+	} `xml:"taskprogress"`
129	+	Runstats struct {
130	+	Text string `xml:",chardata"`
131	+	Finished struct {
132	+	Text string `xml:",chardata"`
133	+	Time string `xml:"time,attr"`
134	+	Timestr string `xml:"timestr,attr"`
135	+	Elapsed string `xml:"elapsed,attr"`
136	+	Summary string `xml:"summary,attr"`
137	+	Exit string `xml:"exit,attr"`
138	+	} `xml:"finished"`
139	+	Hosts struct {
140	+	Text string `xml:",chardata"`
141	+	Up string `xml:"up,attr"`
142	+	Down string `xml:"down,attr"`
143	+	Total string `xml:"total,attr"`
144	+	} `xml:"hosts"`
145	+	} `xml:"runstats"`
146	+	}
147	+
148	+	// NmapRun nmap single scan XML to struct
149	+	type NmapRun struct {
150	+	XMLName xml.Name `xml:"nmaprun"`
151	+	Text string `xml:",chardata"`
152	+	Scanner string `xml:"scanner,attr"`
153	+	Args string `xml:"args,attr"`
154	+	Start string `xml:"start,attr"`
155	+	Startstr string `xml:"startstr,attr"`
156	+	Version string `xml:"version,attr"`
157	+	Xmloutputversion string `xml:"xmloutputversion,attr"`
158	+	Scaninfo struct {
159	+	Text string `xml:",chardata"`
160	+	Type string `xml:"type,attr"`
161	+	Protocol string `xml:"protocol,attr"`
162	+	Numservices string `xml:"numservices,attr"`
163	+	Services string `xml:"services,attr"`
164	+	} `xml:"scaninfo"`
165	+	Verbose struct {
166	+	Text string `xml:",chardata"`
167	+	Level string `xml:"level,attr"`
168	+	} `xml:"verbose"`
169	+	Debugging struct {
170	+	Text string `xml:",chardata"`
171	+	Level string `xml:"level,attr"`
172	+	} `xml:"debugging"`
173	+	Taskbegin []struct {
174	+	Text string `xml:",chardata"`
175	+	Task string `xml:"task,attr"`
176	+	Time string `xml:"time,attr"`
177	+	} `xml:"taskbegin"`
178	+	Taskend []struct {
179	+	Text string `xml:",chardata"`
180	+	Task string `xml:"task,attr"`
181	+	Time string `xml:"time,attr"`
182	+	Extrainfo string `xml:"extrainfo,attr"`
183	+	} `xml:"taskend"`
184	+	Taskprogress []struct {
185	+	Text string `xml:",chardata"`
186	+	Task string `xml:"task,attr"`
187	+	Time string `xml:"time,attr"`
188	+	Percent string `xml:"percent,attr"`
189	+	Remaining string `xml:"remaining,attr"`
190	+	Etc string `xml:"etc,attr"`
191	+	} `xml:"taskprogress"`
192	+
193	+	Host struct {
194	+	Text string `xml:",chardata"`
195	+	Starttime string `xml:"starttime,attr"`
196	+	Endtime string `xml:"endtime,attr"`
197	+	Status struct {
198	+	Text string `xml:",chardata"`
199	+	State string `xml:"state,attr"`
200	+	Reason string `xml:"reason,attr"`
201	+	ReasonTtl string `xml:"reason_ttl,attr"`
202	+	} `xml:"status"`
203	+	Address struct {
204	+	Text string `xml:",chardata"`
205	+	Addr string `xml:"addr,attr"`
206	+	Addrtype string `xml:"addrtype,attr"`
207	+	} `xml:"address"`
208	+	Hostnames struct {
209	+	Text string `xml:",chardata"`
210	+	Hostname struct {
211	+	Text string `xml:",chardata"`
212	+	Name string `xml:"name,attr"`
213	+	Type string `xml:"type,attr"`
214	+	} `xml:"hostname"`
215	+	} `xml:"hostnames"`
216	+	Ports struct {
217	+	Text string `xml:",chardata"`
218	+	Extraports struct {
219	+	Text string `xml:",chardata"`
220	+	State string `xml:"state,attr"`
221	+	Count string `xml:"count,attr"`
222	+	Extrareasons []struct {
223	+	Text string `xml:",chardata"`
224	+	Reason string `xml:"reason,attr"`
225	+	Count string `xml:"count,attr"`
226	+	} `xml:"extrareasons"`
227	+	} `xml:"extraports"`
228	+	Port []struct {
229	+	Text string `xml:",chardata"`
230	+	Protocol string `xml:"protocol,attr"`
231	+	Portid string `xml:"portid,attr"`
232	+	State struct {
233	+	Text string `xml:",chardata"`
234	+	State string `xml:"state,attr"`
235	+	Reason string `xml:"reason,attr"`
236	+	ReasonTtl string `xml:"reason_ttl,attr"`
237	+	} `xml:"state"`
238	+	Service struct {
239	+	Text string `xml:",chardata"`
240	+	Name string `xml:"name,attr"`
241	+	Product string `xml:"product,attr"`
242	+	Devicetype string `xml:"devicetype,attr"`
243	+	Method string `xml:"method,attr"`
244	+	Conf string `xml:"conf,attr"`
245	+	Version string `xml:"version,attr"`
246	+	Extrainfo string `xml:"extrainfo,attr"`
247	+	Ostype string `xml:"ostype,attr"`
248	+	Servicefp string `xml:"servicefp,attr"`
249	+	Cpe []string `xml:"cpe"`
250	+	} `xml:"service"`
251	+	Script struct {
252	+	Text string `xml:",chardata"`
253	+	ID string `xml:"id,attr"`
254	+	Output string `xml:"output,attr"`
255	+	Elem []struct {
256	+	Text string `xml:",chardata"`
257	+	Key string `xml:"key,attr"`
258	+	} `xml:"elem"`
259	+	} `xml:"script"`
260	+	} `xml:"port"`
261	+	} `xml:"ports"`
262	+	Times struct {
263	+	Text string `xml:",chardata"`
264	+	Srtt string `xml:"srtt,attr"`
265	+	Rttvar string `xml:"rttvar,attr"`
266	+	To string `xml:"to,attr"`
267	+	} `xml:"times"`
268	+	} `xml:"host"`
269	+	Runstats struct {
270	+	Text string `xml:",chardata"`
271	+	Finished struct {
272	+	Text string `xml:",chardata"`
273	+	Time string `xml:"time,attr"`
274	+	Timestr string `xml:"timestr,attr"`
275	+	Elapsed string `xml:"elapsed,attr"`
276	+	Summary string `xml:"summary,attr"`
277	+	Exit string `xml:"exit,attr"`
278	+	} `xml:"finished"`
279	+	Hosts struct {
280	+	Text string `xml:",chardata"`
281	+	Up string `xml:"up,attr"`
282	+	Down string `xml:"down,attr"`
283	+	Total string `xml:"total,attr"`
284	+	} `xml:"hosts"`
285	+	} `xml:"runstats"`
286	+	}
287	+
288	+	type Host struct {
289	+	IPAddress string
290	+	Hostname string
291	+	Ports []Port
292	+	}
293	+
294	+	type Port struct {
295	+	Protocol string
296	+	PortID string
297	+	State string
298	+	Service struct {
299	+	Name string
300	+	Product string
301	+	Cpe string
302	+	}
303	+	Script struct {
304	+	ID string
305	+	Output string
306	+	}
307	+	}
308	+
309	+	// ParseNmapXML parse nmap XML result
310	+	func ParseNmapXML(raw string) NmapRun {
311	+	// parsing content
312	+	nmapRun := NmapRun{}
313	+	err := xml.Unmarshal([]byte(raw), &nmapRun)
314	+	if err != nil {
315	+	core.ErrorF("Failed to parse Nmap XML file: %v", err)
316	+	return nmapRun
317	+	}
318	+	return nmapRun
319	+	}
320	+
321	+	// ParseMultipleNmapXML parse nmap XML result
322	+	func ParseMultipleNmapXML(raw string) NmapRuns {
323	+	nmapRuns := NmapRuns{}
324	+	err := xml.Unmarshal([]byte(raw), &nmapRuns)
325	+	if err != nil {
326	+	core.ErrorF("Failed to parse Nmap XML file: %v", err)
327	+	return nmapRuns
328	+	}
329	+	return nmapRuns
330	+	}
331	+
332	+	// GetHosts parse nmap XML and return mutilehost object
333	+	func GetHosts(raw string) []Host {
334	+	var hosts []Host
335	+	if strings.Count(raw, "<address") <= 1 {
336	+	return hosts
337	+	}
338	+	nmapObj := ParseMultipleNmapXML(raw)
339	+	if nmapObj.Args == "" \|\| len(nmapObj.Host) <= 0 {
340	+	core.ErrorF("Failed to parse Nmap XML")
341	+	return hosts
342	+	}
343	+	// really parse something here
344	+	for _, nmapHost := range nmapObj.Host {
345	+	var host Host
346	+	core.DebugF("Parse XML for: %v", host.IPAddress)
347	+
348	+	host.IPAddress = nmapHost.Address.Addr
349	+	host.Hostname = nmapHost.Hostnames.Hostname.Name
350	+	if len(nmapHost.Ports.Port) > 0 {
351	+	for _, port := range nmapHost.Ports.Port {
352	+	var item Port
353	+	item.PortID = port.Portid
354	+	item.Protocol = port.Protocol
355	+	item.State = port.State.State
356	+	// service
357	+	item.Service.Name = port.Service.Name
358	+	item.Service.Product = port.Service.Product
359	+	//item.Service.Cpe = port.Service.Cpe
360	+	item.Script.ID = port.Script.ID
361	+	item.Script.Output = port.Script.Output
362	+	host.Ports = append(host.Ports, item)
363	+	}
364	+	hosts = append(hosts, host)
365	+	}
366	+	}
367	+	return hosts
368	+	}
369	+
370	+	// GetHost parse nmap XML and return host object
371	+	func GetHost(raw string) Host {
372	+	var host Host
373	+	nmapObj := ParseNmapXML(raw)
374	+	if nmapObj.Args == "" {
375	+	core.ErrorF("Failed to parse Nmap XML")
376	+	return host
377	+	}
378	+	host.IPAddress = nmapObj.Host.Address.Addr
379	+	host.Hostname = nmapObj.Host.Hostnames.Hostname.Name
380	+	core.DebugF("Parse XML for: %v", host.IPAddress)
381	+	if len(nmapObj.Host.Ports.Port) > 0 {
382	+	for _, port := range nmapObj.Host.Ports.Port {
383	+	var item Port
384	+	item.PortID = port.Portid
385	+	item.Protocol = port.Protocol
386	+	item.State = port.State.State
387	+	// service
388	+	item.Service.Name = port.Service.Name
389	+	item.Service.Product = port.Service.Product
390	+	//item.Service.Cpe = port.Service.Cpe
391	+	item.Script.ID = port.Script.ID
392	+	item.Script.Output = port.Script.Output
393	+
394	+	host.Ports = append(host.Ports, item)
395	+	}
396	+	}
397	+
398	+	return host
399	+	}
400	+
401	+	// ParsingNmapWithGoquery parse result from nmap XML format using goquery
402	+	func ParsingNmapWithGoquery(raw string, options core.Options) map[string][]string {
403	+	result := make(map[string][]string)
404	+
405	+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(raw))
406	+	if err != nil {
407	+	return result
408	+	}
409	+	doc.Find("host").Each(func(i int, h *goquery.Selection) {
410	+	ip, _ := h.Find("address").First().Attr("addr")
411	+
412	+	h.Find("port").Each(func(j int, s *goquery.Selection) {
413	+	service, _ := s.Find("service").First().Attr("name")
414	+	product, ok := s.Find("service").First().Attr("product")
415	+	if !ok {
416	+	product = ""
417	+	}
418	+	port, _ := s.Attr("portid")
419	+	info := fmt.Sprintf("%v/%v/%v", port, service, product)
420	+	result[ip] = append(result[ip], strings.TrimSpace(info))
421	+	})
422	+
423	+	if options.Scan.NmapScripts != "" {
424	+	h.Find("script").Each(func(j int, s *goquery.Selection) {
425	+	id, _ := s.Attr("id")
426	+	scriptOutput, _ := s.Attr("output")
427	+
428	+	if scriptOutput != "" {
429	+	// grep script output with grepString
430	+	if options.Scan.GrepString != "" {
431	+	var vulnerable bool
432	+	if strings.Contains(scriptOutput, options.Scan.GrepString) {
433	+	vulnerable = true
434	+	} else {
435	+	r, err := regexp.Compile(options.Scan.GrepString)
436	+	if err == nil {
437	+	matches := r.FindStringSubmatch(scriptOutput)
438	+	if len(matches) > 0 {
439	+	vulnerable = true
440	+	}
441	+	}
442	+	}
443	+	if vulnerable {
444	+	vul := fmt.Sprintf("/vulnerable\|%v", id)
445	+	result[ip] = append(result[ip], strings.TrimSpace(vul))
446	+	}
447	+	}
448	+
449	+	scriptOutput = strings.Replace(scriptOutput, "\n", "\\n", -1)
450	+	info := fmt.Sprintf("/script\|%v;;out\|%v", id, scriptOutput)
451	+	result[ip] = append(result[ip], strings.TrimSpace(info))
452	+	}
453	+	})
454	+	}
455	+	})
456	+
457	+	return result
458	+	}
459	+

■ ■ ■ ■ ■ ■

modules/nmap_test.go

1	+	package modules
2	+
3	+	import (
4	+	"fmt"
5	+	"github.com/davecgh/go-spew/spew"
6	+	"github.com/j3ssie/metabigor/core"
7	+	jsoniter "github.com/json-iterator/go"
8	+	"testing"
9	+	)
10	+
11	+	func TestParseNmapXML(t *testing.T) {
12	+	raw := core.GetFileContent("/tmp/nn/vuln.xml")
13	+	nmapRun := ParseNmapXML(raw)
14	+	spew.Dump(nmapRun.Host)
15	+	fmt.Println("------------")
16	+	spew.Dump(nmapRun.Host.Ports)
17	+	//if len(result) == 0 {
18	+	// t.Errorf("Error ParsingMasscan")
19	+	//}
20	+	}
21	+
22	+	func TestGetHost(t *testing.T) {
23	+	raw := core.GetFileContent("/var/folders/lx/q7xk40_d3vd_wvw5dpdj796r0000gn/T/mtg-log/nmap-77.111.191.237-378284373.xml")
24	+	//raw := core.GetFileContent("/tmp/nn/full.xml")
25	+	host := GetHost(raw)
26	+	spew.Dump(host)
27	+
28	+	fmt.Println("------------")
29	+
30	+	// output as JSON
31	+	if data, err := jsoniter.MarshalToString(host); err == nil {
32	+	fmt.Println(data)
33	+	}
34	+
35	+	if len(host.Ports) > 0 {
36	+	for _, port := range host.Ports {
37	+	info := fmt.Sprintf("%v:%v/%v/%v", host.IPAddress, port.PortID, port.Protocol, port.Service.Product)
38	+	fmt.Println(info)
39	+	}
40	+	}
41	+	}
42	+
43	+	func TestGetHosts(t *testing.T) {
44	+	raw := core.GetFileContent("/tmp/nn/multiple.xml")
45	+	hosts := GetHosts(raw)
46	+	//spew.Dump(host)
47	+
48	+	for _, host := range hosts {
49	+	fmt.Println("------------")
50	+	// output as JSON
51	+	if data, err := jsoniter.MarshalToString(host); err == nil {
52	+	fmt.Println(data)
53	+	}
54	+
55	+	if len(host.Ports) > 0 {
56	+	for _, port := range host.Ports {
57	+	info := fmt.Sprintf("%v:%v/%v/%v", host.IPAddress, port.PortID, port.Protocol, port.Service.Product)
58	+	fmt.Println(info)
59	+	}
60	+	}
61	+	}
62	+
63	+	}
64	+

■ ■ ■ ■ ■ ■

modules/scan.go

		skipped 1 lines
2	2
3	3		import (
4	4		"fmt"
	5	+	jsoniter "github.com/json-iterator/go"
5	6		"io/ioutil"
6	7		"os/exec"
7		-	"regexp"
8	8		"strings"
9	9
10		-	"github.com/PuerkitoBio/goquery"
11	10		"github.com/j3ssie/metabigor/core"
12	11		)
13	12
		skipped 52 lines
66	65
67	66		// RunNmap run nmap command and return list of port open
68	67		func RunNmap(input string, ports string, options core.Options) []string {
69		-	// ports := options.Ports
	68	+	// use nmap as overview scan
	69	+	if options.Scan.NmapOverview {
	70	+	ports = options.Scan.Ports
	71	+	}
70	72		if ports == "" {
71	73		ports = "443"
72	74		}
		skipped 17 lines
90	92		var result []string
91	93		realNmapOutput := nmapOutput + ".xml"
92	94		if !core.FileExists(realNmapOutput) {
	95	+	core.ErrorF("Result not found: %v", realNmapOutput)
93	96		return result
94	97		}
95		-	// result := ""
	98	+
96	99		data := core.GetFileContent(realNmapOutput)
97		-	rawResult := ParsingNmap(data, options)
	100	+	result = ParseNmap(data, options)
	101	+	return result
	102	+	}
	103	+
	104	+	// ParseNmap parse nmap XML output
	105	+	func ParseNmap(raw string, options core.Options) []string {
	106	+	var result []string
	107	+	var hosts []Host
	108	+	if strings.Count(raw, "<address") > 1 {
	109	+	hosts = append(hosts, GetHosts(raw)...)
	110	+	} else {
	111	+	hosts = append(hosts, GetHost(raw))
	112	+	}
	113	+
	114	+	for _, host := range hosts {
	115	+	//spew.Dump(host)
	116	+	if len(host.Ports) <= 0 {
	117	+	core.ErrorF("No open port found for %v", host.IPAddress)
	118	+	continue
	119	+	}
	120	+	if options.JsonOutput {
	121	+	if data, err := jsoniter.MarshalToString(host); err == nil {
	122	+	result = append(result, data)
	123	+	}
	124	+	continue
	125	+	}
98	126
99		-	for k, v := range rawResult {
100		-	if options.Scan.Flat {
101		-	result = append(result, fmt.Sprintf("%v:%v", k, strings.Join(v, ",")))
102		-	} else {
103		-	result = append(result, fmt.Sprintf("%v - %v", k, strings.Join(v, ",")))
	127	+	for _, port := range host.Ports {
	128	+	info := fmt.Sprintf("%v:%v/%v/%v", host.IPAddress, port.PortID, port.Protocol, port.Service.Product)
	129	+	//fmt.Println(info)
	130	+	result = append(result, info)
104	131		}
105	132		}
106		-
107	133		return result
108	134		}
109	135
		skipped 15 lines
125	151		return result
126	152		}
127	153
128		-	// ParsingMasscanXML parse result from masscan XML format
129		-	func ParsingMasscanXML(raw string) map[string][]string {
130		-	result := make(map[string][]string)
131		-	doc, err := goquery.NewDocumentFromReader(strings.NewReader(raw))
132		-	if err != nil {
133		-	return result
	154	+	// RunZmap run masscan command and return list of port open
	155	+	func RunZmap(inputFile string, port string, options core.Options) []string {
	156	+	ports := options.Scan.Ports
	157	+	if ports == "" {
	158	+	ports = "443"
	159	+	}
	160	+	zmapOutput := options.Scan.TmpOutput
	161	+	tmpFile, _ := ioutil.TempFile(options.Scan.TmpOutput, "zmap-*.txt")
	162	+	if zmapOutput != "" {
	163	+	tmpFile, _ = ioutil.TempFile(zmapOutput, fmt.Sprintf("zmap-%v-*.txt", core.StripPath(inputFile)))
	164	+	}
	165	+	zmapOutput = tmpFile.Name()
	166	+	zmapCmd := fmt.Sprintf("sudo zmap -p %v -w %v -f 'saddr,sport' -O csv -o %v", port, inputFile, zmapOutput)
	167	+	core.DebugF("Execute: %v", zmapCmd)
	168	+	command := []string{
	169	+	"bash",
	170	+	"-c",
	171	+	zmapCmd,
134	172		}
	173	+	exec.Command(command[0], command[1:]...).CombinedOutput()
135	174
136		-	doc.Find("host").Each(func(i int, s *goquery.Selection) {
137		-	ip, _ := s.Find("address").First().Attr("addr")
138		-	port, _ := s.Find("port").First().Attr("portid")
139		-	result[ip] = append(result[ip], port)
140		-	})
141		-
	175	+	result := ParseZmap(zmapOutput)
142	176		return result
143	177		}
144	178
145		-	// ParsingNmap parse result from nmap XML format
146		-	func ParsingNmap(raw string, options core.Options) map[string][]string {
147		-	result := make(map[string][]string)
148		-
149		-	doc, err := goquery.NewDocumentFromReader(strings.NewReader(raw))
150		-	if err != nil {
	179	+	// ParseZmap parsse zmap data
	180	+	func ParseZmap(zmapOutput string) []string {
	181	+	data := core.GetFileContent(zmapOutput)
	182	+	var result []string
	183	+	if strings.TrimSpace(data) == "" {
151	184		return result
152	185		}
153		-	doc.Find("host").Each(func(i int, h *goquery.Selection) {
154		-	ip, _ := h.Find("address").First().Attr("addr")
155	186
156		-	h.Find("port").Each(func(j int, s *goquery.Selection) {
157		-	service, _ := s.Find("service").First().Attr("name")
158		-	product, ok := s.Find("service").First().Attr("product")
159		-	if !ok {
160		-	product = ""
161		-	}
162		-	port, _ := s.Attr("portid")
163		-	info := fmt.Sprintf("%v/%v/%v", port, service, product)
164		-	result[ip] = append(result[ip], strings.TrimSpace(info))
165		-	})
166		-
167		-	if options.Scan.NmapScripts != "" {
168		-	h.Find("script").Each(func(j int, s *goquery.Selection) {
169		-	id, _ := s.Attr("id")
170		-	scriptOutput, _ := s.Attr("output")
	187	+	raw := strings.Replace(data, ",", ":", -1)
	188	+	raw = strings.Replace(raw, "saddr:sport", "", -1)
	189	+	raw = strings.TrimSpace(raw)
171	190
172		-	if scriptOutput != "" {
173		-	// grep script output with grepString
174		-	if options.Scan.GrepString != "" {
175		-	var vulnerable bool
176		-	if strings.Contains(scriptOutput, options.Scan.GrepString) {
177		-	vulnerable = true
178		-	} else {
179		-	r, err := regexp.Compile(options.Scan.GrepString)
180		-	if err == nil {
181		-	matches := r.FindStringSubmatch(scriptOutput)
182		-	if len(matches) > 0 {
183		-	vulnerable = true
184		-	}
185		-	}
186		-	}
187		-	if vulnerable {
188		-	vul := fmt.Sprintf("/vulnerable\|%v", id)
189		-	result[ip] = append(result[ip], strings.TrimSpace(vul))
190		-	}
191		-	}
192		-
193		-	scriptOutput = strings.Replace(scriptOutput, "\n", "\\n", -1)
194		-	info := fmt.Sprintf("/script\|%v;;out\|%v", id, scriptOutput)
195		-	result[ip] = append(result[ip], strings.TrimSpace(info))
196		-	}
197		-	})
198		-	}
199		-	})
200		-
	191	+	result = strings.Split(raw, "\n")
201	192		return result
202	193		}
203	194

■ ■ ■ ■ ■ ■

modules/scan_test.go

		skipped 19 lines
20	20		options.Scan.NmapScripts = "vulners.nse"
21	21		// options.Input = "103.102.128.0/24"
22	22		raw := core.GetFileContent("/tmp/testttt/samm.xml")
23		-	result := ParsingNmap(raw, options)
	23	+	result := ParseNmap(raw, options)
24	24		fmt.Println(result)
25	25		if len(result) == 0 {
26	26		t.Errorf("Error RunMasscan")
		skipped 9 lines
36	36		}
37	37		}
38	38
	39	+	func TestParseZmap(t *testing.T) {
	40	+	result := ParseZmap("/tmp/nn/p443")
	41	+	fmt.Println(result)
	42	+	if len(result) == 0 {
	43	+	t.Errorf("Error ParsingMasscan")
	44	+	}
	45	+	}
	46	+

Improve nmap parser