STRLCPY/metabigor

Update metabigor
j3ssie committed 4 years ago

3d47d0da

1 parent 587ae50a

Revision indexing in progress... (symbol navigation in revisions will be accurate after indexed)

■ ■ ■ ■ ■ ■

cmd/root.go

		skipped 75 lines
76	76		}
77	77		}
78	78
79		-
80	79		core.InforF("Metabigor %v by %v", core.VERSION, core.AUTHOR)
81	80		core.InforF(fmt.Sprintf("Store log file to: %v", options.LogFile))
82	81		}
		skipped 1 lines

■ ■ ■ ■ ■ ■

cmd/scan.go

		skipped 11 lines
12	12		"github.com/spf13/cobra"
13	13		)
14	14
15		-
16	15		func init() {
17	16		var scanCmd = &cobra.Command{
18	17		Use: "scan",
		skipped 4 lines
23	22
24	23		scanCmd.Flags().StringP("ports", "p", "0-65535", "Port range for previous command")
25	24		scanCmd.Flags().StringP("rate", "r", "5000", "rate limit for masscan command")
26		-	scanCmd.Flags().BoolP("detail","D", false, "Do Nmap scan based on previous output")
	25	+	scanCmd.Flags().BoolP("detail", "D", false, "Do Nmap scan based on previous output")
27	26
28	27		scanCmd.Flags().BoolP("flat", "f", false, "format output like this: 1.2.3.4:443")
29	28		scanCmd.Flags().BoolP("skip-masscan", "s", false, "run nmap from input format like this: 1.2.3.4:443")
30		-	scanCmd.Flags().StringP("script","S", "", "nmap scripts")
31		-	scanCmd.Flags().StringP("grep","g", "", "match string to confirm script success")
	29	+	scanCmd.Flags().StringP("script", "S", "", "nmap scripts")
	30	+	scanCmd.Flags().StringP("grep", "g", "", "match string to confirm script success")
32	31		// only parse scan
33	32		scanCmd.Flags().StringP("result-folder", "R", "", "Result folder")
34	33
		skipped 159 lines

■ ■ ■ ■ ■ ■

cmd/search.go

		skipped 10 lines
11	11		"github.com/spf13/cobra"
12	12		)
13	13
14		-
15	14		func init() {
16	15		var searchCmd = &cobra.Command{
17	16		Use: "search",
		skipped 86 lines
104	103		// add more query by add the country code with original query
105	104		func addMoreQuery(inputs []string, options core.Options) []string {
106	105		var moreQueries []string
107		-	ContriesCode := []string{"AF̵", "AL", "DZ", "AS", "AD", "AO", "AI", "AQ", "AG", "AR", "AM", "AW", "AU", "AT", "AZ", "BS", "BH", "BD", "BB", "BY", "BE", "BZ", "BJ", "BM", "BT", "BO", "BA", "BW", "BV", "BR", "IO", "BN", "BG", "BF", "BI", "KH", "CM", "CA", "CV", "KY", "CF", "TD", "CL", "CN", "CX", "CC", "CO", "KM", "CG", "CD", "CK", "CR", "CI", "HR", "CU", "CY", "CZ", "DK", "DJ", "DM", "DO", "EC", "EG", "EH", "SV", "GQ", "ER", "EE", "ET", "FK", "FO", "FJ", "FI", "FR", "GF", "PF", "TF", "GA", "GM", "GE", "DE", "GH", "GI", "GR", "GL", "GD", "GP", "GU", "GT", "GN", "GW", "GY", "HT", "HM", "HN", "HK", "HU", "IS", "IN", "ID", "IR", "IQ", "IE", "IL", "IT", "JM", "JP", "JO", "KZ", "KE", "KI", "KP", "KR", "KW", "KG", "LA", "LV", "LB", "LS", "LR", "LY", "LI", "LT", "LU", "MO", "MK", "MG", "MW", "MY", "MV", "ML", "MT", "MH", "MQ", "MR", "MU", "YT", "MX", "FM", "MD", "MC", "MN", "MS", "MA", "MZ", "MM", "NA", "NR", "NP", "NL", "AN", "NC", "NZ", "NI", "NE", "NG", "NU", "NF", "MP", "NO", "OM", "PK", "PW", "PS", "PA", "PG", "PY", "PE", "PH", "PN", "PL", "PT", "PR", "QA", "RE", "RO", "RU", "RW", "SH", "KN", "LC", "PM", "VC", "WS", "SM", "ST", "SA", "SN", "CS", "SC", "SL", "SG", "SK", "SI", "SB", "SO", "ZA", "GS", "ES", "LK", "SD", "SR", "SJ", "SZ", "SE", "CH", "SY", "TW", "TJ", "TZ", "TH", "TL", "TG", "TK", "TO", "TT", "TN", "TR", "TM", "TC", "TV", "UG", "UA", "AE", "GB", "US", "UM", "UY", "UZ", "VE", "VU", "VN", "VG", "VI", "WF", "YE", "ZW",}
	106	+	ContriesCode := []string{"AF̵", "AL", "DZ", "AS", "AD", "AO", "AI", "AQ", "AG", "AR", "AM", "AW", "AU", "AT", "AZ", "BS", "BH", "BD", "BB", "BY", "BE", "BZ", "BJ", "BM", "BT", "BO", "BA", "BW", "BV", "BR", "IO", "BN", "BG", "BF", "BI", "KH", "CM", "CA", "CV", "KY", "CF", "TD", "CL", "CN", "CX", "CC", "CO", "KM", "CG", "CD", "CK", "CR", "CI", "HR", "CU", "CY", "CZ", "DK", "DJ", "DM", "DO", "EC", "EG", "EH", "SV", "GQ", "ER", "EE", "ET", "FK", "FO", "FJ", "FI", "FR", "GF", "PF", "TF", "GA", "GM", "GE", "DE", "GH", "GI", "GR", "GL", "GD", "GP", "GU", "GT", "GN", "GW", "GY", "HT", "HM", "HN", "HK", "HU", "IS", "IN", "ID", "IR", "IQ", "IE", "IL", "IT", "JM", "JP", "JO", "KZ", "KE", "KI", "KP", "KR", "KW", "KG", "LA", "LV", "LB", "LS", "LR", "LY", "LI", "LT", "LU", "MO", "MK", "MG", "MW", "MY", "MV", "ML", "MT", "MH", "MQ", "MR", "MU", "YT", "MX", "FM", "MD", "MC", "MN", "MS", "MA", "MZ", "MM", "NA", "NR", "NP", "NL", "AN", "NC", "NZ", "NI", "NE", "NG", "NU", "NF", "MP", "NO", "OM", "PK", "PW", "PS", "PA", "PG", "PY", "PE", "PH", "PN", "PL", "PT", "PR", "QA", "RE", "RO", "RU", "RW", "SH", "KN", "LC", "PM", "VC", "WS", "SM", "ST", "SA", "SN", "CS", "SC", "SL", "SG", "SK", "SI", "SB", "SO", "ZA", "GS", "ES", "LK", "SD", "SR", "SJ", "SZ", "SE", "CH", "SY", "TW", "TJ", "TZ", "TH", "TL", "TG", "TK", "TO", "TT", "TN", "TR", "TM", "TC", "TV", "UG", "UA", "AE", "GB", "US", "UM", "UY", "UZ", "VE", "VU", "VN", "VG", "VI", "WF", "YE", "ZW"}
108	107
109	108		for _, input := range inputs {
110	109		options.Search.Query = input
		skipped 13 lines

■ ■ ■ ■ ■ ■

core/chrome.go

1	+	package core
2	+
3	+	import (
4	+	"context"
5	+	"log"
6	+	"os"
7	+	"path"
8	+	"path/filepath"
9	+	"time"
10	+
11	+	"github.com/chromedp/chromedp"
12	+	)
13	+
14	+	// RequestWithChrome Do request with real browser
15	+	func RequestWithChrome(url string, contentID string, timeout int) string {
16	+	// prepare the chrome options
17	+	opts := append(chromedp.DefaultExecAllocatorOptions[:],
18	+	chromedp.Flag("headless", true),
19	+	chromedp.Flag("ignore-certificate-errors", true),
20	+	chromedp.Flag("disable-gpu", true),
21	+	chromedp.Flag("enable-automation", true),
22	+	chromedp.Flag("disable-extensions", true),
23	+	chromedp.Flag("disable-setuid-sandbox", true),
24	+	chromedp.Flag("disable-web-security", true),
25	+	chromedp.Flag("no-first-run", true),
26	+	chromedp.Flag("no-default-browser-check", true),
27	+	)
28	+
29	+	allocCtx, bcancel := chromedp.NewExecAllocator(context.Background(), opts...)
30	+	defer bcancel()
31	+
32	+	ctx, cancel := chromedp.NewContext(allocCtx, chromedp.WithLogf(log.Printf))
33	+	ctx, cancel = context.WithTimeout(ctx, time.Duration(timeout)*time.Second)
34	+	defer cancel()
35	+
36	+	// run task list
37	+	var data string
38	+	err := chromedp.Run(ctx,
39	+	chromedp.Navigate(url),
40	+	chromedp.OuterHTML(contentID, &data, chromedp.NodeVisible, chromedp.ByID),
41	+	)
42	+	DebugF(data)
43	+
44	+	// clean chromedp-runner folder
45	+	cleanUp()
46	+
47	+	if err != nil {
48	+	InforF("[ERRR] %v", err)
49	+	return ""
50	+	}
51	+	return data
52	+	}
53	+
54	+	func cleanUp() {
55	+	tmpFolder := path.Join(os.TempDir(), "chromedp-runner*")
56	+	if _, err := os.Stat("/tmp/"); !os.IsNotExist(err) {
57	+	tmpFolder = path.Join("/tmp/", "chromedp-runner*")
58	+	}
59	+	junks, err := filepath.Glob(tmpFolder)
60	+	if err != nil {
61	+	return
62	+	}
63	+	for _, junk := range junks {
64	+	os.RemoveAll(junk)
65	+	}
66	+	}
67	+

■ ■ ■ ■ ■ ■

core/options.go

		skipped 46 lines
47	47		More bool
48	48		}
49	49
50		-	// HTTPRequest all information about response
51		-	type HTTPRequest struct {
52		-	Method string
53		-	URL string
54		-	Headers map[string]string
55		-	Body string
	50	+	// Request all information about request
	51	+	type Request struct {
	52	+	Timeout int
	53	+	Repeat int
	54	+	Scheme string
	55	+	Host string
	56	+	Port string
	57	+	Path string
	58	+	URL string
	59	+	Proxy string
	60	+	Method string
	61	+	Redirect bool
	62	+	Headers []map[string]string
	63	+	Body string
	64	+	Beautify string
56	65		}
57	66
58		-	// HTTPResponse all information about response
59		-	type HTTPResponse struct {
60		-	StatusCode int
61		-	Status string
62		-	Headers map[string][]string
63		-	Body string
64		-	ResponseTime float64
65		-	Length int
66		-	Beautify string
	67	+	// Response all information about response
	68	+	type Response struct {
	69	+	HasPopUp bool
	70	+	StatusCode int
	71	+	Status string
	72	+	Headers []map[string]string
	73	+	Body string
	74	+	ResponseTime float64
	75	+	Length int
	76	+	Beautify string
	77	+	BeautifyHeader string
67	78		}
68	79

■ ■ ■ ■ ■ ■

core/request.go

1	1		package core
2	2
3	3		import (
4		-	"context"
5	4		"crypto/tls"
6	5		"fmt"
7		-	"log"
8		-	"os"
9		-	"path"
10		-	"path/filepath"
	6	+	"github.com/go-resty/resty"
	7	+	"github.com/sirupsen/logrus"
	8	+	"io/ioutil"
	9	+	"net/http"
11	10		"strconv"
12	11		"strings"
13	12		"time"
14		-
15		-	"github.com/chromedp/chromedp"
16		-	"github.com/parnurzeal/gorequest"
17	13		)
18	14
19		-	// RequestWithChrome Do request with real browser
20		-	func RequestWithChrome(url string, contentID string, timeout int) string {
21		-	// prepare the chrome options
22		-	opts := append(chromedp.DefaultExecAllocatorOptions[:],
23		-	chromedp.Flag("headless", true),
24		-	chromedp.Flag("ignore-certificate-errors", true),
25		-	chromedp.Flag("disable-gpu", true),
26		-	chromedp.Flag("enable-automation", true),
27		-	chromedp.Flag("disable-extensions", true),
28		-	chromedp.Flag("disable-setuid-sandbox", true),
29		-	chromedp.Flag("disable-web-security", true),
30		-	chromedp.Flag("no-first-run", true),
31		-	chromedp.Flag("no-default-browser-check", true),
32		-	)
33		-
34		-	allocCtx, bcancel := chromedp.NewExecAllocator(context.Background(), opts...)
35		-	defer bcancel()
36		-
37		-	ctx, cancel := chromedp.NewContext(allocCtx, chromedp.WithLogf(log.Printf))
38		-	ctx, cancel = context.WithTimeout(ctx, time.Duration(timeout)*time.Second)
39		-	defer cancel()
40		-
41		-	// run task list
42		-	var data string
43		-	err := chromedp.Run(ctx,
44		-	chromedp.Navigate(url),
45		-	chromedp.OuterHTML(contentID, &data, chromedp.NodeVisible, chromedp.ByID),
46		-	)
47		-	DebugF(data)
48		-
49		-	// clean chromedp-runner folder
50		-	cleanUp()
51		-
52		-	if err != nil {
53		-	InforF("[ERRR] %v", err)
54		-	return ""
55		-	}
56		-	return data
57		-	}
58		-
59		-	func cleanUp() {
60		-	tmpFolder := path.Join(os.TempDir(), "chromedp-runner*")
61		-	if _, err := os.Stat("/tmp/"); !os.IsNotExist(err) {
62		-	tmpFolder = path.Join("/tmp/", "chromedp-runner*")
63		-	}
64		-	junks, err := filepath.Glob(tmpFolder)
65		-	if err != nil {
66		-	return
67		-	}
68		-	for _, junk := range junks {
69		-	os.RemoveAll(junk)
70		-	}
71		-	}
	15	+	var headers map[string]string
72	16
73	17		// SendGET just send GET request
74	18		func SendGET(url string, options Options) string {
75		-	req := HTTPRequest{
76		-	Method: "GET",
77		-	URL: url,
	19	+	headers = map[string]string{
	20	+	"UserAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36",
	21	+	"Accept": "/",
	22	+	"AcceptLang": "en-US,en;q=0.8",
78	23		}
79		-	resp := SendRequest(req, options)
	24	+	resp, _ := JustSend(options, "GET", url, headers)
80	25		return resp.Body
81	26		}
82	27
83	28		// SendPOST just send POST request
84	29		func SendPOST(url string, options Options) string {
85		-	req := HTTPRequest{
86		-	Method: "POST",
87		-	URL: url,
88		-	}
89		-	resp := SendRequest(req, options)
	30	+	resp, _ := JustSend(options, "POST", url, headers)
90	31		return resp.Body
91	32		}
92	33
93		-	// SendRequest just send GET request
94		-	func SendRequest(req HTTPRequest, options Options) HTTPResponse {
95		-	method := req.Method
96		-	url := req.URL
97		-	headers := req.Headers
98		-	body := req.Body
99		-	// default user-agent
100		-	if headers == nil {
101		-	headers = make(map[string]string)
	34	+	// JustSend just sending request
	35	+	func JustSend(options Options, method string, url string, headers map[string]string) (res Response, err error) {
	36	+
	37	+	timeout := options.Timeout
	38	+
	39	+	// disable log when retry
	40	+	logger := logrus.New()
	41	+	if !options.Debug {
	42	+	logger.Out = ioutil.Discard
102	43		}
103		-	headers["User-Agent"] = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/82.0.4052.0 Safari/537.36"
	44	+
	45	+	client := resty.New()
	46	+	client.SetLogger(logger)
	47	+	client.SetTransport(&http.Transport{
	48	+	MaxIdleConns: 100,
	49	+	MaxConnsPerHost: 1000,
	50	+	IdleConnTimeout: time.Duration(timeout) * time.Second,
	51	+	ExpectContinueTimeout: time.Duration(timeout) * time.Second,
	52	+	ResponseHeaderTimeout: time.Duration(timeout) * time.Second,
	53	+	TLSHandshakeTimeout: time.Duration(timeout) * time.Second,
	54	+	DisableCompression: true,
	55	+	TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	56	+	})
	57	+
	58	+	client.SetHeaders(headers)
	59	+	client.SetCloseConnection(true)
104	60
105		-	// new client
106		-	client := gorequest.New().TLSClientConfig(&tls.Config{InsecureSkipVerify: true})
107		-	client.Timeout(time.Duration(options.Timeout) * time.Second)
108		-	if options.Proxy != "" {
109		-	client.Proxy(options.Proxy)
110		-	}
111		-	var res HTTPResponse
112		-	// choose method
	61	+	client.SetTimeout(time.Duration(timeout) * time.Second)
	62	+	client.SetRetryWaitTime(time.Duration(timeout/2) * time.Second)
	63	+	client.SetRetryMaxWaitTime(time.Duration(timeout) * time.Second)
	64	+
	65	+	var resp *resty.Response
	66	+	// really sending things here
	67	+	method = strings.ToLower(strings.TrimSpace(method))
113	68		switch method {
114		-	case "GET":
115		-	client.Get(url)
	69	+	case "get":
	70	+	resp, err = client.R().
	71	+	Get(url)
116	72		break
117		-	case "POST":
118		-	client.Post(url)
119		-	break
120		-	case "PUT":
121		-	client.Put(url)
122		-	break
123		-	case "HEAD":
124		-	client.Head(url)
125		-	break
126		-	case "PATCH":
127		-	client.Patch(url)
128		-	break
129		-	case "DELETE":
130		-	client.Delete(url)
	73	+	case "post":
	74	+	resp, err = client.R().
	75	+	Get(url)
131	76		break
132	77		}
133	78
134		-	timeStart := time.Now()
135		-	for k, v := range headers {
136		-	client.AppendHeader(k, v)
137		-	}
138		-	if body != "" {
139		-	client.Send(body)
	79	+	// in case we want to get redirect stuff
	80	+	if res.StatusCode != 0 {
	81	+	return res, nil
140	82		}
141	83
142		-	// really sending stuff
143		-	resp, resBody, errs := client.End()
144		-	resTime := time.Since(timeStart).Seconds()
145		-
146		-	if len(errs) > 0 && res.StatusCode != 0 {
147		-	return res
148		-	} else if len(errs) > 0 {
149		-	ErrorF("Error sending %v", errs)
150		-	return HTTPResponse{}
	84	+	if err != nil \|\| resp == nil {
	85	+	ErrorF("%v %v", url, err)
	86	+	return Response{}, err
151	87		}
152	88
153		-	resp.Body.Close()
154		-	// return ParseResponse(resp, resBody, resTime), nil
155		-
156		-	return ParseResponse(resp, resBody, resTime)
157		-
	89	+	return ParseResponse(*resp), nil
158	90		}
159	91
160	92		// ParseResponse field to Response
161		-	func ParseResponse(resp gorequest.Response, resBody string, resTime float64) (res HTTPResponse) {
	93	+	func ParseResponse(resp resty.Response) (res Response) {
162	94		// var res libs.Response
163		-	resLength := len(string(resBody))
164		-
	95	+	resLength := len(string(resp.Body()))
165	96		// format the headers
166	97		var resHeaders []map[string]string
167		-	for k, v := range resp.Header {
	98	+	for k, v := range resp.RawResponse.Header {
168	99		element := make(map[string]string)
169	100		element[k] = strings.Join(v[:], "")
170	101		resLength += len(fmt.Sprintf("%s: %s\n", k, strings.Join(v[:], "")))
171	102		resHeaders = append(resHeaders, element)
172	103		}
173		-	// respones time in second
	104	+	// response time in second
	105	+	resTime := float64(resp.Time()) / float64(time.Second)
174	106		resHeaders = append(resHeaders,
175	107		map[string]string{"Total Length": strconv.Itoa(resLength)},
176	108		map[string]string{"Response Time": fmt.Sprintf("%f", resTime)},
177	109		)
178	110
179	111		// set some variable
180		-	res.Headers = resp.Header
181		-	res.StatusCode = resp.StatusCode
182		-	res.Status = resp.Status
183		-	res.Body = resBody
	112	+	res.Headers = resHeaders
	113	+	res.StatusCode = resp.StatusCode()
	114	+	res.Status = fmt.Sprintf("%v %v", resp.Status(), resp.RawResponse.Proto)
	115	+	res.Body = string(resp.Body())
184	116		res.ResponseTime = resTime
185	117		res.Length = resLength
	118	+	// beautify
	119	+	res.Beautify = BeautifyResponse(res)
	120	+	res.BeautifyHeader = BeautifyHeaders(res)
186	121		return res
187	122		}
188	123
	124	+	// BeautifyHeaders beautify response headers
	125	+	func BeautifyHeaders(res Response) string {
	126	+	beautifyHeader := fmt.Sprintf("%v \n", res.Status)
	127	+	for _, header := range res.Headers {
	128	+	for key, value := range header {
	129	+	beautifyHeader += fmt.Sprintf("%v: %v\n", key, value)
	130	+	}
	131	+	}
	132	+	return beautifyHeader
	133	+	}
	134	+
	135	+	// BeautifyResponse beautify response
	136	+	func BeautifyResponse(res Response) string {
	137	+	var beautifyRes string
	138	+	beautifyRes += fmt.Sprintf("%v \n", res.Status)
	139	+
	140	+	for _, header := range res.Headers {
	141	+	for key, value := range header {
	142	+	beautifyRes += fmt.Sprintf("%v: %v\n", key, value)
	143	+	}
	144	+	}
	145	+
	146	+	beautifyRes += fmt.Sprintf("\n%v\n", res.Body)
	147	+	return beautifyRes
	148	+	}
	149	+

■ ■ ■ ■ ■ ■

modules/ip.go

		skipped 69 lines
70	70
71	71		// more info in verbose mode
72	72		if options.Verbose {
73		-	for k,v := range info {
74		-	data := fmt.Sprintf("%v\|%v", k,v)
	73	+	for k, v := range info {
	74	+	data := fmt.Sprintf("%v\|%v", k, v)
75	75		result = append(result, data)
76	76		}
77	77		}
		skipped 3 lines

■ ■ ■ ■ ■ ■

modules/ip_test.go

skipped 15 lines
16 16 }
17 17
18 18 }
19 +

All occurrences

■ ■ ■ ■ ■ ■

modules/netblock.go

		skipped 99 lines
100	100		url := fmt.Sprintf(`https://bgp.he.net/AS%v#_prefixes`, asn)
101	101		core.InforF("Get data from: %v", url)
102	102		var result []string
103		-	content := core.RequestWithChrome(url, "prefixes", options.Timeout * 4)
	103	+	content := core.RequestWithChrome(url, "prefixes", options.Timeout*4)
104	104		doc, err := goquery.NewDocumentFromReader(strings.NewReader(content))
105	105		if err != nil {
106	106		return result
		skipped 63 lines
170	170		})
171	171		return result
172	172		}
173		-
174	173
175	174		// OrgBgbView get Org infor from bgpview.io
176	175		func OrgBgbView(options core.Options) []string {
		skipped 44 lines

■ ■ ■ ■ ■ ■

modules/search.go

		skipped 101 lines
102	102		query := core.Base64Encode(options.Search.Query)
103	103		url := fmt.Sprintf(`https://fofa.so/search/result_stats?qbase64=%v`, query)
104	104		core.DebugF("Get optimize data from: %v", url)
105		-	req := core.HTTPRequest{
106		-	Method: "GET",
107		-	URL: url,
108		-	Headers: map[string]string{
109		-	"X-Requested-With": "XMLHttpRequest",
110		-	},
	105	+
	106	+	headers := map[string]string{
	107	+	"UserAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36",
	108	+	"Accept": "/",
	109	+	"AcceptLang": "en-US,en;q=0.8",
	110	+	"X-Requested-With": "XMLHttpRequest",
111	111		}
112		-	res := core.SendRequest(req, options)
	112	+	res, _ := core.JustSend(options, "GET", url, headers)
113	113		content := res.Body
114	114
115	115		var result []string
		skipped 19 lines

Update metabigor