| skipped 1 lines |
| 2 | 2 | | |
| 3 | 3 | | import ( |
| 4 | 4 | | "crypto/tls" |
| | 5 | + | "errors" |
| 5 | 6 | | "fmt" |
| 6 | 7 | | "github.com/corpix/uarand" |
| 7 | 8 | | "github.com/hbakhtiyor/strsim" |
| 8 | 9 | | "github.com/karlseguin/ccache" |
| | 10 | + | "io" |
| 9 | 11 | | "io/ioutil" |
| 10 | 12 | | "log" |
| 11 | 13 | | "math/rand" |
| | 14 | + | "net" |
| 12 | 15 | | "net/http" |
| 13 | 16 | | "net/http/cookiejar" |
| 14 | 17 | | "net/url" |
| skipped 14 lines |
| 29 | 32 | | |
| 30 | 33 | | // http密码爆破 |
| 31 | 34 | | func HttpRequsetBasic(username string, password string, urlstring string, method string, postdata string, isredirect bool, headers map[string]string) (*Response, error) { |
| 32 | | - | client := GetClient(urlstring) |
| 33 | | - | var err error |
| 34 | | - | if isredirect { |
| 35 | | - | jar, _ := cookiejar.New(nil) |
| 36 | | - | client.Jar = jar |
| 37 | | - | } else { |
| 38 | | - | client.Jar = nil |
| 39 | | - | } |
| 40 | | - | req, err := http.NewRequest(strings.ToUpper(method), urlstring, strings.NewReader(postdata)) |
| 41 | | - | if err != nil { |
| 42 | | - | return nil, err |
| 43 | | - | } |
| 44 | | - | req.SetBasicAuth(username, password) |
| 45 | | - | req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8") |
| 46 | | - | req.Header.Set("User-Agent", uarand.GetRandom()) |
| 47 | | - | SetHeader(&req.Header) |
| 48 | | - | for v, k := range headers { |
| 49 | | - | req.Header[v] = []string{k} |
| 50 | | - | } |
| 51 | | - | var resp *http.Response |
| 52 | | - | |
| 53 | | - | // resp, err = tr.RoundTrip(req) |
| 54 | | - | resp, err = client.Do(req) |
| 55 | | - | if err != nil { |
| 56 | | - | //防止空指针 |
| 57 | | - | return &Response{"999", 999, "", nil, 0, "", ""}, err |
| 58 | | - | } |
| 59 | | - | var location string |
| 60 | | - | var reqbody string |
| 61 | | - | defer resp.Body.Close() |
| 62 | | - | if body, err := ioutil.ReadAll(resp.Body); err == nil { |
| 63 | | - | reqbody = string(body) |
| 64 | | - | } |
| 65 | | - | if resplocation, err := resp.Location(); err == nil { |
| 66 | | - | location = resplocation.String() |
| 67 | | - | } |
| 68 | | - | return &Response{resp.Status, resp.StatusCode, reqbody, &resp.Header, len(reqbody), resp.Request.URL.String(), location}, nil |
| | 35 | + | rsps, _, _, err := GetResponse(username, password, urlstring, method, postdata, isredirect, headers) |
| | 36 | + | return rsps, err |
| 69 | 37 | | } |
| 70 | 38 | | |
| 71 | 39 | | // client缓存 |
| skipped 41 lines |
| 113 | 81 | | } |
| 114 | 82 | | var tr *http.Transport |
| 115 | 83 | | tr = &http.Transport{ |
| 116 | | - | TLSClientConfig: &tls.Config{InsecureSkipVerify: true, MinVersion: tls.VersionTLS10}, |
| 117 | | - | DisableKeepAlives: false, |
| 118 | | - | MaxIdleConns: 300, |
| 119 | | - | IdleConnTimeout: 180, |
| 120 | | - | TLSHandshakeTimeout: 60, |
| 121 | | - | ExpectContinueTimeout: 30, |
| 122 | | - | MaxIdleConnsPerHost: 100, |
| | 84 | + | DialContext: (&net.Dialer{ |
| | 85 | + | Timeout: 30 * time.Second, |
| | 86 | + | KeepAlive: 30 * time.Second, |
| | 87 | + | DualStack: true, |
| | 88 | + | }).DialContext, |
| | 89 | + | MaxIdleConns: 100, |
| | 90 | + | MaxIdleConnsPerHost: 1024, |
| | 91 | + | TLSHandshakeTimeout: 0 * time.Second, |
| | 92 | + | IdleConnTimeout: 90 * time.Second, |
| | 93 | + | ExpectContinueTimeout: 1 * time.Second, |
| | 94 | + | MaxResponseHeaderBytes: 4096, // net/http default is 10Mb |
| | 95 | + | TLSClientConfig: &tls.Config{ |
| | 96 | + | Renegotiation: tls.RenegotiateOnceAsClient, |
| | 97 | + | InsecureSkipVerify: true, |
| | 98 | + | }, |
| | 99 | + | DisableKeepAlives: false, |
| 123 | 100 | | } |
| 124 | 101 | | if HttpProxy != "" { |
| 125 | 102 | | uri, _ := url.Parse(strings.TrimSpace(HttpProxy)) |
| skipped 25 lines |
| 151 | 128 | | } |
| 152 | 129 | | } |
| 153 | 130 | | |
| 154 | | - | // 需要考虑缓存 |
| 155 | | - | // 1、缓解网络不好的情况 |
| 156 | | - | // 2、缓存有效期为当天 |
| 157 | | - | // 3、缓存命中需和请求的数据完全匹配 |
| 158 | | - | func HttpRequset(urlstring string, method string, postdata string, isredirect bool, headers map[string]string) (*Response, error) { |
| | 131 | + | // 数组去重 |
| | 132 | + | func SliceRemoveDuplicates(slice []string) []string { |
| | 133 | + | if nil == slice || 0 == len(slice) { |
| | 134 | + | return slice |
| | 135 | + | } |
| | 136 | + | sort.Strings(slice) |
| | 137 | + | i := 0 |
| | 138 | + | var j int |
| | 139 | + | for { |
| | 140 | + | if i >= len(slice)-1 { |
| | 141 | + | break |
| | 142 | + | } |
| | 143 | + | for j = i + 1; j < len(slice) && slice[i] == slice[j]; j++ { |
| | 144 | + | } |
| | 145 | + | slice = append(slice[:i+1], slice[j:]...) |
| | 146 | + | i++ |
| | 147 | + | } |
| | 148 | + | return slice |
| | 149 | + | } |
| | 150 | + | |
| | 151 | + | func GetResponse(username string, password string, urlstring string, method string, postdata string, isredirect bool, headers map[string]string) (resp1 *Response, reqbody, location string, err error) { |
| 159 | 152 | | client := GetClient(urlstring) |
| 160 | 153 | | if nil == client { |
| 161 | | - | log.Printf("client is nil, url [%s]\n", urlstring) |
| 162 | | - | return nil, nil |
| | 154 | + | return nil, "", "", errors.New(urlstring + " client is nil") |
| 163 | 155 | | } |
| 164 | 156 | | if isredirect { |
| 165 | 157 | | jar, _ := cookiejar.New(nil) |
| skipped 3 lines |
| 169 | 161 | | } |
| 170 | 162 | | req, err := http.NewRequest(strings.ToUpper(method), urlstring, strings.NewReader(postdata)) |
| 171 | 163 | | if err != nil { |
| 172 | | - | return nil, err |
| | 164 | + | return nil, "", "", err |
| 173 | 165 | | } |
| 174 | | - | req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8") |
| 175 | | - | req.Header.Set("User-Agent", uarand.GetRandom()) |
| 176 | | - | // 设置全局自定义头、cookie信息 |
| 177 | | - | SetHeader(&req.Header) |
| 178 | | - | for v, k := range headers { |
| 179 | | - | req.Header[v] = []string{k} |
| | 166 | + | if "" != username && "" != password { |
| | 167 | + | req.SetBasicAuth(username, password) |
| 180 | 168 | | } |
| 181 | | - | resp, err := client.Do(req) |
| 182 | | - | if nil != resp { |
| 183 | | - | defer resp.Body.Close() |
| | 169 | + | req.Header.Add("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8") |
| | 170 | + | req.Header.Add("User-Agent", uarand.GetRandom()) |
| | 171 | + | //req.Header.Add("Connection", "keep-alive")// http1.1 默认 开启 |
| | 172 | + | SetHeader(&req.Header) |
| | 173 | + | for k, v := range headers { |
| | 174 | + | req.Header.Add(k, v) |
| 184 | 175 | | } |
| | 176 | + | |
| | 177 | + | var resp *http.Response |
| | 178 | + | // resp, err = tr.RoundTrip(req) |
| | 179 | + | resp, err = client.Do(req) |
| | 180 | + | defer func() { |
| | 181 | + | req.Body.Close() |
| | 182 | + | if nil != resp { |
| | 183 | + | //io.Copy(ioutil.Discard, resp.Body) |
| | 184 | + | resp.Body.Close() |
| | 185 | + | } |
| | 186 | + | }() |
| | 187 | + | |
| 185 | 188 | | if err != nil { |
| | 189 | + | if nil != resp { |
| | 190 | + | io.Copy(ioutil.Discard, resp.Body) |
| | 191 | + | } |
| 186 | 192 | | //防止空指针 |
| 187 | | - | return &Response{"999", 999, "", nil, 0, "", ""}, err |
| | 193 | + | return &Response{"999", 999, "", nil, 0, "", ""}, "", "", err |
| 188 | 194 | | } |
| 189 | | - | var location string |
| 190 | | - | var reqbody string |
| | 195 | + | |
| 191 | 196 | | if body, err := ioutil.ReadAll(resp.Body); err == nil { |
| 192 | 197 | | reqbody = string(body) |
| 193 | 198 | | } |
| 194 | 199 | | if resplocation, err := resp.Location(); err == nil { |
| 195 | 200 | | location = resplocation.String() |
| 196 | 201 | | } |
| 197 | | - | return &Response{resp.Status, resp.StatusCode, reqbody, &resp.Header, len(reqbody), resp.Request.URL.String(), location}, nil |
| | 202 | + | return &Response{resp.Status, resp.StatusCode, reqbody, &resp.Header, len(reqbody), resp.Request.URL.String(), location}, reqbody, location, nil |
| | 203 | + | } |
| | 204 | + | |
| | 205 | + | // 需要考虑缓存 |
| | 206 | + | // 1、缓解网络不好的情况 |
| | 207 | + | // 2、缓存有效期为当天 |
| | 208 | + | // 3、缓存命中需和请求的数据完全匹配 |
| | 209 | + | func HttpRequset(urlstring string, method string, postdata string, isredirect bool, headers map[string]string) (*Response, error) { |
| | 210 | + | rsps, _, _, err := GetResponse("", "", urlstring, method, postdata, isredirect, headers) |
| | 211 | + | return rsps, err |
| 198 | 212 | | } |
| 199 | 213 | | |
| 200 | 214 | | func Dnslogchek(randomstr string) bool { |
| skipped 134 lines |
