| skipped 4 lines |
| 5 | 5 | | "fmt" |
| 6 | 6 | | "github.com/corpix/uarand" |
| 7 | 7 | | "github.com/hbakhtiyor/strsim" |
| | 8 | + | "github.com/karlseguin/ccache" |
| 8 | 9 | | "io/ioutil" |
| | 10 | + | "log" |
| 9 | 11 | | "math/rand" |
| 10 | 12 | | "net/http" |
| 11 | 13 | | "net/http/cookiejar" |
| skipped 15 lines |
| 27 | 29 | | |
| 28 | 30 | | // http密码爆破 |
| 29 | 31 | | func HttpRequsetBasic(username string, password string, urlstring string, method string, postdata string, isredirect bool, headers map[string]string) (*Response, error) { |
| 30 | | - | var tr *http.Transport |
| | 32 | + | client := GetClient(urlstring) |
| 31 | 33 | | var err error |
| 32 | | - | if HttpProxy != "" { |
| 33 | | - | uri, _ := url.Parse(strings.TrimSpace(HttpProxy)) |
| 34 | | - | tr = &http.Transport{ |
| 35 | | - | MaxIdleConnsPerHost: -1, |
| 36 | | - | TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, |
| 37 | | - | DisableKeepAlives: true, |
| 38 | | - | Proxy: http.ProxyURL(uri), |
| 39 | | - | IdleConnTimeout: 15 * time.Second, |
| 40 | | - | } |
| 41 | | - | } else { |
| 42 | | - | tr = &http.Transport{ |
| 43 | | - | MaxIdleConnsPerHost: -1, |
| 44 | | - | TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, |
| 45 | | - | DisableKeepAlives: true, |
| 46 | | - | IdleConnTimeout: 15 * time.Second, |
| 47 | | - | } |
| 48 | | - | } |
| 49 | | - | |
| 50 | | - | client := &http.Client{ |
| 51 | | - | Timeout: time.Duration(10) * time.Second, |
| 52 | | - | Transport: tr, |
| 53 | | - | CheckRedirect: func(req *http.Request, via []*http.Request) error { |
| 54 | | - | return http.ErrUseLastResponse |
| 55 | | - | }} |
| 56 | 34 | | if isredirect { |
| 57 | 35 | | jar, _ := cookiejar.New(nil) |
| 58 | | - | client = &http.Client{ |
| 59 | | - | Timeout: time.Duration(10) * time.Second, |
| 60 | | - | Transport: tr, |
| 61 | | - | Jar: jar, |
| 62 | | - | } |
| | 36 | + | client.Jar = jar |
| | 37 | + | } else { |
| | 38 | + | client.Jar = nil |
| 63 | 39 | | } |
| 64 | 40 | | req, err := http.NewRequest(strings.ToUpper(method), urlstring, strings.NewReader(postdata)) |
| 65 | 41 | | if err != nil { |
| skipped 26 lines |
| 92 | 68 | | return &Response{resp.Status, resp.StatusCode, reqbody, &resp.Header, len(reqbody), resp.Request.URL.String(), location}, nil |
| 93 | 69 | | } |
| 94 | 70 | | |
| 95 | | - | // 需要考虑缓存 |
| 96 | | - | // 1、缓解网络不好的情况 |
| 97 | | - | // 2、缓存有效期为当天 |
| 98 | | - | // 3、缓存命中需和请求的数据完全匹配 |
| 99 | | - | func HttpRequset(urlstring string, method string, postdata string, isredirect bool, headers map[string]string) (*Response, error) { |
| | 71 | + | // client缓存 |
| | 72 | + | var clientHttpCc *ccache.Cache |
| | 73 | + | |
| | 74 | + | func InitCHcc() { |
| | 75 | + | if nil == clientHttpCc { |
| | 76 | + | configure := ccache.Configure() |
| | 77 | + | configure = configure.MaxSize(10000) |
| | 78 | + | clientHttpCc = ccache.New(configure) |
| | 79 | + | } |
| | 80 | + | } |
| | 81 | + | |
| | 82 | + | func init() { |
| | 83 | + | RegInitFunc(func() { |
| | 84 | + | InitCHcc() |
| | 85 | + | }) |
| | 86 | + | } |
| | 87 | + | |
| | 88 | + | var mUrls = make(map[string]string) |
| | 89 | + | |
| | 90 | + | func GetClient4Cc(szUrl string) *http.Client { |
| | 91 | + | InitCHcc() |
| | 92 | + | oU, err := url.Parse(szUrl) |
| | 93 | + | if nil == err { |
| | 94 | + | if o := clientHttpCc.Get(oU.Host); nil != o { |
| | 95 | + | if v, ok := o.Value().(*http.Client); ok { |
| | 96 | + | return v |
| | 97 | + | } |
| | 98 | + | } |
| | 99 | + | } else { |
| | 100 | + | log.Println("GetClient4Cc url.Parse is err ", err, szUrl) |
| | 101 | + | } |
| | 102 | + | return nil |
| | 103 | + | } |
| | 104 | + | func GetClient(szUrl string) *http.Client { |
| | 105 | + | oU, err := url.Parse(szUrl) |
| | 106 | + | if nil != err { |
| | 107 | + | log.Printf("GetClient url:%s url.Parse err:%v\n", szUrl, err) |
| | 108 | + | return nil |
| | 109 | + | } |
| | 110 | + | client := GetClient4Cc(szUrl) |
| | 111 | + | if nil != client { |
| | 112 | + | return client |
| | 113 | + | } |
| 100 | 114 | | var tr *http.Transport |
| | 115 | + | tr = &http.Transport{ |
| | 116 | + | TLSClientConfig: &tls.Config{InsecureSkipVerify: true, MinVersion: tls.VersionTLS10}, |
| | 117 | + | DisableKeepAlives: false, |
| | 118 | + | MaxIdleConns: 300, |
| | 119 | + | IdleConnTimeout: 180, |
| | 120 | + | TLSHandshakeTimeout: 60, |
| | 121 | + | ExpectContinueTimeout: 30, |
| | 122 | + | MaxIdleConnsPerHost: 100, |
| | 123 | + | } |
| 101 | 124 | | if HttpProxy != "" { |
| 102 | 125 | | uri, _ := url.Parse(strings.TrimSpace(HttpProxy)) |
| 103 | | - | tr = &http.Transport{ |
| 104 | | - | MaxIdleConnsPerHost: -1, |
| 105 | | - | TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, |
| 106 | | - | DisableKeepAlives: true, |
| 107 | | - | Proxy: http.ProxyURL(uri), |
| 108 | | - | } |
| 109 | | - | } else { |
| 110 | | - | tr = &http.Transport{ |
| 111 | | - | MaxIdleConnsPerHost: -1, |
| 112 | | - | TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, |
| 113 | | - | DisableKeepAlives: true, |
| 114 | | - | } |
| | 126 | + | tr.Proxy = http.ProxyURL(uri) |
| 115 | 127 | | } |
| 116 | 128 | | |
| 117 | | - | client := &http.Client{ |
| 118 | | - | Timeout: time.Duration(10) * time.Second, |
| | 129 | + | client = &http.Client{ |
| | 130 | + | Timeout: time.Duration(20) * time.Second, |
| 119 | 131 | | Transport: tr, |
| 120 | 132 | | CheckRedirect: func(req *http.Request, via []*http.Request) error { |
| 121 | 133 | | return http.ErrUseLastResponse |
| 122 | 134 | | }} |
| | 135 | + | mUrls[oU.Host] = "" |
| | 136 | + | clientHttpCc.Set(oU.Host, client, defaultInteractionDuration) |
| | 137 | + | return client |
| | 138 | + | } |
| | 139 | + | |
| | 140 | + | func CloseHttpClient(szUrl string) { |
| | 141 | + | oU, _ := url.Parse(szUrl) |
| | 142 | + | client := GetClient4Cc(szUrl) |
| | 143 | + | if nil != client { |
| | 144 | + | client.CloseIdleConnections() |
| | 145 | + | } |
| | 146 | + | clientHttpCc.Delete(oU.Host) |
| | 147 | + | } |
| | 148 | + | func CloseAllHttpClient() { |
| | 149 | + | for k, _ := range mUrls { |
| | 150 | + | CloseHttpClient("http://" + k) |
| | 151 | + | } |
| | 152 | + | } |
| | 153 | + | |
| | 154 | + | // 需要考虑缓存 |
| | 155 | + | // 1、缓解网络不好的情况 |
| | 156 | + | // 2、缓存有效期为当天 |
| | 157 | + | // 3、缓存命中需和请求的数据完全匹配 |
| | 158 | + | func HttpRequset(urlstring string, method string, postdata string, isredirect bool, headers map[string]string) (*Response, error) { |
| | 159 | + | client := GetClient(urlstring) |
| | 160 | + | if nil == client { |
| | 161 | + | log.Printf("client is nil, url [%s]\n", urlstring) |
| | 162 | + | return nil, nil |
| | 163 | + | } |
| 123 | 164 | | if isredirect { |
| 124 | 165 | | jar, _ := cookiejar.New(nil) |
| 125 | | - | client = &http.Client{ |
| 126 | | - | Timeout: time.Duration(10) * time.Second, |
| 127 | | - | Transport: tr, |
| 128 | | - | Jar: jar, |
| 129 | | - | } |
| | 166 | + | client.Jar = jar |
| | 167 | + | } else { |
| | 168 | + | client.Jar = nil |
| 130 | 169 | | } |
| 131 | 170 | | req, err := http.NewRequest(strings.ToUpper(method), urlstring, strings.NewReader(postdata)) |
| 132 | 171 | | if err != nil { |
| skipped 7 lines |
| 140 | 179 | | req.Header[v] = []string{k} |
| 141 | 180 | | } |
| 142 | 181 | | resp, err := client.Do(req) |
| | 182 | + | if nil != resp { |
| | 183 | + | defer resp.Body.Close() |
| | 184 | + | } |
| 143 | 185 | | if err != nil { |
| 144 | 186 | | //防止空指针 |
| 145 | 187 | | return &Response{"999", 999, "", nil, 0, "", ""}, err |
| 146 | 188 | | } |
| 147 | 189 | | var location string |
| 148 | 190 | | var reqbody string |
| 149 | | - | defer resp.Body.Close() |
| 150 | 191 | | if body, err := ioutil.ReadAll(resp.Body); err == nil { |
| 151 | 192 | | reqbody = string(body) |
| 152 | 193 | | } |
| skipped 141 lines |
