STRLCPY/Taipan

■ ■ ■ ■ ■ ■

Src/ES.Taipan.Application/Scan.fs

		skipped 364 lines
365	365		// this is a very dirty trick. By setting the authentication
366	366		// to Enabled and the type to NoAuthentication,
367	367		// I avoid to follow the Journey path for this specific case.
368		-	instantiateCrawlers([new AuthenticationInfo(Enabled = true); scanContext.Template.HttpRequestorSettings.Authentication])
	368	+	instantiateCrawlers([
	369	+	scanContext.Template.HttpRequestorSettings.Authentication
	370	+
	371	+	// TODO, until the plugin for differential analysis isn't readym this settings is not useful
	372	+	// new AuthenticationInfo(Enabled = true)
	373	+	])
369	374		else
370	375		instantiateCrawlers([new AuthenticationInfo()])
371	376
		skipped 23 lines
395	400		// try to get the IP and verify if the host is reachable
396	401		ip <- Some(Dns.GetHostAddresses(uri.Host) \|> Seq.head)
397	402		let webRequestor = _container.Value.Resolve<IWebPageRequestor>()
	403	+	webRequestor.HttpRequestor.Settings.PermanentDisableAuthentication()
398	404		let mutable webResponse = webRequestor.RequestInitialWebPage(new WebRequest(uri))
399	405
400	406		// check for redirect
		skipped 68 lines

■ ■ ■ ■ ■ ■

Src/ES.Taipan.Fingerprinter/SimpleCrawler.fs

		skipped 31 lines
32	32		]
33	33
34	34		let crawl startUrl limit (httpRequestor: IHttpRequestor) =
	35	+	// disable Auth since we don't need it for fingerpriting
	36	+	httpRequestor.Settings.PermanentDisableAuthentication()
	37	+
35	38		// Concurrent queue for saving collected urls.
36	39		let q = ConcurrentQueue<string>()
37	40
		skipped 116 lines

■ ■ ■ ■ ■ ■

Src/ES.Taipan.Infrastructure/ES.Taipan.Infrastructure.fsproj

		skipped 159 lines
160	160		<Compile Include="Network\HeuristicPageNotFoundIdentifier.fs" />
161	161		<Compile Include="Network\DefaultWebPageRequestor.fs" />
162	162		<Compile Include="Network\CacheableWebPageRequestor.fs" />
163		-	<Compile Include="Network\WebProxy.fs" />
164	163		<Compile Include="Network\WebUtility.fs" />
165	164		<Content Include="Network\Javascript\SendRequest.js">
166	165		<CopyToOutputDirectory>Always</CopyToOutputDirectory>
		skipped 2279 lines

■ ■ ■ ■ ■ ■

Src/ES.Taipan.Infrastructure/Network/DefaultHttpRequestor.fs

		skipped 18 lines
19	19
20	20		type DefaultHttpRequestor(defaultSettings: HttpRequestorSettings, logProvider: ILogProvider) as this =
21	21		let _requestGateTimeout = 1000 * 60 * 60 * 10 // 10 minutes
	22	+	let _authenticationLock = new Object()
22	23		let _maxParallelism = 100
23	24		let _requestGate = new RequestGate(_maxParallelism)
24	25		let _certificationValidate = new Event<CertificationValidateEventArgs>()
		skipped 23 lines
48	49		let headerName = kv.Key
49	50		let headerValue = kv.Value
50	51		let headerAlredyPresent =
51		-	httpRequest.Headers \|> Seq.toList
	52	+	httpRequest.Headers
	53	+	\|> Seq.toList
52	54		\|> List.exists (fun hdr -> hdr.Name.Equals(headerName, StringComparison.Ordinal))
53	55		if not headerAlredyPresent then
54		-	httpRequest.Headers.Add(new HttpHeader(Name=headerName, Value=headerValue))
	56	+	if headerName.Equals("cookie", StringComparison.OrdinalIgnoreCase) then
	57	+	httpRequest.Cookies.AddRange(HttpUtility.parseCookieHeaderValue(headerValue, httpRequest.Uri.Host))
	58	+	else
	59	+	httpRequest.Headers.Add(new HttpHeader(Name=headerName, Value=headerValue))
55	60
56	61		// add cookies
57	62		for kv in _settings.AdditionalCookies do
		skipped 65 lines
123	128		match _seleniumDriver with
124	129		\| Some _ -> ()
125	130		\| None ->
	131	+	if _settings.ProxyUrl.IsSome then
	132	+	let proxyUri = new Uri(_settings.ProxyUrl.Value)
	133	+	_seleniumDriver.Value.ProxyUrl <- Some(String.Format("{0}:{1}", proxyUri.Host, proxyUri.Port))
	134	+
126	135		_seleniumDriver <- Some(new SeleniumDriver(logProvider))
127		-	_seleniumDriver.Value.ProxyUrl <- _settings.ProxyUrl
128	136		_seleniumDriver.Value.Initialize()
129	137		)
130	138
		skipped 4 lines
135	143		let rec sendJourneyTransaction (path: JourneyPath) (transaction: JourneyTransaction) : HttpResponse option =
136	144		let httpRequest = transaction.BuildBaseHttpRequest()
137	145
138		-	// add to setting also the headers of the template request
139		-	httpRequest.Headers
140		-	\|> Seq.filter(fun hdr ->
141		-	["Content"]
142		-	\|> List.exists(fun pattern -> hdr.Name.StartsWith(pattern, StringComparison.OrdinalIgnoreCase))
143		-	\|> not
144		-	)
145		-	\|> Seq.iter(fun hdr -> _settings.AdditionalHttpHeaders.[hdr.Name] <- hdr.Value)
146		-
147	146		// manage parameters
148	147		let data = new StringBuilder()
149	148		let query = new StringBuilder()
		skipped 34 lines
184	183		this.SessionState.Value.RetrieveSessionParametersFromResponse(httpRequest, httpResponse.Value)
185	184
186	185		httpResponse
	186	+
	187	+	let isStatusCodeValidForauthenticationCheck(httpResponse: HttpResponse option) =
	188	+	match httpResponse with
	189	+	\| None -> false
	190	+	\| Some _ ->
	191	+	let sc = int httpResponse.Value.StatusCode
	192	+	sc >= 200 && sc < 400
187	193
188	194		let followPathNavigation() =
189	195		match _settings.Journey.Paths \|> Seq.tryHead with
		skipped 201 lines
391	397		httpResponse.Headers
392	398		\|> Seq.filter(fun hdr -> hdr.Name.Equals("set-cookie", StringComparison.Ordinal))
393	399		\|> Seq.iter(fun hdr ->
394		-	let cookies = HttpRequestorUtility.parseCookieHeaderValue(hdr.Value, httpResponse.ResponseUri.Value.Host)
	400	+	let cookies = HttpUtility.parseCookieHeaderValue(hdr.Value, httpResponse.ResponseUri.Value.Host)
395	401		this.SessionState.Value.AddCookieToSession(httpRequest, cookies)
396	402		)
397	403
		skipped 16 lines
414	420
415	421		member private this.VerifyIfIsNeededToAuthenticate(httpRequest: HttpRequest, httpResponse: HttpResponse option) =
416	422		if not _skipAuthenticationProcess && _settings.Authentication.Enabled then
417		-	let savedValue = _skipAuthenticationProcess
418		-	_skipAuthenticationProcess <- true
	423	+	lock _authenticationLock (fun () ->
	424	+	let savedValue = _skipAuthenticationProcess
	425	+	_skipAuthenticationProcess <- true
419	426
420		-	let httpResponseResult = ref httpResponse
421		-	match _settings.Authentication.Type with
422		-	\| HttpDigest ->
423		-	if httpResponse.IsSome && _httpDigestInfo.IsNone && httpResponse.Value.StatusCode = HttpStatusCode.Unauthorized then
424		-	// retrieve the Auth digest info and re-send the request with the correct token
425		-	_httpDigestInfo <- HttpDigestAuthenticationUtility.retrieveAuthenticationInfo(httpResponse.Value)
426		-	httpResponseResult := this.SendRequestDirect(httpRequest)
	427	+	let httpResponseResult = ref httpResponse
	428	+	match _settings.Authentication.Type with
	429	+	\| HttpDigest ->
	430	+	if httpResponse.IsSome && _httpDigestInfo.IsNone && httpResponse.Value.StatusCode = HttpStatusCode.Unauthorized then
	431	+	// retrieve the Auth digest info and re-send the request with the correct token
	432	+	_httpDigestInfo <- HttpDigestAuthenticationUtility.retrieveAuthenticationInfo(httpResponse.Value)
	433	+	httpResponseResult := this.SendRequestDirect(httpRequest)
427	434
428		-	\| WebForm when httpResponse.IsSome && httpResponse.Value.StatusCode = HttpStatusCode.OK ->
429		-	match this.SessionState with
430		-	\| Some _ ->
431		-	if not(this.AuthenticationSuccessful([\|httpResponse.Value\|])) then
432		-	// a specific logout condition was found, need to re-authenticate by following the Authentication Journey path
433		-	if this.AuthenticationSuccessful(followPathNavigation()) then
434		-	// finally re-do the request in an authentication context
435		-	httpResponseResult := this.SendRequestDirect(httpRequest)
436		-	else
437		-	_logger.AuthenticationFailed()
	435	+	\| WebForm when isStatusCodeValidForauthenticationCheck(httpResponse) ->
	436	+	match this.SessionState with
	437	+	\| Some _ ->
	438	+	if not(this.AuthenticationSuccessful([\|httpResponse.Value\|])) then
	439	+	// a specific logout condition was found, need to re-authenticate by following the Authentication Journey path
	440	+	if this.AuthenticationSuccessful(followPathNavigation()) then
	441	+	// finally re-do the request in an authentication context
	442	+	httpResponseResult := this.SendRequestDirect(httpRequest)
	443	+	else
	444	+	_logger.AuthenticationFailed()
438	445
439		-	\| None -> _logger.SessionStateNullOnWebAuth()
440		-	\| _ ->
441		-	// no authentication process needed
442		-	()
	446	+	\| None -> _logger.SessionStateNullOnWebAuth()
	447	+	\| _ ->
	448	+	// no authentication process needed
	449	+	()
443	450
444		-	// restore value
445		-	_skipAuthenticationProcess <- savedValue
446		-	!httpResponseResult
	451	+	// restore value
	452	+	_skipAuthenticationProcess <- savedValue
	453	+	!httpResponseResult
	454	+	)
447	455
448	456		else
449	457		httpResponse
		skipped 35 lines

■ ■ ■ ■ ■ ■

Src/ES.Taipan.Infrastructure/Network/HttpRequestorSettings.fs

		skipped 37 lines
38	38		/// This property specify if the Javascript engine should be used to send requests
39	39		member val UseJavascriptEngineForRequest = true with get, set
40	40
	41	+	member this.PermanentDisableAuthentication() =
	42	+	this.Authentication.Enabled <- false
	43	+	this.Journey.Paths.Clear()
	44	+
41	45		member this.ToXml() =
42	46		let additionalHttpHeaders = new XElement(x"AdditionalHttpHeaders")
43	47		this.AdditionalHttpHeaders
		skipped 10 lines
54	58		new XElement(x"AllowAutoRedirect", this.AllowAutoRedirect),
55	59		new XElement(x"UseJavascriptEngineForRequest", this.UseJavascriptEngineForRequest),
56	60		new XElement(x"Proxy",
57		-	if this.ProxyUrl.IsSome && Uri.IsWellFormedUriString(this.ProxyUrl.Value, UriKind.Absolute) then this.ProxyUrl.Value
	61	+	if this.ProxyUrl.IsSome
	62	+	then this.ProxyUrl.Value
58	63		else String.Empty
59	64		),
60	65		new XElement(x"StaticExtensions", String.Join(",", this.StaticExtensions)),
		skipped 21 lines
82	87
83	88		let proxyElement = root.Element(x"Proxy")
84	89		this.ProxyUrl <-
85		-	if proxyElement <> null then Some <\| proxyElement.Value.Trim()
	90	+	if proxyElement <> null && not(String.IsNullOrWhiteSpace(proxyElement.Value.Trim()))
	91	+	then Some <\| proxyElement.Value.Trim()
86	92		else None
87	93
88	94		root.Element(x"AdditionalHttpHeaders").Elements(x"HttpHeader")
		skipped 20 lines

■ ■ ■ ■ ■ ■

Src/ES.Taipan.Infrastructure/Network/HttpRequestorUtility.fs

1	1		namespace ES.Taipan.Infrastructure.Network
2	2
3	3		open System
4		-	open System.Collections.Generic
5		-	open System.IO
6	4		open System.Net
7		-	open Brotli
8	5
9	6		module internal HttpRequestorUtility =
10	7
11		-	let parseCookieHeaderValue(hdrValue: String, host: String) =
12		-	let cookies = new List<Cookie>()
13		-	let attributes = [
14		-	"comment"; "domain"; "max-age"
15		-	"path"; "secure"; "version";
16		-	"httponly"
17		-	]
18		-
19		-	let mutable isSecure = false
20		-	let mutable isHttpOnly = false
21		-
22		-	hdrValue.Split(';')
23		-	\|> Array.rev
24		-	\|> Array.map(fun item ->
25		-	let keyValue =
26		-	item.Split('=')
27		-	\|> Array.map(fun s -> s.Trim())
28		-	\|> Array.map(Uri.UnescapeDataString)
29		-	\|> Array.map(Uri.EscapeDataString)
30		-
31		-	if keyValue.Length > 1
32		-	then (keyValue.[0], keyValue.[1])
33		-	else (item, String.Empty)
34		-	)
35		-	\|> Array.iter(fun (name, value) ->
36		-	try
37		-	if attributes \|> List.contains(name.Trim().ToLower()) then
38		-	// it is an attribute
39		-	match name.Trim().ToLower() with
40		-	\| "secure" -> isSecure <- true
41		-	\| "httponly" -> isHttpOnly <- true
42		-	\| _ -> ()
43		-	else
44		-	// create a new cookie
45		-	let cookie = new Cookie(name, value, "/", host, Secure = isSecure, HttpOnly = isHttpOnly)
46		-	cookies.Add(cookie)
47		-	with _ -> ()
48		-	)
49		-	cookies
50		-
51		-	let addPostData(data: String, webRequest: HttpWebRequest) =
52		-	try
53		-	use streamWriter = new StreamWriter(webRequest.GetRequestStream())
54		-	streamWriter.Write(data)
55		-	with
56		-	\| _ -> ()
57		-
58		-	let getCookiesFromHeader(httpWebResponse: HttpWebResponse) =
59		-	let cookies = new List<Cookie>()
60		-	if httpWebResponse.Headers.AllKeys \|> Seq.contains "Set-Cookie" then
61		-	for hdrValue in httpWebResponse.Headers.GetValues("Set-Cookie") do
62		-	cookies.AddRange(parseCookieHeaderValue(hdrValue, httpWebResponse.ResponseUri.Host))
63		-	cookies
64		-
65		-	let addHttpHeader(header: HttpHeader, webRequest: HttpWebRequest) =
66		-	let headerName = header.Name.Replace("-", String.Empty).ToLower()
67		-	match headerName with
68		-	\| "useragent" -> webRequest.UserAgent <- header.Value
69		-	\| "accept" -> webRequest.Accept <- header.Value
70		-	\| "referer" -> webRequest.Referer <- header.Value
71		-	\| "host" -> webRequest.Host <- header.Value
72		-	\| "contenttype" -> webRequest.ContentType <- header.Value
73		-	\| "contentlength" ->
74		-	let contentLen = ref 0L
75		-	if Int64.TryParse(header.Value, contentLen) then webRequest.ContentLength <- !contentLen
76		-	\| "connection"
77		-	\| "proxyconnection" ->
78		-	if header.Value.Equals("Keep-Alive", StringComparison.OrdinalIgnoreCase) then
79		-	webRequest.KeepAlive <- true
80		-	else
81		-	webRequest.Connection <- header.Value
82		-	webRequest.KeepAlive <- false
83		-	\| "date" ->
84		-	let date = ref DateTime.Now
85		-	if DateTime.TryParse(header.Value, date) then webRequest.Date <- !date
86		-	\| "expect" ->
87		-	// 100-Continue must be setted with the System.Net.ServicePointManager.Expect100Continue Setted to true
88		-	// see http://haacked.com/archive/2004/05/15/http-web-request-expect-100-continue.aspx
89		-	if not <\| header.Value.Equals("", StringComparison.OrdinalIgnoreCase) then webRequest.Expect <- header.Value
90		-	\| "ifmodifiedsince" ->
91		-	let date = ref DateTime.Now
92		-	if DateTime.TryParse(header.Value, date) then webRequest.IfModifiedSince <- !date
93		-	\| "transferencoding" ->
94		-	webRequest.SendChunked <- true
95		-	webRequest.TransferEncoding <- header.Value
96		-	\| "cookie" ->
97		-	if webRequest.CookieContainer = null then
98		-	webRequest.CookieContainer <- new CookieContainer()
99		-
100		-	let cookieValue = header.Value
101		-	parseCookieHeaderValue(cookieValue, webRequest.RequestUri.Host)
102		-	\|> Seq.iter(webRequest.CookieContainer.Add)
103		-	\| _ -> webRequest.Headers.[header.Name] <- header.Value
104		-
105	8		let createHttpWebRequest(settings: HttpRequestorSettings, httpRequest: HttpRequest) =
106	9		let httpWebRequest = WebRequest.Create( httpRequest.Uri ) :?> HttpWebRequest
107	10
		skipped 18 lines
126	29
127	30		// set the headers
128	31		for header in httpRequest.Headers do
129		-	addHttpHeader(header, httpWebRequest)
	32	+	HttpUtility.addHttpHeader(header, httpWebRequest)
130	33
131	34		// set the cookies
132	35		if httpWebRequest.CookieContainer = null then
133	36		httpWebRequest.CookieContainer <- new CookieContainer()
	37	+
134	38		for cookie in httpRequest.Cookies do
135	39		httpWebRequest.CookieContainer.Add(cookie)
136	40
137	41		// set the post data if present
138	42		if not <\| String.IsNullOrEmpty(httpRequest.Data) then
139		-	addPostData(httpRequest.Data, httpWebRequest)
	43	+	HttpUtility.addPostData(httpRequest.Data, httpWebRequest)
140	44
141	45		httpWebRequest
142	46
		skipped 27 lines
170	74
171	75		// add cookies to httpResponse by parsing the Set-Cookie header. This since the container doesn't manage cookie on different path very well
172	76		// for more info see: https://stackoverflow.com/questions/3716144/cookiecontainer-handling-of-paths-who-ate-my-cookie
173		-	for cookie in getCookiesFromHeader(httpWebResponse) do
	77	+	for cookie in HttpUtility.getCookiesFromHeader(httpWebResponse) do
174	78		httpResponseResult.Cookies.Add(cookie)
175	79
176	80		httpResponseResult

■ ■ ■ ■ ■ ■

Src/ES.Taipan.Infrastructure/Network/HttpUtility.fs

1	1		namespace ES.Taipan.Infrastructure.Network
2	2
3	3		open System
	4	+	open System.IO
4	5		open System.Collections.Generic
5	6		open System.Net
6	7
7	8		module HttpUtility =
	9	+	let parseCookieHeaderValue(hdrValue: String, host: String) =
	10	+	let cookies = new List<Cookie>()
	11	+	let attributes = [
	12	+	"comment"; "domain"; "max-age"
	13	+	"path"; "secure"; "version";
	14	+	"httponly"
	15	+	]
	16	+
	17	+	let mutable isSecure = false
	18	+	let mutable isHttpOnly = false
	19	+
	20	+	hdrValue.Split(';')
	21	+	\|> Array.rev
	22	+	\|> Array.map(fun item ->
	23	+	let keyValue =
	24	+	item.Split('=')
	25	+	\|> Array.map(fun s -> s.Trim())
	26	+	\|> Array.map(Uri.UnescapeDataString)
	27	+	\|> Array.map(Uri.EscapeDataString)
	28	+
	29	+	if keyValue.Length > 1
	30	+	then (keyValue.[0], keyValue.[1])
	31	+	else (item, String.Empty)
	32	+	)
	33	+	\|> Array.iter(fun (name, value) ->
	34	+	try
	35	+	if attributes \|> List.contains(name.Trim().ToLower()) then
	36	+	// it is an attribute
	37	+	match name.Trim().ToLower() with
	38	+	\| "secure" -> isSecure <- true
	39	+	\| "httponly" -> isHttpOnly <- true
	40	+	\| _ -> ()
	41	+	else
	42	+	// create a new cookie
	43	+	let cookie = new Cookie(name, value, "/", host, Secure = isSecure, HttpOnly = isHttpOnly)
	44	+	cookies.Add(cookie)
	45	+	with _ -> ()
	46	+	)
	47	+	cookies
	48	+
	49	+	let addPostData(data: String, webRequest: HttpWebRequest) =
	50	+	try
	51	+	use streamWriter = new StreamWriter(webRequest.GetRequestStream())
	52	+	streamWriter.Write(data)
	53	+	with
	54	+	\| _ -> ()
	55	+
	56	+	let getCookiesFromHeader(httpWebResponse: HttpWebResponse) =
	57	+	let cookies = new List<Cookie>()
	58	+	if httpWebResponse.Headers.AllKeys \|> Seq.contains "Set-Cookie" then
	59	+	for hdrValue in httpWebResponse.Headers.GetValues("Set-Cookie") do
	60	+	cookies.AddRange(parseCookieHeaderValue(hdrValue, httpWebResponse.ResponseUri.Host))
	61	+	cookies
	62	+
	63	+	let addHttpHeader(header: HttpHeader, webRequest: HttpWebRequest) =
	64	+	let headerName = header.Name.Replace("-", String.Empty).ToLower()
	65	+	match headerName with
	66	+	\| "useragent" -> webRequest.UserAgent <- header.Value
	67	+	\| "accept" -> webRequest.Accept <- header.Value
	68	+	\| "referer" -> webRequest.Referer <- header.Value
	69	+	\| "host" -> webRequest.Host <- header.Value
	70	+	\| "contenttype" -> webRequest.ContentType <- header.Value
	71	+	\| "contentlength" ->
	72	+	let contentLen = ref 0L
	73	+	if Int64.TryParse(header.Value, contentLen) then webRequest.ContentLength <- !contentLen
	74	+	\| "connection"
	75	+	\| "proxyconnection" ->
	76	+	if header.Value.Equals("Keep-Alive", StringComparison.OrdinalIgnoreCase) then
	77	+	webRequest.KeepAlive <- true
	78	+	else
	79	+	webRequest.Connection <- header.Value
	80	+	webRequest.KeepAlive <- false
	81	+	\| "date" ->
	82	+	let date = ref DateTime.Now
	83	+	if DateTime.TryParse(header.Value, date) then webRequest.Date <- !date
	84	+	\| "expect" ->
	85	+	// 100-Continue must be setted with the System.Net.ServicePointManager.Expect100Continue Setted to true
	86	+	// see http://haacked.com/archive/2004/05/15/http-web-request-expect-100-continue.aspx
	87	+	if not <\| header.Value.Equals("", StringComparison.OrdinalIgnoreCase) then webRequest.Expect <- header.Value
	88	+	\| "ifmodifiedsince" ->
	89	+	let date = ref DateTime.Now
	90	+	if DateTime.TryParse(header.Value, date) then webRequest.IfModifiedSince <- !date
	91	+	\| "transferencoding" ->
	92	+	webRequest.SendChunked <- true
	93	+	webRequest.TransferEncoding <- header.Value
	94	+	\| "cookie" ->
	95	+	if webRequest.CookieContainer = null then
	96	+	webRequest.CookieContainer <- new CookieContainer()
	97	+
	98	+	let cookieValue = header.Value
	99	+	parseCookieHeaderValue(cookieValue, webRequest.RequestUri.Host)
	100	+	\|> Seq.iter(webRequest.CookieContainer.Add)
	101	+	\| _ -> webRequest.Headers.[header.Name] <- header.Value
8	102
9	103		/// Return the absolute path directory. Some examples:
10	104		/// http://www.example.com/a/b/c/index.php -> /a/b/c
		skipped 51 lines

■ ■ ■ ■ ■ ■

Src/ES.Taipan.Infrastructure/Network/Journey.fs

1	1		namespace ES.Taipan.Infrastructure.Network
2	2
3	3		open System
	4	+	open System.Net
4	5		open System.Text
5	6		open System.IO
6	7		open System.Collections.Generic
		skipped 53 lines
60	61
61	62		// add headers
62	63		this.TemplateRequest.Headers
63		-	\|> List.iter(fun (name, value) -> httpRequest.Headers.Add(new HttpHeader(Name = name, Value = value)))
	64	+	\|> List.iter(fun (name, value) ->
	65	+	if name.Equals("cookie", StringComparison.OrdinalIgnoreCase) then
	66	+	httpRequest.Cookies.AddRange(HttpUtility.parseCookieHeaderValue(value, httpRequest.Uri.Host))
	67	+	else
	68	+	httpRequest.Headers.Add(new HttpHeader(Name = name, Value = value))
	69	+	)
64	70
65	71		httpRequest
66	72
		skipped 136 lines

■ ■ ■ ■ ■ ■

Src/ES.Taipan.Infrastructure/Network/WebProxy.fs

1	-	namespace ES.Taipan.Infrastructure.Network
2	-
3	-	// TODO: remove this class it is only used in server part
4	-	open System
5	-	open System.Linq
6	-	open System.IO
7	-	open System.Collections.Generic
8	-	open System.Collections.Concurrent
9	-	open System.Net
10	-	open ES.Fslog
11	-
12	-	type WebProxy(remoteAddr: Uri, httpRequestor: IHttpRequestor, logProvider: ILogProvider) =
13	-	let _recordedRequests = new ConcurrentQueue<HttpRequest>()
14	-	let _whitelistedHeaders = new List<String>()
15	-	let _extensionToIgnore = new List<String>()
16	-	let _injectors = new List<String * HttpRequest * HttpResponse -> unit>()
17	-
18	-	let isHeaderAllowed(httpHeader: HttpHeader) =
19	-	_whitelistedHeaders \|> Seq.exists(fun whiteListPrefix -> httpHeader.Name.StartsWith(whiteListPrefix, StringComparison.OrdinalIgnoreCase))
20	-
21	-	new(remoteAddr: Uri) =
22	-	let settings = new HttpRequestorSettings()
23	-	settings.AllowAutoRedirect <- false
24	-	let httpRequestor = new DefaultHttpRequestor(settings, LogProvider.GetDefault())
25	-	new WebProxy(remoteAddr, httpRequestor, LogProvider.GetDefault())
26	-
27	-	member val Id = Guid.NewGuid() with get
28	-
29	-	abstract AnalyzeResponse : String * HttpRequest * HttpResponse -> unit
30	-	default this.AnalyzeResponse(destPath: String, httpRequest: HttpRequest, httpResponse: HttpResponse) =
31	-	// filter headers
32	-	let copyOfHeaders = httpResponse.Headers.ToList()
33	-	httpResponse.Headers.Clear()
34	-	copyOfHeaders \|> Seq.filter isHeaderAllowed \|> Seq.iter httpResponse.Headers.Add
35	-
36	-	// do modification
37	-	_injectors \|> Seq.iter(fun injector -> injector(destPath, httpRequest, httpResponse))
38	-
39	-	member this.AddInjector(codeInjector: String * HttpRequest * HttpResponse -> unit) =
40	-	_injectors.Add(codeInjector)
41	-
42	-	member this.AddExtensionToNotRecord(extension: String) =
43	-	_extensionToIgnore.Add(extension)
44	-
45	-	member this.AddWhitelistedHeader(headerPrefix: String) =
46	-	_whitelistedHeaders.Add(headerPrefix)
47	-
48	-	member this.GetRecordedRequests() =
49	-	_recordedRequests
50	-
51	-	member private this.SendHttpRequest(httpRequest: HttpRequest, destPath: String) =
52	-	match httpRequestor.SendRequest(httpRequest) with
53	-	\| Some httpResponse ->
54	-	if not <\| _extensionToIgnore.Contains(Path.GetExtension(httpRequest.Uri.AbsolutePath)) then
55	-	_recordedRequests.Enqueue(httpRequest)
56	-	this.AnalyzeResponse(destPath, httpRequest, httpResponse)
57	-	Some httpResponse
58	-	\| _ -> None
59	-
60	-	member this.ForwardPost(destPath: String, data: String) =
61	-	let destUri = new Uri(remoteAddr, destPath)
62	-	let httpRequest = new HttpRequest(destUri)
63	-	httpRequest.Method <- HttpMethods.Post
64	-	httpRequest.Data <- data
65	-	this.SendHttpRequest(httpRequest, destPath)
66	-
67	-	member this.ForwardGet(destPath: String) =
68	-	let destUri = new Uri(remoteAddr, destPath)
69	-	let httpRequest = new HttpRequest(destUri)
70	-	this.SendHttpRequest(httpRequest, destPath)
71	-
72	-
73	-

■ ■ ■ ■ ■ ■

Src/ES.Taipan.Infrastructure/Threading/TaskManager.fs

		skipped 9 lines
10	10		(*
11	11		This class is used in order to manage multiple Tasks in order to be Pause and/or Stopped
12	12		in a coherent way. The task execyted through the manager, could be paused or stopped.
	13	+	When Pause/Stop is called, it is guaranteed that all the managed tasks have paused/stopped.
13	14		*)
14	15
15	16		type TaskManager(statusToMonitor: ServiceStateController, releasePauseStatusMonitor: Boolean, releaseStopStatusMonitor: Boolean, concurrentLimit: Int32) as this =
		skipped 111 lines

■ ■ ■ ■ ■ ■

Src/Taipan/ScanReport.fs

1	1		namespace Taipan
2	2
3		-	open ES.Taipan.Application
4	3		open System
	4	+	open System.Reflection
5	5		open System.Linq
6	6		open System.IO
7		-	open Newtonsoft.Json
8		-	open ES.Taipan.Inspector
9	7		open System.Collections.Generic
10	8		open System.Text
	9	+	open Newtonsoft.Json
	10	+	open ES.Taipan.Application
11	11
12	12		type ReportFormat =
13	13		\| Json
		skipped 80 lines
94	94		reportContent.ToString()
95	95
96	96		member this.Save(reportFilename: String option) =
97		-	let defaultName = String.Format("{0}_{1}{2}", scanResult.Scan.Context.StartRequest.HttpRequest.Uri.Host, Guid.NewGuid().ToString("N"), this.Type.ToString())
	97	+	let baseName = String.Format("{0}_{1}{2}", scanResult.Scan.Context.StartRequest.HttpRequest.Uri.Host, Guid.NewGuid().ToString("N"), this.Type.ToString())
	98	+	let reportDirectory = Path.Combine(Path.GetDirectoryName(Assembly.GetEntryAssembly().Location), "Report")
	99	+	Directory.CreateDirectory(reportDirectory) \|> ignore
	100	+	let defaultName = Path.Combine(reportDirectory, baseName)
	101	+
98	102		let filename = defaultArg reportFilename defaultName
99	103
100	104		let reportContent =
		skipped 6 lines

■ ■ ■ ■ ■ ■

Src/templates.fsx

		skipped 62 lines
63	63
64	64		// crawler settings
65	65		defaultProfile.CrawlerSettings.ReCrawlPages <- true
66		-	defaultProfile.CrawlerSettings.MaxNumOfRequestsToTheSamePage <- 100
	66	+	defaultProfile.CrawlerSettings.MaxNumOfRequestsToTheSamePage <- 5
67	67		defaultProfile.CrawlerSettings.MaxNumberOfPagesToCrawl <- 2000
68	68		defaultProfile.CrawlerSettings.Scope <- NavigationScope.EnteredPathAndBelow
69	69		defaultProfile.CrawlerSettings.ActivateAllAddOns <- true
		skipped 251 lines

Improved authenticated scan via WebForm