diff --git a/.gitattributes b/.gitattributes
new file mode 100644
index 0000000..1ff0c42
--- /dev/null
+++ b/.gitattributes
@@ -0,0 +1,63 @@
+###############################################################################
+# Set default behavior to automatically normalize line endings.
+###############################################################################
+* text=auto
+
+###############################################################################
+# Set default behavior for command prompt diff.
+#
+# This is need for earlier builds of msysgit that does not have it on by
+# default for csharp files.
+# Note: This is only used by command line
+###############################################################################
+#*.cs diff=csharp
+
+###############################################################################
+# Set the merge driver for project and solution files
+#
+# Merging from the command prompt will add diff markers to the files if there
+# are conflicts (Merging from VS is not affected by the settings below, in VS
+# the diff markers are never inserted). Diff markers may cause the following
+# file extensions to fail to load in VS. An alternative would be to treat
+# these files as binary and thus will always conflict and require user
+# intervention with every merge. To do so, just uncomment the entries below
+###############################################################################
+#*.sln merge=binary
+#*.csproj merge=binary
+#*.vbproj merge=binary
+#*.vcxproj merge=binary
+#*.vcproj merge=binary
+#*.dbproj merge=binary
+#*.fsproj merge=binary
+#*.lsproj merge=binary
+#*.wixproj merge=binary
+#*.modelproj merge=binary
+#*.sqlproj merge=binary
+#*.wwaproj merge=binary
+
+###############################################################################
+# behavior for image files
+#
+# image files are treated as binary by default.
+###############################################################################
+#*.jpg binary
+#*.png binary
+#*.gif binary
+
+###############################################################################
+# diff behavior for common document formats
+#
+# Convert binary document formats to text before diffing them. This feature
+# is only available from the command line. Turn it on by uncommenting the
+# entries below.
+###############################################################################
+#*.doc diff=astextplain
+#*.DOC diff=astextplain
+#*.docx diff=astextplain
+#*.DOCX diff=astextplain
+#*.dot diff=astextplain
+#*.DOT diff=astextplain
+#*.pdf diff=astextplain
+#*.PDF diff=astextplain
+#*.rtf diff=astextplain
+#*.RTF diff=astextplain
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..b06e864
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,212 @@
+## Ignore Visual Studio temporary files, build results, and
+## files generated by popular Visual Studio add-ons.
+
+# User-specific files
+*.suo
+*.user
+*.userosscache
+*.sln.docstates
+
+# User-specific files (MonoDevelop/Xamarin Studio)
+*.userprefs
+
+# Build results
+[Dd]ebug/
+[Dd]ebugPublic/
+[Rr]elease/
+[Rr]eleases/
+x64/
+x86/
+build/
+bld/
+[Bb]in/
+[Oo]bj/
+
+# Visual Studio 2015 cache/options directory
+.vs/
+
+# MSTest test Results
+[Tt]est[Rr]esult*/
+[Bb]uild[Ll]og.*
+
+# NUNIT
+*.VisualState.xml
+TestResult.xml
+
+# Build Results of an ATL Project
+[Dd]ebugPS/
+[Rr]eleasePS/
+dlldata.c
+
+# DNX
+project.lock.json
+artifacts/
+
+*_i.c
+*_p.c
+*_i.h
+*.ilk
+*.meta
+*.obj
+*.pch
+*.pdb
+*.pgc
+*.pgd
+*.rsp
+*.sbr
+*.tlb
+*.tli
+*.tlh
+*.tmp
+*.tmp_proj
+*.log
+*.vspscc
+*.vssscc
+.builds
+*.pidb
+*.svclog
+*.scc
+
+# Chutzpah Test files
+_Chutzpah*
+
+# Visual C++ cache files
+ipch/
+*.aps
+*.ncb
+*.opensdf
+*.sdf
+*.cachefile
+
+# Visual Studio profiler
+*.psess
+*.vsp
+*.vspx
+
+# TFS 2012 Local Workspace
+$tf/
+
+# Guidance Automation Toolkit
+*.gpState
+
+# ReSharper is a .NET coding add-in
+_ReSharper*/
+*.[Rr]e[Ss]harper
+*.DotSettings.user
+
+# JustCode is a .NET coding add-in
+.JustCode
+
+# TeamCity is a build add-in
+_TeamCity*
+
+# DotCover is a Code Coverage Tool
+*.dotCover
+
+# NCrunch
+_NCrunch_*
+.*crunch*.local.xml
+
+# MightyMoose
+*.mm.*
+AutoTest.Net/
+
+# Web workbench (sass)
+.sass-cache/
+
+# Installshield output folder
+[Ee]xpress/
+
+# DocProject is a documentation generator add-in
+DocProject/buildhelp/
+DocProject/Help/*.HxT
+DocProject/Help/*.HxC
+DocProject/Help/*.hhc
+DocProject/Help/*.hhk
+DocProject/Help/*.hhp
+DocProject/Help/Html2
+DocProject/Help/html
+
+# Click-Once directory
+publish/
+
+# Publish Web Output
+*.[Pp]ublish.xml
+*.azurePubxml
+## TODO: Comment the next line if you want to checkin your
+## web deploy settings but do note that will include unencrypted
+## passwords
+#*.pubxml
+
+*.publishproj
+
+# NuGet Packages
+*.nupkg
+# The packages folder can be ignored because of Package Restore
+**/packages/*
+# except build/, which is used as an MSBuild target.
+!**/packages/build/
+# Uncomment if necessary however generally it will be regenerated when needed
+#!**/packages/repositories.config
+
+# Windows Azure Build Output
+csx/
+*.build.csdef
+
+# Windows Store app package directory
+AppPackages/
+
+# Visual Studio cache files
+# files ending in .cache can be ignored
+*.[Cc]ache
+# but keep track of directories ending in .cache
+!*.[Cc]ache/
+
+# Others
+ClientBin/
+[Ss]tyle[Cc]op.*
+~$*
+*~
+*.dbmdl
+*.dbproj.schemaview
+*.pfx
+*.publishsettings
+node_modules/
+orleans.codegen.cs
+
+# RIA/Silverlight projects
+Generated_Code/
+
+# Backup & report files from converting an old project file
+# to a newer Visual Studio version. Backup files are not needed,
+# because we have git ;-)
+_UpgradeReport_Files/
+Backup*/
+UpgradeLog*.XML
+UpgradeLog*.htm
+
+# SQL Server files
+*.mdf
+*.ldf
+
+# Business Intelligence projects
+*.rdl.data
+*.bim.layout
+*.bim_*.settings
+
+# Microsoft Fakes
+FakesAssemblies/
+
+# Node.js Tools for Visual Studio
+.ntvs_analysis.dat
+
+# Visual Studio 6 build log
+*.plg
+
+# Visual Studio 6 workspace options file
+*.opt
+
+# LightSwitch generated files
+GeneratedArtifacts/
+_Pvt_Extensions/
+ModelManifest.xml
diff --git a/BetterHttpClient.sln b/BetterHttpClient.sln
new file mode 100644
index 0000000..73bf770
--- /dev/null
+++ b/BetterHttpClient.sln
@@ -0,0 +1,58 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio 14
+VisualStudioVersion = 14.0.23107.0
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BetterHttpClient", "BetterHttpClient\BetterHttpClient.csproj", "{FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "UnitTestBetterHttpClient", "UnitTestBetterHttpClient\UnitTestBetterHttpClient.csproj", "{59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}"
+EndProject
+Global
+ GlobalSection(SolutionConfigurationPlatforms) = preSolution
+ Debug|Any CPU = Debug|Any CPU
+ Debug|ARM = Debug|ARM
+ Debug|x64 = Debug|x64
+ Debug|x86 = Debug|x86
+ Release|Any CPU = Release|Any CPU
+ Release|ARM = Release|ARM
+ Release|x64 = Release|x64
+ Release|x86 = Release|x86
+ EndGlobalSection
+ GlobalSection(ProjectConfigurationPlatforms) = postSolution
+ {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Debug|ARM.ActiveCfg = Debug|Any CPU
+ {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Debug|ARM.Build.0 = Debug|Any CPU
+ {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Debug|x64.ActiveCfg = Debug|Any CPU
+ {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Debug|x64.Build.0 = Debug|Any CPU
+ {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Debug|x86.ActiveCfg = Debug|Any CPU
+ {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Debug|x86.Build.0 = Debug|Any CPU
+ {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Release|Any CPU.Build.0 = Release|Any CPU
+ {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Release|ARM.ActiveCfg = Release|Any CPU
+ {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Release|ARM.Build.0 = Release|Any CPU
+ {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Release|x64.ActiveCfg = Release|Any CPU
+ {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Release|x64.Build.0 = Release|Any CPU
+ {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Release|x86.ActiveCfg = Release|Any CPU
+ {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Release|x86.Build.0 = Release|Any CPU
+ {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+ {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Debug|Any CPU.Build.0 = Debug|Any CPU
+ {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Debug|ARM.ActiveCfg = Debug|Any CPU
+ {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Debug|ARM.Build.0 = Debug|Any CPU
+ {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Debug|x64.ActiveCfg = Debug|Any CPU
+ {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Debug|x64.Build.0 = Debug|Any CPU
+ {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Debug|x86.ActiveCfg = Debug|Any CPU
+ {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Debug|x86.Build.0 = Debug|Any CPU
+ {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Release|Any CPU.ActiveCfg = Release|Any CPU
+ {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Release|Any CPU.Build.0 = Release|Any CPU
+ {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Release|ARM.ActiveCfg = Release|Any CPU
+ {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Release|ARM.Build.0 = Release|Any CPU
+ {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Release|x64.ActiveCfg = Release|Any CPU
+ {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Release|x64.Build.0 = Release|Any CPU
+ {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Release|x86.ActiveCfg = Release|Any CPU
+ {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Release|x86.Build.0 = Release|Any CPU
+ EndGlobalSection
+ GlobalSection(SolutionProperties) = preSolution
+ HideSolutionNode = FALSE
+ EndGlobalSection
+EndGlobal
diff --git a/BetterHttpClient/BetterHttpClient.csproj b/BetterHttpClient/BetterHttpClient.csproj
new file mode 100644
index 0000000..b74d6d3
--- /dev/null
+++ b/BetterHttpClient/BetterHttpClient.csproj
@@ -0,0 +1,66 @@
+
+
+
+
+ Debug
+ AnyCPU
+ {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}
+ Library
+ Properties
+ BetterHttpClient
+ BetterHttpClient
+ v4.0
+ 512
+
+
+
+ true
+ full
+ false
+ bin\Debug\
+ DEBUG;TRACE
+ prompt
+ 4
+
+
+ pdbonly
+ true
+ bin\Release\
+ TRACE
+ prompt
+ 4
+ bin\Release\BetterHttpClient.XML
+
+
+
+
+
+
+
+
+
+
+
+
+ Component
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/BetterHttpClient/CheckService/ProxyCheckService.cs b/BetterHttpClient/CheckService/ProxyCheckService.cs
new file mode 100644
index 0000000..15bae09
--- /dev/null
+++ b/BetterHttpClient/CheckService/ProxyCheckService.cs
@@ -0,0 +1,42 @@
+using System;
+
+namespace BetterHttpClient.CheckService
+{
+ public abstract class ProxyJudgeServiceAbstract
+ {
+ private string _myIp = null;
+ private readonly object _syncLock = new object();
+
+ public string MyIp
+ {
+ get
+ {
+ lock (_syncLock)
+ {
+ if(_myIp == null)
+ _myIp = GetMyIp();
+ }
+
+ return _myIp;
+ }
+ }
+ ///
+ /// Set number of attempts which can be made to verify proxy.
+ ///
+ public int NumberOfAttempts { get; set; }
+ ///
+ /// Abstract method. Should return true if proxy is hiding your real ip address.
+ ///
+ ///
+ ///
+ public abstract bool IsProxyAnonymous(Proxy proxy);
+ ///
+ /// Abstract method. Should return real ip addres of user (so without any proxy).
+ ///
+ ///
+ protected abstract string GetMyIp();
+ }
+ public class GetMyIpException : Exception
+ {
+ }
+}
\ No newline at end of file
diff --git a/BetterHttpClient/CheckService/ProxyJudgeProxyCheckService.cs b/BetterHttpClient/CheckService/ProxyJudgeProxyCheckService.cs
new file mode 100644
index 0000000..00d188c
--- /dev/null
+++ b/BetterHttpClient/CheckService/ProxyJudgeProxyCheckService.cs
@@ -0,0 +1,45 @@
+using System.Net;
+using System.Text;
+using System.Text.RegularExpressions;
+
+namespace BetterHttpClient.CheckService
+{
+ public class ProxyJudgeService : ProxyJudgeServiceAbstract
+ {
+ public override bool IsProxyAnonymous(Proxy proxy)
+ {
+ HttpClient client = new HttpClient(proxy, Encoding.UTF8)
+ {
+ NumberOfAttempts = NumberOfAttempts
+ };
+ string page = null;
+
+ try
+ {
+ page = client.Get("http://proxyjudge.info/");
+ }
+ catch (WebException)
+ {
+ proxy.IsOnline = false;
+ }
+
+ if (page == null)
+ return false;
+ if (page.Contains("
Proxyjudge.info") && !page.Contains(MyIp))
+ return true;
+ return false;
+ }
+
+ protected override string GetMyIp()
+ {
+ HttpClient client = new HttpClient(Encoding.UTF8);
+ string page = client.Get("http://proxyjudge.info/");
+
+ Match match = Regex.Match(page, "REMOTE_ADDR = (.*?)\\n");
+ if (!match.Success)
+ throw new GetMyIpException();
+ else
+ return match.Groups[1].Value;
+ }
+ }
+}
\ No newline at end of file
diff --git a/BetterHttpClient/HttpClient.cs b/BetterHttpClient/HttpClient.cs
new file mode 100644
index 0000000..0764aeb
--- /dev/null
+++ b/BetterHttpClient/HttpClient.cs
@@ -0,0 +1,273 @@
+using System;
+using System.Collections.Specialized;
+using System.Net;
+using System.Text;
+using BetterHttpClient.Socks;
+
+namespace BetterHttpClient
+{
+ public class HttpClient : WebClient
+ {
+ private int _numberOfAttempts = 4;
+ private TimeSpan _timeout = TimeSpan.FromSeconds(60);
+ private Proxy _proxy;
+
+ ///
+ /// Cookie container.
+ ///
+ public CookieContainer Cookies { get; set; } = new CookieContainer();
+
+ ///
+ /// Proxy which should be used for request.
+ /// Set null or proxy with type ProxyTypeEnum.None if you want to perform request without proxy.
+ ///
+ public new Proxy Proxy
+ {
+ get { return _proxy; }
+ set { _proxy = value ?? new Proxy(); }
+ }
+ ///
+ /// Timeout for request.
+ ///
+ /// Has to be greater than 5 milliseconds.
+ /// Default value: 60 seconds.
+ public TimeSpan Timeout
+ {
+ get
+ {
+ return _timeout;
+ }
+ set
+ {
+ if (value.TotalMilliseconds < 5)
+ throw new ArgumentOutOfRangeException("Timeout has to be greater or equal than 5 milliseconds.");
+ _timeout = value;
+ }
+ }
+ ///
+ /// Set number of attempts that can be made to execute request.
+ ///
+ /// Default value: 4 attempts.
+ /// Should be greater than 1
+ public int NumberOfAttempts
+ {
+ get { return _numberOfAttempts; }
+ set
+ {
+ if (value < 1)
+ throw new ArgumentOutOfRangeException("Value has to be greater than one.");
+
+ _numberOfAttempts = value;
+ }
+ }
+
+ ///
+ /// Set User-Agent header.
+ ///
+ /// Default value: "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0"
+ public string UserAgent { get; set; } = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0";
+ ///
+ /// Set Accept header.
+ ///
+ /// Default value: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+ public string Accept { get; set; } = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
+ ///
+ /// Set Referer header.
+ ///
+ /// Default value: null
+ public string Referer { get; set; }
+ ///
+ /// Set Accept-Language header.
+ ///
+ /// Default value: "en-US;q=0.7,en;q=0.3"
+ public string AcceptLanguage { get; set; } = "en-US;q=0.7,en;q=0.3";
+ ///
+ /// Set default Accept-Encoding header.
+ ///
+ /// Default value: "gzip, deflate"
+ public string AcceptEncoding { get; set; } = "gzip, deflate";
+ ///
+ /// Set encoding for request.
+ ///
+ /// Default value: UTF-8
+ public new Encoding Encoding
+ {
+ get { return base.Encoding; }
+ set { base.Encoding = value; }
+ }
+ ///
+ /// Enabled automatic redirect. Default: true
+ ///
+ public bool AllowAutoRedirect { get; set; } = true;
+
+ ///
+ /// 内容类型
+ ///
+ public string ContentType { get; set; }
+
+ ///
+ /// Headers collection that will be added to each request
+ ///
+ public NameValueCollection CustomHeaders { get; set; }
+ public HttpClient(Proxy proxy) : this(proxy, Encoding.UTF8) { }
+
+ public HttpClient() : this(new Proxy(), Encoding.UTF8) { }
+ public HttpClient(Encoding encoding) : this(new Proxy(), encoding) { }
+ public HttpClient(Proxy proxy, Encoding encoding)
+ {
+ Encoding = encoding;
+ Proxy = proxy;
+ }
+
+ protected override WebRequest GetWebRequest(Uri address)
+ {
+ WebRequest request = null;
+
+ if (Proxy.ProxyType != ProxyTypeEnum.Socks && Proxy.ProxyType != ProxyTypeEnum.Socks4)
+ {
+ request = base.GetWebRequest(address);
+ request.ContentType= string.IsNullOrEmpty(ContentType) ? base.GetWebRequest(address).ContentType : ContentType;
+ }
+ else if (Proxy.ProxyType == ProxyTypeEnum.Socks)
+ {
+ request = SocksHttpWebRequest.Create(address);
+ request.Method = base.GetWebRequest(address).Method;
+ request.ContentLength = base.GetWebRequest(address).ContentLength;
+ request.ContentType = string.IsNullOrEmpty(ContentType)?base.GetWebRequest(address).ContentType:ContentType;
+ }
+ else if (Proxy.ProxyType == ProxyTypeEnum.Socks4)
+ {
+ request = Socks4HttpWebRequest.Create(address);
+ request.Method = base.GetWebRequest(address).Method;
+ request.ContentLength = base.GetWebRequest(address).ContentLength;
+ request.ContentType = string.IsNullOrEmpty(ContentType) ? base.GetWebRequest(address).ContentType : ContentType;
+ }
+
+
+ request.Headers.Add("Cookie", Cookies.GetCookieHeader(address));
+ request.Headers.Add("Accept-Language", AcceptLanguage);
+ request.Headers.Add("Accept-Encoding", AcceptEncoding);
+ if (CustomHeaders != null)
+ {
+ foreach (string key in CustomHeaders.AllKeys)
+ {
+ request.Headers.Add(key, CustomHeaders[key]);
+ }
+ }
+
+
+ if (Proxy.ProxyType != ProxyTypeEnum.Socks && Proxy.ProxyType != ProxyTypeEnum.Socks4)
+ {
+ var httpRequest = (request as HttpWebRequest);
+ httpRequest.UserAgent = UserAgent;
+ httpRequest.Accept = Accept;
+ httpRequest.Referer = Referer;
+ httpRequest.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip;
+ httpRequest.AllowAutoRedirect = AllowAutoRedirect;
+ }
+ else if (Proxy.ProxyType == ProxyTypeEnum.Socks)
+ {
+ var socksRequest = (request as SocksHttpWebRequest);
+ socksRequest.UserAgent = UserAgent;
+ socksRequest.Accept = Accept;
+ socksRequest.Referer = Referer;
+ socksRequest.AllowAutoRedirect = AllowAutoRedirect;
+ }
+ else if (Proxy.ProxyType == ProxyTypeEnum.Socks4)
+ {
+ var socks4Request = (request as Socks4HttpWebRequest);
+ socks4Request.UserAgent = UserAgent;
+ socks4Request.Accept = Accept;
+ socks4Request.Referer = Referer;
+ socks4Request.AllowAutoRedirect = AllowAutoRedirect;
+ }
+
+ request.Timeout = (int) Timeout.TotalMilliseconds;
+ request.Proxy = Proxy.ProxyItem;
+
+ return request;
+ }
+
+ protected override WebResponse GetWebResponse(WebRequest request)
+ {
+ var response = base.GetWebResponse(request);
+ try
+ {
+ string setCookies = response.Headers["Set-Cookie"];
+ Cookies.SetCookies(request.RequestUri, setCookies);
+ }
+ catch (Exception)
+ {
+
+ }
+
+ return response;
+ }
+ ///
+ /// Execute GET request.
+ ///
+ ///
+ ///
+ public string Get(string url)
+ {
+ return Encoding.GetString(DownloadBytes(url, null));
+ }
+ ///
+ /// Execute POST request.
+ ///
+ ///
+ ///
+ ///
+ public string Post(string url, NameValueCollection data)
+ {
+ return Encoding.GetString(DownloadBytes(url, data));
+ }
+
+ ///
+ /// Execute GET request.
+ ///
+ ///
+ ///
+ public byte[] DownloadBytes(string url)
+ {
+ return DownloadBytes(url, null);
+ }
+ ///
+ /// Excecute POST request.
+ ///
+ ///
+ ///
+ ///
+ public byte[] DownloadBytes(string url, NameValueCollection data)
+ {
+ int counter = 0;
+ WebException lastWebException = null;
+ bool unkownProxy = Proxy.ProxyType == ProxyTypeEnum.Unknown;
+
+ while (counter < NumberOfAttempts + (NumberOfAttempts < 2 && unkownProxy ? 1 : 0)) // min two try for unkonwn proxy type
+ {
+ try
+ {
+ if (unkownProxy && counter % 3 == 0) // every 3rd try is as http proxy
+ Proxy.ProxyType = ProxyTypeEnum.Http;
+ else if (unkownProxy && counter % 3 == 1)
+ Proxy.ProxyType = ProxyTypeEnum.Socks;
+ else if (unkownProxy && counter % 3 == 2)
+ Proxy.ProxyType = ProxyTypeEnum.Socks4;
+ byte[] result = data == null ? Encoding.GetBytes(DownloadString(url)) : UploadValues(url, data);
+ return result;
+ }
+ catch (WebException e)
+ {
+ lastWebException = e;
+ counter++;
+ }
+ }
+
+ if (unkownProxy)
+ Proxy.ProxyType = ProxyTypeEnum.Unknown;
+ // ReSharper disable once PossibleNullReferenceException
+ throw lastWebException;
+ }
+ }
+}
diff --git a/BetterHttpClient/Properties/AssemblyInfo.cs b/BetterHttpClient/Properties/AssemblyInfo.cs
new file mode 100644
index 0000000..30a7f32
--- /dev/null
+++ b/BetterHttpClient/Properties/AssemblyInfo.cs
@@ -0,0 +1,39 @@
+using System.Reflection;
+using System.Runtime.InteropServices;
+
+// General Information about an assembly is controlled through the following
+// set of attributes. Change these attribute values to modify the information
+// associated with an assembly.
+
+[assembly: AssemblyTitle("HttpClient")]
+[assembly: AssemblyDescription("")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("")]
+[assembly: AssemblyProduct("HttpClient")]
+[assembly: AssemblyCopyright("Copyright © 2015")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+// Setting ComVisible to false makes the types in this assembly not visible
+// to COM components. If you need to access a type in this assembly from
+// COM, set the ComVisible attribute to true on that type.
+
+[assembly: ComVisible(false)]
+
+// The following GUID is for the ID of the typelib if this project is exposed to COM
+
+[assembly: Guid("fbc5f420-3f80-4a20-a9ec-b6574e18a1d3")]
+
+// Version information for an assembly consists of the following four values:
+//
+// Major Version
+// Minor Version
+// Build Number
+// Revision
+//
+// You can specify all the values or you can default the Build and Revision Numbers
+// by using the '*' as shown below:
+// [assembly: AssemblyVersion("1.0.*")]
+
+[assembly: AssemblyVersion("1.0.7")]
+[assembly: AssemblyFileVersion("1.0.0")]
\ No newline at end of file
diff --git a/BetterHttpClient/Proxy.cs b/BetterHttpClient/Proxy.cs
new file mode 100644
index 0000000..5bf4617
--- /dev/null
+++ b/BetterHttpClient/Proxy.cs
@@ -0,0 +1,86 @@
+using System;
+using System.Net;
+using BetterHttpClient.CheckService;
+
+namespace BetterHttpClient
+{
+ public class Proxy : ICloneable
+ {
+ private volatile bool _isBusy = false;
+ private bool _isAnonymous;
+ private bool _isChecked = false;
+
+ ///
+ /// Check if proxy is busy.
+ ///
+ public bool IsBusy
+ {
+ get { return _isBusy; }
+ internal set { _isBusy = value; }
+ }
+
+
+ public bool IsOnline { get; set; } = true;
+ public ProxyTypeEnum ProxyType { get; internal set; }
+ internal WebProxy ProxyItem { get; set; }
+
+ public Proxy()
+ {
+ ProxyItem = new WebProxy();
+ ProxyType = ProxyTypeEnum.None;
+ }
+
+ public Proxy(string address)
+ {
+ ProxyItem = new WebProxy(address);
+ }
+
+ public Proxy(string ip, int port)
+ {
+ ProxyItem = new WebProxy(ip, port);
+ }
+
+ public Proxy(string ip, int port, ProxyTypeEnum proxyType)
+ {
+ ProxyItem = new WebProxy(ip, port);
+ ProxyType = proxyType;
+ }
+
+ ///
+ /// Returns true if proxy can hide your ip address
+ ///
+ public bool IsAnonymous(ProxyJudgeService service)
+ {
+ if (_isChecked)
+ return _isAnonymous;
+
+ _isAnonymous = service.IsProxyAnonymous(this);
+ _isChecked = true;
+ IsOnline = _isAnonymous;
+ return _isAnonymous;
+ }
+
+ public object Clone()
+ {
+ var proxy = new Proxy
+ {
+ _isAnonymous = _isAnonymous,
+ _isBusy = _isBusy,
+ _isChecked = _isChecked,
+ IsOnline = IsOnline,
+ ProxyItem = ProxyItem,
+ ProxyType = ProxyType
+ };
+ return proxy;
+ }
+ }
+
+ public enum ProxyTypeEnum
+ {
+ Unknown,
+ None,
+ Http,
+ Socks4,
+ Socks
+ }
+}
\ No newline at end of file
diff --git a/BetterHttpClient/ProxyManager.cs b/BetterHttpClient/ProxyManager.cs
new file mode 100644
index 0000000..2b52bad
--- /dev/null
+++ b/BetterHttpClient/ProxyManager.cs
@@ -0,0 +1,428 @@
+using System;
+using System.Collections.Generic;
+using System.Collections.Specialized;
+using System.IO;
+using System.Linq;
+using System.Net;
+using System.Text;
+using System.Threading;
+using BetterHttpClient.CheckService;
+
+namespace BetterHttpClient
+{
+ public class ProxyManager
+ {
+ private List _proxies = new List();
+ private string _requiredString = string.Empty;
+ private int _numberOfAttemptsPerRequest;
+ private TimeSpan _timeout = TimeSpan.FromSeconds(10);
+ private int _numberOfAttempts = 4;
+ private ProxyJudgeService _proxyJudgeService;
+
+ ///
+ /// Set User-Agent header.
+ ///
+ /// Default value: "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0"
+ public string UserAgent { get; set; } = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0";
+ ///
+ /// Set Accept header.
+ ///
+ /// Default value: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+ public string Accept { get; set; } = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
+ ///
+ /// Set Referer header.
+ ///
+ /// Default value: null
+ public string Referer { get; set; }
+ ///
+ /// Set Accept-Language header.
+ ///
+ /// Default value: "en-US;q=0.7,en;q=0.3"
+ public string AcceptLanguage { get; set; } = "en-US;q=0.7,en;q=0.3";
+ ///
+ /// Set default Accept-Encoding header.
+ ///
+ /// Default value: "gzip, deflate"
+ public string AcceptEncoding { get; set; } = "gzip, deflate";
+
+ ///
+ /// Set proxy check service.
+ /// Must derive from ProxyCheckService class.
+ ///
+ public ProxyJudgeService ProxyJudgeService
+ {
+ get { return _proxyJudgeService; }
+ set
+ {
+ if(value == null)
+ throw new ArgumentNullException();
+ _proxyJudgeService = value;
+ }
+ }
+ ///
+ /// Downloaded page has to contain this tring.
+ /// It helps to check if returned page is the page which we watned to receive.
+ /// Proxy sometimes are returing some other pages.
+ /// Default value string.Empty
+ ///
+ public string RequiredString
+ {
+ get { return _requiredString; }
+ set { _requiredString = value ?? string.Empty; }
+ }
+ ///
+ /// Set encoding for request.
+ ///
+ /// Default value: UTF-8
+ public Encoding Encoding { get; set; } = Encoding.UTF8;
+ ///
+ /// Set this to true if you want to use only anonymous proxies.
+ ///
+ /// Default value: false
+ public bool AnonymousProxyOnly { get; } = false;
+ ///
+ /// Timeout for request.
+ ///
+ /// Has to be greater than 5 milliseconds.
+ /// Default value: 10 seconds.
+ public TimeSpan Timeout
+ {
+ get
+ {
+ return _timeout;
+ }
+ set
+ {
+ if (value.TotalMilliseconds < 5)
+ throw new ArgumentOutOfRangeException("Timeout has to be greater or equal than 5 milliseconds.");
+ _timeout = value;
+ }
+ }
+ ///
+ /// Set how many attempts can be made to execute request on one proxy.
+ /// Default value is default value is equal 4
+ ///
+ public int NumberOfAttempts
+ {
+ get { return _numberOfAttempts; }
+ set
+ {
+ if (value < 1)
+ throw new ArgumentOutOfRangeException("Value has to be greater than one.");
+
+ _numberOfAttempts = value;
+ _proxyJudgeService.NumberOfAttempts = _numberOfAttempts;
+ }
+ }
+
+ /// Proxy list
+ /// Set true if you want to filter proxy list and use only anonymous only
+ /// Proxy judge service is used to determine proxy anonymity level
+ ///
+ public ProxyManager(IEnumerable proxies, bool anonymousOnly, ProxyJudgeService proxyJudgeService) :
+ this(ParseProxies(proxies), anonymousOnly, proxyJudgeService)
+ {
+ }
+
+ /// Proxy list
+ /// Set true if you want to filter proxy list and use only anonymous only
+ /// Proxy judge service is used to determine proxy anonymity level
+ ///
+ public ProxyManager(IEnumerable proxies, bool anonymousOnly, ProxyJudgeService proxyJudgeService)
+ {
+ if (proxies == null || proxyJudgeService == null)
+ throw new ArgumentNullException();
+ foreach (Proxy proxy in proxies)
+ {
+ try
+ {
+ _proxies.Add(proxy);
+ }
+ catch (UriFormatException)
+ {
+ // parsing exception
+ }
+ }
+ AnonymousProxyOnly = anonymousOnly;
+ _proxyJudgeService = proxyJudgeService;
+ _numberOfAttemptsPerRequest = _proxies.Count + 1;
+ _proxyJudgeService.NumberOfAttempts = _numberOfAttempts;
+ }
+ public ProxyManager(string file) : this(File.ReadLines(file), false, new ProxyJudgeService()) { }
+ public ProxyManager(string file, bool anonymousOnly) : this(File.ReadLines(file), anonymousOnly, new ProxyJudgeService()) { }
+ public ProxyManager(string file, bool anonymousOnly, ProxyJudgeService service) : this(File.ReadLines(file), anonymousOnly, service) { }
+
+ ///
+ /// Downloads url using GET.
+ ///
+ /// Url of webpage
+ /// Cookies for request. Left null if you don't want to use cookies
+ /// Specify custom headers for this request
+ ///
+ /// Page has returned 404 not found
+ public string GetPage(string url, string requiredString = null, CookieContainer cookies = null, NameValueCollection customHeaders = null)
+ {
+ return PostPage(url, null, requiredString, cookies, customHeaders);
+ }
+
+ ///
+ /// Downloads url using POST.
+ ///
+ /// Url of webpage
+ /// Post values
+ /// Cookies for request. Left null if you don't want to use cookies
+ /// Specify custom headers for this request
+ /// Page has returned 404 not found
+ ///
+ public string PostPage(string url, NameValueCollection data, string requiredString = null, CookieContainer cookies = null, NameValueCollection customHeaders = null)
+ {
+ if (requiredString == null)
+ requiredString = RequiredString;
+
+ string page = null;
+ int limit = 0;
+
+ do
+ {
+ Proxy proxy = GetAvalibleProxy();
+
+ try
+ {
+ if (AnonymousProxyOnly)
+ {
+ if (AnonymousProxyOnly && !proxy.IsAnonymous(ProxyJudgeService))
+ {
+ continue;
+ }
+ }
+
+ var bytes = DownloadBytes(url, data, proxy, cookies, customHeaders);
+
+ if (bytes != null)
+ {
+ page = Encoding.GetString(bytes);
+ if (!page.Contains(requiredString))
+ {
+ proxy.IsOnline = false;
+ }
+ else
+ {
+ return page;
+ }
+ }
+ }
+ finally
+ {
+ limit++;
+ proxy.IsBusy = false;
+ }
+
+ limit++;
+ } while (limit < _numberOfAttemptsPerRequest);
+
+ throw new AllProxiesBannedException();
+ }
+
+ ///
+ /// Downloads url using POST.
+ ///
+ /// Url of webpage
+ /// Post values
+ /// Cookies for request. Left null if you don't want to use cookies
+ /// Specify custom headers for this request
+ /// Page has returned 404 not found
+ ///
+ public byte[] DownloadBytes(string url, NameValueCollection data, CookieContainer cookies = null, NameValueCollection customHeaders = null)
+ {
+ int limit = 0;
+
+ do
+ {
+ Proxy proxy = GetAvalibleProxy();
+
+ try
+ {
+ if (AnonymousProxyOnly && !proxy.IsAnonymous(ProxyJudgeService))
+ {
+ continue;
+ }
+
+ byte[] result = DownloadBytes(url, data, proxy, cookies, customHeaders);
+
+ if (result != null)
+ {
+ return result;
+ }
+ }
+ finally
+ {
+ limit++;
+ proxy.IsBusy = false;
+ }
+
+ limit++;
+ } while (limit < _numberOfAttemptsPerRequest);
+
+ throw new AllProxiesBannedException();
+ }
+
+ ///
+ /// Downloads url using GET.
+ ///
+ ///
+ ///
+ /// Specify custom headers for this request
+ ///
+ /// Page has returned 404 not found
+ public byte[] DownloadBytes(string url, CookieContainer cookies = null, NameValueCollection customHeaders = null)
+ {
+ return DownloadBytes(url, null, cookies, customHeaders);
+ }
+ ///
+ /// Returns first free (but busy) and working proxy.
+ ///
+ ///
+ /// All proxies are banned. You can't make request.
+ public Proxy GetAvalibleProxy()
+ {
+ lock (_proxies)
+ {
+ Proxy selectedProxy = null;
+
+ do
+ {
+ if (_proxies.Count(t => t.IsOnline) == 0)
+ throw new AllProxiesBannedException();
+
+ selectedProxy = _proxies.FirstOrDefault(t => t.IsOnline && !t.IsBusy);
+ if (selectedProxy == null)
+ {
+ Thread.Sleep(35);
+ }
+
+ } while (selectedProxy == null);
+
+ selectedProxy.IsBusy = true;
+ return selectedProxy;
+ }
+ }
+ ///
+ /// Sets all proxies IsOnline property to true.
+ ///
+ public void SetAllProxyAsOnline()
+ {
+ lock (_proxies)
+ {
+ _proxies.ForEach(t => t.IsOnline = true);
+ }
+ }
+
+ ///
+ /// Return all proxies
+ ///
+ ///
+ public List GetAllProxies()
+ {
+ lock(_proxies)
+ return CloneProxyList(_proxies);
+ }
+
+
+ private List CloneProxyList(List proxyInput)
+ {
+ List proxies = new List(proxyInput.Count);
+ proxies.AddRange(proxyInput.Select(proxy => (Proxy) proxy.Clone()));
+ return proxies;
+ }
+
+ private byte[] DownloadBytes(string url, NameValueCollection data, Proxy proxy, CookieContainer cookies = null, NameValueCollection customHeaders = null)
+ {
+ HttpClient client = CreateHttpClient(customHeaders);
+ client.Proxy = proxy;
+ if (cookies != null) client.Cookies = cookies;
+
+ try
+ {
+ return client.DownloadBytes(url, data);
+ }
+ catch (WebException e)
+ {
+ if (e.Response != null && (e.Response as HttpWebResponse).StatusCode == HttpStatusCode.NotFound)
+ throw new WebPageNotFoundException();
+ proxy.IsOnline = false;
+ }
+
+ return null;
+ }
+ private HttpClient CreateHttpClient(NameValueCollection customHeaders = null)
+ {
+ HttpClient client = new HttpClient(Encoding)
+ {
+ Accept = Accept,
+ AcceptEncoding = AcceptEncoding,
+ AcceptLanguage = AcceptLanguage,
+ NumberOfAttempts = NumberOfAttempts,
+ UserAgent = UserAgent,
+ Referer = Referer,
+ Timeout = Timeout
+ };
+
+ if (customHeaders != null)
+ {
+ if (customHeaders.AllKeys.Contains("Accept"))
+ {
+ client.Accept = customHeaders["Accept"];
+ customHeaders.Remove("Accept");
+ }
+ if (customHeaders.AllKeys.Contains("Accept-Encoding"))
+ {
+ client.AcceptEncoding = customHeaders["Accept-Encoding"];
+ customHeaders.Remove("Accept-Encoding");
+ }
+ if (customHeaders.AllKeys.Contains("Accept-Language"))
+ {
+ client.AcceptLanguage = customHeaders["Accept-Language"];
+ customHeaders.Remove("Accept-Language");
+ }
+ if (customHeaders.AllKeys.Contains("User-Agent"))
+ {
+ client.UserAgent = customHeaders["User-Agente"];
+ customHeaders.Remove("User-Agent");
+ }
+ if (customHeaders.AllKeys.Contains("Referer"))
+ {
+ client.Referer = customHeaders["Referer"];
+ customHeaders.Remove("Referer");
+ }
+
+ client.CustomHeaders = customHeaders;
+ }
+
+ return client;
+ }
+ private static IEnumerable ParseProxies(IEnumerable proxies)
+ {
+ IEnumerable proxyParsed = proxies.Select(t =>
+ {
+ try
+ {
+ Proxy proxy = new Proxy(t);
+ return proxy;
+ }
+ catch (UriFormatException)
+ {
+ // parsing exception
+ return null;
+ }
+ }).Where(t => t != null);
+ return proxyParsed;
+ }
+ }
+
+ public class AllProxiesBannedException : Exception
+ {
+ }
+ public class WebPageNotFoundException : Exception
+ {
+ }
+}
diff --git a/BetterHttpClient/Socks/NeverEndingStream.cs b/BetterHttpClient/Socks/NeverEndingStream.cs
new file mode 100644
index 0000000..cdaf7c6
--- /dev/null
+++ b/BetterHttpClient/Socks/NeverEndingStream.cs
@@ -0,0 +1,22 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BetterHttpClient.Socks
+{
+ public class NeverEndingStream : MemoryStream
+ {
+ public override void Close()
+ {
+ // Ignore
+ }
+
+ public void ForceClose()
+ {
+ base.Close();
+ }
+ }
+}
diff --git a/BetterHttpClient/Socks/Socks4HttpWebRequest.cs b/BetterHttpClient/Socks/Socks4HttpWebRequest.cs
new file mode 100644
index 0000000..74538f4
--- /dev/null
+++ b/BetterHttpClient/Socks/Socks4HttpWebRequest.cs
@@ -0,0 +1,448 @@
+using BetterHttpClient.Socks.Extensions;
+using System;
+using System.Collections.Generic;
+using System.Collections.Specialized;
+using System.IO;
+using System.Linq;
+using System.Net;
+using System.Net.Security;
+using System.Net.Sockets;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BetterHttpClient.Socks
+{
+ internal class Socks4HttpWebRequest: WebRequest
+ {
+ private static readonly StringCollection validHttpVerbs = new StringCollection { "GET", "HEAD", "POST", "PUT", "DELETE", "TRACE", "OPTIONS" };
+ private WebHeaderCollection _HttpRequestHeaders;
+ private string _method;
+ private byte[] _requestContentBuffer;
+ private NeverEndingStream _requestContentStream;
+ private SocksHttpWebResponse _response;
+
+ private Socks4HttpWebRequest(Uri requestUri)
+ {
+ RequestUri = requestUri;
+ }
+
+ public override int Timeout { get; set; }
+
+ public string UserAgent
+ {
+ get { return _HttpRequestHeaders["User-Agent"]; }
+ set { SetSpecialHeaders("User-Agent", value ?? string.Empty); }
+ }
+
+ public string Referer
+ {
+ get { return _HttpRequestHeaders["Referer"]; }
+ set { SetSpecialHeaders("Referer", value ?? string.Empty); }
+ }
+
+ public string Accept
+ {
+ get { return _HttpRequestHeaders["Accept"]; }
+ set { SetSpecialHeaders("Accept", value ?? string.Empty); }
+ }
+
+ public DecompressionMethods AutomaticDecompression
+ {
+ get
+ {
+ var result = DecompressionMethods.None;
+ string encoding = _HttpRequestHeaders["Accept-Encoding"] ?? string.Empty;
+ foreach (string value in encoding.Split(','))
+ {
+ switch (value.Trim())
+ {
+ case "gzip":
+ result |= DecompressionMethods.GZip;
+ break;
+
+ case "deflate":
+ result |= DecompressionMethods.Deflate;
+ break;
+ }
+ }
+
+ return result;
+ }
+ set
+ {
+ string result = string.Empty;
+ if ((value & DecompressionMethods.GZip) != 0)
+ result = "gzip";
+ if ((value & DecompressionMethods.Deflate) != 0)
+ {
+ if (!string.IsNullOrEmpty(result))
+ result += ", ";
+ result += "deflate";
+ }
+
+ SetSpecialHeaders("Accept-Encoding", result);
+ }
+ }
+
+ public override Uri RequestUri { get; }
+
+ public override IWebProxy Proxy { get; set; }
+
+ public override WebHeaderCollection Headers
+ {
+ get { return _HttpRequestHeaders ?? (_HttpRequestHeaders = new WebHeaderCollection()); }
+ set
+ {
+ if (RequestSubmitted)
+ {
+ throw new InvalidOperationException("This operation cannot be performed after the request has been submitted.");
+ }
+ _HttpRequestHeaders = value;
+ }
+ }
+
+ public bool RequestSubmitted { get; private set; }
+
+ public override string Method
+ {
+ get { return _method ?? "GET"; }
+ set
+ {
+ if (validHttpVerbs.Contains(value))
+ {
+ _method = value;
+ }
+ else
+ {
+ throw new ArgumentOutOfRangeException(nameof(value), $"'{value}' is not a known HTTP verb.");
+ }
+ }
+ }
+
+ public override long ContentLength { get; set; }
+
+ public override string ContentType { get; set; }
+ public bool AllowAutoRedirect { get; set; } = true;
+
+ public override WebResponse GetResponse()
+ {
+ if (Proxy == null)
+ {
+ throw new InvalidOperationException("Proxy property cannot be null.");
+ }
+ if (string.IsNullOrEmpty(Method))
+ {
+ throw new InvalidOperationException("Method has not been set.");
+ }
+
+ if (RequestSubmitted)
+ {
+ return _response;
+ }
+ _response = InternalGetResponse();
+ RequestSubmitted = true;
+ return _response;
+ }
+
+ public override Stream GetRequestStream()
+ {
+ if (RequestSubmitted)
+ {
+ throw new InvalidOperationException("This operation cannot be performed after the request has been submitted.");
+ }
+
+ if (_requestContentBuffer == null)
+ {
+ if (ContentLength == 0)
+ {
+ _requestContentStream = new NeverEndingStream();
+ return _requestContentStream;
+ }
+
+ _requestContentBuffer = new byte[ContentLength];
+ }
+ else if (ContentLength == default(long))
+ {
+ _requestContentBuffer = new byte[int.MaxValue];
+ }
+ else if (_requestContentBuffer.Length != ContentLength)
+ {
+ Array.Resize(ref _requestContentBuffer, (int)ContentLength);
+ }
+ return new MemoryStream(_requestContentBuffer);
+ }
+
+ public override IAsyncResult BeginGetResponse(AsyncCallback callback, object state)
+ {
+ if (Proxy == null)
+ {
+ throw new InvalidOperationException("Proxy property cannot be null.");
+ }
+ if (string.IsNullOrEmpty(Method))
+ {
+ throw new InvalidOperationException("Method has not been set.");
+ }
+
+ var task = Task.Factory.StartNew(() => {
+ if (RequestSubmitted)
+ {
+ return _response;
+ }
+ _response = InternalGetResponse();
+ RequestSubmitted = true;
+ return _response;
+ });
+
+ //var task = Task.Run(() =>
+ //{
+ // if (RequestSubmitted)
+ // {
+ // return _response;
+ // }
+ // _response = InternalGetResponse();
+ // RequestSubmitted = true;
+ // return _response;
+ //});
+
+ return task.AsApm(callback, state);
+ }
+
+ public override WebResponse EndGetResponse(IAsyncResult asyncResult)
+ {
+ var task = asyncResult as Task;
+
+ try
+ {
+ return task.Result;
+ }
+ catch (AggregateException ex)
+ {
+ if (ex.InnerException is IOException || ex.InnerException is System.Net.Sockets.SocketException)
+ throw new WebException("Proxy error " + ex.InnerException.Message, ex.InnerException, WebExceptionStatus.ConnectFailure,
+ SocksHttpWebResponse.CreateErrorResponse(HttpStatusCode.GatewayTimeout));
+ throw ex.InnerException;
+ }
+ }
+
+ public new static WebRequest Create(string requestUri)
+ {
+ return new Socks4HttpWebRequest(new Uri(requestUri));
+ }
+
+ public new static WebRequest Create(Uri requestUri)
+ {
+ return new Socks4HttpWebRequest(requestUri);
+ }
+
+ private void SetSpecialHeaders(string headerName, string value)
+ {
+ _HttpRequestHeaders.Remove(headerName);
+ if (value.Length != 0)
+ {
+ _HttpRequestHeaders.Add(headerName, value);
+ }
+ }
+
+ private string BuildHttpRequestMessage(Uri requestUri)
+ {
+ if (RequestSubmitted)
+ {
+ throw new InvalidOperationException("This operation cannot be performed after the request has been submitted.");
+ }
+
+ // See if we have a stream instead of byte array
+ if (_requestContentBuffer == null && _requestContentStream != null)
+ {
+ _requestContentBuffer = _requestContentStream.ToArray();
+ _requestContentStream.ForceClose();
+ _requestContentStream.Dispose();
+ _requestContentStream = null;
+ ContentLength = _requestContentBuffer.Length;
+ }
+
+ var message = new StringBuilder();
+ message.AppendFormat("{0} {1} HTTP/1.1\r\nHost: {2}\r\n", Method, requestUri, requestUri.Host);
+
+ Headers.Set(HttpRequestHeader.Connection, "close");
+
+ // add the headers
+ foreach (var key in Headers.Keys)
+ {
+ string value = Headers[key.ToString()];
+ if (!string.IsNullOrEmpty(value))
+ message.AppendFormat("{0}: {1}\r\n", key, value);
+ }
+
+ if (!string.IsNullOrEmpty(ContentType))
+ {
+ message.AppendFormat("Content-Type: {0}\r\n", ContentType);
+ }
+ if (ContentLength > 0)
+ {
+ message.AppendFormat("Content-Length: {0}\r\n", ContentLength);
+ }
+
+ // add a blank line to indicate the end of the headers
+ message.Append("\r\n");
+
+ // add content
+ if (_requestContentBuffer != null && _requestContentBuffer.Length > 0)
+ {
+ using (var stream = new MemoryStream(_requestContentBuffer, false))
+ {
+ using (var reader = new StreamReader(stream))
+ {
+ message.Append(reader.ReadToEnd());
+ }
+ }
+ }
+ else if (_requestContentStream != null)
+ {
+ _requestContentStream.Position = 0;
+
+ using (var reader = new StreamReader(_requestContentStream))
+ {
+ message.Append(reader.ReadToEnd());
+ }
+
+
+ _requestContentStream.ForceClose();
+ _requestContentStream.Dispose();
+ }
+
+ return message.ToString();
+ }
+
+ private SocksHttpWebResponse InternalGetResponse()
+ {
+ Uri requestUri = RequestUri;
+
+ int redirects = 0;
+ const int maxAutoredirectCount = 10;
+ while (redirects++ < maxAutoredirectCount)
+ {
+ // Loop while redirecting
+
+ var proxyUri = Proxy.GetProxy(requestUri);
+ var ipAddress = GetProxyIpAddress(proxyUri);
+ var response = new List();
+
+ using (var client = new TcpClient(ipAddress.ToString(), proxyUri.Port))
+ {
+ int timeout = Timeout;
+ if (timeout == 0)
+ timeout = 30 * 1000;
+ client.ReceiveTimeout = timeout;
+ client.SendTimeout = timeout;
+ var networkStream = client.GetStream();
+ // auth
+ var buf = new byte[300];
+ int index = 0;
+ buf[index++] = 0x04; // Version
+ buf[index++] = 0x01; // NMETHODS
+ var destIP = Dns.GetHostEntry(requestUri.DnsSafeHost).AddressList[0];
+ var rawBytes = destIP.GetAddressBytes();
+ var portBytes = BitConverter.GetBytes(Uri.UriSchemeHttps == requestUri.Scheme ? 443 : 80);
+ for (var i = portBytes.Length - 3; i >= 0; i--)
+ buf[index++] = portBytes[i];
+
+ rawBytes.CopyTo(buf, index);
+ index += (ushort)rawBytes.Length;
+ networkStream.Write(buf, 0, index);
+
+ networkStream.Read(buf, 0, 2);
+ if (buf[0] != 0)
+ {
+ throw new IOException("Invalid Socks Version");
+ }
+ if (buf[1] == 91|| buf[1] == 92|| buf[1] == 93)
+ {
+ throw new IOException("Socks Server does not support no-auth");
+ }
+ if (buf[1] != 90)
+ {
+ throw new Exception("Socks Server did choose bogus auth");
+ }
+ networkStream.Read(buf, 0, 2);
+ var rport = (ushort)IPAddress.NetworkToHostOrder((short)BitConverter.ToUInt16(buf, 0));
+
+ var rdest = string.Empty;
+ networkStream.Read(buf, 0, 4);
+ var v4 = BitConverter.ToUInt32(buf, 0);
+ rdest = new IPAddress(v4).ToString();
+
+
+ Stream readStream = null;
+ if (Uri.UriSchemeHttps == requestUri.Scheme)
+ {
+ var ssl = new SslStream(networkStream);
+ ssl.AuthenticateAsClient(requestUri.DnsSafeHost);
+ readStream = ssl;
+ }
+ else
+ {
+ readStream = networkStream;
+ }
+
+ string requestString = BuildHttpRequestMessage(requestUri);
+
+ var request = Encoding.ASCII.GetBytes(requestString);
+ readStream.Write(request, 0, request.Length);
+ readStream.Flush();
+
+ var buffer = new byte[client.ReceiveBufferSize];
+
+ var readlen = 0;
+ do
+ {
+ readlen = readStream.Read(buffer, 0, buffer.Length);
+ response.AddRange(buffer.Take(readlen));
+ } while (readlen != 0);
+
+ readStream.Close();
+ }
+
+ var webResponse = new SocksHttpWebResponse(requestUri, response.ToArray());
+
+ if (webResponse.StatusCode == HttpStatusCode.Moved || webResponse.StatusCode == HttpStatusCode.MovedPermanently)
+ {
+ string redirectUrl = webResponse.Headers["Location"];
+ if (string.IsNullOrEmpty(redirectUrl))
+ throw new WebException("Missing location for redirect");
+
+ requestUri = new Uri(requestUri, redirectUrl);
+ if (AllowAutoRedirect)
+ {
+ continue;
+ }
+ return webResponse;
+ }
+
+ if ((int)webResponse.StatusCode < 200 || (int)webResponse.StatusCode > 299)
+ throw new WebException(webResponse.StatusDescription, null, WebExceptionStatus.UnknownError, webResponse);
+
+ return webResponse;
+ }
+
+ throw new WebException("Too many redirects", null, WebExceptionStatus.ProtocolError, SocksHttpWebResponse.CreateErrorResponse(HttpStatusCode.BadRequest));
+ }
+
+ private static IPAddress GetProxyIpAddress(Uri proxyUri)
+ {
+ IPAddress ipAddress;
+ if (!IPAddress.TryParse(proxyUri.Host, out ipAddress))
+ {
+ try
+ {
+ return Dns.GetHostEntry(proxyUri.Host).AddressList[0];
+ }
+ catch (Exception e)
+ {
+ throw new InvalidOperationException($"Unable to resolve proxy hostname '{proxyUri.Host}' to a valid IP address.", e);
+ }
+ }
+ return ipAddress;
+ }
+
+ }
+}
diff --git a/BetterHttpClient/Socks/SocksHttpWebRequest.cs b/BetterHttpClient/Socks/SocksHttpWebRequest.cs
new file mode 100644
index 0000000..aec13d4
--- /dev/null
+++ b/BetterHttpClient/Socks/SocksHttpWebRequest.cs
@@ -0,0 +1,489 @@
+using System;
+using System.Collections.Generic;
+using System.Collections.Specialized;
+using System.IO;
+using System.Linq;
+using System.Net;
+using System.Net.Security;
+using System.Net.Sockets;
+using System.Text;
+using System.Threading.Tasks;
+using BetterHttpClient.Socks.Extensions;
+
+namespace BetterHttpClient.Socks
+{
+ internal class SocksHttpWebRequest : WebRequest
+ {
+ private static readonly StringCollection validHttpVerbs = new StringCollection { "GET", "HEAD", "POST", "PUT", "DELETE", "TRACE", "OPTIONS" };
+ private WebHeaderCollection _HttpRequestHeaders;
+ private string _method;
+ private byte[] _requestContentBuffer;
+ private NeverEndingStream _requestContentStream;
+ private SocksHttpWebResponse _response;
+
+ private SocksHttpWebRequest(Uri requestUri)
+ {
+ RequestUri = requestUri;
+ }
+
+ public override int Timeout { get; set; }
+
+ public string UserAgent
+ {
+ get { return _HttpRequestHeaders["User-Agent"]; }
+ set { SetSpecialHeaders("User-Agent", value ?? string.Empty); }
+ }
+
+ public string Referer
+ {
+ get { return _HttpRequestHeaders["Referer"]; }
+ set { SetSpecialHeaders("Referer", value ?? string.Empty); }
+ }
+
+ public string Accept
+ {
+ get { return _HttpRequestHeaders["Accept"]; }
+ set { SetSpecialHeaders("Accept", value ?? string.Empty); }
+ }
+
+ public DecompressionMethods AutomaticDecompression
+ {
+ get
+ {
+ var result = DecompressionMethods.None;
+ string encoding = _HttpRequestHeaders["Accept-Encoding"] ?? string.Empty;
+ foreach (string value in encoding.Split(','))
+ {
+ switch (value.Trim())
+ {
+ case "gzip":
+ result |= DecompressionMethods.GZip;
+ break;
+
+ case "deflate":
+ result |= DecompressionMethods.Deflate;
+ break;
+ }
+ }
+
+ return result;
+ }
+ set
+ {
+ string result = string.Empty;
+ if ((value & DecompressionMethods.GZip) != 0)
+ result = "gzip";
+ if ((value & DecompressionMethods.Deflate) != 0)
+ {
+ if (!string.IsNullOrEmpty(result))
+ result += ", ";
+ result += "deflate";
+ }
+
+ SetSpecialHeaders("Accept-Encoding", result);
+ }
+ }
+
+ public override Uri RequestUri { get; }
+
+ public override IWebProxy Proxy { get; set; }
+
+ public override WebHeaderCollection Headers
+ {
+ get { return _HttpRequestHeaders ?? (_HttpRequestHeaders = new WebHeaderCollection()); }
+ set
+ {
+ if (RequestSubmitted)
+ {
+ throw new InvalidOperationException("This operation cannot be performed after the request has been submitted.");
+ }
+ _HttpRequestHeaders = value;
+ }
+ }
+
+ public bool RequestSubmitted { get; private set; }
+
+ public override string Method
+ {
+ get { return _method ?? "GET"; }
+ set
+ {
+ if (validHttpVerbs.Contains(value))
+ {
+ _method = value;
+ }
+ else
+ {
+ throw new ArgumentOutOfRangeException(nameof(value), $"'{value}' is not a known HTTP verb.");
+ }
+ }
+ }
+
+ public override long ContentLength { get; set; }
+
+ public override string ContentType { get; set; }
+ public bool AllowAutoRedirect { get; set; } = true;
+
+ public override WebResponse GetResponse()
+ {
+ if (Proxy == null)
+ {
+ throw new InvalidOperationException("Proxy property cannot be null.");
+ }
+ if (string.IsNullOrEmpty(Method))
+ {
+ throw new InvalidOperationException("Method has not been set.");
+ }
+
+ if (RequestSubmitted)
+ {
+ return _response;
+ }
+ _response = InternalGetResponse();
+ RequestSubmitted = true;
+ return _response;
+ }
+
+ public override Stream GetRequestStream()
+ {
+ if (RequestSubmitted)
+ {
+ throw new InvalidOperationException("This operation cannot be performed after the request has been submitted.");
+ }
+
+ if (_requestContentBuffer == null)
+ {
+ if (ContentLength == 0)
+ {
+ _requestContentStream = new NeverEndingStream();
+ return _requestContentStream;
+ }
+
+ _requestContentBuffer = new byte[ContentLength];
+ }
+ else if (ContentLength == default(long))
+ {
+ _requestContentBuffer = new byte[int.MaxValue];
+ }
+ else if (_requestContentBuffer.Length != ContentLength)
+ {
+ Array.Resize(ref _requestContentBuffer, (int)ContentLength);
+ }
+ return new MemoryStream(_requestContentBuffer);
+ }
+
+ public override IAsyncResult BeginGetResponse(AsyncCallback callback, object state)
+ {
+ if (Proxy == null)
+ {
+ throw new InvalidOperationException("Proxy property cannot be null.");
+ }
+ if (string.IsNullOrEmpty(Method))
+ {
+ throw new InvalidOperationException("Method has not been set.");
+ }
+ //.net 4.0
+ var task = Task.Factory.StartNew(() => {
+ if (RequestSubmitted)
+ {
+ return _response;
+ }
+ _response = InternalGetResponse();
+ RequestSubmitted = true;
+ return _response;
+ });
+ //.net 4.5
+ //var task = Task.Run(() =>
+ //{
+ // if (RequestSubmitted)
+ // {
+ // return _response;
+ // }
+ // _response = InternalGetResponse();
+ // RequestSubmitted = true;
+ // return _response;
+ //});
+
+ return task.AsApm(callback, state);
+ }
+
+ public override WebResponse EndGetResponse(IAsyncResult asyncResult)
+ {
+ var task = asyncResult as Task;
+
+ try
+ {
+ return task.Result;
+ }
+ catch (AggregateException ex)
+ {
+ if (ex.InnerException is IOException || ex.InnerException is System.Net.Sockets.SocketException)
+ throw new WebException("Proxy error " + ex.InnerException.Message, ex.InnerException, WebExceptionStatus.ConnectFailure,
+ SocksHttpWebResponse.CreateErrorResponse(HttpStatusCode.GatewayTimeout));
+ throw ex.InnerException;
+ }
+ }
+
+ public new static WebRequest Create(string requestUri)
+ {
+ return new SocksHttpWebRequest(new Uri(requestUri));
+ }
+
+ public new static WebRequest Create(Uri requestUri)
+ {
+ return new SocksHttpWebRequest(requestUri);
+ }
+
+ private void SetSpecialHeaders(string headerName, string value)
+ {
+ _HttpRequestHeaders.Remove(headerName);
+ if (value.Length != 0)
+ {
+ _HttpRequestHeaders.Add(headerName, value);
+ }
+ }
+
+ private string BuildHttpRequestMessage(Uri requestUri)
+ {
+ if (RequestSubmitted)
+ {
+ throw new InvalidOperationException("This operation cannot be performed after the request has been submitted.");
+ }
+
+ // See if we have a stream instead of byte array
+ if (_requestContentBuffer == null && _requestContentStream != null)
+ {
+ _requestContentBuffer = _requestContentStream.ToArray();
+ _requestContentStream.ForceClose();
+ _requestContentStream.Dispose();
+ _requestContentStream = null;
+ ContentLength = _requestContentBuffer.Length;
+ }
+
+ var message = new StringBuilder();
+ message.AppendFormat("{0} {1} HTTP/1.1\r\nHost: {2}\r\n", Method, requestUri, requestUri.Host);
+
+ Headers.Set(HttpRequestHeader.Connection, "close");
+
+ // add the headers
+ foreach (var key in Headers.Keys)
+ {
+ string value = Headers[key.ToString()];
+ if (!string.IsNullOrEmpty(value))
+ message.AppendFormat("{0}: {1}\r\n", key, value);
+ }
+
+ if (!string.IsNullOrEmpty(ContentType))
+ {
+ message.AppendFormat("Content-Type: {0}\r\n", ContentType);
+ }
+ if (ContentLength > 0)
+ {
+ message.AppendFormat("Content-Length: {0}\r\n", ContentLength);
+ }
+
+ // add a blank line to indicate the end of the headers
+ message.Append("\r\n");
+
+ // add content
+ if (_requestContentBuffer != null && _requestContentBuffer.Length > 0)
+ {
+ using (var stream = new MemoryStream(_requestContentBuffer, false))
+ {
+ using (var reader = new StreamReader(stream))
+ {
+ message.Append(reader.ReadToEnd());
+ }
+ }
+ }
+ else if (_requestContentStream != null)
+ {
+ _requestContentStream.Position = 0;
+
+ using (var reader = new StreamReader(_requestContentStream))
+ {
+ message.Append(reader.ReadToEnd());
+ }
+
+
+ _requestContentStream.ForceClose();
+ _requestContentStream.Dispose();
+ }
+
+ return message.ToString();
+ }
+
+ private SocksHttpWebResponse InternalGetResponse()
+ {
+ Uri requestUri = RequestUri;
+
+ int redirects = 0;
+ const int maxAutoredirectCount = 10;
+ while (redirects++ < maxAutoredirectCount)
+ {
+ // Loop while redirecting
+
+ var proxyUri = Proxy.GetProxy(requestUri);
+ var ipAddress = GetProxyIpAddress(proxyUri);
+ var response = new List();
+
+ using (var client = new TcpClient(ipAddress.ToString(), proxyUri.Port))
+ {
+ int timeout = Timeout;
+ if (timeout == 0)
+ timeout = 30 * 1000;
+ client.ReceiveTimeout = timeout;
+ client.SendTimeout = timeout;
+ var networkStream = client.GetStream();
+ // auth
+ var buf = new byte[300];
+ Console.WriteLine(proxyUri.Scheme);
+ buf[0] = 0x05; // Version
+ buf[1] = 0x01; // NMETHODS
+ buf[2] = 0x00; // No auth-method
+ networkStream.Write(buf, 0, 3);
+
+ networkStream.Read(buf, 0, 2);
+ if (buf[0] != 0x05)
+ {
+ throw new IOException("Invalid Socks Version");
+ }
+ if (buf[1] == 0xff)
+ {
+ throw new IOException("Socks Server does not support no-auth");
+ }
+ if (buf[1] != 0x00)
+ {
+ throw new Exception("Socks Server did choose bogus auth");
+ }
+
+ // connect
+ var destIP = Dns.GetHostEntry(requestUri.DnsSafeHost).AddressList[0];
+ var index = 0;
+ buf[index++] = 0x05; // version 5 .
+ buf[index++] = 0x01; // command = connect.
+ buf[index++] = 0x00; // Reserve = must be 0x00
+
+ buf[index++] = 0x01; // Address is full-qualified domain name.
+ var rawBytes = destIP.GetAddressBytes();
+ rawBytes.CopyTo(buf, index);
+ index += (ushort)rawBytes.Length;
+
+ var portBytes = BitConverter.GetBytes(Uri.UriSchemeHttps == requestUri.Scheme ? 443 : 80);
+ for (var i = portBytes.Length - 3; i >= 0; i--)
+ buf[index++] = portBytes[i];
+
+
+ networkStream.Write(buf, 0, index);
+
+ networkStream.Read(buf, 0, 4);
+ if (buf[0] != 0x05)
+ {
+ throw new IOException("Invalid Socks Version");
+ }
+ if (buf[1] != 0x00)
+ {
+ throw new IOException($"Socks Error {buf[1]:X}");
+ }
+
+ var rdest = string.Empty;
+ switch (buf[3])
+ {
+ case 0x01: // IPv4
+ networkStream.Read(buf, 0, 4);
+ var v4 = BitConverter.ToUInt32(buf, 0);
+ rdest = new IPAddress(v4).ToString();
+ break;
+ case 0x03: // Domain name
+ networkStream.Read(buf, 0, 1);
+ if (buf[0] == 0xff)
+ {
+ throw new IOException("Invalid Domain Name");
+ }
+ networkStream.Read(buf, 1, buf[0]);
+ rdest = Encoding.ASCII.GetString(buf, 1, buf[0]);
+ break;
+ case 0x04: // IPv6
+ var octets = new byte[16];
+ networkStream.Read(octets, 0, 16);
+ rdest = new IPAddress(octets).ToString();
+ break;
+ default:
+ throw new IOException("Invalid Address type");
+ }
+ networkStream.Read(buf, 0, 2);
+ var rport = (ushort)IPAddress.NetworkToHostOrder((short)BitConverter.ToUInt16(buf, 0));
+
+ Stream readStream = null;
+ if (Uri.UriSchemeHttps == requestUri.Scheme)
+ {
+ var ssl = new SslStream(networkStream);
+ ssl.AuthenticateAsClient(requestUri.DnsSafeHost);
+ readStream = ssl;
+ }
+ else
+ {
+ readStream = networkStream;
+ }
+
+ string requestString = BuildHttpRequestMessage(requestUri);
+
+ var request = Encoding.ASCII.GetBytes(requestString);
+ readStream.Write(request, 0, request.Length);
+ readStream.Flush();
+
+ var buffer = new byte[client.ReceiveBufferSize];
+
+ var readlen = 0;
+ do
+ {
+ readlen = readStream.Read(buffer, 0, buffer.Length);
+ response.AddRange(buffer.Take(readlen));
+ } while (readlen != 0);
+
+ readStream.Close();
+ }
+
+ var webResponse = new SocksHttpWebResponse(requestUri, response.ToArray());
+
+ if (webResponse.StatusCode == HttpStatusCode.Moved || webResponse.StatusCode == HttpStatusCode.MovedPermanently)
+ {
+ string redirectUrl = webResponse.Headers["Location"];
+ if (string.IsNullOrEmpty(redirectUrl))
+ throw new WebException("Missing location for redirect");
+
+ requestUri = new Uri(requestUri, redirectUrl);
+ if (AllowAutoRedirect)
+ {
+ continue;
+ }
+ return webResponse;
+ }
+
+ if ((int)webResponse.StatusCode < 200 || (int)webResponse.StatusCode > 299)
+ throw new WebException(webResponse.StatusDescription, null, WebExceptionStatus.UnknownError, webResponse);
+
+ return webResponse;
+ }
+
+ throw new WebException("Too many redirects", null, WebExceptionStatus.ProtocolError, SocksHttpWebResponse.CreateErrorResponse(HttpStatusCode.BadRequest));
+ }
+
+ private static IPAddress GetProxyIpAddress(Uri proxyUri)
+ {
+ IPAddress ipAddress;
+ if (!IPAddress.TryParse(proxyUri.Host, out ipAddress))
+ {
+ try
+ {
+ return Dns.GetHostEntry(proxyUri.Host).AddressList[0];
+ }
+ catch (Exception e)
+ {
+ throw new InvalidOperationException($"Unable to resolve proxy hostname '{proxyUri.Host}' to a valid IP address.", e);
+ }
+ }
+ return ipAddress;
+ }
+ }
+}
\ No newline at end of file
diff --git a/BetterHttpClient/Socks/SocksHttpWebResponse.cs b/BetterHttpClient/Socks/SocksHttpWebResponse.cs
new file mode 100644
index 0000000..701b228
--- /dev/null
+++ b/BetterHttpClient/Socks/SocksHttpWebResponse.cs
@@ -0,0 +1,268 @@
+using System;
+using System.IO;
+using System.IO.Compression;
+using System.Net;
+using System.Text;
+
+// Credit: https://github.com/Yozer/BetterHttpClient
+
+namespace BetterHttpClient.Socks
+{
+ public class SocksHttpWebResponse : WebResponse
+ {
+ #region Member Variables
+
+ private WebHeaderCollection _httpResponseHeaders;
+ private Uri responseUri;
+
+ #endregion
+
+ #region Constructors
+
+ public SocksHttpWebResponse(Uri responseUri, byte[] httpResponseMessage)
+ {
+ this.responseUri = responseUri;
+ SetHeadersAndResponseContent(httpResponseMessage);
+ }
+
+ private SocksHttpWebResponse()
+ {
+ }
+
+ public static SocksHttpWebResponse CreateErrorResponse(HttpStatusCode statusCode)
+ {
+ var response = new SocksHttpWebResponse
+ {
+ StatusCode = statusCode
+ };
+
+ return response;
+ }
+
+ #endregion
+
+ #region Methods
+
+ private void SetHeadersAndResponseContent(byte[] responseMessage)
+ {
+ if (responseMessage == null || responseMessage.Length < 4)
+ return;
+
+ var headers = string.Empty;
+ // get headers part;
+ var headerEnd = 0;
+ for (; headerEnd < responseMessage.Length - 3; headerEnd++)
+ {
+ if (responseMessage[headerEnd] == 13 && responseMessage[headerEnd + 1] == 10 && responseMessage[headerEnd + 2] == 13 && responseMessage[headerEnd + 3] == 10)
+ {
+ headers = Encoding.ASCII.GetString(responseMessage, 0, headerEnd);
+ break;
+ }
+ }
+ if (string.IsNullOrEmpty(headers))
+ return;
+
+ var headerValues = headers.Split(new[] { "\r\n" }, StringSplitOptions.RemoveEmptyEntries);
+ // ignore the first line in the header since it is the HTTP response code
+ for (var i = 1; i < headerValues.Length; i++)
+ {
+ var headerEntry = headerValues[i].Split(':');
+ Headers.Add(headerEntry[0], headerEntry[1]);
+ }
+
+ var statusRegex = new System.Text.RegularExpressions.Regex(@"^HTTP/\d+\.\d+ (\d+)(.*)$");
+ var matches = statusRegex.Match(headerValues[0]);
+ if (matches.Groups.Count < 3)
+ throw new InvalidDataException();
+ StatusCode = (HttpStatusCode)int.Parse(matches.Groups[1].Value);
+ StatusDescription = matches.Groups[2].Value;
+
+ ResponseContent = new byte[responseMessage.Length - headerEnd - 4];
+ Array.Copy(responseMessage, headerEnd + 4, ResponseContent, 0, ResponseContent.Length);
+
+ string transferEncoding = Headers["Transfer-Encoding"];
+
+ switch (transferEncoding)
+ {
+ case "chunked":
+ DechunkContent();
+ break;
+ }
+
+ string contentEncoding = Headers["Content-Encoding"];
+
+ switch (contentEncoding)
+ {
+ case "gzip":
+ ResponseContent = Decompress(ResponseContent) ?? ResponseContent;
+ break;
+ }
+ }
+
+ private void DechunkContent()
+ {
+ using (var output = new MemoryStream())
+ using (var ms = new MemoryStream(ResponseContent))
+ {
+ while (true)
+ {
+ var infoLine = new StringBuilder();
+ while (true)
+ {
+ int b = ms.ReadByte();
+
+ if (b == -1)
+ // End of stream
+ break;
+
+ if (b == 13)
+ // Ignore
+ continue;
+
+ if (b == 10)
+ {
+ if (infoLine.Length == 0)
+ // Trim
+ continue;
+ else
+ break;
+ }
+
+ infoLine.Append((char)b);
+ }
+ if (infoLine.Length == 0)
+ break;
+
+ int chunkLength = int.Parse(infoLine.ToString(), System.Globalization.NumberStyles.HexNumber);
+ if (chunkLength == 0)
+ break;
+
+ var buffer = new byte[chunkLength];
+ ms.Read(buffer, 0, chunkLength);
+ output.Write(buffer, 0, buffer.Length);
+ }
+
+ ResponseContent = output.ToArray();
+ }
+ }
+
+ #endregion
+
+ #region WebResponse Members
+
+ public HttpStatusCode StatusCode { get; private set; }
+
+ public string StatusDescription { get; private set; }
+
+ public override Uri ResponseUri
+ {
+ get
+ {
+ return this.responseUri;
+ }
+ }
+
+ private static readonly byte[] GZipHeaderBytes = { 0x1f, 0x8b, 8, 0, 0, 0, 0, 0, 4, 0 };
+ private static readonly byte[] GZipLevel10HeaderBytes = { 0x1f, 0x8b, 8, 0, 0, 0, 0, 0, 2, 0 };
+
+ public static bool IsPossiblyGZippedBytes(byte[] a)
+ {
+ var yes = a.Length > 10;
+
+ if (!yes)
+ {
+ return false;
+ }
+
+ var fail = false;
+ for (var i = 0; i < 10; i++)
+ {
+ if (a[i] != GZipHeaderBytes[i])
+ {
+ fail = true;
+ break;
+ }
+ }
+
+ if (!fail)
+ return true;
+
+ for (var i = 0; i < 10; i++)
+ {
+ if (a[i] != GZipLevel10HeaderBytes[i])
+ {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ private byte[] Decompress(byte[] gzip)
+ {
+ try
+ {
+ using (var stream = new GZipStream(new MemoryStream(gzip), CompressionMode.Decompress))
+ {
+ const int size = 4096;
+ var buffer = new byte[size];
+ using (var memory = new MemoryStream())
+ {
+ var count = 0;
+ do
+ {
+ count = stream.Read(buffer, 0, size);
+ if (count > 0)
+ {
+ memory.Write(buffer, 0, count);
+ }
+ } while (count > 0);
+ return memory.ToArray();
+ }
+ }
+ }
+ catch (Exception)
+ {
+ return null;
+ }
+ }
+
+ public override Stream GetResponseStream()
+ {
+ return ResponseContent.Length == 0 ? Stream.Null : new MemoryStream(ResponseContent);
+ }
+
+ public override void Close()
+ {
+ /* the base implementation throws an exception */
+ }
+
+ public override WebHeaderCollection Headers
+ {
+ get
+ {
+ if (_httpResponseHeaders == null)
+ {
+ _httpResponseHeaders = new WebHeaderCollection();
+ }
+ return _httpResponseHeaders;
+ }
+ }
+
+ public override long ContentLength
+ {
+ get { return ResponseContent.Length; }
+ set { throw new NotSupportedException(); }
+ }
+
+ #endregion
+
+ #region Properties
+
+ private byte[] ResponseContent { get; set; }
+
+ public Encoding CorrectEncoding { get; set; }
+
+ #endregion
+ }
+}
\ No newline at end of file
diff --git a/BetterHttpClient/Socks/TaskExtensions.cs b/BetterHttpClient/Socks/TaskExtensions.cs
new file mode 100644
index 0000000..f490e28
--- /dev/null
+++ b/BetterHttpClient/Socks/TaskExtensions.cs
@@ -0,0 +1,34 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace BetterHttpClient.Socks.Extensions
+{
+ public static class TaskExtensions
+ {
+ public static IAsyncResult AsApm(this Task task,
+ AsyncCallback callback,
+ object state)
+ {
+ if (task == null)
+ throw new ArgumentNullException("task");
+
+ var tcs = new TaskCompletionSource(state);
+ task.ContinueWith(t =>
+ {
+ if (t.IsFaulted)
+ tcs.TrySetException(t.Exception.InnerExceptions);
+ else if (t.IsCanceled)
+ tcs.TrySetCanceled();
+ else
+ tcs.TrySetResult(t.Result);
+
+ if (callback != null)
+ callback(tcs.Task);
+ }, TaskScheduler.Default);
+ return tcs.Task;
+ }
+ }
+}
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..ee6a16a
--- /dev/null
+++ b/README.md
@@ -0,0 +1,74 @@
+# BetterHttpClient
+
+BetterHttpClient will help you making http and https requests with full cookie support.
+
+It supports three types of proxy servers:
+ - HTTP
+ - HTTPS
+ - Socks5
+
+# Http Client
+- Http GET request
+```cs
+HttpClient client = new HttpClient
+{
+ UserAgent = "My custom user-agent",
+ Referer = "My referer",
+ Encoding = Encoding.UTF8,
+ AcceptEncoding = "gzip, deflate" // setup transfer compression algorithm
+};
+string page = client.Get("http://www.google.com");
+```
+
+- Http POST request
+```cs
+HttpClient client = new HttpClient(); // if no headers are specified defaults will be used
+string page = client.Post("https://httpbin.org/post", new NameValueCollection
+{
+ {"custname", "Some post data"},
+});
+```
+
+- Https GET request using proxy
+
+HttpClient will detect proxy type (http/s, socks5) if you don't provide such info.
+```cs
+string proxyAddress = "218.200.66.196:8080";
+HttpClient client = new HttpClient(new Proxy(proxyAddress))
+{
+ UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0"
+};
+
+string page = client.Get("https://httpbin.org/get");
+```
+
+# Proxy Manager
+This class will help you managing large amount of proxy servers.
+
+You can load proxy list from file and then execute GET or POST requests.
+
+If some proxy will be dead ProxyManager will mark them as offline and won't use in future requests.
+
+To perform http/s request it will use first Not Busy and Online proxy (and check for an anonymity level if anonymousOnly is true).
+
+Set anonymousOnly to true if you want to force ProxyManager to use only proxies that really hide your ip. For more info look at ProxyJudgeService class.
+Methods
+- GetPage
+- PostPage
+- DownloadBytes
+
+are thread safe.
+
+Example usage:
+```cs
+ProxyManager proxyManager = new ProxyManager("proxy_list.txt", anonymousOnly: true)
+{
+ UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0"
+ PreserveCookies = false, // if true it will use the same cookies between distinct requests
+ Timeout = TimeSpan.FromSeconds(10),
+ NumberOfAttempts = 2 // max number of attempts that can be made per one request
+};
+
+// ProxyManager will choose first working proxy and use it to perform GET request
+string page = proxyManager.GetPage("http://example.com");
+```
diff --git a/UnitTestBetterHttpClient/Properties/AssemblyInfo.cs b/UnitTestBetterHttpClient/Properties/AssemblyInfo.cs
new file mode 100644
index 0000000..1195150
--- /dev/null
+++ b/UnitTestBetterHttpClient/Properties/AssemblyInfo.cs
@@ -0,0 +1,36 @@
+using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+// General Information about an assembly is controlled through the following
+// set of attributes. Change these attribute values to modify the information
+// associated with an assembly.
+[assembly: AssemblyTitle("UnitTestBetterHttpClient")]
+[assembly: AssemblyDescription("")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("")]
+[assembly: AssemblyProduct("UnitTestBetterHttpClient")]
+[assembly: AssemblyCopyright("Copyright © 2015")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+// Setting ComVisible to false makes the types in this assembly not visible
+// to COM components. If you need to access a type in this assembly from
+// COM, set the ComVisible attribute to true on that type.
+[assembly: ComVisible(false)]
+
+// The following GUID is for the ID of the typelib if this project is exposed to COM
+[assembly: Guid("59acc7d1-547c-4e1c-8de3-ba5351c1a00b")]
+
+// Version information for an assembly consists of the following four values:
+//
+// Major Version
+// Minor Version
+// Build Number
+// Revision
+//
+// You can specify all the values or you can default the Build and Revision Numbers
+// by using the '*' as shown below:
+// [assembly: AssemblyVersion("1.0.*")]
+[assembly: AssemblyVersion("1.0.0.0")]
+[assembly: AssemblyFileVersion("1.0.0.0")]
diff --git a/UnitTestBetterHttpClient/UnitTestBetterHttpClient.csproj b/UnitTestBetterHttpClient/UnitTestBetterHttpClient.csproj
new file mode 100644
index 0000000..850cfa4
--- /dev/null
+++ b/UnitTestBetterHttpClient/UnitTestBetterHttpClient.csproj
@@ -0,0 +1,98 @@
+
+
+
+ Debug
+ AnyCPU
+ {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}
+ Library
+ Properties
+ UnitTestBetterHttpClient
+ UnitTestBetterHttpClient
+ v4.0
+ 512
+ {3AC096D0-A1C2-E12C-1390-A8335801FDAB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}
+ 10.0
+ $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion)
+ $(ProgramFiles)\Common Files\microsoft shared\VSTT\$(VisualStudioVersion)\UITestExtensionPackages
+ False
+ UnitTest
+
+
+
+ true
+ full
+ false
+ bin\Debug\
+ DEBUG;TRACE
+ prompt
+ 4
+
+
+ pdbonly
+ true
+ bin\Release\
+ TRACE
+ prompt
+ 4
+
+
+
+ ..\packages\Newtonsoft.Json.7.0.1\lib\net40\Newtonsoft.Json.dll
+ True
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ {fbc5f420-3f80-4a20-a9ec-b6574e18a1d3}
+ BetterHttpClient
+
+
+
+
+
+
+
+
+
+ False
+
+
+ False
+
+
+ False
+
+
+ False
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/UnitTestBetterHttpClient/UnitTestHttpClient.cs b/UnitTestBetterHttpClient/UnitTestHttpClient.cs
new file mode 100644
index 0000000..2cc1e49
--- /dev/null
+++ b/UnitTestBetterHttpClient/UnitTestHttpClient.cs
@@ -0,0 +1,185 @@
+using System;
+using System.Collections.Specialized;
+using System.Text;
+using BetterHttpClient;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using Newtonsoft.Json;
+
+namespace UnitTestBetterHttpClient
+{
+ [TestClass]
+ public class UnitTestHttpClient
+ {
+ private const string HttpsProxy = "210.245.25.229:3128";
+ private const string Socksproxy = "124.207.126.18:1080";
+ [TestMethod]
+ public void TestGet()
+ {
+ HttpClient client = new HttpClient
+ {
+ UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0"
+ };
+
+ string page = client.Get("http://www.google.com");
+ Assert.IsTrue(page.Contains("Google"));
+ }
+
+ [TestMethod]
+ public void TestPost()
+ {
+ HttpClient client = new HttpClient
+ {
+ UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0",
+ ContentType = "application/x-www-form-urlencoded; charset=UTF-8"
+
+ };
+ //client.Headers.Add("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8");
+
+ string page = client.Post("http://www.newrank.cn/xdnphb/detail/getAccountArticle", new NameValueCollection
+ {
+ { "flag","true"},
+ {"uuid","DCFCC585F745FF5401FD33213B1178EB" },
+ { "nonce","0f3ac361b" },
+ {"xyz","4d86efaf832909233403482879de5cdd" }
+ });
+ Assert.IsTrue(!string.IsNullOrEmpty(page));
+ //Form root = JsonConvert.DeserializeObject(page).form;
+
+ //Assert.AreEqual(root.custname, customerName);
+ //Assert.AreEqual(root.custtel, phone);
+ //Assert.AreEqual(root.custemail, email);
+ //Assert.AreEqual(root.size, size);
+ //Assert.AreEqual(root.topping, topping);
+ //Assert.AreEqual(root.delivery, delivery);
+ //Assert.AreEqual(root.comments, comments);
+
+ }
+
+ [TestMethod]
+ public void TestUserAgent()
+ {
+ HttpClient client = new HttpClient
+ {
+ UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0"
+ };
+
+ string page = client.Get("https://httpbin.org/user-agent");
+ Assert.IsTrue(page.Contains(client.UserAgent));
+ }
+
+ [TestMethod]
+ public void TestGzipDecodingAndReferer()
+ {
+ HttpClient client = new HttpClient
+ {
+ UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0",
+ Referer = "https://httpbin.org/"
+ };
+
+ string page = client.Get("https://httpbin.org/gzip");
+ Assert.IsTrue(page.Contains(client.UserAgent));
+ // check for referer
+ Assert.IsTrue(page.Contains("https://httpbin.org/"));
+ }
+
+ [TestMethod]
+ public void TestHttpProxy()
+ {
+ HttpClient client = new HttpClient(new Proxy(HttpsProxy))
+ {
+ UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0",
+ Encoding = Encoding.GetEncoding("iso-8859-2"),
+ AcceptEncoding = "gzip"
+ };
+
+ string page = client.Get("http://darkwarez.pl");
+ Assert.IsTrue(page.Contains("Polskie Forum Warez! Najnowsze linki"));
+ }
+ [TestMethod]
+ public void TestHttpsProxy()
+ {
+ HttpClient client = new HttpClient(new Proxy(HttpsProxy))
+ {
+ UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0"
+ };
+
+ string page = client.Get("https://httpbin.org/get");
+ Assert.IsTrue(page.Contains(client.UserAgent));
+ }
+ [TestMethod]
+ public void TestSocksHttpProxy()
+ {
+ Proxy proxy = new Proxy("124.207.126.15", 1080, ProxyTypeEnum.Socks);
+ HttpClient client = new HttpClient(proxy)
+ {
+ UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0",
+ Encoding = Encoding.GetEncoding("iso-8859-2"),
+ AcceptEncoding = "deflate"
+ };
+
+ string page = client.Get("https://httpbin.org/get");
+ Assert.IsTrue(page.Contains("darkwarez.pl - Gry, Muzyka, Filmy, Download"));
+ }
+ [TestMethod]
+ public void TestSocks4HttpProxy()
+ {
+ Proxy proxy = new Proxy("115.29.161.103", 1080, ProxyTypeEnum.Socks4);
+
+ HttpClient client = new HttpClient(proxy)
+ {
+ UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0",
+ Encoding = Encoding.GetEncoding("utf-8"),
+ //AcceptEncoding = "deflate"
+ };
+
+ string page = client.Get("http://proxy.mimvp.com/free.php?proxy=in_socks");
+ Assert.IsTrue(page.Contains("\"origin\": \"115.29.161.103\""));
+ }
+ [TestMethod]
+ public void TestSocksHttpsProxyDeflateEncoding()
+ {
+ HttpClient client = new HttpClient(new Proxy(Socksproxy))
+ {
+ UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0",
+ AcceptEncoding = "deflate"
+ };
+
+ string page = client.Get("https://httpbin.org/get");
+ Assert.IsTrue(page.Contains(client.UserAgent));
+ }
+ [TestMethod]
+ public void TestSocksHttpsProxyGzipEndcoding()
+ {
+ HttpClient client = new HttpClient(new Proxy(Socksproxy))
+ {
+ UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0",
+ AcceptEncoding = "gzip"
+ };
+
+ string page = client.Get("https://httpbin.org/get");
+ Assert.IsTrue(page.Contains(client.UserAgent));
+ }
+ public class Form
+ {
+ public string comments { get; set; }
+ public string custemail { get; set; }
+ public string custname { get; set; }
+ public string custtel { get; set; }
+ public string delivery { get; set; }
+ public string size { get; set; }
+ public string topping { get; set; }
+ }
+
+ public class RootObject
+ {
+ public object args { get; set; }
+ public string data { get; set; }
+ public object files { get; set; }
+ public Form form { get; set; }
+ public object headers { get; set; }
+ public object json { get; set; }
+ public string origin { get; set; }
+ public string url { get; set; }
+ }
+ }
+}
diff --git a/UnitTestBetterHttpClient/packages.config b/UnitTestBetterHttpClient/packages.config
new file mode 100644
index 0000000..1b9f231
--- /dev/null
+++ b/UnitTestBetterHttpClient/packages.config
@@ -0,0 +1,4 @@
+
+
+
+
\ No newline at end of file