diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..1ff0c42 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,63 @@ +############################################################################### +# Set default behavior to automatically normalize line endings. +############################################################################### +* text=auto + +############################################################################### +# Set default behavior for command prompt diff. +# +# This is need for earlier builds of msysgit that does not have it on by +# default for csharp files. +# Note: This is only used by command line +############################################################################### +#*.cs diff=csharp + +############################################################################### +# Set the merge driver for project and solution files +# +# Merging from the command prompt will add diff markers to the files if there +# are conflicts (Merging from VS is not affected by the settings below, in VS +# the diff markers are never inserted). Diff markers may cause the following +# file extensions to fail to load in VS. An alternative would be to treat +# these files as binary and thus will always conflict and require user +# intervention with every merge. To do so, just uncomment the entries below +############################################################################### +#*.sln merge=binary +#*.csproj merge=binary +#*.vbproj merge=binary +#*.vcxproj merge=binary +#*.vcproj merge=binary +#*.dbproj merge=binary +#*.fsproj merge=binary +#*.lsproj merge=binary +#*.wixproj merge=binary +#*.modelproj merge=binary +#*.sqlproj merge=binary +#*.wwaproj merge=binary + +############################################################################### +# behavior for image files +# +# image files are treated as binary by default. +############################################################################### +#*.jpg binary +#*.png binary +#*.gif binary + +############################################################################### +# diff behavior for common document formats +# +# Convert binary document formats to text before diffing them. This feature +# is only available from the command line. Turn it on by uncommenting the +# entries below. +############################################################################### +#*.doc diff=astextplain +#*.DOC diff=astextplain +#*.docx diff=astextplain +#*.DOCX diff=astextplain +#*.dot diff=astextplain +#*.DOT diff=astextplain +#*.pdf diff=astextplain +#*.PDF diff=astextplain +#*.rtf diff=astextplain +#*.RTF diff=astextplain diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b06e864 --- /dev/null +++ b/.gitignore @@ -0,0 +1,212 @@ +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. + +# User-specific files +*.suo +*.user +*.userosscache +*.sln.docstates + +# User-specific files (MonoDevelop/Xamarin Studio) +*.userprefs + +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +build/ +bld/ +[Bb]in/ +[Oo]bj/ + +# Visual Studio 2015 cache/options directory +.vs/ + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +# NUNIT +*.VisualState.xml +TestResult.xml + +# Build Results of an ATL Project +[Dd]ebugPS/ +[Rr]eleasePS/ +dlldata.c + +# DNX +project.lock.json +artifacts/ + +*_i.c +*_p.c +*_i.h +*.ilk +*.meta +*.obj +*.pch +*.pdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*.log +*.vspscc +*.vssscc +.builds +*.pidb +*.svclog +*.scc + +# Chutzpah Test files +_Chutzpah* + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opensdf +*.sdf +*.cachefile + +# Visual Studio profiler +*.psess +*.vsp +*.vspx + +# TFS 2012 Local Workspace +$tf/ + +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper*/ +*.[Rr]e[Ss]harper +*.DotSettings.user + +# JustCode is a .NET coding add-in +.JustCode + +# TeamCity is a build add-in +_TeamCity* + +# DotCover is a Code Coverage Tool +*.dotCover + +# NCrunch +_NCrunch_* +.*crunch*.local.xml + +# MightyMoose +*.mm.* +AutoTest.Net/ + +# Web workbench (sass) +.sass-cache/ + +# Installshield output folder +[Ee]xpress/ + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish/ + +# Publish Web Output +*.[Pp]ublish.xml +*.azurePubxml +## TODO: Comment the next line if you want to checkin your +## web deploy settings but do note that will include unencrypted +## passwords +#*.pubxml + +*.publishproj + +# NuGet Packages +*.nupkg +# The packages folder can be ignored because of Package Restore +**/packages/* +# except build/, which is used as an MSBuild target. +!**/packages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/packages/repositories.config + +# Windows Azure Build Output +csx/ +*.build.csdef + +# Windows Store app package directory +AppPackages/ + +# Visual Studio cache files +# files ending in .cache can be ignored +*.[Cc]ache +# but keep track of directories ending in .cache +!*.[Cc]ache/ + +# Others +ClientBin/ +[Ss]tyle[Cc]op.* +~$* +*~ +*.dbmdl +*.dbproj.schemaview +*.pfx +*.publishsettings +node_modules/ +orleans.codegen.cs + +# RIA/Silverlight projects +Generated_Code/ + +# Backup & report files from converting an old project file +# to a newer Visual Studio version. Backup files are not needed, +# because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML +UpgradeLog*.htm + +# SQL Server files +*.mdf +*.ldf + +# Business Intelligence projects +*.rdl.data +*.bim.layout +*.bim_*.settings + +# Microsoft Fakes +FakesAssemblies/ + +# Node.js Tools for Visual Studio +.ntvs_analysis.dat + +# Visual Studio 6 build log +*.plg + +# Visual Studio 6 workspace options file +*.opt + +# LightSwitch generated files +GeneratedArtifacts/ +_Pvt_Extensions/ +ModelManifest.xml diff --git a/BetterHttpClient.sln b/BetterHttpClient.sln new file mode 100644 index 0000000..73bf770 --- /dev/null +++ b/BetterHttpClient.sln @@ -0,0 +1,58 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 14 +VisualStudioVersion = 14.0.23107.0 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BetterHttpClient", "BetterHttpClient\BetterHttpClient.csproj", "{FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "UnitTestBetterHttpClient", "UnitTestBetterHttpClient\UnitTestBetterHttpClient.csproj", "{59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Debug|ARM = Debug|ARM + Debug|x64 = Debug|x64 + Debug|x86 = Debug|x86 + Release|Any CPU = Release|Any CPU + Release|ARM = Release|ARM + Release|x64 = Release|x64 + Release|x86 = Release|x86 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Debug|ARM.ActiveCfg = Debug|Any CPU + {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Debug|ARM.Build.0 = Debug|Any CPU + {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Debug|x64.ActiveCfg = Debug|Any CPU + {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Debug|x64.Build.0 = Debug|Any CPU + {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Debug|x86.ActiveCfg = Debug|Any CPU + {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Debug|x86.Build.0 = Debug|Any CPU + {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Release|Any CPU.Build.0 = Release|Any CPU + {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Release|ARM.ActiveCfg = Release|Any CPU + {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Release|ARM.Build.0 = Release|Any CPU + {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Release|x64.ActiveCfg = Release|Any CPU + {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Release|x64.Build.0 = Release|Any CPU + {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Release|x86.ActiveCfg = Release|Any CPU + {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3}.Release|x86.Build.0 = Release|Any CPU + {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Debug|ARM.ActiveCfg = Debug|Any CPU + {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Debug|ARM.Build.0 = Debug|Any CPU + {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Debug|x64.ActiveCfg = Debug|Any CPU + {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Debug|x64.Build.0 = Debug|Any CPU + {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Debug|x86.ActiveCfg = Debug|Any CPU + {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Debug|x86.Build.0 = Debug|Any CPU + {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Release|Any CPU.Build.0 = Release|Any CPU + {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Release|ARM.ActiveCfg = Release|Any CPU + {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Release|ARM.Build.0 = Release|Any CPU + {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Release|x64.ActiveCfg = Release|Any CPU + {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Release|x64.Build.0 = Release|Any CPU + {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Release|x86.ActiveCfg = Release|Any CPU + {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B}.Release|x86.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/BetterHttpClient/BetterHttpClient.csproj b/BetterHttpClient/BetterHttpClient.csproj new file mode 100644 index 0000000..b74d6d3 --- /dev/null +++ b/BetterHttpClient/BetterHttpClient.csproj @@ -0,0 +1,66 @@ + + + + + Debug + AnyCPU + {FBC5F420-3F80-4A20-A9EC-B6574E18A1D3} + Library + Properties + BetterHttpClient + BetterHttpClient + v4.0 + 512 + + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + bin\Release\BetterHttpClient.XML + + + + + + + + + + + + + Component + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/BetterHttpClient/CheckService/ProxyCheckService.cs b/BetterHttpClient/CheckService/ProxyCheckService.cs new file mode 100644 index 0000000..15bae09 --- /dev/null +++ b/BetterHttpClient/CheckService/ProxyCheckService.cs @@ -0,0 +1,42 @@ +using System; + +namespace BetterHttpClient.CheckService +{ + public abstract class ProxyJudgeServiceAbstract + { + private string _myIp = null; + private readonly object _syncLock = new object(); + + public string MyIp + { + get + { + lock (_syncLock) + { + if(_myIp == null) + _myIp = GetMyIp(); + } + + return _myIp; + } + } + /// + /// Set number of attempts which can be made to verify proxy. + /// + public int NumberOfAttempts { get; set; } + /// + /// Abstract method. Should return true if proxy is hiding your real ip address. + /// + /// + /// + public abstract bool IsProxyAnonymous(Proxy proxy); + /// + /// Abstract method. Should return real ip addres of user (so without any proxy). + /// + /// + protected abstract string GetMyIp(); + } + public class GetMyIpException : Exception + { + } +} \ No newline at end of file diff --git a/BetterHttpClient/CheckService/ProxyJudgeProxyCheckService.cs b/BetterHttpClient/CheckService/ProxyJudgeProxyCheckService.cs new file mode 100644 index 0000000..00d188c --- /dev/null +++ b/BetterHttpClient/CheckService/ProxyJudgeProxyCheckService.cs @@ -0,0 +1,45 @@ +using System.Net; +using System.Text; +using System.Text.RegularExpressions; + +namespace BetterHttpClient.CheckService +{ + public class ProxyJudgeService : ProxyJudgeServiceAbstract + { + public override bool IsProxyAnonymous(Proxy proxy) + { + HttpClient client = new HttpClient(proxy, Encoding.UTF8) + { + NumberOfAttempts = NumberOfAttempts + }; + string page = null; + + try + { + page = client.Get("http://proxyjudge.info/"); + } + catch (WebException) + { + proxy.IsOnline = false; + } + + if (page == null) + return false; + if (page.Contains("Proxyjudge.info") && !page.Contains(MyIp)) + return true; + return false; + } + + protected override string GetMyIp() + { + HttpClient client = new HttpClient(Encoding.UTF8); + string page = client.Get("http://proxyjudge.info/"); + + Match match = Regex.Match(page, "REMOTE_ADDR = (.*?)\\n"); + if (!match.Success) + throw new GetMyIpException(); + else + return match.Groups[1].Value; + } + } +} \ No newline at end of file diff --git a/BetterHttpClient/HttpClient.cs b/BetterHttpClient/HttpClient.cs new file mode 100644 index 0000000..0764aeb --- /dev/null +++ b/BetterHttpClient/HttpClient.cs @@ -0,0 +1,273 @@ +using System; +using System.Collections.Specialized; +using System.Net; +using System.Text; +using BetterHttpClient.Socks; + +namespace BetterHttpClient +{ + public class HttpClient : WebClient + { + private int _numberOfAttempts = 4; + private TimeSpan _timeout = TimeSpan.FromSeconds(60); + private Proxy _proxy; + + /// + /// Cookie container. + /// + public CookieContainer Cookies { get; set; } = new CookieContainer(); + + /// + /// Proxy which should be used for request. + /// Set null or proxy with type ProxyTypeEnum.None if you want to perform request without proxy. + /// + public new Proxy Proxy + { + get { return _proxy; } + set { _proxy = value ?? new Proxy(); } + } + /// + /// Timeout for request. + /// + /// Has to be greater than 5 milliseconds. + /// Default value: 60 seconds. + public TimeSpan Timeout + { + get + { + return _timeout; + } + set + { + if (value.TotalMilliseconds < 5) + throw new ArgumentOutOfRangeException("Timeout has to be greater or equal than 5 milliseconds."); + _timeout = value; + } + } + /// + /// Set number of attempts that can be made to execute request. + /// + /// Default value: 4 attempts. + /// Should be greater than 1 + public int NumberOfAttempts + { + get { return _numberOfAttempts; } + set + { + if (value < 1) + throw new ArgumentOutOfRangeException("Value has to be greater than one."); + + _numberOfAttempts = value; + } + } + + /// + /// Set User-Agent header. + /// + /// Default value: "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0" + public string UserAgent { get; set; } = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0"; + /// + /// Set Accept header. + /// + /// Default value: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" + public string Accept { get; set; } = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"; + /// + /// Set Referer header. + /// + /// Default value: null + public string Referer { get; set; } + /// + /// Set Accept-Language header. + /// + /// Default value: "en-US;q=0.7,en;q=0.3" + public string AcceptLanguage { get; set; } = "en-US;q=0.7,en;q=0.3"; + /// + /// Set default Accept-Encoding header. + /// + /// Default value: "gzip, deflate" + public string AcceptEncoding { get; set; } = "gzip, deflate"; + /// + /// Set encoding for request. + /// + /// Default value: UTF-8 + public new Encoding Encoding + { + get { return base.Encoding; } + set { base.Encoding = value; } + } + /// + /// Enabled automatic redirect. Default: true + /// + public bool AllowAutoRedirect { get; set; } = true; + + /// + /// 内容类型 + /// + public string ContentType { get; set; } + + /// + /// Headers collection that will be added to each request + /// + public NameValueCollection CustomHeaders { get; set; } + public HttpClient(Proxy proxy) : this(proxy, Encoding.UTF8) { } + + public HttpClient() : this(new Proxy(), Encoding.UTF8) { } + public HttpClient(Encoding encoding) : this(new Proxy(), encoding) { } + public HttpClient(Proxy proxy, Encoding encoding) + { + Encoding = encoding; + Proxy = proxy; + } + + protected override WebRequest GetWebRequest(Uri address) + { + WebRequest request = null; + + if (Proxy.ProxyType != ProxyTypeEnum.Socks && Proxy.ProxyType != ProxyTypeEnum.Socks4) + { + request = base.GetWebRequest(address); + request.ContentType= string.IsNullOrEmpty(ContentType) ? base.GetWebRequest(address).ContentType : ContentType; + } + else if (Proxy.ProxyType == ProxyTypeEnum.Socks) + { + request = SocksHttpWebRequest.Create(address); + request.Method = base.GetWebRequest(address).Method; + request.ContentLength = base.GetWebRequest(address).ContentLength; + request.ContentType = string.IsNullOrEmpty(ContentType)?base.GetWebRequest(address).ContentType:ContentType; + } + else if (Proxy.ProxyType == ProxyTypeEnum.Socks4) + { + request = Socks4HttpWebRequest.Create(address); + request.Method = base.GetWebRequest(address).Method; + request.ContentLength = base.GetWebRequest(address).ContentLength; + request.ContentType = string.IsNullOrEmpty(ContentType) ? base.GetWebRequest(address).ContentType : ContentType; + } + + + request.Headers.Add("Cookie", Cookies.GetCookieHeader(address)); + request.Headers.Add("Accept-Language", AcceptLanguage); + request.Headers.Add("Accept-Encoding", AcceptEncoding); + if (CustomHeaders != null) + { + foreach (string key in CustomHeaders.AllKeys) + { + request.Headers.Add(key, CustomHeaders[key]); + } + } + + + if (Proxy.ProxyType != ProxyTypeEnum.Socks && Proxy.ProxyType != ProxyTypeEnum.Socks4) + { + var httpRequest = (request as HttpWebRequest); + httpRequest.UserAgent = UserAgent; + httpRequest.Accept = Accept; + httpRequest.Referer = Referer; + httpRequest.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip; + httpRequest.AllowAutoRedirect = AllowAutoRedirect; + } + else if (Proxy.ProxyType == ProxyTypeEnum.Socks) + { + var socksRequest = (request as SocksHttpWebRequest); + socksRequest.UserAgent = UserAgent; + socksRequest.Accept = Accept; + socksRequest.Referer = Referer; + socksRequest.AllowAutoRedirect = AllowAutoRedirect; + } + else if (Proxy.ProxyType == ProxyTypeEnum.Socks4) + { + var socks4Request = (request as Socks4HttpWebRequest); + socks4Request.UserAgent = UserAgent; + socks4Request.Accept = Accept; + socks4Request.Referer = Referer; + socks4Request.AllowAutoRedirect = AllowAutoRedirect; + } + + request.Timeout = (int) Timeout.TotalMilliseconds; + request.Proxy = Proxy.ProxyItem; + + return request; + } + + protected override WebResponse GetWebResponse(WebRequest request) + { + var response = base.GetWebResponse(request); + try + { + string setCookies = response.Headers["Set-Cookie"]; + Cookies.SetCookies(request.RequestUri, setCookies); + } + catch (Exception) + { + + } + + return response; + } + /// + /// Execute GET request. + /// + /// + /// + public string Get(string url) + { + return Encoding.GetString(DownloadBytes(url, null)); + } + /// + /// Execute POST request. + /// + /// + /// + /// + public string Post(string url, NameValueCollection data) + { + return Encoding.GetString(DownloadBytes(url, data)); + } + + /// + /// Execute GET request. + /// + /// + /// + public byte[] DownloadBytes(string url) + { + return DownloadBytes(url, null); + } + /// + /// Excecute POST request. + /// + /// + /// + /// + public byte[] DownloadBytes(string url, NameValueCollection data) + { + int counter = 0; + WebException lastWebException = null; + bool unkownProxy = Proxy.ProxyType == ProxyTypeEnum.Unknown; + + while (counter < NumberOfAttempts + (NumberOfAttempts < 2 && unkownProxy ? 1 : 0)) // min two try for unkonwn proxy type + { + try + { + if (unkownProxy && counter % 3 == 0) // every 3rd try is as http proxy + Proxy.ProxyType = ProxyTypeEnum.Http; + else if (unkownProxy && counter % 3 == 1) + Proxy.ProxyType = ProxyTypeEnum.Socks; + else if (unkownProxy && counter % 3 == 2) + Proxy.ProxyType = ProxyTypeEnum.Socks4; + byte[] result = data == null ? Encoding.GetBytes(DownloadString(url)) : UploadValues(url, data); + return result; + } + catch (WebException e) + { + lastWebException = e; + counter++; + } + } + + if (unkownProxy) + Proxy.ProxyType = ProxyTypeEnum.Unknown; + // ReSharper disable once PossibleNullReferenceException + throw lastWebException; + } + } +} diff --git a/BetterHttpClient/Properties/AssemblyInfo.cs b/BetterHttpClient/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..30a7f32 --- /dev/null +++ b/BetterHttpClient/Properties/AssemblyInfo.cs @@ -0,0 +1,39 @@ +using System.Reflection; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. + +[assembly: AssemblyTitle("HttpClient")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("HttpClient")] +[assembly: AssemblyCopyright("Copyright © 2015")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. + +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM + +[assembly: Guid("fbc5f420-3f80-4a20-a9ec-b6574e18a1d3")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] + +[assembly: AssemblyVersion("1.0.7")] +[assembly: AssemblyFileVersion("1.0.0")] \ No newline at end of file diff --git a/BetterHttpClient/Proxy.cs b/BetterHttpClient/Proxy.cs new file mode 100644 index 0000000..5bf4617 --- /dev/null +++ b/BetterHttpClient/Proxy.cs @@ -0,0 +1,86 @@ +using System; +using System.Net; +using BetterHttpClient.CheckService; + +namespace BetterHttpClient +{ + public class Proxy : ICloneable + { + private volatile bool _isBusy = false; + private bool _isAnonymous; + private bool _isChecked = false; + + /// + /// Check if proxy is busy. + /// + public bool IsBusy + { + get { return _isBusy; } + internal set { _isBusy = value; } + } + + + public bool IsOnline { get; set; } = true; + public ProxyTypeEnum ProxyType { get; internal set; } + internal WebProxy ProxyItem { get; set; } + + public Proxy() + { + ProxyItem = new WebProxy(); + ProxyType = ProxyTypeEnum.None; + } + + public Proxy(string address) + { + ProxyItem = new WebProxy(address); + } + + public Proxy(string ip, int port) + { + ProxyItem = new WebProxy(ip, port); + } + + public Proxy(string ip, int port, ProxyTypeEnum proxyType) + { + ProxyItem = new WebProxy(ip, port); + ProxyType = proxyType; + } + + /// + /// Returns true if proxy can hide your ip address + /// + public bool IsAnonymous(ProxyJudgeService service) + { + if (_isChecked) + return _isAnonymous; + + _isAnonymous = service.IsProxyAnonymous(this); + _isChecked = true; + IsOnline = _isAnonymous; + return _isAnonymous; + } + + public object Clone() + { + var proxy = new Proxy + { + _isAnonymous = _isAnonymous, + _isBusy = _isBusy, + _isChecked = _isChecked, + IsOnline = IsOnline, + ProxyItem = ProxyItem, + ProxyType = ProxyType + }; + return proxy; + } + } + + public enum ProxyTypeEnum + { + Unknown, + None, + Http, + Socks4, + Socks + } +} \ No newline at end of file diff --git a/BetterHttpClient/ProxyManager.cs b/BetterHttpClient/ProxyManager.cs new file mode 100644 index 0000000..2b52bad --- /dev/null +++ b/BetterHttpClient/ProxyManager.cs @@ -0,0 +1,428 @@ +using System; +using System.Collections.Generic; +using System.Collections.Specialized; +using System.IO; +using System.Linq; +using System.Net; +using System.Text; +using System.Threading; +using BetterHttpClient.CheckService; + +namespace BetterHttpClient +{ + public class ProxyManager + { + private List _proxies = new List(); + private string _requiredString = string.Empty; + private int _numberOfAttemptsPerRequest; + private TimeSpan _timeout = TimeSpan.FromSeconds(10); + private int _numberOfAttempts = 4; + private ProxyJudgeService _proxyJudgeService; + + /// + /// Set User-Agent header. + /// + /// Default value: "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0" + public string UserAgent { get; set; } = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0"; + /// + /// Set Accept header. + /// + /// Default value: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" + public string Accept { get; set; } = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"; + /// + /// Set Referer header. + /// + /// Default value: null + public string Referer { get; set; } + /// + /// Set Accept-Language header. + /// + /// Default value: "en-US;q=0.7,en;q=0.3" + public string AcceptLanguage { get; set; } = "en-US;q=0.7,en;q=0.3"; + /// + /// Set default Accept-Encoding header. + /// + /// Default value: "gzip, deflate" + public string AcceptEncoding { get; set; } = "gzip, deflate"; + + /// + /// Set proxy check service. + /// Must derive from ProxyCheckService class. + /// + public ProxyJudgeService ProxyJudgeService + { + get { return _proxyJudgeService; } + set + { + if(value == null) + throw new ArgumentNullException(); + _proxyJudgeService = value; + } + } + /// + /// Downloaded page has to contain this tring. + /// It helps to check if returned page is the page which we watned to receive. + /// Proxy sometimes are returing some other pages. + /// Default value string.Empty + /// + public string RequiredString + { + get { return _requiredString; } + set { _requiredString = value ?? string.Empty; } + } + /// + /// Set encoding for request. + /// + /// Default value: UTF-8 + public Encoding Encoding { get; set; } = Encoding.UTF8; + /// + /// Set this to true if you want to use only anonymous proxies. + /// + /// Default value: false + public bool AnonymousProxyOnly { get; } = false; + /// + /// Timeout for request. + /// + /// Has to be greater than 5 milliseconds. + /// Default value: 10 seconds. + public TimeSpan Timeout + { + get + { + return _timeout; + } + set + { + if (value.TotalMilliseconds < 5) + throw new ArgumentOutOfRangeException("Timeout has to be greater or equal than 5 milliseconds."); + _timeout = value; + } + } + /// + /// Set how many attempts can be made to execute request on one proxy. + /// Default value is default value is equal 4 + /// + public int NumberOfAttempts + { + get { return _numberOfAttempts; } + set + { + if (value < 1) + throw new ArgumentOutOfRangeException("Value has to be greater than one."); + + _numberOfAttempts = value; + _proxyJudgeService.NumberOfAttempts = _numberOfAttempts; + } + } + + /// Proxy list + /// Set true if you want to filter proxy list and use only anonymous only + /// Proxy judge service is used to determine proxy anonymity level + /// + public ProxyManager(IEnumerable proxies, bool anonymousOnly, ProxyJudgeService proxyJudgeService) : + this(ParseProxies(proxies), anonymousOnly, proxyJudgeService) + { + } + + /// Proxy list + /// Set true if you want to filter proxy list and use only anonymous only + /// Proxy judge service is used to determine proxy anonymity level + /// + public ProxyManager(IEnumerable proxies, bool anonymousOnly, ProxyJudgeService proxyJudgeService) + { + if (proxies == null || proxyJudgeService == null) + throw new ArgumentNullException(); + foreach (Proxy proxy in proxies) + { + try + { + _proxies.Add(proxy); + } + catch (UriFormatException) + { + // parsing exception + } + } + AnonymousProxyOnly = anonymousOnly; + _proxyJudgeService = proxyJudgeService; + _numberOfAttemptsPerRequest = _proxies.Count + 1; + _proxyJudgeService.NumberOfAttempts = _numberOfAttempts; + } + public ProxyManager(string file) : this(File.ReadLines(file), false, new ProxyJudgeService()) { } + public ProxyManager(string file, bool anonymousOnly) : this(File.ReadLines(file), anonymousOnly, new ProxyJudgeService()) { } + public ProxyManager(string file, bool anonymousOnly, ProxyJudgeService service) : this(File.ReadLines(file), anonymousOnly, service) { } + + /// + /// Downloads url using GET. + /// + /// Url of webpage + /// Cookies for request. Left null if you don't want to use cookies + /// Specify custom headers for this request + /// + /// Page has returned 404 not found + public string GetPage(string url, string requiredString = null, CookieContainer cookies = null, NameValueCollection customHeaders = null) + { + return PostPage(url, null, requiredString, cookies, customHeaders); + } + + /// + /// Downloads url using POST. + /// + /// Url of webpage + /// Post values + /// Cookies for request. Left null if you don't want to use cookies + /// Specify custom headers for this request + /// Page has returned 404 not found + /// + public string PostPage(string url, NameValueCollection data, string requiredString = null, CookieContainer cookies = null, NameValueCollection customHeaders = null) + { + if (requiredString == null) + requiredString = RequiredString; + + string page = null; + int limit = 0; + + do + { + Proxy proxy = GetAvalibleProxy(); + + try + { + if (AnonymousProxyOnly) + { + if (AnonymousProxyOnly && !proxy.IsAnonymous(ProxyJudgeService)) + { + continue; + } + } + + var bytes = DownloadBytes(url, data, proxy, cookies, customHeaders); + + if (bytes != null) + { + page = Encoding.GetString(bytes); + if (!page.Contains(requiredString)) + { + proxy.IsOnline = false; + } + else + { + return page; + } + } + } + finally + { + limit++; + proxy.IsBusy = false; + } + + limit++; + } while (limit < _numberOfAttemptsPerRequest); + + throw new AllProxiesBannedException(); + } + + /// + /// Downloads url using POST. + /// + /// Url of webpage + /// Post values + /// Cookies for request. Left null if you don't want to use cookies + /// Specify custom headers for this request + /// Page has returned 404 not found + /// + public byte[] DownloadBytes(string url, NameValueCollection data, CookieContainer cookies = null, NameValueCollection customHeaders = null) + { + int limit = 0; + + do + { + Proxy proxy = GetAvalibleProxy(); + + try + { + if (AnonymousProxyOnly && !proxy.IsAnonymous(ProxyJudgeService)) + { + continue; + } + + byte[] result = DownloadBytes(url, data, proxy, cookies, customHeaders); + + if (result != null) + { + return result; + } + } + finally + { + limit++; + proxy.IsBusy = false; + } + + limit++; + } while (limit < _numberOfAttemptsPerRequest); + + throw new AllProxiesBannedException(); + } + + /// + /// Downloads url using GET. + /// + /// + /// + /// Specify custom headers for this request + /// + /// Page has returned 404 not found + public byte[] DownloadBytes(string url, CookieContainer cookies = null, NameValueCollection customHeaders = null) + { + return DownloadBytes(url, null, cookies, customHeaders); + } + /// + /// Returns first free (but busy) and working proxy. + /// + /// + /// All proxies are banned. You can't make request. + public Proxy GetAvalibleProxy() + { + lock (_proxies) + { + Proxy selectedProxy = null; + + do + { + if (_proxies.Count(t => t.IsOnline) == 0) + throw new AllProxiesBannedException(); + + selectedProxy = _proxies.FirstOrDefault(t => t.IsOnline && !t.IsBusy); + if (selectedProxy == null) + { + Thread.Sleep(35); + } + + } while (selectedProxy == null); + + selectedProxy.IsBusy = true; + return selectedProxy; + } + } + /// + /// Sets all proxies IsOnline property to true. + /// + public void SetAllProxyAsOnline() + { + lock (_proxies) + { + _proxies.ForEach(t => t.IsOnline = true); + } + } + + /// + /// Return all proxies + /// + /// + public List GetAllProxies() + { + lock(_proxies) + return CloneProxyList(_proxies); + } + + + private List CloneProxyList(List proxyInput) + { + List proxies = new List(proxyInput.Count); + proxies.AddRange(proxyInput.Select(proxy => (Proxy) proxy.Clone())); + return proxies; + } + + private byte[] DownloadBytes(string url, NameValueCollection data, Proxy proxy, CookieContainer cookies = null, NameValueCollection customHeaders = null) + { + HttpClient client = CreateHttpClient(customHeaders); + client.Proxy = proxy; + if (cookies != null) client.Cookies = cookies; + + try + { + return client.DownloadBytes(url, data); + } + catch (WebException e) + { + if (e.Response != null && (e.Response as HttpWebResponse).StatusCode == HttpStatusCode.NotFound) + throw new WebPageNotFoundException(); + proxy.IsOnline = false; + } + + return null; + } + private HttpClient CreateHttpClient(NameValueCollection customHeaders = null) + { + HttpClient client = new HttpClient(Encoding) + { + Accept = Accept, + AcceptEncoding = AcceptEncoding, + AcceptLanguage = AcceptLanguage, + NumberOfAttempts = NumberOfAttempts, + UserAgent = UserAgent, + Referer = Referer, + Timeout = Timeout + }; + + if (customHeaders != null) + { + if (customHeaders.AllKeys.Contains("Accept")) + { + client.Accept = customHeaders["Accept"]; + customHeaders.Remove("Accept"); + } + if (customHeaders.AllKeys.Contains("Accept-Encoding")) + { + client.AcceptEncoding = customHeaders["Accept-Encoding"]; + customHeaders.Remove("Accept-Encoding"); + } + if (customHeaders.AllKeys.Contains("Accept-Language")) + { + client.AcceptLanguage = customHeaders["Accept-Language"]; + customHeaders.Remove("Accept-Language"); + } + if (customHeaders.AllKeys.Contains("User-Agent")) + { + client.UserAgent = customHeaders["User-Agente"]; + customHeaders.Remove("User-Agent"); + } + if (customHeaders.AllKeys.Contains("Referer")) + { + client.Referer = customHeaders["Referer"]; + customHeaders.Remove("Referer"); + } + + client.CustomHeaders = customHeaders; + } + + return client; + } + private static IEnumerable ParseProxies(IEnumerable proxies) + { + IEnumerable proxyParsed = proxies.Select(t => + { + try + { + Proxy proxy = new Proxy(t); + return proxy; + } + catch (UriFormatException) + { + // parsing exception + return null; + } + }).Where(t => t != null); + return proxyParsed; + } + } + + public class AllProxiesBannedException : Exception + { + } + public class WebPageNotFoundException : Exception + { + } +} diff --git a/BetterHttpClient/Socks/NeverEndingStream.cs b/BetterHttpClient/Socks/NeverEndingStream.cs new file mode 100644 index 0000000..cdaf7c6 --- /dev/null +++ b/BetterHttpClient/Socks/NeverEndingStream.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BetterHttpClient.Socks +{ + public class NeverEndingStream : MemoryStream + { + public override void Close() + { + // Ignore + } + + public void ForceClose() + { + base.Close(); + } + } +} diff --git a/BetterHttpClient/Socks/Socks4HttpWebRequest.cs b/BetterHttpClient/Socks/Socks4HttpWebRequest.cs new file mode 100644 index 0000000..74538f4 --- /dev/null +++ b/BetterHttpClient/Socks/Socks4HttpWebRequest.cs @@ -0,0 +1,448 @@ +using BetterHttpClient.Socks.Extensions; +using System; +using System.Collections.Generic; +using System.Collections.Specialized; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Security; +using System.Net.Sockets; +using System.Text; +using System.Threading.Tasks; + +namespace BetterHttpClient.Socks +{ + internal class Socks4HttpWebRequest: WebRequest + { + private static readonly StringCollection validHttpVerbs = new StringCollection { "GET", "HEAD", "POST", "PUT", "DELETE", "TRACE", "OPTIONS" }; + private WebHeaderCollection _HttpRequestHeaders; + private string _method; + private byte[] _requestContentBuffer; + private NeverEndingStream _requestContentStream; + private SocksHttpWebResponse _response; + + private Socks4HttpWebRequest(Uri requestUri) + { + RequestUri = requestUri; + } + + public override int Timeout { get; set; } + + public string UserAgent + { + get { return _HttpRequestHeaders["User-Agent"]; } + set { SetSpecialHeaders("User-Agent", value ?? string.Empty); } + } + + public string Referer + { + get { return _HttpRequestHeaders["Referer"]; } + set { SetSpecialHeaders("Referer", value ?? string.Empty); } + } + + public string Accept + { + get { return _HttpRequestHeaders["Accept"]; } + set { SetSpecialHeaders("Accept", value ?? string.Empty); } + } + + public DecompressionMethods AutomaticDecompression + { + get + { + var result = DecompressionMethods.None; + string encoding = _HttpRequestHeaders["Accept-Encoding"] ?? string.Empty; + foreach (string value in encoding.Split(',')) + { + switch (value.Trim()) + { + case "gzip": + result |= DecompressionMethods.GZip; + break; + + case "deflate": + result |= DecompressionMethods.Deflate; + break; + } + } + + return result; + } + set + { + string result = string.Empty; + if ((value & DecompressionMethods.GZip) != 0) + result = "gzip"; + if ((value & DecompressionMethods.Deflate) != 0) + { + if (!string.IsNullOrEmpty(result)) + result += ", "; + result += "deflate"; + } + + SetSpecialHeaders("Accept-Encoding", result); + } + } + + public override Uri RequestUri { get; } + + public override IWebProxy Proxy { get; set; } + + public override WebHeaderCollection Headers + { + get { return _HttpRequestHeaders ?? (_HttpRequestHeaders = new WebHeaderCollection()); } + set + { + if (RequestSubmitted) + { + throw new InvalidOperationException("This operation cannot be performed after the request has been submitted."); + } + _HttpRequestHeaders = value; + } + } + + public bool RequestSubmitted { get; private set; } + + public override string Method + { + get { return _method ?? "GET"; } + set + { + if (validHttpVerbs.Contains(value)) + { + _method = value; + } + else + { + throw new ArgumentOutOfRangeException(nameof(value), $"'{value}' is not a known HTTP verb."); + } + } + } + + public override long ContentLength { get; set; } + + public override string ContentType { get; set; } + public bool AllowAutoRedirect { get; set; } = true; + + public override WebResponse GetResponse() + { + if (Proxy == null) + { + throw new InvalidOperationException("Proxy property cannot be null."); + } + if (string.IsNullOrEmpty(Method)) + { + throw new InvalidOperationException("Method has not been set."); + } + + if (RequestSubmitted) + { + return _response; + } + _response = InternalGetResponse(); + RequestSubmitted = true; + return _response; + } + + public override Stream GetRequestStream() + { + if (RequestSubmitted) + { + throw new InvalidOperationException("This operation cannot be performed after the request has been submitted."); + } + + if (_requestContentBuffer == null) + { + if (ContentLength == 0) + { + _requestContentStream = new NeverEndingStream(); + return _requestContentStream; + } + + _requestContentBuffer = new byte[ContentLength]; + } + else if (ContentLength == default(long)) + { + _requestContentBuffer = new byte[int.MaxValue]; + } + else if (_requestContentBuffer.Length != ContentLength) + { + Array.Resize(ref _requestContentBuffer, (int)ContentLength); + } + return new MemoryStream(_requestContentBuffer); + } + + public override IAsyncResult BeginGetResponse(AsyncCallback callback, object state) + { + if (Proxy == null) + { + throw new InvalidOperationException("Proxy property cannot be null."); + } + if (string.IsNullOrEmpty(Method)) + { + throw new InvalidOperationException("Method has not been set."); + } + + var task = Task.Factory.StartNew(() => { + if (RequestSubmitted) + { + return _response; + } + _response = InternalGetResponse(); + RequestSubmitted = true; + return _response; + }); + + //var task = Task.Run(() => + //{ + // if (RequestSubmitted) + // { + // return _response; + // } + // _response = InternalGetResponse(); + // RequestSubmitted = true; + // return _response; + //}); + + return task.AsApm(callback, state); + } + + public override WebResponse EndGetResponse(IAsyncResult asyncResult) + { + var task = asyncResult as Task; + + try + { + return task.Result; + } + catch (AggregateException ex) + { + if (ex.InnerException is IOException || ex.InnerException is System.Net.Sockets.SocketException) + throw new WebException("Proxy error " + ex.InnerException.Message, ex.InnerException, WebExceptionStatus.ConnectFailure, + SocksHttpWebResponse.CreateErrorResponse(HttpStatusCode.GatewayTimeout)); + throw ex.InnerException; + } + } + + public new static WebRequest Create(string requestUri) + { + return new Socks4HttpWebRequest(new Uri(requestUri)); + } + + public new static WebRequest Create(Uri requestUri) + { + return new Socks4HttpWebRequest(requestUri); + } + + private void SetSpecialHeaders(string headerName, string value) + { + _HttpRequestHeaders.Remove(headerName); + if (value.Length != 0) + { + _HttpRequestHeaders.Add(headerName, value); + } + } + + private string BuildHttpRequestMessage(Uri requestUri) + { + if (RequestSubmitted) + { + throw new InvalidOperationException("This operation cannot be performed after the request has been submitted."); + } + + // See if we have a stream instead of byte array + if (_requestContentBuffer == null && _requestContentStream != null) + { + _requestContentBuffer = _requestContentStream.ToArray(); + _requestContentStream.ForceClose(); + _requestContentStream.Dispose(); + _requestContentStream = null; + ContentLength = _requestContentBuffer.Length; + } + + var message = new StringBuilder(); + message.AppendFormat("{0} {1} HTTP/1.1\r\nHost: {2}\r\n", Method, requestUri, requestUri.Host); + + Headers.Set(HttpRequestHeader.Connection, "close"); + + // add the headers + foreach (var key in Headers.Keys) + { + string value = Headers[key.ToString()]; + if (!string.IsNullOrEmpty(value)) + message.AppendFormat("{0}: {1}\r\n", key, value); + } + + if (!string.IsNullOrEmpty(ContentType)) + { + message.AppendFormat("Content-Type: {0}\r\n", ContentType); + } + if (ContentLength > 0) + { + message.AppendFormat("Content-Length: {0}\r\n", ContentLength); + } + + // add a blank line to indicate the end of the headers + message.Append("\r\n"); + + // add content + if (_requestContentBuffer != null && _requestContentBuffer.Length > 0) + { + using (var stream = new MemoryStream(_requestContentBuffer, false)) + { + using (var reader = new StreamReader(stream)) + { + message.Append(reader.ReadToEnd()); + } + } + } + else if (_requestContentStream != null) + { + _requestContentStream.Position = 0; + + using (var reader = new StreamReader(_requestContentStream)) + { + message.Append(reader.ReadToEnd()); + } + + + _requestContentStream.ForceClose(); + _requestContentStream.Dispose(); + } + + return message.ToString(); + } + + private SocksHttpWebResponse InternalGetResponse() + { + Uri requestUri = RequestUri; + + int redirects = 0; + const int maxAutoredirectCount = 10; + while (redirects++ < maxAutoredirectCount) + { + // Loop while redirecting + + var proxyUri = Proxy.GetProxy(requestUri); + var ipAddress = GetProxyIpAddress(proxyUri); + var response = new List(); + + using (var client = new TcpClient(ipAddress.ToString(), proxyUri.Port)) + { + int timeout = Timeout; + if (timeout == 0) + timeout = 30 * 1000; + client.ReceiveTimeout = timeout; + client.SendTimeout = timeout; + var networkStream = client.GetStream(); + // auth + var buf = new byte[300]; + int index = 0; + buf[index++] = 0x04; // Version + buf[index++] = 0x01; // NMETHODS + var destIP = Dns.GetHostEntry(requestUri.DnsSafeHost).AddressList[0]; + var rawBytes = destIP.GetAddressBytes(); + var portBytes = BitConverter.GetBytes(Uri.UriSchemeHttps == requestUri.Scheme ? 443 : 80); + for (var i = portBytes.Length - 3; i >= 0; i--) + buf[index++] = portBytes[i]; + + rawBytes.CopyTo(buf, index); + index += (ushort)rawBytes.Length; + networkStream.Write(buf, 0, index); + + networkStream.Read(buf, 0, 2); + if (buf[0] != 0) + { + throw new IOException("Invalid Socks Version"); + } + if (buf[1] == 91|| buf[1] == 92|| buf[1] == 93) + { + throw new IOException("Socks Server does not support no-auth"); + } + if (buf[1] != 90) + { + throw new Exception("Socks Server did choose bogus auth"); + } + networkStream.Read(buf, 0, 2); + var rport = (ushort)IPAddress.NetworkToHostOrder((short)BitConverter.ToUInt16(buf, 0)); + + var rdest = string.Empty; + networkStream.Read(buf, 0, 4); + var v4 = BitConverter.ToUInt32(buf, 0); + rdest = new IPAddress(v4).ToString(); + + + Stream readStream = null; + if (Uri.UriSchemeHttps == requestUri.Scheme) + { + var ssl = new SslStream(networkStream); + ssl.AuthenticateAsClient(requestUri.DnsSafeHost); + readStream = ssl; + } + else + { + readStream = networkStream; + } + + string requestString = BuildHttpRequestMessage(requestUri); + + var request = Encoding.ASCII.GetBytes(requestString); + readStream.Write(request, 0, request.Length); + readStream.Flush(); + + var buffer = new byte[client.ReceiveBufferSize]; + + var readlen = 0; + do + { + readlen = readStream.Read(buffer, 0, buffer.Length); + response.AddRange(buffer.Take(readlen)); + } while (readlen != 0); + + readStream.Close(); + } + + var webResponse = new SocksHttpWebResponse(requestUri, response.ToArray()); + + if (webResponse.StatusCode == HttpStatusCode.Moved || webResponse.StatusCode == HttpStatusCode.MovedPermanently) + { + string redirectUrl = webResponse.Headers["Location"]; + if (string.IsNullOrEmpty(redirectUrl)) + throw new WebException("Missing location for redirect"); + + requestUri = new Uri(requestUri, redirectUrl); + if (AllowAutoRedirect) + { + continue; + } + return webResponse; + } + + if ((int)webResponse.StatusCode < 200 || (int)webResponse.StatusCode > 299) + throw new WebException(webResponse.StatusDescription, null, WebExceptionStatus.UnknownError, webResponse); + + return webResponse; + } + + throw new WebException("Too many redirects", null, WebExceptionStatus.ProtocolError, SocksHttpWebResponse.CreateErrorResponse(HttpStatusCode.BadRequest)); + } + + private static IPAddress GetProxyIpAddress(Uri proxyUri) + { + IPAddress ipAddress; + if (!IPAddress.TryParse(proxyUri.Host, out ipAddress)) + { + try + { + return Dns.GetHostEntry(proxyUri.Host).AddressList[0]; + } + catch (Exception e) + { + throw new InvalidOperationException($"Unable to resolve proxy hostname '{proxyUri.Host}' to a valid IP address.", e); + } + } + return ipAddress; + } + + } +} diff --git a/BetterHttpClient/Socks/SocksHttpWebRequest.cs b/BetterHttpClient/Socks/SocksHttpWebRequest.cs new file mode 100644 index 0000000..aec13d4 --- /dev/null +++ b/BetterHttpClient/Socks/SocksHttpWebRequest.cs @@ -0,0 +1,489 @@ +using System; +using System.Collections.Generic; +using System.Collections.Specialized; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Security; +using System.Net.Sockets; +using System.Text; +using System.Threading.Tasks; +using BetterHttpClient.Socks.Extensions; + +namespace BetterHttpClient.Socks +{ + internal class SocksHttpWebRequest : WebRequest + { + private static readonly StringCollection validHttpVerbs = new StringCollection { "GET", "HEAD", "POST", "PUT", "DELETE", "TRACE", "OPTIONS" }; + private WebHeaderCollection _HttpRequestHeaders; + private string _method; + private byte[] _requestContentBuffer; + private NeverEndingStream _requestContentStream; + private SocksHttpWebResponse _response; + + private SocksHttpWebRequest(Uri requestUri) + { + RequestUri = requestUri; + } + + public override int Timeout { get; set; } + + public string UserAgent + { + get { return _HttpRequestHeaders["User-Agent"]; } + set { SetSpecialHeaders("User-Agent", value ?? string.Empty); } + } + + public string Referer + { + get { return _HttpRequestHeaders["Referer"]; } + set { SetSpecialHeaders("Referer", value ?? string.Empty); } + } + + public string Accept + { + get { return _HttpRequestHeaders["Accept"]; } + set { SetSpecialHeaders("Accept", value ?? string.Empty); } + } + + public DecompressionMethods AutomaticDecompression + { + get + { + var result = DecompressionMethods.None; + string encoding = _HttpRequestHeaders["Accept-Encoding"] ?? string.Empty; + foreach (string value in encoding.Split(',')) + { + switch (value.Trim()) + { + case "gzip": + result |= DecompressionMethods.GZip; + break; + + case "deflate": + result |= DecompressionMethods.Deflate; + break; + } + } + + return result; + } + set + { + string result = string.Empty; + if ((value & DecompressionMethods.GZip) != 0) + result = "gzip"; + if ((value & DecompressionMethods.Deflate) != 0) + { + if (!string.IsNullOrEmpty(result)) + result += ", "; + result += "deflate"; + } + + SetSpecialHeaders("Accept-Encoding", result); + } + } + + public override Uri RequestUri { get; } + + public override IWebProxy Proxy { get; set; } + + public override WebHeaderCollection Headers + { + get { return _HttpRequestHeaders ?? (_HttpRequestHeaders = new WebHeaderCollection()); } + set + { + if (RequestSubmitted) + { + throw new InvalidOperationException("This operation cannot be performed after the request has been submitted."); + } + _HttpRequestHeaders = value; + } + } + + public bool RequestSubmitted { get; private set; } + + public override string Method + { + get { return _method ?? "GET"; } + set + { + if (validHttpVerbs.Contains(value)) + { + _method = value; + } + else + { + throw new ArgumentOutOfRangeException(nameof(value), $"'{value}' is not a known HTTP verb."); + } + } + } + + public override long ContentLength { get; set; } + + public override string ContentType { get; set; } + public bool AllowAutoRedirect { get; set; } = true; + + public override WebResponse GetResponse() + { + if (Proxy == null) + { + throw new InvalidOperationException("Proxy property cannot be null."); + } + if (string.IsNullOrEmpty(Method)) + { + throw new InvalidOperationException("Method has not been set."); + } + + if (RequestSubmitted) + { + return _response; + } + _response = InternalGetResponse(); + RequestSubmitted = true; + return _response; + } + + public override Stream GetRequestStream() + { + if (RequestSubmitted) + { + throw new InvalidOperationException("This operation cannot be performed after the request has been submitted."); + } + + if (_requestContentBuffer == null) + { + if (ContentLength == 0) + { + _requestContentStream = new NeverEndingStream(); + return _requestContentStream; + } + + _requestContentBuffer = new byte[ContentLength]; + } + else if (ContentLength == default(long)) + { + _requestContentBuffer = new byte[int.MaxValue]; + } + else if (_requestContentBuffer.Length != ContentLength) + { + Array.Resize(ref _requestContentBuffer, (int)ContentLength); + } + return new MemoryStream(_requestContentBuffer); + } + + public override IAsyncResult BeginGetResponse(AsyncCallback callback, object state) + { + if (Proxy == null) + { + throw new InvalidOperationException("Proxy property cannot be null."); + } + if (string.IsNullOrEmpty(Method)) + { + throw new InvalidOperationException("Method has not been set."); + } + //.net 4.0 + var task = Task.Factory.StartNew(() => { + if (RequestSubmitted) + { + return _response; + } + _response = InternalGetResponse(); + RequestSubmitted = true; + return _response; + }); + //.net 4.5 + //var task = Task.Run(() => + //{ + // if (RequestSubmitted) + // { + // return _response; + // } + // _response = InternalGetResponse(); + // RequestSubmitted = true; + // return _response; + //}); + + return task.AsApm(callback, state); + } + + public override WebResponse EndGetResponse(IAsyncResult asyncResult) + { + var task = asyncResult as Task; + + try + { + return task.Result; + } + catch (AggregateException ex) + { + if (ex.InnerException is IOException || ex.InnerException is System.Net.Sockets.SocketException) + throw new WebException("Proxy error " + ex.InnerException.Message, ex.InnerException, WebExceptionStatus.ConnectFailure, + SocksHttpWebResponse.CreateErrorResponse(HttpStatusCode.GatewayTimeout)); + throw ex.InnerException; + } + } + + public new static WebRequest Create(string requestUri) + { + return new SocksHttpWebRequest(new Uri(requestUri)); + } + + public new static WebRequest Create(Uri requestUri) + { + return new SocksHttpWebRequest(requestUri); + } + + private void SetSpecialHeaders(string headerName, string value) + { + _HttpRequestHeaders.Remove(headerName); + if (value.Length != 0) + { + _HttpRequestHeaders.Add(headerName, value); + } + } + + private string BuildHttpRequestMessage(Uri requestUri) + { + if (RequestSubmitted) + { + throw new InvalidOperationException("This operation cannot be performed after the request has been submitted."); + } + + // See if we have a stream instead of byte array + if (_requestContentBuffer == null && _requestContentStream != null) + { + _requestContentBuffer = _requestContentStream.ToArray(); + _requestContentStream.ForceClose(); + _requestContentStream.Dispose(); + _requestContentStream = null; + ContentLength = _requestContentBuffer.Length; + } + + var message = new StringBuilder(); + message.AppendFormat("{0} {1} HTTP/1.1\r\nHost: {2}\r\n", Method, requestUri, requestUri.Host); + + Headers.Set(HttpRequestHeader.Connection, "close"); + + // add the headers + foreach (var key in Headers.Keys) + { + string value = Headers[key.ToString()]; + if (!string.IsNullOrEmpty(value)) + message.AppendFormat("{0}: {1}\r\n", key, value); + } + + if (!string.IsNullOrEmpty(ContentType)) + { + message.AppendFormat("Content-Type: {0}\r\n", ContentType); + } + if (ContentLength > 0) + { + message.AppendFormat("Content-Length: {0}\r\n", ContentLength); + } + + // add a blank line to indicate the end of the headers + message.Append("\r\n"); + + // add content + if (_requestContentBuffer != null && _requestContentBuffer.Length > 0) + { + using (var stream = new MemoryStream(_requestContentBuffer, false)) + { + using (var reader = new StreamReader(stream)) + { + message.Append(reader.ReadToEnd()); + } + } + } + else if (_requestContentStream != null) + { + _requestContentStream.Position = 0; + + using (var reader = new StreamReader(_requestContentStream)) + { + message.Append(reader.ReadToEnd()); + } + + + _requestContentStream.ForceClose(); + _requestContentStream.Dispose(); + } + + return message.ToString(); + } + + private SocksHttpWebResponse InternalGetResponse() + { + Uri requestUri = RequestUri; + + int redirects = 0; + const int maxAutoredirectCount = 10; + while (redirects++ < maxAutoredirectCount) + { + // Loop while redirecting + + var proxyUri = Proxy.GetProxy(requestUri); + var ipAddress = GetProxyIpAddress(proxyUri); + var response = new List(); + + using (var client = new TcpClient(ipAddress.ToString(), proxyUri.Port)) + { + int timeout = Timeout; + if (timeout == 0) + timeout = 30 * 1000; + client.ReceiveTimeout = timeout; + client.SendTimeout = timeout; + var networkStream = client.GetStream(); + // auth + var buf = new byte[300]; + Console.WriteLine(proxyUri.Scheme); + buf[0] = 0x05; // Version + buf[1] = 0x01; // NMETHODS + buf[2] = 0x00; // No auth-method + networkStream.Write(buf, 0, 3); + + networkStream.Read(buf, 0, 2); + if (buf[0] != 0x05) + { + throw new IOException("Invalid Socks Version"); + } + if (buf[1] == 0xff) + { + throw new IOException("Socks Server does not support no-auth"); + } + if (buf[1] != 0x00) + { + throw new Exception("Socks Server did choose bogus auth"); + } + + // connect + var destIP = Dns.GetHostEntry(requestUri.DnsSafeHost).AddressList[0]; + var index = 0; + buf[index++] = 0x05; // version 5 . + buf[index++] = 0x01; // command = connect. + buf[index++] = 0x00; // Reserve = must be 0x00 + + buf[index++] = 0x01; // Address is full-qualified domain name. + var rawBytes = destIP.GetAddressBytes(); + rawBytes.CopyTo(buf, index); + index += (ushort)rawBytes.Length; + + var portBytes = BitConverter.GetBytes(Uri.UriSchemeHttps == requestUri.Scheme ? 443 : 80); + for (var i = portBytes.Length - 3; i >= 0; i--) + buf[index++] = portBytes[i]; + + + networkStream.Write(buf, 0, index); + + networkStream.Read(buf, 0, 4); + if (buf[0] != 0x05) + { + throw new IOException("Invalid Socks Version"); + } + if (buf[1] != 0x00) + { + throw new IOException($"Socks Error {buf[1]:X}"); + } + + var rdest = string.Empty; + switch (buf[3]) + { + case 0x01: // IPv4 + networkStream.Read(buf, 0, 4); + var v4 = BitConverter.ToUInt32(buf, 0); + rdest = new IPAddress(v4).ToString(); + break; + case 0x03: // Domain name + networkStream.Read(buf, 0, 1); + if (buf[0] == 0xff) + { + throw new IOException("Invalid Domain Name"); + } + networkStream.Read(buf, 1, buf[0]); + rdest = Encoding.ASCII.GetString(buf, 1, buf[0]); + break; + case 0x04: // IPv6 + var octets = new byte[16]; + networkStream.Read(octets, 0, 16); + rdest = new IPAddress(octets).ToString(); + break; + default: + throw new IOException("Invalid Address type"); + } + networkStream.Read(buf, 0, 2); + var rport = (ushort)IPAddress.NetworkToHostOrder((short)BitConverter.ToUInt16(buf, 0)); + + Stream readStream = null; + if (Uri.UriSchemeHttps == requestUri.Scheme) + { + var ssl = new SslStream(networkStream); + ssl.AuthenticateAsClient(requestUri.DnsSafeHost); + readStream = ssl; + } + else + { + readStream = networkStream; + } + + string requestString = BuildHttpRequestMessage(requestUri); + + var request = Encoding.ASCII.GetBytes(requestString); + readStream.Write(request, 0, request.Length); + readStream.Flush(); + + var buffer = new byte[client.ReceiveBufferSize]; + + var readlen = 0; + do + { + readlen = readStream.Read(buffer, 0, buffer.Length); + response.AddRange(buffer.Take(readlen)); + } while (readlen != 0); + + readStream.Close(); + } + + var webResponse = new SocksHttpWebResponse(requestUri, response.ToArray()); + + if (webResponse.StatusCode == HttpStatusCode.Moved || webResponse.StatusCode == HttpStatusCode.MovedPermanently) + { + string redirectUrl = webResponse.Headers["Location"]; + if (string.IsNullOrEmpty(redirectUrl)) + throw new WebException("Missing location for redirect"); + + requestUri = new Uri(requestUri, redirectUrl); + if (AllowAutoRedirect) + { + continue; + } + return webResponse; + } + + if ((int)webResponse.StatusCode < 200 || (int)webResponse.StatusCode > 299) + throw new WebException(webResponse.StatusDescription, null, WebExceptionStatus.UnknownError, webResponse); + + return webResponse; + } + + throw new WebException("Too many redirects", null, WebExceptionStatus.ProtocolError, SocksHttpWebResponse.CreateErrorResponse(HttpStatusCode.BadRequest)); + } + + private static IPAddress GetProxyIpAddress(Uri proxyUri) + { + IPAddress ipAddress; + if (!IPAddress.TryParse(proxyUri.Host, out ipAddress)) + { + try + { + return Dns.GetHostEntry(proxyUri.Host).AddressList[0]; + } + catch (Exception e) + { + throw new InvalidOperationException($"Unable to resolve proxy hostname '{proxyUri.Host}' to a valid IP address.", e); + } + } + return ipAddress; + } + } +} \ No newline at end of file diff --git a/BetterHttpClient/Socks/SocksHttpWebResponse.cs b/BetterHttpClient/Socks/SocksHttpWebResponse.cs new file mode 100644 index 0000000..701b228 --- /dev/null +++ b/BetterHttpClient/Socks/SocksHttpWebResponse.cs @@ -0,0 +1,268 @@ +using System; +using System.IO; +using System.IO.Compression; +using System.Net; +using System.Text; + +// Credit: https://github.com/Yozer/BetterHttpClient + +namespace BetterHttpClient.Socks +{ + public class SocksHttpWebResponse : WebResponse + { + #region Member Variables + + private WebHeaderCollection _httpResponseHeaders; + private Uri responseUri; + + #endregion + + #region Constructors + + public SocksHttpWebResponse(Uri responseUri, byte[] httpResponseMessage) + { + this.responseUri = responseUri; + SetHeadersAndResponseContent(httpResponseMessage); + } + + private SocksHttpWebResponse() + { + } + + public static SocksHttpWebResponse CreateErrorResponse(HttpStatusCode statusCode) + { + var response = new SocksHttpWebResponse + { + StatusCode = statusCode + }; + + return response; + } + + #endregion + + #region Methods + + private void SetHeadersAndResponseContent(byte[] responseMessage) + { + if (responseMessage == null || responseMessage.Length < 4) + return; + + var headers = string.Empty; + // get headers part; + var headerEnd = 0; + for (; headerEnd < responseMessage.Length - 3; headerEnd++) + { + if (responseMessage[headerEnd] == 13 && responseMessage[headerEnd + 1] == 10 && responseMessage[headerEnd + 2] == 13 && responseMessage[headerEnd + 3] == 10) + { + headers = Encoding.ASCII.GetString(responseMessage, 0, headerEnd); + break; + } + } + if (string.IsNullOrEmpty(headers)) + return; + + var headerValues = headers.Split(new[] { "\r\n" }, StringSplitOptions.RemoveEmptyEntries); + // ignore the first line in the header since it is the HTTP response code + for (var i = 1; i < headerValues.Length; i++) + { + var headerEntry = headerValues[i].Split(':'); + Headers.Add(headerEntry[0], headerEntry[1]); + } + + var statusRegex = new System.Text.RegularExpressions.Regex(@"^HTTP/\d+\.\d+ (\d+)(.*)$"); + var matches = statusRegex.Match(headerValues[0]); + if (matches.Groups.Count < 3) + throw new InvalidDataException(); + StatusCode = (HttpStatusCode)int.Parse(matches.Groups[1].Value); + StatusDescription = matches.Groups[2].Value; + + ResponseContent = new byte[responseMessage.Length - headerEnd - 4]; + Array.Copy(responseMessage, headerEnd + 4, ResponseContent, 0, ResponseContent.Length); + + string transferEncoding = Headers["Transfer-Encoding"]; + + switch (transferEncoding) + { + case "chunked": + DechunkContent(); + break; + } + + string contentEncoding = Headers["Content-Encoding"]; + + switch (contentEncoding) + { + case "gzip": + ResponseContent = Decompress(ResponseContent) ?? ResponseContent; + break; + } + } + + private void DechunkContent() + { + using (var output = new MemoryStream()) + using (var ms = new MemoryStream(ResponseContent)) + { + while (true) + { + var infoLine = new StringBuilder(); + while (true) + { + int b = ms.ReadByte(); + + if (b == -1) + // End of stream + break; + + if (b == 13) + // Ignore + continue; + + if (b == 10) + { + if (infoLine.Length == 0) + // Trim + continue; + else + break; + } + + infoLine.Append((char)b); + } + if (infoLine.Length == 0) + break; + + int chunkLength = int.Parse(infoLine.ToString(), System.Globalization.NumberStyles.HexNumber); + if (chunkLength == 0) + break; + + var buffer = new byte[chunkLength]; + ms.Read(buffer, 0, chunkLength); + output.Write(buffer, 0, buffer.Length); + } + + ResponseContent = output.ToArray(); + } + } + + #endregion + + #region WebResponse Members + + public HttpStatusCode StatusCode { get; private set; } + + public string StatusDescription { get; private set; } + + public override Uri ResponseUri + { + get + { + return this.responseUri; + } + } + + private static readonly byte[] GZipHeaderBytes = { 0x1f, 0x8b, 8, 0, 0, 0, 0, 0, 4, 0 }; + private static readonly byte[] GZipLevel10HeaderBytes = { 0x1f, 0x8b, 8, 0, 0, 0, 0, 0, 2, 0 }; + + public static bool IsPossiblyGZippedBytes(byte[] a) + { + var yes = a.Length > 10; + + if (!yes) + { + return false; + } + + var fail = false; + for (var i = 0; i < 10; i++) + { + if (a[i] != GZipHeaderBytes[i]) + { + fail = true; + break; + } + } + + if (!fail) + return true; + + for (var i = 0; i < 10; i++) + { + if (a[i] != GZipLevel10HeaderBytes[i]) + { + return false; + } + } + + return true; + } + + private byte[] Decompress(byte[] gzip) + { + try + { + using (var stream = new GZipStream(new MemoryStream(gzip), CompressionMode.Decompress)) + { + const int size = 4096; + var buffer = new byte[size]; + using (var memory = new MemoryStream()) + { + var count = 0; + do + { + count = stream.Read(buffer, 0, size); + if (count > 0) + { + memory.Write(buffer, 0, count); + } + } while (count > 0); + return memory.ToArray(); + } + } + } + catch (Exception) + { + return null; + } + } + + public override Stream GetResponseStream() + { + return ResponseContent.Length == 0 ? Stream.Null : new MemoryStream(ResponseContent); + } + + public override void Close() + { + /* the base implementation throws an exception */ + } + + public override WebHeaderCollection Headers + { + get + { + if (_httpResponseHeaders == null) + { + _httpResponseHeaders = new WebHeaderCollection(); + } + return _httpResponseHeaders; + } + } + + public override long ContentLength + { + get { return ResponseContent.Length; } + set { throw new NotSupportedException(); } + } + + #endregion + + #region Properties + + private byte[] ResponseContent { get; set; } + + public Encoding CorrectEncoding { get; set; } + + #endregion + } +} \ No newline at end of file diff --git a/BetterHttpClient/Socks/TaskExtensions.cs b/BetterHttpClient/Socks/TaskExtensions.cs new file mode 100644 index 0000000..f490e28 --- /dev/null +++ b/BetterHttpClient/Socks/TaskExtensions.cs @@ -0,0 +1,34 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace BetterHttpClient.Socks.Extensions +{ + public static class TaskExtensions + { + public static IAsyncResult AsApm(this Task task, + AsyncCallback callback, + object state) + { + if (task == null) + throw new ArgumentNullException("task"); + + var tcs = new TaskCompletionSource(state); + task.ContinueWith(t => + { + if (t.IsFaulted) + tcs.TrySetException(t.Exception.InnerExceptions); + else if (t.IsCanceled) + tcs.TrySetCanceled(); + else + tcs.TrySetResult(t.Result); + + if (callback != null) + callback(tcs.Task); + }, TaskScheduler.Default); + return tcs.Task; + } + } +} diff --git a/README.md b/README.md new file mode 100644 index 0000000..ee6a16a --- /dev/null +++ b/README.md @@ -0,0 +1,74 @@ +# BetterHttpClient + +BetterHttpClient will help you making http and https requests with full cookie support. + +It supports three types of proxy servers: + - HTTP + - HTTPS + - Socks5 + +# Http Client +- Http GET request +```cs +HttpClient client = new HttpClient +{ + UserAgent = "My custom user-agent", + Referer = "My referer", + Encoding = Encoding.UTF8, + AcceptEncoding = "gzip, deflate" // setup transfer compression algorithm +}; +string page = client.Get("http://www.google.com"); +``` + +- Http POST request +```cs +HttpClient client = new HttpClient(); // if no headers are specified defaults will be used +string page = client.Post("https://httpbin.org/post", new NameValueCollection +{ + {"custname", "Some post data"}, +}); +``` + +- Https GET request using proxy + +HttpClient will detect proxy type (http/s, socks5) if you don't provide such info. +```cs +string proxyAddress = "218.200.66.196:8080"; +HttpClient client = new HttpClient(new Proxy(proxyAddress)) +{ + UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0" +}; + +string page = client.Get("https://httpbin.org/get"); +``` + +# Proxy Manager +This class will help you managing large amount of proxy servers. + +You can load proxy list from file and then execute GET or POST requests. + +If some proxy will be dead ProxyManager will mark them as offline and won't use in future requests. + +To perform http/s request it will use first Not Busy and Online proxy (and check for an anonymity level if anonymousOnly is true). + +Set anonymousOnly to true if you want to force ProxyManager to use only proxies that really hide your ip. For more info look at ProxyJudgeService class. +Methods +- GetPage +- PostPage +- DownloadBytes + +are thread safe. + +Example usage: +```cs +ProxyManager proxyManager = new ProxyManager("proxy_list.txt", anonymousOnly: true) +{ + UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0" + PreserveCookies = false, // if true it will use the same cookies between distinct requests + Timeout = TimeSpan.FromSeconds(10), + NumberOfAttempts = 2 // max number of attempts that can be made per one request +}; + +// ProxyManager will choose first working proxy and use it to perform GET request +string page = proxyManager.GetPage("http://example.com"); +``` diff --git a/UnitTestBetterHttpClient/Properties/AssemblyInfo.cs b/UnitTestBetterHttpClient/Properties/AssemblyInfo.cs new file mode 100644 index 0000000..1195150 --- /dev/null +++ b/UnitTestBetterHttpClient/Properties/AssemblyInfo.cs @@ -0,0 +1,36 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("UnitTestBetterHttpClient")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("UnitTestBetterHttpClient")] +[assembly: AssemblyCopyright("Copyright © 2015")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("59acc7d1-547c-4e1c-8de3-ba5351c1a00b")] + +// Version information for an assembly consists of the following four values: +// +// Major Version +// Minor Version +// Build Number +// Revision +// +// You can specify all the values or you can default the Build and Revision Numbers +// by using the '*' as shown below: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/UnitTestBetterHttpClient/UnitTestBetterHttpClient.csproj b/UnitTestBetterHttpClient/UnitTestBetterHttpClient.csproj new file mode 100644 index 0000000..850cfa4 --- /dev/null +++ b/UnitTestBetterHttpClient/UnitTestBetterHttpClient.csproj @@ -0,0 +1,98 @@ + + + + Debug + AnyCPU + {59ACC7D1-547C-4E1C-8DE3-BA5351C1A00B} + Library + Properties + UnitTestBetterHttpClient + UnitTestBetterHttpClient + v4.0 + 512 + {3AC096D0-A1C2-E12C-1390-A8335801FDAB};{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC} + 10.0 + $(MSBuildExtensionsPath32)\Microsoft\VisualStudio\v$(VisualStudioVersion) + $(ProgramFiles)\Common Files\microsoft shared\VSTT\$(VisualStudioVersion)\UITestExtensionPackages + False + UnitTest + + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + ..\packages\Newtonsoft.Json.7.0.1\lib\net40\Newtonsoft.Json.dll + True + + + + + + + + + + + + + + + + + + + + + + + {fbc5f420-3f80-4a20-a9ec-b6574e18a1d3} + BetterHttpClient + + + + + + + + + + False + + + False + + + False + + + False + + + + + + + + \ No newline at end of file diff --git a/UnitTestBetterHttpClient/UnitTestHttpClient.cs b/UnitTestBetterHttpClient/UnitTestHttpClient.cs new file mode 100644 index 0000000..2cc1e49 --- /dev/null +++ b/UnitTestBetterHttpClient/UnitTestHttpClient.cs @@ -0,0 +1,185 @@ +using System; +using System.Collections.Specialized; +using System.Text; +using BetterHttpClient; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Newtonsoft.Json; + +namespace UnitTestBetterHttpClient +{ + [TestClass] + public class UnitTestHttpClient + { + private const string HttpsProxy = "210.245.25.229:3128"; + private const string Socksproxy = "124.207.126.18:1080"; + [TestMethod] + public void TestGet() + { + HttpClient client = new HttpClient + { + UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0" + }; + + string page = client.Get("http://www.google.com"); + Assert.IsTrue(page.Contains("Google")); + } + + [TestMethod] + public void TestPost() + { + HttpClient client = new HttpClient + { + UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0", + ContentType = "application/x-www-form-urlencoded; charset=UTF-8" + + }; + //client.Headers.Add("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8"); + + string page = client.Post("http://www.newrank.cn/xdnphb/detail/getAccountArticle", new NameValueCollection + { + { "flag","true"}, + {"uuid","DCFCC585F745FF5401FD33213B1178EB" }, + { "nonce","0f3ac361b" }, + {"xyz","4d86efaf832909233403482879de5cdd" } + }); + Assert.IsTrue(!string.IsNullOrEmpty(page)); + //Form root = JsonConvert.DeserializeObject(page).form; + + //Assert.AreEqual(root.custname, customerName); + //Assert.AreEqual(root.custtel, phone); + //Assert.AreEqual(root.custemail, email); + //Assert.AreEqual(root.size, size); + //Assert.AreEqual(root.topping, topping); + //Assert.AreEqual(root.delivery, delivery); + //Assert.AreEqual(root.comments, comments); + + } + + [TestMethod] + public void TestUserAgent() + { + HttpClient client = new HttpClient + { + UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0" + }; + + string page = client.Get("https://httpbin.org/user-agent"); + Assert.IsTrue(page.Contains(client.UserAgent)); + } + + [TestMethod] + public void TestGzipDecodingAndReferer() + { + HttpClient client = new HttpClient + { + UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0", + Referer = "https://httpbin.org/" + }; + + string page = client.Get("https://httpbin.org/gzip"); + Assert.IsTrue(page.Contains(client.UserAgent)); + // check for referer + Assert.IsTrue(page.Contains("https://httpbin.org/")); + } + + [TestMethod] + public void TestHttpProxy() + { + HttpClient client = new HttpClient(new Proxy(HttpsProxy)) + { + UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0", + Encoding = Encoding.GetEncoding("iso-8859-2"), + AcceptEncoding = "gzip" + }; + + string page = client.Get("http://darkwarez.pl"); + Assert.IsTrue(page.Contains("Polskie Forum Warez! Najnowsze linki")); + } + [TestMethod] + public void TestHttpsProxy() + { + HttpClient client = new HttpClient(new Proxy(HttpsProxy)) + { + UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0" + }; + + string page = client.Get("https://httpbin.org/get"); + Assert.IsTrue(page.Contains(client.UserAgent)); + } + [TestMethod] + public void TestSocksHttpProxy() + { + Proxy proxy = new Proxy("124.207.126.15", 1080, ProxyTypeEnum.Socks); + HttpClient client = new HttpClient(proxy) + { + UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0", + Encoding = Encoding.GetEncoding("iso-8859-2"), + AcceptEncoding = "deflate" + }; + + string page = client.Get("https://httpbin.org/get"); + Assert.IsTrue(page.Contains("darkwarez.pl - Gry, Muzyka, Filmy, Download")); + } + [TestMethod] + public void TestSocks4HttpProxy() + { + Proxy proxy = new Proxy("115.29.161.103", 1080, ProxyTypeEnum.Socks4); + + HttpClient client = new HttpClient(proxy) + { + UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0", + Encoding = Encoding.GetEncoding("utf-8"), + //AcceptEncoding = "deflate" + }; + + string page = client.Get("http://proxy.mimvp.com/free.php?proxy=in_socks"); + Assert.IsTrue(page.Contains("\"origin\": \"115.29.161.103\"")); + } + [TestMethod] + public void TestSocksHttpsProxyDeflateEncoding() + { + HttpClient client = new HttpClient(new Proxy(Socksproxy)) + { + UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0", + AcceptEncoding = "deflate" + }; + + string page = client.Get("https://httpbin.org/get"); + Assert.IsTrue(page.Contains(client.UserAgent)); + } + [TestMethod] + public void TestSocksHttpsProxyGzipEndcoding() + { + HttpClient client = new HttpClient(new Proxy(Socksproxy)) + { + UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0", + AcceptEncoding = "gzip" + }; + + string page = client.Get("https://httpbin.org/get"); + Assert.IsTrue(page.Contains(client.UserAgent)); + } + public class Form + { + public string comments { get; set; } + public string custemail { get; set; } + public string custname { get; set; } + public string custtel { get; set; } + public string delivery { get; set; } + public string size { get; set; } + public string topping { get; set; } + } + + public class RootObject + { + public object args { get; set; } + public string data { get; set; } + public object files { get; set; } + public Form form { get; set; } + public object headers { get; set; } + public object json { get; set; } + public string origin { get; set; } + public string url { get; set; } + } + } +} diff --git a/UnitTestBetterHttpClient/packages.config b/UnitTestBetterHttpClient/packages.config new file mode 100644 index 0000000..1b9f231 --- /dev/null +++ b/UnitTestBetterHttpClient/packages.config @@ -0,0 +1,4 @@ + + + + \ No newline at end of file