C# Http Get Request Program Example
Create a new console application project and you might want to use HttpGetRequestCS as the project and solution names.
|
|
Rename the source file to HttpGetClass to reflect the application.
Add the following using directives.
using System; using System.IO; using System.Net; using System.Text; using System.ComponentModel; using System.Collections; |
Add the following class variables.
public Uri ResourceUri; public IWebProxy HttpProxy; public string LocalSavePath; public bool RecursiveGet; public ArrayList RetrievedUri; |
Next, add the code for the HttpGetClass class constructor.
public HttpGetClass() { ResourceUri = new Uri("http://www.google.com/intl/en/images/about_logo.gif"); // Examples: // http://www.cuilimg.com/static/v2/images/new/flattened/cuil-home_id.png // http://www.google.com/index.html // http://www.cuil.com/search?q=MFC%20programming%20tutorials HttpProxy = WebRequest.DefaultWebProxy; LocalSavePath = "."; RecursiveGet = false; RetrievedUri = new ArrayList(); } |
Then, add the usage() method.
/// <summary> /// Displays simple usage information for the application. /// </summary> static void usage() { Console.WriteLine("Usage: Executable_file_name [-u URI] [-p proxy] [-s local-path]"); Console.WriteLine("Available options:"); Console.WriteLine(" -u URI URI to download (along with linked content)"); Console.WriteLine(" -p proxy Name of proxy server"); Console.WriteLine(" -r Retrieve referenced images and text residing"); Console.WriteLine(" on the same host."); Console.WriteLine(" -s local-path Local path to save content at"); Console.WriteLine(); } |
Next, add the CreateFile() method.
/// <summary> /// Creates any subdirectories and opens the file for writing to. The path is stripped /// from the URI to build the local path to save the file to. This path is appended /// to the local save path supplied. A file is then opened with the same name as /// the retrieved file and the FileStream to it is returned. /// </summary> /// <param name="localSavePath">Local path to append saved resources paths to</param> /// <param name="uriName">URI of destination resource being saved</param> public FileStream CreateFile(string localSavePath, Uri uriName) { FileStream localFile = null; string fileName;
try { string[ ] uriSegments = uriName.Segments; string localDirs = "";
// Retrieve the directory path to the file for (int i = 0; i < uriSegments.Length - 1; i++) { localDirs += uriSegments[i]; }
if (uriSegments.Length > 1) { // Replace the forward slashes with back slashes Console.WriteLine("Replacing the forward slashes with back slashes..."); localDirs = localDirs.Replace(@"/", @"\");
// Remove the escaped spaces Console.WriteLine("Removing the escaped spaces..."); string temp = localDirs.Replace(@"%20", " "); if (temp != null) localDirs = temp;
// Console.WriteLine(" Creating directory: {0}", localSavePath + @"\" + localDirs ); Console.WriteLine("Creating local directory..."); Directory.CreateDirectory(localSavePath + @"\" + localDirs); fileName = uriSegments[uriSegments.Length - 1]; } else { Console.WriteLine("Else, using defaults..."); localDirs = @"\"; fileName = "default.html"; }
// Open the file to write to Console.WriteLine("Opening the file to write to..."); string saveFileName = localSavePath + localDirs + fileName; localFile = File.Open( saveFileName, System.IO.FileMode.Create, System.IO.FileAccess.Write, System.IO.FileShare.None );
Console.WriteLine("Created File: {0}", saveFileName); } catch (Exception ex) { Console.WriteLine("WriteHttpContentToFile failed: {0}", ex.Message); Console.WriteLine("Stack:\n{0}", ex.StackTrace); if (localFile != null) localFile.Close(); } return localFile; } |
Add the ParseHtmlForReferences() method.
/// <summary> /// Parse the HTML content for any links to images and text/HTML documents that /// reside on the same host. For each link call the GetResource method to download /// that image and save it to the local save path. /// </summary> /// <param name="baseUri">URI of resource being parsed for links</param> /// <param name="httpContent">Text content of retrieved resource</param> /// <param name="proxy">Proxy server (if required) to use</param> /// <param name="localSavePath">Local path to append saved resources paths to</param> public void ParseHtmlForReferences(Uri baseUri, string httpContent) { int start, copyCount = 0; string[ ] match = { "<a href=\"", "src=\"" }; char[ ] refFile = new char[1024];
// Search for '<a href="' and 'src="' strings to indicate links to other resources. Console.WriteLine("Searching for \'<a href=\"\' and \'src=\"\' strings to indicate links to other resources..."); for (int i = 0; i < match.Length; i++) { start = 0;
// Search the entire contents of the buffer for occurrences of each match string while (true) { // Find the first reference start = httpContent.IndexOf(match[i], start); if (start == -1) break; // Break if not present
// Offset start to end of match string (so it points the the resource link) start = start + match[i].Length;
// Calculate how many characters make up the link copyCount = httpContent.IndexOf("\"", start); if (copyCount == -1) break;
copyCount = copyCount - start;
// Copy the link to a new string httpContent.CopyTo(start, refFile, 0, copyCount);
string x = new string(refFile, 0, copyCount); Uri newUri = new Uri(baseUri, x);
if ((newUri.Host == baseUri.Host) && (!this.RetrievedUri.Contains(newUri))) { // If link is hosted on the same computer, download it GetResource(newUri, true); Console.WriteLine("\n"); } } } } |
Add the GetResource() method.
/// <summary> /// Retrieve the resource specified by the string URI address. The HTTP proxy /// can be specified if needed to go outside the local network. /// </summary> /// <param name="resourceName">String URI to retrieve</param> /// <param name="proxy">Proxy server to use to access resource</param> /// <param name="localSavePath">Local path to append saved resources paths to</param> /// <returns></returns> public void GetResource(Uri getUri, bool recurse) { HttpWebRequest httpRequest = null; HttpWebResponse httpResponse = null; FileStream localFile = null; string httpContent = null;
try { RetrievedUri.Add(getUri); Console.WriteLine("Retrieving: {0}", getUri.AbsoluteUri);
// Create the HTTP request object Console.WriteLine("Creating the HTTP request object..."); httpRequest = (HttpWebRequest)WebRequest.Create(getUri.AbsoluteUri); // Set some HTTP specific headers Console.WriteLine("Setting some HTTP specific headers..."); httpRequest.UserAgent = "My User Agent/1.0"; // If a proxy was specified create an instance of the WebProxy with it Console.WriteLine("If a proxy was specified create an instance of the WebProxy with it..."); httpRequest.Proxy = HttpProxy; // Get the response object Console.WriteLine("Getting the response object..."); httpResponse = (HttpWebResponse)httpRequest.GetResponse();
byte[ ] byteContent; long byteCount = 0, progress = 0;
// Create the file where resource is to be saved Console.WriteLine("Creating the file where resource is to be saved..."); localFile = CreateFile(LocalSavePath, getUri); Console.WriteLine("Response Type: {0}", httpResponse.ContentType ); if (httpResponse.ContentType.StartsWith(@"image")) { // If this resource is an image, retrieve the content using the binary writer. Console.WriteLine("If this resource is a binary, retrieve the content using the binary writer..."); BinaryReader reader = new BinaryReader(httpResponse.GetResponseStream());
byte[ ] responseBytes;
// Read the response in 4KB chunks Console.Write("Reading the response in 4KB chunks: "); while (true) { responseBytes = reader.ReadBytes(4096); byteCount += responseBytes.Length; if (responseBytes.Length == 0) break; localFile.Write(responseBytes, 0, responseBytes.Length); // Print progress indicator progress = (byteCount * 100) / httpResponse.ContentLength; Console.Write(@"{0}%", progress.ToString().PadLeft(2)); Console.Write("\b\b\b"); } Console.WriteLine(); } else if (httpResponse.ContentType == @"text/html") { // If the resource is HTML text, retrieve using the text stream reader. Console.WriteLine("If the resource is HTML text, retrieve using the text stream reader..."); StreamReader reader = new StreamReader(httpResponse.GetResponseStream(), Encoding.UTF8);
httpContent = reader.ReadToEnd(); byteContent = Encoding.ASCII.GetBytes(httpContent); localFile.Write(byteContent, 0, byteContent.Length);
// For HTML documents, we'll parse them for additional links so // close the open handles as this is a recursive call and we // don't need them anymore. reader.Close(); reader = null; localFile.Close(); localFile = null;
if (recurse == true) { ParseHtmlForReferences(getUri, httpContent); } } } catch (WebException wex) { Console.WriteLine("Exception occurred on request: {0}", wex.Message); Console.WriteLine("Status code: {0}", wex.Status);
if (wex.Status == WebExceptionStatus.ProtocolError) { // If there was a protocol error then the response object is // valid but there was an error retrieving the response. httpResponse = (HttpWebResponse)wex.Response; Console.WriteLine("\nThe protocol returned was: {0}", httpResponse.StatusCode.ToString()); httpResponse.Close(); httpResponse = null; } throw; } finally { // Close the resources if still open if (localFile != null) localFile.Close(); if (httpResponse != null) httpResponse.Close(); } } |
Finally, add the Main() code. Make sure you are not missing the matching braces.
/// <summary> /// This is the main function which parses the command line, initializes the proxy /// if present, and calls the routine to retrieve the specified URI. /// </summary> /// <param name="args">Arguments passed to application</param> static void Main(string[ ] args) { HttpGetClass httpGet = new HttpGetClass(); usage(); // Parse the command line for (int i = 0; i < args.Length; i++) { try { if ((args[i][0] == '-') || (args[i][0] == '/')) { switch (Char.ToLower(args[i][1])) { case 'u': // URI to download (get) httpGet.ResourceUri = new Uri(args[++i]); break; case 'p': // Name of proxy server to use httpGet.HttpProxy = new WebProxy(args[++i]); break; case 'r': // Retrieve all referenced images and text on the same host httpGet.RecursiveGet = true; break; case 's': // Local save path to append to retrieved resources httpGet.LocalSavePath = args[++i]; break; default: usage(); return; } } } catch { usage(); return; } }
// Initialize the proxy server and retrieve the resources Console.WriteLine("Initializing the proxy server and retrieve the resources..."); try { httpGet.GetResource(httpGet.ResourceUri, httpGet.RecursiveGet); } catch (Exception ex) { Console.WriteLine("Exception occurred: {0}", ex.ToString()); } return; } |
Build and run the project.
![]() |
|
The following is a sample output with default arguments.
The saved gif file is shown below.
The following is a sample output that downloads an aspx file. To traverse and download all the links instead a single link, we need to make the program traverses, retrieves and downloads recursively.