VB .NET Http Get Request Program Example
Create a new class library project and you might want to use HttpGetRequestVB as the project and solution names.
|
|
Rename the source file to HttpGetClass to reflect the application that we supposed to develop.
Add the following codes and you may discard the comments.
' This sample illustrates using the HttpWebRequest class to retrieve the ' given URI. The retrieved data is written to a local file. If the -r ' option is specified all the linked text and images URIs are retrieved ' as well. ' ' Usage: ' Executable_file_name [-u URI] [-p proxy] [-s local-path] [-r] ' -u URI URI to download (along with linked content) ' -p proxy Name of proxy server ' -r Retrieve referenced images and text residing ' on the same host. ' -s local-path Local path to save content at ' ' Sample usage: ' Retrieve the given URI and all referenced text and image objects: ' Executable_file_name -u http://myweb -s c:\web -r ' Imports System Imports System.IO Imports System.Net Imports System.Text Imports System.ComponentModel Imports System.Collections
Public Class HttpGetClass Public ResourceUri As Uri Public HttpProxy As IWebProxy Public LocalSavePath As String Public RecursiveGet As Boolean Public RetrievedUri As ArrayList
Public Sub New() ResourceUri = New Uri("http://www.google.com/intl/en/images/about_logo.gif") ' Examples: ' http://www.cuilimg.com/static/v2/images/new/flattened/cuil-home_id.png ' http://www.google.com/index.html ' http://www.cuil.com/search?q=MFC%20programming%20tutorials
HttpProxy = WebRequest.DefaultWebProxy() LocalSavePath = "." RecursiveGet = False RetrievedUri = New ArrayList() End Sub
' Displays simple usage information for the application. Shared Sub usage() Console.WriteLine("Usage: Executable_file_name [-u URI] [-p proxy] [-s local-path]") Console.WriteLine() Console.WriteLine(" -u URI URI to download (along with linked content)") Console.WriteLine(" -p proxy Name of proxy server") Console.WriteLine(" -r Retrieve referenced images and text residing") Console.WriteLine(" on the same host.") Console.WriteLine(" -s local-path Local path to save content at") Console.WriteLine() End Sub
' Creates any subdirectories and opens the file for writing to. The path is stripped ' from the URI to build the local path to save the file to. This path is appended ' to the local save path supplied. A file is then opened with the same name as ' the retrieved file and the FileStream to it is returned. ' <param name="localSavePath">Local path to append saved resources paths to</param> ' <param name="uriName">URI of destination resource being saved</param> Public Function CreateFile(ByVal localSavePath As String, ByVal uriName As Uri) As FileStream Dim localFile As FileStream = Nothing Dim fileName As String
Try Dim uriSegments() As String = uriName.Segments Dim localDirs As String = ""
' Retrieve the directory path to the file Console.WriteLine("Retrieving the directory path to the file...") Dim i As Integer For i = 0 To uriSegments.Length - 2 localDirs += uriSegments(i) Next
If (uriSegments.Length > 1) Then ' Replace the forward slashes with back slashes Console.WriteLine("Replacing the forward slashes with back slashes...") localDirs = localDirs.Replace("/", "\")
' Remove the escaped spaces Console.WriteLine("Removing the escaped spaces...") Dim temp As String temp = localDirs.Replace("%20", " ") If (Not IsNothing(temp)) Then localDirs = temp End If Console.WriteLine(" Creating directory: {0}", localSavePath + "\" + localDirs) Directory.CreateDirectory(localSavePath + "\" + localDirs) fileName = uriSegments(uriSegments.Length - 1) Else localDirs = "\" fileName = "default.html" End If
' Open the file to write to Console.WriteLine("Opening the file to write to...") Dim saveFileName As String saveFileName = localSavePath + localDirs + fileName localFile = File.Open( _ saveFileName, _ System.IO.FileMode.Create, _ System.IO.FileAccess.Write, _ System.IO.FileShare.None _ )
Console.WriteLine("Created File: {0}", saveFileName)
Catch ex As Exception Console.WriteLine("WriteHttpContentToFile failed: {0}", ex.Message) Console.WriteLine("Stack:\n{0}", ex.StackTrace) If (Not IsNothing(localFile)) Then localFile.Close() End If End Try CreateFile = localFile End Function ' Parse the HTML content for any links to images and text/HTML documents that ' reside on the same host. For each link call the GetResource method to download ' that image and save it to the local save path. ' <param name="baseUri">URI of resource being parsed for links</param> ' <param name="httpContent">Text content of retrieved resource</param> ' <param name="proxy">Proxy server (if required) to use</param> ' <param name="localSavePath">Local path to append saved resources paths to</param> Public Sub ParseHtmlForReferences(ByVal baseUri As Uri, ByVal httpContent As String) Dim start As Integer Dim copyCount As Integer = 0 Dim match() As String = {"<a href=""", "src="""} Dim refFile(1024) As Char
Dim i As Integer ' Search for '<a href="' and 'src="' strings to indicate links to other resources.
For i = 0 To match.Length - 1 start = 0
' Search the entire contents of the buffer for occurrences of each match string Console.WriteLine("Searching the entire contents of the buffer for occurrences of each match string...") While (True) ' Find the first reference start = httpContent.IndexOf(match(i), start) If (start = -1) Then Exit While 'Break if not present End If
' Offset start to end of match string (so it points the the resource link) start = start + match(i).Length ' Calculate how many characters make up the link copyCount = httpContent.IndexOf("""", start)
If (copyCount = -1) Then Exit While End If
copyCount = copyCount - start
' Copy the link to a new string httpContent.CopyTo(start, refFile, 0, copyCount)
Dim x As String = New String(refFile, 0, copyCount) Dim newUri As Uri = New Uri(baseUri, x)
If ((newUri.Host = baseUri.Host) And _ (Not Me.RetrievedUri.Contains(newUri))) Then ' If link is hosted on the same computer, download it GetResource(newUri, True) Console.WriteLine("\n") End If End While Next End Sub
' Retrieve the resource specified by the string URI address. The HTTP proxy ' can be specified if needed to go outside the local network. ' <param name="resourceName">String URI to retrieve</param> ' <param name="proxy">Proxy server to use to access resource</param> ' <param name="localSavePath">Local path to append saved resources paths to</param> Public Sub GetResource(ByVal getUri As Uri, ByVal recurse As Boolean) Dim httpRequest As HttpWebRequest = Nothing Dim httpResponse As HttpWebResponse = Nothing Dim localFile As FileStream = Nothing Dim httpContent As String = Nothing
Try RetrievedUri.Add(getUri) Console.WriteLine("Retrieving: {0}", getUri.AbsoluteUri) ' Create the HTTP request object Console.WriteLine("Creating the HTTP request object...") httpRequest = CType(WebRequest.Create(getUri.AbsoluteUri), HttpWebRequest) ' Set some HTTP specific headers Console.WriteLine("Setting some HTTP specific headers...") httpRequest.UserAgent = "My User Agent/1.0" ' If a proxy was specified create an instance of the WebProxy with it Console.WriteLine("If a proxy was specified create an instance of the WebProxy with it...") httpRequest.Proxy = HttpProxy ' Get the response object Console.WriteLine("Getting the response object...") httpResponse = CType(httpRequest.GetResponse(), HttpWebResponse)
Dim byteContent() As Byte Dim byteCount As Long = 0 Dim progress As Long = 0
' Create the file where resource is to be saved Console.WriteLine("Creating the file where resource is to be saved...") localFile = CreateFile(LocalSavePath, getUri) Console.WriteLine("Response Type: {0}", httpResponse.ContentType) If (httpResponse.ContentType.StartsWith("image")) Then ' If this resource is an image, retrieve the content using the ' binary writer. Console.WriteLine("The resource is binary, retrieve the content using the binary writer....") Dim reader As BinaryReader = New BinaryReader(httpResponse.GetResponseStream())
Dim responseBytes() As Byte
' Read the response in 4KB chunks Console.WriteLine("Read the response in 4KB chunks...") Console.Write("Request: ") While (True) responseBytes = reader.ReadBytes(4096) byteCount += responseBytes.Length If (responseBytes.Length = 0) Then Exit While End If localFile.Write(responseBytes, 0, responseBytes.Length) ' Print progress indicator progress = (byteCount * 100) / httpResponse.ContentLength End While Console.Write("{0}%", progress.ToString().PadLeft(2)) Console.WriteLine() ElseIf (httpResponse.ContentType = "text/html") Then ' If the resource is HTML text, retrieve using the text stream reader. Console.WriteLine("The resource is HTML text, retrieve using the text stream reader...")
Dim reader As StreamReader = New StreamReader(httpResponse.GetResponseStream(), Encoding.UTF8)
httpContent = reader.ReadToEnd() byteContent = Encoding.ASCII.GetBytes(httpContent) localFile.Write(byteContent, 0, byteContent.Length)
' For HTML documents, we'll parse them for additional links so ' close the open handles as this is a recursive call and we ' don't need them anymore. Console.WriteLine("HTML text, getting more info...") reader.Close() reader = Nothing localFile.Close() localFile = Nothing If (recurse = True) Then ParseHtmlForReferences(getUri, httpContent) End If End If Catch wex As WebException Console.WriteLine("Exception occurred on request: {0}", wex.Message) Console.WriteLine("Status code: {0}", wex.Status)
If (wex.Status = WebExceptionStatus.ProtocolError) Then ' If there was a protocol error then the response object is ' valid but there was an error retrieving the response. httpResponse = CType(wex.Response, HttpWebResponse) Console.WriteLine("\nThe protocol returned was: {0}", httpResponse.StatusCode.ToString()) httpResponse.Close() httpResponse = Nothing End If Throw Finally ' Close the resources if still open If (Not IsNothing(localFile)) Then localFile.Close() End If If (Not IsNothing(httpResponse)) Then httpResponse.Close() End If End Try End Sub
' This is the main function which parses the command line, initializes the proxy ' if present, and calls the routine to retrieve the specified URI. ' <param name="args">Arguments passed to application</param> Shared Sub Main() Dim httpGet As HttpGetClass = New HttpGetClass()
' Parse the command line Dim args As String() = Environment.GetCommandLineArgs() Dim i As Integer
usage()
For i = 1 To args.GetUpperBound(0) Try Dim CurArg() As Char = args(i).ToCharArray(0, args(i).Length) If (CurArg(0) = "-") Or (CurArg(0) = "/") Then Select Case Char.ToLower(CurArg(1), System.Globalization.CultureInfo.CurrentCulture) Case "u" ' URI to download (get) i = i + 1 httpGet.ResourceUri = New Uri(args(i)) Case "p" ' Name of proxy server to use i = i + 1 httpGet.HttpProxy = New WebProxy(args(i)) Case "r" ' Retrieve all referenced images and text on the same host httpGet.RecursiveGet = True Case "s" ' Local save path to append to retrieved resources i = i + 1 httpGet.LocalSavePath = args(i) Case Else usage() Exit Sub End Select End If Catch e As Exception usage() Exit Sub End Try Next ' Initialize the proxy server and retrieve the resources Console.WriteLine("Initializing the proxy server and retrieve the resources...") Try httpGet.GetResource(httpGet.ResourceUri, httpGet.RecursiveGet) Catch ex As Exception Console.WriteLine("Exception occurred: {0}", ex.ToString()) End Try End Sub End Class
|
In order to test this program, let change the DLL to application/EXE type project so that we can run it from the command prompt. Select the project folder > Right click mouse > Select Properties context menu.
Change the Application type: to Console Application and the Startup object: to Sub Main.
Build and run the project.
The following is the sample output using default argument values.
The following is the downloaded gif image, saved in the local path of the running program.
The following sample output shows the index.html file has been downloaded and saved in the C:\temp folder.