VB .NET Asynchronous Http Get Request Program Example

Create a new class library application project and you might want to use AsyncHttpGetRequestVB as the solution name and project name as shown in the following Figure.

' This sample illustrates the asynchronous BeginGetResponseStream method of the HttpWebRequest

' class. This sample posts an asynchronous GET request for the given URI with a timeout value.

' When the delegate is invoked, the response stream is retrieved and an asynchronous stream

' read is posted. A single asynchronous stream read is posted at a time until the entire stream

' has been received. For each stream read, a DataBuffer object is created which contains the

' data read on the stream. Once the operation completes it is added to a list and an event is

' signaled. The main thread waits on this event and when signaled walks the list of completed

' DataBuffer objects and then writes them to disk. This is done to prevent a blocking write

' call from occurring in the asynchronous delegate for the stream read call.

'

' Usage:

'      usage: Executable_file_name [-u URI] [-p proxy]

'           -u URI          URI to download (along with linked content)

'           -p proxy        Name of proxy server

'

' Sample usage:

'      Executable_file_name -u http://www.microsoft.com -p http://myproxy/

'

Imports System

Imports System.IO

Imports System.Net

Imports System.Text

Imports System.ComponentModel

Imports System.Collections

Imports System.Threading

' Class which contains a byte buffer used to read from an HTTP response stream. This class

' contains the byte array, how many bytes were actually read into the array, and the FileStream

' object that the main thread should write the data to. The 'done' property is set to true

' which indicates the main thread should close the FileStream object.

Public Class DataBuffer

    Public BufferSize As Integer = 4096

    Public byteBuffer() As Byte

    Public bytesRead As Integer

    Public saveFile As FileStream

    Public done As Boolean

    ' Initialize the DataBuffer object by allocating the byte array.

    Public Sub New()

        Console.WriteLine("Initializing the DataBuffer object by allocating the byte array...")

        ReDim byteBuffer(BufferSize)

        done = False

    End Sub

End Class

' This class maintains the context information for each asynchronous web request.

Public Class HttpRequestState

    Public httpRequest As HttpWebRequest

    Public httpResponse As HttpWebResponse

    Public httpResponseStream As Stream

    Public readBuffer As DataBuffer

    Public bufferList As ArrayList

    Public bufferEvent As ManualResetEvent

    Public getUri As Uri

    Public saveFile As FileStream

    Public localSavePath As String

    Public readCount As Integer

    Public requestTimeout As Integer

    Public Sub New(ByVal httpObj As HttpWebRequest, ByVal list As ArrayList, ByVal dataEvent As ManualResetEvent)

        httpRequest = httpObj

        httpResponse = Nothing

        httpResponseStream = Nothing

        bufferList = list

        bufferEvent = dataEvent

        readBuffer = New DataBuffer()

        readCount = 0

        Console.WriteLine("In class maintaining the context information for each asynchronous web request...")

    End Sub

End Class

' This is the main class which contains the methods for initiating the asynchronous web

' requests, handling the completion, and handling any timeouts. This class also contains

' the main routine which parses the command line and starts things off.

Public Class AsyncGetClass

    Public ResourceUri As Uri

    Public HttpProxy As IWebProxy

    Public LocalSavePath As String

    Public RecursiveGet As Boolean

    Public RetrievedUri As ArrayList

    Public PendingUri As ArrayList

    Public PendingBuffer As ArrayList

    Public BufferEvent As ManualResetEvent

    Public RequestTimeout As Integer

    Public Sub New()

        ' ResourceUri = New Uri("http://www.winisp.net/goodrich/Grand Staircase - Escalante/escalante-sunset-02.jpg")

        ResourceUri = New Uri("http://www.mozilla.org/download.html")

        ' Example: http://www.spreadfirefox.com/sites/all/themes/spreadfirefox_RCS/logo.jpg

        HttpProxy = WebRequest.DefaultWebProxy

        LocalSavePath = "."

        RecursiveGet = False

        RetrievedUri = New ArrayList()

        PendingUri = New ArrayList()

        PendingBuffer = New ArrayList()

        BufferEvent = New ManualResetEvent(False)

        RequestTimeout = 10 * 1000 ' 10 second timeout

    End Sub

    ' Displays simple usage information for the application.

    Shared Sub usage()

        Console.WriteLine("Usage: Executable_file_name [-u URI] [-p proxy]")

        Console.WriteLine("Available options:")

        Console.WriteLine("     -u URI          URI to download (along with linked content)")

        Console.WriteLine("     -p proxy        Name of proxy server")

        Console.WriteLine()

    End Sub

    ' Creates any subdirectories and opens the file for writing to. The path is stripped

    ' from the URI to build the local path to save the file to. This path is appended

    ' to the local save path supplied. A file is then opened with the same name as

    ' the retrieved file and the FileStream to it is returned.

    ' <param name="localSavePath">Local path to append saved resources paths to</param>

    ' <param name="uriName">URI of destination resource being saved</param>

    Public Function CreateFile(ByVal localSavePath As String, ByVal uriName As Uri) As FileStream

        Dim localFile As FileStream = Nothing

        Dim fileName As String

        Try

            Dim uriSegments() As String = uriName.Segments

            Dim localDirs As String = ""

            ' Retrieve the directory path to the file

            Console.WriteLine("Retrieving the directory path to the file...")

            Dim i As Integer

            For i = 0 To uriSegments.Length - 2

                localDirs = localDirs + uriSegments(i)

            Next

            If (uriSegments.Length > 1) Then

                ' Replace the forward slashes with back slashes

                Console.WriteLine("Replacing the forward slashes with back slashes...")

                localDirs = localDirs.Replace("/", "\")

                ' Remove the escaped spaces

                Console.WriteLine("Removing the escaped spaces...")

                Dim temp As String = localDirs.Replace("%20", " ")

                If (Not IsNothing(temp)) Then

                    localDirs = temp

                End If

                Console.WriteLine("  Creating directory: {0}", localSavePath + "\" + localDirs)

                Directory.CreateDirectory(localSavePath + "\" + localDirs)

                fileName = uriSegments(uriSegments.Length - 1)

            Else

                Console.WriteLine("Using default file name...")

                localDirs = "\"

                fileName = "default.html"

            End If

            ' Open the file to write to

            Console.WriteLine("Opening the file to write to...")

            Dim saveFileName As String = localSavePath + localDirs + fileName

            localFile = File.Open( _

                saveFileName, _

                System.IO.FileMode.Create, _

                System.IO.FileAccess.Write, _

                System.IO.FileShare.None _

                )

            Console.WriteLine("Created File: {0}", saveFileName)

        Catch ex As Exception

            Console.WriteLine("WriteHttpContentToFile failed: {0}", ex.Message)

            Console.WriteLine("Stack:\n{0}", ex.StackTrace)

            If (Not IsNothing(localFile)) Then

                localFile.Close()

            End If

        End Try

        CreateFile = localFile

    End Function

    Private Sub RequestTimeoutCallback(ByVal state As Object, ByVal timedOut As Boolean)

        If (timedOut) Then

            Console.WriteLine("RequestTimeoutCallback timed out!")

            Dim httpState As HttpRequestState = CType(state, HttpRequestState)

            If (Not IsNothing(httpState) And Not IsNothing(httpState.httpRequest)) Then

                httpState.httpRequest.Abort()

            End If

        End If

    End Sub

    Private Sub HttpStreamReadCallback(ByVal ar As IAsyncResult)

        Dim httpState As HttpRequestState = CType(ar.AsyncState, HttpRequestState)

        Try

            httpState.readBuffer.bytesRead = httpState.httpResponseStream.EndRead(ar)

            Console.WriteLine("HttpStreamReadCallback returned: {0} bytes read (content length = {1})", _

                httpState.readBuffer.bytesRead, httpState.httpResponse.ContentLength)

            ' Set the FileStream in the DatBuffer object

            Console.WriteLine("Setting the FileStream in the DatBuffer object...")

            httpState.readBuffer.saveFile = httpState.saveFile

            If (httpState.readBuffer.bytesRead > 0) Then

                httpState.bufferList.Add(httpState.readBuffer)

                httpState.bufferEvent.Set()

                httpState.readBuffer = New DataBuffer()

                PostResponseRead(httpState)

            Else

                httpState.httpResponseStream.Close()

                httpState.httpResponse.Close()

                httpState.readBuffer.done = True

                httpState.bufferList.Add(httpState.readBuffer)

                httpState.bufferEvent.Set()

            End If

        Catch e As WebException

            Console.WriteLine("ReadCallBack Exception raised!")

            Console.WriteLine("Message:{0}", e.Message)

            Console.WriteLine("Status:{0}", e.Status)

        End Try

    End Sub

    Private Sub HttpResponseCallback(ByVal ar As IAsyncResult)

        Dim httpState As HttpRequestState = CType(ar.AsyncState, HttpRequestState)

        Console.WriteLine("HttpResponseCallback() invoked...")

        Try

            ' Complete the asynchronous request

            Console.WriteLine("Completing the asynchronous request...")

            httpState.httpResponse = CType(httpState.httpRequest.EndGetResponse(ar), HttpWebResponse)

            ' Read the response into a Stream object.

            Console.WriteLine("Reading the response into a Stream object...")

            httpState.httpResponseStream = httpState.httpResponse.GetResponseStream()

            ' Create the file where resource is to be saved

            Console.WriteLine("Creating the file where resource is to be saved...")

            httpState.saveFile = CreateFile(httpState.localSavePath, httpState.getUri)

            PostResponseRead(httpState)

            Console.WriteLine("BeginRead() invoked on response stream...")

        Catch e As WebException

            Console.WriteLine("RespCallback() Exception raised!")

            Console.WriteLine("Message:{0}", e.Message)

            Console.WriteLine("Status:{0}", e.Status)

        End Try

    End Sub

    Public Sub PostResponseRead(ByVal state As HttpRequestState)

        Monitor.Enter(state)

        'state.readBuffer.Count = Interlocked.Increment( ref state.readCount )

        ' Begin the Reading of the contents of the HTML page and print it to the console.

        Console.WriteLine("Reading the contents of the HTML page and print it to the console...")

        Dim buffer As DataBuffer = New DataBuffer()

        Dim asyncStreamRead As IAsyncResult = state.httpResponseStream.BeginRead( _

            state.readBuffer.byteBuffer, _

            0, _

            buffer.BufferSize, _

            New AsyncCallback(AddressOf HttpStreamReadCallback), _

            state _

            )

        Monitor.Exit(state)

    End Sub

    ' Retrieve the resource specified by the string URI address. The HTTP proxy

    ' can be specified if needed to go outside the local network.

    ' <param name="resourceName">String URI to retrieve</param>

    ' <param name="proxy">Proxy server to use to access resource</param>

    ' <param name="localSavePath">Local path to append saved resources paths to</param>

    ' <returns></returns>

    Public Sub GetResource(ByVal getUri As Uri, ByVal recurse As Boolean)

        Dim httpRequest As HttpWebRequest = Nothing

        Dim httpResponse As HttpWebResponse = Nothing

        Dim localFile As FileStream = Nothing

        Try

            RetrievedUri.Add(getUri)

            Console.WriteLine("Retrieving: {0}", getUri.AbsoluteUri)

            ' Create the HTTP request object

            Console.WriteLine("Creating the HTTP request object...")

            httpRequest = CType(WebRequest.Create(getUri.AbsoluteUri), HttpWebRequest)

            ' Set some HTTP specific headers

            Console.WriteLine("Setting some HTTP specific headers...")

            httpRequest.UserAgent = "My User Agent/1.0"

            ' If a proxy was specified create an instance of the WebProxy with it

            Console.WriteLine("If a proxy was specified create an instance of the WebProxy with it...")

            httpRequest.Proxy = HttpProxy

            Dim getState As HttpRequestState = New HttpRequestState(httpRequest, Me.PendingBuffer, Me.BufferEvent)

            ' Set the local save path in the context block

            Console.WriteLine("Setting the local save path in the context block...")

            getState.localSavePath = Me.LocalSavePath

            getState.getUri = getUri

            ' Get the response object

            Console.WriteLine("Getting the response object...")

            Dim ar As IAsyncResult = httpRequest.BeginGetResponse( _

                New AsyncCallback(AddressOf HttpResponseCallback), _

                getState _

                )

            ThreadPool.RegisterWaitForSingleObject( _

                ar.AsyncWaitHandle, _

                New WaitOrTimerCallback(AddressOf RequestTimeoutCallback), _

                getState, _

                RequestTimeout, _

                True _

                )

        Catch wex As WebException

            Console.WriteLine("Exception occurred on request: {0}", wex.Message)

            Console.WriteLine("Status code: {0}", wex.Status)

            If (wex.Status = WebExceptionStatus.ProtocolError) Then

                ' If there was a protocol error then the response object is

                '    valid but there was an error retrieving the response.

                httpResponse = CType(wex.Response, HttpWebResponse)

                Console.WriteLine("The protocol returned was: {0}", httpResponse.StatusCode.ToString())

                httpResponse.Close()

                httpResponse = Nothing

            End If

            Throw

        Finally

            ' Close the resources if still open

            Console.WriteLine("Closing all the resources if still open...")

            If (Not IsNothing(localFile)) Then

                localFile.Close()

            End If

            If (Not IsNothing(httpResponse)) Then

                httpResponse.Close()

            End If

        End Try

    End Sub

    ' This is the main function which parses the command line, initializes the proxy

    ' if present, and calls the routine to retrieve the specified URI.   

    ' <param name="args">Arguments passed to application</param>

    Shared Sub Main()

        Dim httpGet As AsyncGetClass = New AsyncGetClass()

        ' Parse the command line

        Dim args As String() = Environment.GetCommandLineArgs()

        Dim i As Integer

        usage()

        For i = 1 To args.GetUpperBound(0)

            Try

                Dim CurArg() As Char = args(i).ToCharArray(0, args(i).Length)

                If (CurArg(0) = "-") Or (CurArg(0) = "/") Then

                    Select Case Char.ToLower(CurArg(1), System.Globalization.CultureInfo.CurrentCulture)

                        Case "u"

                            ' URI to download (get)

                            i = i + 1

                            httpGet.ResourceUri = New Uri(args(i))

                        Case "p"

                            ' Name of proxy server to use

                            i = i + 1

                            httpGet.HttpProxy = New WebProxy(args(i))

                        Case "r"

                            ' Retrieve all referenced images and text on the same host

                            httpGet.RecursiveGet = True

                        Case "s"

                            ' Local save path to append to retrieved resources

                            i = i + 1

                            httpGet.LocalSavePath = args(i)

                        Case Else

                            usage()

                            Exit Sub

                    End Select

                End If

            Catch e As Exception

                usage()

                Exit Sub

            End Try

        Next

        ' Initialize the proxy server and retrieve the resources

        Console.WriteLine("Initializing the proxy server and retrieve the resources...")

        Try

            Dim getCompleted As Boolean = False

            httpGet.GetResource(httpGet.ResourceUri, httpGet.RecursiveGet)

            While (getCompleted = False)

                If (httpGet.BufferEvent.WaitOne(5000, True) = True) Then

                    Console.WriteLine("Buffer event signaled...")

                    While (httpGet.PendingBuffer.Count > 0)

                        Dim data As DataBuffer = CType(httpGet.PendingBuffer(0), DataBuffer)

                        httpGet.PendingBuffer.RemoveAt(0)

                        If (data.done = True) Then

                            data.saveFile.Close()

                            Console.WriteLine("End of stream, closing file...")

                            getCompleted = True

                            Exit While

                        ElseIf (data.byteBuffer.Length > 0) Then

                            data.saveFile.Write(data.byteBuffer, 0, data.bytesRead)

                        End If

                    End While

                    httpGet.BufferEvent.Reset()

                Else

                    Console.WriteLine("Buffer event timed out")

                End If

            End While

        Catch ex As Exception

            Console.WriteLine("Exception occurred: {0}", ex.ToString())

        End Try

    End Sub

End Class