C# Asynchronous Http Get Request Program Example

Create a new console application project and you might want to use AsyncHttpGetRequestCS as the solution name and project name as shown in the following Figure.

// This sample illustrates the asynchronous BeginGetResponseStream() method of the HttpWebRequest

// class. This sample posts an asynchronous GET request for the given URI with a timeout value.

// When the delegate is invoked, the response stream is retrieved and an asynchronous stream

// read is posted. A single asynchronous stream read is posted at a time until the entire stream

// has been received. For each stream read, a DataBuffer object is created which contains the

// data read on the stream. Once the operation completes it is added to a list and an event is

// signaled. The main thread waits on this event and when signaled walks the list of completed

// DataBuffer objects and then writes them to disk. This is done to prevent a blocking write

// call from occurring in the asynchronous delegate for the stream read call.

//

// Usage:

//      usage: Executable_file_name [-u URI] [-p proxy]

//           -u URI          URI to download (along with linked content)

//           -p proxy        Name of proxy server

//

// Sample usage:

//      Executable_file_name -u http://www.microsoft.com -p http://myproxy/

using System;

using System.IO;

using System.Net;

using System.Text;

using System.ComponentModel;

using System.Collections;

using System.Threading;

public class DataBuffer

    {

        public const int BufferSize = 4096;

        public byte[] byteBuffer;

        public int bytesRead;

        public FileStream saveFile;

        public bool done;

        /// <summary>

        /// Initialize the DataBuffer object by allocating the byte array.

        /// </summary>

        public DataBuffer()

        {

            byteBuffer = new byte[BufferSize];

            done = false;

        }

    }

    /// <summary>

    /// This class maintains the context information for each asynchronous web request.

    /// </summary>

    public class HttpRequestState

    {

        public HttpWebRequest httpRequest;

        public HttpWebResponse httpResponse;

        public Stream httpResponseStream;

        public DataBuffer readBuffer;

        public ArrayList bufferList;

        public ManualResetEvent bufferEvent;

        public Uri getUri;

        public FileStream saveFile;

        public string localSavePath;

        public int readCount;

        public int requestTimeout;

        public HttpRequestState(HttpWebRequest httpObj, ArrayList list, ManualResetEvent dataEvent)

        {

            httpRequest = httpObj;

            httpResponse = null;

            httpResponseStream = null;

            bufferList = list;

            bufferEvent = dataEvent;

            readBuffer = new DataBuffer();

            readCount = 0;

        }

    }

    class AsyncGetClass

    {

    public Uri                       ResourceUri;

    public IWebProxy         HttpProxy;

    public string                  LocalSavePath;

    public bool                    RecursiveGet;

    public ArrayList            RetrievedUri,

                                           PendingUri,

                                           PendingBuffer;

    public ManualResetEvent       BufferEvent;

    public int                        RequestTimeout;

    public AsyncGetClass()

    {

        ResourceUri = new Uri("http://www.google.com/intl/en/images/about_logo.gif");

        // ResourceUri = new Uri("http://labs.google.com/index.html");

        HttpProxy = WebRequest.DefaultWebProxy;

        LocalSavePath = ".";

        RecursiveGet = false;

        RetrievedUri = new ArrayList();

        PendingUri = new ArrayList();

        PendingBuffer = new ArrayList();

        BufferEvent = new ManualResetEvent( false );

        RequestTimeout = 10 * 1000; // 10 second timeout

    }

    /// <summary>

    /// Displays simple usage information for the application.

    /// </summary>

    static void usage()

    {

        Console.WriteLine("Usage: Executable_file_name [-u URI] [-p proxy]");

        Console.WriteLine("Available options:");

        Console.WriteLine("     -u URI          URI to download (along with linked content)");

        Console.WriteLine("     -p proxy        Name of proxy server");

        Console.WriteLine();

    }

    /// <summary>

    /// Creates any subdirectories and opens the file for writing to. The path is stripped

    /// from the URI to build the local path to save the file to. This path is appended

    /// to the local save path supplied. A file is then opened with the same name as

    /// the retrieved file and the FileStream to it is returned.

    /// </summary>

    /// <param name="localSavePath">Local path to append saved resources paths to</param>

    /// <param name="uriName">URI of destination resource being saved</param>

    public static FileStream CreateFile( string localSavePath, Uri uriName )

    {

        FileStream  localFile = null;

        string fileName;

        try

        {

            string [ ]   uriSegments = uriName.Segments;

            string       localDirs = "";

            // Retrieve the directory path to the file

            Console.WriteLine("Retrieving the directory path to the file...");

            for(int i=0; i < uriSegments.Length-1 ;i++)

            {

                localDirs += uriSegments[i];

            }

            if ( uriSegments.Length > 1 )

            {

                // Replace the forward slashes with back slashes

                Console.WriteLine("Replacing the forward slashes with back slashes...");

                localDirs = localDirs.Replace(@"/", @"\");

                // Remove the escaped spaces

                Console.WriteLine("Removing the escaped spaces...");

                string temp = localDirs.Replace(@"%20", " ");

                if ( temp != null )

                    localDirs = temp;

                Console.WriteLine("  Creating directory: {0}", localSavePath + @"\" + localDirs );

                Directory.CreateDirectory( localSavePath + @"\" + localDirs );

                fileName = uriSegments[ uriSegments.Length - 1 ];

            }

            else

            {

                Console.WriteLine("Using default file name...");

                localDirs = @"\";

                fileName = "default.html";

            }

            // Open the file to write to

            Console.WriteLine("Opening the file to write to...");

            string saveFileName = localSavePath + localDirs + fileName;

            localFile = File.Open(

                saveFileName,

                System.IO.FileMode.Create,

                System.IO.FileAccess.Write,

                System.IO.FileShare.None

                );

            Console.WriteLine("Created File: {0}", saveFileName);

        }

        catch ( Exception ex )

        {

            Console.WriteLine("WriteHttpContentToFile failed: {0}", ex.Message);

            Console.WriteLine("Stack:\n{0}", ex.StackTrace);

            if ( localFile != null )

                localFile.Close();

        }

        return localFile;

    }

    private static void RequestTimeoutCallback( object state, bool timedOut )

    {

        if ( timedOut )

        {

            Console.WriteLine("RequestTimeoutCallback timed out!");

            HttpRequestState httpState = state as HttpRequestState;

            if ( ( httpState != null ) && ( httpState.httpRequest != null ) )

            {

                httpState.httpRequest.Abort();

            }

        }

    }

    private static  void HttpStreamReadCallback( IAsyncResult ar )

    {

        HttpRequestState     httpState = (HttpRequestState) ar.AsyncState;

        try

        {

            httpState.readBuffer.bytesRead = httpState.httpResponseStream.EndRead( ar );

            Console.WriteLine("HttpStreamReadCallback() returned: {0} bytes read (content length = {1})",

                httpState.readBuffer.bytesRead, httpState.httpResponse.ContentLength );

            // Set the FileStream in the DatBuffer object

            Console.WriteLine("Setting the FileStream in the DatBuffer object...");

            httpState.readBuffer.saveFile = httpState.saveFile;

            if ( httpState.readBuffer.bytesRead > 0 )

            {

                httpState.bufferList.Add( httpState.readBuffer );

                httpState.bufferEvent.Set();

                httpState.readBuffer = new DataBuffer();

                PostResponseRead( httpState );

            }

            else

            {

                httpState.httpResponseStream.Close();

                httpState.httpResponse.Close();

                httpState.readBuffer.done = true;

                httpState.bufferList.Add( httpState.readBuffer );

                httpState.bufferEvent.Set();

            }

        }

        catch(WebException e)

        {

            Console.WriteLine("\nReadCallBack() Exception raised!");

            Console.WriteLine("\nMessage:{0}",e.Message);

            Console.WriteLine("\nStatus:{0}",e.Status);

        }

    }

    private static void HttpResponseCallback(IAsyncResult ar)

    {

        HttpRequestState    httpState = (HttpRequestState) ar.AsyncState;

        Console.WriteLine("HttpResponseCallback() invoked...");

        try

        {

            // Complete the asynchronous request

            Console.WriteLine("Completing the asynchronous request...");

            httpState.httpResponse = (HttpWebResponse) httpState.httpRequest.EndGetResponse( ar );

            // Read the response into a Stream object.

            Console.WriteLine("Reading the response into a Stream object...");

            httpState.httpResponseStream = httpState.httpResponse.GetResponseStream();

            // Create the file where resource is to be saved

            Console.WriteLine("Creating the file where resource is to be saved...");

            httpState.saveFile = CreateFile( httpState.localSavePath, httpState.getUri );

            PostResponseRead( httpState );

            Console.WriteLine("BeginRead() invoked on response stream...");

            return;

        }

        catch(WebException e)

        {

            Console.WriteLine("\nRespCallback() Exception raised!");

            Console.WriteLine("\nMessage:{0}",e.Message);

            Console.WriteLine("\nStatus:{0}",e.Status);

        }

    }

    public static void PostResponseRead( HttpRequestState state )

    {

        Monitor.Enter( state );

        // state.readBuffer.Count = Interlocked.Increment( ref state.readCount );

        // Begin the Reading of the contents of the HTML page and print it to the console.

        Console.WriteLine("Begin the Reading of the contents of the HTML page and print it to the console...");

        IAsyncResult asyncStreamRead = state.httpResponseStream.BeginRead(

            state.readBuffer.byteBuffer,

            0,

            DataBuffer.BufferSize,

            new AsyncCallback( HttpStreamReadCallback ),

            state

            );

        Monitor.Exit( state );

    }

    /// <summary>

    /// Retrieve the resource specified by the string URI address. The HTTP proxy

    /// can be specified if needed to go outside the local network.

    /// </summary>

    /// <param name="resourceName">String URI to retrieve</param>

    /// <param name="proxy">Proxy server to use to access resource</param>

    /// <param name="localSavePath">Local path to append saved resources paths to</param>

    /// <returns></returns>

    public void GetResource(Uri getUri, bool recurse)

    {

        HttpWebRequest httpRequest = null;

        HttpWebResponse httpResponse = null;

        FileStream localFile = null;

        try

        {

            RetrievedUri.Add(getUri);

            Console.WriteLine("Retrieving: {0}", getUri.AbsoluteUri);

            // Create the HTTP request object

            Console.WriteLine("Creating the HTTP request object...");

            httpRequest = (HttpWebRequest)WebRequest.Create(getUri.AbsoluteUri);

            // Set some HTTP specific headers

            Console.WriteLine("Setting some HTTP specific headers...");

            httpRequest.UserAgent = "My User Agent/1.0";

            // If a proxy was specified create an instance of the WebProxy with it

            Console.WriteLine("If a proxy was specified create an instance of the WebProxy with it...");

            httpRequest.Proxy = HttpProxy;

            HttpRequestState getState = new HttpRequestState(httpRequest, this.PendingBuffer, this.BufferEvent);

            // Set the local save path in the context block

            Console.WriteLine("Setting the local save path in the context block...");

            getState.localSavePath = this.LocalSavePath;

            getState.getUri = getUri;

            // Get the response object

            Console.WriteLine("Getting the response object...");

            IAsyncResult ar = httpRequest.BeginGetResponse(new AsyncCallback(HttpResponseCallback), getState);

            ThreadPool.RegisterWaitForSingleObject(

                ar.AsyncWaitHandle,

                new WaitOrTimerCallback(RequestTimeoutCallback),

                getState,

                RequestTimeout,

                true

                );

        }

        catch (WebException wex)

        {

            Console.WriteLine("Exception occurred on request: {0}", wex.Message);

            Console.WriteLine("Status code: {0}", wex.Status);

            if (wex.Status == WebExceptionStatus.ProtocolError)

            {

                // If there was a protocol error then the response object is

                //    valid but there was an error retrieving the response.

                httpResponse = (HttpWebResponse)wex.Response;

                Console.WriteLine("\nThe protocol returned was: {0}", httpResponse.StatusCode.ToString());

                httpResponse.Close();

                httpResponse = null;

            }

            throw;

        }

        finally

        {

            // Close the resources if still open

            if (localFile != null)

                localFile.Close();

            if (httpResponse != null)

                httpResponse.Close();

        }

    }

        /// <summary>

        /// This is the main function which parses the command line, initializes the proxy

        /// if present, and calls the routine to retrieve the specified URI.

        /// </summary>

        /// <param name="args">Arguments passed to application</param>

        static void Main(string[ ] args)

        {

            AsyncGetClass    httpGet = new AsyncGetClass();

            usage();

        // Parse the command line

        for(int i=0; i < args.Length ;i++)

        {

            try

            {

                if ( ( args[i][0] == '-' ) || ( args[i][0] == '/' ) )

                {

                    switch ( Char.ToLower( args[i][1] ) )

                    {

                        case 'u':

                            // URI to download (get)

                            httpGet.ResourceUri = new Uri( args[ ++i ] );

                            break;

                        case 'p':

                            // Name of proxy server to use

                            httpGet.HttpProxy = new WebProxy( args[ ++i ] );

                            break;

                        case 'r':

                            // Retrieve all referenced images and text on the same host

                            httpGet.RecursiveGet = true;

                            break;

                        case 's':

                            // Local save path to append to retrieved resources

                            httpGet.LocalSavePath = args[ ++i ];

                            break;

                        default:

                            usage();

                            return;

                    }

                }

            }

            catch

            {

                usage();

                return;

            }

        }

        // Initialize the proxy server and retrieve the resources

        Console.WriteLine("Initializing the proxy server and retrieve the resources...");

        try

        {

            bool getCompleted = false;

            httpGet.GetResource( httpGet.ResourceUri, httpGet.RecursiveGet );

            while ( getCompleted == false )

            {

                if ( httpGet.BufferEvent.WaitOne( 5000, true ) == true )

                {

                    Console.WriteLine("Buffer event signaled...");

                    while ( httpGet.PendingBuffer.Count > 0 )

                    {

                        DataBuffer data = (DataBuffer) httpGet.PendingBuffer[0];

                        httpGet.PendingBuffer.RemoveAt(0);

                        if ( data.done == true )

                        {

                            data.saveFile.Close();

                            Console.WriteLine("End of stream, closing file...");

                            getCompleted = true;

                            break;

                        }

                        else if ( data.byteBuffer.Length > 0 )

                        {

                            data.saveFile.Write( data.byteBuffer, 0, data.bytesRead );

                        }

                    }

                    httpGet.BufferEvent.Reset();

                }

                else

                {

                    Console.WriteLine("Buffer event timed out");

                }

            }

        }

        catch( Exception ex )

        {

            Console.WriteLine("Exception occurred: {0}", ex.ToString());

        }

        return;

        }

    }