Mono on Mac OS X - Parallel HTTP Downloads limited

Question

I'm using Mono to develop a program (for Mac OS X and Debian) that can simultaneously download multiple files.

However i'm only able to download 2 files at the same time although i use the constructor new RollingDownload(10). The Code i'm using is this

using System;
using System.Collections.Generic;
using System.Collections.Concurrent;
using System.Threading;
using System.Net;
using System.Diagnostics;
using System.IO;
namespace worker
{
    public class RollingDownload
    {
        private static ConcurrentQueue<Download> _downloads = new ConcurrentQueue<Download>();
        private static short _NumberOfThreads;
        private static short DefaultTimeoutSeconds = 20;
        public RollingDownload (short NumberOfThreads)
        {
            _NumberOfThreads = NumberOfThreads;
        }

        public void addDownload(Download download) {
            _downloads.Enqueue(download);
        }
        public void SpawnWebRequests()
        {
            ServicePointManager.DefaultConnectionLimit = _NumberOfThreads;
            ServicePointManager.MaxServicePoints = _NumberOfThreads;
            IList<Thread> threadList = new List<Thread>();

            for (int i = 0; i < _NumberOfThreads; i++)
            {
                var thread = new Thread(ProcessWebRequests);
                threadList.Add(thread);
                thread.Start();
            }

            for (int i = 0; i < _NumberOfThreads; i++)
            {
                threadList[i].Join();
            }
        }

        private static void ProcessWebRequests()
        {
            while (true)
            {
                Download download;
                Console.WriteLine (Thread.CurrentThread.ManagedThreadId);
                if(_downloads.TryDequeue(out download)) {
                    ProcessWebRequest(download);
                } else {
                    break;
                }
            }
        }

        private static void ProcessWebRequest(Download download)
        {
            try
            {
                var request = (HttpWebRequest)WebRequest.Create(download.Url);
                request.Method = "GET"; // or "GET", since some sites (Amazon) don't allow HEAD
                request.Timeout = DefaultTimeoutSeconds * 1000;
                request.AllowAutoRedirect = true;
                //request.ServicePoint.ConnectionLimit = _NumberOfThreads;
                request.KeepAlive = false;
                //request.ServicePoint = ServicePointManager.FindServicePoint(new Uri(download.Url));
                // Get the response (throws an exception if status != 200)
                using (var response = (HttpWebResponse)request.GetResponse())
                {
                    if (response.StatusCode == HttpStatusCode.OK) {
                        /*string contents = "";
                        using (StreamReader reader = new StreamReader(response.GetResponseStream()))
                        {
                            contents = reader.ReadToEnd();
                        }*/
                        download.onCompleted(response.GetResponseStream(), response.StatusCode);
                    }
                }
            }
            catch (WebException ex)
            {
                var response = ((HttpWebResponse)ex.Response);
                var status = response != null
                                 ? response.StatusCode
                                 : HttpStatusCode.RequestTimeout;

                Console.WriteLine(String.Format("Broken link ({0}): {1}", status, download.Url), ex);

                // Don't rethrow, as this is an expected exception in many cases
            }
            catch (Exception ex)
            {
                Console.WriteLine(String.Format("Error processing link {0}", download.Url), ex);

                // Rethrow, something went wrong
                throw;
            }
        }
    }
public class Download
    {
        public string Url { get; set; }

        public string PathToSave { get; set; }

        public Download (String Url)
        {
            this.Url = Url;
        }

        public void onCompleted (Stream response, HttpStatusCode httpcode)
        {
            Console.WriteLine ("hello everybody: " + httpcode.ToString ());
        }
    }
}

Well i don't know. Someone in the #mono IRC Channel meant, i should use this ticket to resolve the issues but i don't know where to find machine.config or how to add it in monodevelop.

The application i'm developing is a Console-Application (no ASP!) using C#.

Would be great to hear from you guys.

Answer 1

Are your downloads from the same host? If they are, you'll need to add some code to the constructor of RollingDownload (or other initialisation code):

string downloadHost = ...;
ServicePoint sp = ServicePointManager.FindServicePoint(new Uri(downloadHost));
sp.ConnectionLimit = _NumberOfThreads;

[Credit to this blog that helped me with a similar problem earlier.]