A69WD/Src/A69WD/Service/Service_Crawler.cs

using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.Windows.Forms;
using A69WD.Net;



namespace A69WD.Service {
    public class Service_Crawler {

        // Thread worker che consuma la coda
        private Thread _workerThread;

        // Coda thread-safe di URL da scaricare
        private readonly BlockingCollection<iLink> _urlQueue = new BlockingCollection<iLink>(new ConcurrentQueue<iLink>());

        // Semaforo per limitare il numero massimo di download simultanei
        private readonly SemaphoreSlim _semaphore = null;

        // Contatori
        private int _inCorso = 0;
        private int _completati = 0;
        private int _errori = 0;

        // Riferimenti ai controlli UI per aggiornare i contatori e la progress bar
        private readonly StatusStrip _sb;


        public Service_Crawler(StatusStrip sb) {
            _semaphore = new SemaphoreSlim(Cfg.Settings.nThread);
            _sb = sb;
        }



        /// <summary>
        /// Avvia il thread worker che consuma la coda di URL.
        /// </summary>
        public void StartWorker() {
            _workerThread = new Thread(ProcessQueue) { IsBackground = true };
            _workerThread.Start();
        }

        /// <summary>
        /// Ferma il worker completando la coda.
        /// </summary>
        public void StopWorker() {
            _urlQueue.CompleteAdding();
        }

        /// <summary>
        /// Aggiunge un URL alla coda FIFO.
        /// </summary>
        public void EnqueueUrl(iLink link) {
            _urlQueue.Add(link);
            UpdateCounters();
        }

        /// <summary>
        /// Thread worker: consuma la coda ed avvia task di download.
        /// </summary>
        private async void ProcessQueue() {

            // Verifico cartelle di destinazione:
            if (!Directory.Exists(Cfg.Settings.PathIdxDone)) Directory.CreateDirectory(Cfg.Settings.PathIdxDone);
            if (!Directory.Exists(Cfg.Settings.PathSrcDone)) Directory.CreateDirectory(Cfg.Settings.PathSrcDone);
            if (!Directory.Exists(Cfg.Settings.PathSrcErrOnExtract)) Directory.CreateDirectory(Cfg.Settings.PathSrcErrOnExtract);
            if (!Directory.Exists(Cfg.Settings.PathSrcErrOnSave)) Directory.CreateDirectory(Cfg.Settings.PathSrcErrOnSave);

            // Autenticazione ai siti:
            if (!cLink_A69.Autenticated) { await cLink_A69.Authenticate(); }
            if (!cLink_eRac.Autenticated) { await cLink_eRac.Authenticate(); }

            // Ciclo per ciascun elemento della FIFO:
            foreach (iLink link in _urlQueue.GetConsumingEnumerable()) {
                Logger.Information("ProcessQueue: Start url " + link.FileName);

                Task.Run(async () => {
                    Interlocked.Increment(ref _inCorso);
                    UpdateCounters();

                    await _semaphore.WaitAsync();
                    try {
                        List<iLink> urls = null;
                        if (link is cLink_A69) {
                            cLink_A69 clk = link as cLink_A69;
                            urls = await clk.CrawlPageAsync();
                        } else {
                            if (link is cLink_eRac) {
                                cLink_eRac clk = link as cLink_eRac;
                                urls = await clk.CrawlPageAsync();
                            }
                        }
                        Logger.Information("ProcessQueue:   end url " + link.FileName + " - URL trovati: " + urls.Count.ToString("000"));
                        foreach (iLink url in urls) {
                            if (url.Validate) {
                                //    Logger.Warning("URL Validato: " + url.FileName);
                                App.DB.LinkLog(url);
                                _urlQueue.Add(url);
                                //} else {
                                //    Logger.Warning("URL NON Validato: " + url.FileName);
                            }
                        }
                    } catch (Exception ex) {
                        Interlocked.Increment(ref _errori);
                        Logger.Error($"Errore su {link}: {ex.Message}");
                    } finally {
                        Interlocked.Decrement(ref _inCorso);
                        Interlocked.Increment(ref _completati);
                        _semaphore.Release();
                        UpdateCounters();
                    }
                });
            }
        }



        /// <summary>
        /// Aggiorna i contatori e la progress bar sulla UI.
        /// </summary>
        private void UpdateCounters() {
            if (_sb.InvokeRequired) {
                _sb.Invoke(new Action(UpdateCounters));
                return;
            }

            int daFare = _urlQueue.Count;
            int totale = _inCorso + _completati + _errori + daFare;
            int fatti = _completati + _errori;

            //_sb.Items["sbStatusLabel"].Text = $"In corso: {_inCorso} - Completati: {_completati} - Errori: {_errori} - Da fare: {daFare}";
            _sb.Items["sbStatusLabel"].Text = $"{totale}|{_inCorso}|{fatti}|{_errori}";
            _sb.Items["sbStatusLabel"].ToolTipText = "Totali | In Corso | Fatti | Error";

            (_sb.Items["sbProgressBar"] as ToolStripProgressBar).Maximum = totale > 0 ? totale : 1;
            (_sb.Items["sbProgressBar"] as ToolStripProgressBar).Value = fatti;
        }





    }

}