diff --git a/Src/A69WD/Net/cLink_A69.cs b/Src/A69WD/Net/cLink_A69.cs index 3cfa612..a3ea3f2 100644 --- a/Src/A69WD/Net/cLink_A69.cs +++ b/Src/A69WD/Net/cLink_A69.cs @@ -6,6 +6,7 @@ using System.Runtime.Remoting.Messaging; using System.Security.Policy; using System.Text; using System.Threading.Tasks; +using A69WD.Service; namespace A69WD.Net { public class cLink_A69 : iLink { @@ -70,7 +71,26 @@ namespace A69WD.Net { return false; } } + + public static bool InsertStartLink(Service_Crawler service) { + //int i = 1; + //try { + // foreach (string cat in categorie) { + // string url = LinkIdxBase.Replace("{Categoria}", cat).Replace("{ID}", i.ToString()); + // string name = string.Format("_A69_{0}_{1}.html", cat, i.ToString("0000")); + // service.EnqueueUrl(new cLink_A69(url, name)); + // } + // return true; + //} catch (Exception ex) { + // Logger.Error("Errore inserendo A69 Start Link - " + ex.Message); + // return false; + //} + string url = LinkIdxBase.Replace("{Categoria}", "trio").Replace("{ID}", "1"); + string name = "_A69_trio_0001.html"; + service.EnqueueUrl(new cLink_A69(url, name)); + return true; + } }//class cLink_A69 diff --git a/Src/A69WD/Service/Service_Crawler.cs b/Src/A69WD/Service/Service_Crawler.cs index 103aad3..fb4f2dc 100644 --- a/Src/A69WD/Service/Service_Crawler.cs +++ b/Src/A69WD/Service/Service_Crawler.cs @@ -7,6 +7,8 @@ using System.Text; using System.Threading; using System.Threading.Tasks; using System.Windows.Forms; +using A69WD.Net; +using A69WD.Site; @@ -17,10 +19,10 @@ namespace A69WD.Service { private Thread _workerThread; // Coda thread-safe di URL da scaricare - private readonly BlockingCollection _urlQueue = new BlockingCollection(new ConcurrentQueue()); + private readonly BlockingCollection _urlQueue = new BlockingCollection(new ConcurrentQueue()); // Semaforo per limitare il numero massimo di download simultanei - private readonly SemaphoreSlim _semaphore = new SemaphoreSlim(1); + private readonly SemaphoreSlim _semaphore = new SemaphoreSlim(99); // Contatori private int _inCorso = 0; @@ -28,7 +30,7 @@ namespace A69WD.Service { private int _errori = 0; // HttpClient autenticato (fornito da AuthService) - private HttpClient _client; + private iSiteType _site; public Service_Crawler(Label lblInCorso) { @@ -42,10 +44,10 @@ namespace A69WD.Service { // Riferimenti ai controlli UI per aggiornare i contatori e la progress bar private readonly Label _lblInCorso; - private readonly Label _lblCompletati; - private readonly Label _lblDaFare; - private readonly Label _lblErrori; - private readonly ProgressBar _progressBar; + //private readonly Label _lblCompletati; + //private readonly Label _lblDaFare; + //private readonly Label _lblErrori; + //private readonly ProgressBar _progressBar; @@ -54,8 +56,8 @@ namespace A69WD.Service { /// /// Avvia il thread worker che consuma la coda di URL. /// - public void StartWorker(HttpClient client) { - _client = client; + public void StartWorker(iSiteType site) { + _site = site; _workerThread = new Thread(ProcessQueue) { IsBackground = true }; _workerThread.Start(); } @@ -70,8 +72,8 @@ namespace A69WD.Service { /// /// Aggiunge un URL alla coda FIFO. /// - public void EnqueueUrl(string url) { - _urlQueue.Add(url); + public void EnqueueUrl(iLink link) { + _urlQueue.Add(link); UpdateCounters(); } @@ -79,28 +81,23 @@ namespace A69WD.Service { /// Thread worker: consuma la coda ed avvia task di download. /// private void ProcessQueue() { - foreach (var url in _urlQueue.GetConsumingEnumerable()) { + foreach (iLink link in _urlQueue.GetConsumingEnumerable()) { + Logger.Information("ProcessQueue: Start url " + link.FileName); Task.Run(async () => { Interlocked.Increment(ref _inCorso); UpdateCounters(); await _semaphore.WaitAsync(); try { - // Scarica contenuto della pagina - var html = await _client.GetStringAsync(url); - - Interlocked.Increment(ref _completati); - - // Salvo il file downloadato - System.IO.StreamWriter sw = new System.IO.StreamWriter("A69_Idx_altro_" + _completati.ToString("000") + ".html", false, Encoding.UTF8); - sw.Write(html); - - Logger.Information($"Scaricato {url} ({html.Length} bytes)"); + List urls = await _site.CrawlPageAsync(link); + Logger.Information("ProcessQueue: end url " + link.FileName + " - URL trovati: " + urls.Count.ToString("000")); + foreach (iLink url in urls) { _urlQueue.Add(url); } } catch (Exception ex) { Interlocked.Increment(ref _errori); - Logger.Error($"Errore su {url}: {ex.Message}"); + Logger.Error($"Errore su {link}: {ex.Message}"); } finally { Interlocked.Decrement(ref _inCorso); + Interlocked.Increment(ref _completati); _semaphore.Release(); UpdateCounters(); } diff --git a/Src/A69WD/UI/frmDownloader.cs b/Src/A69WD/UI/frmDownloader.cs index cb8d4a6..3f2ba44 100644 --- a/Src/A69WD/UI/frmDownloader.cs +++ b/Src/A69WD/UI/frmDownloader.cs @@ -82,20 +82,31 @@ namespace A69WD { // i++; // } - // Inserisco i link iniziali sulla coda dei download: - if (!cLink_A69.InsertStartLink(_urlQueue)) { + // // Inserisco i link iniziali sulla coda dei download: + // if (!cLink_A69.InsertStartLink(_urlQueue)) { + // lblST.Text = "Inserimento A69 start link fault."; + // return; + // } + // lblST.Text = "In coda vi sono: " + _urlQueue.Count.ToString() + " url."; + // + // // Test estrazione link dal primo indice: + // iLink link = new cLink_A69("https://www.annunci69.it/racconti-erotici/racconti_lista.php?genere=altro&pagina=1", "Idx_altro_1.html"); + // Logger.Information("Start url " + link.FileName); + // List urls = await siteA69.CrawlPageAsync(link); + // Logger.Information(" end url " + link.FileName + " - URL trovati: " + urls.Count.ToString("000")); + // foreach (iLink url in urls) { _urlQueue.Add(url); } + // lblST.Text = "In coda vi sono: " + _urlQueue.Count.ToString() + " url."; + + + // Preparo il servizio + service = new Service_Crawler(lblST); + // Inserisco i link iniziali nella coda + if (!cLink_A69.InsertStartLink(service)) { lblST.Text = "Inserimento A69 start link fault."; return; } - lblST.Text = "In coda vi sono: " + _urlQueue.Count.ToString() + " url."; - - // Test estrazione link dal primo indice: - iLink link = new cLink_A69("https://www.annunci69.it/racconti-erotici/racconti_lista.php?genere=altro&pagina=1", "Idx_altro_1.html"); - Logger.Information("Start url " + link.FileName); - List urls = await siteA69.CrawlPageAsync(link); - Logger.Information(" end url " + link.FileName + " - URL trovati: " + urls.Count.ToString("000")); - foreach (iLink url in urls) { _urlQueue.Add(url); } - lblST.Text = "In coda vi sono: " + _urlQueue.Count.ToString() + " url."; + // avvio il servizio + service.StartWorker(siteA69); } @@ -114,6 +125,10 @@ namespace A69WD { private void btnClose_Click(object sender, EventArgs e) { //if (engine.Stop()) this.Close(); + + // fermo il servizio + if (service != null) service.StopWorker(); + this.Close(); }