From 0d2b5a1f493b7c502486d664d40833580dbc8df9 Mon Sep 17 00:00:00 2001 From: CSiCoAlm Date: Thu, 1 Jan 2026 15:51:38 +0100 Subject: [PATCH] Upgrade dello skraping Refactoring (eliminata la iSite) e cartelle di destinazione distinte in base alla diagnostica. --- Src/A69WD/General/cTools.cs | 6 ++ Src/A69WD/Net/cLink_A69.cs | 107 +++++++++++++++------------ Src/A69WD/Net/iLink.cs | 27 +++++++ Src/A69WD/Service/Service_Crawler.cs | 22 ++++-- Src/A69WD/Site/cSite_A69.cs | 28 +++---- Src/A69WD/UI/frmReader.Designer.cs | 13 ++++ Src/A69WD/UI/frmReader.cs | 7 ++ 7 files changed, 142 insertions(+), 68 deletions(-) diff --git a/Src/A69WD/General/cTools.cs b/Src/A69WD/General/cTools.cs index 67d9d91..230de02 100644 --- a/Src/A69WD/General/cTools.cs +++ b/Src/A69WD/General/cTools.cs @@ -47,6 +47,12 @@ namespace A69WD { } + public static void SaveToFile(string txtHTML, string filePath, string fileName) { + System.IO.StreamWriter tsw = new System.IO.StreamWriter(System.IO.Path.Combine(filePath, fileName), false, Encoding.UTF8); + tsw.Write(txtHTML); + } + + /// /// Metodo che legge un testo da file (UTF8) /// diff --git a/Src/A69WD/Net/cLink_A69.cs b/Src/A69WD/Net/cLink_A69.cs index cceba20..b7337ea 100644 --- a/Src/A69WD/Net/cLink_A69.cs +++ b/Src/A69WD/Net/cLink_A69.cs @@ -745,7 +745,7 @@ namespace A69WD.Net { /// /// Indica se siamo autenticati al sito /// - public bool Autenticated => autenticated; + public static bool Autenticated => autenticated; /// @@ -867,14 +867,11 @@ namespace A69WD.Net { /// public async Task> CrawlPageAsync() { var newlinks = new List(); - if (!autenticated) { - // Sito non autenticato, non posso fare nulla - Logger.Warning("A69 - Sito non autenticato. Download non permessi."); - return newlinks; - } + try { Console.WriteLine($"A69 - Scarico: {URL}"); HtmlDocument tdoc = null; + string thtml = ""; #region ToConfirm if (Status == eLinkState.ToConfirm) { @@ -900,13 +897,11 @@ namespace A69WD.Net { return newlinks; } // Scarico la pagina - var thtml = await client.GetStringAsync(URL); + thtml = await client.GetStringAsync(URL); tdoc = new HtmlDocument(); tdoc.LoadHtml(thtml); // Salvo il file downloadato - //System.IO.StreamWriter tsw = new System.IO.StreamWriter(lk.DstFilePath, false, Encoding.UTF8); - System.IO.StreamWriter tsw = new System.IO.StreamWriter(System.IO.Path.Combine(FilePath, FileName), false, Encoding.UTF8); - tsw.Write(thtml); + //SaveToFile(thtml, lk.FilePath, lk.FileName); Status = eLinkState.ToAnalize; } @@ -916,6 +911,7 @@ namespace A69WD.Net { { if (Status == eLinkState.ToAnalize) { List tmpLk = new List(); + #region Estraggo gli url ad indici: { /* @@ -934,22 +930,22 @@ namespace A69WD.Net { * * */ - // Estraggo i link alle altre pagine: - var tnodes = tdoc.DocumentNode.SelectNodes("//div[@id='pagination']//ul//li//a[@href]"); - if (tnodes != null) { - foreach (var link in tnodes) { - var href = link.GetAttributeValue("href", ""); - if (!string.IsNullOrEmpty(href) && href.StartsWith("/racconti-erotici/racconti_lista.php?genere=")) - tmpLk.Add(href); //tmpLk.Add(cLink_A69.PageBase + href); - } - } - // Rimuove duplicati e ordina alfabeticamente - tmpLk = tmpLk - .OrderBy(s => s) // ordina alfabeticamente - .Distinct() // elimina duplicati - .ToList(); - // Inserisco i link trovati try { + // Estraggo i link alle altre pagine: + var tnodes = tdoc.DocumentNode.SelectNodes("//div[@id='pagination']//ul//li//a[@href]"); + if (tnodes != null) { + foreach (var link in tnodes) { + var href = link.GetAttributeValue("href", ""); + if (!string.IsNullOrEmpty(href) && href.StartsWith("/racconti-erotici/racconti_lista.php?genere=")) + tmpLk.Add(href); //tmpLk.Add(cLink_A69.PageBase + href); + } + } + // Rimuove duplicati e ordina alfabeticamente + tmpLk = tmpLk + .OrderBy(s => s) // ordina alfabeticamente + .Distinct() // elimina duplicati + .ToList(); + // Inserisco i link trovati foreach (string ilk in tmpLk) { string[] separator = new string[] { "&pagina=" }; string[] txt = ilk.Split(separator, StringSplitOptions.RemoveEmptyEntries); @@ -957,7 +953,10 @@ namespace A69WD.Net { string name = txt[0].Replace("/racconti-erotici/racconti_lista.php?genere=", "_A69_") + "_" + int.Parse(txt[1]).ToString("0000") + ".html"; newlinks.Add(new cLink_A69(url, name)); } - } catch (Exception ex) { Logger.Error("Errore analisi Idx link dal " + FileName + " - " + ex.Message); } + } catch (Exception ex) { + Logger.Error("Errore analisi Idx link dal " + FileName + " - " + ex.Message); + cTools.SaveToFile(thtml, Cfg.Settings.PathSrcErrOnExtract, FileName); + } } #endregion @@ -970,30 +969,33 @@ namespace A69WD.Net { * Continua a leggere ยป * */ - // Estraggo i link ai racconti: - tmpLk = new List(); - // Estraggo i link alle altre pagine: - var tnodes = tdoc.DocumentNode.SelectNodes("//div[@class='continua']//a[@href]"); - if (tnodes != null) { - foreach (var link in tnodes) { - var href = link.GetAttributeValue("href", ""); - if (!string.IsNullOrEmpty(href) && href.StartsWith("/racconti-erotici/") && (href.EndsWith(".html") || href.EndsWith(".htm"))) - tmpLk.Add(href); //tmpLk.Add(cLink_A69.PageBase + href); - } - } - // Rimuove duplicati e ordina alfabeticamente - tmpLk = tmpLk - .OrderBy(s => s) // ordina alfabeticamente - .Distinct() // elimina duplicati - .ToList(); - // Inserisco i link trovati try { + // Estraggo i link ai racconti: + tmpLk = new List(); + // Estraggo i link alle altre pagine: + var tnodes = tdoc.DocumentNode.SelectNodes("//div[@class='continua']//a[@href]"); + if (tnodes != null) { + foreach (var link in tnodes) { + var href = link.GetAttributeValue("href", ""); + if (!string.IsNullOrEmpty(href) && href.StartsWith("/racconti-erotici/") && (href.EndsWith(".html") || href.EndsWith(".htm"))) + tmpLk.Add(href); //tmpLk.Add(cLink_A69.PageBase + href); + } + } + // Rimuove duplicati e ordina alfabeticamente + tmpLk = tmpLk + .OrderBy(s => s) // ordina alfabeticamente + .Distinct() // elimina duplicati + .ToList(); + // Inserisco i link trovati foreach (string ilk in tmpLk) { string url = cLink_A69.PageBase + ilk; string name = ilk.Replace("/racconti-erotici/", "A69_").Replace(@"/", "_"); newlinks.Add(new cLink_A69(url, name)); } - } catch (Exception ex) { Logger.Error("Errore analisi Idx link dal " + FileName + " - " + ex.Message); } + } catch (Exception ex) { + Logger.Error("Errore analisi Idx link dal " + FileName + " - " + ex.Message); + cTools.SaveToFile(thtml, Cfg.Settings.PathSrcErrOnExtract, FileName); + } } #endregion @@ -1003,8 +1005,21 @@ namespace A69WD.Net { cRac rac = EstraiRac(tdoc); if (rac != null) { // Racconto estratto correttamente, lo salvo sul DB - App.DB.ImportRac(rac); + int res = App.DB.ImportRac(rac); + if (res > 0) { + // Racconto salvato correttamente sul DB + cTools.SaveToFile(thtml, Cfg.Settings.PathSrcDone, FileName); + } else { + // Errore nel salvataggio sul DB + cTools.SaveToFile(thtml, Cfg.Settings.PathSrcErrOnSave, FileName); + } + } else { + // Estrazione del racconto fallita + cTools.SaveToFile(thtml, Cfg.Settings.PathSrcErrOnExtract, FileName); } + } else { + // Link ad un indice + cTools.SaveToFile(thtml, Cfg.Settings.PathIdxDone, FileName); } #endregion @@ -1039,7 +1054,7 @@ namespace A69WD.Net { } catch (Exception ex) { Console.WriteLine($"Errore con {URL}: {ex.Message}"); Status = eLinkState.Error; - //App.DB.LinkLog(lk); + App.DB.LinkLog(this); } return newlinks; } diff --git a/Src/A69WD/Net/iLink.cs b/Src/A69WD/Net/iLink.cs index 59da2de..2a43c80 100644 --- a/Src/A69WD/Net/iLink.cs +++ b/Src/A69WD/Net/iLink.cs @@ -47,6 +47,33 @@ namespace A69WD.Net { cRac EstraiRac(HtmlDocument doc); + + + + // #region Gestione Sito + // + // /// + // /// Booleano per sapere se siamo autenticati al sito + // /// (sempre true per i siti senza autenticazione) + // /// + // bool Autenticated { get; } + // + // /// + // /// Esegue l'autenticazione al sito + // /// + // Task Authenticate(); + // + // /// + // /// Elaborazione della pagina linkata: + // /// Scarica il link e analizza quanto downloadato + // /// + // /// + // /// + // Task> CrawlPageAsync(iLink lk); + // + // #endregion + + }//interface iLink }//namespace A69WD.Net diff --git a/Src/A69WD/Service/Service_Crawler.cs b/Src/A69WD/Service/Service_Crawler.cs index 468c15a..73523fe 100644 --- a/Src/A69WD/Service/Service_Crawler.cs +++ b/Src/A69WD/Service/Service_Crawler.cs @@ -71,23 +71,29 @@ namespace A69WD.Service { /// /// Thread worker: consuma la coda ed avvia task di download. /// - private void ProcessQueue() { + private async void ProcessQueue() { + + // Verifico cartelle di destinazione: + if (!Directory.Exists(Cfg.Settings.PathIdxDone)) Directory.CreateDirectory(Cfg.Settings.PathIdxDone); + if (!Directory.Exists(Cfg.Settings.PathSrcDone)) Directory.CreateDirectory(Cfg.Settings.PathSrcDone); + if (!Directory.Exists(Cfg.Settings.PathSrcErrOnExtract)) Directory.CreateDirectory(Cfg.Settings.PathSrcErrOnExtract); + if (!Directory.Exists(Cfg.Settings.PathSrcErrOnSave)) Directory.CreateDirectory(Cfg.Settings.PathSrcErrOnSave); + + // Autenticazione ai siti: + if (!cLink_A69.Autenticated) { await cLink_A69.Authenticate(); } + + // Ciclo per ciascun elemento della FIFO: foreach (iLink link in _urlQueue.GetConsumingEnumerable()) { Logger.Information("ProcessQueue: Start url " + link.FileName); - // Verifico cartelle di destinazione - if (!Directory.Exists(Cfg.Settings.PathIdxDone)) Directory.CreateDirectory(Cfg.Settings.PathIdxDone); - if (!Directory.Exists(Cfg.Settings.PathSrcDone)) Directory.CreateDirectory(Cfg.Settings.PathSrcDone); - if (!Directory.Exists(Cfg.Settings.PathSrcErrOnExtract)) Directory.CreateDirectory(Cfg.Settings.PathSrcErrOnExtract); - if (!Directory.Exists(Cfg.Settings.PathSrcErrOnSave)) Directory.CreateDirectory(Cfg.Settings.PathSrcErrOnSave); - Task.Run(async () => { Interlocked.Increment(ref _inCorso); UpdateCounters(); await _semaphore.WaitAsync(); try { - List urls = await _site.CrawlPageAsync(link); + cLink_A69 clk = link as cLink_A69; + List urls = await clk.CrawlPageAsync(); Logger.Information("ProcessQueue: end url " + link.FileName + " - URL trovati: " + urls.Count.ToString("000")); foreach (iLink url in urls) { if (url.Validate) { diff --git a/Src/A69WD/Site/cSite_A69.cs b/Src/A69WD/Site/cSite_A69.cs index fc79a31..ee5ae9a 100644 --- a/Src/A69WD/Site/cSite_A69.cs +++ b/Src/A69WD/Site/cSite_A69.cs @@ -187,14 +187,14 @@ namespace A69WD.Site { .OrderBy(s => s) // ordina alfabeticamente .Distinct() // elimina duplicati .ToList(); - // MRK // Inserisco i link trovati - // MRK foreach (string ilk in tmpLk) { - // MRK string[] separator = new string[] { "&pagina=" }; - // MRK string[] txt = ilk.Split(separator, StringSplitOptions.RemoveEmptyEntries); - // MRK string url = cLink_A69.PageBase + ilk; - // MRK string name = txt[0].Replace("/racconti-erotici/racconti_lista.php?genere=", "_A69_") + "_" + int.Parse(txt[1]).ToString("0000") + ".html"; - // MRK newlinks.Add(new cLink_A69(url, name)); - // MRK } + // Inserisco i link trovati + foreach (string ilk in tmpLk) { + string[] separator = new string[] { "&pagina=" }; + string[] txt = ilk.Split(separator, StringSplitOptions.RemoveEmptyEntries); + string url = cLink_A69.PageBase + ilk; + string name = txt[0].Replace("/racconti-erotici/racconti_lista.php?genere=", "_A69_") + "_" + int.Parse(txt[1]).ToString("0000") + ".html"; + newlinks.Add(new cLink_A69(url, name)); + } } catch (Exception ex) { Logger.Error("Errore analisi Idx link dal " + lk.FileName + " - " + ex.Message); SaveToFile(thtml, Cfg.Settings.PathSrcErrOnExtract, lk.FileName); @@ -228,12 +228,12 @@ namespace A69WD.Site { .OrderBy(s => s) // ordina alfabeticamente .Distinct() // elimina duplicati .ToList(); - // MRK // Inserisco i link trovati - // MRK foreach (string ilk in tmpLk) { - // MRK string url = cLink_A69.PageBase + ilk; - // MRK string name = ilk.Replace("/racconti-erotici/", "A69_").Replace(@"/", "_"); - // MRK newlinks.Add(new cLink_A69(url, name)); - // MRK } + // Inserisco i link trovati + foreach (string ilk in tmpLk) { + string url = cLink_A69.PageBase + ilk; + string name = ilk.Replace("/racconti-erotici/", "A69_").Replace(@"/", "_"); + newlinks.Add(new cLink_A69(url, name)); + } } catch (Exception ex) { Logger.Error("Errore analisi Idx link dal " + lk.FileName + " - " + ex.Message); SaveToFile(thtml, Cfg.Settings.PathSrcErrOnExtract, lk.FileName); diff --git a/Src/A69WD/UI/frmReader.Designer.cs b/Src/A69WD/UI/frmReader.Designer.cs index 9b2a899..e6d3a34 100644 --- a/Src/A69WD/UI/frmReader.Designer.cs +++ b/Src/A69WD/UI/frmReader.Designer.cs @@ -33,6 +33,7 @@ this.btnTest3 = new System.Windows.Forms.Button(); this.pnlTop = new System.Windows.Forms.Panel(); this.btn = new System.Windows.Forms.Button(); + this.btnTest5 = new System.Windows.Forms.Button(); this.sb.SuspendLayout(); this.pnlTop.SuspendLayout(); this.SuspendLayout(); @@ -128,6 +129,7 @@ // // pnlTop // + this.pnlTop.Controls.Add(this.btnTest5); this.pnlTop.Controls.Add(this.btn); this.pnlTop.Controls.Add(this.btnTest3); this.pnlTop.Controls.Add(this.btnTest); @@ -148,6 +150,16 @@ this.btn.UseVisualStyleBackColor = true; this.btn.Click += new System.EventHandler(this.btn_Click); // + // btnTest5 + // + this.btnTest5.Location = new System.Drawing.Point(283, 5); + this.btnTest5.Name = "btnTest5"; + this.btnTest5.Size = new System.Drawing.Size(62, 32); + this.btnTest5.TabIndex = 8; + this.btnTest5.Text = "Test 5"; + this.btnTest5.UseVisualStyleBackColor = true; + this.btnTest5.Click += new System.EventHandler(this.btnTest5_Click); + // // frmReader // this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F); @@ -180,5 +192,6 @@ private System.Windows.Forms.Button btnTest3; private System.Windows.Forms.Panel pnlTop; private System.Windows.Forms.Button btn; + private System.Windows.Forms.Button btnTest5; } } \ No newline at end of file diff --git a/Src/A69WD/UI/frmReader.cs b/Src/A69WD/UI/frmReader.cs index 2a23161..01cbbd7 100644 --- a/Src/A69WD/UI/frmReader.cs +++ b/Src/A69WD/UI/frmReader.cs @@ -880,6 +880,13 @@ namespace A69WD.UI { } + private void btnTest5_Click(object sender, EventArgs e) { + // Test download multipli con client diversi ma che condividono solo la sessione + // Nuovo refactoring eliminando il Site inglobandolo sul Link + + + + } } }