MrkPi/A69WD
3
0 Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

Migliorato lo skraping #5

Merged
Mrk merged 3 commits from mrk/Skraping Upgrade AGit into master 2026-01-01 16:14:22 +01:00
Conversation 0 Commits 3 Files Changed 13
+142 -68
7 changed files with 142 additions and 68 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 0d2b5a1f49 - Show all commits

No files matched your search

Src/A69WD/General/cTools.cs
+6
View File
@@ -47,6 +47,12 @@ namespace A69WD {
}
public static void SaveToFile(string txtHTML, string filePath, string fileName) {
System.IO.StreamWriter tsw = new System.IO.StreamWriter(System.IO.Path.Combine(filePath, fileName), false, Encoding.UTF8);
tsw.Write(txtHTML);
}
/// <summary>
/// Metodo che legge un testo da file (UTF8)
/// </summary>
Src/A69WD/Net/cLink_A69.cs
+61 -46
View File
@@ -745,7 +745,7 @@ namespace A69WD.Net {
/// <summary>
/// Indica se siamo autenticati al sito
/// </summary>
public bool Autenticated => autenticated;
public static bool Autenticated => autenticated;
/// <summary>
@@ -867,14 +867,11 @@ namespace A69WD.Net {
/// <returns></returns>
public async Task<List<iLink>> CrawlPageAsync() {
var newlinks = new List<iLink>();
if (!autenticated) {
// Sito non autenticato, non posso fare nulla
Logger.Warning("A69 - Sito non autenticato. Download non permessi.");
return newlinks;
}
try {
Console.WriteLine($"A69 - Scarico: {URL}");
HtmlDocument tdoc = null;
string thtml = "";
#region ToConfirm
if (Status == eLinkState.ToConfirm) {
@@ -900,13 +897,11 @@ namespace A69WD.Net {
return newlinks;
}
// Scarico la pagina
var thtml = await client.GetStringAsync(URL);
thtml = await client.GetStringAsync(URL);
tdoc = new HtmlDocument();
tdoc.LoadHtml(thtml);
// Salvo il file downloadato
//System.IO.StreamWriter tsw = new System.IO.StreamWriter(lk.DstFilePath, false, Encoding.UTF8);
System.IO.StreamWriter tsw = new System.IO.StreamWriter(System.IO.Path.Combine(FilePath, FileName), false, Encoding.UTF8);
tsw.Write(thtml);
//SaveToFile(thtml, lk.FilePath, lk.FileName);
Status = eLinkState.ToAnalize;
}
@@ -916,6 +911,7 @@ namespace A69WD.Net {
{
if (Status == eLinkState.ToAnalize) {
List<string> tmpLk = new List<string>();
#region Estraggo gli url ad indici:
{
/*
@@ -934,22 +930,22 @@ namespace A69WD.Net {
* </div>
*
*/
// Estraggo i link alle altre pagine:
var tnodes = tdoc.DocumentNode.SelectNodes("//div[@id='pagination']//ul//li//a[@href]");
if (tnodes != null) {
foreach (var link in tnodes) {
var href = link.GetAttributeValue("href", "");
if (!string.IsNullOrEmpty(href) && href.StartsWith("/racconti-erotici/racconti_lista.php?genere="))
tmpLk.Add(href); //tmpLk.Add(cLink_A69.PageBase + href);
}
}
// Rimuove duplicati e ordina alfabeticamente
tmpLk = tmpLk
.OrderBy(s => s) // ordina alfabeticamente
.Distinct() // elimina duplicati
.ToList();
// Inserisco i link trovati
try {
// Estraggo i link alle altre pagine:
var tnodes = tdoc.DocumentNode.SelectNodes("//div[@id='pagination']//ul//li//a[@href]");
if (tnodes != null) {
foreach (var link in tnodes) {
var href = link.GetAttributeValue("href", "");
if (!string.IsNullOrEmpty(href) && href.StartsWith("/racconti-erotici/racconti_lista.php?genere="))
tmpLk.Add(href); //tmpLk.Add(cLink_A69.PageBase + href);
}
}
// Rimuove duplicati e ordina alfabeticamente
tmpLk = tmpLk
.OrderBy(s => s) // ordina alfabeticamente
.Distinct() // elimina duplicati
.ToList();
// Inserisco i link trovati
foreach (string ilk in tmpLk) {
string[] separator = new string[] { "&pagina=" };
string[] txt = ilk.Split(separator, StringSplitOptions.RemoveEmptyEntries);
@@ -957,7 +953,10 @@ namespace A69WD.Net {
string name = txt[0].Replace("/racconti-erotici/racconti_lista.php?genere=", "_A69_") + "_" + int.Parse(txt[1]).ToString("0000") + ".html";
newlinks.Add(new cLink_A69(url, name));
}
} catch (Exception ex) { Logger.Error("Errore analisi Idx link dal " + FileName + " - " + ex.Message); }
} catch (Exception ex) {
Logger.Error("Errore analisi Idx link dal " + FileName + " - " + ex.Message);
cTools.SaveToFile(thtml, Cfg.Settings.PathSrcErrOnExtract, FileName);
}
}
#endregion
@@ -970,30 +969,33 @@ namespace A69WD.Net {
* <a href="/racconti-erotici/altro/Tutto-e-iniziato-per-gioco-pazza-famiglia_166770.html">Continua a leggere » </a>
* </div>
*/
// Estraggo i link ai racconti:
tmpLk = new List<string>();
// Estraggo i link alle altre pagine:
var tnodes = tdoc.DocumentNode.SelectNodes("//div[@class='continua']//a[@href]");
if (tnodes != null) {
foreach (var link in tnodes) {
var href = link.GetAttributeValue("href", "");
if (!string.IsNullOrEmpty(href) && href.StartsWith("/racconti-erotici/") && (href.EndsWith(".html") || href.EndsWith(".htm")))
tmpLk.Add(href); //tmpLk.Add(cLink_A69.PageBase + href);
}
}
// Rimuove duplicati e ordina alfabeticamente
tmpLk = tmpLk
.OrderBy(s => s) // ordina alfabeticamente
.Distinct() // elimina duplicati
.ToList();
// Inserisco i link trovati
try {
// Estraggo i link ai racconti:
tmpLk = new List<string>();
// Estraggo i link alle altre pagine:
var tnodes = tdoc.DocumentNode.SelectNodes("//div[@class='continua']//a[@href]");
if (tnodes != null) {
foreach (var link in tnodes) {
var href = link.GetAttributeValue("href", "");
if (!string.IsNullOrEmpty(href) && href.StartsWith("/racconti-erotici/") && (href.EndsWith(".html") || href.EndsWith(".htm")))
tmpLk.Add(href); //tmpLk.Add(cLink_A69.PageBase + href);
}
}
// Rimuove duplicati e ordina alfabeticamente
tmpLk = tmpLk
.OrderBy(s => s) // ordina alfabeticamente
.Distinct() // elimina duplicati
.ToList();
// Inserisco i link trovati
foreach (string ilk in tmpLk) {
string url = cLink_A69.PageBase + ilk;
string name = ilk.Replace("/racconti-erotici/", "A69_").Replace(@"/", "_");
newlinks.Add(new cLink_A69(url, name));
}
} catch (Exception ex) { Logger.Error("Errore analisi Idx link dal " + FileName + " - " + ex.Message); }
} catch (Exception ex) {
Logger.Error("Errore analisi Idx link dal " + FileName + " - " + ex.Message);
cTools.SaveToFile(thtml, Cfg.Settings.PathSrcErrOnExtract, FileName);
}
}
#endregion
@@ -1003,8 +1005,21 @@ namespace A69WD.Net {
cRac rac = EstraiRac(tdoc);
if (rac != null) {
// Racconto estratto correttamente, lo salvo sul DB
App.DB.ImportRac(rac);
int res = App.DB.ImportRac(rac);
if (res > 0) {
// Racconto salvato correttamente sul DB
cTools.SaveToFile(thtml, Cfg.Settings.PathSrcDone, FileName);
} else {
// Errore nel salvataggio sul DB
cTools.SaveToFile(thtml, Cfg.Settings.PathSrcErrOnSave, FileName);
}
} else {
// Estrazione del racconto fallita
cTools.SaveToFile(thtml, Cfg.Settings.PathSrcErrOnExtract, FileName);
}
} else {
// Link ad un indice
cTools.SaveToFile(thtml, Cfg.Settings.PathIdxDone, FileName);
}
#endregion
@@ -1039,7 +1054,7 @@ namespace A69WD.Net {
} catch (Exception ex) {
Console.WriteLine($"Errore con {URL}: {ex.Message}");
Status = eLinkState.Error;
//App.DB.LinkLog(lk);
App.DB.LinkLog(this);
}
return newlinks;
}
Src/A69WD/Net/iLink.cs
+27
View File
@@ -47,6 +47,33 @@ namespace A69WD.Net {
cRac EstraiRac(HtmlDocument doc);
// #region Gestione Sito
//
// /// <summary>
// /// Booleano per sapere se siamo autenticati al sito
// /// (sempre true per i siti senza autenticazione)
// /// </summary>
// bool Autenticated { get; }
//
// /// <summary>
// /// Esegue l'autenticazione al sito
// /// </summary>
// Task<bool> Authenticate();
//
// /// <summary>
// /// Elaborazione della pagina linkata:
// /// Scarica il link e analizza quanto downloadato
// /// </summary>
// /// <param name="lk"></param>
// /// <returns></returns>
// Task<List<iLink>> CrawlPageAsync(iLink lk);
//
// #endregion
}//interface iLink
}//namespace A69WD.Net
Src/A69WD/Service/Service_Crawler.cs
+14 -8
View File
@@ -71,23 +71,29 @@ namespace A69WD.Service {
/// <summary>
/// Thread worker: consuma la coda ed avvia task di download.
/// </summary>
private void ProcessQueue() {
private async void ProcessQueue() {
// Verifico cartelle di destinazione:
if (!Directory.Exists(Cfg.Settings.PathIdxDone)) Directory.CreateDirectory(Cfg.Settings.PathIdxDone);
if (!Directory.Exists(Cfg.Settings.PathSrcDone)) Directory.CreateDirectory(Cfg.Settings.PathSrcDone);
if (!Directory.Exists(Cfg.Settings.PathSrcErrOnExtract)) Directory.CreateDirectory(Cfg.Settings.PathSrcErrOnExtract);
if (!Directory.Exists(Cfg.Settings.PathSrcErrOnSave)) Directory.CreateDirectory(Cfg.Settings.PathSrcErrOnSave);
// Autenticazione ai siti:
if (!cLink_A69.Autenticated) { await cLink_A69.Authenticate(); }
// Ciclo per ciascun elemento della FIFO:
foreach (iLink link in _urlQueue.GetConsumingEnumerable()) {
Logger.Information("ProcessQueue: Start url " + link.FileName);
// Verifico cartelle di destinazione
if (!Directory.Exists(Cfg.Settings.PathIdxDone)) Directory.CreateDirectory(Cfg.Settings.PathIdxDone);
if (!Directory.Exists(Cfg.Settings.PathSrcDone)) Directory.CreateDirectory(Cfg.Settings.PathSrcDone);
if (!Directory.Exists(Cfg.Settings.PathSrcErrOnExtract)) Directory.CreateDirectory(Cfg.Settings.PathSrcErrOnExtract);
if (!Directory.Exists(Cfg.Settings.PathSrcErrOnSave)) Directory.CreateDirectory(Cfg.Settings.PathSrcErrOnSave);
Task.Run(async () => {
Interlocked.Increment(ref _inCorso);
UpdateCounters();
await _semaphore.WaitAsync();
try {
List<iLink> urls = await _site.CrawlPageAsync(link);
cLink_A69 clk = link as cLink_A69;
List<iLink> urls = await clk.CrawlPageAsync();
Logger.Information("ProcessQueue: end url " + link.FileName + " - URL trovati: " + urls.Count.ToString("000"));
foreach (iLink url in urls) {
if (url.Validate) {
Src/A69WD/Site/cSite_A69.cs
+14 -14
View File
@@ -187,14 +187,14 @@ namespace A69WD.Site {
.OrderBy(s => s) // ordina alfabeticamente
.Distinct() // elimina duplicati
.ToList();
// MRK // Inserisco i link trovati
// MRK foreach (string ilk in tmpLk) {
// MRK string[] separator = new string[] { "&pagina=" };
// MRK string[] txt = ilk.Split(separator, StringSplitOptions.RemoveEmptyEntries);
// MRK string url = cLink_A69.PageBase + ilk;
// MRK string name = txt[0].Replace("/racconti-erotici/racconti_lista.php?genere=", "_A69_") + "_" + int.Parse(txt[1]).ToString("0000") + ".html";
// MRK newlinks.Add(new cLink_A69(url, name));
// MRK }
// Inserisco i link trovati
foreach (string ilk in tmpLk) {
string[] separator = new string[] { "&pagina=" };
string[] txt = ilk.Split(separator, StringSplitOptions.RemoveEmptyEntries);
string url = cLink_A69.PageBase + ilk;
string name = txt[0].Replace("/racconti-erotici/racconti_lista.php?genere=", "_A69_") + "_" + int.Parse(txt[1]).ToString("0000") + ".html";
newlinks.Add(new cLink_A69(url, name));
}
} catch (Exception ex) {
Logger.Error("Errore analisi Idx link dal " + lk.FileName + " - " + ex.Message);
SaveToFile(thtml, Cfg.Settings.PathSrcErrOnExtract, lk.FileName);
@@ -228,12 +228,12 @@ namespace A69WD.Site {
.OrderBy(s => s) // ordina alfabeticamente
.Distinct() // elimina duplicati
.ToList();
// MRK // Inserisco i link trovati
// MRK foreach (string ilk in tmpLk) {
// MRK string url = cLink_A69.PageBase + ilk;
// MRK string name = ilk.Replace("/racconti-erotici/", "A69_").Replace(@"/", "_");
// MRK newlinks.Add(new cLink_A69(url, name));
// MRK }
// Inserisco i link trovati
foreach (string ilk in tmpLk) {
string url = cLink_A69.PageBase + ilk;
string name = ilk.Replace("/racconti-erotici/", "A69_").Replace(@"/", "_");
newlinks.Add(new cLink_A69(url, name));
}
} catch (Exception ex) {
Logger.Error("Errore analisi Idx link dal " + lk.FileName + " - " + ex.Message);
SaveToFile(thtml, Cfg.Settings.PathSrcErrOnExtract, lk.FileName);
Src/A69WD/UI/frmReader.Designer.cs
Generated
+13
View File
@@ -33,6 +33,7 @@
this.btnTest3 = new System.Windows.Forms.Button();
this.pnlTop = new System.Windows.Forms.Panel();
this.btn = new System.Windows.Forms.Button();
this.btnTest5 = new System.Windows.Forms.Button();
this.sb.SuspendLayout();
this.pnlTop.SuspendLayout();
this.SuspendLayout();
@@ -128,6 +129,7 @@
//
// pnlTop
//
this.pnlTop.Controls.Add(this.btnTest5);
this.pnlTop.Controls.Add(this.btn);
this.pnlTop.Controls.Add(this.btnTest3);
this.pnlTop.Controls.Add(this.btnTest);
@@ -148,6 +150,16 @@
this.btn.UseVisualStyleBackColor = true;
this.btn.Click += new System.EventHandler(this.btn_Click);
//
// btnTest5
//
this.btnTest5.Location = new System.Drawing.Point(283, 5);
this.btnTest5.Name = "btnTest5";
this.btnTest5.Size = new System.Drawing.Size(62, 32);
this.btnTest5.TabIndex = 8;
this.btnTest5.Text = "Test 5";
this.btnTest5.UseVisualStyleBackColor = true;
this.btnTest5.Click += new System.EventHandler(this.btnTest5_Click);
//
// frmReader
//
this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F);
@@ -180,5 +192,6 @@
private System.Windows.Forms.Button btnTest3;
private System.Windows.Forms.Panel pnlTop;
private System.Windows.Forms.Button btn;
private System.Windows.Forms.Button btnTest5;
}
}
Src/A69WD/UI/frmReader.cs
+7
View File
@@ -880,6 +880,13 @@ namespace A69WD.UI {
}
private void btnTest5_Click(object sender, EventArgs e) {
// Test download multipli con client diversi ma che condividono solo la sessione
// Nuovo refactoring eliminando il Site inglobandolo sul Link
}
}
}