diff --git a/Src/A69WD/General/cTools.cs b/Src/A69WD/General/cTools.cs
index 67d9d91..230de02 100644
--- a/Src/A69WD/General/cTools.cs
+++ b/Src/A69WD/General/cTools.cs
@@ -47,6 +47,12 @@ namespace A69WD {
}
+ public static void SaveToFile(string txtHTML, string filePath, string fileName) {
+ System.IO.StreamWriter tsw = new System.IO.StreamWriter(System.IO.Path.Combine(filePath, fileName), false, Encoding.UTF8);
+ tsw.Write(txtHTML);
+ }
+
+
///
/// Metodo che legge un testo da file (UTF8)
///
diff --git a/Src/A69WD/Net/cLink_A69.cs b/Src/A69WD/Net/cLink_A69.cs
index cceba20..b7337ea 100644
--- a/Src/A69WD/Net/cLink_A69.cs
+++ b/Src/A69WD/Net/cLink_A69.cs
@@ -745,7 +745,7 @@ namespace A69WD.Net {
///
/// Indica se siamo autenticati al sito
///
- public bool Autenticated => autenticated;
+ public static bool Autenticated => autenticated;
///
@@ -867,14 +867,11 @@ namespace A69WD.Net {
///
public async Task> CrawlPageAsync() {
var newlinks = new List();
- if (!autenticated) {
- // Sito non autenticato, non posso fare nulla
- Logger.Warning("A69 - Sito non autenticato. Download non permessi.");
- return newlinks;
- }
+
try {
Console.WriteLine($"A69 - Scarico: {URL}");
HtmlDocument tdoc = null;
+ string thtml = "";
#region ToConfirm
if (Status == eLinkState.ToConfirm) {
@@ -900,13 +897,11 @@ namespace A69WD.Net {
return newlinks;
}
// Scarico la pagina
- var thtml = await client.GetStringAsync(URL);
+ thtml = await client.GetStringAsync(URL);
tdoc = new HtmlDocument();
tdoc.LoadHtml(thtml);
// Salvo il file downloadato
- //System.IO.StreamWriter tsw = new System.IO.StreamWriter(lk.DstFilePath, false, Encoding.UTF8);
- System.IO.StreamWriter tsw = new System.IO.StreamWriter(System.IO.Path.Combine(FilePath, FileName), false, Encoding.UTF8);
- tsw.Write(thtml);
+ //SaveToFile(thtml, lk.FilePath, lk.FileName);
Status = eLinkState.ToAnalize;
}
@@ -916,6 +911,7 @@ namespace A69WD.Net {
{
if (Status == eLinkState.ToAnalize) {
List tmpLk = new List();
+
#region Estraggo gli url ad indici:
{
/*
@@ -934,22 +930,22 @@ namespace A69WD.Net {
*
*
*/
- // Estraggo i link alle altre pagine:
- var tnodes = tdoc.DocumentNode.SelectNodes("//div[@id='pagination']//ul//li//a[@href]");
- if (tnodes != null) {
- foreach (var link in tnodes) {
- var href = link.GetAttributeValue("href", "");
- if (!string.IsNullOrEmpty(href) && href.StartsWith("/racconti-erotici/racconti_lista.php?genere="))
- tmpLk.Add(href); //tmpLk.Add(cLink_A69.PageBase + href);
- }
- }
- // Rimuove duplicati e ordina alfabeticamente
- tmpLk = tmpLk
- .OrderBy(s => s) // ordina alfabeticamente
- .Distinct() // elimina duplicati
- .ToList();
- // Inserisco i link trovati
try {
+ // Estraggo i link alle altre pagine:
+ var tnodes = tdoc.DocumentNode.SelectNodes("//div[@id='pagination']//ul//li//a[@href]");
+ if (tnodes != null) {
+ foreach (var link in tnodes) {
+ var href = link.GetAttributeValue("href", "");
+ if (!string.IsNullOrEmpty(href) && href.StartsWith("/racconti-erotici/racconti_lista.php?genere="))
+ tmpLk.Add(href); //tmpLk.Add(cLink_A69.PageBase + href);
+ }
+ }
+ // Rimuove duplicati e ordina alfabeticamente
+ tmpLk = tmpLk
+ .OrderBy(s => s) // ordina alfabeticamente
+ .Distinct() // elimina duplicati
+ .ToList();
+ // Inserisco i link trovati
foreach (string ilk in tmpLk) {
string[] separator = new string[] { "&pagina=" };
string[] txt = ilk.Split(separator, StringSplitOptions.RemoveEmptyEntries);
@@ -957,7 +953,10 @@ namespace A69WD.Net {
string name = txt[0].Replace("/racconti-erotici/racconti_lista.php?genere=", "_A69_") + "_" + int.Parse(txt[1]).ToString("0000") + ".html";
newlinks.Add(new cLink_A69(url, name));
}
- } catch (Exception ex) { Logger.Error("Errore analisi Idx link dal " + FileName + " - " + ex.Message); }
+ } catch (Exception ex) {
+ Logger.Error("Errore analisi Idx link dal " + FileName + " - " + ex.Message);
+ cTools.SaveToFile(thtml, Cfg.Settings.PathSrcErrOnExtract, FileName);
+ }
}
#endregion
@@ -970,30 +969,33 @@ namespace A69WD.Net {
* Continua a leggere ยป
*
*/
- // Estraggo i link ai racconti:
- tmpLk = new List();
- // Estraggo i link alle altre pagine:
- var tnodes = tdoc.DocumentNode.SelectNodes("//div[@class='continua']//a[@href]");
- if (tnodes != null) {
- foreach (var link in tnodes) {
- var href = link.GetAttributeValue("href", "");
- if (!string.IsNullOrEmpty(href) && href.StartsWith("/racconti-erotici/") && (href.EndsWith(".html") || href.EndsWith(".htm")))
- tmpLk.Add(href); //tmpLk.Add(cLink_A69.PageBase + href);
- }
- }
- // Rimuove duplicati e ordina alfabeticamente
- tmpLk = tmpLk
- .OrderBy(s => s) // ordina alfabeticamente
- .Distinct() // elimina duplicati
- .ToList();
- // Inserisco i link trovati
try {
+ // Estraggo i link ai racconti:
+ tmpLk = new List();
+ // Estraggo i link alle altre pagine:
+ var tnodes = tdoc.DocumentNode.SelectNodes("//div[@class='continua']//a[@href]");
+ if (tnodes != null) {
+ foreach (var link in tnodes) {
+ var href = link.GetAttributeValue("href", "");
+ if (!string.IsNullOrEmpty(href) && href.StartsWith("/racconti-erotici/") && (href.EndsWith(".html") || href.EndsWith(".htm")))
+ tmpLk.Add(href); //tmpLk.Add(cLink_A69.PageBase + href);
+ }
+ }
+ // Rimuove duplicati e ordina alfabeticamente
+ tmpLk = tmpLk
+ .OrderBy(s => s) // ordina alfabeticamente
+ .Distinct() // elimina duplicati
+ .ToList();
+ // Inserisco i link trovati
foreach (string ilk in tmpLk) {
string url = cLink_A69.PageBase + ilk;
string name = ilk.Replace("/racconti-erotici/", "A69_").Replace(@"/", "_");
newlinks.Add(new cLink_A69(url, name));
}
- } catch (Exception ex) { Logger.Error("Errore analisi Idx link dal " + FileName + " - " + ex.Message); }
+ } catch (Exception ex) {
+ Logger.Error("Errore analisi Idx link dal " + FileName + " - " + ex.Message);
+ cTools.SaveToFile(thtml, Cfg.Settings.PathSrcErrOnExtract, FileName);
+ }
}
#endregion
@@ -1003,8 +1005,21 @@ namespace A69WD.Net {
cRac rac = EstraiRac(tdoc);
if (rac != null) {
// Racconto estratto correttamente, lo salvo sul DB
- App.DB.ImportRac(rac);
+ int res = App.DB.ImportRac(rac);
+ if (res > 0) {
+ // Racconto salvato correttamente sul DB
+ cTools.SaveToFile(thtml, Cfg.Settings.PathSrcDone, FileName);
+ } else {
+ // Errore nel salvataggio sul DB
+ cTools.SaveToFile(thtml, Cfg.Settings.PathSrcErrOnSave, FileName);
+ }
+ } else {
+ // Estrazione del racconto fallita
+ cTools.SaveToFile(thtml, Cfg.Settings.PathSrcErrOnExtract, FileName);
}
+ } else {
+ // Link ad un indice
+ cTools.SaveToFile(thtml, Cfg.Settings.PathIdxDone, FileName);
}
#endregion
@@ -1039,7 +1054,7 @@ namespace A69WD.Net {
} catch (Exception ex) {
Console.WriteLine($"Errore con {URL}: {ex.Message}");
Status = eLinkState.Error;
- //App.DB.LinkLog(lk);
+ App.DB.LinkLog(this);
}
return newlinks;
}
diff --git a/Src/A69WD/Net/iLink.cs b/Src/A69WD/Net/iLink.cs
index 59da2de..2a43c80 100644
--- a/Src/A69WD/Net/iLink.cs
+++ b/Src/A69WD/Net/iLink.cs
@@ -47,6 +47,33 @@ namespace A69WD.Net {
cRac EstraiRac(HtmlDocument doc);
+
+
+
+ // #region Gestione Sito
+ //
+ // ///
+ // /// Booleano per sapere se siamo autenticati al sito
+ // /// (sempre true per i siti senza autenticazione)
+ // ///
+ // bool Autenticated { get; }
+ //
+ // ///
+ // /// Esegue l'autenticazione al sito
+ // ///
+ // Task Authenticate();
+ //
+ // ///
+ // /// Elaborazione della pagina linkata:
+ // /// Scarica il link e analizza quanto downloadato
+ // ///
+ // ///
+ // ///
+ // Task> CrawlPageAsync(iLink lk);
+ //
+ // #endregion
+
+
}//interface iLink
}//namespace A69WD.Net
diff --git a/Src/A69WD/Service/Service_Crawler.cs b/Src/A69WD/Service/Service_Crawler.cs
index 468c15a..73523fe 100644
--- a/Src/A69WD/Service/Service_Crawler.cs
+++ b/Src/A69WD/Service/Service_Crawler.cs
@@ -71,23 +71,29 @@ namespace A69WD.Service {
///
/// Thread worker: consuma la coda ed avvia task di download.
///
- private void ProcessQueue() {
+ private async void ProcessQueue() {
+
+ // Verifico cartelle di destinazione:
+ if (!Directory.Exists(Cfg.Settings.PathIdxDone)) Directory.CreateDirectory(Cfg.Settings.PathIdxDone);
+ if (!Directory.Exists(Cfg.Settings.PathSrcDone)) Directory.CreateDirectory(Cfg.Settings.PathSrcDone);
+ if (!Directory.Exists(Cfg.Settings.PathSrcErrOnExtract)) Directory.CreateDirectory(Cfg.Settings.PathSrcErrOnExtract);
+ if (!Directory.Exists(Cfg.Settings.PathSrcErrOnSave)) Directory.CreateDirectory(Cfg.Settings.PathSrcErrOnSave);
+
+ // Autenticazione ai siti:
+ if (!cLink_A69.Autenticated) { await cLink_A69.Authenticate(); }
+
+ // Ciclo per ciascun elemento della FIFO:
foreach (iLink link in _urlQueue.GetConsumingEnumerable()) {
Logger.Information("ProcessQueue: Start url " + link.FileName);
- // Verifico cartelle di destinazione
- if (!Directory.Exists(Cfg.Settings.PathIdxDone)) Directory.CreateDirectory(Cfg.Settings.PathIdxDone);
- if (!Directory.Exists(Cfg.Settings.PathSrcDone)) Directory.CreateDirectory(Cfg.Settings.PathSrcDone);
- if (!Directory.Exists(Cfg.Settings.PathSrcErrOnExtract)) Directory.CreateDirectory(Cfg.Settings.PathSrcErrOnExtract);
- if (!Directory.Exists(Cfg.Settings.PathSrcErrOnSave)) Directory.CreateDirectory(Cfg.Settings.PathSrcErrOnSave);
-
Task.Run(async () => {
Interlocked.Increment(ref _inCorso);
UpdateCounters();
await _semaphore.WaitAsync();
try {
- List urls = await _site.CrawlPageAsync(link);
+ cLink_A69 clk = link as cLink_A69;
+ List urls = await clk.CrawlPageAsync();
Logger.Information("ProcessQueue: end url " + link.FileName + " - URL trovati: " + urls.Count.ToString("000"));
foreach (iLink url in urls) {
if (url.Validate) {
diff --git a/Src/A69WD/Site/cSite_A69.cs b/Src/A69WD/Site/cSite_A69.cs
index fc79a31..ee5ae9a 100644
--- a/Src/A69WD/Site/cSite_A69.cs
+++ b/Src/A69WD/Site/cSite_A69.cs
@@ -187,14 +187,14 @@ namespace A69WD.Site {
.OrderBy(s => s) // ordina alfabeticamente
.Distinct() // elimina duplicati
.ToList();
- // MRK // Inserisco i link trovati
- // MRK foreach (string ilk in tmpLk) {
- // MRK string[] separator = new string[] { "&pagina=" };
- // MRK string[] txt = ilk.Split(separator, StringSplitOptions.RemoveEmptyEntries);
- // MRK string url = cLink_A69.PageBase + ilk;
- // MRK string name = txt[0].Replace("/racconti-erotici/racconti_lista.php?genere=", "_A69_") + "_" + int.Parse(txt[1]).ToString("0000") + ".html";
- // MRK newlinks.Add(new cLink_A69(url, name));
- // MRK }
+ // Inserisco i link trovati
+ foreach (string ilk in tmpLk) {
+ string[] separator = new string[] { "&pagina=" };
+ string[] txt = ilk.Split(separator, StringSplitOptions.RemoveEmptyEntries);
+ string url = cLink_A69.PageBase + ilk;
+ string name = txt[0].Replace("/racconti-erotici/racconti_lista.php?genere=", "_A69_") + "_" + int.Parse(txt[1]).ToString("0000") + ".html";
+ newlinks.Add(new cLink_A69(url, name));
+ }
} catch (Exception ex) {
Logger.Error("Errore analisi Idx link dal " + lk.FileName + " - " + ex.Message);
SaveToFile(thtml, Cfg.Settings.PathSrcErrOnExtract, lk.FileName);
@@ -228,12 +228,12 @@ namespace A69WD.Site {
.OrderBy(s => s) // ordina alfabeticamente
.Distinct() // elimina duplicati
.ToList();
- // MRK // Inserisco i link trovati
- // MRK foreach (string ilk in tmpLk) {
- // MRK string url = cLink_A69.PageBase + ilk;
- // MRK string name = ilk.Replace("/racconti-erotici/", "A69_").Replace(@"/", "_");
- // MRK newlinks.Add(new cLink_A69(url, name));
- // MRK }
+ // Inserisco i link trovati
+ foreach (string ilk in tmpLk) {
+ string url = cLink_A69.PageBase + ilk;
+ string name = ilk.Replace("/racconti-erotici/", "A69_").Replace(@"/", "_");
+ newlinks.Add(new cLink_A69(url, name));
+ }
} catch (Exception ex) {
Logger.Error("Errore analisi Idx link dal " + lk.FileName + " - " + ex.Message);
SaveToFile(thtml, Cfg.Settings.PathSrcErrOnExtract, lk.FileName);
diff --git a/Src/A69WD/UI/frmReader.Designer.cs b/Src/A69WD/UI/frmReader.Designer.cs
index 9b2a899..e6d3a34 100644
--- a/Src/A69WD/UI/frmReader.Designer.cs
+++ b/Src/A69WD/UI/frmReader.Designer.cs
@@ -33,6 +33,7 @@
this.btnTest3 = new System.Windows.Forms.Button();
this.pnlTop = new System.Windows.Forms.Panel();
this.btn = new System.Windows.Forms.Button();
+ this.btnTest5 = new System.Windows.Forms.Button();
this.sb.SuspendLayout();
this.pnlTop.SuspendLayout();
this.SuspendLayout();
@@ -128,6 +129,7 @@
//
// pnlTop
//
+ this.pnlTop.Controls.Add(this.btnTest5);
this.pnlTop.Controls.Add(this.btn);
this.pnlTop.Controls.Add(this.btnTest3);
this.pnlTop.Controls.Add(this.btnTest);
@@ -148,6 +150,16 @@
this.btn.UseVisualStyleBackColor = true;
this.btn.Click += new System.EventHandler(this.btn_Click);
//
+ // btnTest5
+ //
+ this.btnTest5.Location = new System.Drawing.Point(283, 5);
+ this.btnTest5.Name = "btnTest5";
+ this.btnTest5.Size = new System.Drawing.Size(62, 32);
+ this.btnTest5.TabIndex = 8;
+ this.btnTest5.Text = "Test 5";
+ this.btnTest5.UseVisualStyleBackColor = true;
+ this.btnTest5.Click += new System.EventHandler(this.btnTest5_Click);
+ //
// frmReader
//
this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F);
@@ -180,5 +192,6 @@
private System.Windows.Forms.Button btnTest3;
private System.Windows.Forms.Panel pnlTop;
private System.Windows.Forms.Button btn;
+ private System.Windows.Forms.Button btnTest5;
}
}
\ No newline at end of file
diff --git a/Src/A69WD/UI/frmReader.cs b/Src/A69WD/UI/frmReader.cs
index 2a23161..01cbbd7 100644
--- a/Src/A69WD/UI/frmReader.cs
+++ b/Src/A69WD/UI/frmReader.cs
@@ -880,6 +880,13 @@ namespace A69WD.UI {
}
+ private void btnTest5_Click(object sender, EventArgs e) {
+ // Test download multipli con client diversi ma che condividono solo la sessione
+ // Nuovo refactoring eliminando il Site inglobandolo sul Link
+
+
+
+ }
}
}