diff --git a/Src/A69WD/Data/DBTools.cs b/Src/A69WD/Data/DBTools.cs
index e73bf57..3a2f511 100644
--- a/Src/A69WD/Data/DBTools.cs
+++ b/Src/A69WD/Data/DBTools.cs
@@ -1075,7 +1075,7 @@ namespace A69WD {
/// Importa il racconto passato sul DB
///
/// Racconto d aimportare
- /// 1 se tutto ok, -1 se importazione abortita per errore, -10 se importazione abortita per duplicato, ....
+ /// IDrac se tutto ok, -1 se importazione abortita per errore, -10 se importazione abortita per duplicato, ....
public int ImportRac(cRac rac) {
lock (objLock) {
// Inizializzo le variabili che occorrono
diff --git a/Src/A69WD/General/Cfg.cs b/Src/A69WD/General/Cfg.cs
index 61b3126..f048c40 100644
--- a/Src/A69WD/General/Cfg.cs
+++ b/Src/A69WD/General/Cfg.cs
@@ -237,6 +237,12 @@ namespace A69WD {
}
private string pathAnalizadedRaccFile = "";
+
+ public virtual string PathSrcDone => System.IO.Path.Combine(PathDownloadWorking, "Done");
+ public virtual string PathIdxDone => System.IO.Path.Combine(PathDownloadWorking, "Idx");
+ public virtual string PathSrcErrOnExtract => System.IO.Path.Combine(PathDownloadWorking, "ErrOnExtract");
+ public virtual string PathSrcErrOnSave => System.IO.Path.Combine(PathDownloadWorking, "ErrOnSave");
+
#endregion
diff --git a/Src/A69WD/Service/Service_Crawler.cs b/Src/A69WD/Service/Service_Crawler.cs
index dfe5e41..468c15a 100644
--- a/Src/A69WD/Service/Service_Crawler.cs
+++ b/Src/A69WD/Service/Service_Crawler.cs
@@ -1,6 +1,7 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
+using System.IO;
using System.Linq;
using System.Net.Http;
using System.Text;
@@ -73,6 +74,13 @@ namespace A69WD.Service {
private void ProcessQueue() {
foreach (iLink link in _urlQueue.GetConsumingEnumerable()) {
Logger.Information("ProcessQueue: Start url " + link.FileName);
+
+ // Verifico cartelle di destinazione
+ if (!Directory.Exists(Cfg.Settings.PathIdxDone)) Directory.CreateDirectory(Cfg.Settings.PathIdxDone);
+ if (!Directory.Exists(Cfg.Settings.PathSrcDone)) Directory.CreateDirectory(Cfg.Settings.PathSrcDone);
+ if (!Directory.Exists(Cfg.Settings.PathSrcErrOnExtract)) Directory.CreateDirectory(Cfg.Settings.PathSrcErrOnExtract);
+ if (!Directory.Exists(Cfg.Settings.PathSrcErrOnSave)) Directory.CreateDirectory(Cfg.Settings.PathSrcErrOnSave);
+
Task.Run(async () => {
Interlocked.Increment(ref _inCorso);
UpdateCounters();
diff --git a/Src/A69WD/Service/Service_Importer.cs b/Src/A69WD/Service/Service_Importer.cs
index 0cdfe9c..bff0b7d 100644
--- a/Src/A69WD/Service/Service_Importer.cs
+++ b/Src/A69WD/Service/Service_Importer.cs
@@ -95,12 +95,10 @@ namespace A69WD.Service {
UpdateCounters();
// Verifico cartelle di destinazione
- string pathErrOnExtract = System.IO.Path.Combine(Cfg.Settings.PathDownloadWorking, "ErrOnExtract");
- if (!Directory.Exists(pathErrOnExtract)) Directory.CreateDirectory(pathErrOnExtract);
- string pathErrOnSave = System.IO.Path.Combine(Cfg.Settings.PathDownloadWorking, "ErrOnSave");
- if (!Directory.Exists(pathErrOnSave)) Directory.CreateDirectory(pathErrOnSave);
- string pathDone = System.IO.Path.Combine(Cfg.Settings.PathDownloadWorking, "Done");
- if (!Directory.Exists(pathDone)) Directory.CreateDirectory(pathDone);
+ if (!Directory.Exists(Cfg.Settings.PathSrcDone)) Directory.CreateDirectory(Cfg.Settings.PathSrcDone);
+ if (!Directory.Exists(Cfg.Settings.PathSrcErrOnExtract)) Directory.CreateDirectory(Cfg.Settings.PathSrcErrOnExtract);
+ if (!Directory.Exists(Cfg.Settings.PathSrcErrOnSave)) Directory.CreateDirectory(Cfg.Settings.PathSrcErrOnSave);
+
// Ciclo su ciascun file
foreach (string file in fileCasuali) {
string name = System.IO.Path.GetFileName(file);
@@ -117,19 +115,19 @@ namespace A69WD.Service {
Logger.Warning($"Import - errore nel salvataggio: {name}");
Interlocked.Increment(ref _errori);
// - Sposto il file
- System.IO.File.Move(file, System.IO.Path.Combine(pathErrOnSave, System.IO.Path.GetFileName(file)));
+ System.IO.File.Move(file, System.IO.Path.Combine(Cfg.Settings.PathSrcErrOnSave, System.IO.Path.GetFileName(file)));
} else {
// salvataggio ok
Logger.Warning($"Import - racconto salvato: {name}");
Interlocked.Increment(ref _completati);
// - Sposto il file
- System.IO.File.Move(file, System.IO.Path.Combine(pathDone, System.IO.Path.GetFileName(file)));
+ System.IO.File.Move(file, System.IO.Path.Combine(Cfg.Settings.PathSrcDone, System.IO.Path.GetFileName(file)));
}
} else {
Logger.Warning($"Import - estrazione errata: {name}");
Interlocked.Increment(ref _errori);
// - Sposto il file
- System.IO.File.Move(file, System.IO.Path.Combine(pathErrOnExtract, System.IO.Path.GetFileName(file)));
+ System.IO.File.Move(file, System.IO.Path.Combine(Cfg.Settings.PathSrcErrOnExtract, System.IO.Path.GetFileName(file)));
}
UpdateCounters();
}
diff --git a/Src/A69WD/Site/cSite_A69.cs b/Src/A69WD/Site/cSite_A69.cs
index 86d164e..fc79a31 100644
--- a/Src/A69WD/Site/cSite_A69.cs
+++ b/Src/A69WD/Site/cSite_A69.cs
@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
+using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
@@ -8,6 +9,7 @@ using System.Threading.Tasks;
using A69WD.Data;
using A69WD.Net;
using HtmlAgilityPack;
+using static System.Windows.Forms.VisualStyles.VisualStyleElement.TrackBar;
namespace A69WD.Site {
@@ -107,9 +109,11 @@ namespace A69WD.Site {
///
public async Task> CrawlPageAsync(iLink lk) {
var newlinks = new List();
+
try {
Console.WriteLine($"A69 - Scarico: {lk.URL}");
HtmlDocument tdoc = null;
+ string thtml = "";
#region ToConfirm
if (lk.Status == eLinkState.ToConfirm) {
@@ -135,13 +139,11 @@ namespace A69WD.Site {
return newlinks;
}
// Scarico la pagina
- var thtml = await client.GetStringAsync(lk.URL);
+ thtml = await client.GetStringAsync(lk.URL);
tdoc = new HtmlDocument();
tdoc.LoadHtml(thtml);
// Salvo il file downloadato
- //System.IO.StreamWriter tsw = new System.IO.StreamWriter(lk.DstFilePath, false, Encoding.UTF8);
- System.IO.StreamWriter tsw = new System.IO.StreamWriter(System.IO.Path.Combine(lk.FilePath, lk.FileName), false, Encoding.UTF8);
- tsw.Write(thtml);
+ //SaveToFile(thtml, lk.FilePath, lk.FileName);
lk.Status = eLinkState.ToAnalize;
}
@@ -151,6 +153,7 @@ namespace A69WD.Site {
{
if (lk.Status == eLinkState.ToAnalize) {
List tmpLk = new List();
+
#region Estraggo gli url ad indici:
{
/*
@@ -169,30 +172,33 @@ namespace A69WD.Site {
*
*
*/
- // Estraggo i link alle altre pagine:
- var tnodes = tdoc.DocumentNode.SelectNodes("//div[@id='pagination']//ul//li//a[@href]");
- if (tnodes != null) {
- foreach (var link in tnodes) {
- var href = link.GetAttributeValue("href", "");
- if (!string.IsNullOrEmpty(href) && href.StartsWith("/racconti-erotici/racconti_lista.php?genere="))
- tmpLk.Add(href); //tmpLk.Add(cLink_A69.PageBase + href);
- }
- }
- // Rimuove duplicati e ordina alfabeticamente
- tmpLk = tmpLk
- .OrderBy(s => s) // ordina alfabeticamente
- .Distinct() // elimina duplicati
- .ToList();
- // Inserisco i link trovati
try {
- foreach (string ilk in tmpLk) {
- string[] separator = new string[] { "&pagina=" };
- string[] txt = ilk.Split(separator, StringSplitOptions.RemoveEmptyEntries);
- string url = cLink_A69.PageBase + ilk;
- string name = txt[0].Replace("/racconti-erotici/racconti_lista.php?genere=", "_A69_") + "_" + int.Parse(txt[1]).ToString("0000") + ".html";
- newlinks.Add(new cLink_A69(url, name));
+ // Estraggo i link alle altre pagine:
+ var tnodes = tdoc.DocumentNode.SelectNodes("//div[@id='pagination']//ul//li//a[@href]");
+ if (tnodes != null) {
+ foreach (var link in tnodes) {
+ var href = link.GetAttributeValue("href", "");
+ if (!string.IsNullOrEmpty(href) && href.StartsWith("/racconti-erotici/racconti_lista.php?genere="))
+ tmpLk.Add(href); //tmpLk.Add(cLink_A69.PageBase + href);
+ }
}
- } catch (Exception ex) { Logger.Error("Errore analisi Idx link dal " + lk.FileName + " - " + ex.Message); }
+ // Rimuove duplicati e ordina alfabeticamente
+ tmpLk = tmpLk
+ .OrderBy(s => s) // ordina alfabeticamente
+ .Distinct() // elimina duplicati
+ .ToList();
+ // MRK // Inserisco i link trovati
+ // MRK foreach (string ilk in tmpLk) {
+ // MRK string[] separator = new string[] { "&pagina=" };
+ // MRK string[] txt = ilk.Split(separator, StringSplitOptions.RemoveEmptyEntries);
+ // MRK string url = cLink_A69.PageBase + ilk;
+ // MRK string name = txt[0].Replace("/racconti-erotici/racconti_lista.php?genere=", "_A69_") + "_" + int.Parse(txt[1]).ToString("0000") + ".html";
+ // MRK newlinks.Add(new cLink_A69(url, name));
+ // MRK }
+ } catch (Exception ex) {
+ Logger.Error("Errore analisi Idx link dal " + lk.FileName + " - " + ex.Message);
+ SaveToFile(thtml, Cfg.Settings.PathSrcErrOnExtract, lk.FileName);
+ }
}
#endregion
@@ -205,30 +211,33 @@ namespace A69WD.Site {
* Continua a leggere »
*
*/
- // Estraggo i link ai racconti:
- tmpLk = new List();
- // Estraggo i link alle altre pagine:
- var tnodes = tdoc.DocumentNode.SelectNodes("//div[@class='continua']//a[@href]");
- if (tnodes != null) {
- foreach (var link in tnodes) {
- var href = link.GetAttributeValue("href", "");
- if (!string.IsNullOrEmpty(href) && href.StartsWith("/racconti-erotici/") && (href.EndsWith(".html") || href.EndsWith(".htm")))
- tmpLk.Add(href); //tmpLk.Add(cLink_A69.PageBase + href);
- }
- }
- // Rimuove duplicati e ordina alfabeticamente
- tmpLk = tmpLk
- .OrderBy(s => s) // ordina alfabeticamente
- .Distinct() // elimina duplicati
- .ToList();
- // Inserisco i link trovati
try {
- foreach (string ilk in tmpLk) {
- string url = cLink_A69.PageBase + ilk;
- string name = ilk.Replace("/racconti-erotici/", "A69_").Replace(@"/","_");
- newlinks.Add(new cLink_A69(url, name));
+ // Estraggo i link ai racconti:
+ tmpLk = new List();
+ // Estraggo i link alle altre pagine:
+ var tnodes = tdoc.DocumentNode.SelectNodes("//div[@class='continua']//a[@href]");
+ if (tnodes != null) {
+ foreach (var link in tnodes) {
+ var href = link.GetAttributeValue("href", "");
+ if (!string.IsNullOrEmpty(href) && href.StartsWith("/racconti-erotici/") && (href.EndsWith(".html") || href.EndsWith(".htm")))
+ tmpLk.Add(href); //tmpLk.Add(cLink_A69.PageBase + href);
+ }
}
- } catch (Exception ex) { Logger.Error("Errore analisi Idx link dal " + lk.FileName + " - " + ex.Message); }
+ // Rimuove duplicati e ordina alfabeticamente
+ tmpLk = tmpLk
+ .OrderBy(s => s) // ordina alfabeticamente
+ .Distinct() // elimina duplicati
+ .ToList();
+ // MRK // Inserisco i link trovati
+ // MRK foreach (string ilk in tmpLk) {
+ // MRK string url = cLink_A69.PageBase + ilk;
+ // MRK string name = ilk.Replace("/racconti-erotici/", "A69_").Replace(@"/", "_");
+ // MRK newlinks.Add(new cLink_A69(url, name));
+ // MRK }
+ } catch (Exception ex) {
+ Logger.Error("Errore analisi Idx link dal " + lk.FileName + " - " + ex.Message);
+ SaveToFile(thtml, Cfg.Settings.PathSrcErrOnExtract, lk.FileName);
+ }
}
#endregion
@@ -238,8 +247,21 @@ namespace A69WD.Site {
cRac rac = lk.EstraiRac(tdoc);
if (rac != null) {
// Racconto estratto correttamente, lo salvo sul DB
- App.DB.ImportRac(rac);
+ int res = App.DB.ImportRac(rac);
+ if (res>0) {
+ // Racconto salvato correttamente sul DB
+ SaveToFile(thtml, Cfg.Settings.PathSrcDone, lk.FileName);
+ } else {
+ // Errore nel salvataggio sul DB
+ SaveToFile(thtml, Cfg.Settings.PathSrcErrOnSave, lk.FileName);
+ }
+ } else {
+ // Estrazione del racconto fallita
+ SaveToFile(thtml, Cfg.Settings.PathSrcErrOnExtract, lk.FileName);
}
+ } else {
+ // Link ad un indice
+ SaveToFile(thtml, Cfg.Settings.PathIdxDone, lk.FileName);
}
#endregion
@@ -274,10 +296,14 @@ namespace A69WD.Site {
} catch (Exception ex) {
Console.WriteLine($"Errore con {lk.URL}: {ex.Message}");
lk.Status = eLinkState.Error;
- //App.DB.LinkLog(lk);
+ App.DB.LinkLog(lk);
}
return newlinks;
}
+ private void SaveToFile(string txtHTML, string filePath, string fileName) {
+ System.IO.StreamWriter tsw = new System.IO.StreamWriter(System.IO.Path.Combine(filePath, fileName), false, Encoding.UTF8);
+ tsw.Write(txtHTML);
+ }
public async Task> CrawlPageAsyncTest1(iLink lk) {
var newlinks = new List();
diff --git a/Src/A69WD/UI/frmReader.Designer.cs b/Src/A69WD/UI/frmReader.Designer.cs
index f73280f..9b2a899 100644
--- a/Src/A69WD/UI/frmReader.Designer.cs
+++ b/Src/A69WD/UI/frmReader.Designer.cs
@@ -31,21 +31,22 @@
this.btnTest = new System.Windows.Forms.Button();
this.btnTest2 = new System.Windows.Forms.Button();
this.btnTest3 = new System.Windows.Forms.Button();
+ this.pnlTop = new System.Windows.Forms.Panel();
+ this.btn = new System.Windows.Forms.Button();
this.sb.SuspendLayout();
+ this.pnlTop.SuspendLayout();
this.SuspendLayout();
//
// rtb
//
- this.rtb.Anchor = ((System.Windows.Forms.AnchorStyles)((((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Bottom)
- | System.Windows.Forms.AnchorStyles.Left)
- | System.Windows.Forms.AnchorStyles.Right)));
this.rtb.BackColor = System.Drawing.Color.White;
this.rtb.BorderStyle = System.Windows.Forms.BorderStyle.None;
+ this.rtb.Dock = System.Windows.Forms.DockStyle.Fill;
this.rtb.Font = new System.Drawing.Font("Verdana", 11.25F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
- this.rtb.Location = new System.Drawing.Point(0, 54);
+ this.rtb.Location = new System.Drawing.Point(0, 42);
this.rtb.Name = "rtb";
- this.rtb.Size = new System.Drawing.Size(700, 366);
- this.rtb.TabIndex = 3;
+ this.rtb.Size = new System.Drawing.Size(700, 378);
+ this.rtb.TabIndex = 2;
this.rtb.Text = "";
this.rtb.KeyDown += new System.Windows.Forms.KeyEventHandler(this.KeyDownAnalisi);
//
@@ -59,7 +60,7 @@
this.sb.Location = new System.Drawing.Point(0, 420);
this.sb.Name = "sb";
this.sb.Size = new System.Drawing.Size(700, 24);
- this.sb.TabIndex = 2;
+ this.sb.TabIndex = 1;
this.sb.Text = "statusStrip1";
//
// sbMessage
@@ -97,9 +98,9 @@
//
// btnTest
//
- this.btnTest.Location = new System.Drawing.Point(4, 4);
+ this.btnTest.Location = new System.Drawing.Point(11, 5);
this.btnTest.Name = "btnTest";
- this.btnTest.Size = new System.Drawing.Size(62, 48);
+ this.btnTest.Size = new System.Drawing.Size(62, 32);
this.btnTest.TabIndex = 4;
this.btnTest.Text = "Test";
this.btnTest.UseVisualStyleBackColor = true;
@@ -107,9 +108,9 @@
//
// btnTest2
//
- this.btnTest2.Location = new System.Drawing.Point(72, 4);
+ this.btnTest2.Location = new System.Drawing.Point(79, 5);
this.btnTest2.Name = "btnTest2";
- this.btnTest2.Size = new System.Drawing.Size(62, 48);
+ this.btnTest2.Size = new System.Drawing.Size(62, 32);
this.btnTest2.TabIndex = 5;
this.btnTest2.Text = "Test 2";
this.btnTest2.UseVisualStyleBackColor = true;
@@ -117,24 +118,44 @@
//
// btnTest3
//
- this.btnTest3.Location = new System.Drawing.Point(140, 4);
+ this.btnTest3.Location = new System.Drawing.Point(147, 5);
this.btnTest3.Name = "btnTest3";
- this.btnTest3.Size = new System.Drawing.Size(62, 48);
+ this.btnTest3.Size = new System.Drawing.Size(62, 32);
this.btnTest3.TabIndex = 6;
this.btnTest3.Text = "Test 3";
this.btnTest3.UseVisualStyleBackColor = true;
this.btnTest3.Click += new System.EventHandler(this.btnTest3_Click);
//
+ // pnlTop
+ //
+ this.pnlTop.Controls.Add(this.btn);
+ this.pnlTop.Controls.Add(this.btnTest3);
+ this.pnlTop.Controls.Add(this.btnTest);
+ this.pnlTop.Controls.Add(this.btnTest2);
+ this.pnlTop.Dock = System.Windows.Forms.DockStyle.Top;
+ this.pnlTop.Location = new System.Drawing.Point(0, 0);
+ this.pnlTop.Name = "pnlTop";
+ this.pnlTop.Size = new System.Drawing.Size(700, 42);
+ this.pnlTop.TabIndex = 0;
+ //
+ // btn
+ //
+ this.btn.Location = new System.Drawing.Point(215, 5);
+ this.btn.Name = "btn";
+ this.btn.Size = new System.Drawing.Size(62, 32);
+ this.btn.TabIndex = 7;
+ this.btn.Text = "Test 4";
+ this.btn.UseVisualStyleBackColor = true;
+ this.btn.Click += new System.EventHandler(this.btn_Click);
+ //
// frmReader
//
this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F);
this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font;
this.ClientSize = new System.Drawing.Size(700, 444);
this.ControlBox = false;
- this.Controls.Add(this.btnTest3);
- this.Controls.Add(this.btnTest2);
- this.Controls.Add(this.btnTest);
this.Controls.Add(this.rtb);
+ this.Controls.Add(this.pnlTop);
this.Controls.Add(this.sb);
this.Name = "frmReader";
this.Text = "PlugIn_App_Reader.UI Reader (DB vrs1.3, App" +
@@ -142,6 +163,7 @@
this.FormClosing += new System.Windows.Forms.FormClosingEventHandler(this.frmReader_FormClosing);
this.sb.ResumeLayout(false);
this.sb.PerformLayout();
+ this.pnlTop.ResumeLayout(false);
this.ResumeLayout(false);
}
@@ -156,5 +178,7 @@
private System.Windows.Forms.Button btnTest;
private System.Windows.Forms.Button btnTest2;
private System.Windows.Forms.Button btnTest3;
+ private System.Windows.Forms.Panel pnlTop;
+ private System.Windows.Forms.Button btn;
}
}
\ No newline at end of file
diff --git a/Src/A69WD/UI/frmReader.cs b/Src/A69WD/UI/frmReader.cs
index 00782d8..2a23161 100644
--- a/Src/A69WD/UI/frmReader.cs
+++ b/Src/A69WD/UI/frmReader.cs
@@ -850,6 +850,35 @@ namespace A69WD.UI {
}
+ private async void btn_Click(object sender, EventArgs e) {
+
+ // - Creo istanza del client per il sito con autenticazione:
+ sbStatusLabel.Text = "Login on A69";
+ siteA69 = new cSite_A69();
+
+ // - Eseguo l'autenticazione
+ bool res = await siteA69.Authenticate();
+ if (res) {
+ sbStatusLabel.Text = "A69 autenticated";
+ Logger.Information("Sito A69 loggato: " + res.ToString());
+ } else {
+ Logger.Information("Login Fault");
+ return;
+ }
+
+ // - Verifico path per il download
+ if (!System.IO.Directory.Exists(Cfg.Settings.PathDownloadWorking)) System.IO.Directory.CreateDirectory(Cfg.Settings.PathDownloadWorking);
+
+ // Preparo il servizio
+ string url = "https://www.annunci69.it/racconti-erotici/racconti_lista.php?genere=altro&pagina=1";
+ string name = "_A69_altro_0001.html";
+ serviceCrawler = new Service_Crawler(sb);
+ // Inserisco i link iniziali nella coda
+ serviceCrawler.EnqueueUrl(new cLink_A69(url, name));
+ // avvio il servizio
+ serviceCrawler.StartWorker(siteA69);
+
+ }
}