Upgradato lo skraping impostando salvataggi diversi per ciasucn file.

Src/A69WD/Data/DBTools.cs

@@ -1075,7 +1075,7 @@ namespace A69WD {
         /// Importa il racconto passato sul DB
         /// </summary>
         /// <param name="rac">Racconto d aimportare</param>
-        /// <returns>1 se tutto ok, -1 se importazione abortita per errore, -10 se importazione abortita per duplicato, ....</returns>
+        /// <returns>IDrac se tutto ok, -1 se importazione abortita per errore, -10 se importazione abortita per duplicato, ....</returns>
         public int ImportRac(cRac rac) {
             lock (objLock) {
                 // Inizializzo le variabili che occorrono

Src/A69WD/General/Cfg.cs

@@ -237,6 +237,12 @@ namespace A69WD {
         }
         private string pathAnalizadedRaccFile = "";
 
+
+        public virtual string PathSrcDone => System.IO.Path.Combine(PathDownloadWorking, "Done");
+        public virtual string PathIdxDone => System.IO.Path.Combine(PathDownloadWorking, "Idx");
+        public virtual string PathSrcErrOnExtract => System.IO.Path.Combine(PathDownloadWorking, "ErrOnExtract");
+        public virtual string PathSrcErrOnSave => System.IO.Path.Combine(PathDownloadWorking, "ErrOnSave");
+
         #endregion
 
 

Src/A69WD/Service/Service_Crawler.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
+using System.IO;
 using System.Linq;
 using System.Net.Http;
 using System.Text;
@@ -73,6 +74,13 @@ namespace A69WD.Service {
         private void ProcessQueue() {
             foreach (iLink link in _urlQueue.GetConsumingEnumerable()) {
                 Logger.Information("ProcessQueue: Start url " + link.FileName);
+
+                // Verifico cartelle di destinazione
+                if (!Directory.Exists(Cfg.Settings.PathIdxDone)) Directory.CreateDirectory(Cfg.Settings.PathIdxDone);
+                if (!Directory.Exists(Cfg.Settings.PathSrcDone)) Directory.CreateDirectory(Cfg.Settings.PathSrcDone);
+                if (!Directory.Exists(Cfg.Settings.PathSrcErrOnExtract)) Directory.CreateDirectory(Cfg.Settings.PathSrcErrOnExtract);
+                if (!Directory.Exists(Cfg.Settings.PathSrcErrOnSave)) Directory.CreateDirectory(Cfg.Settings.PathSrcErrOnSave);
+
                 Task.Run(async () => {
                     Interlocked.Increment(ref _inCorso);
                     UpdateCounters();

Src/A69WD/Service/Service_Importer.cs

@@ -95,12 +95,10 @@ namespace A69WD.Service {
             UpdateCounters();
 
             // Verifico cartelle di destinazione
-            string pathErrOnExtract = System.IO.Path.Combine(Cfg.Settings.PathDownloadWorking, "ErrOnExtract");
+            if (!Directory.Exists(Cfg.Settings.PathSrcDone)) Directory.CreateDirectory(Cfg.Settings.PathSrcDone);
-            if (!Directory.Exists(pathErrOnExtract)) Directory.CreateDirectory(pathErrOnExtract);
+            if (!Directory.Exists(Cfg.Settings.PathSrcErrOnExtract)) Directory.CreateDirectory(Cfg.Settings.PathSrcErrOnExtract);
-            string pathErrOnSave = System.IO.Path.Combine(Cfg.Settings.PathDownloadWorking, "ErrOnSave");
+            if (!Directory.Exists(Cfg.Settings.PathSrcErrOnSave)) Directory.CreateDirectory(Cfg.Settings.PathSrcErrOnSave);
-            if (!Directory.Exists(pathErrOnSave)) Directory.CreateDirectory(pathErrOnSave);
+
-            string pathDone = System.IO.Path.Combine(Cfg.Settings.PathDownloadWorking, "Done");
-            if (!Directory.Exists(pathDone)) Directory.CreateDirectory(pathDone);
             // Ciclo su ciascun file
             foreach (string file in fileCasuali) {
                 string name = System.IO.Path.GetFileName(file);
@@ -117,19 +115,19 @@ namespace A69WD.Service {
                         Logger.Warning($"Import - errore nel salvataggio: {name}");
                         Interlocked.Increment(ref _errori);
                         // - Sposto il file
-                        System.IO.File.Move(file, System.IO.Path.Combine(pathErrOnSave, System.IO.Path.GetFileName(file)));
+                        System.IO.File.Move(file, System.IO.Path.Combine(Cfg.Settings.PathSrcErrOnSave, System.IO.Path.GetFileName(file)));
                     } else {
                         // salvataggio ok
                         Logger.Warning($"Import - racconto salvato: {name}");
                         Interlocked.Increment(ref _completati);
                         // - Sposto il file
-                        System.IO.File.Move(file, System.IO.Path.Combine(pathDone, System.IO.Path.GetFileName(file)));
+                        System.IO.File.Move(file, System.IO.Path.Combine(Cfg.Settings.PathSrcDone, System.IO.Path.GetFileName(file)));
                     }
                 } else {
                     Logger.Warning($"Import - estrazione errata:   {name}");
                     Interlocked.Increment(ref _errori);
                     // - Sposto il file
-                    System.IO.File.Move(file, System.IO.Path.Combine(pathErrOnExtract, System.IO.Path.GetFileName(file)));
+                    System.IO.File.Move(file, System.IO.Path.Combine(Cfg.Settings.PathSrcErrOnExtract, System.IO.Path.GetFileName(file)));
                 }
                 UpdateCounters();
             }

Src/A69WD/Site/cSite_A69.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.IO;
 using System.Linq;
 using System.Net;
 using System.Net.Http;
@@ -8,6 +9,7 @@ using System.Threading.Tasks;
 using A69WD.Data;
 using A69WD.Net;
 using HtmlAgilityPack;
+using static System.Windows.Forms.VisualStyles.VisualStyleElement.TrackBar;
 
 namespace A69WD.Site {
 
@@ -107,9 +109,11 @@ namespace A69WD.Site {
         /// <returns></returns>
         public async Task<List<iLink>> CrawlPageAsync(iLink lk) {
             var newlinks = new List<iLink>();
+
             try {
                 Console.WriteLine($"A69 - Scarico: {lk.URL}");
                 HtmlDocument tdoc = null;
+                string thtml = "";
 
                 #region ToConfirm
                 if (lk.Status == eLinkState.ToConfirm) {
@@ -135,13 +139,11 @@ namespace A69WD.Site {
                         return newlinks;
                     }
                     // Scarico la pagina
-                    var thtml = await client.GetStringAsync(lk.URL);
+                    thtml = await client.GetStringAsync(lk.URL);
                     tdoc = new HtmlDocument();
                     tdoc.LoadHtml(thtml);
                     // Salvo il file downloadato
-                    //System.IO.StreamWriter tsw = new System.IO.StreamWriter(lk.DstFilePath, false, Encoding.UTF8);
+                    //SaveToFile(thtml, lk.FilePath, lk.FileName);
-                    System.IO.StreamWriter tsw = new System.IO.StreamWriter(System.IO.Path.Combine(lk.FilePath, lk.FileName), false, Encoding.UTF8);
-                    tsw.Write(thtml);
 
                     lk.Status = eLinkState.ToAnalize;
                 }
@@ -151,6 +153,7 @@ namespace A69WD.Site {
                 {
                     if (lk.Status == eLinkState.ToAnalize) {
                         List<string> tmpLk = new List<string>();
+
                         #region Estraggo gli url ad indici:
                         {
                             /*
@@ -169,6 +172,7 @@ namespace A69WD.Site {
                              * </div>
                              * 
                              */
+                            try {
                                 // Estraggo i link alle altre pagine:
                                 var tnodes = tdoc.DocumentNode.SelectNodes("//div[@id='pagination']//ul//li//a[@href]");
                                 if (tnodes != null) {
@@ -183,16 +187,18 @@ namespace A69WD.Site {
                                     .OrderBy(s => s)    // ordina alfabeticamente
                                     .Distinct()         // elimina duplicati
                                     .ToList();
-                            // Inserisco i link trovati
+        // MRK                        // Inserisco i link trovati
-                            try {
+        // MRK                        foreach (string ilk in tmpLk) {
-                                foreach (string ilk in tmpLk) {
+        // MRK                            string[] separator = new string[] { "&pagina=" };
-                                    string[] separator = new string[] { "&pagina=" };
+        // MRK                            string[] txt = ilk.Split(separator, StringSplitOptions.RemoveEmptyEntries);
-                                    string[] txt = ilk.Split(separator, StringSplitOptions.RemoveEmptyEntries);
+        // MRK                            string url = cLink_A69.PageBase + ilk;
-                                    string url = cLink_A69.PageBase + ilk;
+        // MRK                            string name = txt[0].Replace("/racconti-erotici/racconti_lista.php?genere=", "_A69_") + "_" + int.Parse(txt[1]).ToString("0000") + ".html";
-                                    string name = txt[0].Replace("/racconti-erotici/racconti_lista.php?genere=", "_A69_") + "_" + int.Parse(txt[1]).ToString("0000") + ".html";
+        // MRK                            newlinks.Add(new cLink_A69(url, name));
-                                    newlinks.Add(new cLink_A69(url, name));
+        // MRK                        }
+                            } catch (Exception ex) {
+                                Logger.Error("Errore analisi Idx link dal " + lk.FileName + "  -  " + ex.Message);
+                                SaveToFile(thtml, Cfg.Settings.PathSrcErrOnExtract, lk.FileName);
                             }
-                            } catch (Exception ex) { Logger.Error("Errore analisi Idx link dal " + lk.FileName + "  -  " + ex.Message); }
                         }
                         #endregion
 
@@ -205,6 +211,7 @@ namespace A69WD.Site {
                              * 	<a href="/racconti-erotici/altro/Tutto-e-iniziato-per-gioco-pazza-famiglia_166770.html">Continua a leggere » </a>
                              * </div>
                              */
+                            try {
                                 // Estraggo i link ai racconti:
                                 tmpLk = new List<string>();
                                 // Estraggo i link alle altre pagine:
@@ -221,14 +228,16 @@ namespace A69WD.Site {
                                     .OrderBy(s => s)    // ordina alfabeticamente
                                     .Distinct()         // elimina duplicati
                                     .ToList();
-                            // Inserisco i link trovati
+        // MRK                        // Inserisco i link trovati
-                            try {
+        // MRK                        foreach (string ilk in tmpLk) {
-                                foreach (string ilk in tmpLk) {
+        // MRK                            string url = cLink_A69.PageBase + ilk;
-                                    string url = cLink_A69.PageBase + ilk;
+        // MRK                            string name = ilk.Replace("/racconti-erotici/", "A69_").Replace(@"/", "_");
-                                    string name = ilk.Replace("/racconti-erotici/", "A69_").Replace(@"/","_");
+        // MRK                            newlinks.Add(new cLink_A69(url, name));
-                                    newlinks.Add(new cLink_A69(url, name));
+        // MRK                        }
+                            } catch (Exception ex) {
+                                Logger.Error("Errore analisi Idx link dal " + lk.FileName + "  -  " + ex.Message);
+                                SaveToFile(thtml, Cfg.Settings.PathSrcErrOnExtract, lk.FileName);
                             }
-                            } catch (Exception ex) { Logger.Error("Errore analisi Idx link dal " + lk.FileName + "  -  " + ex.Message); }
                         }
                         #endregion
 
@@ -238,8 +247,21 @@ namespace A69WD.Site {
                             cRac rac = lk.EstraiRac(tdoc);
                             if (rac != null) {
                                 // Racconto estratto correttamente, lo salvo sul DB
-                                App.DB.ImportRac(rac);
+                                int res = App.DB.ImportRac(rac);
+                                if (res>0) {
+                                    // Racconto salvato correttamente sul DB
+                                    SaveToFile(thtml, Cfg.Settings.PathSrcDone, lk.FileName);
+                                } else {
+                                    // Errore nel salvataggio sul DB
+                                    SaveToFile(thtml, Cfg.Settings.PathSrcErrOnSave, lk.FileName);
                                 }
+                            } else {
+                                // Estrazione del racconto fallita
+                                SaveToFile(thtml, Cfg.Settings.PathSrcErrOnExtract, lk.FileName);
+                            }
+                        } else {
+                            // Link ad un indice
+                            SaveToFile(thtml, Cfg.Settings.PathIdxDone, lk.FileName);
                         }
                         #endregion
 
@@ -274,10 +296,14 @@ namespace A69WD.Site {
             } catch (Exception ex) {
                 Console.WriteLine($"Errore con {lk.URL}: {ex.Message}");
                 lk.Status = eLinkState.Error;
-                //App.DB.LinkLog(lk);
+                App.DB.LinkLog(lk);
             }
             return newlinks;
         }
+        private void SaveToFile(string txtHTML, string filePath, string fileName) {
+            System.IO.StreamWriter tsw = new System.IO.StreamWriter(System.IO.Path.Combine(filePath, fileName), false, Encoding.UTF8);
+            tsw.Write(txtHTML);
+        }
 
         public async Task<List<iLink>> CrawlPageAsyncTest1(iLink lk) {
             var newlinks = new List<iLink>();

Src/A69WD/UI/frmReader.Designer.cs

@@ -31,21 +31,22 @@
             this.btnTest = new System.Windows.Forms.Button();
             this.btnTest2 = new System.Windows.Forms.Button();
             this.btnTest3 = new System.Windows.Forms.Button();
+            this.pnlTop = new System.Windows.Forms.Panel();
+            this.btn = new System.Windows.Forms.Button();
             this.sb.SuspendLayout();
+            this.pnlTop.SuspendLayout();
             this.SuspendLayout();
             // 
             // rtb
             // 
-            this.rtb.Anchor = ((System.Windows.Forms.AnchorStyles)((((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Bottom) 
-            | System.Windows.Forms.AnchorStyles.Left) 
-            | System.Windows.Forms.AnchorStyles.Right)));
             this.rtb.BackColor = System.Drawing.Color.White;
             this.rtb.BorderStyle = System.Windows.Forms.BorderStyle.None;
+            this.rtb.Dock = System.Windows.Forms.DockStyle.Fill;
             this.rtb.Font = new System.Drawing.Font("Verdana", 11.25F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
-            this.rtb.Location = new System.Drawing.Point(0, 54);
+            this.rtb.Location = new System.Drawing.Point(0, 42);
             this.rtb.Name = "rtb";
-            this.rtb.Size = new System.Drawing.Size(700, 366);
+            this.rtb.Size = new System.Drawing.Size(700, 378);
-            this.rtb.TabIndex = 3;
+            this.rtb.TabIndex = 2;
             this.rtb.Text = "";
             this.rtb.KeyDown += new System.Windows.Forms.KeyEventHandler(this.KeyDownAnalisi);
             // 
@@ -59,7 +60,7 @@
             this.sb.Location = new System.Drawing.Point(0, 420);
             this.sb.Name = "sb";
             this.sb.Size = new System.Drawing.Size(700, 24);
-            this.sb.TabIndex = 2;
+            this.sb.TabIndex = 1;
             this.sb.Text = "statusStrip1";
             // 
             // sbMessage
@@ -97,9 +98,9 @@
             // 
             // btnTest
             // 
-            this.btnTest.Location = new System.Drawing.Point(4, 4);
+            this.btnTest.Location = new System.Drawing.Point(11, 5);
             this.btnTest.Name = "btnTest";
-            this.btnTest.Size = new System.Drawing.Size(62, 48);
+            this.btnTest.Size = new System.Drawing.Size(62, 32);
             this.btnTest.TabIndex = 4;
             this.btnTest.Text = "Test";
             this.btnTest.UseVisualStyleBackColor = true;
@@ -107,9 +108,9 @@
             // 
             // btnTest2
             // 
-            this.btnTest2.Location = new System.Drawing.Point(72, 4);
+            this.btnTest2.Location = new System.Drawing.Point(79, 5);
             this.btnTest2.Name = "btnTest2";
-            this.btnTest2.Size = new System.Drawing.Size(62, 48);
+            this.btnTest2.Size = new System.Drawing.Size(62, 32);
             this.btnTest2.TabIndex = 5;
             this.btnTest2.Text = "Test 2";
             this.btnTest2.UseVisualStyleBackColor = true;
@@ -117,24 +118,44 @@
             // 
             // btnTest3
             // 
-            this.btnTest3.Location = new System.Drawing.Point(140, 4);
+            this.btnTest3.Location = new System.Drawing.Point(147, 5);
             this.btnTest3.Name = "btnTest3";
-            this.btnTest3.Size = new System.Drawing.Size(62, 48);
+            this.btnTest3.Size = new System.Drawing.Size(62, 32);
             this.btnTest3.TabIndex = 6;
             this.btnTest3.Text = "Test 3";
             this.btnTest3.UseVisualStyleBackColor = true;
             this.btnTest3.Click += new System.EventHandler(this.btnTest3_Click);
             // 
+            // pnlTop
+            // 
+            this.pnlTop.Controls.Add(this.btn);
+            this.pnlTop.Controls.Add(this.btnTest3);
+            this.pnlTop.Controls.Add(this.btnTest);
+            this.pnlTop.Controls.Add(this.btnTest2);
+            this.pnlTop.Dock = System.Windows.Forms.DockStyle.Top;
+            this.pnlTop.Location = new System.Drawing.Point(0, 0);
+            this.pnlTop.Name = "pnlTop";
+            this.pnlTop.Size = new System.Drawing.Size(700, 42);
+            this.pnlTop.TabIndex = 0;
+            // 
+            // btn
+            // 
+            this.btn.Location = new System.Drawing.Point(215, 5);
+            this.btn.Name = "btn";
+            this.btn.Size = new System.Drawing.Size(62, 32);
+            this.btn.TabIndex = 7;
+            this.btn.Text = "Test 4";
+            this.btn.UseVisualStyleBackColor = true;
+            this.btn.Click += new System.EventHandler(this.btn_Click);
+            // 
             // frmReader
             // 
             this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F);
             this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font;
             this.ClientSize = new System.Drawing.Size(700, 444);
             this.ControlBox = false;
-            this.Controls.Add(this.btnTest3);
-            this.Controls.Add(this.btnTest2);
-            this.Controls.Add(this.btnTest);
             this.Controls.Add(this.rtb);
+            this.Controls.Add(this.pnlTop);
             this.Controls.Add(this.sb);
             this.Name = "frmReader";
             this.Text = "PlugIn_App_Reader.UI Reader                                       (DB vrs1.3, App" +
@@ -142,6 +163,7 @@
             this.FormClosing += new System.Windows.Forms.FormClosingEventHandler(this.frmReader_FormClosing);
             this.sb.ResumeLayout(false);
             this.sb.PerformLayout();
+            this.pnlTop.ResumeLayout(false);
             this.ResumeLayout(false);
 
         }
@@ -156,5 +178,7 @@
         private System.Windows.Forms.Button btnTest;
         private System.Windows.Forms.Button btnTest2;
         private System.Windows.Forms.Button btnTest3;
+        private System.Windows.Forms.Panel pnlTop;
+        private System.Windows.Forms.Button btn;
     }
 }

Src/A69WD/UI/frmReader.cs

@@ -850,6 +850,35 @@ namespace A69WD.UI {
 
         }
 
+        private async void btn_Click(object sender, EventArgs e) {
+
+            // - Creo istanza del client per il sito con autenticazione:
+            sbStatusLabel.Text = "Login on A69";
+            siteA69 = new cSite_A69();
+
+            // - Eseguo l'autenticazione
+            bool res = await siteA69.Authenticate();
+            if (res) {
+                sbStatusLabel.Text = "A69 autenticated";
+                Logger.Information("Sito A69 loggato: " + res.ToString());
+            } else {
+                Logger.Information("Login Fault");
+                return;
+            }
+
+            // - Verifico path per il download
+            if (!System.IO.Directory.Exists(Cfg.Settings.PathDownloadWorking)) System.IO.Directory.CreateDirectory(Cfg.Settings.PathDownloadWorking);
+
+            // Preparo il servizio
+            string url = "https://www.annunci69.it/racconti-erotici/racconti_lista.php?genere=altro&pagina=1";
+            string name = "_A69_altro_0001.html";
+            serviceCrawler = new Service_Crawler(sb);
+            // Inserisco i link iniziali nella coda
+            serviceCrawler.EnqueueUrl(new cLink_A69(url, name));
+            // avvio il servizio
+            serviceCrawler.StartWorker(siteA69);
+        
+        }
 
 
     }