- Autocommit - 2022/09/22 07:48:34

Src/A69WD/Net/cA69Engine2.cs

@@ -1,15 +1,17 @@
 using System;
+using System.IO;
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
-
+using System.Windows.Forms;
-using A69WD.Net;
+using System.Security.Policy;
-
+using System.Net;
 
 namespace A69WD {
 
     public class cA69Engine2 {
 
+
         #region Property:  State,  NumDownload,  Tipologie,  BASE_URL, BASE_LINK_IDX, BASE_LINK_RAC
 
         public EngineStatusEnum State { get { return state; } }
@@ -25,7 +27,7 @@ namespace A69WD {
         #endregion
 
 
-        #region Fields:  allLinks,  toDownload,  downloaded,  racLinks,  dtStartTime,  downRac,  state,  wb
+        #region Fields:  allLinks,  toDownload,  downloaded,  racLinks,  dtStartTime,  downRac,  state
 
         List<string> allLinks = null;
         List<string> toDownload = null;
@@ -38,6 +40,7 @@ namespace A69WD {
 
         EngineStatusEnum state = EngineStatusEnum.NotReady;
 
+
         #endregion
 
 
@@ -52,8 +55,6 @@ namespace A69WD {
                 racLinks = new List<string>();
                 int res = App.DB.LoadData(allLinks, toDownload, downloaded, racLinks);
                 RaiseEvent_Updating("Ricaricati " + res.ToString("000") + " link di racconti da scaricare");
-                //wb = WB;
-                //wb.DocumentCompleted += Wb_DocumentCompleted;
                 // Per ora non serializzo gli archivi e quindi carico le pagine iniziali delle 11 tipologie:
                 foreach (string tipologia in Tipologie) {
                     toDownload.Add(BASE_URL.Replace("{Tip}", tipologia));
@@ -69,8 +70,99 @@ namespace A69WD {
         #endregion
 
 
+        #region Method: 
+
+        public bool Start(bool DownRac) {
+            if (state != EngineStatusEnum.StandBy) return false;
+
+            dtStartTime = DateTime.Now;
+            downRac = DownRac;
+
+            RaiseEvent_Updating("Inizio lo scrap dei link dei racconti");
+
+            state = EngineStatusEnum.Starting;
+
+            if (toDownload.Count > 0) {
+                // proseguo con il download:
+                string lk = toDownload.First();
+                toDownload.Remove(lk);
+                using (WebClient client = new WebClient()) {
+                    client.Encoding = Encoding.ASCII;
+                    client.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0");
+                    client.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
+                    client.Headers.Add("Cookie", "euconsent-v2=CPG74goPG74goAKAeAENBcCsAP_AAH_AAAwIH3tf_X__bX9j-_59__t0eY1f9_7_v-wzjhfdt-8N2P_X_L0X42E7PF36pq4KuR4Eu3LBIQNlHOHUTUmw6okVrTPsak2Mr7NKJ7LEmnMZe2dYGHtfn91TuZKY7_78_9fz3_-v_v___9f3r-3_3__59X---_e_V399zLv9__3__9gfiASYal8AF2JY4Mk0aVQogQhWEh0AoAKKAYWiawgYXBTsrAI9QQMAEJqAjAiBBiCjFgEAAgEASERASAHggEQBEAgABACpAQgAImAQWAFgYBAAKAaFiBFAEIEhBkcFRymBARItFBPZWAJRd7GmEIZRYAUCj-iowEShBAsDISFg5gAA.YAAAAAAAAAAA");
+                    client.Headers.Add("DNT", "1");
+                    client.Headers.Add("TE", "Trailers");
+                    client.DownloadFile(new Uri(lk), fileName);
+                }
+            }
+
+            return true;
+        }
+
+        public bool Stop() {
+            switch (state) {
+                case EngineStatusEnum.NotReady:
+                case EngineStatusEnum.StandBy:
+                    SaveState();
+                    state = EngineStatusEnum.Stopped;
+                    return true;
+                case EngineStatusEnum.Starting:
+                case EngineStatusEnum.Started:
+                case EngineStatusEnum.Stopping:
+                    state = EngineStatusEnum.Stopping;
+                    return false;
+                case EngineStatusEnum.Stopped:
+                    return true;
+            }
+            return true;
+        }
+
+        public int ImportLinkFromFile(string SrcFilePath) {
+            int res = 0;
+            try {
+                if (File.Exists(SrcFilePath)) {
+                    StreamReader sr = new StreamReader(SrcFilePath, Encoding.UTF8);
+                    string row = sr.ReadLine().Trim();
+                    while (row != null) {
+                        // verifico che non sia un commento:
+                        if (!(row.StartsWith("#") || row.StartsWith("//"))) {
+                            // Se link nuovo:
+                            if (!allLinks.Contains(row)) {
+                                // Link nuovo, mi comporto di conseguenza alla tipologia:
+                                if (IsRacLink(row)) {
+                                    // Link ad un racconto
+                                    allLinks.Add(row);
+                                    toDownload.Add(row);
+                                    racLinks.Add(row);
+                                    res++;
+                                } else {
+                                    if (IsIdxLink(row)) {
+                                        // Link ad un indice
+                                        allLinks.Add(row);
+                                        toDownload.Add(row);
+                                        res++;
+                                    } else {
+                                        // Link non riconosciuto, lo scarto
+                                        ;
+                                    }
+                                }
+                            }
+                        }
+                        //Read the next line
+                        row = sr.ReadLine();
+                    }
+                }
+
+            } catch (Exception ex) {
+                Logger.Log(new object[] { "Error Code: DB07403", "Errore in ImportLinkFromFile()", ex });
+                if (res > 0) res = -1 * res;
+            }
+            return res;
+        }
 
 
+        #endregion
 
 
         #region Eventi:
@@ -105,5 +197,143 @@ namespace A69WD {
         #endregion
 
 
-    }
+        int conteggioEvt = 0;
-}
+
+
+        private void Wb_DocumentCompleted(object sender, WebBrowserDocumentCompletedEventArgs e) {
+            object obj = new Object();
+            lock (obj) {
+
+                conteggioEvt++;
+
+                StringBuilder sb;
+                if (state == EngineStatusEnum.Starting) state = EngineStatusEnum.Started;
+
+                WebBrowser wb = sender as WebBrowser;
+                string link = wb.Url.ToString();
+
+                // Se link di logon, devo terminare i download
+                if (link == "https://www.annunci69.it/accedi.php?nop") {
+                    // Finito download:
+                    state = EngineStatusEnum.Stopping;
+                    // salvo gli archivi:
+                    SaveState();
+                    return;
+                }
+
+                // Elimino ulteriori ripetizioni dell'evento:
+                if (downloaded.Contains(link)) return;
+
+                // aggiorno UI
+                RaiseEvent_Updating(wb.Name + " - (Evt" + conteggioEvt.ToString("0000") + ") Donwloaded:  " + link.Replace("https://www.annunci69.it/racconti-erotici/", "./"));
+
+                // Se pagina è un racconto, lo salvo
+                if (IsRacLink(link)) {
+                    sb = new StringBuilder();
+                    sb.Append(wb.Document.Body.InnerHtml);
+                    string fn = Path.Combine(Cfg.Settings.PathAnalizadedRaccFile, GetFileName(link));
+                    //cTools.WriteTextFile(fn, sb, Encoding.GetEncoding(wb.Document.Encoding));
+                    cTools.WriteTextFile(fn, sb);
+                }
+
+                // Registro che ho eseguito il download del link
+                downloaded.Add(link);
+
+                // Ciclo su tutti i link estratti dalla pagina:
+                foreach (HtmlElement item in wb.Document.Links) {
+                    link = item.GetAttribute("href");
+                    // Se link nuovo:
+                    if (!allLinks.Contains(link)) {
+                        // elimino il link di logon
+                        if (link == "https://www.annunci69.it/accedi.php?nop") break;
+                        // Link nuovo, mi comporto di conseguenza alla tipologia:
+                        if (IsRacLink(link)) {
+                            // Link ad un racconto
+                            allLinks.Add(link);
+                            if (downRac) { toDownload.Add(link); }
+                            racLinks.Add(link);
+                        } else {
+                            if (IsIdxLink(link)) {
+                                // Link ad un indice
+                                allLinks.Add(link);
+                                toDownload.Add(link);
+                            } else {
+                                // Link non riconosciuto, lo scarto
+                                ;
+                            }
+                        }
+                    }
+                } // fine ciclo sui link estratti
+
+                // Se ho raggiunto il massimo numero di link, o non vi sono link da downloadare, mi fermo:
+                if ((toDownload.Count > 0) && (state == EngineStatusEnum.Started)) {
+                    // proseguo con il download:
+                    string lk = toDownload.First();
+                    toDownload.Remove(lk);
+                    wb.Navigate(lk);
+                } else {
+                    // Finito download:
+                    state = EngineStatusEnum.Stopping;
+                    // salvo gli archivi:
+                    SaveState();
+                }
+
+            }//lock
+        }
+
+
+        private void SaveState() {
+            int res = App.DB.SaveData(allLinks, toDownload, downloaded, racLinks);
+
+            StringBuilder sb;
+
+            // Finito download, salvo gli archivi:
+            string dt = DateTime.Now.ToString("yyyy-MM-dd_HH-mm-ss");
+
+            //   Salvo archivio: AllLinks
+            sb = new StringBuilder();
+            allLinks.Sort();
+            foreach (string lnk in allLinks) { sb.AppendLine(lnk); }
+            cTools.WriteTextFile(Path.Combine(Cfg.Settings.PathAnalizadedIndexFile, string.Format("_A69_AllLinks_{0}.tes", dt)), sb);
+
+            //   Salvo archivio: toDownload
+            sb = new StringBuilder();
+            toDownload.Sort();
+            foreach (string lnk in toDownload) { sb.AppendLine(lnk); }
+            cTools.WriteTextFile(Path.Combine(Cfg.Settings.PathAnalizadedIndexFile, string.Format("_A69_toDownload_{0}.tes", dt)), sb);
+
+            //   Salvo archivio: Downloaded
+            sb = new StringBuilder();
+            downloaded.Sort();
+            foreach (string lnk in downloaded) { sb.AppendLine(lnk); }
+            cTools.WriteTextFile(Path.Combine(Cfg.Settings.PathAnalizadedIndexFile, string.Format("_A69_Downloaded_{0}.tes", dt)), sb);
+
+            //   Salvo archivio: RacLinks
+            sb = new StringBuilder();
+            racLinks.Sort();
+            foreach (string lnk in racLinks) { sb.AppendLine(lnk); }
+            cTools.WriteTextFile(Path.Combine(Cfg.Settings.PathAnalizadedIndexFile, string.Format("_A69_RacLinks_{0}.tes", dt)), sb);
+
+            // avviso utente:
+            RaiseEvent_Updating(string.Format("Finito il download in {0}", DateTime.Now - dtStartTime));
+
+            state = EngineStatusEnum.Stopped;
+
+        }
+
+
+        private string GetFileName(string url) { return url.Replace("https://www.annunci69.it/racconti-erotici/", "A69_").Replace("/", "_").Replace(":", "_").Replace(@"\", "_"); }
+        static private bool IsIdxLink(string link) { return link.StartsWith(BASE_LINK_IDX); }
+        static private bool IsRacLink(string link) { foreach (string tip in Tipologie) { if (link.StartsWith(BASE_LINK_RAC + tip + "/")) return true; } return false; }
+
+        static public string LinkType(string link) {
+            if (IsIdxLink(link)) return "Idx";
+            if (IsRacLink(link)) return "Rac";
+            //return "Unknown";
+            return "Unk";
+        }
+
+    }//class cA69Engine2
+
+
+}//namespace A69WD

Src/A69WD/Net/frmDownloader.Designer.cs

@@ -33,6 +33,7 @@
             this.btnShowLog = new System.Windows.Forms.Button();
             this.btnDownloadAll2 = new System.Windows.Forms.Button();
             this.btnTest = new System.Windows.Forms.Button();
+            this.button1 = new System.Windows.Forms.Button();
             this.SuspendLayout();
             // 
             // btnClose
@@ -129,15 +130,26 @@
             this.btnTest.Name = "btnTest";
             this.btnTest.Size = new System.Drawing.Size(97, 33);
             this.btnTest.TabIndex = 19;
-            this.btnTest.Text = "Test";
+            this.btnTest.Text = "Test WebClient";
             this.btnTest.UseVisualStyleBackColor = true;
             this.btnTest.Click += new System.EventHandler(this.btnTest_Click);
             // 
+            // button1
+            // 
+            this.button1.Location = new System.Drawing.Point(290, 1);
+            this.button1.Name = "button1";
+            this.button1.Size = new System.Drawing.Size(97, 33);
+            this.button1.TabIndex = 20;
+            this.button1.Text = "Test Engine2";
+            this.button1.UseVisualStyleBackColor = true;
+            this.button1.Click += new System.EventHandler(this.button1_Click);
+            // 
             // frmDownloader
             // 
             this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F);
             this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font;
             this.ClientSize = new System.Drawing.Size(524, 74);
+            this.Controls.Add(this.button1);
             this.Controls.Add(this.btnTest);
             this.Controls.Add(this.btnDownloadAll2);
             this.Controls.Add(this.btnShowLog);
@@ -165,5 +177,6 @@
         private System.Windows.Forms.Button btnShowLog;
         private System.Windows.Forms.Button btnDownloadAll2;
         private System.Windows.Forms.Button btnTest;
+        private System.Windows.Forms.Button button1;
     }
 }

Src/A69WD/Net/frmDownloader.cs

@@ -94,7 +94,7 @@ namespace A69WD {
 
 
 
-        #region Test download 
+        #region Test WEBCLIENT
 
         private void btnTest_Click(object sender, EventArgs e) {
             ////Run download file with multiple thread
@@ -110,8 +110,8 @@ namespace A69WD {
             //    thread.Start();
             //}
 
-            string url = "https://www.annunci69.it/racconti-erotici/racconti_lista.php?genere=altro&pagina=3";
+            string url = "https://www.annunci69.it/racconti-erotici/racconti_lista.php?genere=altro&pagina=98";
-            string fileName = @"C:\VMS\Prj\C#\A69WD\Test\Logs\_A69_genere_altro_3.html";
+            string fileName = @"C:\VMS\Prj\C#\A69WD\Test\Logs\_A69_genere_altro_98.html";
 
             //// Test 1: Questo dà errore:
             //WebClient client;
@@ -123,20 +123,35 @@ namespace A69WD {
             //cTools.WriteTextFile(fileName, sb);
             //// Test 1
 
-            // Test
+            //   // Test 2: OK
-            WebClient client = new WebClient();
+            //   WebClient client = new WebClient();
-            //client.Headers["User-Agent"] = "User-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36";
+            //   //client.Headers["User-Agent"] = "User-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36";
-            client.Encoding = Encoding.ASCII;
+            //   client.Encoding = Encoding.ASCII;
-            client.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0");
+            //   client.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0");
-            client.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
+            //   client.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
-            //client.Headers.Add("Accept-Encoding", "gzip, deflate, br");
+            //   //client.Headers.Add("Accept-Encoding", "gzip, deflate, br");
-            //client.Headers.Add("Connection", "keep-alive");
+            //   //client.Headers.Add("Connection", "keep-alive");
-            client.Headers.Add("Cookie", "euconsent-v2=CPG74goPG74goAKAeAENBcCsAP_AAH_AAAwIH3tf_X__bX9j-_59__t0eY1f9_7_v-wzjhfdt-8N2P_X_L0X42E7PF36pq4KuR4Eu3LBIQNlHOHUTUmw6okVrTPsak2Mr7NKJ7LEmnMZe2dYGHtfn91TuZKY7_78_9fz3_-v_v___9f3r-3_3__59X---_e_V399zLv9__3__9gfiASYal8AF2JY4Mk0aVQogQhWEh0AoAKKAYWiawgYXBTsrAI9QQMAEJqAjAiBBiCjFgEAAgEASERASAHggEQBEAgABACpAQgAImAQWAFgYBAAKAaFiBFAEIEhBkcFRymBARItFBPZWAJRd7GmEIZRYAUCj-iowEShBAsDISFg5gAA.YAAAAAAAAAAA");
+            //   client.Headers.Add("Cookie", "euconsent-v2=CPG74goPG74goAKAeAENBcCsAP_AAH_AAAwIH3tf_X__bX9j-_59__t0eY1f9_7_v-wzjhfdt-8N2P_X_L0X42E7PF36pq4KuR4Eu3LBIQNlHOHUTUmw6okVrTPsak2Mr7NKJ7LEmnMZe2dYGHtfn91TuZKY7_78_9fz3_-v_v___9f3r-3_3__59X---_e_V399zLv9__3__9gfiASYal8AF2JY4Mk0aVQogQhWEh0AoAKKAYWiawgYXBTsrAI9QQMAEJqAjAiBBiCjFgEAAgEASERASAHggEQBEAgABACpAQgAImAQWAFgYBAAKAaFiBFAEIEhBkcFRymBARItFBPZWAJRd7GmEIZRYAUCj-iowEShBAsDISFg5gAA.YAAAAAAAAAAA");
-            client.Headers.Add("DNT", "1");
+            //   client.Headers.Add("DNT", "1");
-            client.Headers.Add("TE", "Trailers");
+            //   client.Headers.Add("TE", "Trailers");
-            client.DownloadFile(new Uri(url), fileName);
+            //   client.DownloadFile(new Uri(url), fileName);
-
+            //   // Test 2: OK
 
+            // Test 3: OK
+            using (WebClient client = new WebClient()) {
+                for (int i = 90; i < 110; i++) {
+                    url = "https://www.annunci69.it/racconti-erotici/racconti_lista.php?genere=altro&pagina=" + i.ToString();
+                    fileName = @"C:\VMS\Prj\C#\A69WD\Test\Logs\_A69_genere_altro_" + i.ToString() + ".html";
+                    client.Encoding = Encoding.ASCII;
+                    client.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0");
+                    client.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
+                    client.Headers.Add("Cookie", "euconsent-v2=CPG74goPG74goAKAeAENBcCsAP_AAH_AAAwIH3tf_X__bX9j-_59__t0eY1f9_7_v-wzjhfdt-8N2P_X_L0X42E7PF36pq4KuR4Eu3LBIQNlHOHUTUmw6okVrTPsak2Mr7NKJ7LEmnMZe2dYGHtfn91TuZKY7_78_9fz3_-v_v___9f3r-3_3__59X---_e_V399zLv9__3__9gfiASYal8AF2JY4Mk0aVQogQhWEh0AoAKKAYWiawgYXBTsrAI9QQMAEJqAjAiBBiCjFgEAAgEASERASAHggEQBEAgABACpAQgAImAQWAFgYBAAKAaFiBFAEIEhBkcFRymBARItFBPZWAJRd7GmEIZRYAUCj-iowEShBAsDISFg5gAA.YAAAAAAAAAAA");
+                    client.Headers.Add("DNT", "1");
+                    client.Headers.Add("TE", "Trailers");
+                    client.DownloadFile(new Uri(url), fileName);
+                }
+            }
+            // Test 3: OK
 
 
         }
@@ -157,6 +172,18 @@ namespace A69WD {
             //}));
         }
 
+
+
+        #endregion
+
+
+        #region Test ENGINE2
+
+        private void button1_Click(object sender, EventArgs e) {
+
+        }
+
+
         #endregion