From 43186b9875e69e7e9a545cf95da4b0c4f3518ffd Mon Sep 17 00:00:00 2001 From: CSiCoAlm Date: Thu, 22 Sep 2022 07:48:34 +0200 Subject: [PATCH] - Autocommit - 2022/09/22 07:48:34 --- Src/A69WD/Net/cA69Engine2.cs | 246 +++++++++++++++++++++++- Src/A69WD/Net/frmDownloader.Designer.cs | 15 +- Src/A69WD/Net/frmDownloader.cs | 59 ++++-- 3 files changed, 295 insertions(+), 25 deletions(-) diff --git a/Src/A69WD/Net/cA69Engine2.cs b/Src/A69WD/Net/cA69Engine2.cs index 64d1abe..ecab307 100644 --- a/Src/A69WD/Net/cA69Engine2.cs +++ b/Src/A69WD/Net/cA69Engine2.cs @@ -1,15 +1,17 @@ using System; +using System.IO; using System.Collections.Generic; using System.Linq; using System.Text; - -using A69WD.Net; - +using System.Windows.Forms; +using System.Security.Policy; +using System.Net; namespace A69WD { public class cA69Engine2 { + #region Property: State, NumDownload, Tipologie, BASE_URL, BASE_LINK_IDX, BASE_LINK_RAC public EngineStatusEnum State { get { return state; } } @@ -25,7 +27,7 @@ namespace A69WD { #endregion - #region Fields: allLinks, toDownload, downloaded, racLinks, dtStartTime, downRac, state, wb + #region Fields: allLinks, toDownload, downloaded, racLinks, dtStartTime, downRac, state List allLinks = null; List toDownload = null; @@ -38,6 +40,7 @@ namespace A69WD { EngineStatusEnum state = EngineStatusEnum.NotReady; + #endregion @@ -52,8 +55,6 @@ namespace A69WD { racLinks = new List(); int res = App.DB.LoadData(allLinks, toDownload, downloaded, racLinks); RaiseEvent_Updating("Ricaricati " + res.ToString("000") + " link di racconti da scaricare"); - //wb = WB; - //wb.DocumentCompleted += Wb_DocumentCompleted; // Per ora non serializzo gli archivi e quindi carico le pagine iniziali delle 11 tipologie: foreach (string tipologia in Tipologie) { toDownload.Add(BASE_URL.Replace("{Tip}", tipologia)); @@ -69,8 +70,99 @@ namespace A69WD { #endregion + #region Method: + + public bool Start(bool DownRac) { + if (state != EngineStatusEnum.StandBy) return false; + + dtStartTime = DateTime.Now; + downRac = DownRac; + + RaiseEvent_Updating("Inizio lo scrap dei link dei racconti"); + + state = EngineStatusEnum.Starting; + + if (toDownload.Count > 0) { + // proseguo con il download: + string lk = toDownload.First(); + toDownload.Remove(lk); + using (WebClient client = new WebClient()) { + client.Encoding = Encoding.ASCII; + client.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0"); + client.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"); + client.Headers.Add("Cookie", "euconsent-v2=CPG74goPG74goAKAeAENBcCsAP_AAH_AAAwIH3tf_X__bX9j-_59__t0eY1f9_7_v-wzjhfdt-8N2P_X_L0X42E7PF36pq4KuR4Eu3LBIQNlHOHUTUmw6okVrTPsak2Mr7NKJ7LEmnMZe2dYGHtfn91TuZKY7_78_9fz3_-v_v___9f3r-3_3__59X---_e_V399zLv9__3__9gfiASYal8AF2JY4Mk0aVQogQhWEh0AoAKKAYWiawgYXBTsrAI9QQMAEJqAjAiBBiCjFgEAAgEASERASAHggEQBEAgABACpAQgAImAQWAFgYBAAKAaFiBFAEIEhBkcFRymBARItFBPZWAJRd7GmEIZRYAUCj-iowEShBAsDISFg5gAA.YAAAAAAAAAAA"); + client.Headers.Add("DNT", "1"); + client.Headers.Add("TE", "Trailers"); + client.DownloadFile(new Uri(lk), fileName); + } + } + + return true; + } + + public bool Stop() { + switch (state) { + case EngineStatusEnum.NotReady: + case EngineStatusEnum.StandBy: + SaveState(); + state = EngineStatusEnum.Stopped; + return true; + case EngineStatusEnum.Starting: + case EngineStatusEnum.Started: + case EngineStatusEnum.Stopping: + state = EngineStatusEnum.Stopping; + return false; + case EngineStatusEnum.Stopped: + return true; + } + return true; + } + + public int ImportLinkFromFile(string SrcFilePath) { + int res = 0; + try { + if (File.Exists(SrcFilePath)) { + StreamReader sr = new StreamReader(SrcFilePath, Encoding.UTF8); + string row = sr.ReadLine().Trim(); + while (row != null) { + // verifico che non sia un commento: + if (!(row.StartsWith("#") || row.StartsWith("//"))) { + // Se link nuovo: + if (!allLinks.Contains(row)) { + // Link nuovo, mi comporto di conseguenza alla tipologia: + if (IsRacLink(row)) { + // Link ad un racconto + allLinks.Add(row); + toDownload.Add(row); + racLinks.Add(row); + res++; + } else { + if (IsIdxLink(row)) { + // Link ad un indice + allLinks.Add(row); + toDownload.Add(row); + res++; + } else { + // Link non riconosciuto, lo scarto + ; + } + } + } + } + //Read the next line + row = sr.ReadLine(); + } + } + + } catch (Exception ex) { + Logger.Log(new object[] { "Error Code: DB07403", "Errore in ImportLinkFromFile()", ex }); + if (res > 0) res = -1 * res; + } + return res; + } + #endregion #region Eventi: @@ -105,5 +197,143 @@ namespace A69WD { #endregion - } -} + int conteggioEvt = 0; + + + private void Wb_DocumentCompleted(object sender, WebBrowserDocumentCompletedEventArgs e) { + object obj = new Object(); + lock (obj) { + + conteggioEvt++; + + StringBuilder sb; + if (state == EngineStatusEnum.Starting) state = EngineStatusEnum.Started; + + WebBrowser wb = sender as WebBrowser; + string link = wb.Url.ToString(); + + // Se link di logon, devo terminare i download + if (link == "https://www.annunci69.it/accedi.php?nop") { + // Finito download: + state = EngineStatusEnum.Stopping; + // salvo gli archivi: + SaveState(); + return; + } + + // Elimino ulteriori ripetizioni dell'evento: + if (downloaded.Contains(link)) return; + + // aggiorno UI + RaiseEvent_Updating(wb.Name + " - (Evt" + conteggioEvt.ToString("0000") + ") Donwloaded: " + link.Replace("https://www.annunci69.it/racconti-erotici/", "./")); + + // Se pagina è un racconto, lo salvo + if (IsRacLink(link)) { + sb = new StringBuilder(); + sb.Append(wb.Document.Body.InnerHtml); + string fn = Path.Combine(Cfg.Settings.PathAnalizadedRaccFile, GetFileName(link)); + //cTools.WriteTextFile(fn, sb, Encoding.GetEncoding(wb.Document.Encoding)); + cTools.WriteTextFile(fn, sb); + } + + // Registro che ho eseguito il download del link + downloaded.Add(link); + + // Ciclo su tutti i link estratti dalla pagina: + foreach (HtmlElement item in wb.Document.Links) { + link = item.GetAttribute("href"); + // Se link nuovo: + if (!allLinks.Contains(link)) { + // elimino il link di logon + if (link == "https://www.annunci69.it/accedi.php?nop") break; + // Link nuovo, mi comporto di conseguenza alla tipologia: + if (IsRacLink(link)) { + // Link ad un racconto + allLinks.Add(link); + if (downRac) { toDownload.Add(link); } + racLinks.Add(link); + } else { + if (IsIdxLink(link)) { + // Link ad un indice + allLinks.Add(link); + toDownload.Add(link); + } else { + // Link non riconosciuto, lo scarto + ; + } + } + } + } // fine ciclo sui link estratti + + // Se ho raggiunto il massimo numero di link, o non vi sono link da downloadare, mi fermo: + if ((toDownload.Count > 0) && (state == EngineStatusEnum.Started)) { + // proseguo con il download: + string lk = toDownload.First(); + toDownload.Remove(lk); + wb.Navigate(lk); + } else { + // Finito download: + state = EngineStatusEnum.Stopping; + // salvo gli archivi: + SaveState(); + } + + }//lock + } + + + private void SaveState() { + int res = App.DB.SaveData(allLinks, toDownload, downloaded, racLinks); + + StringBuilder sb; + + // Finito download, salvo gli archivi: + string dt = DateTime.Now.ToString("yyyy-MM-dd_HH-mm-ss"); + + // Salvo archivio: AllLinks + sb = new StringBuilder(); + allLinks.Sort(); + foreach (string lnk in allLinks) { sb.AppendLine(lnk); } + cTools.WriteTextFile(Path.Combine(Cfg.Settings.PathAnalizadedIndexFile, string.Format("_A69_AllLinks_{0}.tes", dt)), sb); + + // Salvo archivio: toDownload + sb = new StringBuilder(); + toDownload.Sort(); + foreach (string lnk in toDownload) { sb.AppendLine(lnk); } + cTools.WriteTextFile(Path.Combine(Cfg.Settings.PathAnalizadedIndexFile, string.Format("_A69_toDownload_{0}.tes", dt)), sb); + + // Salvo archivio: Downloaded + sb = new StringBuilder(); + downloaded.Sort(); + foreach (string lnk in downloaded) { sb.AppendLine(lnk); } + cTools.WriteTextFile(Path.Combine(Cfg.Settings.PathAnalizadedIndexFile, string.Format("_A69_Downloaded_{0}.tes", dt)), sb); + + // Salvo archivio: RacLinks + sb = new StringBuilder(); + racLinks.Sort(); + foreach (string lnk in racLinks) { sb.AppendLine(lnk); } + cTools.WriteTextFile(Path.Combine(Cfg.Settings.PathAnalizadedIndexFile, string.Format("_A69_RacLinks_{0}.tes", dt)), sb); + + // avviso utente: + RaiseEvent_Updating(string.Format("Finito il download in {0}", DateTime.Now - dtStartTime)); + + state = EngineStatusEnum.Stopped; + + } + + + private string GetFileName(string url) { return url.Replace("https://www.annunci69.it/racconti-erotici/", "A69_").Replace("/", "_").Replace(":", "_").Replace(@"\", "_"); } + static private bool IsIdxLink(string link) { return link.StartsWith(BASE_LINK_IDX); } + static private bool IsRacLink(string link) { foreach (string tip in Tipologie) { if (link.StartsWith(BASE_LINK_RAC + tip + "/")) return true; } return false; } + + static public string LinkType(string link) { + if (IsIdxLink(link)) return "Idx"; + if (IsRacLink(link)) return "Rac"; + //return "Unknown"; + return "Unk"; + } + + }//class cA69Engine2 + + +}//namespace A69WD diff --git a/Src/A69WD/Net/frmDownloader.Designer.cs b/Src/A69WD/Net/frmDownloader.Designer.cs index 98611c8..23ecede 100644 --- a/Src/A69WD/Net/frmDownloader.Designer.cs +++ b/Src/A69WD/Net/frmDownloader.Designer.cs @@ -33,6 +33,7 @@ this.btnShowLog = new System.Windows.Forms.Button(); this.btnDownloadAll2 = new System.Windows.Forms.Button(); this.btnTest = new System.Windows.Forms.Button(); + this.button1 = new System.Windows.Forms.Button(); this.SuspendLayout(); // // btnClose @@ -129,15 +130,26 @@ this.btnTest.Name = "btnTest"; this.btnTest.Size = new System.Drawing.Size(97, 33); this.btnTest.TabIndex = 19; - this.btnTest.Text = "Test"; + this.btnTest.Text = "Test WebClient"; this.btnTest.UseVisualStyleBackColor = true; this.btnTest.Click += new System.EventHandler(this.btnTest_Click); // + // button1 + // + this.button1.Location = new System.Drawing.Point(290, 1); + this.button1.Name = "button1"; + this.button1.Size = new System.Drawing.Size(97, 33); + this.button1.TabIndex = 20; + this.button1.Text = "Test Engine2"; + this.button1.UseVisualStyleBackColor = true; + this.button1.Click += new System.EventHandler(this.button1_Click); + // // frmDownloader // this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F); this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; this.ClientSize = new System.Drawing.Size(524, 74); + this.Controls.Add(this.button1); this.Controls.Add(this.btnTest); this.Controls.Add(this.btnDownloadAll2); this.Controls.Add(this.btnShowLog); @@ -165,5 +177,6 @@ private System.Windows.Forms.Button btnShowLog; private System.Windows.Forms.Button btnDownloadAll2; private System.Windows.Forms.Button btnTest; + private System.Windows.Forms.Button button1; } } \ No newline at end of file diff --git a/Src/A69WD/Net/frmDownloader.cs b/Src/A69WD/Net/frmDownloader.cs index c92cc36..17d4a81 100644 --- a/Src/A69WD/Net/frmDownloader.cs +++ b/Src/A69WD/Net/frmDownloader.cs @@ -94,7 +94,7 @@ namespace A69WD { - #region Test download + #region Test WEBCLIENT private void btnTest_Click(object sender, EventArgs e) { ////Run download file with multiple thread @@ -110,8 +110,8 @@ namespace A69WD { // thread.Start(); //} - string url = "https://www.annunci69.it/racconti-erotici/racconti_lista.php?genere=altro&pagina=3"; - string fileName = @"C:\VMS\Prj\C#\A69WD\Test\Logs\_A69_genere_altro_3.html"; + string url = "https://www.annunci69.it/racconti-erotici/racconti_lista.php?genere=altro&pagina=98"; + string fileName = @"C:\VMS\Prj\C#\A69WD\Test\Logs\_A69_genere_altro_98.html"; //// Test 1: Questo dà errore: //WebClient client; @@ -123,20 +123,35 @@ namespace A69WD { //cTools.WriteTextFile(fileName, sb); //// Test 1 - // Test - WebClient client = new WebClient(); - //client.Headers["User-Agent"] = "User-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36"; - client.Encoding = Encoding.ASCII; - client.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0"); - client.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"); - //client.Headers.Add("Accept-Encoding", "gzip, deflate, br"); - //client.Headers.Add("Connection", "keep-alive"); - client.Headers.Add("Cookie", "euconsent-v2=CPG74goPG74goAKAeAENBcCsAP_AAH_AAAwIH3tf_X__bX9j-_59__t0eY1f9_7_v-wzjhfdt-8N2P_X_L0X42E7PF36pq4KuR4Eu3LBIQNlHOHUTUmw6okVrTPsak2Mr7NKJ7LEmnMZe2dYGHtfn91TuZKY7_78_9fz3_-v_v___9f3r-3_3__59X---_e_V399zLv9__3__9gfiASYal8AF2JY4Mk0aVQogQhWEh0AoAKKAYWiawgYXBTsrAI9QQMAEJqAjAiBBiCjFgEAAgEASERASAHggEQBEAgABACpAQgAImAQWAFgYBAAKAaFiBFAEIEhBkcFRymBARItFBPZWAJRd7GmEIZRYAUCj-iowEShBAsDISFg5gAA.YAAAAAAAAAAA"); - client.Headers.Add("DNT", "1"); - client.Headers.Add("TE", "Trailers"); - client.DownloadFile(new Uri(url), fileName); - + // // Test 2: OK + // WebClient client = new WebClient(); + // //client.Headers["User-Agent"] = "User-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36"; + // client.Encoding = Encoding.ASCII; + // client.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0"); + // client.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"); + // //client.Headers.Add("Accept-Encoding", "gzip, deflate, br"); + // //client.Headers.Add("Connection", "keep-alive"); + // client.Headers.Add("Cookie", "euconsent-v2=CPG74goPG74goAKAeAENBcCsAP_AAH_AAAwIH3tf_X__bX9j-_59__t0eY1f9_7_v-wzjhfdt-8N2P_X_L0X42E7PF36pq4KuR4Eu3LBIQNlHOHUTUmw6okVrTPsak2Mr7NKJ7LEmnMZe2dYGHtfn91TuZKY7_78_9fz3_-v_v___9f3r-3_3__59X---_e_V399zLv9__3__9gfiASYal8AF2JY4Mk0aVQogQhWEh0AoAKKAYWiawgYXBTsrAI9QQMAEJqAjAiBBiCjFgEAAgEASERASAHggEQBEAgABACpAQgAImAQWAFgYBAAKAaFiBFAEIEhBkcFRymBARItFBPZWAJRd7GmEIZRYAUCj-iowEShBAsDISFg5gAA.YAAAAAAAAAAA"); + // client.Headers.Add("DNT", "1"); + // client.Headers.Add("TE", "Trailers"); + // client.DownloadFile(new Uri(url), fileName); + // // Test 2: OK + // Test 3: OK + using (WebClient client = new WebClient()) { + for (int i = 90; i < 110; i++) { + url = "https://www.annunci69.it/racconti-erotici/racconti_lista.php?genere=altro&pagina=" + i.ToString(); + fileName = @"C:\VMS\Prj\C#\A69WD\Test\Logs\_A69_genere_altro_" + i.ToString() + ".html"; + client.Encoding = Encoding.ASCII; + client.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0"); + client.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"); + client.Headers.Add("Cookie", "euconsent-v2=CPG74goPG74goAKAeAENBcCsAP_AAH_AAAwIH3tf_X__bX9j-_59__t0eY1f9_7_v-wzjhfdt-8N2P_X_L0X42E7PF36pq4KuR4Eu3LBIQNlHOHUTUmw6okVrTPsak2Mr7NKJ7LEmnMZe2dYGHtfn91TuZKY7_78_9fz3_-v_v___9f3r-3_3__59X---_e_V399zLv9__3__9gfiASYal8AF2JY4Mk0aVQogQhWEh0AoAKKAYWiawgYXBTsrAI9QQMAEJqAjAiBBiCjFgEAAgEASERASAHggEQBEAgABACpAQgAImAQWAFgYBAAKAaFiBFAEIEhBkcFRymBARItFBPZWAJRd7GmEIZRYAUCj-iowEShBAsDISFg5gAA.YAAAAAAAAAAA"); + client.Headers.Add("DNT", "1"); + client.Headers.Add("TE", "Trailers"); + client.DownloadFile(new Uri(url), fileName); + } + } + // Test 3: OK } @@ -157,6 +172,18 @@ namespace A69WD { //})); } + + + #endregion + + + #region Test ENGINE2 + + private void button1_Click(object sender, EventArgs e) { + + } + + #endregion