From c3355d8bf368233314fdea91e05abbec424546e5 Mon Sep 17 00:00:00 2001 From: CSiCoAlm Date: Thu, 22 Sep 2022 11:29:19 +0200 Subject: [PATCH] Versione per test webclient --- Src/A69WD/A69WD.csproj | 2 +- Src/A69WD/A69WD.csproj.user | 2 +- Src/A69WD/Net/cA69Engine2.cs | 55 +++--------- Src/A69WD/Net/cA69Engine3.cs | 161 ++++++++++++++++++++++++++++++----- 4 files changed, 156 insertions(+), 64 deletions(-) diff --git a/Src/A69WD/A69WD.csproj b/Src/A69WD/A69WD.csproj index b1a8a36..f5da4c5 100644 --- a/Src/A69WD/A69WD.csproj +++ b/Src/A69WD/A69WD.csproj @@ -63,8 +63,8 @@ - + Form diff --git a/Src/A69WD/A69WD.csproj.user b/Src/A69WD/A69WD.csproj.user index 1efe784..e482657 100644 --- a/Src/A69WD/A69WD.csproj.user +++ b/Src/A69WD/A69WD.csproj.user @@ -1,6 +1,6 @@  - ProjectFiles + ShowAllFiles \ No newline at end of file diff --git a/Src/A69WD/Net/cA69Engine2.cs b/Src/A69WD/Net/cA69Engine2.cs index 302b6b1..ec2195a 100644 --- a/Src/A69WD/Net/cA69Engine2.cs +++ b/Src/A69WD/Net/cA69Engine2.cs @@ -13,7 +13,7 @@ using System.Collections; namespace A69WD { - public class cA69Engine3 { + public class cA69Engine2 { #region Property: State, NumDownload, Tipologie, BASE_URL, BASE_LINK_IDX, BASE_LINK_RAC @@ -50,7 +50,7 @@ namespace A69WD { #region Costruttore e distruttore - public cA69Engine3() { + public cA69Engine2() { try { // Inizializzo gli archivi (dai file in memoria) allLinks = new List(); @@ -90,8 +90,8 @@ namespace A69WD { // proseguo con il download: string lk = toDownload.First(); toDownload.Remove(lk); - string fn = Path.Combine(Cfg.Settings.PathAnalizadedRaccFile, GetFileName(lk)); - using (MyWebClient client = new MyWebClient(lk, fn)) { + using (WebClient client = new WebClient()) { + string fn = Path.Combine(Cfg.Settings.PathAnalizadedRaccFile, GetFileName(lk)); //client.Encoding = Encoding.ASCII; client.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0"); client.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"); @@ -334,13 +334,7 @@ namespace A69WD { } - private string GetFileName(string url) { - if (IsRacLink(url)) { - return url.Replace("https://www.annunci69.it/racconti-erotici/", "A69_").Replace("/", "_").Replace(":", "_").Replace(@"\", "_"); - } else { - return url.Replace("https://www.annunci69.it/racconti-erotici/", "_A69_").Replace("/", "_").Replace(":", "_").Replace(@"\", "_"); - } - } + private string GetFileName(string url) { return url.Replace("https://www.annunci69.it/racconti-erotici/", "A69_").Replace("/", "_").Replace(":", "_").Replace(@"\", "_"); } static private bool IsIdxLink(string link) { return link.StartsWith(BASE_LINK_IDX); } static private bool IsRacLink(string link) { foreach (string tip in Tipologie) { if (link.StartsWith(BASE_LINK_RAC + tip + "/")) return true; } return false; } @@ -473,29 +467,19 @@ namespace A69WD { conteggioEvt++; - MyWebClient mwb = sender as MyWebClient; - if (state == EngineStatusEnum.Starting) state = EngineStatusEnum.Started; - - // aggiorno UI - RaiseEvent_Updating("(Evt" + conteggioEvt.ToString("0000") + ") Donwloaded: " + mwb.URL.Replace("https://www.annunci69.it/racconti-erotici/", "./")); - - // Salvo il testo downloadato if (!e.Cancelled && e.Error == null) { string textData = e.Result; - string FullFileName = mwb.FileName; + string FullFileName = @"C:\VMS\Prj\C#\A69WD\Test\Txt\Src\2022-09-22\A69_altroZZ.html"; + //System.IO.StreamWriter sw = new System.IO.StreamWriter(FullFileName, false, Encoding.Unicode); System.IO.StreamWriter sw = new System.IO.StreamWriter(FullFileName,false, Encoding.UTF8); sw.Write(textData); sw.Flush(); sw.Close(); - } - // Se ho raggiunto il massimo numero di link, o non vi sono link da downloadare, mi fermo: - if ((toDownload.Count > 0) && (state == EngineStatusEnum.Started)) { - // proseguo con il download: string lk = toDownload.First(); toDownload.Remove(lk); - string fn = Path.Combine(Cfg.Settings.PathAnalizadedRaccFile, GetFileName(lk)); - using (MyWebClient client = new MyWebClient(lk, fn)) { + using (WebClient client = new WebClient()) { + string fn = Path.Combine(Cfg.Settings.PathAnalizadedRaccFile, GetFileName(lk)); //client.Encoding = Encoding.ASCII; client.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0"); client.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"); @@ -507,15 +491,11 @@ namespace A69WD { client.DownloadFileCompleted += Client_DownloadFileCompleted; client.DownloadDataCompleted += Client_DownloadDataCompleted; client.DownloadStringCompleted += Client_DownloadStringCompleted; - //client.DownloadFileAsync(new Uri(lk), fn); + client.DownloadFileAsync(new Uri(lk), fn); //client.DownloadDataAsync(new Uri(lk)); - client.DownloadStringAsync(new Uri(lk)); + //client.DownloadStringAsync(new Uri(fn)); } - } else { - // Finito download: - state = EngineStatusEnum.Stopping; - // salvo gli archivi: - SaveState(); + } }//lock @@ -535,16 +515,5 @@ namespace A69WD { } }//class cA69Engine2 - public class MyWebClient : WebClient { - - public string URL; - public string FileName; - public MyWebClient(string url, string fileName) { - this.URL = url; - this.FileName = fileName; - } - - - } }//namespace A69WD diff --git a/Src/A69WD/Net/cA69Engine3.cs b/Src/A69WD/Net/cA69Engine3.cs index ec2195a..2f99ad1 100644 --- a/Src/A69WD/Net/cA69Engine3.cs +++ b/Src/A69WD/Net/cA69Engine3.cs @@ -10,10 +10,11 @@ using static System.Windows.Forms.LinkLabel; using System.ComponentModel; using static System.Net.Mime.MediaTypeNames; using System.Collections; +using System.Text.RegularExpressions; namespace A69WD { - public class cA69Engine2 { + public class cA69Engine3 { #region Property: State, NumDownload, Tipologie, BASE_URL, BASE_LINK_IDX, BASE_LINK_RAC @@ -50,7 +51,7 @@ namespace A69WD { #region Costruttore e distruttore - public cA69Engine2() { + public cA69Engine3() { try { // Inizializzo gli archivi (dai file in memoria) allLinks = new List(); @@ -90,8 +91,8 @@ namespace A69WD { // proseguo con il download: string lk = toDownload.First(); toDownload.Remove(lk); - using (WebClient client = new WebClient()) { - string fn = Path.Combine(Cfg.Settings.PathAnalizadedRaccFile, GetFileName(lk)); + string fn = Path.Combine(Cfg.Settings.PathAnalizadedRaccFile, GetFileName(lk)); + using (MyWebClient client = new MyWebClient(lk, fn)) { //client.Encoding = Encoding.ASCII; client.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0"); client.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"); @@ -334,7 +335,13 @@ namespace A69WD { } - private string GetFileName(string url) { return url.Replace("https://www.annunci69.it/racconti-erotici/", "A69_").Replace("/", "_").Replace(":", "_").Replace(@"\", "_"); } + private string GetFileName(string url) { + if (IsRacLink(url)) { + return url.Replace("https://www.annunci69.it/racconti-erotici/", "A69_").Replace("/", "_").Replace(":", "_").Replace(@"\", "_"); + } else { + return url.Replace("https://www.annunci69.it/racconti-erotici/", "_A69_").Replace("/", "_").Replace(":", "_").Replace(@"\", "_"); + } + } static private bool IsIdxLink(string link) { return link.StartsWith(BASE_LINK_IDX); } static private bool IsRacLink(string link) { foreach (string tip in Tipologie) { if (link.StartsWith(BASE_LINK_RAC + tip + "/")) return true; } return false; } @@ -467,19 +474,71 @@ namespace A69WD { conteggioEvt++; - if (!e.Cancelled && e.Error == null) { - string textData = e.Result; - string FullFileName = @"C:\VMS\Prj\C#\A69WD\Test\Txt\Src\2022-09-22\A69_altroZZ.html"; - //System.IO.StreamWriter sw = new System.IO.StreamWriter(FullFileName, false, Encoding.Unicode); - System.IO.StreamWriter sw = new System.IO.StreamWriter(FullFileName,false, Encoding.UTF8); - sw.Write(textData); - sw.Flush(); - sw.Close(); + MyWebClient mwb = sender as MyWebClient; + if (state == EngineStatusEnum.Starting) state = EngineStatusEnum.Started; + + // Verifico se download corretto + if (!e.Cancelled && e.Error == null) { + // aggiorno UI + RaiseEvent_Updating("(Evt" + conteggioEvt.ToString("0000") + ") Donwloaded: " + mwb.URL.Replace("https://www.annunci69.it/racconti-erotici/", "./")); + + // Se pagina è un racconto, lo salvo + if (IsRacLink(mwb.URL)) { + string textData = e.Result; + string FullFileName = mwb.FileName; + System.IO.StreamWriter sw = new System.IO.StreamWriter(FullFileName, false, Encoding.UTF8); + sw.Write(textData); + sw.Flush(); + sw.Close(); + } + + // Registro che ho eseguito il download del link + downloaded.Add(mwb.URL); + + // Estraggo tutti i link net testo: + List links = new List(); + //if (EstraiA69Link(e.Result, links)) { + if (EstraiLink(e.Result, "/racconti-erotici/", links)) { + + // Ciclo su tutti i link estratti dalla pagina: + foreach (string link in links) { + if (!allLinks.Contains(link)) { + // elimino il link di logon + if (link == "https://www.annunci69.it/accedi.php?nop") break; + // Link nuovo, mi comporto di conseguenza alla tipologia: + if (IsRacLink(link)) { + // Link ad un racconto + allLinks.Add(link); + if (downRac) { toDownload.Add(link); } + racLinks.Add(link); + } else { + if (IsIdxLink(link)) { + // Link ad un indice + allLinks.Add(link); + toDownload.Add(link); + } else { + // Link non riconosciuto, lo scarto + ; + } + } + } + } + } // fine ciclo sui link estratti + + } else { + // Download con errore, informo l'utente + RaiseEvent_Updating("(Evt" + conteggioEvt.ToString("0000") + ") Donwload error: " + mwb.URL.Replace("https://www.annunci69.it/racconti-erotici/", "./")); + } + + + // Se ho raggiunto il massimo numero di link, o non vi sono link da downloadare, mi fermo: + if ((toDownload.Count > 0) && (state == EngineStatusEnum.Started)) { + // proseguo con il download: string lk = toDownload.First(); toDownload.Remove(lk); - using (WebClient client = new WebClient()) { - string fn = Path.Combine(Cfg.Settings.PathAnalizadedRaccFile, GetFileName(lk)); + string fn = Path.Combine(Cfg.Settings.PathAnalizadedRaccFile, GetFileName(lk)); + using (MyWebClient client = new MyWebClient(lk, fn)) { //client.Encoding = Encoding.ASCII; client.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0"); client.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"); @@ -491,11 +550,15 @@ namespace A69WD { client.DownloadFileCompleted += Client_DownloadFileCompleted; client.DownloadDataCompleted += Client_DownloadDataCompleted; client.DownloadStringCompleted += Client_DownloadStringCompleted; - client.DownloadFileAsync(new Uri(lk), fn); + //client.DownloadFileAsync(new Uri(lk), fn); //client.DownloadDataAsync(new Uri(lk)); - //client.DownloadStringAsync(new Uri(fn)); + client.DownloadStringAsync(new Uri(lk)); } - + } else { + // Finito download: + state = EngineStatusEnum.Stopping; + // salvo gli archivi: + SaveState(); } }//lock @@ -513,7 +576,67 @@ namespace A69WD { // progressBar.Value = int.Parse(Math.Truncate(percentage).ToString()); //})); } - }//class cA69Engine2 + + + + + + + public static bool EstraiA69Link(string txt, List links) { + try { + Regex regex = new Regex("(?<=href=\")\\S+?(?=\")"); + Match matchObj = regex.Match(txt); + while (matchObj.Success) { + if (matchObj.Value.StartsWith("/racconti-erotici/")) { links.Add(matchObj.Value); } + if (matchObj.Groups.Count > 1) { + for (int i = 1; i < matchObj.Groups.Count; i++) { + Group group = matchObj.Groups[i]; + if (group.Success) { + if (group.Value.StartsWith("/racconti-erotici/")) { links.Add(group.Value); } + } + } + } + matchObj = matchObj.NextMatch(); + } + return true; + } catch { return false; } + } + + public static bool EstraiLink(string txt, string startWith, List links) { + try { + Regex regex = new Regex("(?<=href=\")\\S+?(?=\")"); + Match matchObj = regex.Match(txt); + while (matchObj.Success) { + if (matchObj.Value.StartsWith(startWith)) { links.Add(matchObj.Value.Replace(startWith, "https://www.annunci69.it/racconti-erotici/")); } + if (matchObj.Groups.Count > 1) { + for (int i = 1; i < matchObj.Groups.Count; i++) { + Group group = matchObj.Groups[i]; + if (group.Success) { + if (group.Value.StartsWith(startWith)) { links.Add(group.Value.Replace(startWith, "https://www.annunci69.it/racconti-erotici/")); } + } + } + } + matchObj = matchObj.NextMatch(); + } + return true; + } catch { return false; } + } + + + }//class cA69Engine3 + + public class MyWebClient : WebClient { + + public string URL; + public string FileName; + public MyWebClient(string url, string fileName) { + this.URL = url; + this.FileName = fileName; + } + + + } + }//namespace A69WD