diff --git a/Src/A69WD/A69WD.csproj b/Src/A69WD/A69WD.csproj index f5da4c5..23c8034 100644 --- a/Src/A69WD/A69WD.csproj +++ b/Src/A69WD/A69WD.csproj @@ -63,8 +63,6 @@ - - Form diff --git a/Src/A69WD/Net/cA69Engine.cs b/Src/A69WD/Net/cA69Engine.cs index 012904d..fb4bbd3 100644 --- a/Src/A69WD/Net/cA69Engine.cs +++ b/Src/A69WD/Net/cA69Engine.cs @@ -4,14 +4,15 @@ using System.Collections.Generic; using System.Linq; using System.Text; using System.Windows.Forms; - +using System.Net; +using System.Text.RegularExpressions; namespace A69WD { public class cA69Engine { - #region Property: State, NumDownload, Tipologie, BASE_URL, BASE_LINK_IDX, BASE_LINK_RAC + #region Property: State, NumDownload, Tipologie, BASE_URL, BASE_LINK_IDX, BASE_LINK_RAC, conteggioEvt public EngineStatusEnum State { get { return state; } } @@ -23,10 +24,12 @@ namespace A69WD { const string BASE_LINK_IDX = "https://www.annunci69.it/racconti-erotici/racconti_lista.php"; const string BASE_LINK_RAC = "https://www.annunci69.it/racconti-erotici/"; + int conteggioEvt = 0; + #endregion - #region Fields: allLinks, toDownload, downloaded, racLinks, dtStartTime, downRac, state, wb + #region Fields: allLinks, toDownload, downloaded, racLinks, dtStartTime, downRac, state List allLinks = null; List toDownload = null; @@ -39,14 +42,12 @@ namespace A69WD { EngineStatusEnum state = EngineStatusEnum.NotReady; - WebBrowser wb = null; - #endregion #region Costruttore e distruttore - public cA69Engine(WebBrowser WB) { + public cA69Engine() { try { // Inizializzo gli archivi (dai file in memoria) allLinks = new List(); @@ -54,14 +55,15 @@ namespace A69WD { downloaded = new List(); racLinks = new List(); int res = App.DB.LoadData(allLinks, toDownload, downloaded, racLinks); + RaiseEvent_Updating("Ricaricati " + res.ToString("000") + " link di racconti da scaricare"); - wb = WB; - wb.DocumentCompleted += Wb_DocumentCompleted; + // Per ora non serializzo gli archivi e quindi carico le pagine iniziali delle 11 tipologie: foreach (string tipologia in Tipologie) { toDownload.Add(BASE_URL.Replace("{Tip}", tipologia)); allLinks.Add(BASE_URL.Replace("{Tip}", tipologia)); } + } catch (Exception ex) { Logger.Log(new object[] { "Error Code: ER02220", "Errore in cA69Engine()", ex }); throw ex; @@ -72,7 +74,7 @@ namespace A69WD { #endregion - #region Method: + #region Method: Start, Stop, ImportLinkFromFile public bool Start(bool DownRac) { if (state != EngineStatusEnum.StandBy) return false; @@ -88,7 +90,23 @@ namespace A69WD { // proseguo con il download: string lk = toDownload.First(); toDownload.Remove(lk); - wb.Navigate(lk); + string fn = Path.Combine(Cfg.Settings.PathAnalizadedRaccFile, GetFileName(lk)); + using (MyWebClient client = new MyWebClient(lk, fn)) { + //client.Encoding = Encoding.ASCII; + client.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0"); + client.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"); + client.Headers.Add("Cookie", "euconsent-v2=CPG74goPG74goAKAeAENBcCsAP_AAH_AAAwIH3tf_X__bX9j-_59__t0eY1f9_7_v-wzjhfdt-8N2P_X_L0X42E7PF36pq4KuR4Eu3LBIQNlHOHUTUmw6okVrTPsak2Mr7NKJ7LEmnMZe2dYGHtfn91TuZKY7_78_9fz3_-v_v___9f3r-3_3__59X---_e_V399zLv9__3__9gfiASYal8AF2JY4Mk0aVQogQhWEh0AoAKKAYWiawgYXBTsrAI9QQMAEJqAjAiBBiCjFgEAAgEASERASAHggEQBEAgABACpAQgAImAQWAFgYBAAKAaFiBFAEIEhBkcFRymBARItFBPZWAJRd7GmEIZRYAUCj-iowEShBAsDISFg5gAA.YAAAAAAAAAAA"); + client.Headers.Add("DNT", "1"); + client.Headers.Add("TE", "Trailers"); + ////client.DownloadFile(new Uri(lk), fn); + //client.DownloadProgressChanged += Client_DownloadProgressChanged; + //client.DownloadFileCompleted += Client_DownloadFileCompleted; + //client.DownloadDataCompleted += Client_DownloadDataCompleted; + //client.DownloadStringCompleted += Client_DownloadStringCompleted; + ////client.DownloadFileAsync(new Uri(lk), fn); + ////client.DownloadDataAsync(new Uri(lk)); + client.DownloadStringAsync(new Uri(lk)); + } } return true; @@ -159,7 +177,7 @@ namespace A69WD { #endregion - #region Eventi: + #region Eventi: Updating, Client_DownloadStringCompleted /// /// Evento che indica un cambiamento di stato nell'AnalizeManager @@ -167,6 +185,98 @@ namespace A69WD { public event EventHandler Updating; + private void Client_DownloadStringCompleted(object sender, DownloadStringCompletedEventArgs e) { + object obj = new Object(); + lock (obj) { + + conteggioEvt++; + + MyWebClient mwb = sender as MyWebClient; + if (state == EngineStatusEnum.Starting) state = EngineStatusEnum.Started; + + + // Verifico se download corretto + if (!e.Cancelled && e.Error == null) { + // aggiorno UI + RaiseEvent_Updating("(Evt" + conteggioEvt.ToString("0000") + ") Donwloaded: " + mwb.URL.Replace("https://www.annunci69.it/racconti-erotici/", "./")); + + // Se pagina è un racconto, lo salvo + if (IsRacLink(mwb.URL)) { + string textData = e.Result; + string FullFileName = mwb.FileName; + System.IO.StreamWriter sw = new System.IO.StreamWriter(FullFileName, false, Encoding.UTF8); + sw.Write(textData); + sw.Flush(); + sw.Close(); + } + + // Registro che ho eseguito il download del link + downloaded.Add(mwb.URL); + + // Estraggo tutti i link net testo: + List links = new List(); + //if (EstraiA69Link(e.Result, links)) { + if (EstraiLink(e.Result, "/racconti-erotici/", links)) { + + // Ciclo su tutti i link estratti dalla pagina: + foreach (string link in links) { + if (!allLinks.Contains(link)) { + // elimino il link di logon + if (link == "https://www.annunci69.it/accedi.php?nop") break; + // Link nuovo, mi comporto di conseguenza alla tipologia: + if (IsRacLink(link)) { + // Link ad un racconto + allLinks.Add(link); + if (downRac) { toDownload.Add(link); } + racLinks.Add(link); + } else { + if (IsIdxLink(link)) { + // Link ad un indice + allLinks.Add(link); + toDownload.Add(link); + } else { + // Link non riconosciuto, lo scarto + ; + } + } + } + } + } // fine ciclo sui link estratti + + } else { + // Download con errore, informo l'utente + RaiseEvent_Updating("(Evt" + conteggioEvt.ToString("0000") + ") Donwload error: " + mwb.URL.Replace("https://www.annunci69.it/racconti-erotici/", "./")); + } + + + // Se ho raggiunto il massimo numero di link, o non vi sono link da downloadare, mi fermo: + if ((toDownload.Count > 0) && (state == EngineStatusEnum.Started)) { + // proseguo con il download: + string lk = toDownload.First(); + toDownload.Remove(lk); + string fn = Path.Combine(Cfg.Settings.PathAnalizadedRaccFile, GetFileName(lk)); + using (MyWebClient client = new MyWebClient(lk, fn)) { + client.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0"); + client.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"); + client.Headers.Add("Cookie", "euconsent-v2=CPG74goPG74goAKAeAENBcCsAP_AAH_AAAwIH3tf_X__bX9j-_59__t0eY1f9_7_v-wzjhfdt-8N2P_X_L0X42E7PF36pq4KuR4Eu3LBIQNlHOHUTUmw6okVrTPsak2Mr7NKJ7LEmnMZe2dYGHtfn91TuZKY7_78_9fz3_-v_v___9f3r-3_3__59X---_e_V399zLv9__3__9gfiASYal8AF2JY4Mk0aVQogQhWEh0AoAKKAYWiawgYXBTsrAI9QQMAEJqAjAiBBiCjFgEAAgEASERASAHggEQBEAgABACpAQgAImAQWAFgYBAAKAaFiBFAEIEhBkcFRymBARItFBPZWAJRd7GmEIZRYAUCj-iowEShBAsDISFg5gAA.YAAAAAAAAAAA"); + client.Headers.Add("DNT", "1"); + client.Headers.Add("TE", "Trailers"); + client.DownloadStringCompleted += Client_DownloadStringCompleted; + client.DownloadStringAsync(new Uri(lk)); + } + } else { + // Finito download: + state = EngineStatusEnum.Stopping; + // salvo gli archivi: + SaveState(); + } + + }//lock + + } + + + #region Generazione degli eventi dell'engine verso le classi chiamanti: /// @@ -191,94 +301,11 @@ namespace A69WD { #endregion - int conteggioEvt = 0; - - - private void Wb_DocumentCompleted(object sender, WebBrowserDocumentCompletedEventArgs e) { - object obj = new Object(); - lock (obj) { - - conteggioEvt++; - - StringBuilder sb; - if (state == EngineStatusEnum.Starting) state = EngineStatusEnum.Started; - - WebBrowser wb = sender as WebBrowser; - string link = wb.Url.ToString(); - - // Se link di logon, devo terminare i download - if (link== "https://www.annunci69.it/accedi.php?nop") { - // Finito download: - state = EngineStatusEnum.Stopping; - // salvo gli archivi: - SaveState(); - return; - } - - // Elimino ulteriori ripetizioni dell'evento: - if (downloaded.Contains(link)) return; - - // aggiorno UI - RaiseEvent_Updating(wb.Name + " - (Evt" + conteggioEvt.ToString("0000") + ") Donwloaded: " + link.Replace("https://www.annunci69.it/racconti-erotici/","./")); - - // Se pagina è un racconto, lo salvo - if (IsRacLink(link)) { - sb = new StringBuilder(); - sb.Append(wb.Document.Body.InnerHtml); - string fn = Path.Combine(Cfg.Settings.PathAnalizadedRaccFile, GetFileName(link)); - //cTools.WriteTextFile(fn, sb, Encoding.GetEncoding(wb.Document.Encoding)); - cTools.WriteTextFile(fn, sb); - } - - // Registro che ho eseguito il download del link - downloaded.Add(link); - - // Ciclo su tutti i link estratti dalla pagina: - foreach (HtmlElement item in wb.Document.Links) { - link = item.GetAttribute("href"); - // Se link nuovo: - if (!allLinks.Contains(link)) { - // elimino il link di logon - if (link == "https://www.annunci69.it/accedi.php?nop") break; - // Link nuovo, mi comporto di conseguenza alla tipologia: - if (IsRacLink(link)) { - // Link ad un racconto - allLinks.Add(link); - if (downRac) { toDownload.Add(link); } - racLinks.Add(link); - } else { - if (IsIdxLink(link)) { - // Link ad un indice - allLinks.Add(link); - toDownload.Add(link); - } else { - // Link non riconosciuto, lo scarto - ; - } - } - } - } // fine ciclo sui link estratti - - // Se ho raggiunto il massimo numero di link, o non vi sono link da downloadare, mi fermo: - if ((toDownload.Count > 0) && (state==EngineStatusEnum.Started)) { - // proseguo con il download: - string lk = toDownload.First(); - toDownload.Remove(lk); - wb.Navigate(lk); - } else { - // Finito download: - state = EngineStatusEnum.Stopping; - // salvo gli archivi: - SaveState(); - } - - }//lock - } - + #region Internal Method: SaveState, GetFileName, IsIdxLink, IsRacLink, LinkType, EstraiLink private void SaveState() { int res = App.DB.SaveData(allLinks, toDownload, downloaded, racLinks); - + StringBuilder sb; // Finito download, salvo gli archivi: @@ -315,11 +342,16 @@ namespace A69WD { } + private string GetFileName(string url) { + if (IsRacLink(url)) { + return url.Replace("https://www.annunci69.it/racconti-erotici/", "A69_").Replace("/", "_").Replace(":", "_").Replace(@"\", "_"); + } else { + return url.Replace("https://www.annunci69.it/racconti-erotici/", "_A69_").Replace("/", "_").Replace(":", "_").Replace(@"\", "_"); + } + } - private string GetFileName(string url) { return url.Replace("https://www.annunci69.it/racconti-erotici/", "A69_").Replace("/", "_").Replace(":", "_").Replace(@"\", "_"); } static private bool IsIdxLink(string link) { return link.StartsWith(BASE_LINK_IDX); } static private bool IsRacLink(string link) { foreach (string tip in Tipologie) { if (link.StartsWith(BASE_LINK_RAC + tip + "/")) return true; } return false; } - static public string LinkType(string link) { if (IsIdxLink(link)) return "Idx"; if (IsRacLink(link)) return "Rac"; @@ -327,6 +359,29 @@ namespace A69WD { return "Unk"; } + public static bool EstraiLink(string txt, string startWith, List links) { + try { + Regex regex = new Regex("(?<=href=\")\\S+?(?=\")"); + Match matchObj = regex.Match(txt); + while (matchObj.Success) { + if (matchObj.Value.StartsWith(startWith)) { links.Add(matchObj.Value.Replace(startWith, "https://www.annunci69.it/racconti-erotici/")); } + if (matchObj.Groups.Count > 1) { + for (int i = 1; i < matchObj.Groups.Count; i++) { + Group group = matchObj.Groups[i]; + if (group.Success) { + if (group.Value.StartsWith(startWith)) { links.Add(group.Value.Replace(startWith, "https://www.annunci69.it/racconti-erotici/")); } + } + } + } + matchObj = matchObj.NextMatch(); + } + return true; + } catch { return false; } + } + + #endregion + + }//class cA69Engine @@ -383,5 +438,17 @@ namespace A69WD { } // class EngineEventArgs + public class MyWebClient : WebClient { + + public string URL; + public string FileName; + public MyWebClient(string url, string fileName) { + this.URL = url; + this.FileName = fileName; + } + + + } // class MyWebClient + }//namespace A69WD diff --git a/Src/A69WD/Net/cA69Engine2.cs b/Src/A69WD/Net/cA69Engine2.cs deleted file mode 100644 index ec2195a..0000000 --- a/Src/A69WD/Net/cA69Engine2.cs +++ /dev/null @@ -1,519 +0,0 @@ -using System; -using System.IO; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Windows.Forms; -using System.Security.Policy; -using System.Net; -using static System.Windows.Forms.LinkLabel; -using System.ComponentModel; -using static System.Net.Mime.MediaTypeNames; -using System.Collections; - -namespace A69WD { - - public class cA69Engine2 { - - - #region Property: State, NumDownload, Tipologie, BASE_URL, BASE_LINK_IDX, BASE_LINK_RAC - - public EngineStatusEnum State { get { return state; } } - - public int NumDownload { get { return downloaded.Count; } } - - static string[] Tipologie = { "altro", "bdsm", "gay", "incesto", "lesbo", "orge", "prime", "scambio", "tradimenti", "trans", "trio" }; - - const string BASE_URL = "https://www.annunci69.it/racconti-erotici/{Tip}.html"; - const string BASE_LINK_IDX = "https://www.annunci69.it/racconti-erotici/racconti_lista.php"; - const string BASE_LINK_RAC = "https://www.annunci69.it/racconti-erotici/"; - - #endregion - - - #region Fields: allLinks, toDownload, downloaded, racLinks, dtStartTime, downRac, state - - List allLinks = null; - List toDownload = null; - List downloaded = null; - List racLinks = null; - - DateTime dtStartTime; - - bool downRac = false; - - EngineStatusEnum state = EngineStatusEnum.NotReady; - - - #endregion - - - #region Costruttore e distruttore - - public cA69Engine2() { - try { - // Inizializzo gli archivi (dai file in memoria) - allLinks = new List(); - toDownload = new List(); - downloaded = new List(); - racLinks = new List(); - int res = App.DB.LoadData(allLinks, toDownload, downloaded, racLinks); - RaiseEvent_Updating("Ricaricati " + res.ToString("000") + " link di racconti da scaricare"); - // Per ora non serializzo gli archivi e quindi carico le pagine iniziali delle 11 tipologie: - foreach (string tipologia in Tipologie) { - toDownload.Add(BASE_URL.Replace("{Tip}", tipologia)); - allLinks.Add(BASE_URL.Replace("{Tip}", tipologia)); - } - } catch (Exception ex) { - Logger.Log(new object[] { "Error Code: ER02220", "Errore in cA69Engine()", ex }); - throw ex; - } - state = EngineStatusEnum.StandBy; - } - - #endregion - - - #region Method: - - public bool Start(bool DownRac) { - if (state != EngineStatusEnum.StandBy) return false; - - dtStartTime = DateTime.Now; - downRac = DownRac; - - RaiseEvent_Updating("Inizio lo scrap dei link dei racconti"); - - state = EngineStatusEnum.Starting; - - if (toDownload.Count > 0) { - // proseguo con il download: - string lk = toDownload.First(); - toDownload.Remove(lk); - using (WebClient client = new WebClient()) { - string fn = Path.Combine(Cfg.Settings.PathAnalizadedRaccFile, GetFileName(lk)); - //client.Encoding = Encoding.ASCII; - client.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0"); - client.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"); - client.Headers.Add("Cookie", "euconsent-v2=CPG74goPG74goAKAeAENBcCsAP_AAH_AAAwIH3tf_X__bX9j-_59__t0eY1f9_7_v-wzjhfdt-8N2P_X_L0X42E7PF36pq4KuR4Eu3LBIQNlHOHUTUmw6okVrTPsak2Mr7NKJ7LEmnMZe2dYGHtfn91TuZKY7_78_9fz3_-v_v___9f3r-3_3__59X---_e_V399zLv9__3__9gfiASYal8AF2JY4Mk0aVQogQhWEh0AoAKKAYWiawgYXBTsrAI9QQMAEJqAjAiBBiCjFgEAAgEASERASAHggEQBEAgABACpAQgAImAQWAFgYBAAKAaFiBFAEIEhBkcFRymBARItFBPZWAJRd7GmEIZRYAUCj-iowEShBAsDISFg5gAA.YAAAAAAAAAAA"); - client.Headers.Add("DNT", "1"); - client.Headers.Add("TE", "Trailers"); - //client.DownloadFile(new Uri(lk), fn); - client.DownloadProgressChanged += Client_DownloadProgressChanged; - client.DownloadFileCompleted += Client_DownloadFileCompleted; - client.DownloadDataCompleted += Client_DownloadDataCompleted; - client.DownloadStringCompleted += Client_DownloadStringCompleted; - //client.DownloadFileAsync(new Uri(lk), fn); - //client.DownloadDataAsync(new Uri(lk)); - client.DownloadStringAsync(new Uri(lk)); - } - } - - return true; - } - - public bool Stop() { - switch (state) { - case EngineStatusEnum.NotReady: - case EngineStatusEnum.StandBy: - SaveState(); - state = EngineStatusEnum.Stopped; - return true; - case EngineStatusEnum.Starting: - case EngineStatusEnum.Started: - case EngineStatusEnum.Stopping: - state = EngineStatusEnum.Stopping; - return false; - case EngineStatusEnum.Stopped: - return true; - } - return true; - } - - public int ImportLinkFromFile(string SrcFilePath) { - int res = 0; - try { - if (File.Exists(SrcFilePath)) { - StreamReader sr = new StreamReader(SrcFilePath, Encoding.UTF8); - string row = sr.ReadLine().Trim(); - while (row != null) { - // verifico che non sia un commento: - if (!(row.StartsWith("#") || row.StartsWith("//"))) { - // Se link nuovo: - if (!allLinks.Contains(row)) { - // Link nuovo, mi comporto di conseguenza alla tipologia: - if (IsRacLink(row)) { - // Link ad un racconto - allLinks.Add(row); - toDownload.Add(row); - racLinks.Add(row); - res++; - } else { - if (IsIdxLink(row)) { - // Link ad un indice - allLinks.Add(row); - toDownload.Add(row); - res++; - } else { - // Link non riconosciuto, lo scarto - ; - } - } - } - } - //Read the next line - row = sr.ReadLine(); - } - } - - } catch (Exception ex) { - Logger.Log(new object[] { "Error Code: DB07403", "Errore in ImportLinkFromFile()", ex }); - if (res > 0) res = -1 * res; - } - return res; - } - - - #endregion - - - #region Eventi: - - /// - /// Evento che indica un cambiamento di stato nell'AnalizeManager - /// - public event EventHandler Updating; - - - #region Generazione degli eventi dell'engine verso le classi chiamanti: - - /// - /// Attiva l'evento che segnala un cambiamento di stato nel engine (per esempio, terminato il download di un singolo file) - /// - void RaiseEvent_Updating(string msg, int tot, int toDown, int down) { - EventHandler tmp = Updating; - if (null != tmp) tmp(this, new EngineEventArgs(msg, tot, toDown, down)); - } - - /// - /// Attiva l'evento che segnala un cambiamento di stato nel engine (per esempio, terminato il download di un singolo file) - /// - void RaiseEvent_Updating(string msg) { - EventHandler tmp = Updating; - if (null != tmp) tmp(this, new EngineEventArgs(msg, allLinks.Count, toDownload.Count, downloaded.Count)); - } - - #endregion - - - #endregion - - - int conteggioEvt = 0; - - - private void Wb_DocumentCompleted(object sender, WebBrowserDocumentCompletedEventArgs e) { - object obj = new Object(); - lock (obj) { - - conteggioEvt++; - - StringBuilder sb; - if (state == EngineStatusEnum.Starting) state = EngineStatusEnum.Started; - - WebBrowser wb = sender as WebBrowser; - string link = wb.Url.ToString(); - - // Se link di logon, devo terminare i download - if (link == "https://www.annunci69.it/accedi.php?nop") { - // Finito download: - state = EngineStatusEnum.Stopping; - // salvo gli archivi: - SaveState(); - return; - } - - // Elimino ulteriori ripetizioni dell'evento: - if (downloaded.Contains(link)) return; - - // aggiorno UI - RaiseEvent_Updating(wb.Name + " - (Evt" + conteggioEvt.ToString("0000") + ") Donwloaded: " + link.Replace("https://www.annunci69.it/racconti-erotici/", "./")); - - // Se pagina è un racconto, lo salvo - if (IsRacLink(link)) { - sb = new StringBuilder(); - sb.Append(wb.Document.Body.InnerHtml); - string fn = Path.Combine(Cfg.Settings.PathAnalizadedRaccFile, GetFileName(link)); - //cTools.WriteTextFile(fn, sb, Encoding.GetEncoding(wb.Document.Encoding)); - cTools.WriteTextFile(fn, sb); - } - - // Registro che ho eseguito il download del link - downloaded.Add(link); - - // Ciclo su tutti i link estratti dalla pagina: - foreach (HtmlElement item in wb.Document.Links) { - link = item.GetAttribute("href"); - // Se link nuovo: - if (!allLinks.Contains(link)) { - // elimino il link di logon - if (link == "https://www.annunci69.it/accedi.php?nop") break; - // Link nuovo, mi comporto di conseguenza alla tipologia: - if (IsRacLink(link)) { - // Link ad un racconto - allLinks.Add(link); - if (downRac) { toDownload.Add(link); } - racLinks.Add(link); - } else { - if (IsIdxLink(link)) { - // Link ad un indice - allLinks.Add(link); - toDownload.Add(link); - } else { - // Link non riconosciuto, lo scarto - ; - } - } - } - } // fine ciclo sui link estratti - - // Se ho raggiunto il massimo numero di link, o non vi sono link da downloadare, mi fermo: - if ((toDownload.Count > 0) && (state == EngineStatusEnum.Started)) { - // proseguo con il download: - string lk = toDownload.First(); - toDownload.Remove(lk); - wb.Navigate(lk); - } else { - // Finito download: - state = EngineStatusEnum.Stopping; - // salvo gli archivi: - SaveState(); - } - - }//lock - } - - - private void SaveState() { - int res = App.DB.SaveData(allLinks, toDownload, downloaded, racLinks); - - StringBuilder sb; - - // Finito download, salvo gli archivi: - string dt = DateTime.Now.ToString("yyyy-MM-dd_HH-mm-ss"); - - // Salvo archivio: AllLinks - sb = new StringBuilder(); - allLinks.Sort(); - foreach (string lnk in allLinks) { sb.AppendLine(lnk); } - cTools.WriteTextFile(Path.Combine(Cfg.Settings.PathAnalizadedIndexFile, string.Format("_A69_AllLinks_{0}.tes", dt)), sb); - - // Salvo archivio: toDownload - sb = new StringBuilder(); - toDownload.Sort(); - foreach (string lnk in toDownload) { sb.AppendLine(lnk); } - cTools.WriteTextFile(Path.Combine(Cfg.Settings.PathAnalizadedIndexFile, string.Format("_A69_toDownload_{0}.tes", dt)), sb); - - // Salvo archivio: Downloaded - sb = new StringBuilder(); - downloaded.Sort(); - foreach (string lnk in downloaded) { sb.AppendLine(lnk); } - cTools.WriteTextFile(Path.Combine(Cfg.Settings.PathAnalizadedIndexFile, string.Format("_A69_Downloaded_{0}.tes", dt)), sb); - - // Salvo archivio: RacLinks - sb = new StringBuilder(); - racLinks.Sort(); - foreach (string lnk in racLinks) { sb.AppendLine(lnk); } - cTools.WriteTextFile(Path.Combine(Cfg.Settings.PathAnalizadedIndexFile, string.Format("_A69_RacLinks_{0}.tes", dt)), sb); - - // avviso utente: - RaiseEvent_Updating(string.Format("Finito il download in {0}", DateTime.Now - dtStartTime)); - - state = EngineStatusEnum.Stopped; - - } - - - private string GetFileName(string url) { return url.Replace("https://www.annunci69.it/racconti-erotici/", "A69_").Replace("/", "_").Replace(":", "_").Replace(@"\", "_"); } - static private bool IsIdxLink(string link) { return link.StartsWith(BASE_LINK_IDX); } - static private bool IsRacLink(string link) { foreach (string tip in Tipologie) { if (link.StartsWith(BASE_LINK_RAC + tip + "/")) return true; } return false; } - - static public string LinkType(string link) { - if (IsIdxLink(link)) return "Idx"; - if (IsRacLink(link)) return "Rac"; - //return "Unknown"; - return "Unk"; - } - - - - - - - - private void Client_DownloadFileCompleted(object sender, AsyncCompletedEventArgs e) { - object obj = new Object(); - lock (obj) { - - conteggioEvt++; - - StringBuilder sb; - if (state == EngineStatusEnum.Starting) state = EngineStatusEnum.Started; - - WebClient wb = sender as WebClient; - string link = wb.BaseAddress; - - return; - /* - // Se link di logon, devo terminare i download - if (link == "https://www.annunci69.it/accedi.php?nop") { - // Finito download: - state = EngineStatusEnum.Stopping; - // salvo gli archivi: - SaveState(); - return; - } - - // Elimino ulteriori ripetizioni dell'evento: - if (downloaded.Contains(link)) return; - - // aggiorno UI - RaiseEvent_Updating(wb.Name + " - (Evt" + conteggioEvt.ToString("0000") + ") Donwloaded: " + link.Replace("https://www.annunci69.it/racconti-erotici/", "./")); - - // Se pagina è un racconto, lo salvo - if (IsRacLink(link)) { - sb = new StringBuilder(); - sb.Append(wb.Document.Body.InnerHtml); - string fn = Path.Combine(Cfg.Settings.PathAnalizadedRaccFile, GetFileName(link)); - //cTools.WriteTextFile(fn, sb, Encoding.GetEncoding(wb.Document.Encoding)); - cTools.WriteTextFile(fn, sb); - } - - // Registro che ho eseguito il download del link - downloaded.Add(link); - - // Ciclo su tutti i link estratti dalla pagina: - foreach (HtmlElement item in wb.Document.Links) { - link = item.GetAttribute("href"); - // Se link nuovo: - if (!allLinks.Contains(link)) { - // elimino il link di logon - if (link == "https://www.annunci69.it/accedi.php?nop") break; - // Link nuovo, mi comporto di conseguenza alla tipologia: - if (IsRacLink(link)) { - // Link ad un racconto - allLinks.Add(link); - if (downRac) { toDownload.Add(link); } - racLinks.Add(link); - } else { - if (IsIdxLink(link)) { - // Link ad un indice - allLinks.Add(link); - toDownload.Add(link); - } else { - // Link non riconosciuto, lo scarto - ; - } - } - } - } // fine ciclo sui link estratti - - // Se ho raggiunto il massimo numero di link, o non vi sono link da downloadare, mi fermo: - if ((toDownload.Count > 0) && (state == EngineStatusEnum.Started)) { - // proseguo con il download: - string lk = toDownload.First(); - toDownload.Remove(lk); - wb.Navigate(lk); - } else { - // Finito download: - state = EngineStatusEnum.Stopping; - // salvo gli archivi: - SaveState(); - } - - //*/ - }//lock - - } - private void Client_DownloadDataCompleted(object sender, DownloadDataCompletedEventArgs e) { - //System.Threading.AutoResetEvent waiter = (System.Threading.AutoResetEvent)e.UserState; - - try { - // If the request was not canceled and did not throw - // an exception, display the resource. - if (!e.Cancelled && e.Error == null) { - byte[] data = (byte[])e.Result; - string textData = System.Text.Encoding.Unicode.GetString(data); - string FullFileName = @"C:\VMS\Prj\C#\A69WD\Test\Txt\Src\2022-09-22\A69_altroz.html"; - - //System.IO.StreamWriter sw = new System.IO.StreamWriter(FullFileName, false, Encoding.Unicode); - //sw.Write(textData); - //sw.Flush(); - //sw.Close(); - - using (var fs = new FileStream(FullFileName, FileMode.Create, FileAccess.Write)) { - fs.Write(data, 0, data.Length); - } - } - } finally { - // Let the main application thread resume. - //waiter.Set(); - } - } - - private void Client_DownloadStringCompleted(object sender, DownloadStringCompletedEventArgs e) { - object obj = new Object(); - lock (obj) { - - conteggioEvt++; - - if (!e.Cancelled && e.Error == null) { - string textData = e.Result; - string FullFileName = @"C:\VMS\Prj\C#\A69WD\Test\Txt\Src\2022-09-22\A69_altroZZ.html"; - //System.IO.StreamWriter sw = new System.IO.StreamWriter(FullFileName, false, Encoding.Unicode); - System.IO.StreamWriter sw = new System.IO.StreamWriter(FullFileName,false, Encoding.UTF8); - sw.Write(textData); - sw.Flush(); - sw.Close(); - - string lk = toDownload.First(); - toDownload.Remove(lk); - using (WebClient client = new WebClient()) { - string fn = Path.Combine(Cfg.Settings.PathAnalizadedRaccFile, GetFileName(lk)); - //client.Encoding = Encoding.ASCII; - client.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0"); - client.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"); - client.Headers.Add("Cookie", "euconsent-v2=CPG74goPG74goAKAeAENBcCsAP_AAH_AAAwIH3tf_X__bX9j-_59__t0eY1f9_7_v-wzjhfdt-8N2P_X_L0X42E7PF36pq4KuR4Eu3LBIQNlHOHUTUmw6okVrTPsak2Mr7NKJ7LEmnMZe2dYGHtfn91TuZKY7_78_9fz3_-v_v___9f3r-3_3__59X---_e_V399zLv9__3__9gfiASYal8AF2JY4Mk0aVQogQhWEh0AoAKKAYWiawgYXBTsrAI9QQMAEJqAjAiBBiCjFgEAAgEASERASAHggEQBEAgABACpAQgAImAQWAFgYBAAKAaFiBFAEIEhBkcFRymBARItFBPZWAJRd7GmEIZRYAUCj-iowEShBAsDISFg5gAA.YAAAAAAAAAAA"); - client.Headers.Add("DNT", "1"); - client.Headers.Add("TE", "Trailers"); - //client.DownloadFile(new Uri(lk), fn); - client.DownloadProgressChanged += Client_DownloadProgressChanged; - client.DownloadFileCompleted += Client_DownloadFileCompleted; - client.DownloadDataCompleted += Client_DownloadDataCompleted; - client.DownloadStringCompleted += Client_DownloadStringCompleted; - client.DownloadFileAsync(new Uri(lk), fn); - //client.DownloadDataAsync(new Uri(lk)); - //client.DownloadStringAsync(new Uri(fn)); - } - - } - - }//lock - - } - - private void Client_DownloadProgressChanged(object sender, DownloadProgressChangedEventArgs e) { - ////Update progress bar & label - //Invoke(new MethodInvoker(delegate () { - // progressBar.Minimum = 0; - // double receive = double.Parse(e.BytesReceived.ToString()); - // double total = double.Parse(e.TotalBytesToReceive.ToString()); - // double percentage = receive / total * 100; - // lblStatus.Text = $"Downloaded {string.Format("{0:0.##}", percentage)}%"; - // progressBar.Value = int.Parse(Math.Truncate(percentage).ToString()); - //})); - } - }//class cA69Engine2 - - -}//namespace A69WD diff --git a/Src/A69WD/Net/cA69Engine3.cs b/Src/A69WD/Net/cA69Engine3.cs deleted file mode 100644 index 2f99ad1..0000000 --- a/Src/A69WD/Net/cA69Engine3.cs +++ /dev/null @@ -1,642 +0,0 @@ -using System; -using System.IO; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Windows.Forms; -using System.Security.Policy; -using System.Net; -using static System.Windows.Forms.LinkLabel; -using System.ComponentModel; -using static System.Net.Mime.MediaTypeNames; -using System.Collections; -using System.Text.RegularExpressions; - -namespace A69WD { - - public class cA69Engine3 { - - - #region Property: State, NumDownload, Tipologie, BASE_URL, BASE_LINK_IDX, BASE_LINK_RAC - - public EngineStatusEnum State { get { return state; } } - - public int NumDownload { get { return downloaded.Count; } } - - static string[] Tipologie = { "altro", "bdsm", "gay", "incesto", "lesbo", "orge", "prime", "scambio", "tradimenti", "trans", "trio" }; - - const string BASE_URL = "https://www.annunci69.it/racconti-erotici/{Tip}.html"; - const string BASE_LINK_IDX = "https://www.annunci69.it/racconti-erotici/racconti_lista.php"; - const string BASE_LINK_RAC = "https://www.annunci69.it/racconti-erotici/"; - - #endregion - - - #region Fields: allLinks, toDownload, downloaded, racLinks, dtStartTime, downRac, state - - List allLinks = null; - List toDownload = null; - List downloaded = null; - List racLinks = null; - - DateTime dtStartTime; - - bool downRac = false; - - EngineStatusEnum state = EngineStatusEnum.NotReady; - - - #endregion - - - #region Costruttore e distruttore - - public cA69Engine3() { - try { - // Inizializzo gli archivi (dai file in memoria) - allLinks = new List(); - toDownload = new List(); - downloaded = new List(); - racLinks = new List(); - int res = App.DB.LoadData(allLinks, toDownload, downloaded, racLinks); - RaiseEvent_Updating("Ricaricati " + res.ToString("000") + " link di racconti da scaricare"); - // Per ora non serializzo gli archivi e quindi carico le pagine iniziali delle 11 tipologie: - foreach (string tipologia in Tipologie) { - toDownload.Add(BASE_URL.Replace("{Tip}", tipologia)); - allLinks.Add(BASE_URL.Replace("{Tip}", tipologia)); - } - } catch (Exception ex) { - Logger.Log(new object[] { "Error Code: ER02220", "Errore in cA69Engine()", ex }); - throw ex; - } - state = EngineStatusEnum.StandBy; - } - - #endregion - - - #region Method: - - public bool Start(bool DownRac) { - if (state != EngineStatusEnum.StandBy) return false; - - dtStartTime = DateTime.Now; - downRac = DownRac; - - RaiseEvent_Updating("Inizio lo scrap dei link dei racconti"); - - state = EngineStatusEnum.Starting; - - if (toDownload.Count > 0) { - // proseguo con il download: - string lk = toDownload.First(); - toDownload.Remove(lk); - string fn = Path.Combine(Cfg.Settings.PathAnalizadedRaccFile, GetFileName(lk)); - using (MyWebClient client = new MyWebClient(lk, fn)) { - //client.Encoding = Encoding.ASCII; - client.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0"); - client.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"); - client.Headers.Add("Cookie", "euconsent-v2=CPG74goPG74goAKAeAENBcCsAP_AAH_AAAwIH3tf_X__bX9j-_59__t0eY1f9_7_v-wzjhfdt-8N2P_X_L0X42E7PF36pq4KuR4Eu3LBIQNlHOHUTUmw6okVrTPsak2Mr7NKJ7LEmnMZe2dYGHtfn91TuZKY7_78_9fz3_-v_v___9f3r-3_3__59X---_e_V399zLv9__3__9gfiASYal8AF2JY4Mk0aVQogQhWEh0AoAKKAYWiawgYXBTsrAI9QQMAEJqAjAiBBiCjFgEAAgEASERASAHggEQBEAgABACpAQgAImAQWAFgYBAAKAaFiBFAEIEhBkcFRymBARItFBPZWAJRd7GmEIZRYAUCj-iowEShBAsDISFg5gAA.YAAAAAAAAAAA"); - client.Headers.Add("DNT", "1"); - client.Headers.Add("TE", "Trailers"); - //client.DownloadFile(new Uri(lk), fn); - client.DownloadProgressChanged += Client_DownloadProgressChanged; - client.DownloadFileCompleted += Client_DownloadFileCompleted; - client.DownloadDataCompleted += Client_DownloadDataCompleted; - client.DownloadStringCompleted += Client_DownloadStringCompleted; - //client.DownloadFileAsync(new Uri(lk), fn); - //client.DownloadDataAsync(new Uri(lk)); - client.DownloadStringAsync(new Uri(lk)); - } - } - - return true; - } - - public bool Stop() { - switch (state) { - case EngineStatusEnum.NotReady: - case EngineStatusEnum.StandBy: - SaveState(); - state = EngineStatusEnum.Stopped; - return true; - case EngineStatusEnum.Starting: - case EngineStatusEnum.Started: - case EngineStatusEnum.Stopping: - state = EngineStatusEnum.Stopping; - return false; - case EngineStatusEnum.Stopped: - return true; - } - return true; - } - - public int ImportLinkFromFile(string SrcFilePath) { - int res = 0; - try { - if (File.Exists(SrcFilePath)) { - StreamReader sr = new StreamReader(SrcFilePath, Encoding.UTF8); - string row = sr.ReadLine().Trim(); - while (row != null) { - // verifico che non sia un commento: - if (!(row.StartsWith("#") || row.StartsWith("//"))) { - // Se link nuovo: - if (!allLinks.Contains(row)) { - // Link nuovo, mi comporto di conseguenza alla tipologia: - if (IsRacLink(row)) { - // Link ad un racconto - allLinks.Add(row); - toDownload.Add(row); - racLinks.Add(row); - res++; - } else { - if (IsIdxLink(row)) { - // Link ad un indice - allLinks.Add(row); - toDownload.Add(row); - res++; - } else { - // Link non riconosciuto, lo scarto - ; - } - } - } - } - //Read the next line - row = sr.ReadLine(); - } - } - - } catch (Exception ex) { - Logger.Log(new object[] { "Error Code: DB07403", "Errore in ImportLinkFromFile()", ex }); - if (res > 0) res = -1 * res; - } - return res; - } - - - #endregion - - - #region Eventi: - - /// - /// Evento che indica un cambiamento di stato nell'AnalizeManager - /// - public event EventHandler Updating; - - - #region Generazione degli eventi dell'engine verso le classi chiamanti: - - /// - /// Attiva l'evento che segnala un cambiamento di stato nel engine (per esempio, terminato il download di un singolo file) - /// - void RaiseEvent_Updating(string msg, int tot, int toDown, int down) { - EventHandler tmp = Updating; - if (null != tmp) tmp(this, new EngineEventArgs(msg, tot, toDown, down)); - } - - /// - /// Attiva l'evento che segnala un cambiamento di stato nel engine (per esempio, terminato il download di un singolo file) - /// - void RaiseEvent_Updating(string msg) { - EventHandler tmp = Updating; - if (null != tmp) tmp(this, new EngineEventArgs(msg, allLinks.Count, toDownload.Count, downloaded.Count)); - } - - #endregion - - - #endregion - - - int conteggioEvt = 0; - - - private void Wb_DocumentCompleted(object sender, WebBrowserDocumentCompletedEventArgs e) { - object obj = new Object(); - lock (obj) { - - conteggioEvt++; - - StringBuilder sb; - if (state == EngineStatusEnum.Starting) state = EngineStatusEnum.Started; - - WebBrowser wb = sender as WebBrowser; - string link = wb.Url.ToString(); - - // Se link di logon, devo terminare i download - if (link == "https://www.annunci69.it/accedi.php?nop") { - // Finito download: - state = EngineStatusEnum.Stopping; - // salvo gli archivi: - SaveState(); - return; - } - - // Elimino ulteriori ripetizioni dell'evento: - if (downloaded.Contains(link)) return; - - // aggiorno UI - RaiseEvent_Updating(wb.Name + " - (Evt" + conteggioEvt.ToString("0000") + ") Donwloaded: " + link.Replace("https://www.annunci69.it/racconti-erotici/", "./")); - - // Se pagina è un racconto, lo salvo - if (IsRacLink(link)) { - sb = new StringBuilder(); - sb.Append(wb.Document.Body.InnerHtml); - string fn = Path.Combine(Cfg.Settings.PathAnalizadedRaccFile, GetFileName(link)); - //cTools.WriteTextFile(fn, sb, Encoding.GetEncoding(wb.Document.Encoding)); - cTools.WriteTextFile(fn, sb); - } - - // Registro che ho eseguito il download del link - downloaded.Add(link); - - // Ciclo su tutti i link estratti dalla pagina: - foreach (HtmlElement item in wb.Document.Links) { - link = item.GetAttribute("href"); - // Se link nuovo: - if (!allLinks.Contains(link)) { - // elimino il link di logon - if (link == "https://www.annunci69.it/accedi.php?nop") break; - // Link nuovo, mi comporto di conseguenza alla tipologia: - if (IsRacLink(link)) { - // Link ad un racconto - allLinks.Add(link); - if (downRac) { toDownload.Add(link); } - racLinks.Add(link); - } else { - if (IsIdxLink(link)) { - // Link ad un indice - allLinks.Add(link); - toDownload.Add(link); - } else { - // Link non riconosciuto, lo scarto - ; - } - } - } - } // fine ciclo sui link estratti - - // Se ho raggiunto il massimo numero di link, o non vi sono link da downloadare, mi fermo: - if ((toDownload.Count > 0) && (state == EngineStatusEnum.Started)) { - // proseguo con il download: - string lk = toDownload.First(); - toDownload.Remove(lk); - wb.Navigate(lk); - } else { - // Finito download: - state = EngineStatusEnum.Stopping; - // salvo gli archivi: - SaveState(); - } - - }//lock - } - - - private void SaveState() { - int res = App.DB.SaveData(allLinks, toDownload, downloaded, racLinks); - - StringBuilder sb; - - // Finito download, salvo gli archivi: - string dt = DateTime.Now.ToString("yyyy-MM-dd_HH-mm-ss"); - - // Salvo archivio: AllLinks - sb = new StringBuilder(); - allLinks.Sort(); - foreach (string lnk in allLinks) { sb.AppendLine(lnk); } - cTools.WriteTextFile(Path.Combine(Cfg.Settings.PathAnalizadedIndexFile, string.Format("_A69_AllLinks_{0}.tes", dt)), sb); - - // Salvo archivio: toDownload - sb = new StringBuilder(); - toDownload.Sort(); - foreach (string lnk in toDownload) { sb.AppendLine(lnk); } - cTools.WriteTextFile(Path.Combine(Cfg.Settings.PathAnalizadedIndexFile, string.Format("_A69_toDownload_{0}.tes", dt)), sb); - - // Salvo archivio: Downloaded - sb = new StringBuilder(); - downloaded.Sort(); - foreach (string lnk in downloaded) { sb.AppendLine(lnk); } - cTools.WriteTextFile(Path.Combine(Cfg.Settings.PathAnalizadedIndexFile, string.Format("_A69_Downloaded_{0}.tes", dt)), sb); - - // Salvo archivio: RacLinks - sb = new StringBuilder(); - racLinks.Sort(); - foreach (string lnk in racLinks) { sb.AppendLine(lnk); } - cTools.WriteTextFile(Path.Combine(Cfg.Settings.PathAnalizadedIndexFile, string.Format("_A69_RacLinks_{0}.tes", dt)), sb); - - // avviso utente: - RaiseEvent_Updating(string.Format("Finito il download in {0}", DateTime.Now - dtStartTime)); - - state = EngineStatusEnum.Stopped; - - } - - - private string GetFileName(string url) { - if (IsRacLink(url)) { - return url.Replace("https://www.annunci69.it/racconti-erotici/", "A69_").Replace("/", "_").Replace(":", "_").Replace(@"\", "_"); - } else { - return url.Replace("https://www.annunci69.it/racconti-erotici/", "_A69_").Replace("/", "_").Replace(":", "_").Replace(@"\", "_"); - } - } - static private bool IsIdxLink(string link) { return link.StartsWith(BASE_LINK_IDX); } - static private bool IsRacLink(string link) { foreach (string tip in Tipologie) { if (link.StartsWith(BASE_LINK_RAC + tip + "/")) return true; } return false; } - - static public string LinkType(string link) { - if (IsIdxLink(link)) return "Idx"; - if (IsRacLink(link)) return "Rac"; - //return "Unknown"; - return "Unk"; - } - - - - - - - - private void Client_DownloadFileCompleted(object sender, AsyncCompletedEventArgs e) { - object obj = new Object(); - lock (obj) { - - conteggioEvt++; - - StringBuilder sb; - if (state == EngineStatusEnum.Starting) state = EngineStatusEnum.Started; - - WebClient wb = sender as WebClient; - string link = wb.BaseAddress; - - return; - /* - // Se link di logon, devo terminare i download - if (link == "https://www.annunci69.it/accedi.php?nop") { - // Finito download: - state = EngineStatusEnum.Stopping; - // salvo gli archivi: - SaveState(); - return; - } - - // Elimino ulteriori ripetizioni dell'evento: - if (downloaded.Contains(link)) return; - - // aggiorno UI - RaiseEvent_Updating(wb.Name + " - (Evt" + conteggioEvt.ToString("0000") + ") Donwloaded: " + link.Replace("https://www.annunci69.it/racconti-erotici/", "./")); - - // Se pagina è un racconto, lo salvo - if (IsRacLink(link)) { - sb = new StringBuilder(); - sb.Append(wb.Document.Body.InnerHtml); - string fn = Path.Combine(Cfg.Settings.PathAnalizadedRaccFile, GetFileName(link)); - //cTools.WriteTextFile(fn, sb, Encoding.GetEncoding(wb.Document.Encoding)); - cTools.WriteTextFile(fn, sb); - } - - // Registro che ho eseguito il download del link - downloaded.Add(link); - - // Ciclo su tutti i link estratti dalla pagina: - foreach (HtmlElement item in wb.Document.Links) { - link = item.GetAttribute("href"); - // Se link nuovo: - if (!allLinks.Contains(link)) { - // elimino il link di logon - if (link == "https://www.annunci69.it/accedi.php?nop") break; - // Link nuovo, mi comporto di conseguenza alla tipologia: - if (IsRacLink(link)) { - // Link ad un racconto - allLinks.Add(link); - if (downRac) { toDownload.Add(link); } - racLinks.Add(link); - } else { - if (IsIdxLink(link)) { - // Link ad un indice - allLinks.Add(link); - toDownload.Add(link); - } else { - // Link non riconosciuto, lo scarto - ; - } - } - } - } // fine ciclo sui link estratti - - // Se ho raggiunto il massimo numero di link, o non vi sono link da downloadare, mi fermo: - if ((toDownload.Count > 0) && (state == EngineStatusEnum.Started)) { - // proseguo con il download: - string lk = toDownload.First(); - toDownload.Remove(lk); - wb.Navigate(lk); - } else { - // Finito download: - state = EngineStatusEnum.Stopping; - // salvo gli archivi: - SaveState(); - } - - //*/ - }//lock - - } - private void Client_DownloadDataCompleted(object sender, DownloadDataCompletedEventArgs e) { - //System.Threading.AutoResetEvent waiter = (System.Threading.AutoResetEvent)e.UserState; - - try { - // If the request was not canceled and did not throw - // an exception, display the resource. - if (!e.Cancelled && e.Error == null) { - byte[] data = (byte[])e.Result; - string textData = System.Text.Encoding.Unicode.GetString(data); - string FullFileName = @"C:\VMS\Prj\C#\A69WD\Test\Txt\Src\2022-09-22\A69_altroz.html"; - - //System.IO.StreamWriter sw = new System.IO.StreamWriter(FullFileName, false, Encoding.Unicode); - //sw.Write(textData); - //sw.Flush(); - //sw.Close(); - - using (var fs = new FileStream(FullFileName, FileMode.Create, FileAccess.Write)) { - fs.Write(data, 0, data.Length); - } - } - } finally { - // Let the main application thread resume. - //waiter.Set(); - } - } - - private void Client_DownloadStringCompleted(object sender, DownloadStringCompletedEventArgs e) { - object obj = new Object(); - lock (obj) { - - conteggioEvt++; - - MyWebClient mwb = sender as MyWebClient; - if (state == EngineStatusEnum.Starting) state = EngineStatusEnum.Started; - - - // Verifico se download corretto - if (!e.Cancelled && e.Error == null) { - // aggiorno UI - RaiseEvent_Updating("(Evt" + conteggioEvt.ToString("0000") + ") Donwloaded: " + mwb.URL.Replace("https://www.annunci69.it/racconti-erotici/", "./")); - - // Se pagina è un racconto, lo salvo - if (IsRacLink(mwb.URL)) { - string textData = e.Result; - string FullFileName = mwb.FileName; - System.IO.StreamWriter sw = new System.IO.StreamWriter(FullFileName, false, Encoding.UTF8); - sw.Write(textData); - sw.Flush(); - sw.Close(); - } - - // Registro che ho eseguito il download del link - downloaded.Add(mwb.URL); - - // Estraggo tutti i link net testo: - List links = new List(); - //if (EstraiA69Link(e.Result, links)) { - if (EstraiLink(e.Result, "/racconti-erotici/", links)) { - - // Ciclo su tutti i link estratti dalla pagina: - foreach (string link in links) { - if (!allLinks.Contains(link)) { - // elimino il link di logon - if (link == "https://www.annunci69.it/accedi.php?nop") break; - // Link nuovo, mi comporto di conseguenza alla tipologia: - if (IsRacLink(link)) { - // Link ad un racconto - allLinks.Add(link); - if (downRac) { toDownload.Add(link); } - racLinks.Add(link); - } else { - if (IsIdxLink(link)) { - // Link ad un indice - allLinks.Add(link); - toDownload.Add(link); - } else { - // Link non riconosciuto, lo scarto - ; - } - } - } - } - } // fine ciclo sui link estratti - - } else { - // Download con errore, informo l'utente - RaiseEvent_Updating("(Evt" + conteggioEvt.ToString("0000") + ") Donwload error: " + mwb.URL.Replace("https://www.annunci69.it/racconti-erotici/", "./")); - } - - - // Se ho raggiunto il massimo numero di link, o non vi sono link da downloadare, mi fermo: - if ((toDownload.Count > 0) && (state == EngineStatusEnum.Started)) { - // proseguo con il download: - string lk = toDownload.First(); - toDownload.Remove(lk); - string fn = Path.Combine(Cfg.Settings.PathAnalizadedRaccFile, GetFileName(lk)); - using (MyWebClient client = new MyWebClient(lk, fn)) { - //client.Encoding = Encoding.ASCII; - client.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0"); - client.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"); - client.Headers.Add("Cookie", "euconsent-v2=CPG74goPG74goAKAeAENBcCsAP_AAH_AAAwIH3tf_X__bX9j-_59__t0eY1f9_7_v-wzjhfdt-8N2P_X_L0X42E7PF36pq4KuR4Eu3LBIQNlHOHUTUmw6okVrTPsak2Mr7NKJ7LEmnMZe2dYGHtfn91TuZKY7_78_9fz3_-v_v___9f3r-3_3__59X---_e_V399zLv9__3__9gfiASYal8AF2JY4Mk0aVQogQhWEh0AoAKKAYWiawgYXBTsrAI9QQMAEJqAjAiBBiCjFgEAAgEASERASAHggEQBEAgABACpAQgAImAQWAFgYBAAKAaFiBFAEIEhBkcFRymBARItFBPZWAJRd7GmEIZRYAUCj-iowEShBAsDISFg5gAA.YAAAAAAAAAAA"); - client.Headers.Add("DNT", "1"); - client.Headers.Add("TE", "Trailers"); - //client.DownloadFile(new Uri(lk), fn); - client.DownloadProgressChanged += Client_DownloadProgressChanged; - client.DownloadFileCompleted += Client_DownloadFileCompleted; - client.DownloadDataCompleted += Client_DownloadDataCompleted; - client.DownloadStringCompleted += Client_DownloadStringCompleted; - //client.DownloadFileAsync(new Uri(lk), fn); - //client.DownloadDataAsync(new Uri(lk)); - client.DownloadStringAsync(new Uri(lk)); - } - } else { - // Finito download: - state = EngineStatusEnum.Stopping; - // salvo gli archivi: - SaveState(); - } - - }//lock - - } - - private void Client_DownloadProgressChanged(object sender, DownloadProgressChangedEventArgs e) { - ////Update progress bar & label - //Invoke(new MethodInvoker(delegate () { - // progressBar.Minimum = 0; - // double receive = double.Parse(e.BytesReceived.ToString()); - // double total = double.Parse(e.TotalBytesToReceive.ToString()); - // double percentage = receive / total * 100; - // lblStatus.Text = $"Downloaded {string.Format("{0:0.##}", percentage)}%"; - // progressBar.Value = int.Parse(Math.Truncate(percentage).ToString()); - //})); - } - - - - - - - - - public static bool EstraiA69Link(string txt, List links) { - try { - Regex regex = new Regex("(?<=href=\")\\S+?(?=\")"); - Match matchObj = regex.Match(txt); - while (matchObj.Success) { - if (matchObj.Value.StartsWith("/racconti-erotici/")) { links.Add(matchObj.Value); } - if (matchObj.Groups.Count > 1) { - for (int i = 1; i < matchObj.Groups.Count; i++) { - Group group = matchObj.Groups[i]; - if (group.Success) { - if (group.Value.StartsWith("/racconti-erotici/")) { links.Add(group.Value); } - } - } - } - matchObj = matchObj.NextMatch(); - } - return true; - } catch { return false; } - } - - public static bool EstraiLink(string txt, string startWith, List links) { - try { - Regex regex = new Regex("(?<=href=\")\\S+?(?=\")"); - Match matchObj = regex.Match(txt); - while (matchObj.Success) { - if (matchObj.Value.StartsWith(startWith)) { links.Add(matchObj.Value.Replace(startWith, "https://www.annunci69.it/racconti-erotici/")); } - if (matchObj.Groups.Count > 1) { - for (int i = 1; i < matchObj.Groups.Count; i++) { - Group group = matchObj.Groups[i]; - if (group.Success) { - if (group.Value.StartsWith(startWith)) { links.Add(group.Value.Replace(startWith, "https://www.annunci69.it/racconti-erotici/")); } - } - } - } - matchObj = matchObj.NextMatch(); - } - return true; - } catch { return false; } - } - - - }//class cA69Engine3 - - public class MyWebClient : WebClient { - - public string URL; - public string FileName; - public MyWebClient(string url, string fileName) { - this.URL = url; - this.FileName = fileName; - } - - - } - -}//namespace A69WD diff --git a/Src/A69WD/Net/frmDownloader.Designer.cs b/Src/A69WD/Net/frmDownloader.Designer.cs index 892972d..a40faf3 100644 --- a/Src/A69WD/Net/frmDownloader.Designer.cs +++ b/Src/A69WD/Net/frmDownloader.Designer.cs @@ -27,14 +27,9 @@ this.btnHide = new System.Windows.Forms.Button(); this.btnDownloadAll = new System.Windows.Forms.Button(); this.lblST = new System.Windows.Forms.Label(); - this.wb1 = new System.Windows.Forms.WebBrowser(); this.btnImportLinks = new System.Windows.Forms.Button(); this.openFileDialog1 = new System.Windows.Forms.OpenFileDialog(); this.btnShowLog = new System.Windows.Forms.Button(); - this.btnDownloadAll2 = new System.Windows.Forms.Button(); - this.btnTest = new System.Windows.Forms.Button(); - this.button1 = new System.Windows.Forms.Button(); - this.button2 = new System.Windows.Forms.Button(); this.SuspendLayout(); // // btnClose @@ -78,18 +73,6 @@ this.lblST.Text = "Status"; this.lblST.TextAlign = System.Drawing.ContentAlignment.MiddleLeft; // - // wb1 - // - this.wb1.Location = new System.Drawing.Point(0, 1); - this.wb1.MinimumSize = new System.Drawing.Size(20, 20); - this.wb1.Name = "wb1"; - this.wb1.ScriptErrorsSuppressed = true; - this.wb1.ScrollBarsEnabled = false; - this.wb1.Size = new System.Drawing.Size(63, 44); - this.wb1.TabIndex = 12; - this.wb1.TabStop = false; - this.wb1.Visible = false; - // // btnImportLinks // this.btnImportLinks.Location = new System.Drawing.Point(115, 12); @@ -115,62 +98,17 @@ this.btnShowLog.UseVisualStyleBackColor = true; this.btnShowLog.Click += new System.EventHandler(this.btnShowLog_Click); // - // btnDownloadAll2 - // - this.btnDownloadAll2.Location = new System.Drawing.Point(45, 1); - this.btnDownloadAll2.Name = "btnDownloadAll2"; - this.btnDownloadAll2.Size = new System.Drawing.Size(97, 33); - this.btnDownloadAll2.TabIndex = 18; - this.btnDownloadAll2.Text = "Download All"; - this.btnDownloadAll2.UseVisualStyleBackColor = true; - this.btnDownloadAll2.Click += new System.EventHandler(this.btnDownloadAll2_Click); - // - // btnTest - // - this.btnTest.Location = new System.Drawing.Point(148, 1); - this.btnTest.Name = "btnTest"; - this.btnTest.Size = new System.Drawing.Size(97, 33); - this.btnTest.TabIndex = 19; - this.btnTest.Text = "Test WebClient"; - this.btnTest.UseVisualStyleBackColor = true; - this.btnTest.Click += new System.EventHandler(this.btnTest_Click); - // - // button1 - // - this.button1.Location = new System.Drawing.Point(251, 1); - this.button1.Name = "button1"; - this.button1.Size = new System.Drawing.Size(97, 33); - this.button1.TabIndex = 20; - this.button1.Text = "Test Engine2"; - this.button1.UseVisualStyleBackColor = true; - this.button1.Click += new System.EventHandler(this.button1_Click); - // - // button2 - // - this.button2.Location = new System.Drawing.Point(354, 1); - this.button2.Name = "button2"; - this.button2.Size = new System.Drawing.Size(97, 33); - this.button2.TabIndex = 21; - this.button2.Text = "Test Engine3"; - this.button2.UseVisualStyleBackColor = true; - this.button2.Click += new System.EventHandler(this.button2_Click); - // // frmDownloader // this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F); this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font; this.ClientSize = new System.Drawing.Size(524, 74); - this.Controls.Add(this.button2); - this.Controls.Add(this.button1); - this.Controls.Add(this.btnTest); - this.Controls.Add(this.btnDownloadAll2); this.Controls.Add(this.btnShowLog); this.Controls.Add(this.btnImportLinks); this.Controls.Add(this.btnClose); this.Controls.Add(this.btnHide); this.Controls.Add(this.btnDownloadAll); this.Controls.Add(this.lblST); - this.Controls.Add(this.wb1); this.FormBorderStyle = System.Windows.Forms.FormBorderStyle.FixedDialog; this.Name = "frmDownloader"; this.Text = "frmDownloader"; @@ -183,13 +121,8 @@ private System.Windows.Forms.Button btnHide; private System.Windows.Forms.Button btnDownloadAll; private System.Windows.Forms.Label lblST; - private System.Windows.Forms.WebBrowser wb1; private System.Windows.Forms.Button btnImportLinks; private System.Windows.Forms.OpenFileDialog openFileDialog1; private System.Windows.Forms.Button btnShowLog; - private System.Windows.Forms.Button btnDownloadAll2; - private System.Windows.Forms.Button btnTest; - private System.Windows.Forms.Button button1; - private System.Windows.Forms.Button button2; } } \ No newline at end of file diff --git a/Src/A69WD/Net/frmDownloader.cs b/Src/A69WD/Net/frmDownloader.cs index 82006e9..48c8fc1 100644 --- a/Src/A69WD/Net/frmDownloader.cs +++ b/Src/A69WD/Net/frmDownloader.cs @@ -21,17 +21,11 @@ namespace A69WD { public frmDownloader() { InitializeComponent(); this.Text = String.Format("A69WD - Dowloader Testi Vrs {0}", cTools.AssemblyVersion); - engine = new cA69Engine(this.wb1); + engine = new cA69Engine(); engine.Updating += Engine_Updating; - engine2 = new cA69Engine2(); - engine2.Updating += Engine_Updating; - engine3 = new cA69Engine3(); - engine3.Updating += Engine_Updating; } cA69Engine engine = null; - cA69Engine2 engine2 = null; - cA69Engine3 engine3 = null; /// @@ -95,108 +89,6 @@ namespace A69WD { #endregion - private void btnDownloadAll2_Click(object sender, EventArgs e) { engine.Start(false); } - - - - - #region Test WEBCLIENT - - private void btnTest_Click(object sender, EventArgs e) { - ////Run download file with multiple thread - //string url = "https://www.annunci69.it/racconti-erotici/racconti_lista.php?genere=altro&pagina=103"; - //if (!string.IsNullOrEmpty(url)) { - // Thread thread = new Thread(() => - // { - // Uri uri = new Uri(url); - // //string fileName = System.IO.Path.GetFileName(uri.AbsolutePath); - // string fileName = System.IO.Path.Combine(Cfg.Settings.PathAnalizadedRaccFile, url.Replace("https://www.annunci69.it/racconti-erotici/", "A69_").Replace("/", "_").Replace(":", "_").Replace(@"\", "_")); - // client.DownloadFileAsync(uri, @"C:\VMS\Prj\C#\A69WD\Test\Txt\Src\_A69_genere_altro_103.html"); - // }); - // thread.Start(); - //} - - string url = "https://www.annunci69.it/racconti-erotici/racconti_lista.php?genere=altro&pagina=98"; - string fileName = @"C:\VMS\Prj\C#\A69WD\Test\Logs\_A69_genere_altro_98.html"; - - //// Test 1: Questo dà errore: - //WebClient client; - //client = new WebClient(); - //client.DownloadProgressChanged += Client_DownloadProgressChanged; - //client.DownloadFileCompleted += Client_DownloadFileCompleted; - ////client.DownloadFile(url, fileName); - //StringBuilder sb = new StringBuilder(client.DownloadString(url)); - //cTools.WriteTextFile(fileName, sb); - //// Test 1 - - // // Test 2: OK - // WebClient client = new WebClient(); - // //client.Headers["User-Agent"] = "User-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36"; - // client.Encoding = Encoding.ASCII; - // client.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0"); - // client.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"); - // //client.Headers.Add("Accept-Encoding", "gzip, deflate, br"); - // //client.Headers.Add("Connection", "keep-alive"); - // client.Headers.Add("Cookie", "euconsent-v2=CPG74goPG74goAKAeAENBcCsAP_AAH_AAAwIH3tf_X__bX9j-_59__t0eY1f9_7_v-wzjhfdt-8N2P_X_L0X42E7PF36pq4KuR4Eu3LBIQNlHOHUTUmw6okVrTPsak2Mr7NKJ7LEmnMZe2dYGHtfn91TuZKY7_78_9fz3_-v_v___9f3r-3_3__59X---_e_V399zLv9__3__9gfiASYal8AF2JY4Mk0aVQogQhWEh0AoAKKAYWiawgYXBTsrAI9QQMAEJqAjAiBBiCjFgEAAgEASERASAHggEQBEAgABACpAQgAImAQWAFgYBAAKAaFiBFAEIEhBkcFRymBARItFBPZWAJRd7GmEIZRYAUCj-iowEShBAsDISFg5gAA.YAAAAAAAAAAA"); - // client.Headers.Add("DNT", "1"); - // client.Headers.Add("TE", "Trailers"); - // client.DownloadFile(new Uri(url), fileName); - // // Test 2: OK - - // Test 3: OK - using (WebClient client = new WebClient()) { - for (int i = 90; i < 110; i++) { - url = "https://www.annunci69.it/racconti-erotici/racconti_lista.php?genere=altro&pagina=" + i.ToString(); - fileName = @"C:\VMS\Prj\C#\A69WD\Test\Logs\_A69_genere_altro_" + i.ToString() + ".html"; - client.Encoding = Encoding.ASCII; - client.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0"); - client.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"); - client.Headers.Add("Cookie", "euconsent-v2=CPG74goPG74goAKAeAENBcCsAP_AAH_AAAwIH3tf_X__bX9j-_59__t0eY1f9_7_v-wzjhfdt-8N2P_X_L0X42E7PF36pq4KuR4Eu3LBIQNlHOHUTUmw6okVrTPsak2Mr7NKJ7LEmnMZe2dYGHtfn91TuZKY7_78_9fz3_-v_v___9f3r-3_3__59X---_e_V399zLv9__3__9gfiASYal8AF2JY4Mk0aVQogQhWEh0AoAKKAYWiawgYXBTsrAI9QQMAEJqAjAiBBiCjFgEAAgEASERASAHggEQBEAgABACpAQgAImAQWAFgYBAAKAaFiBFAEIEhBkcFRymBARItFBPZWAJRd7GmEIZRYAUCj-iowEShBAsDISFg5gAA.YAAAAAAAAAAA"); - client.Headers.Add("DNT", "1"); - client.Headers.Add("TE", "Trailers"); - client.DownloadFile(new Uri(url), fileName); - } - } - // Test 3: OK - - - } - - private void Client_DownloadFileCompleted(object sender, AsyncCompletedEventArgs e) { - MessageBox.Show("Download complete !", "Message", MessageBoxButtons.OK, MessageBoxIcon.Information); - } - - private void Client_DownloadProgressChanged(object sender, DownloadProgressChangedEventArgs e) { - ////Update progress bar & label - //Invoke(new MethodInvoker(delegate () { - // progressBar.Minimum = 0; - // double receive = double.Parse(e.BytesReceived.ToString()); - // double total = double.Parse(e.TotalBytesToReceive.ToString()); - // double percentage = receive / total * 100; - // lblStatus.Text = $"Downloaded {string.Format("{0:0.##}", percentage)}%"; - // progressBar.Value = int.Parse(Math.Truncate(percentage).ToString()); - //})); - } - - - - #endregion - - - #region Test ENGINE2 - - private void button1_Click(object sender, EventArgs e) { - engine2.Start(true); - } - - - - - #endregion - - private void button2_Click(object sender, EventArgs e) { - engine3.Start(true); - } diff --git a/Src/A69WD/Properties/AssemblyInfo.cs b/Src/A69WD/Properties/AssemblyInfo.cs index 9876639..8bc6c3d 100644 --- a/Src/A69WD/Properties/AssemblyInfo.cs +++ b/Src/A69WD/Properties/AssemblyInfo.cs @@ -36,6 +36,16 @@ using System.Runtime.InteropServices; [assembly: AssemblyFileVersion("0.2.0.2")] +/* + * + * Vrs 0.3.0.0 + * + * Compilata da CSiCoAlm il 2022/09/22 + * - Versione in cui ho sostituito l'oggetto WebBrowser con la classe WebClient per poter gestire in parallelo i download. + * + */ + + /* * * Vrs 0.2.0.2