MrkPi/A69WD
3
0 Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

- Autocommit - 2022/09/22 11:48:33

Browse Source
This commit is contained in:
CSiCoAlm committed 2022-09-22 11:48:33 +02:00
1 parent c3355d8bf3
commit 2112a5050c
7 files changed
+176 -1437
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Src/A69WD/A69WD.csproj
-2
View File
@@ -63,8 +63,6 @@
<Compile Include="Net\ATLink.cs" />
<Compile Include="Net\ATLinkState.cs" />
<Compile Include="Net\cA69Engine.cs" />
<Compile Include="Net\cA69Engine2.cs" />
<Compile Include="Net\cA69Engine3.cs" />
<Compile Include="Net\DownloadClient.cs" />
<Compile Include="Net\frmDownloader.cs">
<SubType>Form</SubType>
Src/A69WD/Net/cA69Engine.cs
+165 -98
View File
@@ -4,14 +4,15 @@ using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Windows.Forms;
using System.Net;
using System.Text.RegularExpressions;
namespace A69WD {
public class cA69Engine {
#region Property: State, NumDownload, Tipologie, BASE_URL, BASE_LINK_IDX, BASE_LINK_RAC
#region Property: State, NumDownload, Tipologie, BASE_URL, BASE_LINK_IDX, BASE_LINK_RAC, conteggioEvt
public EngineStatusEnum State { get { return state; } }
@@ -23,10 +24,12 @@ namespace A69WD {
const string BASE_LINK_IDX = "https://www.annunci69.it/racconti-erotici/racconti_lista.php";
const string BASE_LINK_RAC = "https://www.annunci69.it/racconti-erotici/";
int conteggioEvt = 0;
#endregion
#region Fields: allLinks, toDownload, downloaded, racLinks, dtStartTime, downRac, state, wb
#region Fields: allLinks, toDownload, downloaded, racLinks, dtStartTime, downRac, state
List<string> allLinks = null;
List<string> toDownload = null;
@@ -39,14 +42,12 @@ namespace A69WD {
EngineStatusEnum state = EngineStatusEnum.NotReady;
WebBrowser wb = null;
#endregion
#region Costruttore e distruttore
public cA69Engine(WebBrowser WB) {
public cA69Engine() {
try {
// Inizializzo gli archivi (dai file in memoria)
allLinks = new List<string>();
@@ -54,14 +55,15 @@ namespace A69WD {
downloaded = new List<string>();
racLinks = new List<string>();
int res = App.DB.LoadData(allLinks, toDownload, downloaded, racLinks);
RaiseEvent_Updating("Ricaricati " + res.ToString("000") + " link di racconti da scaricare");
wb = WB;
wb.DocumentCompleted += Wb_DocumentCompleted;
// Per ora non serializzo gli archivi e quindi carico le pagine iniziali delle 11 tipologie:
foreach (string tipologia in Tipologie) {
toDownload.Add(BASE_URL.Replace("{Tip}", tipologia));
allLinks.Add(BASE_URL.Replace("{Tip}", tipologia));
}
} catch (Exception ex) {
Logger.Log(new object[] { "Error Code: ER02220", "Errore in cA69Engine()", ex });
throw ex;
@@ -72,7 +74,7 @@ namespace A69WD {
#endregion
#region Method:
#region Method: Start, Stop, ImportLinkFromFile
public bool Start(bool DownRac) {
if (state != EngineStatusEnum.StandBy) return false;
@@ -88,7 +90,23 @@ namespace A69WD {
// proseguo con il download:
string lk = toDownload.First();
toDownload.Remove(lk);
wb.Navigate(lk);
string fn = Path.Combine(Cfg.Settings.PathAnalizadedRaccFile, GetFileName(lk));
using (MyWebClient client = new MyWebClient(lk, fn)) {
//client.Encoding = Encoding.ASCII;
client.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0");
client.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
client.Headers.Add("Cookie", "euconsent-v2=CPG74goPG74goAKAeAENBcCsAP_AAH_AAAwIH3tf_X__bX9j-_59__t0eY1f9_7_v-wzjhfdt-8N2P_X_L0X42E7PF36pq4KuR4Eu3LBIQNlHOHUTUmw6okVrTPsak2Mr7NKJ7LEmnMZe2dYGHtfn91TuZKY7_78_9fz3_-v_v___9f3r-3_3__59X---_e_V399zLv9__3__9gfiASYal8AF2JY4Mk0aVQogQhWEh0AoAKKAYWiawgYXBTsrAI9QQMAEJqAjAiBBiCjFgEAAgEASERASAHggEQBEAgABACpAQgAImAQWAFgYBAAKAaFiBFAEIEhBkcFRymBARItFBPZWAJRd7GmEIZRYAUCj-iowEShBAsDISFg5gAA.YAAAAAAAAAAA");
client.Headers.Add("DNT", "1");
client.Headers.Add("TE", "Trailers");
////client.DownloadFile(new Uri(lk), fn);
//client.DownloadProgressChanged += Client_DownloadProgressChanged;
//client.DownloadFileCompleted += Client_DownloadFileCompleted;
//client.DownloadDataCompleted += Client_DownloadDataCompleted;
//client.DownloadStringCompleted += Client_DownloadStringCompleted;
////client.DownloadFileAsync(new Uri(lk), fn);
////client.DownloadDataAsync(new Uri(lk));
client.DownloadStringAsync(new Uri(lk));
}
}
return true;
@@ -159,7 +177,7 @@ namespace A69WD {
#endregion
#region Eventi:
#region Eventi: Updating, Client_DownloadStringCompleted
/// <summary>
/// Evento che indica un cambiamento di stato nell'AnalizeManager
@@ -167,6 +185,98 @@ namespace A69WD {
public event EventHandler<EngineEventArgs> Updating;
private void Client_DownloadStringCompleted(object sender, DownloadStringCompletedEventArgs e) {
object obj = new Object();
lock (obj) {
conteggioEvt++;
MyWebClient mwb = sender as MyWebClient;
if (state == EngineStatusEnum.Starting) state = EngineStatusEnum.Started;
// Verifico se download corretto
if (!e.Cancelled && e.Error == null) {
// aggiorno UI
RaiseEvent_Updating("(Evt" + conteggioEvt.ToString("0000") + ") Donwloaded: " + mwb.URL.Replace("https://www.annunci69.it/racconti-erotici/", "./"));
// Se pagina è un racconto, lo salvo
if (IsRacLink(mwb.URL)) {
string textData = e.Result;
string FullFileName = mwb.FileName;
System.IO.StreamWriter sw = new System.IO.StreamWriter(FullFileName, false, Encoding.UTF8);
sw.Write(textData);
sw.Flush();
sw.Close();
}
// Registro che ho eseguito il download del link
downloaded.Add(mwb.URL);
// Estraggo tutti i link net testo:
List<string> links = new List<string>();
//if (EstraiA69Link(e.Result, links)) {
if (EstraiLink(e.Result, "/racconti-erotici/", links)) {
// Ciclo su tutti i link estratti dalla pagina:
foreach (string link in links) {
if (!allLinks.Contains(link)) {
// elimino il link di logon
if (link == "https://www.annunci69.it/accedi.php?nop") break;
// Link nuovo, mi comporto di conseguenza alla tipologia:
if (IsRacLink(link)) {
// Link ad un racconto
allLinks.Add(link);
if (downRac) { toDownload.Add(link); }
racLinks.Add(link);
} else {
if (IsIdxLink(link)) {
// Link ad un indice
allLinks.Add(link);
toDownload.Add(link);
} else {
// Link non riconosciuto, lo scarto
;
}
}
}
}
} // fine ciclo sui link estratti
} else {
// Download con errore, informo l'utente
RaiseEvent_Updating("(Evt" + conteggioEvt.ToString("0000") + ") Donwload error: " + mwb.URL.Replace("https://www.annunci69.it/racconti-erotici/", "./"));
}
// Se ho raggiunto il massimo numero di link, o non vi sono link da downloadare, mi fermo:
if ((toDownload.Count > 0) && (state == EngineStatusEnum.Started)) {
// proseguo con il download:
string lk = toDownload.First();
toDownload.Remove(lk);
string fn = Path.Combine(Cfg.Settings.PathAnalizadedRaccFile, GetFileName(lk));
using (MyWebClient client = new MyWebClient(lk, fn)) {
client.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0");
client.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
client.Headers.Add("Cookie", "euconsent-v2=CPG74goPG74goAKAeAENBcCsAP_AAH_AAAwIH3tf_X__bX9j-_59__t0eY1f9_7_v-wzjhfdt-8N2P_X_L0X42E7PF36pq4KuR4Eu3LBIQNlHOHUTUmw6okVrTPsak2Mr7NKJ7LEmnMZe2dYGHtfn91TuZKY7_78_9fz3_-v_v___9f3r-3_3__59X---_e_V399zLv9__3__9gfiASYal8AF2JY4Mk0aVQogQhWEh0AoAKKAYWiawgYXBTsrAI9QQMAEJqAjAiBBiCjFgEAAgEASERASAHggEQBEAgABACpAQgAImAQWAFgYBAAKAaFiBFAEIEhBkcFRymBARItFBPZWAJRd7GmEIZRYAUCj-iowEShBAsDISFg5gAA.YAAAAAAAAAAA");
client.Headers.Add("DNT", "1");
client.Headers.Add("TE", "Trailers");
client.DownloadStringCompleted += Client_DownloadStringCompleted;
client.DownloadStringAsync(new Uri(lk));
}
} else {
// Finito download:
state = EngineStatusEnum.Stopping;
// salvo gli archivi:
SaveState();
}
}//lock
}
#region Generazione degli eventi dell'engine verso le classi chiamanti:
/// <summary>
@@ -191,94 +301,11 @@ namespace A69WD {
#endregion
int conteggioEvt = 0;
private void Wb_DocumentCompleted(object sender, WebBrowserDocumentCompletedEventArgs e) {
object obj = new Object();
lock (obj) {
conteggioEvt++;
StringBuilder sb;
if (state == EngineStatusEnum.Starting) state = EngineStatusEnum.Started;
WebBrowser wb = sender as WebBrowser;
string link = wb.Url.ToString();
// Se link di logon, devo terminare i download
if (link== "https://www.annunci69.it/accedi.php?nop") {
// Finito download:
state = EngineStatusEnum.Stopping;
// salvo gli archivi:
SaveState();
return;
}
// Elimino ulteriori ripetizioni dell'evento:
if (downloaded.Contains(link)) return;
// aggiorno UI
RaiseEvent_Updating(wb.Name + " - (Evt" + conteggioEvt.ToString("0000") + ") Donwloaded: " + link.Replace("https://www.annunci69.it/racconti-erotici/","./"));
// Se pagina è un racconto, lo salvo
if (IsRacLink(link)) {
sb = new StringBuilder();
sb.Append(wb.Document.Body.InnerHtml);
string fn = Path.Combine(Cfg.Settings.PathAnalizadedRaccFile, GetFileName(link));
//cTools.WriteTextFile(fn, sb, Encoding.GetEncoding(wb.Document.Encoding));
cTools.WriteTextFile(fn, sb);
}
// Registro che ho eseguito il download del link
downloaded.Add(link);
// Ciclo su tutti i link estratti dalla pagina:
foreach (HtmlElement item in wb.Document.Links) {
link = item.GetAttribute("href");
// Se link nuovo:
if (!allLinks.Contains(link)) {
// elimino il link di logon
if (link == "https://www.annunci69.it/accedi.php?nop") break;
// Link nuovo, mi comporto di conseguenza alla tipologia:
if (IsRacLink(link)) {
// Link ad un racconto
allLinks.Add(link);
if (downRac) { toDownload.Add(link); }
racLinks.Add(link);
} else {
if (IsIdxLink(link)) {
// Link ad un indice
allLinks.Add(link);
toDownload.Add(link);
} else {
// Link non riconosciuto, lo scarto
;
}
}
}
} // fine ciclo sui link estratti
// Se ho raggiunto il massimo numero di link, o non vi sono link da downloadare, mi fermo:
if ((toDownload.Count > 0) && (state==EngineStatusEnum.Started)) {
// proseguo con il download:
string lk = toDownload.First();
toDownload.Remove(lk);
wb.Navigate(lk);
} else {
// Finito download:
state = EngineStatusEnum.Stopping;
// salvo gli archivi:
SaveState();
}
}//lock
}
#region Internal Method: SaveState, GetFileName, IsIdxLink, IsRacLink, LinkType, EstraiLink
private void SaveState() {
int res = App.DB.SaveData(allLinks, toDownload, downloaded, racLinks);
StringBuilder sb;
// Finito download, salvo gli archivi:
@@ -315,11 +342,16 @@ namespace A69WD {
}
private string GetFileName(string url) {
if (IsRacLink(url)) {
return url.Replace("https://www.annunci69.it/racconti-erotici/", "A69_").Replace("/", "_").Replace(":", "_").Replace(@"\", "_");
} else {
return url.Replace("https://www.annunci69.it/racconti-erotici/", "_A69_").Replace("/", "_").Replace(":", "_").Replace(@"\", "_");
}
}
private string GetFileName(string url) { return url.Replace("https://www.annunci69.it/racconti-erotici/", "A69_").Replace("/", "_").Replace(":", "_").Replace(@"\", "_"); }
static private bool IsIdxLink(string link) { return link.StartsWith(BASE_LINK_IDX); }
static private bool IsRacLink(string link) { foreach (string tip in Tipologie) { if (link.StartsWith(BASE_LINK_RAC + tip + "/")) return true; } return false; }
static public string LinkType(string link) {
if (IsIdxLink(link)) return "Idx";
if (IsRacLink(link)) return "Rac";
@@ -327,6 +359,29 @@ namespace A69WD {
return "Unk";
}
public static bool EstraiLink(string txt, string startWith, List<string> links) {
try {
Regex regex = new Regex("(?<=href=\")\\S+?(?=\")");
Match matchObj = regex.Match(txt);
while (matchObj.Success) {
if (matchObj.Value.StartsWith(startWith)) { links.Add(matchObj.Value.Replace(startWith, "https://www.annunci69.it/racconti-erotici/")); }
if (matchObj.Groups.Count > 1) {
for (int i = 1; i < matchObj.Groups.Count; i++) {
Group group = matchObj.Groups[i];
if (group.Success) {
if (group.Value.StartsWith(startWith)) { links.Add(group.Value.Replace(startWith, "https://www.annunci69.it/racconti-erotici/")); }
}
}
}
matchObj = matchObj.NextMatch();
}
return true;
} catch { return false; }
}
#endregion
}//class cA69Engine
@@ -383,5 +438,17 @@ namespace A69WD {
} // class EngineEventArgs
public class MyWebClient : WebClient {
public string URL;
public string FileName;
public MyWebClient(string url, string fileName) {
this.URL = url;
this.FileName = fileName;
}
} // class MyWebClient
}//namespace A69WD
Src/A69WD/Net/cA69Engine2.cs
-519
View File
@@ -1,519 +0,0 @@
using System;
using System.IO;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Windows.Forms;
using System.Security.Policy;
using System.Net;
using static System.Windows.Forms.LinkLabel;
using System.ComponentModel;
using static System.Net.Mime.MediaTypeNames;
using System.Collections;
namespace A69WD {
public class cA69Engine2 {
#region Property: State, NumDownload, Tipologie, BASE_URL, BASE_LINK_IDX, BASE_LINK_RAC
public EngineStatusEnum State { get { return state; } }
public int NumDownload { get { return downloaded.Count; } }
static string[] Tipologie = { "altro", "bdsm", "gay", "incesto", "lesbo", "orge", "prime", "scambio", "tradimenti", "trans", "trio" };
const string BASE_URL = "https://www.annunci69.it/racconti-erotici/{Tip}.html";
const string BASE_LINK_IDX = "https://www.annunci69.it/racconti-erotici/racconti_lista.php";
const string BASE_LINK_RAC = "https://www.annunci69.it/racconti-erotici/";
#endregion
#region Fields: allLinks, toDownload, downloaded, racLinks, dtStartTime, downRac, state
List<string> allLinks = null;
List<string> toDownload = null;
List<string> downloaded = null;
List<string> racLinks = null;
DateTime dtStartTime;
bool downRac = false;
EngineStatusEnum state = EngineStatusEnum.NotReady;
#endregion
#region Costruttore e distruttore
public cA69Engine2() {
try {
// Inizializzo gli archivi (dai file in memoria)
allLinks = new List<string>();
toDownload = new List<string>();
downloaded = new List<string>();
racLinks = new List<string>();
int res = App.DB.LoadData(allLinks, toDownload, downloaded, racLinks);
RaiseEvent_Updating("Ricaricati " + res.ToString("000") + " link di racconti da scaricare");
// Per ora non serializzo gli archivi e quindi carico le pagine iniziali delle 11 tipologie:
foreach (string tipologia in Tipologie) {
toDownload.Add(BASE_URL.Replace("{Tip}", tipologia));
allLinks.Add(BASE_URL.Replace("{Tip}", tipologia));
}
} catch (Exception ex) {
Logger.Log(new object[] { "Error Code: ER02220", "Errore in cA69Engine()", ex });
throw ex;
}
state = EngineStatusEnum.StandBy;
}
#endregion
#region Method:
public bool Start(bool DownRac) {
if (state != EngineStatusEnum.StandBy) return false;
dtStartTime = DateTime.Now;
downRac = DownRac;
RaiseEvent_Updating("Inizio lo scrap dei link dei racconti");
state = EngineStatusEnum.Starting;
if (toDownload.Count > 0) {
// proseguo con il download:
string lk = toDownload.First();
toDownload.Remove(lk);
using (WebClient client = new WebClient()) {
string fn = Path.Combine(Cfg.Settings.PathAnalizadedRaccFile, GetFileName(lk));
//client.Encoding = Encoding.ASCII;
client.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0");
client.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
client.Headers.Add("Cookie", "euconsent-v2=CPG74goPG74goAKAeAENBcCsAP_AAH_AAAwIH3tf_X__bX9j-_59__t0eY1f9_7_v-wzjhfdt-8N2P_X_L0X42E7PF36pq4KuR4Eu3LBIQNlHOHUTUmw6okVrTPsak2Mr7NKJ7LEmnMZe2dYGHtfn91TuZKY7_78_9fz3_-v_v___9f3r-3_3__59X---_e_V399zLv9__3__9gfiASYal8AF2JY4Mk0aVQogQhWEh0AoAKKAYWiawgYXBTsrAI9QQMAEJqAjAiBBiCjFgEAAgEASERASAHggEQBEAgABACpAQgAImAQWAFgYBAAKAaFiBFAEIEhBkcFRymBARItFBPZWAJRd7GmEIZRYAUCj-iowEShBAsDISFg5gAA.YAAAAAAAAAAA");
client.Headers.Add("DNT", "1");
client.Headers.Add("TE", "Trailers");
//client.DownloadFile(new Uri(lk), fn);
client.DownloadProgressChanged += Client_DownloadProgressChanged;
client.DownloadFileCompleted += Client_DownloadFileCompleted;
client.DownloadDataCompleted += Client_DownloadDataCompleted;
client.DownloadStringCompleted += Client_DownloadStringCompleted;
//client.DownloadFileAsync(new Uri(lk), fn);
//client.DownloadDataAsync(new Uri(lk));
client.DownloadStringAsync(new Uri(lk));
}
}
return true;
}
public bool Stop() {
switch (state) {
case EngineStatusEnum.NotReady:
case EngineStatusEnum.StandBy:
SaveState();
state = EngineStatusEnum.Stopped;
return true;
case EngineStatusEnum.Starting:
case EngineStatusEnum.Started:
case EngineStatusEnum.Stopping:
state = EngineStatusEnum.Stopping;
return false;
case EngineStatusEnum.Stopped:
return true;
}
return true;
}
public int ImportLinkFromFile(string SrcFilePath) {
int res = 0;
try {
if (File.Exists(SrcFilePath)) {
StreamReader sr = new StreamReader(SrcFilePath, Encoding.UTF8);
string row = sr.ReadLine().Trim();
while (row != null) {
// verifico che non sia un commento:
if (!(row.StartsWith("#") || row.StartsWith("//"))) {
// Se link nuovo:
if (!allLinks.Contains(row)) {
// Link nuovo, mi comporto di conseguenza alla tipologia:
if (IsRacLink(row)) {
// Link ad un racconto
allLinks.Add(row);
toDownload.Add(row);
racLinks.Add(row);
res++;
} else {
if (IsIdxLink(row)) {
// Link ad un indice
allLinks.Add(row);
toDownload.Add(row);
res++;
} else {
// Link non riconosciuto, lo scarto
;
}
}
}
}
//Read the next line
row = sr.ReadLine();
}
}
} catch (Exception ex) {
Logger.Log(new object[] { "Error Code: DB07403", "Errore in ImportLinkFromFile()", ex });
if (res > 0) res = -1 * res;
}
return res;
}
#endregion
#region Eventi:
/// <summary>
/// Evento che indica un cambiamento di stato nell'AnalizeManager
/// </summary>
public event EventHandler<EngineEventArgs> Updating;
#region Generazione degli eventi dell'engine verso le classi chiamanti:
/// <summary>
/// Attiva l'evento che segnala un cambiamento di stato nel engine (per esempio, terminato il download di un singolo file)
/// </summary>
void RaiseEvent_Updating(string msg, int tot, int toDown, int down) {
EventHandler<EngineEventArgs> tmp = Updating;
if (null != tmp) tmp(this, new EngineEventArgs(msg, tot, toDown, down));
}
/// <summary>
/// Attiva l'evento che segnala un cambiamento di stato nel engine (per esempio, terminato il download di un singolo file)
/// </summary>
void RaiseEvent_Updating(string msg) {
EventHandler<EngineEventArgs> tmp = Updating;
if (null != tmp) tmp(this, new EngineEventArgs(msg, allLinks.Count, toDownload.Count, downloaded.Count));
}
#endregion
#endregion
int conteggioEvt = 0;
private void Wb_DocumentCompleted(object sender, WebBrowserDocumentCompletedEventArgs e) {
object obj = new Object();
lock (obj) {
conteggioEvt++;
StringBuilder sb;
if (state == EngineStatusEnum.Starting) state = EngineStatusEnum.Started;
WebBrowser wb = sender as WebBrowser;
string link = wb.Url.ToString();
// Se link di logon, devo terminare i download
if (link == "https://www.annunci69.it/accedi.php?nop") {
// Finito download:
state = EngineStatusEnum.Stopping;
// salvo gli archivi:
SaveState();
return;
}
// Elimino ulteriori ripetizioni dell'evento:
if (downloaded.Contains(link)) return;
// aggiorno UI
RaiseEvent_Updating(wb.Name + " - (Evt" + conteggioEvt.ToString("0000") + ") Donwloaded: " + link.Replace("https://www.annunci69.it/racconti-erotici/", "./"));
// Se pagina è un racconto, lo salvo
if (IsRacLink(link)) {
sb = new StringBuilder();
sb.Append(wb.Document.Body.InnerHtml);
string fn = Path.Combine(Cfg.Settings.PathAnalizadedRaccFile, GetFileName(link));
//cTools.WriteTextFile(fn, sb, Encoding.GetEncoding(wb.Document.Encoding));
cTools.WriteTextFile(fn, sb);
}
// Registro che ho eseguito il download del link
downloaded.Add(link);
// Ciclo su tutti i link estratti dalla pagina:
foreach (HtmlElement item in wb.Document.Links) {
link = item.GetAttribute("href");
// Se link nuovo:
if (!allLinks.Contains(link)) {
// elimino il link di logon
if (link == "https://www.annunci69.it/accedi.php?nop") break;
// Link nuovo, mi comporto di conseguenza alla tipologia:
if (IsRacLink(link)) {
// Link ad un racconto
allLinks.Add(link);
if (downRac) { toDownload.Add(link); }
racLinks.Add(link);
} else {
if (IsIdxLink(link)) {
// Link ad un indice
allLinks.Add(link);
toDownload.Add(link);
} else {
// Link non riconosciuto, lo scarto
;
}
}
}
} // fine ciclo sui link estratti
// Se ho raggiunto il massimo numero di link, o non vi sono link da downloadare, mi fermo:
if ((toDownload.Count > 0) && (state == EngineStatusEnum.Started)) {
// proseguo con il download:
string lk = toDownload.First();
toDownload.Remove(lk);
wb.Navigate(lk);
} else {
// Finito download:
state = EngineStatusEnum.Stopping;
// salvo gli archivi:
SaveState();
}
}//lock
}
private void SaveState() {
int res = App.DB.SaveData(allLinks, toDownload, downloaded, racLinks);
StringBuilder sb;
// Finito download, salvo gli archivi:
string dt = DateTime.Now.ToString("yyyy-MM-dd_HH-mm-ss");
// Salvo archivio: AllLinks
sb = new StringBuilder();
allLinks.Sort();
foreach (string lnk in allLinks) { sb.AppendLine(lnk); }
cTools.WriteTextFile(Path.Combine(Cfg.Settings.PathAnalizadedIndexFile, string.Format("_A69_AllLinks_{0}.tes", dt)), sb);
// Salvo archivio: toDownload
sb = new StringBuilder();
toDownload.Sort();
foreach (string lnk in toDownload) { sb.AppendLine(lnk); }
cTools.WriteTextFile(Path.Combine(Cfg.Settings.PathAnalizadedIndexFile, string.Format("_A69_toDownload_{0}.tes", dt)), sb);
// Salvo archivio: Downloaded
sb = new StringBuilder();
downloaded.Sort();
foreach (string lnk in downloaded) { sb.AppendLine(lnk); }
cTools.WriteTextFile(Path.Combine(Cfg.Settings.PathAnalizadedIndexFile, string.Format("_A69_Downloaded_{0}.tes", dt)), sb);
// Salvo archivio: RacLinks
sb = new StringBuilder();
racLinks.Sort();
foreach (string lnk in racLinks) { sb.AppendLine(lnk); }
cTools.WriteTextFile(Path.Combine(Cfg.Settings.PathAnalizadedIndexFile, string.Format("_A69_RacLinks_{0}.tes", dt)), sb);
// avviso utente:
RaiseEvent_Updating(string.Format("Finito il download in {0}", DateTime.Now - dtStartTime));
state = EngineStatusEnum.Stopped;
}
private string GetFileName(string url) { return url.Replace("https://www.annunci69.it/racconti-erotici/", "A69_").Replace("/", "_").Replace(":", "_").Replace(@"\", "_"); }
static private bool IsIdxLink(string link) { return link.StartsWith(BASE_LINK_IDX); }
static private bool IsRacLink(string link) { foreach (string tip in Tipologie) { if (link.StartsWith(BASE_LINK_RAC + tip + "/")) return true; } return false; }
static public string LinkType(string link) {
if (IsIdxLink(link)) return "Idx";
if (IsRacLink(link)) return "Rac";
//return "Unknown";
return "Unk";
}
private void Client_DownloadFileCompleted(object sender, AsyncCompletedEventArgs e) {
object obj = new Object();
lock (obj) {
conteggioEvt++;
StringBuilder sb;
if (state == EngineStatusEnum.Starting) state = EngineStatusEnum.Started;
WebClient wb = sender as WebClient;
string link = wb.BaseAddress;
return;
/*
// Se link di logon, devo terminare i download
if (link == "https://www.annunci69.it/accedi.php?nop") {
// Finito download:
state = EngineStatusEnum.Stopping;
// salvo gli archivi:
SaveState();
return;
}
// Elimino ulteriori ripetizioni dell'evento:
if (downloaded.Contains(link)) return;
// aggiorno UI
RaiseEvent_Updating(wb.Name + " - (Evt" + conteggioEvt.ToString("0000") + ") Donwloaded: " + link.Replace("https://www.annunci69.it/racconti-erotici/", "./"));
// Se pagina è un racconto, lo salvo
if (IsRacLink(link)) {
sb = new StringBuilder();
sb.Append(wb.Document.Body.InnerHtml);
string fn = Path.Combine(Cfg.Settings.PathAnalizadedRaccFile, GetFileName(link));
//cTools.WriteTextFile(fn, sb, Encoding.GetEncoding(wb.Document.Encoding));
cTools.WriteTextFile(fn, sb);
}
// Registro che ho eseguito il download del link
downloaded.Add(link);
// Ciclo su tutti i link estratti dalla pagina:
foreach (HtmlElement item in wb.Document.Links) {
link = item.GetAttribute("href");
// Se link nuovo:
if (!allLinks.Contains(link)) {
// elimino il link di logon
if (link == "https://www.annunci69.it/accedi.php?nop") break;
// Link nuovo, mi comporto di conseguenza alla tipologia:
if (IsRacLink(link)) {
// Link ad un racconto
allLinks.Add(link);
if (downRac) { toDownload.Add(link); }
racLinks.Add(link);
} else {
if (IsIdxLink(link)) {
// Link ad un indice
allLinks.Add(link);
toDownload.Add(link);
} else {
// Link non riconosciuto, lo scarto
;
}
}
}
} // fine ciclo sui link estratti
// Se ho raggiunto il massimo numero di link, o non vi sono link da downloadare, mi fermo:
if ((toDownload.Count > 0) && (state == EngineStatusEnum.Started)) {
// proseguo con il download:
string lk = toDownload.First();
toDownload.Remove(lk);
wb.Navigate(lk);
} else {
// Finito download:
state = EngineStatusEnum.Stopping;
// salvo gli archivi:
SaveState();
}
//*/
}//lock
}
private void Client_DownloadDataCompleted(object sender, DownloadDataCompletedEventArgs e) {
//System.Threading.AutoResetEvent waiter = (System.Threading.AutoResetEvent)e.UserState;
try {
// If the request was not canceled and did not throw
// an exception, display the resource.
if (!e.Cancelled && e.Error == null) {
byte[] data = (byte[])e.Result;
string textData = System.Text.Encoding.Unicode.GetString(data);
string FullFileName = @"C:\VMS\Prj\C#\A69WD\Test\Txt\Src\2022-09-22\A69_altroz.html";
//System.IO.StreamWriter sw = new System.IO.StreamWriter(FullFileName, false, Encoding.Unicode);
//sw.Write(textData);
//sw.Flush();
//sw.Close();
using (var fs = new FileStream(FullFileName, FileMode.Create, FileAccess.Write)) {
fs.Write(data, 0, data.Length);
}
}
} finally {
// Let the main application thread resume.
//waiter.Set();
}
}
private void Client_DownloadStringCompleted(object sender, DownloadStringCompletedEventArgs e) {
object obj = new Object();
lock (obj) {
conteggioEvt++;
if (!e.Cancelled && e.Error == null) {
string textData = e.Result;
string FullFileName = @"C:\VMS\Prj\C#\A69WD\Test\Txt\Src\2022-09-22\A69_altroZZ.html";
//System.IO.StreamWriter sw = new System.IO.StreamWriter(FullFileName, false, Encoding.Unicode);
System.IO.StreamWriter sw = new System.IO.StreamWriter(FullFileName,false, Encoding.UTF8);
sw.Write(textData);
sw.Flush();
sw.Close();
string lk = toDownload.First();
toDownload.Remove(lk);
using (WebClient client = new WebClient()) {
string fn = Path.Combine(Cfg.Settings.PathAnalizadedRaccFile, GetFileName(lk));
//client.Encoding = Encoding.ASCII;
client.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0");
client.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
client.Headers.Add("Cookie", "euconsent-v2=CPG74goPG74goAKAeAENBcCsAP_AAH_AAAwIH3tf_X__bX9j-_59__t0eY1f9_7_v-wzjhfdt-8N2P_X_L0X42E7PF36pq4KuR4Eu3LBIQNlHOHUTUmw6okVrTPsak2Mr7NKJ7LEmnMZe2dYGHtfn91TuZKY7_78_9fz3_-v_v___9f3r-3_3__59X---_e_V399zLv9__3__9gfiASYal8AF2JY4Mk0aVQogQhWEh0AoAKKAYWiawgYXBTsrAI9QQMAEJqAjAiBBiCjFgEAAgEASERASAHggEQBEAgABACpAQgAImAQWAFgYBAAKAaFiBFAEIEhBkcFRymBARItFBPZWAJRd7GmEIZRYAUCj-iowEShBAsDISFg5gAA.YAAAAAAAAAAA");
client.Headers.Add("DNT", "1");
client.Headers.Add("TE", "Trailers");
//client.DownloadFile(new Uri(lk), fn);
client.DownloadProgressChanged += Client_DownloadProgressChanged;
client.DownloadFileCompleted += Client_DownloadFileCompleted;
client.DownloadDataCompleted += Client_DownloadDataCompleted;
client.DownloadStringCompleted += Client_DownloadStringCompleted;
client.DownloadFileAsync(new Uri(lk), fn);
//client.DownloadDataAsync(new Uri(lk));
//client.DownloadStringAsync(new Uri(fn));
}
}
}//lock
}
private void Client_DownloadProgressChanged(object sender, DownloadProgressChangedEventArgs e) {
////Update progress bar & label
//Invoke(new MethodInvoker(delegate () {
// progressBar.Minimum = 0;
// double receive = double.Parse(e.BytesReceived.ToString());
// double total = double.Parse(e.TotalBytesToReceive.ToString());
// double percentage = receive / total * 100;
// lblStatus.Text = $"Downloaded {string.Format("{0:0.##}", percentage)}%";
// progressBar.Value = int.Parse(Math.Truncate(percentage).ToString());
//}));
}
}//class cA69Engine2
}//namespace A69WD
Src/A69WD/Net/cA69Engine3.cs
-642
View File
@@ -1,642 +0,0 @@
using System;
using System.IO;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Windows.Forms;
using System.Security.Policy;
using System.Net;
using static System.Windows.Forms.LinkLabel;
using System.ComponentModel;
using static System.Net.Mime.MediaTypeNames;
using System.Collections;
using System.Text.RegularExpressions;
namespace A69WD {
public class cA69Engine3 {
#region Property: State, NumDownload, Tipologie, BASE_URL, BASE_LINK_IDX, BASE_LINK_RAC
public EngineStatusEnum State { get { return state; } }
public int NumDownload { get { return downloaded.Count; } }
static string[] Tipologie = { "altro", "bdsm", "gay", "incesto", "lesbo", "orge", "prime", "scambio", "tradimenti", "trans", "trio" };
const string BASE_URL = "https://www.annunci69.it/racconti-erotici/{Tip}.html";
const string BASE_LINK_IDX = "https://www.annunci69.it/racconti-erotici/racconti_lista.php";
const string BASE_LINK_RAC = "https://www.annunci69.it/racconti-erotici/";
#endregion
#region Fields: allLinks, toDownload, downloaded, racLinks, dtStartTime, downRac, state
List<string> allLinks = null;
List<string> toDownload = null;
List<string> downloaded = null;
List<string> racLinks = null;
DateTime dtStartTime;
bool downRac = false;
EngineStatusEnum state = EngineStatusEnum.NotReady;
#endregion
#region Costruttore e distruttore
public cA69Engine3() {
try {
// Inizializzo gli archivi (dai file in memoria)
allLinks = new List<string>();
toDownload = new List<string>();
downloaded = new List<string>();
racLinks = new List<string>();
int res = App.DB.LoadData(allLinks, toDownload, downloaded, racLinks);
RaiseEvent_Updating("Ricaricati " + res.ToString("000") + " link di racconti da scaricare");
// Per ora non serializzo gli archivi e quindi carico le pagine iniziali delle 11 tipologie:
foreach (string tipologia in Tipologie) {
toDownload.Add(BASE_URL.Replace("{Tip}", tipologia));
allLinks.Add(BASE_URL.Replace("{Tip}", tipologia));
}
} catch (Exception ex) {
Logger.Log(new object[] { "Error Code: ER02220", "Errore in cA69Engine()", ex });
throw ex;
}
state = EngineStatusEnum.StandBy;
}
#endregion
#region Method:
public bool Start(bool DownRac) {
if (state != EngineStatusEnum.StandBy) return false;
dtStartTime = DateTime.Now;
downRac = DownRac;
RaiseEvent_Updating("Inizio lo scrap dei link dei racconti");
state = EngineStatusEnum.Starting;
if (toDownload.Count > 0) {
// proseguo con il download:
string lk = toDownload.First();
toDownload.Remove(lk);
string fn = Path.Combine(Cfg.Settings.PathAnalizadedRaccFile, GetFileName(lk));
using (MyWebClient client = new MyWebClient(lk, fn)) {
//client.Encoding = Encoding.ASCII;
client.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0");
client.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
client.Headers.Add("Cookie", "euconsent-v2=CPG74goPG74goAKAeAENBcCsAP_AAH_AAAwIH3tf_X__bX9j-_59__t0eY1f9_7_v-wzjhfdt-8N2P_X_L0X42E7PF36pq4KuR4Eu3LBIQNlHOHUTUmw6okVrTPsak2Mr7NKJ7LEmnMZe2dYGHtfn91TuZKY7_78_9fz3_-v_v___9f3r-3_3__59X---_e_V399zLv9__3__9gfiASYal8AF2JY4Mk0aVQogQhWEh0AoAKKAYWiawgYXBTsrAI9QQMAEJqAjAiBBiCjFgEAAgEASERASAHggEQBEAgABACpAQgAImAQWAFgYBAAKAaFiBFAEIEhBkcFRymBARItFBPZWAJRd7GmEIZRYAUCj-iowEShBAsDISFg5gAA.YAAAAAAAAAAA");
client.Headers.Add("DNT", "1");
client.Headers.Add("TE", "Trailers");
//client.DownloadFile(new Uri(lk), fn);
client.DownloadProgressChanged += Client_DownloadProgressChanged;
client.DownloadFileCompleted += Client_DownloadFileCompleted;
client.DownloadDataCompleted += Client_DownloadDataCompleted;
client.DownloadStringCompleted += Client_DownloadStringCompleted;
//client.DownloadFileAsync(new Uri(lk), fn);
//client.DownloadDataAsync(new Uri(lk));
client.DownloadStringAsync(new Uri(lk));
}
}
return true;
}
public bool Stop() {
switch (state) {
case EngineStatusEnum.NotReady:
case EngineStatusEnum.StandBy:
SaveState();
state = EngineStatusEnum.Stopped;
return true;
case EngineStatusEnum.Starting:
case EngineStatusEnum.Started:
case EngineStatusEnum.Stopping:
state = EngineStatusEnum.Stopping;
return false;
case EngineStatusEnum.Stopped:
return true;
}
return true;
}
public int ImportLinkFromFile(string SrcFilePath) {
int res = 0;
try {
if (File.Exists(SrcFilePath)) {
StreamReader sr = new StreamReader(SrcFilePath, Encoding.UTF8);
string row = sr.ReadLine().Trim();
while (row != null) {
// verifico che non sia un commento:
if (!(row.StartsWith("#") || row.StartsWith("//"))) {
// Se link nuovo:
if (!allLinks.Contains(row)) {
// Link nuovo, mi comporto di conseguenza alla tipologia:
if (IsRacLink(row)) {
// Link ad un racconto
allLinks.Add(row);
toDownload.Add(row);
racLinks.Add(row);
res++;
} else {
if (IsIdxLink(row)) {
// Link ad un indice
allLinks.Add(row);
toDownload.Add(row);
res++;
} else {
// Link non riconosciuto, lo scarto
;
}
}
}
}
//Read the next line
row = sr.ReadLine();
}
}
} catch (Exception ex) {
Logger.Log(new object[] { "Error Code: DB07403", "Errore in ImportLinkFromFile()", ex });
if (res > 0) res = -1 * res;
}
return res;
}
#endregion
#region Eventi:
/// <summary>
/// Evento che indica un cambiamento di stato nell'AnalizeManager
/// </summary>
public event EventHandler<EngineEventArgs> Updating;
#region Generazione degli eventi dell'engine verso le classi chiamanti:
/// <summary>
/// Attiva l'evento che segnala un cambiamento di stato nel engine (per esempio, terminato il download di un singolo file)
/// </summary>
void RaiseEvent_Updating(string msg, int tot, int toDown, int down) {
EventHandler<EngineEventArgs> tmp = Updating;
if (null != tmp) tmp(this, new EngineEventArgs(msg, tot, toDown, down));
}
/// <summary>
/// Attiva l'evento che segnala un cambiamento di stato nel engine (per esempio, terminato il download di un singolo file)
/// </summary>
void RaiseEvent_Updating(string msg) {
EventHandler<EngineEventArgs> tmp = Updating;
if (null != tmp) tmp(this, new EngineEventArgs(msg, allLinks.Count, toDownload.Count, downloaded.Count));
}
#endregion
#endregion
int conteggioEvt = 0;
private void Wb_DocumentCompleted(object sender, WebBrowserDocumentCompletedEventArgs e) {
object obj = new Object();
lock (obj) {
conteggioEvt++;
StringBuilder sb;
if (state == EngineStatusEnum.Starting) state = EngineStatusEnum.Started;
WebBrowser wb = sender as WebBrowser;
string link = wb.Url.ToString();
// Se link di logon, devo terminare i download
if (link == "https://www.annunci69.it/accedi.php?nop") {
// Finito download:
state = EngineStatusEnum.Stopping;
// salvo gli archivi:
SaveState();
return;
}
// Elimino ulteriori ripetizioni dell'evento:
if (downloaded.Contains(link)) return;
// aggiorno UI
RaiseEvent_Updating(wb.Name + " - (Evt" + conteggioEvt.ToString("0000") + ") Donwloaded: " + link.Replace("https://www.annunci69.it/racconti-erotici/", "./"));
// Se pagina è un racconto, lo salvo
if (IsRacLink(link)) {
sb = new StringBuilder();
sb.Append(wb.Document.Body.InnerHtml);
string fn = Path.Combine(Cfg.Settings.PathAnalizadedRaccFile, GetFileName(link));
//cTools.WriteTextFile(fn, sb, Encoding.GetEncoding(wb.Document.Encoding));
cTools.WriteTextFile(fn, sb);
}
// Registro che ho eseguito il download del link
downloaded.Add(link);
// Ciclo su tutti i link estratti dalla pagina:
foreach (HtmlElement item in wb.Document.Links) {
link = item.GetAttribute("href");
// Se link nuovo:
if (!allLinks.Contains(link)) {
// elimino il link di logon
if (link == "https://www.annunci69.it/accedi.php?nop") break;
// Link nuovo, mi comporto di conseguenza alla tipologia:
if (IsRacLink(link)) {
// Link ad un racconto
allLinks.Add(link);
if (downRac) { toDownload.Add(link); }
racLinks.Add(link);
} else {
if (IsIdxLink(link)) {
// Link ad un indice
allLinks.Add(link);
toDownload.Add(link);
} else {
// Link non riconosciuto, lo scarto
;
}
}
}
} // fine ciclo sui link estratti
// Se ho raggiunto il massimo numero di link, o non vi sono link da downloadare, mi fermo:
if ((toDownload.Count > 0) && (state == EngineStatusEnum.Started)) {
// proseguo con il download:
string lk = toDownload.First();
toDownload.Remove(lk);
wb.Navigate(lk);
} else {
// Finito download:
state = EngineStatusEnum.Stopping;
// salvo gli archivi:
SaveState();
}
}//lock
}
private void SaveState() {
int res = App.DB.SaveData(allLinks, toDownload, downloaded, racLinks);
StringBuilder sb;
// Finito download, salvo gli archivi:
string dt = DateTime.Now.ToString("yyyy-MM-dd_HH-mm-ss");
// Salvo archivio: AllLinks
sb = new StringBuilder();
allLinks.Sort();
foreach (string lnk in allLinks) { sb.AppendLine(lnk); }
cTools.WriteTextFile(Path.Combine(Cfg.Settings.PathAnalizadedIndexFile, string.Format("_A69_AllLinks_{0}.tes", dt)), sb);
// Salvo archivio: toDownload
sb = new StringBuilder();
toDownload.Sort();
foreach (string lnk in toDownload) { sb.AppendLine(lnk); }
cTools.WriteTextFile(Path.Combine(Cfg.Settings.PathAnalizadedIndexFile, string.Format("_A69_toDownload_{0}.tes", dt)), sb);
// Salvo archivio: Downloaded
sb = new StringBuilder();
downloaded.Sort();
foreach (string lnk in downloaded) { sb.AppendLine(lnk); }
cTools.WriteTextFile(Path.Combine(Cfg.Settings.PathAnalizadedIndexFile, string.Format("_A69_Downloaded_{0}.tes", dt)), sb);
// Salvo archivio: RacLinks
sb = new StringBuilder();
racLinks.Sort();
foreach (string lnk in racLinks) { sb.AppendLine(lnk); }
cTools.WriteTextFile(Path.Combine(Cfg.Settings.PathAnalizadedIndexFile, string.Format("_A69_RacLinks_{0}.tes", dt)), sb);
// avviso utente:
RaiseEvent_Updating(string.Format("Finito il download in {0}", DateTime.Now - dtStartTime));
state = EngineStatusEnum.Stopped;
}
private string GetFileName(string url) {
if (IsRacLink(url)) {
return url.Replace("https://www.annunci69.it/racconti-erotici/", "A69_").Replace("/", "_").Replace(":", "_").Replace(@"\", "_");
} else {
return url.Replace("https://www.annunci69.it/racconti-erotici/", "_A69_").Replace("/", "_").Replace(":", "_").Replace(@"\", "_");
}
}
static private bool IsIdxLink(string link) { return link.StartsWith(BASE_LINK_IDX); }
static private bool IsRacLink(string link) { foreach (string tip in Tipologie) { if (link.StartsWith(BASE_LINK_RAC + tip + "/")) return true; } return false; }
static public string LinkType(string link) {
if (IsIdxLink(link)) return "Idx";
if (IsRacLink(link)) return "Rac";
//return "Unknown";
return "Unk";
}
private void Client_DownloadFileCompleted(object sender, AsyncCompletedEventArgs e) {
object obj = new Object();
lock (obj) {
conteggioEvt++;
StringBuilder sb;
if (state == EngineStatusEnum.Starting) state = EngineStatusEnum.Started;
WebClient wb = sender as WebClient;
string link = wb.BaseAddress;
return;
/*
// Se link di logon, devo terminare i download
if (link == "https://www.annunci69.it/accedi.php?nop") {
// Finito download:
state = EngineStatusEnum.Stopping;
// salvo gli archivi:
SaveState();
return;
}
// Elimino ulteriori ripetizioni dell'evento:
if (downloaded.Contains(link)) return;
// aggiorno UI
RaiseEvent_Updating(wb.Name + " - (Evt" + conteggioEvt.ToString("0000") + ") Donwloaded: " + link.Replace("https://www.annunci69.it/racconti-erotici/", "./"));
// Se pagina è un racconto, lo salvo
if (IsRacLink(link)) {
sb = new StringBuilder();
sb.Append(wb.Document.Body.InnerHtml);
string fn = Path.Combine(Cfg.Settings.PathAnalizadedRaccFile, GetFileName(link));
//cTools.WriteTextFile(fn, sb, Encoding.GetEncoding(wb.Document.Encoding));
cTools.WriteTextFile(fn, sb);
}
// Registro che ho eseguito il download del link
downloaded.Add(link);
// Ciclo su tutti i link estratti dalla pagina:
foreach (HtmlElement item in wb.Document.Links) {
link = item.GetAttribute("href");
// Se link nuovo:
if (!allLinks.Contains(link)) {
// elimino il link di logon
if (link == "https://www.annunci69.it/accedi.php?nop") break;
// Link nuovo, mi comporto di conseguenza alla tipologia:
if (IsRacLink(link)) {
// Link ad un racconto
allLinks.Add(link);
if (downRac) { toDownload.Add(link); }
racLinks.Add(link);
} else {
if (IsIdxLink(link)) {
// Link ad un indice
allLinks.Add(link);
toDownload.Add(link);
} else {
// Link non riconosciuto, lo scarto
;
}
}
}
} // fine ciclo sui link estratti
// Se ho raggiunto il massimo numero di link, o non vi sono link da downloadare, mi fermo:
if ((toDownload.Count > 0) && (state == EngineStatusEnum.Started)) {
// proseguo con il download:
string lk = toDownload.First();
toDownload.Remove(lk);
wb.Navigate(lk);
} else {
// Finito download:
state = EngineStatusEnum.Stopping;
// salvo gli archivi:
SaveState();
}
//*/
}//lock
}
private void Client_DownloadDataCompleted(object sender, DownloadDataCompletedEventArgs e) {
//System.Threading.AutoResetEvent waiter = (System.Threading.AutoResetEvent)e.UserState;
try {
// If the request was not canceled and did not throw
// an exception, display the resource.
if (!e.Cancelled && e.Error == null) {
byte[] data = (byte[])e.Result;
string textData = System.Text.Encoding.Unicode.GetString(data);
string FullFileName = @"C:\VMS\Prj\C#\A69WD\Test\Txt\Src\2022-09-22\A69_altroz.html";
//System.IO.StreamWriter sw = new System.IO.StreamWriter(FullFileName, false, Encoding.Unicode);
//sw.Write(textData);
//sw.Flush();
//sw.Close();
using (var fs = new FileStream(FullFileName, FileMode.Create, FileAccess.Write)) {
fs.Write(data, 0, data.Length);
}
}
} finally {
// Let the main application thread resume.
//waiter.Set();
}
}
private void Client_DownloadStringCompleted(object sender, DownloadStringCompletedEventArgs e) {
object obj = new Object();
lock (obj) {
conteggioEvt++;
MyWebClient mwb = sender as MyWebClient;
if (state == EngineStatusEnum.Starting) state = EngineStatusEnum.Started;
// Verifico se download corretto
if (!e.Cancelled && e.Error == null) {
// aggiorno UI
RaiseEvent_Updating("(Evt" + conteggioEvt.ToString("0000") + ") Donwloaded: " + mwb.URL.Replace("https://www.annunci69.it/racconti-erotici/", "./"));
// Se pagina è un racconto, lo salvo
if (IsRacLink(mwb.URL)) {
string textData = e.Result;
string FullFileName = mwb.FileName;
System.IO.StreamWriter sw = new System.IO.StreamWriter(FullFileName, false, Encoding.UTF8);
sw.Write(textData);
sw.Flush();
sw.Close();
}
// Registro che ho eseguito il download del link
downloaded.Add(mwb.URL);
// Estraggo tutti i link net testo:
List<string> links = new List<string>();
//if (EstraiA69Link(e.Result, links)) {
if (EstraiLink(e.Result, "/racconti-erotici/", links)) {
// Ciclo su tutti i link estratti dalla pagina:
foreach (string link in links) {
if (!allLinks.Contains(link)) {
// elimino il link di logon
if (link == "https://www.annunci69.it/accedi.php?nop") break;
// Link nuovo, mi comporto di conseguenza alla tipologia:
if (IsRacLink(link)) {
// Link ad un racconto
allLinks.Add(link);
if (downRac) { toDownload.Add(link); }
racLinks.Add(link);
} else {
if (IsIdxLink(link)) {
// Link ad un indice
allLinks.Add(link);
toDownload.Add(link);
} else {
// Link non riconosciuto, lo scarto
;
}
}
}
}
} // fine ciclo sui link estratti
} else {
// Download con errore, informo l'utente
RaiseEvent_Updating("(Evt" + conteggioEvt.ToString("0000") + ") Donwload error: " + mwb.URL.Replace("https://www.annunci69.it/racconti-erotici/", "./"));
}
// Se ho raggiunto il massimo numero di link, o non vi sono link da downloadare, mi fermo:
if ((toDownload.Count > 0) && (state == EngineStatusEnum.Started)) {
// proseguo con il download:
string lk = toDownload.First();
toDownload.Remove(lk);
string fn = Path.Combine(Cfg.Settings.PathAnalizadedRaccFile, GetFileName(lk));
using (MyWebClient client = new MyWebClient(lk, fn)) {
//client.Encoding = Encoding.ASCII;
client.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0");
client.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
client.Headers.Add("Cookie", "euconsent-v2=CPG74goPG74goAKAeAENBcCsAP_AAH_AAAwIH3tf_X__bX9j-_59__t0eY1f9_7_v-wzjhfdt-8N2P_X_L0X42E7PF36pq4KuR4Eu3LBIQNlHOHUTUmw6okVrTPsak2Mr7NKJ7LEmnMZe2dYGHtfn91TuZKY7_78_9fz3_-v_v___9f3r-3_3__59X---_e_V399zLv9__3__9gfiASYal8AF2JY4Mk0aVQogQhWEh0AoAKKAYWiawgYXBTsrAI9QQMAEJqAjAiBBiCjFgEAAgEASERASAHggEQBEAgABACpAQgAImAQWAFgYBAAKAaFiBFAEIEhBkcFRymBARItFBPZWAJRd7GmEIZRYAUCj-iowEShBAsDISFg5gAA.YAAAAAAAAAAA");
client.Headers.Add("DNT", "1");
client.Headers.Add("TE", "Trailers");
//client.DownloadFile(new Uri(lk), fn);
client.DownloadProgressChanged += Client_DownloadProgressChanged;
client.DownloadFileCompleted += Client_DownloadFileCompleted;
client.DownloadDataCompleted += Client_DownloadDataCompleted;
client.DownloadStringCompleted += Client_DownloadStringCompleted;
//client.DownloadFileAsync(new Uri(lk), fn);
//client.DownloadDataAsync(new Uri(lk));
client.DownloadStringAsync(new Uri(lk));
}
} else {
// Finito download:
state = EngineStatusEnum.Stopping;
// salvo gli archivi:
SaveState();
}
}//lock
}
private void Client_DownloadProgressChanged(object sender, DownloadProgressChangedEventArgs e) {
////Update progress bar & label
//Invoke(new MethodInvoker(delegate () {
// progressBar.Minimum = 0;
// double receive = double.Parse(e.BytesReceived.ToString());
// double total = double.Parse(e.TotalBytesToReceive.ToString());
// double percentage = receive / total * 100;
// lblStatus.Text = $"Downloaded {string.Format("{0:0.##}", percentage)}%";
// progressBar.Value = int.Parse(Math.Truncate(percentage).ToString());
//}));
}
public static bool EstraiA69Link(string txt, List<string> links) {
try {
Regex regex = new Regex("(?<=href=\")\\S+?(?=\")");
Match matchObj = regex.Match(txt);
while (matchObj.Success) {
if (matchObj.Value.StartsWith("/racconti-erotici/")) { links.Add(matchObj.Value); }
if (matchObj.Groups.Count > 1) {
for (int i = 1; i < matchObj.Groups.Count; i++) {
Group group = matchObj.Groups[i];
if (group.Success) {
if (group.Value.StartsWith("/racconti-erotici/")) { links.Add(group.Value); }
}
}
}
matchObj = matchObj.NextMatch();
}
return true;
} catch { return false; }
}
public static bool EstraiLink(string txt, string startWith, List<string> links) {
try {
Regex regex = new Regex("(?<=href=\")\\S+?(?=\")");
Match matchObj = regex.Match(txt);
while (matchObj.Success) {
if (matchObj.Value.StartsWith(startWith)) { links.Add(matchObj.Value.Replace(startWith, "https://www.annunci69.it/racconti-erotici/")); }
if (matchObj.Groups.Count > 1) {
for (int i = 1; i < matchObj.Groups.Count; i++) {
Group group = matchObj.Groups[i];
if (group.Success) {
if (group.Value.StartsWith(startWith)) { links.Add(group.Value.Replace(startWith, "https://www.annunci69.it/racconti-erotici/")); }
}
}
}
matchObj = matchObj.NextMatch();
}
return true;
} catch { return false; }
}
}//class cA69Engine3
public class MyWebClient : WebClient {
public string URL;
public string FileName;
public MyWebClient(string url, string fileName) {
this.URL = url;
this.FileName = fileName;
}
}
}//namespace A69WD
Src/A69WD/Net/frmDownloader.Designer.cs
Generated
-67
View File
@@ -27,14 +27,9 @@
this.btnHide = new System.Windows.Forms.Button();
this.btnDownloadAll = new System.Windows.Forms.Button();
this.lblST = new System.Windows.Forms.Label();
this.wb1 = new System.Windows.Forms.WebBrowser();
this.btnImportLinks = new System.Windows.Forms.Button();
this.openFileDialog1 = new System.Windows.Forms.OpenFileDialog();
this.btnShowLog = new System.Windows.Forms.Button();
this.btnDownloadAll2 = new System.Windows.Forms.Button();
this.btnTest = new System.Windows.Forms.Button();
this.button1 = new System.Windows.Forms.Button();
this.button2 = new System.Windows.Forms.Button();
this.SuspendLayout();
//
// btnClose
@@ -78,18 +73,6 @@
this.lblST.Text = "Status";
this.lblST.TextAlign = System.Drawing.ContentAlignment.MiddleLeft;
//
// wb1
//
this.wb1.Location = new System.Drawing.Point(0, 1);
this.wb1.MinimumSize = new System.Drawing.Size(20, 20);
this.wb1.Name = "wb1";
this.wb1.ScriptErrorsSuppressed = true;
this.wb1.ScrollBarsEnabled = false;
this.wb1.Size = new System.Drawing.Size(63, 44);
this.wb1.TabIndex = 12;
this.wb1.TabStop = false;
this.wb1.Visible = false;
//
// btnImportLinks
//
this.btnImportLinks.Location = new System.Drawing.Point(115, 12);
@@ -115,62 +98,17 @@
this.btnShowLog.UseVisualStyleBackColor = true;
this.btnShowLog.Click += new System.EventHandler(this.btnShowLog_Click);
//
// btnDownloadAll2
//
this.btnDownloadAll2.Location = new System.Drawing.Point(45, 1);
this.btnDownloadAll2.Name = "btnDownloadAll2";
this.btnDownloadAll2.Size = new System.Drawing.Size(97, 33);
this.btnDownloadAll2.TabIndex = 18;
this.btnDownloadAll2.Text = "Download All";
this.btnDownloadAll2.UseVisualStyleBackColor = true;
this.btnDownloadAll2.Click += new System.EventHandler(this.btnDownloadAll2_Click);
//
// btnTest
//
this.btnTest.Location = new System.Drawing.Point(148, 1);
this.btnTest.Name = "btnTest";
this.btnTest.Size = new System.Drawing.Size(97, 33);
this.btnTest.TabIndex = 19;
this.btnTest.Text = "Test WebClient";
this.btnTest.UseVisualStyleBackColor = true;
this.btnTest.Click += new System.EventHandler(this.btnTest_Click);
//
// button1
//
this.button1.Location = new System.Drawing.Point(251, 1);
this.button1.Name = "button1";
this.button1.Size = new System.Drawing.Size(97, 33);
this.button1.TabIndex = 20;
this.button1.Text = "Test Engine2";
this.button1.UseVisualStyleBackColor = true;
this.button1.Click += new System.EventHandler(this.button1_Click);
//
// button2
//
this.button2.Location = new System.Drawing.Point(354, 1);
this.button2.Name = "button2";
this.button2.Size = new System.Drawing.Size(97, 33);
this.button2.TabIndex = 21;
this.button2.Text = "Test Engine3";
this.button2.UseVisualStyleBackColor = true;
this.button2.Click += new System.EventHandler(this.button2_Click);
//
// frmDownloader
//
this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F);
this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font;
this.ClientSize = new System.Drawing.Size(524, 74);
this.Controls.Add(this.button2);
this.Controls.Add(this.button1);
this.Controls.Add(this.btnTest);
this.Controls.Add(this.btnDownloadAll2);
this.Controls.Add(this.btnShowLog);
this.Controls.Add(this.btnImportLinks);
this.Controls.Add(this.btnClose);
this.Controls.Add(this.btnHide);
this.Controls.Add(this.btnDownloadAll);
this.Controls.Add(this.lblST);
this.Controls.Add(this.wb1);
this.FormBorderStyle = System.Windows.Forms.FormBorderStyle.FixedDialog;
this.Name = "frmDownloader";
this.Text = "frmDownloader";
@@ -183,13 +121,8 @@
private System.Windows.Forms.Button btnHide;
private System.Windows.Forms.Button btnDownloadAll;
private System.Windows.Forms.Label lblST;
private System.Windows.Forms.WebBrowser wb1;
private System.Windows.Forms.Button btnImportLinks;
private System.Windows.Forms.OpenFileDialog openFileDialog1;
private System.Windows.Forms.Button btnShowLog;
private System.Windows.Forms.Button btnDownloadAll2;
private System.Windows.Forms.Button btnTest;
private System.Windows.Forms.Button button1;
private System.Windows.Forms.Button button2;
}
}
Src/A69WD/Net/frmDownloader.cs
+1 -109
View File
@@ -21,17 +21,11 @@ namespace A69WD {
public frmDownloader() {
InitializeComponent();
this.Text = String.Format("A69WD - Dowloader Testi Vrs {0}", cTools.AssemblyVersion);
engine = new cA69Engine(this.wb1);
engine = new cA69Engine();
engine.Updating += Engine_Updating;
engine2 = new cA69Engine2();
engine2.Updating += Engine_Updating;
engine3 = new cA69Engine3();
engine3.Updating += Engine_Updating;
}
cA69Engine engine = null;
cA69Engine2 engine2 = null;
cA69Engine3 engine3 = null;
/// <summary>
@@ -95,108 +89,6 @@ namespace A69WD {
#endregion
private void btnDownloadAll2_Click(object sender, EventArgs e) { engine.Start(false); }
#region Test WEBCLIENT
private void btnTest_Click(object sender, EventArgs e) {
////Run download file with multiple thread
//string url = "https://www.annunci69.it/racconti-erotici/racconti_lista.php?genere=altro&pagina=103";
//if (!string.IsNullOrEmpty(url)) {
// Thread thread = new Thread(() =>
// {
// Uri uri = new Uri(url);
// //string fileName = System.IO.Path.GetFileName(uri.AbsolutePath);
// string fileName = System.IO.Path.Combine(Cfg.Settings.PathAnalizadedRaccFile, url.Replace("https://www.annunci69.it/racconti-erotici/", "A69_").Replace("/", "_").Replace(":", "_").Replace(@"\", "_"));
// client.DownloadFileAsync(uri, @"C:\VMS\Prj\C#\A69WD\Test\Txt\Src\_A69_genere_altro_103.html");
// });
// thread.Start();
//}
string url = "https://www.annunci69.it/racconti-erotici/racconti_lista.php?genere=altro&pagina=98";
string fileName = @"C:\VMS\Prj\C#\A69WD\Test\Logs\_A69_genere_altro_98.html";
//// Test 1: Questo dà errore:
//WebClient client;
//client = new WebClient();
//client.DownloadProgressChanged += Client_DownloadProgressChanged;
//client.DownloadFileCompleted += Client_DownloadFileCompleted;
////client.DownloadFile(url, fileName);
//StringBuilder sb = new StringBuilder(client.DownloadString(url));
//cTools.WriteTextFile(fileName, sb);
//// Test 1
// // Test 2: OK
// WebClient client = new WebClient();
// //client.Headers["User-Agent"] = "User-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36";
// client.Encoding = Encoding.ASCII;
// client.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0");
// client.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
// //client.Headers.Add("Accept-Encoding", "gzip, deflate, br");
// //client.Headers.Add("Connection", "keep-alive");
// client.Headers.Add("Cookie", "euconsent-v2=CPG74goPG74goAKAeAENBcCsAP_AAH_AAAwIH3tf_X__bX9j-_59__t0eY1f9_7_v-wzjhfdt-8N2P_X_L0X42E7PF36pq4KuR4Eu3LBIQNlHOHUTUmw6okVrTPsak2Mr7NKJ7LEmnMZe2dYGHtfn91TuZKY7_78_9fz3_-v_v___9f3r-3_3__59X---_e_V399zLv9__3__9gfiASYal8AF2JY4Mk0aVQogQhWEh0AoAKKAYWiawgYXBTsrAI9QQMAEJqAjAiBBiCjFgEAAgEASERASAHggEQBEAgABACpAQgAImAQWAFgYBAAKAaFiBFAEIEhBkcFRymBARItFBPZWAJRd7GmEIZRYAUCj-iowEShBAsDISFg5gAA.YAAAAAAAAAAA");
// client.Headers.Add("DNT", "1");
// client.Headers.Add("TE", "Trailers");
// client.DownloadFile(new Uri(url), fileName);
// // Test 2: OK
// Test 3: OK
using (WebClient client = new WebClient()) {
for (int i = 90; i < 110; i++) {
url = "https://www.annunci69.it/racconti-erotici/racconti_lista.php?genere=altro&pagina=" + i.ToString();
fileName = @"C:\VMS\Prj\C#\A69WD\Test\Logs\_A69_genere_altro_" + i.ToString() + ".html";
client.Encoding = Encoding.ASCII;
client.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0");
client.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
client.Headers.Add("Cookie", "euconsent-v2=CPG74goPG74goAKAeAENBcCsAP_AAH_AAAwIH3tf_X__bX9j-_59__t0eY1f9_7_v-wzjhfdt-8N2P_X_L0X42E7PF36pq4KuR4Eu3LBIQNlHOHUTUmw6okVrTPsak2Mr7NKJ7LEmnMZe2dYGHtfn91TuZKY7_78_9fz3_-v_v___9f3r-3_3__59X---_e_V399zLv9__3__9gfiASYal8AF2JY4Mk0aVQogQhWEh0AoAKKAYWiawgYXBTsrAI9QQMAEJqAjAiBBiCjFgEAAgEASERASAHggEQBEAgABACpAQgAImAQWAFgYBAAKAaFiBFAEIEhBkcFRymBARItFBPZWAJRd7GmEIZRYAUCj-iowEShBAsDISFg5gAA.YAAAAAAAAAAA");
client.Headers.Add("DNT", "1");
client.Headers.Add("TE", "Trailers");
client.DownloadFile(new Uri(url), fileName);
}
}
// Test 3: OK
}
private void Client_DownloadFileCompleted(object sender, AsyncCompletedEventArgs e) {
MessageBox.Show("Download complete !", "Message", MessageBoxButtons.OK, MessageBoxIcon.Information);
}
private void Client_DownloadProgressChanged(object sender, DownloadProgressChangedEventArgs e) {
////Update progress bar & label
//Invoke(new MethodInvoker(delegate () {
// progressBar.Minimum = 0;
// double receive = double.Parse(e.BytesReceived.ToString());
// double total = double.Parse(e.TotalBytesToReceive.ToString());
// double percentage = receive / total * 100;
// lblStatus.Text = $"Downloaded {string.Format("{0:0.##}", percentage)}%";
// progressBar.Value = int.Parse(Math.Truncate(percentage).ToString());
//}));
}
#endregion
#region Test ENGINE2
private void button1_Click(object sender, EventArgs e) {
engine2.Start(true);
}
#endregion
private void button2_Click(object sender, EventArgs e) {
engine3.Start(true);
}
Src/A69WD/Properties/AssemblyInfo.cs
+10
View File
@@ -36,6 +36,16 @@ using System.Runtime.InteropServices;
[assembly: AssemblyFileVersion("0.2.0.2")]
/*
*
* Vrs 0.3.0.0
*
* Compilata da CSiCoAlm il 2022/09/22
* - Versione in cui ho sostituito l'oggetto WebBrowser con la classe WebClient per poter gestire in parallelo i download.
*
*/
/*
*
* Vrs 0.2.0.2