MrkPi/A69WD
3
0 Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

- Autocommit - 2022/09/22 07:48:34

Browse Source
This commit is contained in:
CSiCoAlm committed 2022-09-22 07:48:34 +02:00
1 parent 01021811f3
commit 43186b9875
3 files changed
+295 -25
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Src/A69WD/Net/cA69Engine2.cs
+238 -8
View File
@@ -1,15 +1,17 @@
using System;
using System.IO;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using A69WD.Net;
using System.Windows.Forms;
using System.Security.Policy;
using System.Net;
namespace A69WD {
public class cA69Engine2 {
#region Property: State, NumDownload, Tipologie, BASE_URL, BASE_LINK_IDX, BASE_LINK_RAC
public EngineStatusEnum State { get { return state; } }
@@ -25,7 +27,7 @@ namespace A69WD {
#endregion
#region Fields: allLinks, toDownload, downloaded, racLinks, dtStartTime, downRac, state, wb
#region Fields: allLinks, toDownload, downloaded, racLinks, dtStartTime, downRac, state
List<string> allLinks = null;
List<string> toDownload = null;
@@ -38,6 +40,7 @@ namespace A69WD {
EngineStatusEnum state = EngineStatusEnum.NotReady;
#endregion
@@ -52,8 +55,6 @@ namespace A69WD {
racLinks = new List<string>();
int res = App.DB.LoadData(allLinks, toDownload, downloaded, racLinks);
RaiseEvent_Updating("Ricaricati " + res.ToString("000") + " link di racconti da scaricare");
//wb = WB;
//wb.DocumentCompleted += Wb_DocumentCompleted;
// Per ora non serializzo gli archivi e quindi carico le pagine iniziali delle 11 tipologie:
foreach (string tipologia in Tipologie) {
toDownload.Add(BASE_URL.Replace("{Tip}", tipologia));
@@ -69,8 +70,99 @@ namespace A69WD {
#endregion
#region Method:
public bool Start(bool DownRac) {
if (state != EngineStatusEnum.StandBy) return false;
dtStartTime = DateTime.Now;
downRac = DownRac;
RaiseEvent_Updating("Inizio lo scrap dei link dei racconti");
state = EngineStatusEnum.Starting;
if (toDownload.Count > 0) {
// proseguo con il download:
string lk = toDownload.First();
toDownload.Remove(lk);
using (WebClient client = new WebClient()) {
client.Encoding = Encoding.ASCII;
client.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0");
client.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
client.Headers.Add("Cookie", "euconsent-v2=CPG74goPG74goAKAeAENBcCsAP_AAH_AAAwIH3tf_X__bX9j-_59__t0eY1f9_7_v-wzjhfdt-8N2P_X_L0X42E7PF36pq4KuR4Eu3LBIQNlHOHUTUmw6okVrTPsak2Mr7NKJ7LEmnMZe2dYGHtfn91TuZKY7_78_9fz3_-v_v___9f3r-3_3__59X---_e_V399zLv9__3__9gfiASYal8AF2JY4Mk0aVQogQhWEh0AoAKKAYWiawgYXBTsrAI9QQMAEJqAjAiBBiCjFgEAAgEASERASAHggEQBEAgABACpAQgAImAQWAFgYBAAKAaFiBFAEIEhBkcFRymBARItFBPZWAJRd7GmEIZRYAUCj-iowEShBAsDISFg5gAA.YAAAAAAAAAAA");
client.Headers.Add("DNT", "1");
client.Headers.Add("TE", "Trailers");
client.DownloadFile(new Uri(lk), fileName);
}
}
return true;
}
public bool Stop() {
switch (state) {
case EngineStatusEnum.NotReady:
case EngineStatusEnum.StandBy:
SaveState();
state = EngineStatusEnum.Stopped;
return true;
case EngineStatusEnum.Starting:
case EngineStatusEnum.Started:
case EngineStatusEnum.Stopping:
state = EngineStatusEnum.Stopping;
return false;
case EngineStatusEnum.Stopped:
return true;
}
return true;
}
public int ImportLinkFromFile(string SrcFilePath) {
int res = 0;
try {
if (File.Exists(SrcFilePath)) {
StreamReader sr = new StreamReader(SrcFilePath, Encoding.UTF8);
string row = sr.ReadLine().Trim();
while (row != null) {
// verifico che non sia un commento:
if (!(row.StartsWith("#") || row.StartsWith("//"))) {
// Se link nuovo:
if (!allLinks.Contains(row)) {
// Link nuovo, mi comporto di conseguenza alla tipologia:
if (IsRacLink(row)) {
// Link ad un racconto
allLinks.Add(row);
toDownload.Add(row);
racLinks.Add(row);
res++;
} else {
if (IsIdxLink(row)) {
// Link ad un indice
allLinks.Add(row);
toDownload.Add(row);
res++;
} else {
// Link non riconosciuto, lo scarto
;
}
}
}
}
//Read the next line
row = sr.ReadLine();
}
}
} catch (Exception ex) {
Logger.Log(new object[] { "Error Code: DB07403", "Errore in ImportLinkFromFile()", ex });
if (res > 0) res = -1 * res;
}
return res;
}
#endregion
#region Eventi:
@@ -105,5 +197,143 @@ namespace A69WD {
#endregion
}
}
int conteggioEvt = 0;
private void Wb_DocumentCompleted(object sender, WebBrowserDocumentCompletedEventArgs e) {
object obj = new Object();
lock (obj) {
conteggioEvt++;
StringBuilder sb;
if (state == EngineStatusEnum.Starting) state = EngineStatusEnum.Started;
WebBrowser wb = sender as WebBrowser;
string link = wb.Url.ToString();
// Se link di logon, devo terminare i download
if (link == "https://www.annunci69.it/accedi.php?nop") {
// Finito download:
state = EngineStatusEnum.Stopping;
// salvo gli archivi:
SaveState();
return;
}
// Elimino ulteriori ripetizioni dell'evento:
if (downloaded.Contains(link)) return;
// aggiorno UI
RaiseEvent_Updating(wb.Name + " - (Evt" + conteggioEvt.ToString("0000") + ") Donwloaded: " + link.Replace("https://www.annunci69.it/racconti-erotici/", "./"));
// Se pagina è un racconto, lo salvo
if (IsRacLink(link)) {
sb = new StringBuilder();
sb.Append(wb.Document.Body.InnerHtml);
string fn = Path.Combine(Cfg.Settings.PathAnalizadedRaccFile, GetFileName(link));
//cTools.WriteTextFile(fn, sb, Encoding.GetEncoding(wb.Document.Encoding));
cTools.WriteTextFile(fn, sb);
}
// Registro che ho eseguito il download del link
downloaded.Add(link);
// Ciclo su tutti i link estratti dalla pagina:
foreach (HtmlElement item in wb.Document.Links) {
link = item.GetAttribute("href");
// Se link nuovo:
if (!allLinks.Contains(link)) {
// elimino il link di logon
if (link == "https://www.annunci69.it/accedi.php?nop") break;
// Link nuovo, mi comporto di conseguenza alla tipologia:
if (IsRacLink(link)) {
// Link ad un racconto
allLinks.Add(link);
if (downRac) { toDownload.Add(link); }
racLinks.Add(link);
} else {
if (IsIdxLink(link)) {
// Link ad un indice
allLinks.Add(link);
toDownload.Add(link);
} else {
// Link non riconosciuto, lo scarto
;
}
}
}
} // fine ciclo sui link estratti
// Se ho raggiunto il massimo numero di link, o non vi sono link da downloadare, mi fermo:
if ((toDownload.Count > 0) && (state == EngineStatusEnum.Started)) {
// proseguo con il download:
string lk = toDownload.First();
toDownload.Remove(lk);
wb.Navigate(lk);
} else {
// Finito download:
state = EngineStatusEnum.Stopping;
// salvo gli archivi:
SaveState();
}
}//lock
}
private void SaveState() {
int res = App.DB.SaveData(allLinks, toDownload, downloaded, racLinks);
StringBuilder sb;
// Finito download, salvo gli archivi:
string dt = DateTime.Now.ToString("yyyy-MM-dd_HH-mm-ss");
// Salvo archivio: AllLinks
sb = new StringBuilder();
allLinks.Sort();
foreach (string lnk in allLinks) { sb.AppendLine(lnk); }
cTools.WriteTextFile(Path.Combine(Cfg.Settings.PathAnalizadedIndexFile, string.Format("_A69_AllLinks_{0}.tes", dt)), sb);
// Salvo archivio: toDownload
sb = new StringBuilder();
toDownload.Sort();
foreach (string lnk in toDownload) { sb.AppendLine(lnk); }
cTools.WriteTextFile(Path.Combine(Cfg.Settings.PathAnalizadedIndexFile, string.Format("_A69_toDownload_{0}.tes", dt)), sb);
// Salvo archivio: Downloaded
sb = new StringBuilder();
downloaded.Sort();
foreach (string lnk in downloaded) { sb.AppendLine(lnk); }
cTools.WriteTextFile(Path.Combine(Cfg.Settings.PathAnalizadedIndexFile, string.Format("_A69_Downloaded_{0}.tes", dt)), sb);
// Salvo archivio: RacLinks
sb = new StringBuilder();
racLinks.Sort();
foreach (string lnk in racLinks) { sb.AppendLine(lnk); }
cTools.WriteTextFile(Path.Combine(Cfg.Settings.PathAnalizadedIndexFile, string.Format("_A69_RacLinks_{0}.tes", dt)), sb);
// avviso utente:
RaiseEvent_Updating(string.Format("Finito il download in {0}", DateTime.Now - dtStartTime));
state = EngineStatusEnum.Stopped;
}
private string GetFileName(string url) { return url.Replace("https://www.annunci69.it/racconti-erotici/", "A69_").Replace("/", "_").Replace(":", "_").Replace(@"\", "_"); }
static private bool IsIdxLink(string link) { return link.StartsWith(BASE_LINK_IDX); }
static private bool IsRacLink(string link) { foreach (string tip in Tipologie) { if (link.StartsWith(BASE_LINK_RAC + tip + "/")) return true; } return false; }
static public string LinkType(string link) {
if (IsIdxLink(link)) return "Idx";
if (IsRacLink(link)) return "Rac";
//return "Unknown";
return "Unk";
}
}//class cA69Engine2
}//namespace A69WD
Src/A69WD/Net/frmDownloader.Designer.cs
Generated
+14 -1
View File
@@ -33,6 +33,7 @@
this.btnShowLog = new System.Windows.Forms.Button();
this.btnDownloadAll2 = new System.Windows.Forms.Button();
this.btnTest = new System.Windows.Forms.Button();
this.button1 = new System.Windows.Forms.Button();
this.SuspendLayout();
//
// btnClose
@@ -129,15 +130,26 @@
this.btnTest.Name = "btnTest";
this.btnTest.Size = new System.Drawing.Size(97, 33);
this.btnTest.TabIndex = 19;
this.btnTest.Text = "Test";
this.btnTest.Text = "Test WebClient";
this.btnTest.UseVisualStyleBackColor = true;
this.btnTest.Click += new System.EventHandler(this.btnTest_Click);
//
// button1
//
this.button1.Location = new System.Drawing.Point(290, 1);
this.button1.Name = "button1";
this.button1.Size = new System.Drawing.Size(97, 33);
this.button1.TabIndex = 20;
this.button1.Text = "Test Engine2";
this.button1.UseVisualStyleBackColor = true;
this.button1.Click += new System.EventHandler(this.button1_Click);
//
// frmDownloader
//
this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F);
this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font;
this.ClientSize = new System.Drawing.Size(524, 74);
this.Controls.Add(this.button1);
this.Controls.Add(this.btnTest);
this.Controls.Add(this.btnDownloadAll2);
this.Controls.Add(this.btnShowLog);
@@ -165,5 +177,6 @@
private System.Windows.Forms.Button btnShowLog;
private System.Windows.Forms.Button btnDownloadAll2;
private System.Windows.Forms.Button btnTest;
private System.Windows.Forms.Button button1;
}
}
Src/A69WD/Net/frmDownloader.cs
+43 -16
View File
@@ -94,7 +94,7 @@ namespace A69WD {
#region Test download
#region Test WEBCLIENT
private void btnTest_Click(object sender, EventArgs e) {
////Run download file with multiple thread
@@ -110,8 +110,8 @@ namespace A69WD {
// thread.Start();
//}
string url = "https://www.annunci69.it/racconti-erotici/racconti_lista.php?genere=altro&pagina=3";
string fileName = @"C:\VMS\Prj\C#\A69WD\Test\Logs\_A69_genere_altro_3.html";
string url = "https://www.annunci69.it/racconti-erotici/racconti_lista.php?genere=altro&pagina=98";
string fileName = @"C:\VMS\Prj\C#\A69WD\Test\Logs\_A69_genere_altro_98.html";
//// Test 1: Questo dà errore:
//WebClient client;
@@ -123,20 +123,35 @@ namespace A69WD {
//cTools.WriteTextFile(fileName, sb);
//// Test 1
// Test
WebClient client = new WebClient();
//client.Headers["User-Agent"] = "User-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36";
client.Encoding = Encoding.ASCII;
client.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0");
client.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
//client.Headers.Add("Accept-Encoding", "gzip, deflate, br");
//client.Headers.Add("Connection", "keep-alive");
client.Headers.Add("Cookie", "euconsent-v2=CPG74goPG74goAKAeAENBcCsAP_AAH_AAAwIH3tf_X__bX9j-_59__t0eY1f9_7_v-wzjhfdt-8N2P_X_L0X42E7PF36pq4KuR4Eu3LBIQNlHOHUTUmw6okVrTPsak2Mr7NKJ7LEmnMZe2dYGHtfn91TuZKY7_78_9fz3_-v_v___9f3r-3_3__59X---_e_V399zLv9__3__9gfiASYal8AF2JY4Mk0aVQogQhWEh0AoAKKAYWiawgYXBTsrAI9QQMAEJqAjAiBBiCjFgEAAgEASERASAHggEQBEAgABACpAQgAImAQWAFgYBAAKAaFiBFAEIEhBkcFRymBARItFBPZWAJRd7GmEIZRYAUCj-iowEShBAsDISFg5gAA.YAAAAAAAAAAA");
client.Headers.Add("DNT", "1");
client.Headers.Add("TE", "Trailers");
client.DownloadFile(new Uri(url), fileName);
// // Test 2: OK
// WebClient client = new WebClient();
// //client.Headers["User-Agent"] = "User-Agent: Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36";
// client.Encoding = Encoding.ASCII;
// client.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0");
// client.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
// //client.Headers.Add("Accept-Encoding", "gzip, deflate, br");
// //client.Headers.Add("Connection", "keep-alive");
// client.Headers.Add("Cookie", "euconsent-v2=CPG74goPG74goAKAeAENBcCsAP_AAH_AAAwIH3tf_X__bX9j-_59__t0eY1f9_7_v-wzjhfdt-8N2P_X_L0X42E7PF36pq4KuR4Eu3LBIQNlHOHUTUmw6okVrTPsak2Mr7NKJ7LEmnMZe2dYGHtfn91TuZKY7_78_9fz3_-v_v___9f3r-3_3__59X---_e_V399zLv9__3__9gfiASYal8AF2JY4Mk0aVQogQhWEh0AoAKKAYWiawgYXBTsrAI9QQMAEJqAjAiBBiCjFgEAAgEASERASAHggEQBEAgABACpAQgAImAQWAFgYBAAKAaFiBFAEIEhBkcFRymBARItFBPZWAJRd7GmEIZRYAUCj-iowEShBAsDISFg5gAA.YAAAAAAAAAAA");
// client.Headers.Add("DNT", "1");
// client.Headers.Add("TE", "Trailers");
// client.DownloadFile(new Uri(url), fileName);
// // Test 2: OK
// Test 3: OK
using (WebClient client = new WebClient()) {
for (int i = 90; i < 110; i++) {
url = "https://www.annunci69.it/racconti-erotici/racconti_lista.php?genere=altro&pagina=" + i.ToString();
fileName = @"C:\VMS\Prj\C#\A69WD\Test\Logs\_A69_genere_altro_" + i.ToString() + ".html";
client.Encoding = Encoding.ASCII;
client.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0");
client.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
client.Headers.Add("Cookie", "euconsent-v2=CPG74goPG74goAKAeAENBcCsAP_AAH_AAAwIH3tf_X__bX9j-_59__t0eY1f9_7_v-wzjhfdt-8N2P_X_L0X42E7PF36pq4KuR4Eu3LBIQNlHOHUTUmw6okVrTPsak2Mr7NKJ7LEmnMZe2dYGHtfn91TuZKY7_78_9fz3_-v_v___9f3r-3_3__59X---_e_V399zLv9__3__9gfiASYal8AF2JY4Mk0aVQogQhWEh0AoAKKAYWiawgYXBTsrAI9QQMAEJqAjAiBBiCjFgEAAgEASERASAHggEQBEAgABACpAQgAImAQWAFgYBAAKAaFiBFAEIEhBkcFRymBARItFBPZWAJRd7GmEIZRYAUCj-iowEShBAsDISFg5gAA.YAAAAAAAAAAA");
client.Headers.Add("DNT", "1");
client.Headers.Add("TE", "Trailers");
client.DownloadFile(new Uri(url), fileName);
}
}
// Test 3: OK
}
@@ -157,6 +172,18 @@ namespace A69WD {
//}));
}
#endregion
#region Test ENGINE2
private void button1_Click(object sender, EventArgs e) {
}
#endregion