MrkPi/A69WD
3
0 Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

Versione per test webclient

Browse Source
This commit is contained in:
CSiCoAlm committed 2022-09-22 11:29:19 +02:00
1 parent e12801e1a1
commit c3355d8bf3
4 files changed
+156 -64
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Src/A69WD/A69WD.csproj
+1 -1
View File
@@ -63,8 +63,8 @@
<Compile Include="Net\ATLink.cs" />
<Compile Include="Net\ATLinkState.cs" />
<Compile Include="Net\cA69Engine.cs" />
<Compile Include="Net\cA69Engine3.cs" />
<Compile Include="Net\cA69Engine2.cs" />
<Compile Include="Net\cA69Engine3.cs" />
<Compile Include="Net\DownloadClient.cs" />
<Compile Include="Net\frmDownloader.cs">
<SubType>Form</SubType>
Src/A69WD/A69WD.csproj.user
+1 -1
View File
@@ -1,6 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<Project ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
<PropertyGroup>
<ProjectView>ProjectFiles</ProjectView>
<ProjectView>ShowAllFiles</ProjectView>
</PropertyGroup>
</Project>
Src/A69WD/Net/cA69Engine2.cs
+12 -43
View File
@@ -13,7 +13,7 @@ using System.Collections;
namespace A69WD {
public class cA69Engine3 {
public class cA69Engine2 {
#region Property: State, NumDownload, Tipologie, BASE_URL, BASE_LINK_IDX, BASE_LINK_RAC
@@ -50,7 +50,7 @@ namespace A69WD {
#region Costruttore e distruttore
public cA69Engine3() {
public cA69Engine2() {
try {
// Inizializzo gli archivi (dai file in memoria)
allLinks = new List<string>();
@@ -90,8 +90,8 @@ namespace A69WD {
// proseguo con il download:
string lk = toDownload.First();
toDownload.Remove(lk);
string fn = Path.Combine(Cfg.Settings.PathAnalizadedRaccFile, GetFileName(lk));
using (MyWebClient client = new MyWebClient(lk, fn)) {
using (WebClient client = new WebClient()) {
string fn = Path.Combine(Cfg.Settings.PathAnalizadedRaccFile, GetFileName(lk));
//client.Encoding = Encoding.ASCII;
client.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0");
client.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
@@ -334,13 +334,7 @@ namespace A69WD {
}
private string GetFileName(string url) {
if (IsRacLink(url)) {
return url.Replace("https://www.annunci69.it/racconti-erotici/", "A69_").Replace("/", "_").Replace(":", "_").Replace(@"\", "_");
} else {
return url.Replace("https://www.annunci69.it/racconti-erotici/", "_A69_").Replace("/", "_").Replace(":", "_").Replace(@"\", "_");
}
}
private string GetFileName(string url) { return url.Replace("https://www.annunci69.it/racconti-erotici/", "A69_").Replace("/", "_").Replace(":", "_").Replace(@"\", "_"); }
static private bool IsIdxLink(string link) { return link.StartsWith(BASE_LINK_IDX); }
static private bool IsRacLink(string link) { foreach (string tip in Tipologie) { if (link.StartsWith(BASE_LINK_RAC + tip + "/")) return true; } return false; }
@@ -473,29 +467,19 @@ namespace A69WD {
conteggioEvt++;
MyWebClient mwb = sender as MyWebClient;
if (state == EngineStatusEnum.Starting) state = EngineStatusEnum.Started;
// aggiorno UI
RaiseEvent_Updating("(Evt" + conteggioEvt.ToString("0000") + ") Donwloaded: " + mwb.URL.Replace("https://www.annunci69.it/racconti-erotici/", "./"));
// Salvo il testo downloadato
if (!e.Cancelled && e.Error == null) {
string textData = e.Result;
string FullFileName = mwb.FileName;
string FullFileName = @"C:\VMS\Prj\C#\A69WD\Test\Txt\Src\2022-09-22\A69_altroZZ.html";
//System.IO.StreamWriter sw = new System.IO.StreamWriter(FullFileName, false, Encoding.Unicode);
System.IO.StreamWriter sw = new System.IO.StreamWriter(FullFileName,false, Encoding.UTF8);
sw.Write(textData);
sw.Flush();
sw.Close();
}
// Se ho raggiunto il massimo numero di link, o non vi sono link da downloadare, mi fermo:
if ((toDownload.Count > 0) && (state == EngineStatusEnum.Started)) {
// proseguo con il download:
string lk = toDownload.First();
toDownload.Remove(lk);
string fn = Path.Combine(Cfg.Settings.PathAnalizadedRaccFile, GetFileName(lk));
using (MyWebClient client = new MyWebClient(lk, fn)) {
using (WebClient client = new WebClient()) {
string fn = Path.Combine(Cfg.Settings.PathAnalizadedRaccFile, GetFileName(lk));
//client.Encoding = Encoding.ASCII;
client.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0");
client.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
@@ -507,15 +491,11 @@ namespace A69WD {
client.DownloadFileCompleted += Client_DownloadFileCompleted;
client.DownloadDataCompleted += Client_DownloadDataCompleted;
client.DownloadStringCompleted += Client_DownloadStringCompleted;
//client.DownloadFileAsync(new Uri(lk), fn);
client.DownloadFileAsync(new Uri(lk), fn);
//client.DownloadDataAsync(new Uri(lk));
client.DownloadStringAsync(new Uri(lk));
//client.DownloadStringAsync(new Uri(fn));
}
} else {
// Finito download:
state = EngineStatusEnum.Stopping;
// salvo gli archivi:
SaveState();
}
}//lock
@@ -535,16 +515,5 @@ namespace A69WD {
}
}//class cA69Engine2
public class MyWebClient : WebClient {
public string URL;
public string FileName;
public MyWebClient(string url, string fileName) {
this.URL = url;
this.FileName = fileName;
}
}
}//namespace A69WD
Src/A69WD/Net/cA69Engine3.cs
+142 -19
View File
@@ -10,10 +10,11 @@ using static System.Windows.Forms.LinkLabel;
using System.ComponentModel;
using static System.Net.Mime.MediaTypeNames;
using System.Collections;
using System.Text.RegularExpressions;
namespace A69WD {
public class cA69Engine2 {
public class cA69Engine3 {
#region Property: State, NumDownload, Tipologie, BASE_URL, BASE_LINK_IDX, BASE_LINK_RAC
@@ -50,7 +51,7 @@ namespace A69WD {
#region Costruttore e distruttore
public cA69Engine2() {
public cA69Engine3() {
try {
// Inizializzo gli archivi (dai file in memoria)
allLinks = new List<string>();
@@ -90,8 +91,8 @@ namespace A69WD {
// proseguo con il download:
string lk = toDownload.First();
toDownload.Remove(lk);
using (WebClient client = new WebClient()) {
string fn = Path.Combine(Cfg.Settings.PathAnalizadedRaccFile, GetFileName(lk));
string fn = Path.Combine(Cfg.Settings.PathAnalizadedRaccFile, GetFileName(lk));
using (MyWebClient client = new MyWebClient(lk, fn)) {
//client.Encoding = Encoding.ASCII;
client.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0");
client.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
@@ -334,7 +335,13 @@ namespace A69WD {
}
private string GetFileName(string url) { return url.Replace("https://www.annunci69.it/racconti-erotici/", "A69_").Replace("/", "_").Replace(":", "_").Replace(@"\", "_"); }
private string GetFileName(string url) {
if (IsRacLink(url)) {
return url.Replace("https://www.annunci69.it/racconti-erotici/", "A69_").Replace("/", "_").Replace(":", "_").Replace(@"\", "_");
} else {
return url.Replace("https://www.annunci69.it/racconti-erotici/", "_A69_").Replace("/", "_").Replace(":", "_").Replace(@"\", "_");
}
}
static private bool IsIdxLink(string link) { return link.StartsWith(BASE_LINK_IDX); }
static private bool IsRacLink(string link) { foreach (string tip in Tipologie) { if (link.StartsWith(BASE_LINK_RAC + tip + "/")) return true; } return false; }
@@ -467,19 +474,71 @@ namespace A69WD {
conteggioEvt++;
if (!e.Cancelled && e.Error == null) {
string textData = e.Result;
string FullFileName = @"C:\VMS\Prj\C#\A69WD\Test\Txt\Src\2022-09-22\A69_altroZZ.html";
//System.IO.StreamWriter sw = new System.IO.StreamWriter(FullFileName, false, Encoding.Unicode);
System.IO.StreamWriter sw = new System.IO.StreamWriter(FullFileName,false, Encoding.UTF8);
sw.Write(textData);
sw.Flush();
sw.Close();
MyWebClient mwb = sender as MyWebClient;
if (state == EngineStatusEnum.Starting) state = EngineStatusEnum.Started;
// Verifico se download corretto
if (!e.Cancelled && e.Error == null) {
// aggiorno UI
RaiseEvent_Updating("(Evt" + conteggioEvt.ToString("0000") + ") Donwloaded: " + mwb.URL.Replace("https://www.annunci69.it/racconti-erotici/", "./"));
// Se pagina è un racconto, lo salvo
if (IsRacLink(mwb.URL)) {
string textData = e.Result;
string FullFileName = mwb.FileName;
System.IO.StreamWriter sw = new System.IO.StreamWriter(FullFileName, false, Encoding.UTF8);
sw.Write(textData);
sw.Flush();
sw.Close();
}
// Registro che ho eseguito il download del link
downloaded.Add(mwb.URL);
// Estraggo tutti i link net testo:
List<string> links = new List<string>();
//if (EstraiA69Link(e.Result, links)) {
if (EstraiLink(e.Result, "/racconti-erotici/", links)) {
// Ciclo su tutti i link estratti dalla pagina:
foreach (string link in links) {
if (!allLinks.Contains(link)) {
// elimino il link di logon
if (link == "https://www.annunci69.it/accedi.php?nop") break;
// Link nuovo, mi comporto di conseguenza alla tipologia:
if (IsRacLink(link)) {
// Link ad un racconto
allLinks.Add(link);
if (downRac) { toDownload.Add(link); }
racLinks.Add(link);
} else {
if (IsIdxLink(link)) {
// Link ad un indice
allLinks.Add(link);
toDownload.Add(link);
} else {
// Link non riconosciuto, lo scarto
;
}
}
}
}
} // fine ciclo sui link estratti
} else {
// Download con errore, informo l'utente
RaiseEvent_Updating("(Evt" + conteggioEvt.ToString("0000") + ") Donwload error: " + mwb.URL.Replace("https://www.annunci69.it/racconti-erotici/", "./"));
}
// Se ho raggiunto il massimo numero di link, o non vi sono link da downloadare, mi fermo:
if ((toDownload.Count > 0) && (state == EngineStatusEnum.Started)) {
// proseguo con il download:
string lk = toDownload.First();
toDownload.Remove(lk);
using (WebClient client = new WebClient()) {
string fn = Path.Combine(Cfg.Settings.PathAnalizadedRaccFile, GetFileName(lk));
string fn = Path.Combine(Cfg.Settings.PathAnalizadedRaccFile, GetFileName(lk));
using (MyWebClient client = new MyWebClient(lk, fn)) {
//client.Encoding = Encoding.ASCII;
client.Headers.Add("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:88.0) Gecko/20100101 Firefox/88.0");
client.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8");
@@ -491,11 +550,15 @@ namespace A69WD {
client.DownloadFileCompleted += Client_DownloadFileCompleted;
client.DownloadDataCompleted += Client_DownloadDataCompleted;
client.DownloadStringCompleted += Client_DownloadStringCompleted;
client.DownloadFileAsync(new Uri(lk), fn);
//client.DownloadFileAsync(new Uri(lk), fn);
//client.DownloadDataAsync(new Uri(lk));
//client.DownloadStringAsync(new Uri(fn));
client.DownloadStringAsync(new Uri(lk));
}
} else {
// Finito download:
state = EngineStatusEnum.Stopping;
// salvo gli archivi:
SaveState();
}
}//lock
@@ -513,7 +576,67 @@ namespace A69WD {
// progressBar.Value = int.Parse(Math.Truncate(percentage).ToString());
//}));
}
}//class cA69Engine2
public static bool EstraiA69Link(string txt, List<string> links) {
try {
Regex regex = new Regex("(?<=href=\")\\S+?(?=\")");
Match matchObj = regex.Match(txt);
while (matchObj.Success) {
if (matchObj.Value.StartsWith("/racconti-erotici/")) { links.Add(matchObj.Value); }
if (matchObj.Groups.Count > 1) {
for (int i = 1; i < matchObj.Groups.Count; i++) {
Group group = matchObj.Groups[i];
if (group.Success) {
if (group.Value.StartsWith("/racconti-erotici/")) { links.Add(group.Value); }
}
}
}
matchObj = matchObj.NextMatch();
}
return true;
} catch { return false; }
}
public static bool EstraiLink(string txt, string startWith, List<string> links) {
try {
Regex regex = new Regex("(?<=href=\")\\S+?(?=\")");
Match matchObj = regex.Match(txt);
while (matchObj.Success) {
if (matchObj.Value.StartsWith(startWith)) { links.Add(matchObj.Value.Replace(startWith, "https://www.annunci69.it/racconti-erotici/")); }
if (matchObj.Groups.Count > 1) {
for (int i = 1; i < matchObj.Groups.Count; i++) {
Group group = matchObj.Groups[i];
if (group.Success) {
if (group.Value.StartsWith(startWith)) { links.Add(group.Value.Replace(startWith, "https://www.annunci69.it/racconti-erotici/")); }
}
}
}
matchObj = matchObj.NextMatch();
}
return true;
} catch { return false; }
}
}//class cA69Engine3
public class MyWebClient : WebClient {
public string URL;
public string FileName;
public MyWebClient(string url, string fileName) {
this.URL = url;
this.FileName = fileName;
}
}
}//namespace A69WD