MrkPi/A69WD
3
0 Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

Upgradato lo skraping impostando salvataggi diversi per ciasucn file.

Browse Source
This commit is contained in:
CSiCoAlm committed 2026-01-01 15:10:37 +01:00
1 parent 1c372a3632
commit cc828ff4ee
7 files changed
+167 -76
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

Src/A69WD/Data/DBTools.cs
+1 -1
View File
@@ -1075,7 +1075,7 @@ namespace A69WD {
/// Importa il racconto passato sul DB
/// </summary>
/// <param name="rac">Racconto d aimportare</param>
/// <returns>1 se tutto ok, -1 se importazione abortita per errore, -10 se importazione abortita per duplicato, ....</returns>
/// <returns>IDrac se tutto ok, -1 se importazione abortita per errore, -10 se importazione abortita per duplicato, ....</returns>
public int ImportRac(cRac rac) {
lock (objLock) {
// Inizializzo le variabili che occorrono
Src/A69WD/General/Cfg.cs
+6
View File
@@ -237,6 +237,12 @@ namespace A69WD {
}
private string pathAnalizadedRaccFile = "";
public virtual string PathSrcDone => System.IO.Path.Combine(PathDownloadWorking, "Done");
public virtual string PathIdxDone => System.IO.Path.Combine(PathDownloadWorking, "Idx");
public virtual string PathSrcErrOnExtract => System.IO.Path.Combine(PathDownloadWorking, "ErrOnExtract");
public virtual string PathSrcErrOnSave => System.IO.Path.Combine(PathDownloadWorking, "ErrOnSave");
#endregion
Src/A69WD/Service/Service_Crawler.cs
+8
View File
@@ -1,6 +1,7 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Text;
@@ -73,6 +74,13 @@ namespace A69WD.Service {
private void ProcessQueue() {
foreach (iLink link in _urlQueue.GetConsumingEnumerable()) {
Logger.Information("ProcessQueue: Start url " + link.FileName);
// Verifico cartelle di destinazione
if (!Directory.Exists(Cfg.Settings.PathIdxDone)) Directory.CreateDirectory(Cfg.Settings.PathIdxDone);
if (!Directory.Exists(Cfg.Settings.PathSrcDone)) Directory.CreateDirectory(Cfg.Settings.PathSrcDone);
if (!Directory.Exists(Cfg.Settings.PathSrcErrOnExtract)) Directory.CreateDirectory(Cfg.Settings.PathSrcErrOnExtract);
if (!Directory.Exists(Cfg.Settings.PathSrcErrOnSave)) Directory.CreateDirectory(Cfg.Settings.PathSrcErrOnSave);
Task.Run(async () => {
Interlocked.Increment(ref _inCorso);
UpdateCounters();
Src/A69WD/Service/Service_Importer.cs
+7 -9
View File
@@ -95,12 +95,10 @@ namespace A69WD.Service {
UpdateCounters();
// Verifico cartelle di destinazione
string pathErrOnExtract = System.IO.Path.Combine(Cfg.Settings.PathDownloadWorking, "ErrOnExtract");
if (!Directory.Exists(pathErrOnExtract)) Directory.CreateDirectory(pathErrOnExtract);
string pathErrOnSave = System.IO.Path.Combine(Cfg.Settings.PathDownloadWorking, "ErrOnSave");
if (!Directory.Exists(pathErrOnSave)) Directory.CreateDirectory(pathErrOnSave);
string pathDone = System.IO.Path.Combine(Cfg.Settings.PathDownloadWorking, "Done");
if (!Directory.Exists(pathDone)) Directory.CreateDirectory(pathDone);
if (!Directory.Exists(Cfg.Settings.PathSrcDone)) Directory.CreateDirectory(Cfg.Settings.PathSrcDone);
if (!Directory.Exists(Cfg.Settings.PathSrcErrOnExtract)) Directory.CreateDirectory(Cfg.Settings.PathSrcErrOnExtract);
if (!Directory.Exists(Cfg.Settings.PathSrcErrOnSave)) Directory.CreateDirectory(Cfg.Settings.PathSrcErrOnSave);
// Ciclo su ciascun file
foreach (string file in fileCasuali) {
string name = System.IO.Path.GetFileName(file);
@@ -117,19 +115,19 @@ namespace A69WD.Service {
Logger.Warning($"Import - errore nel salvataggio: {name}");
Interlocked.Increment(ref _errori);
// - Sposto il file
System.IO.File.Move(file, System.IO.Path.Combine(pathErrOnSave, System.IO.Path.GetFileName(file)));
System.IO.File.Move(file, System.IO.Path.Combine(Cfg.Settings.PathSrcErrOnSave, System.IO.Path.GetFileName(file)));
} else {
// salvataggio ok
Logger.Warning($"Import - racconto salvato: {name}");
Interlocked.Increment(ref _completati);
// - Sposto il file
System.IO.File.Move(file, System.IO.Path.Combine(pathDone, System.IO.Path.GetFileName(file)));
System.IO.File.Move(file, System.IO.Path.Combine(Cfg.Settings.PathSrcDone, System.IO.Path.GetFileName(file)));
}
} else {
Logger.Warning($"Import - estrazione errata: {name}");
Interlocked.Increment(ref _errori);
// - Sposto il file
System.IO.File.Move(file, System.IO.Path.Combine(pathErrOnExtract, System.IO.Path.GetFileName(file)));
System.IO.File.Move(file, System.IO.Path.Combine(Cfg.Settings.PathSrcErrOnExtract, System.IO.Path.GetFileName(file)));
}
UpdateCounters();
}
Src/A69WD/Site/cSite_A69.cs
+76 -50
View File
@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
@@ -8,6 +9,7 @@ using System.Threading.Tasks;
using A69WD.Data;
using A69WD.Net;
using HtmlAgilityPack;
using static System.Windows.Forms.VisualStyles.VisualStyleElement.TrackBar;
namespace A69WD.Site {
@@ -107,9 +109,11 @@ namespace A69WD.Site {
/// <returns></returns>
public async Task<List<iLink>> CrawlPageAsync(iLink lk) {
var newlinks = new List<iLink>();
try {
Console.WriteLine($"A69 - Scarico: {lk.URL}");
HtmlDocument tdoc = null;
string thtml = "";
#region ToConfirm
if (lk.Status == eLinkState.ToConfirm) {
@@ -135,13 +139,11 @@ namespace A69WD.Site {
return newlinks;
}
// Scarico la pagina
var thtml = await client.GetStringAsync(lk.URL);
thtml = await client.GetStringAsync(lk.URL);
tdoc = new HtmlDocument();
tdoc.LoadHtml(thtml);
// Salvo il file downloadato
//System.IO.StreamWriter tsw = new System.IO.StreamWriter(lk.DstFilePath, false, Encoding.UTF8);
System.IO.StreamWriter tsw = new System.IO.StreamWriter(System.IO.Path.Combine(lk.FilePath, lk.FileName), false, Encoding.UTF8);
tsw.Write(thtml);
//SaveToFile(thtml, lk.FilePath, lk.FileName);
lk.Status = eLinkState.ToAnalize;
}
@@ -151,6 +153,7 @@ namespace A69WD.Site {
{
if (lk.Status == eLinkState.ToAnalize) {
List<string> tmpLk = new List<string>();
#region Estraggo gli url ad indici:
{
/*
@@ -169,30 +172,33 @@ namespace A69WD.Site {
* </div>
*
*/
// Estraggo i link alle altre pagine:
var tnodes = tdoc.DocumentNode.SelectNodes("//div[@id='pagination']//ul//li//a[@href]");
if (tnodes != null) {
foreach (var link in tnodes) {
var href = link.GetAttributeValue("href", "");
if (!string.IsNullOrEmpty(href) && href.StartsWith("/racconti-erotici/racconti_lista.php?genere="))
tmpLk.Add(href); //tmpLk.Add(cLink_A69.PageBase + href);
}
}
// Rimuove duplicati e ordina alfabeticamente
tmpLk = tmpLk
.OrderBy(s => s) // ordina alfabeticamente
.Distinct() // elimina duplicati
.ToList();
// Inserisco i link trovati
try {
foreach (string ilk in tmpLk) {
string[] separator = new string[] { "&pagina=" };
string[] txt = ilk.Split(separator, StringSplitOptions.RemoveEmptyEntries);
string url = cLink_A69.PageBase + ilk;
string name = txt[0].Replace("/racconti-erotici/racconti_lista.php?genere=", "_A69_") + "_" + int.Parse(txt[1]).ToString("0000") + ".html";
newlinks.Add(new cLink_A69(url, name));
// Estraggo i link alle altre pagine:
var tnodes = tdoc.DocumentNode.SelectNodes("//div[@id='pagination']//ul//li//a[@href]");
if (tnodes != null) {
foreach (var link in tnodes) {
var href = link.GetAttributeValue("href", "");
if (!string.IsNullOrEmpty(href) && href.StartsWith("/racconti-erotici/racconti_lista.php?genere="))
tmpLk.Add(href); //tmpLk.Add(cLink_A69.PageBase + href);
}
}
} catch (Exception ex) { Logger.Error("Errore analisi Idx link dal " + lk.FileName + " - " + ex.Message); }
// Rimuove duplicati e ordina alfabeticamente
tmpLk = tmpLk
.OrderBy(s => s) // ordina alfabeticamente
.Distinct() // elimina duplicati
.ToList();
// MRK // Inserisco i link trovati
// MRK foreach (string ilk in tmpLk) {
// MRK string[] separator = new string[] { "&pagina=" };
// MRK string[] txt = ilk.Split(separator, StringSplitOptions.RemoveEmptyEntries);
// MRK string url = cLink_A69.PageBase + ilk;
// MRK string name = txt[0].Replace("/racconti-erotici/racconti_lista.php?genere=", "_A69_") + "_" + int.Parse(txt[1]).ToString("0000") + ".html";
// MRK newlinks.Add(new cLink_A69(url, name));
// MRK }
} catch (Exception ex) {
Logger.Error("Errore analisi Idx link dal " + lk.FileName + " - " + ex.Message);
SaveToFile(thtml, Cfg.Settings.PathSrcErrOnExtract, lk.FileName);
}
}
#endregion
@@ -205,30 +211,33 @@ namespace A69WD.Site {
* <a href="/racconti-erotici/altro/Tutto-e-iniziato-per-gioco-pazza-famiglia_166770.html">Continua a leggere » </a>
* </div>
*/
// Estraggo i link ai racconti:
tmpLk = new List<string>();
// Estraggo i link alle altre pagine:
var tnodes = tdoc.DocumentNode.SelectNodes("//div[@class='continua']//a[@href]");
if (tnodes != null) {
foreach (var link in tnodes) {
var href = link.GetAttributeValue("href", "");
if (!string.IsNullOrEmpty(href) && href.StartsWith("/racconti-erotici/") && (href.EndsWith(".html") || href.EndsWith(".htm")))
tmpLk.Add(href); //tmpLk.Add(cLink_A69.PageBase + href);
}
}
// Rimuove duplicati e ordina alfabeticamente
tmpLk = tmpLk
.OrderBy(s => s) // ordina alfabeticamente
.Distinct() // elimina duplicati
.ToList();
// Inserisco i link trovati
try {
foreach (string ilk in tmpLk) {
string url = cLink_A69.PageBase + ilk;
string name = ilk.Replace("/racconti-erotici/", "A69_").Replace(@"/","_");
newlinks.Add(new cLink_A69(url, name));
// Estraggo i link ai racconti:
tmpLk = new List<string>();
// Estraggo i link alle altre pagine:
var tnodes = tdoc.DocumentNode.SelectNodes("//div[@class='continua']//a[@href]");
if (tnodes != null) {
foreach (var link in tnodes) {
var href = link.GetAttributeValue("href", "");
if (!string.IsNullOrEmpty(href) && href.StartsWith("/racconti-erotici/") && (href.EndsWith(".html") || href.EndsWith(".htm")))
tmpLk.Add(href); //tmpLk.Add(cLink_A69.PageBase + href);
}
}
} catch (Exception ex) { Logger.Error("Errore analisi Idx link dal " + lk.FileName + " - " + ex.Message); }
// Rimuove duplicati e ordina alfabeticamente
tmpLk = tmpLk
.OrderBy(s => s) // ordina alfabeticamente
.Distinct() // elimina duplicati
.ToList();
// MRK // Inserisco i link trovati
// MRK foreach (string ilk in tmpLk) {
// MRK string url = cLink_A69.PageBase + ilk;
// MRK string name = ilk.Replace("/racconti-erotici/", "A69_").Replace(@"/", "_");
// MRK newlinks.Add(new cLink_A69(url, name));
// MRK }
} catch (Exception ex) {
Logger.Error("Errore analisi Idx link dal " + lk.FileName + " - " + ex.Message);
SaveToFile(thtml, Cfg.Settings.PathSrcErrOnExtract, lk.FileName);
}
}
#endregion
@@ -238,8 +247,21 @@ namespace A69WD.Site {
cRac rac = lk.EstraiRac(tdoc);
if (rac != null) {
// Racconto estratto correttamente, lo salvo sul DB
App.DB.ImportRac(rac);
int res = App.DB.ImportRac(rac);
if (res>0) {
// Racconto salvato correttamente sul DB
SaveToFile(thtml, Cfg.Settings.PathSrcDone, lk.FileName);
} else {
// Errore nel salvataggio sul DB
SaveToFile(thtml, Cfg.Settings.PathSrcErrOnSave, lk.FileName);
}
} else {
// Estrazione del racconto fallita
SaveToFile(thtml, Cfg.Settings.PathSrcErrOnExtract, lk.FileName);
}
} else {
// Link ad un indice
SaveToFile(thtml, Cfg.Settings.PathIdxDone, lk.FileName);
}
#endregion
@@ -274,10 +296,14 @@ namespace A69WD.Site {
} catch (Exception ex) {
Console.WriteLine($"Errore con {lk.URL}: {ex.Message}");
lk.Status = eLinkState.Error;
//App.DB.LinkLog(lk);
App.DB.LinkLog(lk);
}
return newlinks;
}
private void SaveToFile(string txtHTML, string filePath, string fileName) {
System.IO.StreamWriter tsw = new System.IO.StreamWriter(System.IO.Path.Combine(filePath, fileName), false, Encoding.UTF8);
tsw.Write(txtHTML);
}
public async Task<List<iLink>> CrawlPageAsyncTest1(iLink lk) {
var newlinks = new List<iLink>();
Src/A69WD/UI/frmReader.Designer.cs
Generated
+40 -16
View File
@@ -31,21 +31,22 @@
this.btnTest = new System.Windows.Forms.Button();
this.btnTest2 = new System.Windows.Forms.Button();
this.btnTest3 = new System.Windows.Forms.Button();
this.pnlTop = new System.Windows.Forms.Panel();
this.btn = new System.Windows.Forms.Button();
this.sb.SuspendLayout();
this.pnlTop.SuspendLayout();
this.SuspendLayout();
//
// rtb
//
this.rtb.Anchor = ((System.Windows.Forms.AnchorStyles)((((System.Windows.Forms.AnchorStyles.Top | System.Windows.Forms.AnchorStyles.Bottom)
| System.Windows.Forms.AnchorStyles.Left)
| System.Windows.Forms.AnchorStyles.Right)));
this.rtb.BackColor = System.Drawing.Color.White;
this.rtb.BorderStyle = System.Windows.Forms.BorderStyle.None;
this.rtb.Dock = System.Windows.Forms.DockStyle.Fill;
this.rtb.Font = new System.Drawing.Font("Verdana", 11.25F, System.Drawing.FontStyle.Regular, System.Drawing.GraphicsUnit.Point, ((byte)(0)));
this.rtb.Location = new System.Drawing.Point(0, 54);
this.rtb.Location = new System.Drawing.Point(0, 42);
this.rtb.Name = "rtb";
this.rtb.Size = new System.Drawing.Size(700, 366);
this.rtb.TabIndex = 3;
this.rtb.Size = new System.Drawing.Size(700, 378);
this.rtb.TabIndex = 2;
this.rtb.Text = "";
this.rtb.KeyDown += new System.Windows.Forms.KeyEventHandler(this.KeyDownAnalisi);
//
@@ -59,7 +60,7 @@
this.sb.Location = new System.Drawing.Point(0, 420);
this.sb.Name = "sb";
this.sb.Size = new System.Drawing.Size(700, 24);
this.sb.TabIndex = 2;
this.sb.TabIndex = 1;
this.sb.Text = "statusStrip1";
//
// sbMessage
@@ -97,9 +98,9 @@
//
// btnTest
//
this.btnTest.Location = new System.Drawing.Point(4, 4);
this.btnTest.Location = new System.Drawing.Point(11, 5);
this.btnTest.Name = "btnTest";
this.btnTest.Size = new System.Drawing.Size(62, 48);
this.btnTest.Size = new System.Drawing.Size(62, 32);
this.btnTest.TabIndex = 4;
this.btnTest.Text = "Test";
this.btnTest.UseVisualStyleBackColor = true;
@@ -107,9 +108,9 @@
//
// btnTest2
//
this.btnTest2.Location = new System.Drawing.Point(72, 4);
this.btnTest2.Location = new System.Drawing.Point(79, 5);
this.btnTest2.Name = "btnTest2";
this.btnTest2.Size = new System.Drawing.Size(62, 48);
this.btnTest2.Size = new System.Drawing.Size(62, 32);
this.btnTest2.TabIndex = 5;
this.btnTest2.Text = "Test 2";
this.btnTest2.UseVisualStyleBackColor = true;
@@ -117,24 +118,44 @@
//
// btnTest3
//
this.btnTest3.Location = new System.Drawing.Point(140, 4);
this.btnTest3.Location = new System.Drawing.Point(147, 5);
this.btnTest3.Name = "btnTest3";
this.btnTest3.Size = new System.Drawing.Size(62, 48);
this.btnTest3.Size = new System.Drawing.Size(62, 32);
this.btnTest3.TabIndex = 6;
this.btnTest3.Text = "Test 3";
this.btnTest3.UseVisualStyleBackColor = true;
this.btnTest3.Click += new System.EventHandler(this.btnTest3_Click);
//
// pnlTop
//
this.pnlTop.Controls.Add(this.btn);
this.pnlTop.Controls.Add(this.btnTest3);
this.pnlTop.Controls.Add(this.btnTest);
this.pnlTop.Controls.Add(this.btnTest2);
this.pnlTop.Dock = System.Windows.Forms.DockStyle.Top;
this.pnlTop.Location = new System.Drawing.Point(0, 0);
this.pnlTop.Name = "pnlTop";
this.pnlTop.Size = new System.Drawing.Size(700, 42);
this.pnlTop.TabIndex = 0;
//
// btn
//
this.btn.Location = new System.Drawing.Point(215, 5);
this.btn.Name = "btn";
this.btn.Size = new System.Drawing.Size(62, 32);
this.btn.TabIndex = 7;
this.btn.Text = "Test 4";
this.btn.UseVisualStyleBackColor = true;
this.btn.Click += new System.EventHandler(this.btn_Click);
//
// frmReader
//
this.AutoScaleDimensions = new System.Drawing.SizeF(6F, 13F);
this.AutoScaleMode = System.Windows.Forms.AutoScaleMode.Font;
this.ClientSize = new System.Drawing.Size(700, 444);
this.ControlBox = false;
this.Controls.Add(this.btnTest3);
this.Controls.Add(this.btnTest2);
this.Controls.Add(this.btnTest);
this.Controls.Add(this.rtb);
this.Controls.Add(this.pnlTop);
this.Controls.Add(this.sb);
this.Name = "frmReader";
this.Text = "PlugIn_App_Reader.UI Reader (DB vrs1.3, App" +
@@ -142,6 +163,7 @@
this.FormClosing += new System.Windows.Forms.FormClosingEventHandler(this.frmReader_FormClosing);
this.sb.ResumeLayout(false);
this.sb.PerformLayout();
this.pnlTop.ResumeLayout(false);
this.ResumeLayout(false);
}
@@ -156,5 +178,7 @@
private System.Windows.Forms.Button btnTest;
private System.Windows.Forms.Button btnTest2;
private System.Windows.Forms.Button btnTest3;
private System.Windows.Forms.Panel pnlTop;
private System.Windows.Forms.Button btn;
}
}
Src/A69WD/UI/frmReader.cs
+29
View File
@@ -850,6 +850,35 @@ namespace A69WD.UI {
}
private async void btn_Click(object sender, EventArgs e) {
// - Creo istanza del client per il sito con autenticazione:
sbStatusLabel.Text = "Login on A69";
siteA69 = new cSite_A69();
// - Eseguo l'autenticazione
bool res = await siteA69.Authenticate();
if (res) {
sbStatusLabel.Text = "A69 autenticated";
Logger.Information("Sito A69 loggato: " + res.ToString());
} else {
Logger.Information("Login Fault");
return;
}
// - Verifico path per il download
if (!System.IO.Directory.Exists(Cfg.Settings.PathDownloadWorking)) System.IO.Directory.CreateDirectory(Cfg.Settings.PathDownloadWorking);
// Preparo il servizio
string url = "https://www.annunci69.it/racconti-erotici/racconti_lista.php?genere=altro&pagina=1";
string name = "_A69_altro_0001.html";
serviceCrawler = new Service_Crawler(sb);
// Inserisco i link iniziali nella coda
serviceCrawler.EnqueueUrl(new cLink_A69(url, name));
// avvio il servizio
serviceCrawler.StartWorker(siteA69);
}
}