Mikescher/WordpressEbookScraper2
1
0
Fork 0
Code

Fix download Pale (sometimes missing next-chapter links)

Browse Source
This commit is contained in:
Mike Schwörer 2023-11-12 19:45:37 +01:00
parent 5228f3e840
commit ff68d714ee
Signed by: Mikescher
GPG Key ID: D3C7172E0A70F8CF
1 changed files with 15 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
Scraper/Scraper.cs
View File

@ -510,7 +510,7 @@ public class Scraper
if (next == null) if (next == null)
next = nodeContent.Descendants() next = nodeContent.Descendants()
.Where(p => p.Name.ToLower() == "a") .Where(p => p.Name.ToLower() == "a")
.Where(p => Helper.Striptease(p) == "next chapter" || Helper.Striptease(p) == "next") .Where(p => Helper.Striptease(p) == "next chapter" || Helper.Striptease(p) == "next" || Helper.Striptease(p) == "ext chapt")
.Where(p => p.Attributes.Contains("href")) .Where(p => p.Attributes.Contains("href"))
.FirstOrDefault(); .FirstOrDefault();
@ -523,7 +523,7 @@ public class Scraper
if (next == null) if (next == null)
next = Helper.RecursiveDescendants(nodeContent) next = Helper.RecursiveDescendants(nodeContent)
.Where(p => p.Name.ToLower() == "a") .Where(p => p.Name.ToLower() == "a")
.Where(p => Helper.Striptease(p) == "next chapter" || Helper.Striptease(p) == "next") .Where(p => Helper.Striptease(p) == "next chapter" || Helper.Striptease(p) == "next" || Helper.Striptease(p) == "ext chapt")
.Where(p => p.Attributes.Contains("href")) .Where(p => p.Attributes.Contains("href"))
.FirstOrDefault(); .FirstOrDefault();
@ -533,6 +533,18 @@ public class Scraper
.Where(p => p.Attributes.Any(q => q.Name == "rel" && q.Value == "next")) .Where(p => p.Attributes.Any(q => q.Name == "rel" && q.Value == "next"))
.FirstOrDefault(); .FirstOrDefault();
if (next == null && ACTIVE_BOOK.Title == "Pale")
{
var nextLS = Helper.RecursiveDescendants(doc.DocumentNode)
.Where(p => p.Name.ToLower() == "a")
.Where(p => p.Attributes.Any(q => q.Name == "rel" && q.Value == "next"))
.GroupBy(p => p.Attributes["href"].Value.Trim())
.ToList();
if (nextLS.Count == 1) next = nextLS.Single().FirstOrDefault();
}
if (next != null && next.Attributes["href"].Value.Trim() == "(https://palewebserial.wordpress.com/2023/10/10/end/") next = null; // do not process author-notes from Pale
if (next != null) if (next != null)
{ {
var next_url = next.Attributes["href"].Value.Trim(); var next_url = next.Attributes["href"].Value.Trim();
@ -557,7 +569,7 @@ public class Scraper
} }
} }
if (next == null) prt(" > (!) No next URL found"); if (next == null) prt(" > (!) No next URL found");
#endregion #endregion