From 55a3a10528a574dc781268dccd1e84a54d6a6659 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?old=E6=98=93?= <156663459@qq.com> Date: Wed, 11 Dec 2024 15:34:23 +0800 Subject: [PATCH] =?UTF-8?q?=E5=85=A8=E5=9B=BD=E6=96=B0=E9=97=BB=E8=B5=84?= =?UTF-8?q?=E8=AE=AF=E9=87=87=E9=9B=86=E6=A8=A1=E5=9D=97=E5=AE=8C=E5=96=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../HostedService/JobTimedSpiderService.cs | 103 ++++++++++++------ .../HtmlAgSpider/NationWideNewsAgHelper.cs | 102 +++++++++++++++++ 2 files changed, 174 insertions(+), 31 deletions(-) create mode 100644 New_College.Tasks/HtmlAgSpider/NationWideNewsAgHelper.cs diff --git a/New_College.Tasks/HostedService/JobTimedSpiderService.cs b/New_College.Tasks/HostedService/JobTimedSpiderService.cs index de4a8ac..8955be3 100644 --- a/New_College.Tasks/HostedService/JobTimedSpiderService.cs +++ b/New_College.Tasks/HostedService/JobTimedSpiderService.cs @@ -17,10 +17,12 @@ namespace New_College.Tasks /// /// private ID_NewsInfoServices newsInfoServices; + private ISysRegionServices sysRegionServices; // 这里可以注入 - public JobTimedSpiderService(ID_NewsInfoServices d_NewsInfoServices) + public JobTimedSpiderService(ID_NewsInfoServices d_NewsInfoServices, ISysRegionServices sysRegionServices) { newsInfoServices = d_NewsInfoServices; + this.sysRegionServices = sysRegionServices; } public Task StartAsync(CancellationToken cancellationToken) @@ -32,42 +34,81 @@ namespace New_College.Tasks return Task.CompletedTask; } - private void DoWork(object state) + private async void DoWork(object state) { - try + //try + //{ + // HtmlAgHelper agHelper = new HtmlAgHelper(); + // var list = agHelper.HtmlCreatePageData(); + // list.ForEach(async c => + // { + // var newsinfo = await newsInfoServices.Query(e => e.Title.Trim() == c.title); + // if (!newsinfo.Any() && c.pubtime.Year > DateTime.Now.Year - 1) + // { + + // await newsInfoServices.Add(new Model.Models.D_NewsInfo() + // { + // Author = c.author, + // CategoryId = 1, + // CreateBy = "spdier", + // ProvinceCode = "370000", + // CreateId = 1, + // CreateTime = c.pubtime, + // Detail = c.detail, + // CoverImg = "https://static-data.ycymedu.com/static/newstop.png", + // OrderSort = 0, + // IsDelete = false, + // Title = c.title, + // Summary = HtmlHelper.ReplaceHtmlTag(c.detail).Length > 200 ? HtmlHelper.ReplaceHtmlTag(c.detail).Substring(0, 200) : HtmlHelper.ReplaceHtmlTag(c.detail) + // }); + + // } + // }); + // ConsoleHelper.WriteWarningLine($"Job spider success： {DateTime.Now}-{list.Count}"); + //} + //catch (Exception ex) + //{ + // ConsoleHelper.WriteWarningLine($"Job spider 抓取异常"); + //} + NationWideNewsAgHelper anghelper = new NationWideNewsAgHelper(); + var provineInfo = await this.sysRegionServices.Query(e => e.Level == 1 && !e.RegionCode.Contains("-")); + provineInfo.ForEach(p => { - HtmlAgHelper agHelper = new HtmlAgHelper(); - var list = agHelper.HtmlCreatePageData(); - list.ForEach(async c => + try { - var newsinfo = await newsInfoServices.Query(e => e.Title.Trim() == c.title); - if (!newsinfo.Any() && c.pubtime.Year > DateTime.Now.Year - 1) + var list = anghelper.HtmlCreatePageData(p.RegionCode); + list.ForEach(async c => { - - await newsInfoServices.Add(new Model.Models.D_NewsInfo() + var newsinfo = await newsInfoServices.Query(e => e.Title.Trim() == c.title); + if (!newsinfo.Any() && c.pubtime.Year > DateTime.Now.Year - 1) { - Author = c.author, - CategoryId = 1, - CreateBy = "spdier", - ProvinceCode = "370000", - CreateId = 1, - CreateTime = c.pubtime, - Detail = c.detail, - CoverImg = "https://static-data.ycymedu.com/static/newstop.png", - OrderSort = 0, - IsDelete = false, - Title = c.title, - Summary = HtmlHelper.ReplaceHtmlTag(c.detail).Length > 200 ? HtmlHelper.ReplaceHtmlTag(c.detail).Substring(0, 200) : HtmlHelper.ReplaceHtmlTag(c.detail) - }); + await newsInfoServices.Add(new Model.Models.D_NewsInfo() + { + Author = c.author, + CategoryId = 1, + CreateBy = "spdier", + ProvinceCode = p.RegionCode, + CreateId = 1, + CreateTime = c.pubtime, + Detail = c.detail, + CoverImg = "https://static-data.ycymedu.com/static/newstop.png", + OrderSort = 0, + IsDelete = false, + Title = c.title, + Summary = HtmlHelper.ReplaceHtmlTag(c.detail).Length > 200 ? HtmlHelper.ReplaceHtmlTag(c.detail).Substring(0, 200) : HtmlHelper.ReplaceHtmlTag(c.detail) + }); + + } + }); + ConsoleHelper.WriteWarningLine($"Job spider success： {DateTime.Now}-{list.Count}"); + } + catch (Exception ex) + { + ConsoleHelper.WriteWarningLine($"Job spider 抓取异常"); + } + }); + - } - }); - ConsoleHelper.WriteWarningLine($"Job spider success： {DateTime.Now}-{list.Count}"); - } - catch (Exception ex) - { - ConsoleHelper.WriteWarningLine($"Job spider 抓取异常"); - } } public Task StopAsync(CancellationToken cancellationToken) diff --git a/New_College.Tasks/HtmlAgSpider/NationWideNewsAgHelper.cs b/New_College.Tasks/HtmlAgSpider/NationWideNewsAgHelper.cs new file mode 100644 index 0000000..a442a7a --- /dev/null +++ b/New_College.Tasks/HtmlAgSpider/NationWideNewsAgHelper.cs @@ -0,0 +1,102 @@ +using Aliyun.OSS; +using HtmlAgilityPack; +using Microsoft.AspNetCore.Mvc.RazorPages; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Text; +using System.Threading.Tasks; +using System.Text.Json; +using New_College.Common.Helper; +using System.Threading; +namespace New_College.Tasks +{ + public class NationWideNewsAgHelper + { + + private string itemUrl = "https://gaokao.chsi.com.cn"; + public List HtmlCreatePageData(string provinceCode) + { + HtmlWeb webClient = new HtmlWeb(); + + var apiUrl = "https://gaokao.chsi.com.cn/wap/news/search/5018267?ps=20&ss="; + var list = new List(); + var baseUrl = $"{apiUrl}{provinceCode.Replace("0000", "")}&_t={DateTimeOffset.Now.ToUnixTimeSeconds()}"; + using var httpClient = new HttpClient(); + var response = httpClient.GetAsync(baseUrl).Result; + if (response.IsSuccessStatusCode) + { + var jsonData = response.Content.ReadAsStringAsync().Result; + Console.WriteLine(jsonData); + var resultlist = JsonSerializer.Deserialize(jsonData); + if (resultlist.msg.Any()) + { + resultlist.msg.ForEach(o => + { + Thread.Sleep(100); + string newsUrl = $"{itemUrl}{o.uri}"; + HtmlDocument doc = webClient.Load(newsUrl); + var inntertitle = doc.DocumentNode.SelectSingleNode("//*[@id=\"app\"]/div[2]/h2").InnerText; + var author = doc.DocumentNode.SelectSingleNode("//*[@id=\"app\"]/div[3]/div[1]/span[2]").InnerText.Replace("来源：", ""); + var createtime = doc.DocumentNode.SelectSingleNode("//*[@id=\"app\"]/div[3]/div[1]/span[1]").InnerText; + foreach (var linkNode in doc.DocumentNode.SelectNodes("//a[@href]") ?? new HtmlNodeCollection(null)) + { + string href = linkNode.GetAttributeValue("href", ""); + if (!string.IsNullOrEmpty(href) && href.StartsWith("/")) + { + string fullUrl = "https://gaokao.chsi.com.cn" + href; + linkNode.SetAttributeValue("href", fullUrl); + } + } + var innerhtml = doc.DocumentNode.SelectSingleNode("//*[@id=\"article_dnull\"]").InnerHtml; + if (!HtmlHelper.ReplaceHtmlTag(innerhtml).Contains("浏览器")) + { + + if (innerhtml.Contains(".pdf") || innerhtml.Contains(".doc") || innerhtml.Contains(".docx") || innerhtml.Contains(".xls") || innerhtml.Contains(".xlsx")) + { + innerhtml = $"{innerhtml}\n若有附件详情，请至本省招生考试院下载附件!!!"; + } + + list.Add(new NewsModels() + { + title = inntertitle, + author = author, + pubtime = Convert.ToDateTime(createtime), + detail = innerhtml.Replace("src=\"", "src=\"https://gaokao.chsi.com.cn") + }); + } + }); + } + } + else + { + Console.WriteLine($"Failed to fetch page {baseUrl}, Status Code: {response.StatusCode}"); + } + + + + + + + + + return list; + } + } +} + +public class GaokaoObject +{ + public List msg { get; set; } + public bool flag { get; set; } +} + +public class Msg +{ + public string title { get; set; } + public string truncTitle { get; set; } + public string uri { get; set; } + public string displayDate { get; set; } +}