using New_College.Common.Helper;
using Microsoft.Extensions.Hosting;
using System;
using System.Threading;
using System.Threading.Tasks;
using New_College.IServices;
using System.Linq;
using New_College.Model.Models;
using System.Collections.Generic;
namespace New_College.Tasks
{
public class JobTimedSpiderService : IHostedService, IDisposable
{
private Timer _timer;
///
///
///
private ID_NewsInfoServices newsInfoServices;
private ISysRegionServices sysRegionServices;
// 这里可以注入
public JobTimedSpiderService(ID_NewsInfoServices d_NewsInfoServices, ISysRegionServices sysRegionServices)
{
newsInfoServices = d_NewsInfoServices;
this.sysRegionServices = sysRegionServices;
}
public Task StartAsync(CancellationToken cancellationToken)
{
Console.WriteLine("Job spider is starting.");
_timer = new Timer(DoWork, null, TimeSpan.Zero,
TimeSpan.FromSeconds(60 * 60 * 8));//两个小时
return Task.CompletedTask;
}
private async void DoWork(object state)
{
// BatchListAgHelper batchListAgHelper = new BatchListAgHelper();
NationWideNewsAgHelper anghelper = new NationWideNewsAgHelper();
var provineInfo = await this.sysRegionServices.Query(e => e.Level == 1 && !e.RegionCode.Contains("-"));
provineInfo.ForEach(p =>
{
//var years = new List() { "2024", "2023", "2022" };
//years.ForEach(y =>
//{
// batchListAgHelper.HtmlCreatePageData(p.RegionCode, y);
//});
try
{
var list = anghelper.HtmlCreatePageData(p.RegionCode);
list.ForEach(async c =>
{
c.title = c.title.Replace($"{p.SimpleName}:", "");
var newsinfo = await newsInfoServices.Query(e => e.Title.Trim() == c.title);
if (!newsinfo.Any() && c.pubtime.Year > DateTime.Now.Year - 1)
{
await newsInfoServices.Add(new Model.Models.D_NewsInfo()
{
Author = c.author,
CategoryId = 1,
CreateBy = "spdier",
ProvinceCode = p.RegionCode,
CreateId = 1,
CreateTime = c.pubtime,
Detail = c.detail,
CoverImg = "https://static-data.ycymedu.com/static/newstop.png",
OrderSort = 0,
IsDelete = false,
Title = c.title,
Summary = HtmlHelper.ReplaceHtmlTag(c.detail).Length > 200 ? HtmlHelper.ReplaceHtmlTag(c.detail).Substring(0, 200) : HtmlHelper.ReplaceHtmlTag(c.detail)
});
}
});
ConsoleHelper.WriteWarningLine($"Job spider success: {DateTime.Now}-{list.Count}");
}
catch (Exception ex)
{
ConsoleHelper.WriteWarningLine($"Job spider 抓取异常");
}
});
}
public Task StopAsync(CancellationToken cancellationToken)
{
Console.WriteLine("Job spider is stopping.");
_timer?.Change(Timeout.Infinite, 0);
return Task.CompletedTask;
}
public void Dispose()
{
_timer?.Dispose();
}
}
}