From 8b5e73f664fca40cfcaac56a1b8348e05a017d15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?old=E6=98=93?= <156663459@qq.com> Date: Thu, 12 Dec 2024 15:55:09 +0800 Subject: [PATCH] bug fixed --- New_College.Model/ViewModels/BaseResult.cs | 68 +++++++++++++++ .../ViewModels/Query/SubjectSelectQuery.cs | 4 + .../T_EnrollmentPlanedescServices.cs | 84 ++++++++++++------- .../HostedService/JobTimedSpiderService.cs | 38 +-------- .../HtmlAgSpider/BatchListAgHelper.cs | 49 +++++++++++ 5 files changed, 177 insertions(+), 66 deletions(-) create mode 100644 New_College.Tasks/HtmlAgSpider/BatchListAgHelper.cs diff --git a/New_College.Model/ViewModels/BaseResult.cs b/New_College.Model/ViewModels/BaseResult.cs index 4a12660..6f09768 100644 --- a/New_College.Model/ViewModels/BaseResult.cs +++ b/New_College.Model/ViewModels/BaseResult.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Drawing.Drawing2D; using System.Text; namespace New_College.Model.ViewModels @@ -31,3 +32,70 @@ namespace New_College.Model.ViewModels public bool success { get; set; } = true; } } + +public class Batchobject +{ + public int code { get; set; } + public string msg { get; set; } + public BatchData data { get; set; } +} + +public class BatchData +{ + public Configlist configList { get; set; } + public List list { get; set; } +} + +public class Configlist +{ + public string proDefault { get; set; } + public int yearDefault { get; set; } + public int subjectDefault { get; set; } + public Prolist[] proList { get; set; } + public Yearlist[] yearList { get; set; } + public Subjectlist[] subjectList { get; set; } + public Batchlist[] batchList { get; set; } +} + +public class Prolist +{ + public int id { get; set; } + public string name { get; set; } + public int parent_id { get; set; } + public string code { get; set; } + public int order { get; set; } + public int parent { get; set; } + public int list_order_by_py { get; set; } + public bool selected { get; set; } +} + +public class Yearlist +{ + public int year { get; set; } + public bool selected { get; set; } +} + +public class Subjectlist +{ + public int subject_id { get; set; } + public string subject_name { get; set; } + public bool selected { get; set; } +} + +public class Batchlist +{ + public int batch_id { get; set; } + public string batch_name { get; set; } +} + +public class ListItem +{ + public int year { get; set; } + public int province_code { get; set; } + public int subject_id { get; set; } + public int batch_id { get; set; } + public int score { get; set; } + public string province_name { get; set; } + public string batch_name { get; set; } + public string subject_name { get; set; } +} diff --git a/New_College.Model/ViewModels/Query/SubjectSelectQuery.cs b/New_College.Model/ViewModels/Query/SubjectSelectQuery.cs index 1c10b84..4de3edd 100644 --- a/New_College.Model/ViewModels/Query/SubjectSelectQuery.cs +++ b/New_College.Model/ViewModels/Query/SubjectSelectQuery.cs @@ -98,5 +98,9 @@ namespace New_College.Model.ViewModels /// 省份名称 /// public string AreaName { get; set; } + + + public string ProvinceCode { get; set; } + } } diff --git a/New_College.Services/T_EnrollmentPlanedescServices.cs b/New_College.Services/T_EnrollmentPlanedescServices.cs index e38a133..b2fc6a3 100644 --- a/New_College.Services/T_EnrollmentPlanedescServices.cs +++ b/New_College.Services/T_EnrollmentPlanedescServices.cs @@ -19,6 +19,7 @@ using SqlSugar; using System.Linq.Expressions; using static New_College.Services.T_EnrollmentPlaneServices; using NPOI.SS.Formula.Functions; +using System.Net.Http; namespace New_College.Services { @@ -1608,40 +1609,61 @@ namespace New_College.Services /// public async Task>> GetBatchByYearArea(YearAreaQuery query) { - query.Year = query.Year.HasValue && query.Year >= 2024 ? 2024 : query.Year;//获取批次信息新数据出来2023后需更新成2024 - Expression> expression = Expressionable.Create() - .And(c => c.IsDelete == false) - .And(c => !c.Batch_name.Contains("提前")) - .And(c => !c.Batch_name.Contains("零志愿")) - .And(c => c.AreaName == query.AreaName) - .AndIF(query.Year.HasValue, c => c.Year == query.Year) - .ToExpression(); - var info = await t_EnrollmentBatchRepository.Query(expression); - if (info.Count <= 0) - return new MessageModel>() { success = false, msg = "数据为空" }; - List list = new List() { }; - foreach (var item in info.OrderByDescending(c => c.Year).ToList()) + //query.Year = query.Year.HasValue && query.Year >= 2024 ? 2024 : query.Year;//获取批次信息新数据出来2023后需更新成2024 + //Expression> expression = Expressionable.Create() + // .And(c => c.IsDelete == false) + // .And(c => !c.Batch_name.Contains("提前")) + // .And(c => !c.Batch_name.Contains("零志愿")) + // .And(c => c.AreaName == query.AreaName) + // .AndIF(query.Year.HasValue, c => c.Year == query.Year) + // .ToExpression(); + //var info = await t_EnrollmentBatchRepository.Query(expression); + //if (info.Count <= 0) + // return new MessageModel>() { success = false, msg = "数据为空" }; + //List list = new List() { }; + //foreach (var item in info.OrderByDescending(c => c.Year).ToList()) + //{ + // if (item.Type != "不分文理") + // { + // list.Add(new IdNameResult() + // { + // Id = item.Id, + // Name = item.Batch_name, + // TypeName = item.Type, + // Year = item.Year, + // Score = item.ScoreLine + // }); + // } + // else + // { + // list.Add(new IdNameResult() + // { + // Id = item.Id, + // Name = item.Batch_name, + // Year = item.Year, + // Score = item.ScoreLine + // }); + // } + //} + + var baseurl = "https://static-data.ycymedu.com/batchitems/"; + var list = new List(); + using var httpClient = new System.Net.Http.HttpClient(); + string url = $"{baseurl}{query.ProvinceCode}.json"; + var response = httpClient.GetAsync(url).Result; + if (response.IsSuccessStatusCode) { - if (item.Type != "不分文理") + var jsonData = System.Text.Json.JsonSerializer.Deserialize(response.Content.ReadAsStringAsync().Result); + if (jsonData?.code == 1) { - list.Add(new IdNameResult() + list = jsonData.data.list.Select(s => new IdNameResult() { - Id = item.Id, - Name = item.Batch_name, - TypeName = item.Type, - Year = item.Year, - Score = item.ScoreLine - }); - } - else - { - list.Add(new IdNameResult() - { - Id = item.Id, - Name = item.Batch_name, - Year = item.Year, - Score = item.ScoreLine - }); + Id = s.subject_id, + Name = s.batch_name, + Year = s.year, + TypeName = s.subject_name, + Score = s.score + }).ToList(); } } return new MessageModel>() diff --git a/New_College.Tasks/HostedService/JobTimedSpiderService.cs b/New_College.Tasks/HostedService/JobTimedSpiderService.cs index 2d469e0..d2602eb 100644 --- a/New_College.Tasks/HostedService/JobTimedSpiderService.cs +++ b/New_College.Tasks/HostedService/JobTimedSpiderService.cs @@ -36,50 +36,18 @@ namespace New_College.Tasks private async void DoWork(object state) { - //try - //{ - // HtmlAgHelper agHelper = new HtmlAgHelper(); - // var list = agHelper.HtmlCreatePageData(); - // list.ForEach(async c => - // { - // var newsinfo = await newsInfoServices.Query(e => e.Title.Trim() == c.title); - // if (!newsinfo.Any() && c.pubtime.Year > DateTime.Now.Year - 1) - // { - - // await newsInfoServices.Add(new Model.Models.D_NewsInfo() - // { - // Author = c.author, - // CategoryId = 1, - // CreateBy = "spdier", - // ProvinceCode = "370000", - // CreateId = 1, - // CreateTime = c.pubtime, - // Detail = c.detail, - // CoverImg = "https://static-data.ycymedu.com/static/newstop.png", - // OrderSort = 0, - // IsDelete = false, - // Title = c.title, - // Summary = HtmlHelper.ReplaceHtmlTag(c.detail).Length > 200 ? HtmlHelper.ReplaceHtmlTag(c.detail).Substring(0, 200) : HtmlHelper.ReplaceHtmlTag(c.detail) - // }); - - // } - // }); - // ConsoleHelper.WriteWarningLine($"Job spider success： {DateTime.Now}-{list.Count}"); - //} - //catch (Exception ex) - //{ - // ConsoleHelper.WriteWarningLine($"Job spider 抓取异常"); - //} + // BatchListAgHelper batchListAgHelper = new BatchListAgHelper(); NationWideNewsAgHelper anghelper = new NationWideNewsAgHelper(); var provineInfo = await this.sysRegionServices.Query(e => e.Level == 1 && !e.RegionCode.Contains("-")); provineInfo.ForEach(p => { + // batchListAgHelper.HtmlCreatePageData(p.RegionCode); try { var list = anghelper.HtmlCreatePageData(p.RegionCode); list.ForEach(async c => { - c.title = c.title.Replace($"{p.SimpleName}：", ""); + c.title = c.title.Replace($"{p.SimpleName}：", ""); var newsinfo = await newsInfoServices.Query(e => e.Title.Trim() == c.title); if (!newsinfo.Any() && c.pubtime.Year > DateTime.Now.Year - 1) { diff --git a/New_College.Tasks/HtmlAgSpider/BatchListAgHelper.cs b/New_College.Tasks/HtmlAgSpider/BatchListAgHelper.cs new file mode 100644 index 0000000..7cde34c --- /dev/null +++ b/New_College.Tasks/HtmlAgSpider/BatchListAgHelper.cs @@ -0,0 +1,49 @@ +using Aliyun.OSS; +using Microsoft.AspNetCore.Mvc.RazorPages; +using NPOI.SS.Formula.Functions; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace New_College.Tasks +{ + + + public class BatchListAgHelper + { + private string bucketName = "static-ycymedu-minpro"; // + private string filePrefix = "batchitems/"; + // 初始化 OSS 客户端 + private OssClient ossClient = new OssClient("https://oss-cn-shanghai.aliyuncs.com", "LTAI5tKs3TXSbt7E4WMDcxwR", "EvC8MjRaQC1kHubgU4MtecZnofOb0v"); + public string baseUrl = "http://api.xuexingaokao.com/api/tool/batch/index?pro="; + public void HtmlCreatePageData(string provinceCode) + { + using var httpClient = new HttpClient(); + Thread.Sleep(100); + // 动态构建 API URL + var gourl = $"{baseUrl}{provinceCode}&year=&subject=&batch=×tamp={DateTimeOffset.Now.ToUnixTimeSeconds()}"; + Console.WriteLine($"gourl {gourl}..."); + var response = httpClient.GetAsync(gourl).Result; + if (response.IsSuccessStatusCode) + { + var jsonData = response.Content.ReadAsStringAsync().Result; + // 构建文件名和路径 + string objectName = $"{filePrefix}{provinceCode}.json"; + // 上传到 OSS（同步上传） + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(jsonData)); + ossClient.PutObject(bucketName, objectName, stream); + Console.WriteLine($"Uploaded to OSS as {objectName}"); + } + else + { + Console.WriteLine($"Failed to fetch, Status Code: {response.StatusCode}"); + } + + } + } +}