diff --git a/New_College.Api/Controllers/HealthCheckController.cs b/New_College.Api/Controllers/HealthCheckController.cs index 0365c56..4fdd370 100644 --- a/New_College.Api/Controllers/HealthCheckController.cs +++ b/New_College.Api/Controllers/HealthCheckController.cs @@ -24,7 +24,7 @@ namespace New_College.Controllers /// [HttpGet] public IActionResult Get() - { + { return Ok(); } @@ -39,7 +39,14 @@ namespace New_College.Controllers // return t_EnrollmentPlane.Import(); // return t_EnrollmentPlane.Importuniverbaseinfo(); - return t_EnrollmentPlane.universitydetailupdate(); + // return t_EnrollmentPlane.universitydetailupdate(); + + //return t_EnrollmentPlane.tradeupadte(); + + // return t_EnrollmentPlane.categoryupdate(); + + return t_EnrollmentPlane.ClassChildItems(); + } } diff --git a/New_College.Api/New_College.Model.xml b/New_College.Api/New_College.Model.xml index b348c65..1ed7116 100644 --- a/New_College.Api/New_College.Model.xml +++ b/New_College.Api/New_College.Model.xml @@ -242,7 +242,7 @@ - 本科/专科大类 1本科 2 专科 + 本科/专科大类 1本科 2 专科|3职业本科 diff --git a/New_College.IServices/IT_EnrollmentPlaneServices.cs b/New_College.IServices/IT_EnrollmentPlaneServices.cs index f8280a3..a8e924c 100644 --- a/New_College.IServices/IT_EnrollmentPlaneServices.cs +++ b/New_College.IServices/IT_EnrollmentPlaneServices.cs @@ -13,7 +13,12 @@ namespace New_College.IServices /// public interface IT_EnrollmentPlaneServices : IBaseServices { + + Task ClassChildItems(); Task Import(); + + Task categoryupdate(); + Task tradeupadte(); Task universitydetailupdate(); Task Importuniverbaseinfo(); /// diff --git a/New_College.Model/Models/D_Major.cs b/New_College.Model/Models/D_Major.cs index 21584a4..1f3cf7b 100644 --- a/New_College.Model/Models/D_Major.cs +++ b/New_College.Model/Models/D_Major.cs @@ -31,7 +31,7 @@ namespace New_College.Model.Models /// 热度 /// [SugarColumn(IsNullable = true)] - public int Hot { get; set; } + public decimal Hot { get; set; } /// /// 专业分类 1文科 2理科 3不区分 diff --git a/New_College.Model/Models/D_MajorCategory.cs b/New_College.Model/Models/D_MajorCategory.cs index 9d3024c..9988219 100644 --- a/New_College.Model/Models/D_MajorCategory.cs +++ b/New_College.Model/Models/D_MajorCategory.cs @@ -12,18 +12,18 @@ namespace New_College.Model.Models /// /// /// - [SugarColumn(IsNullable = true)] + [SugarColumn(IsNullable = true)] public string TradeNumber { get; set; } /// /// /// - [SugarColumn(IsNullable = true)] + [SugarColumn(IsNullable = true)] public string TradeName { get; set; } /// - /// 本科/专科大类 1本科 2 专科 + /// 本科/专科大类 1本科 2 专科|3职业本科 /// - [SugarColumn(IsNullable = true)] + [SugarColumn(IsNullable = true)] public int Type { get; set; } } diff --git a/New_College.Services/T_EnrollmentPlaneServices.cs b/New_College.Services/T_EnrollmentPlaneServices.cs index f677dde..390f7d8 100644 --- a/New_College.Services/T_EnrollmentPlaneServices.cs +++ b/New_College.Services/T_EnrollmentPlaneServices.cs @@ -31,7 +31,9 @@ namespace New_College.Services private readonly ID_MajorRepository d_Major; private readonly IT_EnrollmentBatchRepository t_EnrollmentBatch; private readonly IT_EnrollmentPlanedescRepository t_EnrollmentPlanedesc; - public T_EnrollmentPlaneServices(IBaseRepository dal, IT_EnrollmentPlaneRepository t_EnrollmentPlaneRepository, ID_UniversityRepository d_UniversityRepository, ID_MajorRepository d_MajorRepository, IT_EnrollmentBatchRepository t_EnrollmentBatchRepository, IT_EnrollmentPlanedescRepository t_EnrollmentPlanedescRepository) + private readonly ID_MajorCategoryRepository d_MajorCategoryRepository; + private readonly ID_MajorClassRepository d_MajorClassRepository; + public T_EnrollmentPlaneServices(IBaseRepository dal, IT_EnrollmentPlaneRepository t_EnrollmentPlaneRepository, ID_UniversityRepository d_UniversityRepository, ID_MajorRepository d_MajorRepository, IT_EnrollmentBatchRepository t_EnrollmentBatchRepository, IT_EnrollmentPlanedescRepository t_EnrollmentPlanedescRepository, ID_MajorCategoryRepository d_MajorCategoryRepository, ID_MajorClassRepository majorClassRepository) { this._dal = dal; base.BaseDal = dal; @@ -40,6 +42,8 @@ namespace New_College.Services this.d_Major = d_MajorRepository; this.t_EnrollmentBatch = t_EnrollmentBatchRepository; this.t_EnrollmentPlanedesc = t_EnrollmentPlanedescRepository; + this.d_MajorCategoryRepository = d_MajorCategoryRepository; + this.d_MajorClassRepository = majorClassRepository; } /// @@ -70,7 +74,7 @@ namespace New_College.Services Thread.Sleep(100); if (!string.IsNullOrWhiteSpace(c.description)) { - if (universitylist.Any(k => k.Name == c.name.Trim())&& string.IsNullOrWhiteSpace(universitylist.FirstOrDefault(k => k.Name == c.name.Trim()).Description)) + if (universitylist.Any(k => k.Name == c.name.Trim()) && string.IsNullOrWhiteSpace(universitylist.FirstOrDefault(k => k.Name == c.name.Trim()).Description)) { var defaultinfo = universitylist.FirstOrDefault(k => k.Name == c.name.Trim()); @@ -83,6 +87,7 @@ namespace New_College.Services + }); @@ -93,6 +98,184 @@ namespace New_College.Services + /// + /// 更新目录 + /// + /// + public async Task tradeupadte() + { + var jsontext = UniversityTypeRelsove.GetFileJson(@"D:\\Files\\1060.json"); + var result = JsonConvert.DeserializeObject(jsontext); + var newtrade = result.msg; + var majorcategory = await this.d_MajorCategoryRepository.Query(x => x.IsDelete == false); + var majorcategorylist = new List(); + newtrade.ToList().ForEach(async a => + { + + if (majorcategory.Any(n => n.TradeName == a.name && n.Type == 2)) + { + var majorcategorydetail = majorcategory.FirstOrDefault(n => n.TradeName == a.name && n.Type == 2); + majorcategorydetail.TradeNumber = a.key; + majorcategorydetail.ModifyTime = DateTime.Now; + await this.d_MajorCategoryRepository.Update(majorcategorydetail); + + } + + //if (!majorcategory.Any(n => n.TradeName == a.name && n.Type == 3)) + //{ + // majorcategorylist.Add(new D_MajorCategory() + // { + // CreateTime = DateTime.Now, + // IsDelete = false, + // ModifyTime = DateTime.Now, + // Name = a.name, + // OrderSort = 0, + // TradeName = a.name, + // TradeNumber = a.key, + // Type = 3, + // }); + //} + }); + //if (majorcategorylist.Any()) + // await this.d_MajorCategoryRepository.Add(majorcategorylist); + return true; + } + + + + /// + /// 专业类别二级目录 + /// + /// + public async Task categoryupdate() + { + string[] files = Directory.GetFiles(@"D:\\Files\\Type\\"); + var msglist = new List(); + + var majorclassitems = new List(); + var majorcategory = await this.d_MajorCategoryRepository.Query(x => x.IsDelete == false && x.Type == 3); + var majorclass = await this.d_MajorClassRepository.Query(x => x.IsDelete == false); + foreach (string itemfile in files) + { + if (itemfile.Contains("1070")) + { + var jsontext = UniversityTypeRelsove.GetFileJson(itemfile); + var result = JsonConvert.DeserializeObject(jsontext); + msglist.AddRange(result.msg); + } + } + + var majorclasslist = majorclass.ToList(); + msglist.ForEach(x => + { + + //if (majorclass.Any(n=> n.Name == x.name)) + //{ + + // var majorclassdefault = majorclass.FirstOrDefault(k => k.Name == x.name); + // majorclassdefault.ModifyTime = DateTime.Now; + // majorclassdefault.OrderSort = int.Parse(x.key); + // await this.d_MajorClassRepository.Update(majorclassdefault); + //} + //else + //{ + var tradenumber = x.key.Remove(x.key.Length - 2, 2); + var singletrade = majorcategory.FirstOrDefault(c => c.TradeNumber == tradenumber); + if (singletrade != null) + { + majorclassitems.Add(new D_MajorClass() + { + + Categoryname = x.name, + CreateTime = DateTime.Now, + IsDelete = false, + ModifyTime = DateTime.Now, + Name = x.name, + OrderSort = int.Parse(x.key), + TradeId = singletrade.Id + }); + } + // } + + + }); + if (majorclassitems.Any()) + { + + await this.d_MajorClassRepository.Add(majorclassitems); + } + + + return true; + } + + /// + /// 专业名称 + /// + /// + public async Task ClassChildItems() + { + string[] files = Directory.GetFiles(@"D:\\Files\\TypeChild\\"); + var msglist = new List(); + var majorclass = await this.d_MajorClassRepository.Query(x => x.IsDelete == false); + var majorinfolist = await this.d_Major.Query(x => x.IsDelete == false); + foreach (string itemfile in files) + { + if (itemfile.Contains("1070")) + { + int ordersort = int.Parse("1070" + itemfile.Split("1070")[1].Replace(".json", "")); + var categoryid = majorclass.FirstOrDefault(c => c.IsDelete == false && c.OrderSort == ordersort); + + var jsontext = UniversityTypeRelsove.GetFileJson(itemfile); + var result = JsonConvert.DeserializeObject(jsontext); + result.msg.ToList().ForEach(msg => + { + msg.categoryid = categoryid.Id; + msglist.Add(msg); + }); + + } + } + + var majoritems = new List(); + /// + msglist.ForEach(a => + { + var anyinfo = majorinfolist.Any(c => a.categoryid == c.CategoryClass_Id && a.zydm == c.MajorCode); + if (!anyinfo) + { + majoritems.Add(new D_Major() + { + Hot = a.zymyd==""?0:decimal.Parse(a.zymyd), + CategoryClass_Id = a.categoryid, + CreateTime = DateTime.Now, + IsDelete = false, + MajorCode = a.zydm, + MajorName = a.zymc, + ModifyTime = DateTime.Now, + Name = a.zymc, + OrderSort =0, + Type = 3, + + }); + } + }); + await d_Major.Add(majoritems); + + return true; + } + + public class Rootobject + { + public Msg[] msg { get; set; } + public bool flag { get; set; } + } + + public class Msg + { + public string key { get; set; } + public string name { get; set; } + } public class Class1 @@ -103,6 +286,25 @@ namespace New_College.Services } + public class MajorItem + { + public MsgITEMS[] msg { get; set; } + public bool flag { get; set; } + } + + public class MsgITEMS + { + public string zydm { get; set; } + public string zymc { get; set; } + public string specId { get; set; } + public string zymyd { get; set; } + + public int categoryid { get; set; } + } + + + + /// /// /// diff --git a/New_Spider/HtmlAgMajorHelper.cs b/New_Spider/HtmlAgMajorHelper.cs new file mode 100644 index 0000000..d30e955 --- /dev/null +++ b/New_Spider/HtmlAgMajorHelper.cs @@ -0,0 +1,119 @@ +using HtmlAgilityPack; +using New_College.Common.Helper; +using Newtonsoft.Json; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; + +namespace New_Spider +{ + public class HtmlAgMajorHelper + { + + public async void HtmlCreatePageData() + { + WebClient wc = new WebClient(); + wc.DownloadProgressChanged += (sender, args) => Console.WriteLine(args.ProgressPercentage + "% complete"); + Task.Delay(10000).ContinueWith(ant => wc.CancelAsync()); //如果超过限制时间，则取消下载 + + string pathFile = AppDomain.CurrentDomain.SetupInformation.ApplicationBase + "Files/"; + //判断文件夹是否存在 + if (!Directory.Exists(pathFile)) + { + Directory.CreateDirectory(pathFile); + } + + + string type = "https://gaokao.chsi.com.cn/zyk/zybk/xkCategory/107021?_t=1694748927326"; + + string childtype = "https://gaokao.chsi.com.cn/zyk/zybk/specialityesByCategory/10702101?_t=1694748927386"; + + + await wc.DownloadFileTaskAsync("https://gaokao.chsi.com.cn/zyk/zybk/mlCategory/1070?_t=1694748266332", pathFile + "1070.json"); // await 是C# 5.0中实现异步操作的关键字 + + + + } + + + /// + /// 下载二级分类 + /// + public void DownloadTypeListFile() + { + + var basejson = UniversityTypeRelsove.GetFileJson(AppDomain.CurrentDomain.SetupInformation.ApplicationBase + "Files/1070.json"); + // var jsontext2 = Regex.Unescape(jsontext.Remove(jsontext.Length - 1, 1).Remove(0, 1)); + + var jsons = JsonConvert.DeserializeObject(basejson); + string pathFile = AppDomain.CurrentDomain.SetupInformation.ApplicationBase + "Files/Type/"; + //判断文件夹是否存在 + if (!Directory.Exists(pathFile)) + { + Directory.CreateDirectory(pathFile); + } + WebClient wc = new WebClient(); + + var list = jsons.msg.ToList(); + list.ForEach(c => + { + + wc.DownloadFile(string.Format("https://gaokao.chsi.com.cn/zyk/zybk/xkCategory/{0}?_t=1694748927326", c.key), string.Format(pathFile + "{0}.json", c.key)); + }); + + } + + /// + /// 下载三级分类 + /// + public void DownloadChildTypeListFile() + { + + string[] files = Directory.GetFiles(AppDomain.CurrentDomain.SetupInformation.ApplicationBase + "Files/Type/"); + foreach (string itemfile in files) + { + var basejson = UniversityTypeRelsove.GetFileJson(itemfile); + var jsons = JsonConvert.DeserializeObject(basejson); + string pathFile = AppDomain.CurrentDomain.SetupInformation.ApplicationBase + "Files/TypeChild/"; + //判断文件夹是否存在 + if (!Directory.Exists(pathFile)) + { + Directory.CreateDirectory(pathFile); + } + WebClient wc = new WebClient(); + + var list = jsons.msg.ToList(); + list.ForEach(c => + { + + wc.DownloadFile(string.Format("https://gaokao.chsi.com.cn/zyk/zybk/specialityesByCategory/{0}?_t=1694748927326", c.key), string.Format(pathFile + "{0}.json", c.key)); + }); + + Thread.Sleep(300); + } + } + + + + + + + public class Rootobject + { + public Msg[] msg { get; set; } + public bool flag { get; set; } + } + + public class Msg + { + public string key { get; set; } + public string name { get; set; } + } + + + } +} diff --git a/New_Spider/New_Spider.csproj b/New_Spider/New_Spider.csproj index da66fe9..91f8ed1 100644 --- a/New_Spider/New_Spider.csproj +++ b/New_Spider/New_Spider.csproj @@ -12,4 +12,8 @@ + + + + diff --git a/New_Spider/Program.cs b/New_Spider/Program.cs index e38bbc5..4be4f6c 100644 --- a/New_Spider/Program.cs +++ b/New_Spider/Program.cs @@ -3,9 +3,13 @@ using HtmlAgilityPack; using New_Spider; using System.Text.RegularExpressions; -HtmlAgHelper htmlAgHelper = new HtmlAgHelper(); +//HtmlAgHelper htmlAgHelper = new HtmlAgHelper(); + +//htmlAgHelper.HtmlCreatePageData(); +HtmlAgMajorHelper agMajorHelper = new HtmlAgMajorHelper(); +//agMajorHelper.DownloadTypeListFile(); +agMajorHelper.DownloadChildTypeListFile(); -htmlAgHelper.HtmlCreatePageData(); Console.Read();