faeat:专业数据更新
old易
parent
9edc8a7aa2
commit
9c8cd9c151
|
|
@ -39,7 +39,14 @@ namespace New_College.Controllers
|
|||
// return t_EnrollmentPlane.Import();
|
||||
// return t_EnrollmentPlane.Importuniverbaseinfo();
|
||||
|
||||
return t_EnrollmentPlane.universitydetailupdate();
|
||||
// return t_EnrollmentPlane.universitydetailupdate();
|
||||
|
||||
//return t_EnrollmentPlane.tradeupadte();
|
||||
|
||||
// return t_EnrollmentPlane.categoryupdate();
|
||||
|
||||
return t_EnrollmentPlane.ClassChildItems();
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -242,7 +242,7 @@
|
|||
</member>
|
||||
<member name="P:New_College.Model.Models.D_MajorCategory.Type">
|
||||
<summary>
|
||||
本科/专科大类 1本科 2 专科
|
||||
本科/专科大类 1本科 2 专科|3职业本科
|
||||
</summary>
|
||||
</member>
|
||||
<member name="P:New_College.Model.Models.D_MajorClass.Categoryname">
|
||||
|
|
|
|||
|
|
@ -13,7 +13,12 @@ namespace New_College.IServices
|
|||
/// </summary>
|
||||
public interface IT_EnrollmentPlaneServices : IBaseServices<T_EnrollmentPlane>
|
||||
{
|
||||
|
||||
Task<bool> ClassChildItems();
|
||||
Task<bool> Import();
|
||||
|
||||
Task<bool> categoryupdate();
|
||||
Task<bool> tradeupadte();
|
||||
Task<bool> universitydetailupdate();
|
||||
Task<bool> Importuniverbaseinfo();
|
||||
/// <summary>
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ namespace New_College.Model.Models
|
|||
/// 热度
|
||||
/// </summary>
|
||||
[SugarColumn(IsNullable = true)]
|
||||
public int Hot { get; set; }
|
||||
public decimal Hot { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// 专业分类 1文科 2理科 3不区分
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ namespace New_College.Model.Models
|
|||
public string TradeName { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// 本科/专科大类 1本科 2 专科
|
||||
/// 本科/专科大类 1本科 2 专科|3职业本科
|
||||
/// </summary>
|
||||
[SugarColumn(IsNullable = true)]
|
||||
public int Type { get; set; }
|
||||
|
|
|
|||
|
|
@ -31,7 +31,9 @@ namespace New_College.Services
|
|||
private readonly ID_MajorRepository d_Major;
|
||||
private readonly IT_EnrollmentBatchRepository t_EnrollmentBatch;
|
||||
private readonly IT_EnrollmentPlanedescRepository t_EnrollmentPlanedesc;
|
||||
public T_EnrollmentPlaneServices(IBaseRepository<T_EnrollmentPlane> dal, IT_EnrollmentPlaneRepository t_EnrollmentPlaneRepository, ID_UniversityRepository d_UniversityRepository, ID_MajorRepository d_MajorRepository, IT_EnrollmentBatchRepository t_EnrollmentBatchRepository, IT_EnrollmentPlanedescRepository t_EnrollmentPlanedescRepository)
|
||||
private readonly ID_MajorCategoryRepository d_MajorCategoryRepository;
|
||||
private readonly ID_MajorClassRepository d_MajorClassRepository;
|
||||
public T_EnrollmentPlaneServices(IBaseRepository<T_EnrollmentPlane> dal, IT_EnrollmentPlaneRepository t_EnrollmentPlaneRepository, ID_UniversityRepository d_UniversityRepository, ID_MajorRepository d_MajorRepository, IT_EnrollmentBatchRepository t_EnrollmentBatchRepository, IT_EnrollmentPlanedescRepository t_EnrollmentPlanedescRepository, ID_MajorCategoryRepository d_MajorCategoryRepository, ID_MajorClassRepository majorClassRepository)
|
||||
{
|
||||
this._dal = dal;
|
||||
base.BaseDal = dal;
|
||||
|
|
@ -40,6 +42,8 @@ namespace New_College.Services
|
|||
this.d_Major = d_MajorRepository;
|
||||
this.t_EnrollmentBatch = t_EnrollmentBatchRepository;
|
||||
this.t_EnrollmentPlanedesc = t_EnrollmentPlanedescRepository;
|
||||
this.d_MajorCategoryRepository = d_MajorCategoryRepository;
|
||||
this.d_MajorClassRepository = majorClassRepository;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -70,7 +74,7 @@ namespace New_College.Services
|
|||
Thread.Sleep(100);
|
||||
if (!string.IsNullOrWhiteSpace(c.description))
|
||||
{
|
||||
if (universitylist.Any(k => k.Name == c.name.Trim())&& string.IsNullOrWhiteSpace(universitylist.FirstOrDefault(k => k.Name == c.name.Trim()).Description))
|
||||
if (universitylist.Any(k => k.Name == c.name.Trim()) && string.IsNullOrWhiteSpace(universitylist.FirstOrDefault(k => k.Name == c.name.Trim()).Description))
|
||||
{
|
||||
var defaultinfo = universitylist.FirstOrDefault(k => k.Name == c.name.Trim());
|
||||
|
||||
|
|
@ -83,6 +87,7 @@ namespace New_College.Services
|
|||
|
||||
|
||||
|
||||
|
||||
});
|
||||
|
||||
|
||||
|
|
@ -93,6 +98,184 @@ namespace New_College.Services
|
|||
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// 更新目录
|
||||
/// </summary>
|
||||
/// <returns></returns>
|
||||
public async Task<bool> tradeupadte()
|
||||
{
|
||||
var jsontext = UniversityTypeRelsove.GetFileJson(@"D:\\Files\\1060.json");
|
||||
var result = JsonConvert.DeserializeObject<Rootobject>(jsontext);
|
||||
var newtrade = result.msg;
|
||||
var majorcategory = await this.d_MajorCategoryRepository.Query(x => x.IsDelete == false);
|
||||
var majorcategorylist = new List<D_MajorCategory>();
|
||||
newtrade.ToList().ForEach(async a =>
|
||||
{
|
||||
|
||||
if (majorcategory.Any(n => n.TradeName == a.name && n.Type == 2))
|
||||
{
|
||||
var majorcategorydetail = majorcategory.FirstOrDefault(n => n.TradeName == a.name && n.Type == 2);
|
||||
majorcategorydetail.TradeNumber = a.key;
|
||||
majorcategorydetail.ModifyTime = DateTime.Now;
|
||||
await this.d_MajorCategoryRepository.Update(majorcategorydetail);
|
||||
|
||||
}
|
||||
|
||||
//if (!majorcategory.Any(n => n.TradeName == a.name && n.Type == 3))
|
||||
//{
|
||||
// majorcategorylist.Add(new D_MajorCategory()
|
||||
// {
|
||||
// CreateTime = DateTime.Now,
|
||||
// IsDelete = false,
|
||||
// ModifyTime = DateTime.Now,
|
||||
// Name = a.name,
|
||||
// OrderSort = 0,
|
||||
// TradeName = a.name,
|
||||
// TradeNumber = a.key,
|
||||
// Type = 3,
|
||||
// });
|
||||
//}
|
||||
});
|
||||
//if (majorcategorylist.Any())
|
||||
// await this.d_MajorCategoryRepository.Add(majorcategorylist);
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// 专业类别二级目录
|
||||
/// </summary>
|
||||
/// <returns></returns>
|
||||
public async Task<bool> categoryupdate()
|
||||
{
|
||||
string[] files = Directory.GetFiles(@"D:\\Files\\Type\\");
|
||||
var msglist = new List<Msg>();
|
||||
|
||||
var majorclassitems = new List<D_MajorClass>();
|
||||
var majorcategory = await this.d_MajorCategoryRepository.Query(x => x.IsDelete == false && x.Type == 3);
|
||||
var majorclass = await this.d_MajorClassRepository.Query(x => x.IsDelete == false);
|
||||
foreach (string itemfile in files)
|
||||
{
|
||||
if (itemfile.Contains("1070"))
|
||||
{
|
||||
var jsontext = UniversityTypeRelsove.GetFileJson(itemfile);
|
||||
var result = JsonConvert.DeserializeObject<Rootobject>(jsontext);
|
||||
msglist.AddRange(result.msg);
|
||||
}
|
||||
}
|
||||
|
||||
var majorclasslist = majorclass.ToList();
|
||||
msglist.ForEach(x =>
|
||||
{
|
||||
|
||||
//if (majorclass.Any(n=> n.Name == x.name))
|
||||
//{
|
||||
|
||||
// var majorclassdefault = majorclass.FirstOrDefault(k => k.Name == x.name);
|
||||
// majorclassdefault.ModifyTime = DateTime.Now;
|
||||
// majorclassdefault.OrderSort = int.Parse(x.key);
|
||||
// await this.d_MajorClassRepository.Update(majorclassdefault);
|
||||
//}
|
||||
//else
|
||||
//{
|
||||
var tradenumber = x.key.Remove(x.key.Length - 2, 2);
|
||||
var singletrade = majorcategory.FirstOrDefault(c => c.TradeNumber == tradenumber);
|
||||
if (singletrade != null)
|
||||
{
|
||||
majorclassitems.Add(new D_MajorClass()
|
||||
{
|
||||
|
||||
Categoryname = x.name,
|
||||
CreateTime = DateTime.Now,
|
||||
IsDelete = false,
|
||||
ModifyTime = DateTime.Now,
|
||||
Name = x.name,
|
||||
OrderSort = int.Parse(x.key),
|
||||
TradeId = singletrade.Id
|
||||
});
|
||||
}
|
||||
// }
|
||||
|
||||
|
||||
});
|
||||
if (majorclassitems.Any())
|
||||
{
|
||||
|
||||
await this.d_MajorClassRepository.Add(majorclassitems);
|
||||
}
|
||||
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 专业名称
|
||||
/// </summary>
|
||||
/// <returns></returns>
|
||||
public async Task<bool> ClassChildItems()
|
||||
{
|
||||
string[] files = Directory.GetFiles(@"D:\\Files\\TypeChild\\");
|
||||
var msglist = new List<MsgITEMS>();
|
||||
var majorclass = await this.d_MajorClassRepository.Query(x => x.IsDelete == false);
|
||||
var majorinfolist = await this.d_Major.Query(x => x.IsDelete == false);
|
||||
foreach (string itemfile in files)
|
||||
{
|
||||
if (itemfile.Contains("1070"))
|
||||
{
|
||||
int ordersort = int.Parse("1070" + itemfile.Split("1070")[1].Replace(".json", ""));
|
||||
var categoryid = majorclass.FirstOrDefault(c => c.IsDelete == false && c.OrderSort == ordersort);
|
||||
|
||||
var jsontext = UniversityTypeRelsove.GetFileJson(itemfile);
|
||||
var result = JsonConvert.DeserializeObject<MajorItem>(jsontext);
|
||||
result.msg.ToList().ForEach(msg =>
|
||||
{
|
||||
msg.categoryid = categoryid.Id;
|
||||
msglist.Add(msg);
|
||||
});
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
var majoritems = new List<D_Major>();
|
||||
///
|
||||
msglist.ForEach(a =>
|
||||
{
|
||||
var anyinfo = majorinfolist.Any(c => a.categoryid == c.CategoryClass_Id && a.zydm == c.MajorCode);
|
||||
if (!anyinfo)
|
||||
{
|
||||
majoritems.Add(new D_Major()
|
||||
{
|
||||
Hot = a.zymyd==""?0:decimal.Parse(a.zymyd),
|
||||
CategoryClass_Id = a.categoryid,
|
||||
CreateTime = DateTime.Now,
|
||||
IsDelete = false,
|
||||
MajorCode = a.zydm,
|
||||
MajorName = a.zymc,
|
||||
ModifyTime = DateTime.Now,
|
||||
Name = a.zymc,
|
||||
OrderSort =0,
|
||||
Type = 3,
|
||||
|
||||
});
|
||||
}
|
||||
});
|
||||
await d_Major.Add(majoritems);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
public class Rootobject
|
||||
{
|
||||
public Msg[] msg { get; set; }
|
||||
public bool flag { get; set; }
|
||||
}
|
||||
|
||||
public class Msg
|
||||
{
|
||||
public string key { get; set; }
|
||||
public string name { get; set; }
|
||||
}
|
||||
|
||||
|
||||
public class Class1
|
||||
|
|
@ -103,6 +286,25 @@ namespace New_College.Services
|
|||
}
|
||||
|
||||
|
||||
public class MajorItem
|
||||
{
|
||||
public MsgITEMS[] msg { get; set; }
|
||||
public bool flag { get; set; }
|
||||
}
|
||||
|
||||
public class MsgITEMS
|
||||
{
|
||||
public string zydm { get; set; }
|
||||
public string zymc { get; set; }
|
||||
public string specId { get; set; }
|
||||
public string zymyd { get; set; }
|
||||
|
||||
public int categoryid { get; set; }
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
/// <summary>
|
||||
///
|
||||
/// </summary>
|
||||
|
|
|
|||
|
|
@ -0,0 +1,119 @@
|
|||
using HtmlAgilityPack;
|
||||
using New_College.Common.Helper;
|
||||
using Newtonsoft.Json;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
namespace New_Spider
|
||||
{
|
||||
public class HtmlAgMajorHelper
|
||||
{
|
||||
|
||||
public async void HtmlCreatePageData()
|
||||
{
|
||||
WebClient wc = new WebClient();
|
||||
wc.DownloadProgressChanged += (sender, args) => Console.WriteLine(args.ProgressPercentage + "% complete");
|
||||
Task.Delay(10000).ContinueWith(ant => wc.CancelAsync()); //如果超过限制时间，则取消下载
|
||||
|
||||
string pathFile = AppDomain.CurrentDomain.SetupInformation.ApplicationBase + "Files/";
|
||||
//判断文件夹是否存在
|
||||
if (!Directory.Exists(pathFile))
|
||||
{
|
||||
Directory.CreateDirectory(pathFile);
|
||||
}
|
||||
|
||||
|
||||
string type = "https://gaokao.chsi.com.cn/zyk/zybk/xkCategory/107021?_t=1694748927326";
|
||||
|
||||
string childtype = "https://gaokao.chsi.com.cn/zyk/zybk/specialityesByCategory/10702101?_t=1694748927386";
|
||||
|
||||
|
||||
await wc.DownloadFileTaskAsync("https://gaokao.chsi.com.cn/zyk/zybk/mlCategory/1070?_t=1694748266332", pathFile + "1070.json"); // await 是C# 5.0中实现异步操作的关键字
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// 下载二级分类
|
||||
/// </summary>
|
||||
public void DownloadTypeListFile()
|
||||
{
|
||||
|
||||
var basejson = UniversityTypeRelsove.GetFileJson(AppDomain.CurrentDomain.SetupInformation.ApplicationBase + "Files/1070.json");
|
||||
// var jsontext2 = Regex.Unescape(jsontext.Remove(jsontext.Length - 1, 1).Remove(0, 1));
|
||||
|
||||
var jsons = JsonConvert.DeserializeObject<Rootobject>(basejson);
|
||||
string pathFile = AppDomain.CurrentDomain.SetupInformation.ApplicationBase + "Files/Type/";
|
||||
//判断文件夹是否存在
|
||||
if (!Directory.Exists(pathFile))
|
||||
{
|
||||
Directory.CreateDirectory(pathFile);
|
||||
}
|
||||
WebClient wc = new WebClient();
|
||||
|
||||
var list = jsons.msg.ToList();
|
||||
list.ForEach(c =>
|
||||
{
|
||||
|
||||
wc.DownloadFile(string.Format("https://gaokao.chsi.com.cn/zyk/zybk/xkCategory/{0}?_t=1694748927326", c.key), string.Format(pathFile + "{0}.json", c.key));
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// 下载三级分类
|
||||
/// </summary>
|
||||
public void DownloadChildTypeListFile()
|
||||
{
|
||||
|
||||
string[] files = Directory.GetFiles(AppDomain.CurrentDomain.SetupInformation.ApplicationBase + "Files/Type/");
|
||||
foreach (string itemfile in files)
|
||||
{
|
||||
var basejson = UniversityTypeRelsove.GetFileJson(itemfile);
|
||||
var jsons = JsonConvert.DeserializeObject<Rootobject>(basejson);
|
||||
string pathFile = AppDomain.CurrentDomain.SetupInformation.ApplicationBase + "Files/TypeChild/";
|
||||
//判断文件夹是否存在
|
||||
if (!Directory.Exists(pathFile))
|
||||
{
|
||||
Directory.CreateDirectory(pathFile);
|
||||
}
|
||||
WebClient wc = new WebClient();
|
||||
|
||||
var list = jsons.msg.ToList();
|
||||
list.ForEach(c =>
|
||||
{
|
||||
|
||||
wc.DownloadFile(string.Format("https://gaokao.chsi.com.cn/zyk/zybk/specialityesByCategory/{0}?_t=1694748927326", c.key), string.Format(pathFile + "{0}.json", c.key));
|
||||
});
|
||||
|
||||
Thread.Sleep(300);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
public class Rootobject
|
||||
{
|
||||
public Msg[] msg { get; set; }
|
||||
public bool flag { get; set; }
|
||||
}
|
||||
|
||||
public class Msg
|
||||
{
|
||||
public string key { get; set; }
|
||||
public string name { get; set; }
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
|
@ -12,4 +12,8 @@
|
|||
<PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\New_College.Common\New_College.Common.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
|
|
|
|||
|
|
@ -3,9 +3,13 @@ using HtmlAgilityPack;
|
|||
using New_Spider;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
HtmlAgHelper htmlAgHelper = new HtmlAgHelper();
|
||||
//HtmlAgHelper htmlAgHelper = new HtmlAgHelper();
|
||||
|
||||
//htmlAgHelper.HtmlCreatePageData();
|
||||
HtmlAgMajorHelper agMajorHelper = new HtmlAgMajorHelper();
|
||||
//agMajorHelper.DownloadTypeListFile();
|
||||
agMajorHelper.DownloadChildTypeListFile();
|
||||
|
||||
htmlAgHelper.HtmlCreatePageData();
|
||||
|
||||
Console.Read();
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue