mirror of
https://gitee.com/novel_dev_team/novel
synced 2026-07-20 17:52:45 +08:00
增加爬虫智能化配置
This commit is contained in:
xiongxiaoyang
11
novel-admin/src/main/resources/application-crawl.yml
Normal file
11
novel-admin/src/main/resources/application-crawl.yml
Normal file
@@ -0,0 +1,11 @@
|
||||
crawl:
|
||||
#爬虫线程数
|
||||
threadCount: 1
|
||||
#爬取优先级 1：评分优先 2：更新时间优先
|
||||
priority: 1
|
||||
#小说最低评分
|
||||
lowestScore: 7.8
|
||||
#小说最小更新时间
|
||||
minUptTime: 2000-01-01
|
||||
#爬取最大条数
|
||||
maxNumber: 100000
|
||||
@@ -31,6 +31,7 @@ spring:
|
||||
type: redis
|
||||
|
||||
|
||||
|
||||
datasource:
|
||||
type: com.alibaba.druid.pool.DruidDataSource
|
||||
driverClassName: com.mysql.jdbc.Driver
|
||||
@@ -76,6 +77,8 @@ spring:
|
||||
max-active: 100
|
||||
# 连接池最大阻塞等待时间（使用负值表示没有限制）
|
||||
max-wait: -1
|
||||
profiles:
|
||||
include: crawl
|
||||
mybatis:
|
||||
configuration:
|
||||
#自动将数据库带下划线的表字段值映射到Java类的驼峰字段上
|
||||
|
||||
@@ -2,9 +2,9 @@
|
||||
threadCount=1
|
||||
#<23><>ȡ<EFBFBD><C8A1><EFBFBD>ȼ<EFBFBD> 1<><31><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> 2<><32><EFBFBD><EFBFBD><EFBFBD><EFBFBD>ʱ<EFBFBD><CAB1><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
|
||||
priority=1
|
||||
#С˵<D0A1><CBB5><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>֣<EFBFBD>0<EFBFBD><EFBFBD>ʾ<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
|
||||
#С˵<D0A1><CBB5><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
|
||||
lowestScore=0
|
||||
#С˵<D0A1><CBB5>С<EFBFBD><D0A1><EFBFBD><EFBFBD>ʱ<EFBFBD><CAB1>
|
||||
minUptTime=2000-01-01 00:00:00
|
||||
#<23><>ȡ<EFBFBD><C8A1><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>,0<><30>ʾ<EFBFBD><CABE><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
|
||||
maxNumber=0
|
||||
minUptTime=2000-01-01
|
||||
#<23><>ȡ<EFBFBD><C8A1><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
|
||||
maxNumber=100000
|
||||
|
||||
@@ -163,7 +163,7 @@ function edit(){
|
||||
console.log('打开配置页面');
|
||||
layer.open({
|
||||
type : 2,
|
||||
title : '增加',
|
||||
title : '爬虫配置修改',
|
||||
maxmin : true,
|
||||
shadeClose : false,
|
||||
area : [ '800px', '520px' ],
|
||||
|
||||
@@ -9,41 +9,57 @@
|
||||
<div class="ibox float-e-margins">
|
||||
<div class="ibox-content">
|
||||
<form class="form-horizontal m-t" id="signupForm">
|
||||
<input id="id" name="id" th:value="${bookCrawl.id}"
|
||||
type="hidden">
|
||||
<div class="form-group">
|
||||
<label class="col-sm-3 control-label">：</label>
|
||||
<div class="col-sm-8">
|
||||
<input id="crawlWebName" name="crawlWebName" class="form-control"
|
||||
th:value="${bookCrawl.crawlWebName}"
|
||||
type="text">
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label class="col-sm-3 control-label">爬虫线程数: </label>
|
||||
<div class="col-sm-8">
|
||||
<label class="radio-inline" > <input th:field="*{property.threadCount}"
|
||||
type="radio" name="threadCount" th:value="1" />单线程
|
||||
</label>
|
||||
<label class="radio-inline" > <input th:field="*{property.threadCount}"
|
||||
type="radio" name="threadCount" th:value="3" />3线程
|
||||
</label>
|
||||
<label class="radio-inline" > <input th:field="*{property.threadCount}"
|
||||
type="radio" name="threadCount" th:value="7" />7线程
|
||||
</label>
|
||||
</div>
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label class="col-sm-3 control-label">爬取优先级：</label>
|
||||
<div class="col-sm-8">
|
||||
<div class="col-sm-8">
|
||||
<label class="radio-inline" > <input th:field="*{property.priority}"
|
||||
type="radio" name="priority" th:value="1" />评分优先
|
||||
</label>
|
||||
<label class="radio-inline" > <input th:field="*{property.priority}"
|
||||
type="radio" name="priority" th:value="2" />更新时间优先
|
||||
</label>
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label class="col-sm-3 control-label">：</label>
|
||||
<div class="col-sm-8">
|
||||
<input id="crawlWebUrl" name="crawlWebUrl" class="form-control"
|
||||
th:value="${bookCrawl.crawlWebUrl}"
|
||||
type="text">
|
||||
</div>
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label class="col-sm-3 control-label">：</label>
|
||||
<div class="col-sm-8">
|
||||
<input id="crawlWebCode" name="crawlWebCode" class="form-control"
|
||||
th:value="${bookCrawl.crawlWebCode}"
|
||||
type="text">
|
||||
</div>
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label class="col-sm-3 control-label">：</label>
|
||||
<div class="col-sm-8">
|
||||
<input id="status" name="status" class="form-control"
|
||||
th:value="${bookCrawl.status}"
|
||||
type="text">
|
||||
</div>
|
||||
</div>
|
||||
<div class="form-group">
|
||||
</div>
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label class="col-sm-3 control-label">小说最低评分：</label>
|
||||
<div class="col-sm-8">
|
||||
<input id="lowestScore" name="lowestScore" class="form-control"
|
||||
th:value="${property.lowestScore}"
|
||||
type="number" max="10" min="0">
|
||||
</div>
|
||||
</div>
|
||||
<div class="form-group">
|
||||
<label class="col-sm-3 control-label">小说最小更新时间：</label>
|
||||
<div class="col-sm-8">
|
||||
<input type="text" class="laydate-icon layer-date form-control" id="minUptTime" name="minUptTime" th:value="${property.minUptTime}==null?null:${#dates.format(property.minUptTime,'yyyy-MM-dd')}" placeholder="请选择小说最小更新时间"
|
||||
onclick="laydate({istime: true, format: 'YYYY-MM-DD'})" style="background-color: #fff;" readonly="readonly"/>
|
||||
</div>
|
||||
</div>
|
||||
<!-- <div class="form-group">
|
||||
<label class="col-sm-3 control-label">爬取最大条数：</label>
|
||||
<div class="col-sm-8">
|
||||
<input id="maxNumber" name="maxNumber" class="form-control"
|
||||
th:value="${property.maxNumber}"
|
||||
type="number" max="100000" min="1">
|
||||
</div>
|
||||
</div>-->
|
||||
<div class="form-group">
|
||||
<div class="col-sm-8 col-sm-offset-3">
|
||||
<button type="submit" class="btn btn-primary">提交</button>
|
||||
</div>
|
||||
|
||||
Reference in New Issue
Block a user