调整爬取速度,上传视频URL前等待3秒
This commit is contained in:
parent
30a9056751
commit
ec5309255b
@ -1,5 +1,5 @@
|
|||||||
# 使用官方Python基础镜像
|
# 使用国内Python基础镜像
|
||||||
FROM python:3.9-bookworm
|
FROM swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/python:3.9-bookworm
|
||||||
|
|
||||||
# 设置工作目录
|
# 设置工作目录
|
||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
@ -221,6 +221,8 @@ class AliyunVodPipeline:
|
|||||||
|
|
||||||
# 2. 通过URL上传视频,获取JobId
|
# 2. 通过URL上传视频,获取JobId
|
||||||
title = item.get('title', '')
|
title = item.get('title', '')
|
||||||
|
# 延迟3秒上传
|
||||||
|
time.sleep(3)
|
||||||
job_id = self.upload_media_by_url(
|
job_id = self.upload_media_by_url(
|
||||||
video_url=video_url,
|
video_url=video_url,
|
||||||
title=title,
|
title=title,
|
||||||
@ -378,7 +380,7 @@ class MariaDBPipeline:
|
|||||||
Returns:
|
Returns:
|
||||||
item: 处理后的item
|
item: 处理后的item
|
||||||
"""
|
"""
|
||||||
# 这里不需要处理item,因为我们要从SQLite读取数据
|
# 从SQLite读取数据
|
||||||
return item
|
return item
|
||||||
|
|
||||||
def migrate_data(self):
|
def migrate_data(self):
|
||||||
@ -425,7 +427,8 @@ class MariaDBPipeline:
|
|||||||
'video_remote_id': video.aliyun_video_id or '',
|
'video_remote_id': video.aliyun_video_id or '',
|
||||||
'video_url': '',
|
'video_url': '',
|
||||||
'video_list': video.video_list or 0,
|
'video_list': video.video_list or 0,
|
||||||
'month': '1'
|
'month': '1',
|
||||||
|
'video_status': video.aliyun_status or '',
|
||||||
}
|
}
|
||||||
|
|
||||||
# 映射到wz_video_data表
|
# 映射到wz_video_data表
|
||||||
@ -447,12 +450,12 @@ class MariaDBPipeline:
|
|||||||
cid, title, css, thumb, keywords, remark, block, url,
|
cid, title, css, thumb, keywords, remark, block, url,
|
||||||
status, route, publisher, addtime, updatetime, area,
|
status, route, publisher, addtime, updatetime, area,
|
||||||
category, theme, year, video_remote_id, video_url,
|
category, theme, year, video_remote_id, video_url,
|
||||||
video_list, month
|
video_list, month, video_status
|
||||||
) VALUES (
|
) VALUES (
|
||||||
:cid, :title, :css, :thumb, :keywords, :remark, :block, :url,
|
:cid, :title, :css, :thumb, :keywords, :remark, :block, :url,
|
||||||
:status, :route, :publisher, :addtime, :updatetime, :area,
|
:status, :route, :publisher, :addtime, :updatetime, :area,
|
||||||
:category, :theme, :year, :video_remote_id, :video_url,
|
:category, :theme, :year, :video_remote_id, :video_url,
|
||||||
:video_list, :month
|
:video_list, :month, :video_status
|
||||||
)"""),
|
)"""),
|
||||||
wz_video
|
wz_video
|
||||||
)
|
)
|
||||||
|
@ -216,7 +216,7 @@
|
|||||||
<th>爬虫名称</th>
|
<th>爬虫名称</th>
|
||||||
<th>URL</th>
|
<th>URL</th>
|
||||||
<th>剧集ID</th>
|
<th>剧集ID</th>
|
||||||
<th>状态</th>
|
<th>定时任务开关</th>
|
||||||
<th>运行状态</th>
|
<th>运行状态</th>
|
||||||
<th>操作</th>
|
<th>操作</th>
|
||||||
</tr>
|
</tr>
|
||||||
@ -558,7 +558,7 @@
|
|||||||
<td>${task.spider_name}</td>
|
<td>${task.spider_name}</td>
|
||||||
<td>${task.url}</td>
|
<td>${task.url}</td>
|
||||||
<td>${task.video_list}</td>
|
<td>${task.video_list}</td>
|
||||||
<td>
|
<td align="center">
|
||||||
<label class="switch">
|
<label class="switch">
|
||||||
<input type="checkbox" ${task.enabled ? "checked" : ""}
|
<input type="checkbox" ${task.enabled ? "checked" : ""}
|
||||||
onchange="toggleTask(${task.id}, this.checked)">
|
onchange="toggleTask(${task.id}, this.checked)">
|
||||||
@ -572,7 +572,7 @@
|
|||||||
<button class="btn btn-primary btn-sm" onclick="runTask(${
|
<button class="btn btn-primary btn-sm" onclick="runTask(${
|
||||||
task.id
|
task.id
|
||||||
})">
|
})">
|
||||||
立即执行
|
手动运行
|
||||||
</button>
|
</button>
|
||||||
<button class="btn btn-warning btn-sm" onclick="editTask(${
|
<button class="btn btn-warning btn-sm" onclick="editTask(${
|
||||||
task.id
|
task.id
|
||||||
|
Loading…
Reference in New Issue
Block a user