调整爬取速度,上传视频URL前等待3秒
This commit is contained in:
parent
30a9056751
commit
ec5309255b
@ -1,5 +1,5 @@
|
||||
# 使用官方Python基础镜像
|
||||
FROM python:3.9-bookworm
|
||||
# 使用国内Python基础镜像
|
||||
FROM swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/python:3.9-bookworm
|
||||
|
||||
# 设置工作目录
|
||||
WORKDIR /app
|
||||
|
@ -221,6 +221,8 @@ class AliyunVodPipeline:
|
||||
|
||||
# 2. 通过URL上传视频,获取JobId
|
||||
title = item.get('title', '')
|
||||
# 延迟3秒上传
|
||||
time.sleep(3)
|
||||
job_id = self.upload_media_by_url(
|
||||
video_url=video_url,
|
||||
title=title,
|
||||
@ -378,7 +380,7 @@ class MariaDBPipeline:
|
||||
Returns:
|
||||
item: 处理后的item
|
||||
"""
|
||||
# 这里不需要处理item,因为我们要从SQLite读取数据
|
||||
# 从SQLite读取数据
|
||||
return item
|
||||
|
||||
def migrate_data(self):
|
||||
@ -425,7 +427,8 @@ class MariaDBPipeline:
|
||||
'video_remote_id': video.aliyun_video_id or '',
|
||||
'video_url': '',
|
||||
'video_list': video.video_list or 0,
|
||||
'month': '1'
|
||||
'month': '1',
|
||||
'video_status': video.aliyun_status or '',
|
||||
}
|
||||
|
||||
# 映射到wz_video_data表
|
||||
@ -447,12 +450,12 @@ class MariaDBPipeline:
|
||||
cid, title, css, thumb, keywords, remark, block, url,
|
||||
status, route, publisher, addtime, updatetime, area,
|
||||
category, theme, year, video_remote_id, video_url,
|
||||
video_list, month
|
||||
video_list, month, video_status
|
||||
) VALUES (
|
||||
:cid, :title, :css, :thumb, :keywords, :remark, :block, :url,
|
||||
:status, :route, :publisher, :addtime, :updatetime, :area,
|
||||
:category, :theme, :year, :video_remote_id, :video_url,
|
||||
:video_list, :month
|
||||
:video_list, :month, :video_status
|
||||
)"""),
|
||||
wz_video
|
||||
)
|
||||
|
@ -216,7 +216,7 @@
|
||||
<th>爬虫名称</th>
|
||||
<th>URL</th>
|
||||
<th>剧集ID</th>
|
||||
<th>状态</th>
|
||||
<th>定时任务开关</th>
|
||||
<th>运行状态</th>
|
||||
<th>操作</th>
|
||||
</tr>
|
||||
@ -558,7 +558,7 @@
|
||||
<td>${task.spider_name}</td>
|
||||
<td>${task.url}</td>
|
||||
<td>${task.video_list}</td>
|
||||
<td>
|
||||
<td align="center">
|
||||
<label class="switch">
|
||||
<input type="checkbox" ${task.enabled ? "checked" : ""}
|
||||
onchange="toggleTask(${task.id}, this.checked)">
|
||||
@ -572,7 +572,7 @@
|
||||
<button class="btn btn-primary btn-sm" onclick="runTask(${
|
||||
task.id
|
||||
})">
|
||||
立即执行
|
||||
手动运行
|
||||
</button>
|
||||
<button class="btn btn-warning btn-sm" onclick="editTask(${
|
||||
task.id
|
||||
|
Loading…
Reference in New Issue
Block a user