调整爬取速度,上传视频URL前等待3秒

This commit is contained in:
ifui 2025-08-10 17:30:58 +08:00
parent 30a9056751
commit ec5309255b
3 changed files with 12 additions and 9 deletions

View File

@ -1,5 +1,5 @@
# 使用官方Python基础镜像
FROM python:3.9-bookworm
# 使用国内Python基础镜像
FROM swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/python:3.9-bookworm
# 设置工作目录
WORKDIR /app

View File

@ -221,6 +221,8 @@ class AliyunVodPipeline:
# 2. 通过URL上传视频获取JobId
title = item.get('title', '')
# 延迟3秒上传
time.sleep(3)
job_id = self.upload_media_by_url(
video_url=video_url,
title=title,
@ -378,7 +380,7 @@ class MariaDBPipeline:
Returns:
item: 处理后的item
"""
# 这里不需要处理item因为我们要从SQLite读取数据
# 从SQLite读取数据
return item
def migrate_data(self):
@ -425,7 +427,8 @@ class MariaDBPipeline:
'video_remote_id': video.aliyun_video_id or '',
'video_url': '',
'video_list': video.video_list or 0,
'month': '1'
'month': '1',
'video_status': video.aliyun_status or '',
}
# 映射到wz_video_data表
@ -447,12 +450,12 @@ class MariaDBPipeline:
cid, title, css, thumb, keywords, remark, block, url,
status, route, publisher, addtime, updatetime, area,
category, theme, year, video_remote_id, video_url,
video_list, month
video_list, month, video_status
) VALUES (
:cid, :title, :css, :thumb, :keywords, :remark, :block, :url,
:status, :route, :publisher, :addtime, :updatetime, :area,
:category, :theme, :year, :video_remote_id, :video_url,
:video_list, :month
:video_list, :month, :video_status
)"""),
wz_video
)

View File

@ -216,7 +216,7 @@
<th>爬虫名称</th>
<th>URL</th>
<th>剧集ID</th>
<th>状态</th>
<th>定时任务开关</th>
<th>运行状态</th>
<th>操作</th>
</tr>
@ -558,7 +558,7 @@
<td>${task.spider_name}</td>
<td>${task.url}</td>
<td>${task.video_list}</td>
<td>
<td align="center">
<label class="switch">
<input type="checkbox" ${task.enabled ? "checked" : ""}
onchange="toggleTask(${task.id}, this.checked)">
@ -572,7 +572,7 @@
<button class="btn btn-primary btn-sm" onclick="runTask(${
task.id
})">
立即执
手动运
</button>
<button class="btn btn-warning btn-sm" onclick="editTask(${
task.id