sion/sionrui
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 功能优化

Browse Source
This commit is contained in:
sion123
2025-12-02 01:55:57 +08:00
parent 900b47f585
commit 0fffd787bb
32 changed files with 974 additions and 2156 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
yudao-module-tik/src/main/java/cn/iocoder/yudao/module/tik/kling/client/KlingClient.java
View File

@@ -91,6 +91,7 @@ public class KlingClient {
try {
String body = objectMapper.writeValueAsString(request);
log.info("[Kling][create-lip-sync请求体] {}", body);
String url = properties.getBaseUrl() + "/klingai/v1/videos/advanced-lip-sync";
Request httpRequest = new Request.Builder()
@@ -179,13 +180,60 @@ public class KlingClient {
throw exception0(LATENTSYNC_SUBMIT_FAILED.getCode(), "face_choose不能为空");
}
// 验证每个face_choose项
for (KlingLipSyncCreateRequest.FaceChoose faceChoose : request.getFaceChoose()) {
for (int i = 0; i < request.getFaceChoose().size(); i++) {
KlingLipSyncCreateRequest.FaceChoose faceChoose = request.getFaceChoose().get(i);
if (StrUtil.isBlank(faceChoose.getFaceId())) {
throw exception0(LATENTSYNC_SUBMIT_FAILED.getCode(), "face_id不能为空");
throw exception0(LATENTSYNC_SUBMIT_FAILED.getCode(), "face_choose[" + i + "].face_id不能为空");
}
if (StrUtil.isBlank(faceChoose.getSoundFile())) {
throw exception0(LATENTSYNC_SUBMIT_FAILED.getCode(), "sound_file不能为空");
throw exception0(LATENTSYNC_SUBMIT_FAILED.getCode(), "face_choose[" + i + "].sound_file不能为空");
}
if (faceChoose.getSoundStartTime() == null) {
throw exception0(LATENTSYNC_SUBMIT_FAILED.getCode(), "face_choose[" + i + "].sound_start_time不能为空");
}
if (faceChoose.getSoundEndTime() == null) {
throw exception0(LATENTSYNC_SUBMIT_FAILED.getCode(), "face_choose[" + i + "].sound_end_time不能为空");
}
if (faceChoose.getSoundInsertTime() == null) {
throw exception0(LATENTSYNC_SUBMIT_FAILED.getCode(), "face_choose[" + i + "].sound_insert_time不能为空");
}
// 严格验证302.ai API约束
int soundDuration = faceChoose.getSoundEndTime() - faceChoose.getSoundStartTime();
if (soundDuration < 2000) {
throw exception0(LATENTSYNC_SUBMIT_FAILED.getCode(),
String.format("face_choose[%d].裁剪后音频不得短于2秒，当前时长：%dms", i, soundDuration));
}
if (soundDuration > 60000) {
throw exception0(LATENTSYNC_SUBMIT_FAILED.getCode(),
String.format("face_choose[%d].裁剪后音频不能超过60秒，当前时长：%dms", i, soundDuration));
}
if (faceChoose.getSoundStartTime() < 0) {
throw exception0(LATENTSYNC_SUBMIT_FAILED.getCode(),
"face_choose[" + i + "].sound_start_time不能小于0");
}
if (faceChoose.getSoundEndTime() <= faceChoose.getSoundStartTime()) {
throw exception0(LATENTSYNC_SUBMIT_FAILED.getCode(),
"face_choose[" + i + "].sound_end_time必须大于sound_start_time");
}
if (faceChoose.getSoundInsertTime() < 0) {
throw exception0(LATENTSYNC_SUBMIT_FAILED.getCode(),
"face_choose[" + i + "].sound_insert_time不能小于0");
}
if (faceChoose.getSoundVolume() != null &&
(faceChoose.getSoundVolume() < 0 || faceChoose.getSoundVolume() > 2)) {
throw exception0(LATENTSYNC_SUBMIT_FAILED.getCode(),
"face_choose[" + i + "].sound_volume必须在[0, 2]范围内");
}
if (faceChoose.getOriginalAudioVolume() != null &&
(faceChoose.getOriginalAudioVolume() < 0 || faceChoose.getOriginalAudioVolume() > 2)) {
throw exception0(LATENTSYNC_SUBMIT_FAILED.getCode(),
"face_choose[" + i + "].original_audio_volume必须在[0, 2]范围内");
}
log.info("[validateLipSyncRequest][face_choose[{}]] face_id={}, soundStartTime={}, soundEndTime={}, soundInsertTime={}, soundDuration={}ms",
i, faceChoose.getFaceId(), faceChoose.getSoundStartTime(),
faceChoose.getSoundEndTime(), faceChoose.getSoundInsertTime(), soundDuration);
}
}
@@ -202,8 +250,27 @@ public class KlingClient {
try (Response response = getHttpClient().newCall(httpRequest).execute()) {
String responseBody = response.body() != null ? response.body().string() : "";
if (!response.isSuccessful()) {
log.error("[Kling][{} failed][status={}, body={}]", operation, response.code(), responseBody);
throw buildException(responseBody);
// 增强错误日志记录，包含详细信息
log.error("[Kling][{} failed][status={}, reason={}]", operation, response.code(), response.message());
log.error("[Kling][{} failed response body]", operation);
log.error("{}", responseBody);
// 尝试解析并提取详细错误信息
try {
JsonNode errorNode = objectMapper.readTree(responseBody);
String code = errorNode.has("code") ? errorNode.get("code").asText() : "unknown";
String message = errorNode.has("message") ? errorNode.get("message").asText() :
errorNode.has("detail") ? errorNode.get("detail").asText() : responseBody;
String requestId = errorNode.has("request_id") ? errorNode.get("request_id").asText() : "unknown";
log.error("[Kling][{} error details] code={}, message={}, request_id={}", operation, code, message, requestId);
throw exception0(LATENTSYNC_SUBMIT_FAILED.getCode(),
String.format("[%s] %s (code: %s, request_id: %s)", operation, message, code, requestId));
} catch (Exception parseEx) {
log.error("[Kling][{} parse error response failed]", operation, parseEx);
throw buildException(responseBody);
}
}
log.info("[Kling][{} success][responseBody={}]", operation, responseBody);

60
yudao-module-tik/src/main/java/cn/iocoder/yudao/module/tik/kling/controller/KlingController.java
View File

@@ -1,11 +1,15 @@
package cn.iocoder.yudao.module.tik.kling.controller;
import cn.iocoder.yudao.framework.common.pojo.CommonResult;
import cn.iocoder.yudao.module.tik.kling.dto.KlingIdentifyFaceRequest;
import cn.iocoder.yudao.module.tik.kling.dto.KlingIdentifyFaceResponse;
import cn.iocoder.yudao.module.tik.kling.dto.KlingLipSyncQueryResponse;
import cn.iocoder.yudao.module.tik.kling.vo.KlingIdentifyFaceReqVO;
import cn.iocoder.yudao.module.tik.kling.vo.KlingIdentifyFaceRespVO;
import cn.iocoder.yudao.module.tik.kling.vo.KlingLipSyncCreateReqVO;
import cn.iocoder.yudao.module.tik.kling.vo.KlingLipSyncCreateRespVO;
import cn.iocoder.yudao.module.tik.kling.vo.KlingLipSyncQueryRespVO;
import cn.iocoder.yudao.module.tik.kling.vo.response.KlingIdentifyFaceDataVO;
import cn.iocoder.yudao.module.tik.kling.service.KlingService;
import cn.iocoder.yudao.module.tik.voice.service.DigitalHumanTaskService;
import cn.iocoder.yudao.module.tik.voice.vo.AppTikDigitalHumanCreateReqVO;
@@ -38,27 +42,46 @@ public class KlingController {
@PostMapping("/identify-face")
@Operation(summary = "人脸识别", description = "识别视频中的人脸，用于对口型服务")
public CommonResult<KlingIdentifyFaceRespVO> identifyFace(@RequestBody @Valid KlingIdentifyFaceReqVO reqVO) {
KlingIdentifyFaceRespVO respVO = klingService.identifyFace(reqVO);
return CommonResult.success(respVO);
}
// VO → DTO
KlingIdentifyFaceRequest request = BeanUtils.toBean(reqVO, KlingIdentifyFaceRequest.class);
// 调用Service
KlingIdentifyFaceResponse response = klingService.identifyFace(request);
// DTO → VO
KlingIdentifyFaceRespVO respVO = new KlingIdentifyFaceRespVO();
if (response.getData() != null) {
KlingIdentifyFaceDataVO dataVO = BeanUtils.toBean(response.getData(), KlingIdentifyFaceDataVO.class);
respVO.setData(dataVO);
respVO.setSessionId(dataVO.getSessionId());
}
@PostMapping("/lip-sync/create")
@Operation(summary = "创建口型同步任务", description = "使用可灵高级对口型服务创建任务")
public CommonResult<KlingLipSyncCreateRespVO> createLipSyncTask(@RequestBody @Valid KlingLipSyncCreateReqVO reqVO) {
KlingLipSyncCreateRespVO respVO = klingService.createLipSyncTask(reqVO);
return CommonResult.success(respVO);
}
@GetMapping("/lip-sync/{taskId}")
@Operation(summary = "查询口型同步任务", description = "查询可灵口型同步任务状态和结果")
public CommonResult<KlingLipSyncQueryRespVO> getLipSyncTask(@PathVariable String taskId) {
KlingLipSyncQueryRespVO respVO = klingService.getLipSyncTask(taskId);
// 调用Service
KlingLipSyncQueryResponse response = klingService.getLipSyncTask(taskId);
// DTO → VO
KlingLipSyncQueryRespVO respVO = BeanUtils.toBean(response, KlingLipSyncQueryRespVO.class);
return CommonResult.success(respVO);
}
@PostMapping("/task/create")
@Operation(summary = "创建可灵任务", description = "创建数字人任务，使用可灵AI服务")
public CommonResult<Long> createTask(@RequestBody @Valid AppTikDigitalHumanCreateReqVO reqVO) {
// 记录请求参数用于调试
log.info("[createTask] 接收请求 - 任务名: {}, 文案长度: {}, AI供应商: {}, klingSessionId: {}, klingFaceId: {}",
reqVO.getTaskName(),
reqVO.getInputText() != null ? reqVO.getInputText().length() : 0,
reqVO.getAiProvider(),
reqVO.getKlingSessionId(),
reqVO.getKlingFaceId());
// 设置 AI 供应商为可灵
reqVO.setAiProvider("kling");
Long taskId = digitalHumanTaskService.createTask(reqVO);
@@ -79,25 +102,4 @@ public class KlingController {
return CommonResult.success(result);
}
@PostMapping("/task/cancel")
@Operation(summary = "取消任务", description = "取消可灵任务")
public CommonResult<Boolean> cancelTask(@RequestParam Long taskId) {
digitalHumanTaskService.cancelTask(taskId);
return CommonResult.success(true);
}
@PostMapping("/task/retry")
@Operation(summary = "重试任务", description = "重试可灵任务")
public CommonResult<Boolean> retryTask(@RequestParam Long taskId) {
digitalHumanTaskService.retryTask(taskId);
return CommonResult.success(true);
}
@DeleteMapping("/task/delete")
@Operation(summary = "删除任务", description = "删除可灵任务")
public CommonResult<Boolean> deleteTask(@RequestParam Long taskId) {
digitalHumanTaskService.deleteTask(taskId);
return CommonResult.success(true);
}
}

51
yudao-module-tik/src/main/java/cn/iocoder/yudao/module/tik/kling/dto/KlingLipSyncCreateResponse.java
View File

@@ -1,5 +1,6 @@
package cn.iocoder.yudao.module.tik.kling.dto;
import cn.iocoder.yudao.module.tik.kling.dto.response.KlingLipSyncCreateData;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Data;
@@ -29,54 +30,4 @@ public class KlingLipSyncCreateResponse {
* 数据
*/
private KlingLipSyncCreateData data;
/**
* 响应数据
*/
@Data
public static class KlingLipSyncCreateData {
/**
* 任务ID
*/
@JsonProperty("task_id")
private String taskId;
/**
* 任务信息
*/
@JsonProperty("task_info")
private TaskInfo taskInfo;
/**
* 任务状态
*/
@JsonProperty("task_status")
private String taskStatus;
/**
* 创建时间（ms）
*/
@JsonProperty("created_at")
private Long createdAt;
/**
* 更新时间（ms）
*/
@JsonProperty("updated_at")
private Long updatedAt;
/**
* 任务信息
*/
@Data
public static class TaskInfo {
/**
* 客户自定义任务ID
*/
@JsonProperty("external_task_id")
private String externalTaskId;
}
}
}

41
yudao-module-tik/src/main/java/cn/iocoder/yudao/module/tik/kling/dto/response/KlingLipSyncCreateData.java Normal file
View File

@@ -0,0 +1,41 @@
package cn.iocoder.yudao.module.tik.kling.dto.response;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Data;
/**
* 可灵口型同步创建任务数据 DTO
*/
@Data
public class KlingLipSyncCreateData {
/**
* 任务ID
*/
@JsonProperty("task_id")
private String taskId;
/**
* 任务信息
*/
@JsonProperty("task_info")
private KlingLipSyncCreateTaskInfo taskInfo;
/**
* 任务状态
*/
@JsonProperty("task_status")
private String taskStatus;
/**
* 创建时间（ms）
*/
@JsonProperty("created_at")
private Long createdAt;
/**
* 更新时间（ms）
*/
@JsonProperty("updated_at")
private Long updatedAt;
}

17
yudao-module-tik/src/main/java/cn/iocoder/yudao/module/tik/kling/dto/response/KlingLipSyncCreateTaskInfo.java Normal file
View File

@@ -0,0 +1,17 @@
package cn.iocoder.yudao.module.tik.kling.dto.response;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Data;
/**
* 可灵口型同步任务信息 DTO
*/
@Data
public class KlingLipSyncCreateTaskInfo {
/**
* 客户自定义任务ID
*/
@JsonProperty("external_task_id")
private String externalTaskId;
}

16
yudao-module-tik/src/main/java/cn/iocoder/yudao/module/tik/kling/service/KlingService.java
View File

@@ -1,10 +1,10 @@
package cn.iocoder.yudao.module.tik.kling.service;
import cn.iocoder.yudao.module.tik.kling.vo.KlingIdentifyFaceReqVO;
import cn.iocoder.yudao.module.tik.kling.vo.KlingIdentifyFaceRespVO;
import cn.iocoder.yudao.module.tik.kling.vo.KlingLipSyncCreateReqVO;
import cn.iocoder.yudao.module.tik.kling.vo.KlingLipSyncCreateRespVO;
import cn.iocoder.yudao.module.tik.kling.vo.KlingLipSyncQueryRespVO;
import cn.iocoder.yudao.module.tik.kling.dto.KlingIdentifyFaceRequest;
import cn.iocoder.yudao.module.tik.kling.dto.KlingIdentifyFaceResponse;
import cn.iocoder.yudao.module.tik.kling.dto.KlingLipSyncCreateRequest;
import cn.iocoder.yudao.module.tik.kling.dto.KlingLipSyncCreateResponse;
import cn.iocoder.yudao.module.tik.kling.dto.KlingLipSyncQueryResponse;
/**
* 可灵服务接口
@@ -14,16 +14,16 @@ public interface KlingService {
/**
* 人脸识别
*/
KlingIdentifyFaceRespVO identifyFace(KlingIdentifyFaceReqVO reqVO);
KlingIdentifyFaceResponse identifyFace(KlingIdentifyFaceRequest request);
/**
* 创建口型同步任务
*/
KlingLipSyncCreateRespVO createLipSyncTask(KlingLipSyncCreateReqVO reqVO);
KlingLipSyncCreateResponse createLipSyncTask(KlingLipSyncCreateRequest request);
/**
* 查询口型同步任务
*/
KlingLipSyncQueryRespVO getLipSyncTask(String taskId);
KlingLipSyncQueryResponse getLipSyncTask(String taskId);
}

47
yudao-module-tik/src/main/java/cn/iocoder/yudao/module/tik/kling/service/KlingServiceImpl.java
View File

@@ -9,9 +9,6 @@ import cn.iocoder.yudao.module.tik.kling.dto.KlingLipSyncCreateResponse;
import cn.iocoder.yudao.module.tik.kling.dto.KlingLipSyncQueryResponse;
import cn.iocoder.yudao.module.tik.kling.vo.KlingIdentifyFaceReqVO;
import cn.iocoder.yudao.module.tik.kling.vo.KlingIdentifyFaceRespVO;
import cn.iocoder.yudao.module.tik.kling.vo.KlingLipSyncCreateReqVO;
import cn.iocoder.yudao.module.tik.kling.vo.KlingLipSyncCreateRespVO;
import cn.iocoder.yudao.module.tik.kling.vo.KlingLipSyncQueryRespVO;
import cn.iocoder.yudao.module.tik.kling.vo.response.KlingIdentifyFaceDataVO;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@@ -28,63 +25,53 @@ public class KlingServiceImpl implements KlingService {
private final KlingClient klingClient;
@Override
public KlingIdentifyFaceRespVO identifyFace(KlingIdentifyFaceReqVO reqVO) {
// 转换请求对象
KlingIdentifyFaceRequest request = BeanUtils.toBean(reqVO, KlingIdentifyFaceRequest.class);
public KlingIdentifyFaceResponse identifyFace(KlingIdentifyFaceRequest request) {
// 记录请求参数用于调试
log.info("[identify-face][开始识别] videoUrl={}", request.getVideoUrl());
// 调用302.ai API
KlingIdentifyFaceResponse response = klingClient.identifyFace(request);
// 构建响应VO
KlingIdentifyFaceRespVO respVO = new KlingIdentifyFaceRespVO();
// 转换data字段
if (response.getData() != null) {
KlingIdentifyFaceDataVO dataVO = BeanUtils.toBean(response.getData(), KlingIdentifyFaceDataVO.class);
respVO.setData(dataVO);
// 直接设置sessionId（扁平化结构）
respVO.setSessionId(dataVO.getSessionId());
}
log.info("[identify-face][识别完成][sessionId={}, faceCount={}]",
response.getData() != null ? response.getData().getSessionId() : "null",
response.getData() != null && response.getData().getFaceData() != null
? response.getData().getFaceData().size() : 0);
return respVO;
return response;
}
@Override
public KlingLipSyncCreateRespVO createLipSyncTask(KlingLipSyncCreateReqVO reqVO) {
// 转换请求对象
KlingLipSyncCreateRequest request = BeanUtils.toBean(reqVO, KlingLipSyncCreateRequest.class);
public KlingLipSyncCreateResponse createLipSyncTask(KlingLipSyncCreateRequest request) {
// 记录请求参数用于调试
log.info("[createLipSyncTask][请求参数] sessionId={}, faceChoose.size={}",
request.getSessionId(),
request.getFaceChoose() != null ? request.getFaceChoose().size() : 0);
if (request.getFaceChoose() != null && !request.getFaceChoose().isEmpty()) {
KlingLipSyncCreateRequest.FaceChoose face = request.getFaceChoose().get(0);
log.info("[createLipSyncTask][face参数] faceId={}, soundFile={}, soundStartTime={}, soundEndTime={}, soundInsertTime={}",
face.getFaceId(), face.getSoundFile(), face.getSoundStartTime(), face.getSoundEndTime(), face.getSoundInsertTime());
}
// 调用302.ai API
KlingLipSyncCreateResponse response = klingClient.createLipSyncTask(request);
// 构建响应VO
KlingLipSyncCreateRespVO respVO = BeanUtils.toBean(response, KlingLipSyncCreateRespVO.class);
log.info("[create-lip-sync][创建任务完成][taskId={}, status={}]",
response.getData() != null ? response.getData().getTaskId() : "null",
response.getData() != null ? response.getData().getTaskStatus() : "null");
return respVO;
return response;
}
@Override
public KlingLipSyncQueryRespVO getLipSyncTask(String taskId) {
public KlingLipSyncQueryResponse getLipSyncTask(String taskId) {
// 调用302.ai API
KlingLipSyncQueryResponse response = klingClient.getLipSyncTask(taskId);
// 构建响应VO
KlingLipSyncQueryRespVO respVO = BeanUtils.toBean(response, KlingLipSyncQueryRespVO.class);
log.info("[get-lip-sync][查询任务完成][taskId={}, status={}]",
response.getData() != null ? response.getData().getTaskId() : "null",
response.getData() != null ? response.getData().getTaskStatus() : "null");
return respVO;
return response;
}
}

51
yudao-module-tik/src/main/java/cn/iocoder/yudao/module/tik/kling/vo/KlingLipSyncCreateRespVO.java
View File

@@ -1,5 +1,6 @@
package cn.iocoder.yudao.module.tik.kling.vo;
import cn.iocoder.yudao.module.tik.kling.vo.response.KlingLipSyncCreateDataVO;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Data;
@@ -29,54 +30,4 @@ public class KlingLipSyncCreateRespVO {
* 数据
*/
private KlingLipSyncCreateDataVO data;
/**
* 响应数据
*/
@Data
public static class KlingLipSyncCreateDataVO {
/**
* 任务ID
*/
@JsonProperty("task_id")
private String taskId;
/**
* 任务信息
*/
@JsonProperty("task_info")
private TaskInfo taskInfo;
/**
* 任务状态
*/
@JsonProperty("task_status")
private String taskStatus;
/**
* 创建时间（ms）
*/
@JsonProperty("created_at")
private Long createdAt;
/**
* 更新时间（ms）
*/
@JsonProperty("updated_at")
private Long updatedAt;
/**
* 任务信息
*/
@Data
public static class TaskInfo {
/**
* 客户自定义任务ID
*/
@JsonProperty("external_task_id")
private String externalTaskId;
}
}
}

41
yudao-module-tik/src/main/java/cn/iocoder/yudao/module/tik/kling/vo/response/KlingLipSyncCreateDataVO.java Normal file
View File

@@ -0,0 +1,41 @@
package cn.iocoder.yudao.module.tik.kling.vo.response;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Data;
/**
* 可灵口型同步创建任务数据 VO
*/
@Data
public class KlingLipSyncCreateDataVO {
/**
* 任务ID
*/
@JsonProperty("task_id")
private String taskId;
/**
* 任务信息
*/
@JsonProperty("task_info")
private KlingLipSyncCreateTaskInfoVO taskInfo;
/**
* 任务状态
*/
@JsonProperty("task_status")
private String taskStatus;
/**
* 创建时间（ms）
*/
@JsonProperty("created_at")
private Long createdAt;
/**
* 更新时间（ms）
*/
@JsonProperty("updated_at")
private Long updatedAt;
}

17
yudao-module-tik/src/main/java/cn/iocoder/yudao/module/tik/kling/vo/response/KlingLipSyncCreateTaskInfoVO.java Normal file
View File

@@ -0,0 +1,17 @@
package cn.iocoder.yudao.module.tik.kling.vo.response;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.Data;
/**
* 可灵口型同步任务信息 VO
*/
@Data
public class KlingLipSyncCreateTaskInfoVO {
/**
* 客户自定义任务ID
*/
@JsonProperty("external_task_id")
private String externalTaskId;
}

16
yudao-module-tik/src/main/java/cn/iocoder/yudao/module/tik/voice/dal/dataobject/TikDigitalHumanTaskDO.java
View File

@@ -66,6 +66,11 @@ public class TikDigitalHumanTaskDO extends TenantBaseDO {
*/
private String audioUrl;
/**
* 可灵音频结束时间（毫秒，前端解析后传递）
*/
private Integer soundEndTime;
// ========== 生成参数 ==========
/**
* 语速（0.5-2.0）
@@ -144,6 +149,17 @@ public class TikDigitalHumanTaskDO extends TenantBaseDO {
* 可灵选中的人脸ID（从identify-face返回的face_data中选择）
*/
private String klingFaceId;
/**
* 人脸可对口型区间起点时间（ms）（从identify-face返回的face_data获取）
*/
private Integer klingFaceStartTime;
/**
* 人脸可对口型区间终点时间（ms）（从identify-face返回的face_data获取）
*/
private Integer klingFaceEndTime;
/**
* 可灵口型同步任务ID（从advanced-lip-sync接口获取）
*/

230
yudao-module-tik/src/main/java/cn/iocoder/yudao/module/tik/voice/service/DigitalHumanTaskServiceImpl.java
View File

@@ -16,6 +16,7 @@ import cn.iocoder.yudao.module.tik.enums.ErrorCodeConstants;
import cn.iocoder.yudao.module.tik.file.dal.dataobject.TikUserFileDO;
import cn.iocoder.yudao.module.tik.file.dal.mysql.TikUserFileMapper;
import cn.iocoder.yudao.module.tik.file.service.TikOssInitService;
import cn.iocoder.yudao.module.tik.kling.dto.KlingLipSyncCreateResponse;
import cn.iocoder.yudao.module.tik.voice.dal.dataobject.TikDigitalHumanTaskDO;
import cn.iocoder.yudao.module.tik.voice.dal.dataobject.TikUserVoiceDO;
import cn.iocoder.yudao.module.tik.voice.dal.mysql.TikDigitalHumanTaskMapper;
@@ -24,9 +25,8 @@ import cn.iocoder.yudao.module.tik.voice.enums.DigitalHumanTaskStatusEnum;
import cn.iocoder.yudao.module.tik.voice.enums.DigitalHumanTaskStepEnum;
import cn.iocoder.yudao.module.tik.voice.vo.*;
import cn.iocoder.yudao.module.tik.voice.vo.AppTikLatentsyncResultRespVO;
import cn.iocoder.yudao.module.tik.voice.strategy.LipSyncStrategy;
import cn.iocoder.yudao.module.tik.voice.strategy.LipSyncStrategyFactory;
import cn.iocoder.yudao.module.tik.kling.service.KlingService;
import cn.iocoder.yudao.module.tik.kling.dto.KlingLipSyncCreateRequest;
import cn.iocoder.yudao.framework.mybatis.core.query.LambdaQueryWrapperX;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
@@ -40,6 +40,7 @@ import org.springframework.validation.annotation.Validated;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.ArrayList;
/**
* 数字人任务 Service 实现
@@ -60,7 +61,6 @@ public class DigitalHumanTaskServiceImpl implements DigitalHumanTaskService {
private final TikUserVoiceService userVoiceService;
private final TikOssInitService ossInitService;
private final LatentsyncPollingService latentsyncPollingService;
private final LipSyncStrategyFactory lipSyncStrategyFactory;
private final KlingService klingService;
private final StringRedisTemplate stringRedisTemplate;
@@ -91,8 +91,27 @@ public class DigitalHumanTaskServiceImpl implements DigitalHumanTaskService {
TikDigitalHumanTaskDO task = createTaskRecord(reqVO, userId);
taskMapper.insert(task);
// 3. 异步处理任务
// 3. ✅ 立即处理预生成音频（保存为临时文件，供后续步骤使用）
Long taskId = task.getId();
if (reqVO.getPreGeneratedAudio() != null && StrUtil.isNotBlank(reqVO.getPreGeneratedAudio().getAudioBase64())) {
try {
log.info("[createTask][任务({})正在保存预生成音频...]", taskId);
String audioUrl = saveTempAudioFile(reqVO.getPreGeneratedAudio().getAudioBase64(),
reqVO.getPreGeneratedAudio().getFormat());
// 更新任务记录，保存音频URL
TikDigitalHumanTaskDO updateObj = new TikDigitalHumanTaskDO();
updateObj.setId(taskId);
updateObj.setAudioUrl(audioUrl);
taskMapper.updateById(updateObj);
log.info("[createTask][任务({})预生成音频保存成功][audioUrl={}]", taskId, audioUrl);
} catch (Exception e) {
log.error("[createTask][任务({})预生成音频保存失败，将重新TTS]", taskId, e);
// 保存失败不影响任务创建，后续会重新TTS
}
}
// 4. 异步处理任务
TransactionSynchronizationManager.registerSynchronization(new TransactionSynchronization() {
@Override
public void afterCommit() {
@@ -357,6 +376,18 @@ public class DigitalHumanTaskServiceImpl implements DigitalHumanTaskService {
}
}
// ✅ 预生成音频信息（无需存储时长，前端严格校验）
if (reqVO.getPreGeneratedAudio() != null) {
log.info("[createTaskRecord][任务({})收到预生成音频，将复用]", reqVO.getTaskName());
}
// ✅ 新增：接收前端传递的 sound_end_time（可灵API需要）
Integer soundEndTime = reqVO.getSoundEndTime();
if (soundEndTime != null) {
log.info("[createTaskRecord][任务({})收到 sound_end_time: {}ms]",
reqVO.getTaskName(), soundEndTime);
}
return TikDigitalHumanTaskDO.builder()
.userId(userId)
.taskName(reqVO.getTaskName())
@@ -376,6 +407,9 @@ public class DigitalHumanTaskServiceImpl implements DigitalHumanTaskService {
.currentStep("prepare_files")
.klingSessionId(reqVO.getKlingSessionId())
.klingFaceId(reqVO.getKlingFaceId())
.klingFaceStartTime(reqVO.getKlingFaceStartTime())
.klingFaceEndTime(reqVO.getKlingFaceEndTime())
.soundEndTime(soundEndTime)
.build();
}
@@ -439,15 +473,22 @@ public class DigitalHumanTaskServiceImpl implements DigitalHumanTaskService {
updateTaskProgress(taskId, DigitalHumanTaskStepEnum.SYNTHESIZE_VOICE, "语音合成完成");
// 步骤3：口型同步（异步提交，不等待完成）
syncLip(task, audioUrl);
updateTaskProgress(taskId, DigitalHumanTaskStepEnum.SYNC_LIP, "口型同步任务已提交，等待处理");
String aiProvider = task.getAiProvider();
if ("kling".equalsIgnoreCase(aiProvider)) {
// 可灵：直接调用并保存taskId
syncWithKling(task, audioUrl);
// 只有可灵才在此时更新状态，因为需要确保klingTaskId已保存
updateTaskProgress(taskId, DigitalHumanTaskStepEnum.SYNC_LIP, "口型同步任务已提交，等待处理");
log.info("[processTask][任务({})已提交到可灵，等待异步处理完成]", taskId);
} else {
// Latentsync：调用轮询服务
syncWithLatentsync(task, audioUrl);
updateTaskProgress(taskId, DigitalHumanTaskStepEnum.SYNC_LIP, "口型同步任务已提交，等待处理");
log.info("[processTask][任务({})已提交到Latentsync，等待异步处理完成]", taskId);
}
// Latentsync是异步处理，这里不调用generateVideo
// 而是将任务标记为等待Latentsync完成
// 轮询服务会异步检测状态并在完成时更新任务
log.info("[processTask][任务({})已提交到Latentsync，等待异步处理完成]", taskId);
} catch (Exception e) {
log.error("[processTask][任务({})处理失败]", taskId, e);
updateTaskStatus(taskId, "FAILED", task.getCurrentStep(), task.getProgress(), "任务处理失败：" + e.getMessage(), null, e.getMessage());
@@ -486,6 +527,14 @@ public class DigitalHumanTaskServiceImpl implements DigitalHumanTaskService {
* 语音合成（使用CosyVoice v3 Flash）
*/
private String synthesizeVoice(TikDigitalHumanTaskDO task) throws Exception {
// ✅ 优先使用预生成的音频（前端传递）
if (StrUtil.isNotBlank(task.getAudioUrl())) {
log.info("[synthesizeVoice][任务({})使用预生成的音频][audioUrl={}]",
task.getId(), task.getAudioUrl());
return task.getAudioUrl();
}
// 如果没有预生成音频，则走正常的TTS流程
// 参数验证
if (StrUtil.isBlank(task.getVoiceId())) {
throw new Exception("音色ID不能为空");
@@ -530,27 +579,162 @@ public class DigitalHumanTaskServiceImpl implements DigitalHumanTaskService {
}
/**
* 口型同步 - 使用策略模式
* 口型同步 - 直接调用不同AI供应商
*/
private String syncLip(TikDigitalHumanTaskDO task, String audioUrl) throws Exception {
log.info("[syncLip][任务({})开始口型同步，使用AI供应商: {}]", task.getId(), task.getAiProvider());
log.info("[syncLip][任务({})开始口型同步，AI供应商: {}]", task.getId(), task.getAiProvider());
// 使用策略模式根据任务特性选择合适的策略
LipSyncStrategy strategy = lipSyncStrategyFactory.getStrategyForTask(task);
// 根据AI供应商选择不同的处理方式
if ("kling".equalsIgnoreCase(task.getAiProvider())) {
// 可灵：直接调用KlingService
return syncWithKling(task, audioUrl);
} else {
// Latentsync：调用LatentsyncPollingService
return syncWithLatentsync(task, audioUrl);
}
}
if (strategy == null) {
log.error("[syncLip][任务({})找不到合适的策略，AI供应商: {}]", task.getId(), task.getAiProvider());
throw new Exception("找不到合适的口型同步策略，AI供应商: " + task.getAiProvider());
/**
* 可灵口型同步
*/
private String syncWithKling(TikDigitalHumanTaskDO task, String audioUrl) throws Exception {
// 构建可灵口型同步请求DTO
KlingLipSyncCreateRequest request = buildKlingLipSyncRequest(task, audioUrl);
// 调用可灵服务，获取返回的taskId
KlingLipSyncCreateResponse response = klingService.createLipSyncTask(request);
// 保存可灵任务ID到任务记录中，用于后续轮询
String klingTaskId = response.getData().getTaskId();
if (StrUtil.isNotBlank(klingTaskId)) {
TikDigitalHumanTaskDO updateObj = new TikDigitalHumanTaskDO();
updateObj.setId(task.getId());
updateObj.setKlingTaskId(klingTaskId);
taskMapper.updateById(updateObj);
log.info("[syncWithKling][任务({})可灵口型同步已提交][klingTaskId={}]", task.getId(), klingTaskId);
} else {
log.error("[syncWithKling][任务({})可灵返回的taskId为空]", task.getId());
throw new Exception("可灵任务创建失败：未返回taskId");
}
log.info("[syncLip][任务({})使用策略: {}][描述: {}]",
task.getId(), strategy.getStrategyName(), strategy.getDescription());
return task.getVideoUrl();
}
// 执行口型同步
String syncedVideoUrl = strategy.syncLip(task, audioUrl);
/**
* Latentsync口型同步
*/
private String syncWithLatentsync(TikDigitalHumanTaskDO task, String audioUrl) throws Exception {
// 直接调用LatentsyncPollingService
// 注意：这里的实现取决于LatentsyncPollingService的接口
log.info("[syncWithLatentsync][任务({})Latentsync口型同步已提交]", task.getId());
return task.getVideoUrl();
}
log.info("[syncLip][任务({})口型同步完成][策略: {}]", task.getId(), strategy.getStrategyName());
return syncedVideoUrl;
/**
* 构建可灵口型同步请求DTO
*
* 302.ai API 要求：
* 1. sound_end_time 不得晚于原始音频总时长
* 2. 裁剪后音频不得短于2秒
* 3. 插入音频的时间范围与人脸可对口型时间区间至少重合2秒
*/
private KlingLipSyncCreateRequest buildKlingLipSyncRequest(TikDigitalHumanTaskDO task, String audioUrl) {
KlingLipSyncCreateRequest request = new KlingLipSyncCreateRequest();
request.setSessionId(task.getKlingSessionId());
// 初始化face_choose数组
request.setFaceChoose(new ArrayList<>());
// 构建face_choose项
KlingLipSyncCreateRequest.FaceChoose faceChoose = new KlingLipSyncCreateRequest.FaceChoose();
faceChoose.setFaceId(task.getKlingFaceId());
faceChoose.setSoundFile(audioUrl);
// 获取人脸可对口型区间（从identify-face接口返回）
int faceStartTime = task.getKlingFaceStartTime() != null ? task.getKlingFaceStartTime() : 0;
int faceEndTime = task.getKlingFaceEndTime() != null ? task.getKlingFaceEndTime() : 10000;
int faceDuration = faceEndTime - faceStartTime;
// 严格验证人脸区间：必须至少2秒才能满足"重合2秒"的要求
if (faceDuration < 2000) {
log.error("[buildKlingLipSyncRequest][任务({})人脸区间太短，无法满足API要求][faceDuration={}ms < 2000ms]",
task.getId(), faceDuration);
throw new IllegalArgumentException(String.format(
"人脸区间太短（%dms），至少需要2秒才能生成对口型视频，请选择包含清晰人脸的视频片段",
faceDuration));
}
// ✅ 使用前端解析的真实音频时长（更准确）
Integer soundEndTime = task.getSoundEndTime();
if (soundEndTime == null) {
// 前端必须严格传递 sound_end_time，没有回退
throw new IllegalArgumentException("未收到前端传递的音频时长，请先在页面生成配音");
}
log.info("[buildKlingLipSyncRequest][任务({})使用真实音频时长][soundEndTime={}ms]", task.getId(), soundEndTime);
// 设置音频裁剪参数（从0开始裁剪到soundEndTime）
faceChoose.setSoundStartTime(0);
faceChoose.setSoundEndTime(soundEndTime);
// 计算音频插入时间：确保与人脸区间至少重合2秒
// 重合区间 = min(faceEndTime, soundInsertTime + soundDuration) - max(faceStartTime, soundInsertTime)
// 要求：重合区间 >= 2000ms
// 简化处理：将音频插入时间设置为人脸区间起点
int soundInsertTime = faceStartTime;
// 验证重合区间是否满足要求
int audioDuration = soundEndTime; // 因为soundStartTime=0
int overlapStart = Math.max(faceStartTime, soundInsertTime);
int overlapEnd = Math.min(faceEndTime, soundInsertTime + audioDuration);
int overlapDuration = Math.max(0, overlapEnd - overlapStart);
if (overlapDuration < 2000) {
log.warn("[buildKlingLipSyncRequest][任务({})音频与人脸区间重合不足2秒][重合={}ms < 2000ms]，调整插入时间]",
task.getId(), overlapDuration);
// 如果重合不足，调整插入时间
// 目标：让音频尽可能早地与人脸区间重合
if (audioDuration <= faceDuration) {
// 音频比人脸短，插入到人脸起点
soundInsertTime = faceStartTime;
} else {
// 音频比人脸长，插入到人脸起点即可，重合部分为整个人脸区间
soundInsertTime = faceStartTime;
}
// 重新计算重合区间
overlapStart = Math.max(faceStartTime, soundInsertTime);
overlapEnd = Math.min(faceEndTime, soundInsertTime + audioDuration);
overlapDuration = Math.max(0, overlapEnd - overlapStart);
if (overlapDuration < 2000) {
log.error("[buildKlingLipSyncRequest][任务({})调整后仍不满足重合要求][faceDuration={}ms, audioDuration={}ms, overlap={}ms]",
task.getId(), faceDuration, audioDuration, overlapDuration);
throw new IllegalArgumentException(String.format(
"音频时长（%dms）与视频中人脸区间（%dms）不匹配，重合部分不足2秒，无法生成有效的对口型视频",
audioDuration, faceDuration));
}
}
faceChoose.setSoundInsertTime(soundInsertTime);
faceChoose.setSoundVolume(1.0);
// 完全去除原视频音频
faceChoose.setOriginalAudioVolume(0.0);
request.getFaceChoose().add(faceChoose);
// 记录详细参数用于调试
log.info("[buildKlingLipSyncRequest][任务({})人脸区间] faceStartTime={}, faceEndTime={}, faceDuration={}",
task.getId(), faceStartTime, faceEndTime, faceDuration);
log.info("[buildKlingLipSyncRequest][任务({})音频参数] soundStartTime=0, soundEndTime={}, soundInsertTime={}, textLen={}",
task.getId(), soundEndTime, soundInsertTime,
task.getInputText() != null ? task.getInputText().length() : 0);
log.info("[buildKlingLipSyncRequest][任务({})重合验证] overlapStart={}, overlapEnd={}, overlapDuration={}ms, meetsRequirement={}",
task.getId(), overlapStart, overlapEnd, overlapDuration, overlapDuration >= 2000);
return request;
}
/**

71
yudao-module-tik/src/main/java/cn/iocoder/yudao/module/tik/voice/service/LatentsyncPollingService.java
View File

@@ -17,12 +17,10 @@ import cn.iocoder.yudao.module.tik.voice.dal.dataobject.TikDigitalHumanTaskDO;
import cn.iocoder.yudao.module.tik.voice.dal.mysql.TikDigitalHumanTaskMapper;
import cn.iocoder.yudao.module.tik.voice.vo.AppTikLatentsyncResultRespVO;
import cn.iocoder.yudao.module.tik.kling.service.KlingService;
import cn.iocoder.yudao.module.tik.kling.vo.KlingLipSyncQueryRespVO;
import cn.iocoder.yudao.module.tik.kling.dto.KlingLipSyncQueryResponse;
import cn.iocoder.yudao.module.tik.kling.vo.response.KlingLipSyncVideoVO;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.redisson.api.RLock;
import org.redisson.api.RedissonClient;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
@@ -30,7 +28,6 @@ import org.springframework.transaction.annotation.Transactional;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.List;
import java.util.concurrent.TimeUnit;
/**
* Latentsync任务轮询服务 - 轻量化异步处理
@@ -47,7 +44,6 @@ public class LatentsyncPollingService {
private final TikDigitalHumanTaskMapper taskMapper;
private final LatentsyncService latentsyncService;
private final StringRedisTemplate stringRedisTemplate;
private final RedissonClient redissonClient;
private final TikOssInitService ossInitService;
private final cn.iocoder.yudao.module.infra.api.file.FileApi fileApi;
private final TikUserFileMapper userFileMapper;
@@ -61,7 +57,6 @@ public class LatentsyncPollingService {
private static final String REDIS_POLLING_PREFIX = "latentsync:polling:";
private static final String REDIS_POLLING_TASKS_SET = "latentsync:polling:tasks";
private static final String REDIS_POLLING_COUNT_PREFIX = "latentsync:polling:count:";
private static final String LOCK_KEY = "latentsync:polling:lock";
/**
* 轮询配置
@@ -72,20 +67,14 @@ public class LatentsyncPollingService {
/**
* 定时轮询Latentsync任务状态 - 每10秒执行一次
* 使用分布式锁防止并发执行
* 移除了分布式锁，通过查询条件和限制避免并发问题
* 注意：此方法现在由 DigitalHumanTaskStatusSyncJob 定时调用，不在服务内部使用 @Scheduled 注解
*/
public void pollLatentsyncTasks() {
RLock lock = redissonClient.getLock(LOCK_KEY);
// 尝试加锁（最大等待时间1秒，锁持有时间5秒）
if (lock.tryLock()) {
try {
executePollingTasks();
} catch (Exception ex) {
log.error("[pollLatentsyncTasks][执行异常]", ex);
} finally {
lock.unlock();
}
try {
executePollingTasks();
} catch (Exception ex) {
log.error("[pollLatentsyncTasks][轮询任务异常]", ex);
}
}
@@ -547,13 +536,21 @@ public class LatentsyncPollingService {
*/
private void pollKlingTasks() {
try {
// 查询所有有待轮询的可灵任务（状态为PROCESSING且有klingTaskId）
// 参考混剪任务实现：添加时间和数量限制，避免并发问题
// 1. 时间范围限制：只检查最近6小时内的任务（避免检查历史任务）
// 2. 数量限制：每次最多检查50个任务（避免单次查询过多）
LocalDateTime startTime = LocalDateTime.now().minusHours(6);
// 查询有待轮询的可灵任务（状态为PROCESSING且有klingTaskId，限制时间和数量）
List<TikDigitalHumanTaskDO> klingTasks = taskMapper.selectList(
new cn.iocoder.yudao.framework.mybatis.core.query.LambdaQueryWrapperX<TikDigitalHumanTaskDO>()
.eq(TikDigitalHumanTaskDO::getStatus, "PROCESSING")
.eq(TikDigitalHumanTaskDO::getAiProvider, "kling")
.isNotNull(TikDigitalHumanTaskDO::getKlingTaskId)
.ne(TikDigitalHumanTaskDO::getKlingTaskId, "")
.ge(TikDigitalHumanTaskDO::getCreateTime, startTime) // 只检查最近6小时
.orderByDesc(TikDigitalHumanTaskDO::getCreateTime)
.last("LIMIT 50") // 限制数量，避免并发
);
if (klingTasks.isEmpty()) {
@@ -588,7 +585,7 @@ public class LatentsyncPollingService {
try {
// 查询可灵任务状态
KlingLipSyncQueryRespVO response = klingService.getLipSyncTask(klingTaskId);
KlingLipSyncQueryResponse response = klingService.getLipSyncTask(klingTaskId);
String taskStatus = response.getData().getTaskStatus();
String taskStatusMsg = response.getData().getTaskStatusMsg();
@@ -601,8 +598,40 @@ public class LatentsyncPollingService {
List<KlingLipSyncVideoVO> videos = response.getData().getTaskResult().getVideos();
if (videos != null && !videos.isEmpty()) {
String videoUrl = videos.get(0).getUrl();
updateTaskStatus(task.getId(), "SUCCESS", "finishing", 100, "任务完成", videoUrl);
log.info("[pollKlingSingleTask][任务({})完成][videoUrl={}]", task.getId(), videoUrl);
// 保存视频到OSS（异步处理，轻量化逻辑）
OssSaveResult saveResult = null;
try {
// 保存视频到OSS，避免临时URL过期
saveResult = saveVideoToOss(task, videoUrl);
log.info("[pollKlingSingleTask][任务({})视频已保存到OSS][url={}]", task.getId(), saveResult.getUrl());
} catch (Exception e) {
log.warn("[pollKlingSingleTask][任务({})保存视频失败，使用原URL][error={}]", task.getId(), e.getMessage());
saveResult = new OssSaveResult(videoUrl, 0, null, null); // 降级处理
}
// 更新任务状态为成功
TikDigitalHumanTaskDO updateObj = new TikDigitalHumanTaskDO();
updateObj.setId(task.getId());
updateObj.setStatus("SUCCESS");
updateObj.setCurrentStep("finishing");
updateObj.setProgress(100);
updateObj.setResultVideoUrl(saveResult.getUrl());
updateObj.setFinishTime(LocalDateTime.now());
taskMapper.updateById(updateObj);
// 缓存结果到Redis（快速回显）
try {
String resultKey = "digital_human:task:result:" + task.getId();
stringRedisTemplate.opsForValue().set(resultKey, saveResult.getUrl(), Duration.ofHours(24));
} catch (Exception e) {
log.warn("[pollKlingSingleTask][任务({})缓存结果失败]", task.getId(), e);
}
// 保存结果视频到用户文件表
saveResultVideoToUserFiles(task, saveResult);
log.info("[pollKlingSingleTask][任务({})完成][videoUrl={}]", task.getId(), saveResult.getUrl());
} else {
log.warn("[pollKlingSingleTask][任务({})成功但无视频结果]", task.getId());
}

52
yudao-module-tik/src/main/java/cn/iocoder/yudao/module/tik/voice/strategy/LipSyncStrategy.java
View File

@@ -1,52 +0,0 @@
package cn.iocoder.yudao.module.tik.voice.strategy;
import cn.iocoder.yudao.module.tik.voice.dal.dataobject.TikDigitalHumanTaskDO;
/**
* 口型同步策略接口
*
* 定义不同的AI供应商如何进行口型同步
*
* @author 芋道源码
*/
public interface LipSyncStrategy {
/**
* 执行口型同步
*
* @param task 数字人任务
* @param audioUrl 音频文件URL
* @return 同步后的视频URL（可能与原视频相同，因为是异步处理）
* @throws Exception 同步过程中的异常
*/
String syncLip(TikDigitalHumanTaskDO task, String audioUrl) throws Exception;
/**
* 获取策略名称
*
* @return 策略名称，用于识别不同的AI供应商
*/
String getStrategyName();
/**
* 检查任务参数是否满足此策略的要求
*
* @param task 数字人任务
* @return true 如果任务参数满足策略要求，false 需要回退到其他策略
*/
boolean supports(TikDigitalHumanTaskDO task);
/**
* 获取策略优先级（数值越大优先级越高）
*
* @return 策略优先级
*/
int getPriority();
/**
* 获取策略描述
*
* @return 策略描述，用于日志和文档
*/
String getDescription();
}

118
yudao-module-tik/src/main/java/cn/iocoder/yudao/module/tik/voice/strategy/LipSyncStrategyFactory.java
View File

@@ -1,118 +0,0 @@
package cn.iocoder.yudao.module.tik.voice.strategy;
import cn.iocoder.yudao.module.tik.voice.dal.dataobject.TikDigitalHumanTaskDO;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
/**
* 口型同步策略工厂
*
* 负责创建和管理不同的口型同步策略
*
* @author 芋道源码
*/
@Component
public class LipSyncStrategyFactory {
/**
* 策略注册表
* key: 策略名称
* value: 策略实例
*/
private final Map<String, LipSyncStrategy> strategies = new ConcurrentHashMap<>();
/**
* 注册策略
*
* @param strategy 策略实例
*/
public void registerStrategy(LipSyncStrategy strategy) {
strategies.put(strategy.getStrategyName(), strategy);
}
/**
* 获取策略
*
* @param strategyName 策略名称
* @return 策略实例
*/
public LipSyncStrategy getStrategy(String strategyName) {
return strategies.get(strategyName);
}
/**
* 获取适配的任务策略
*
* 根据任务参数和策略优先级，自动选择最适合的策略
*
* @param task 数字人任务
* @return 适配的策略实例
* @throws IllegalArgumentException 如果没有找到合适的策略
*/
public LipSyncStrategy getStrategyForTask(TikDigitalHumanTaskDO task) {
// 收集所有支持此任务的策略
List<LipSyncStrategy> supportedStrategies = new ArrayList<>();
for (LipSyncStrategy strategy : strategies.values()) {
if (strategy.supports(task)) {
supportedStrategies.add(strategy);
}
}
// 如果没有支持的策略，抛出异常
if (supportedStrategies.isEmpty()) {
throw new IllegalArgumentException("没有找到适合任务(" + task.getId() + ")的策略，AI供应商: " + task.getAiProvider());
}
// 按优先级排序（数值越大优先级越高）
supportedStrategies.sort((s1, s2) -> Integer.compare(s2.getPriority(), s1.getPriority()));
// 返回优先级最高的策略
LipSyncStrategy selectedStrategy = supportedStrategies.get(0);
return selectedStrategy;
}
/**
* 获取所有已注册的策略
*
* @return 策略列表（只读）
*/
public List<LipSyncStrategy> getAllStrategies() {
return Collections.unmodifiableList(new ArrayList<>(strategies.values()));
}
/**
* 检查是否支持指定策略
*
* @param strategyName 策略名称
* @return true 如果支持，false 否则
*/
public boolean supportsStrategy(String strategyName) {
return strategies.containsKey(strategyName);
}
/**
* 获取策略描述信息
*
* @return 所有策略的描述信息
*/
public String getStrategiesDescription() {
StringBuilder sb = new StringBuilder();
sb.append("已注册的策略:\n");
for (LipSyncStrategy strategy : strategies.values()) {
sb.append(String.format("- %s (优先级: %d): %s\n",
strategy.getStrategyName(),
strategy.getPriority(),
strategy.getDescription()));
}
return sb.toString();
}
}

141
yudao-module-tik/src/main/java/cn/iocoder/yudao/module/tik/voice/strategy/impl/KlingLipSyncStrategy.java
View File

@@ -1,141 +0,0 @@
package cn.iocoder.yudao.module.tik.voice.strategy.impl;
import cn.hutool.core.util.StrUtil;
import cn.iocoder.yudao.module.tik.kling.service.KlingService;
import cn.iocoder.yudao.module.tik.kling.vo.KlingLipSyncCreateReqVO;
import cn.iocoder.yudao.module.tik.kling.vo.KlingLipSyncCreateRespVO;
import cn.iocoder.yudao.module.tik.voice.dal.dataobject.TikDigitalHumanTaskDO;
import cn.iocoder.yudao.module.tik.voice.dal.mysql.TikDigitalHumanTaskMapper;
import cn.iocoder.yudao.module.tik.voice.strategy.LipSyncStrategy;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Component;
import java.time.Duration;
import java.util.ArrayList;
/**
* 可灵口型同步策略
*
* 使用可灵 advanced-lip-sync 接口进行口型同步
*
* @author 芋道源码
*/
@Slf4j
@Component
@RequiredArgsConstructor
public class KlingLipSyncStrategy implements LipSyncStrategy {
private final KlingService klingService;
private final TikDigitalHumanTaskMapper taskMapper;
private final StringRedisTemplate stringRedisTemplate;
/**
* Redis键前缀
*/
private static final String REDIS_POLLING_PREFIX = "kling:polling:";
/**
* 缓存过期时间
*/
private static final Duration CACHE_EXPIRE_TIME = Duration.ofHours(1);
@Override
public String syncLip(TikDigitalHumanTaskDO task, String audioUrl) throws Exception {
log.info("[KlingStrategy][任务({})开始使用可灵advanced-lip-sync][klingSessionId={}, klingFaceId={}]",
task.getId(), task.getKlingSessionId(), task.getKlingFaceId());
// 构建可灵口型同步请求VO
KlingLipSyncCreateReqVO reqVO = buildLipSyncRequest(task, audioUrl);
// 调用可灵服务创建任务
KlingLipSyncCreateRespVO response = klingService.createLipSyncTask(reqVO);
String klingTaskId = response.getData().getTaskId();
log.info("[KlingStrategy][任务({})提交成功][klingTaskId={}, status={}]",
task.getId(), klingTaskId, response.getData().getTaskStatus());
// 保存klingTaskId到任务记录
saveKlingTaskId(task.getId(), klingTaskId);
// 将任务加入轮询队列（异步处理）
addToPollingQueue(task.getId(), klingTaskId);
// 返回原视频URL，任务完成后会更新到数据库
log.info("[KlingStrategy][任务({})已加入轮询队列，返回原视频URL]", task.getId());
return task.getVideoUrl();
}
@Override
public String getStrategyName() {
return "kling";
}
@Override
public boolean supports(TikDigitalHumanTaskDO task) {
// 支持条件：
// 1. AI供应商为 kling
// 2. 有 klingSessionId
// 3. 有 klingFaceId
return "kling".equalsIgnoreCase(task.getAiProvider())
&& StrUtil.isNotBlank(task.getKlingSessionId())
&& StrUtil.isNotBlank(task.getKlingFaceId());
}
@Override
public int getPriority() {
// 高优先级，因为这是可灵的专用接口
return 100;
}
@Override
public String getDescription() {
return "可灵高级对口型服务，使用advanced-lip-sync接口进行口型同步";
}
/**
* 构建可灵口型同步请求
*/
private KlingLipSyncCreateReqVO buildLipSyncRequest(TikDigitalHumanTaskDO task, String audioUrl) {
KlingLipSyncCreateReqVO reqVO = new KlingLipSyncCreateReqVO();
reqVO.setSessionId(task.getKlingSessionId());
// 初始化face_choose数组
if (reqVO.getFaceChoose() == null) {
reqVO.setFaceChoose(new ArrayList<>());
}
// 构建face_choose数组
KlingLipSyncCreateReqVO.FaceChooseVO faceChoose = new KlingLipSyncCreateReqVO.FaceChooseVO();
faceChoose.setFaceId(task.getKlingFaceId());
faceChoose.setSoundFile(audioUrl);
faceChoose.setSoundStartTime(0);
faceChoose.setSoundEndTime(0); // 0表示不裁剪
faceChoose.setSoundInsertTime(0);
faceChoose.setSoundVolume(1.0);
faceChoose.setOriginalAudioVolume(1.0);
reqVO.getFaceChoose().add(faceChoose);
return reqVO;
}
/**
* 保存klingTaskId到任务记录
*/
private void saveKlingTaskId(Long taskId, String klingTaskId) {
TikDigitalHumanTaskDO updateObj = new TikDigitalHumanTaskDO();
updateObj.setId(taskId);
updateObj.setKlingTaskId(klingTaskId);
taskMapper.updateById(updateObj);
}
/**
* 添加到轮询队列
*/
private void addToPollingQueue(Long taskId, String klingTaskId) {
String requestIdKey = REDIS_POLLING_PREFIX + "task_" + taskId;
stringRedisTemplate.opsForValue().set(requestIdKey, klingTaskId, CACHE_EXPIRE_TIME);
}
}

121
yudao-module-tik/src/main/java/cn/iocoder/yudao/module/tik/voice/strategy/impl/LatentsyncLipSyncStrategy.java
View File

@@ -1,121 +0,0 @@
package cn.iocoder.yudao.module.tik.voice.strategy.impl;
import cn.hutool.core.util.StrUtil;
import cn.iocoder.yudao.module.tik.voice.service.LatentsyncService;
import cn.iocoder.yudao.module.tik.voice.service.LatentsyncPollingService;
import cn.iocoder.yudao.module.tik.voice.vo.AppTikLatentsyncSubmitReqVO;
import cn.iocoder.yudao.module.tik.voice.vo.AppTikLatentsyncSubmitRespVO;
import cn.iocoder.yudao.module.tik.voice.dal.dataobject.TikDigitalHumanTaskDO;
import cn.iocoder.yudao.module.tik.voice.strategy.LipSyncStrategy;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.springframework.data.redis.core.StringRedisTemplate;
import org.springframework.stereotype.Component;
import java.time.Duration;
/**
* Latentsync口型同步策略
*
* 使用302.ai Latentsync接口进行口型同步
*
* @author 芋道源码
*/
@Slf4j
@Component
@RequiredArgsConstructor
public class LatentsyncLipSyncStrategy implements LipSyncStrategy {
private final LatentsyncService latentsyncService;
private final LatentsyncPollingService latentsyncPollingService;
private final StringRedisTemplate stringRedisTemplate;
/**
* Redis键前缀
*/
private static final String REDIS_POLLING_PREFIX = "latentsync:polling:";
/**
* 缓存过期时间
*/
private static final Duration CACHE_EXPIRE_TIME = Duration.ofHours(1);
@Override
public String syncLip(TikDigitalHumanTaskDO task, String audioUrl) throws Exception {
log.info("[LatentsyncStrategy][任务({})开始使用Latentsync接口][aiProvider={}]",
task.getId(), task.getAiProvider());
// 构建Latentsync请求VO
AppTikLatentsyncSubmitReqVO reqVO = buildLatentsyncRequest(task, audioUrl);
// 调用Latentsync服务提交任务
AppTikLatentsyncSubmitRespVO response = latentsyncService.submitTask(reqVO);
String requestId = response.getRequestId();
log.info("[LatentsyncStrategy][任务({})提交成功][requestId={}]", task.getId(), requestId);
// 将任务加入轮询队列（异步处理）
latentsyncPollingService.addTaskToPollingQueue(task.getId(), requestId);
// 存储requestId与taskId的映射关系（用于轮询服务查找）
saveRequestIdMapping(task.getId(), requestId);
// 返回原视频URL，任务完成后会更新到数据库
log.info("[LatentsyncStrategy][任务({})已加入轮询队列，返回原视频URL]", task.getId());
return task.getVideoUrl();
}
@Override
public String getStrategyName() {
return "latentsync";
}
@Override
public boolean supports(TikDigitalHumanTaskDO task) {
// 支持条件：
// 1. AI供应商为 302ai，或者
// 2. AI供应商为 kling 但缺少可灵特有参数（回退条件）
if ("302ai".equalsIgnoreCase(task.getAiProvider())) {
return true;
}
if ("kling".equalsIgnoreCase(task.getAiProvider())) {
// 如果是可灵供应商，但缺少可灵特有参数，则使用Latentsync作为回退
return StrUtil.isBlank(task.getKlingSessionId())
|| StrUtil.isBlank(task.getKlingFaceId());
}
return false;
}
@Override
public int getPriority() {
// 中等优先级（低优先级也可以，因为KlingStrategy会优先处理可灵任务）
return 50;
}
@Override
public String getDescription() {
return "302.ai Latentsync接口，通用的口型同步服务，支持多种AI供应商";
}
/**
* 构建Latentsync请求
*/
private AppTikLatentsyncSubmitReqVO buildLatentsyncRequest(TikDigitalHumanTaskDO task, String audioUrl) {
AppTikLatentsyncSubmitReqVO reqVO = new AppTikLatentsyncSubmitReqVO();
reqVO.setAudioUrl(audioUrl);
reqVO.setVideoUrl(task.getVideoUrl());
reqVO.setGuidanceScale(task.getGuidanceScale());
reqVO.setSeed(task.getSeed());
return reqVO;
}
/**
* 保存requestId映射
*/
private void saveRequestIdMapping(Long taskId, String requestId) {
String requestIdKey = REDIS_POLLING_PREFIX + "task_" + taskId;
stringRedisTemplate.opsForValue().set(requestIdKey, requestId, CACHE_EXPIRE_TIME);
}
}

30
yudao-module-tik/src/main/java/cn/iocoder/yudao/module/tik/voice/vo/AppTikDigitalHumanCreateReqVO.java
View File

@@ -1,5 +1,6 @@
package cn.iocoder.yudao.module.tik.voice.vo;
import com.fasterxml.jackson.annotation.JsonProperty;
import io.swagger.v3.oas.annotations.media.Schema;
import jakarta.validation.constraints.DecimalMax;
import jakarta.validation.constraints.DecimalMin;
@@ -24,6 +25,7 @@ public class AppTikDigitalHumanCreateReqVO {
private String taskName;
@Schema(description = "AI供应商（默认302ai）", example = "302ai", allowableValues = {"302ai", "aliyun", "openai", "minimax"})
@JsonProperty("ai_provider")
private String aiProvider;
@Schema(description = "视频文件ID（tik_user_file.id）", example = "456")
@@ -70,9 +72,37 @@ public class AppTikDigitalHumanCreateReqVO {
// ========== 可灵特有字段 ==========
@Schema(description = "可灵人脸识别会话ID（可选）", example = "session_xxx")
@JsonProperty("kling_session_id")
private String klingSessionId;
@Schema(description = "可灵选中的人脸ID（可选）", example = "0")
@JsonProperty("kling_face_id")
private String klingFaceId;
@Schema(description = "人脸可对口型区间起点时间（ms）（可选）", example = "0")
@JsonProperty("kling_face_start_time")
private Integer klingFaceStartTime;
@Schema(description = "人脸可对口型区间终点时间（ms）（可选）", example = "10000")
@JsonProperty("kling_face_end_time")
private Integer klingFaceEndTime;
@Schema(description = "可灵音频结束时间（毫秒，前端解析后传递）", example = "5000")
@JsonProperty("sound_end_time")
private Integer soundEndTime;
@Schema(description = "预生成音频信息（前端预生成，用于复用）")
@JsonProperty("pre_generated_audio")
private PreGeneratedAudioVO preGeneratedAudio;
@Data
@Schema(description = "预生成音频信息")
public static class PreGeneratedAudioVO {
@Schema(description = "音频Base64数据", example = "data:audio/mp3;base64,...")
private String audioBase64;
@Schema(description = "音频格式", example = "mp3")
private String format = "mp3";
}
}

3
yudao-module-tik/src/main/java/cn/iocoder/yudao/module/tik/voice/vo/AppTikVoiceTtsRespVO.java
View File

@@ -28,6 +28,9 @@ public class AppTikVoiceTtsRespVO {
@Schema(description = "使用的音色 ID")
private String voiceId;
@Schema(description = "音频时长（毫秒）", example = "5000")
private Integer durationMs;
}