Przeglądaj źródła

feat:新增根据文案获取语种接口

private_deployment
xmzhao71 1 rok temu
rodzic
commit
e030bba7aa
9 zmienionych plików z 245 dodań i 51 usunięć
  1. +27
    -0
      suimangCApi/src/main/java/com/iformall/controller/VoiceLanguageController.java
  2. +12
    -0
      suimangService/src/main/java/com/iformall/domain/dto/neuver/GetLanguageDTO.java
  3. +5
    -0
      suimangService/src/main/java/com/iformall/mapper/VoiceLanguageMapper.java
  4. +1
    -0
      suimangService/src/main/java/com/iformall/service/sm/VoiceLanguageService.java
  5. +26
    -0
      suimangService/src/main/java/com/iformall/service/sm/impl/VoiceLanguageServiceImpl.java
  6. +18
    -0
      suimangService/src/main/resources/mapper/VoiceLanguageMapper.xml
  7. +52
    -0
      suimangVideo/src/main/java/com/iformall/constant/LanguageEnums.java
  8. +59
    -51
      suimangVideo/src/main/java/com/iformall/language/LanguageDetect.java
  9. +45
    -0
      suimangVideo/src/main/java/com/iformall/util/DetectUtils.java

+ 27
- 0
suimangCApi/src/main/java/com/iformall/controller/VoiceLanguageController.java Wyświetl plik

@@ -0,0 +1,27 @@
package com.iformall.controller;

import com.iformall.annotation.AuthIgnore;
import com.iformall.common.ResultData;
import com.iformall.domain.dto.neuver.GetLanguageDTO;
import com.iformall.service.sm.VoiceLanguageService;
import io.swagger.annotations.ApiOperation;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;

@RestController
@RequestMapping("/api/VoiceLanguage")
public class VoiceLanguageController {

@Autowired
private VoiceLanguageService voiceLanguageService;

@AuthIgnore
@ApiOperation("根据文案获取语种")
@PostMapping("getLanguage")
public ResultData getLanguage(@RequestBody GetLanguageDTO dto) {
return new ResultData(voiceLanguageService.getLanguage(dto.getPaperwork()));
}
}

+ 12
- 0
suimangService/src/main/java/com/iformall/domain/dto/neuver/GetLanguageDTO.java Wyświetl plik

@@ -0,0 +1,12 @@
package com.iformall.domain.dto.neuver;

import io.swagger.annotations.ApiModel;
import io.swagger.annotations.ApiModelProperty;
import lombok.Data;

@ApiModel(value = "获取语种")
@Data
public class GetLanguageDTO {
@ApiModelProperty(value = "文案")
private String paperwork;
}

+ 5
- 0
suimangService/src/main/java/com/iformall/mapper/VoiceLanguageMapper.java Wyświetl plik

@@ -6,8 +6,13 @@ import com.iformall.domain.po.sm.VoiceLanguage;
import org.apache.ibatis.annotations.Param;

import java.util.HashSet;
import java.util.List;

public interface VoiceLanguageMapper extends CommonMapper<VoiceLanguage, Long> {
void saveBatch(@Param("set") HashSet<VoiceLanguage> set);

List<VoiceLanguage> listByLocal(@Param("local") String local);

List<VoiceLanguage> listByLanguage(@Param("language") String language);
}


+ 1
- 0
suimangService/src/main/java/com/iformall/service/sm/VoiceLanguageService.java Wyświetl plik

@@ -9,4 +9,5 @@ public interface VoiceLanguageService {

List<VoiceLanguage> voiceTotal();

VoiceLanguage getLanguage(String paperwork);
}

+ 26
- 0
suimangService/src/main/java/com/iformall/service/sm/impl/VoiceLanguageServiceImpl.java Wyświetl plik

@@ -3,14 +3,19 @@ package com.iformall.service.sm.impl;

import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.google.common.collect.Lists;
import com.iformall.constant.LanguageEnums;
import com.iformall.domain.po.sm.VoiceLanguage;
import com.iformall.language.LanguageDetect;
import com.iformall.mapper.VoiceLanguageMapper;
import com.iformall.service.sm.VoiceLanguageService;
import com.iformall.util.DetectUtils;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import java.util.List;
import java.util.Optional;

@Service
public class VoiceLanguageServiceImpl implements VoiceLanguageService {
@@ -23,4 +28,25 @@ public class VoiceLanguageServiceImpl implements VoiceLanguageService {
List<VoiceLanguage> languages = voiceLanguageMapper.selectList(new LambdaQueryWrapper<VoiceLanguage>().eq(VoiceLanguage::getIsDel, 0).orderByAsc(VoiceLanguage::getLocal));
return CollectionUtils.isEmpty(languages) ? Lists.newArrayList() : languages;
}

@Override
public VoiceLanguage getLanguage(String paperwork) {
// 文案中若包含指定语种,则规定为该语种
String detectLanguage = DetectUtils.detectLanguage(paperwork);
detectLanguage = Optional.ofNullable(detectLanguage).orElseGet(() -> LanguageDetect.detect(paperwork));
detectLanguage = DetectUtils.getLocalLanguage(detectLanguage);

List<VoiceLanguage> voiceLanguages;
// 判断语种是否包含-
String split = ".*-.*";
if (detectLanguage == null) {
voiceLanguages = voiceLanguageMapper.listByLocal(LanguageEnums.en_US.getLocal());
} else if (detectLanguage.matches(split)) {
voiceLanguages = voiceLanguageMapper.listByLocal(detectLanguage);
} else {
voiceLanguages = voiceLanguageMapper.listByLanguage(detectLanguage);
}

return voiceLanguages.get(0);
}
}

+ 18
- 0
suimangService/src/main/resources/mapper/VoiceLanguageMapper.xml Wyświetl plik

@@ -18,4 +18,22 @@
)
</foreach>
</insert>
<select id="listByLocal" resultType="com.iformall.domain.po.sm.VoiceLanguage">
SELECT
*
FROM
voice_language
WHERE
is_del = 0 AND
LOWER( `local` ) = #{local}
</select>
<select id="listByLanguage" resultType="com.iformall.domain.po.sm.VoiceLanguage">
SELECT
*
FROM
voice_language
WHERE
is_del = 0 AND
LOWER( `language` ) = #{language}
</select>
</mapper>

+ 52
- 0
suimangVideo/src/main/java/com/iformall/constant/LanguageEnums.java Wyświetl plik

@@ -0,0 +1,52 @@
package com.iformall.constant;


/**
* 地区语言列表(具体到地区语言)
* 主要用来做默认语言常量
*
* @author xmzhao71
* @date 2023-10-13
*/
public enum LanguageEnums {
en_US("en", "US", "en-US", "英语(美国)"),
zh_CN("zh", "CN", "zh-CN", "中文(普通话,简体)"),
;

LanguageEnums(String language, String country, String local, String desc) {
this.language = language;
this.country = country;
this.local = local;
this.desc = desc;
}

private String language;
private String country;
private String local;
private String desc;

public String getLanguage() {
return language;
}

public String getCountry() {
return country;
}

public String getLocal() {
return local;
}

public String getDesc() {
return desc;
}

public static LanguageEnums getEnum(String language) {
for (LanguageEnums value : values()) {
if (value.getLanguage().equals(language)) {
return value;
}
}
return null;
}
}

+ 59
- 51
suimangVideo/src/main/java/com/iformall/language/LanguageDetect.java Wyświetl plik

@@ -1,105 +1,113 @@
package com.iformall.language;

import com.alibaba.fastjson.JSONObject;
import com.cybozu.labs.langdetect.Detector;
import com.cybozu.labs.langdetect.DetectorFactory;
import com.cybozu.labs.langdetect.LangDetectException;
import com.google.common.io.ByteStreams;
import lombok.extern.slf4j.Slf4j;
import lombok.var;
import org.apache.commons.io.IOUtils;
import org.springframework.core.io.Resource;
import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
import org.springframework.util.StreamUtils;
import sun.misc.IOUtils;

import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;

@Slf4j
public class LanguageDetect {

private static Detector initDetector(){
private static Detector initDetector() {
try {
// DetectorFactory.loadProfile(Thread.currentThread().getContextClassLoader().getResource("profiles").getPath());
// return DetectorFactory.create();
List<String> profile = new ArrayList<>();

try {
PathMatchingResourcePatternResolver resolver = new PathMatchingResourcePatternResolver();
Resource[] resources = resolver.getResources("classpath:profiles/*");
for (Resource resource:resources) {
InputStream inputStream = resource.getInputStream();
// String str = StreamUtils.copyToString(inputStream, StandardCharsets.UTF_8);
String str = new String(ByteStreams.toByteArray(inputStream));
profile.add(str);
// InputStream inputStream = resource.getInputStream();
// byte[] bytes = new byte[(int) resource.contentLength()];
// inputStream.read(bytes);
// profile.add(new String(bytes));
for (Resource resource : resources) {
InputStream inputStream = resource.getInputStream();
String str = new String(IOUtils.toByteArray(inputStream));
profile.add(str);
}
} catch (IOException e) {
e.printStackTrace();
}
if(!profile.isEmpty()){
if (!profile.isEmpty()) {
DetectorFactory.clear();
DetectorFactory.loadProfile(profile);
return DetectorFactory.create();
}
} catch (LangDetectException e) {
e.printStackTrace();
log.error("语言检测初始化错误。"+e.getMessage());
log.error("语言检测初始化错误。" + e.getMessage());
}
return null;
}

public static String detect(String str){
public static String detect(String str) {
Detector detector = initDetector();
if(detector == null){
if (detector == null) {
return null;
}
detector.append(str);
// try {
// detect.getProbabilities();
// } catch (LangDetectException e) {
// e.printStackTrace();
// }

try {
String language = detector.detect();
log.info("语种识别(" + str + ")-----" + language);
return language;
} catch (LangDetectException e) {
e.printStackTrace();
log.error("语言检测错误。");
log.error("语言检测错误", e);
}
return null;
}


public static void main(String[] args) {
// var resolver = new PathMatchingResourcePatternResolver();
// Resource[] resources = new Resource[0];
// try {
// resources = resolver.getResources("classpath:profiles/*");
// } catch (IOException e) {
// e.printStackTrace();
// }
// for (var resource:resources) {
// System.out.println(resource.getFilename());
// try {
// byte[] bytes = new byte[0];
// bytes = new byte[resource.getInputStream().available()];
// resource.getInputStream().read(bytes);
// String str = new String(bytes);
// System.out.println(str);
// } catch (IOException e) {
// e.printStackTrace();
// }
//
// }
PathMatchingResourcePatternResolver resolver = new PathMatchingResourcePatternResolver();
Resource[] resources = new Resource[0];
try {
resources = resolver.getResources("classpath:profiles/*");
} catch (IOException e) {
e.printStackTrace();
}
for (Resource resource : resources) {
System.out.println(resource.getFilename());
try {
byte[] bytes = new byte[0];
bytes = new byte[resource.getInputStream().available()];
resource.getInputStream().read(bytes);
String str = new String(bytes);
System.out.println(str);
} catch (IOException e) {
e.printStackTrace();
}
}
System.out.println(detect("科技公司"));
System.out.println(detect("科技公司有哪几家做的还不错呢,其中就有我们这一家"));
System.out.println(detect("中国科技还行吧"));
System.out.println(detect("中国"));
System.out.println(detect("梦想"));
System.out.println(detect("I am a teacher"));
System.out.println(detect("I am a 老师"));
System.out.println(detect("行人從非機動車道變換車道到機動車道,激動的叫起來了"));
System.out.println(detect("おはよう"));

System.out.println("=======================================");
String regex = ".*[\u4e00-\u9fa5]+.*";
// String regex2 = "\b.*[a-zA-Z]+.*\b";

String regex3 = ".*-.*";
System.out.println("I am a 老师".matches(regex));
System.out.println("I am a teacher".matches(regex));
System.out.println("I am a 我們".matches(regex));
System.out.println("行人從非機動車道變換車道到機動車道,激動的叫起來了".matches(regex));

System.out.println("=======================================");
// System.out.println("I am a 老师".matches(regex2));
// System.out.println("I am a teacher".matches(regex2));
// System.out.println("abcdefg".matches(regex2));
// System.out.println("abc d e fg".matches(regex2));

System.out.println("=======================================");
System.out.println("abc".matches(regex3));
System.out.println("a-bc".matches(regex3));
}
}

+ 45
- 0
suimangVideo/src/main/java/com/iformall/util/DetectUtils.java Wyświetl plik

@@ -0,0 +1,45 @@
package com.iformall.util;

import com.iformall.constant.LanguageEnums;

import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
* 语种检测类
*
* @author xmzhao71
* @date 2023-10-13
*/
public class DetectUtils {

/**
* 检测语言是否包含指定语种
*
* @param text 文案
* @return {@link String} 指定地区语种
*/
public static String detectLanguage(String text) {
String zhCNRegex = ".*[\u4e00-\u9fa5]+.*";
if (text.matches(zhCNRegex)) {
return LanguageEnums.zh_CN.getLocal();
}
return null;
}

/**
* 获取地区语言
*
* @param detectLanguage 检测语言
* @return {@link String} 指定地区语言
*/
public static String getLocalLanguage(String detectLanguage) {
if (detectLanguage == null) {
return null;
}
if (detectLanguage.equalsIgnoreCase(LanguageEnums.en_US.getLanguage())) {
return LanguageEnums.en_US.getLocal();
}
return detectLanguage;
}
}

Ładowanie…
Anuluj
Zapisz