From e030bba7aaa5612ace558fffb65fd010e70da993 Mon Sep 17 00:00:00 2001 From: xmzhao71 Date: Fri, 13 Oct 2023 17:01:16 +0800 Subject: [PATCH] =?UTF-8?q?feat=EF=BC=9A=E6=96=B0=E5=A2=9E=E6=A0=B9?= =?UTF-8?q?=E6=8D=AE=E6=96=87=E6=A1=88=E8=8E=B7=E5=8F=96=E8=AF=AD=E7=A7=8D?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../controller/VoiceLanguageController.java | 27 +++++ .../domain/dto/neuver/GetLanguageDTO.java | 12 ++ .../iformall/mapper/VoiceLanguageMapper.java | 5 + .../service/sm/VoiceLanguageService.java | 1 + .../sm/impl/VoiceLanguageServiceImpl.java | 26 +++++ .../resources/mapper/VoiceLanguageMapper.xml | 18 +++ .../com/iformall/constant/LanguageEnums.java | 52 +++++++++ .../com/iformall/language/LanguageDetect.java | 110 ++++++++++-------- .../java/com/iformall/util/DetectUtils.java | 45 +++++++ 9 files changed, 245 insertions(+), 51 deletions(-) create mode 100644 suimangCApi/src/main/java/com/iformall/controller/VoiceLanguageController.java create mode 100644 suimangService/src/main/java/com/iformall/domain/dto/neuver/GetLanguageDTO.java create mode 100644 suimangVideo/src/main/java/com/iformall/constant/LanguageEnums.java create mode 100644 suimangVideo/src/main/java/com/iformall/util/DetectUtils.java diff --git a/suimangCApi/src/main/java/com/iformall/controller/VoiceLanguageController.java b/suimangCApi/src/main/java/com/iformall/controller/VoiceLanguageController.java new file mode 100644 index 0000000..a329d62 --- /dev/null +++ b/suimangCApi/src/main/java/com/iformall/controller/VoiceLanguageController.java @@ -0,0 +1,27 @@ +package com.iformall.controller; + +import com.iformall.annotation.AuthIgnore; +import com.iformall.common.ResultData; +import com.iformall.domain.dto.neuver.GetLanguageDTO; +import com.iformall.service.sm.VoiceLanguageService; +import io.swagger.annotations.ApiOperation; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequestMapping("/api/VoiceLanguage") +public class VoiceLanguageController { + + @Autowired + private VoiceLanguageService voiceLanguageService; + + @AuthIgnore + @ApiOperation("根据文案获取语种") + @PostMapping("getLanguage") + public ResultData getLanguage(@RequestBody GetLanguageDTO dto) { + return new ResultData(voiceLanguageService.getLanguage(dto.getPaperwork())); + } +} diff --git a/suimangService/src/main/java/com/iformall/domain/dto/neuver/GetLanguageDTO.java b/suimangService/src/main/java/com/iformall/domain/dto/neuver/GetLanguageDTO.java new file mode 100644 index 0000000..ecae1cd --- /dev/null +++ b/suimangService/src/main/java/com/iformall/domain/dto/neuver/GetLanguageDTO.java @@ -0,0 +1,12 @@ +package com.iformall.domain.dto.neuver; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +@ApiModel(value = "获取语种") +@Data +public class GetLanguageDTO { + @ApiModelProperty(value = "文案") + private String paperwork; +} diff --git a/suimangService/src/main/java/com/iformall/mapper/VoiceLanguageMapper.java b/suimangService/src/main/java/com/iformall/mapper/VoiceLanguageMapper.java index 1d5e054..e9bb894 100644 --- a/suimangService/src/main/java/com/iformall/mapper/VoiceLanguageMapper.java +++ b/suimangService/src/main/java/com/iformall/mapper/VoiceLanguageMapper.java @@ -6,8 +6,13 @@ import com.iformall.domain.po.sm.VoiceLanguage; import org.apache.ibatis.annotations.Param; import java.util.HashSet; +import java.util.List; public interface VoiceLanguageMapper extends CommonMapper { void saveBatch(@Param("set") HashSet set); + + List listByLocal(@Param("local") String local); + + List listByLanguage(@Param("language") String language); } diff --git a/suimangService/src/main/java/com/iformall/service/sm/VoiceLanguageService.java b/suimangService/src/main/java/com/iformall/service/sm/VoiceLanguageService.java index d16cfb8..302b0c8 100644 --- a/suimangService/src/main/java/com/iformall/service/sm/VoiceLanguageService.java +++ b/suimangService/src/main/java/com/iformall/service/sm/VoiceLanguageService.java @@ -9,4 +9,5 @@ public interface VoiceLanguageService { List voiceTotal(); + VoiceLanguage getLanguage(String paperwork); } diff --git a/suimangService/src/main/java/com/iformall/service/sm/impl/VoiceLanguageServiceImpl.java b/suimangService/src/main/java/com/iformall/service/sm/impl/VoiceLanguageServiceImpl.java index e52501e..19336de 100644 --- a/suimangService/src/main/java/com/iformall/service/sm/impl/VoiceLanguageServiceImpl.java +++ b/suimangService/src/main/java/com/iformall/service/sm/impl/VoiceLanguageServiceImpl.java @@ -3,14 +3,19 @@ package com.iformall.service.sm.impl; import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper; import com.google.common.collect.Lists; +import com.iformall.constant.LanguageEnums; import com.iformall.domain.po.sm.VoiceLanguage; +import com.iformall.language.LanguageDetect; import com.iformall.mapper.VoiceLanguageMapper; import com.iformall.service.sm.VoiceLanguageService; +import com.iformall.util.DetectUtils; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import java.util.List; +import java.util.Optional; @Service public class VoiceLanguageServiceImpl implements VoiceLanguageService { @@ -23,4 +28,25 @@ public class VoiceLanguageServiceImpl implements VoiceLanguageService { List languages = voiceLanguageMapper.selectList(new LambdaQueryWrapper().eq(VoiceLanguage::getIsDel, 0).orderByAsc(VoiceLanguage::getLocal)); return CollectionUtils.isEmpty(languages) ? Lists.newArrayList() : languages; } + + @Override + public VoiceLanguage getLanguage(String paperwork) { + // 文案中若包含指定语种,则规定为该语种 + String detectLanguage = DetectUtils.detectLanguage(paperwork); + detectLanguage = Optional.ofNullable(detectLanguage).orElseGet(() -> LanguageDetect.detect(paperwork)); + detectLanguage = DetectUtils.getLocalLanguage(detectLanguage); + + List voiceLanguages; + // 判断语种是否包含- + String split = ".*-.*"; + if (detectLanguage == null) { + voiceLanguages = voiceLanguageMapper.listByLocal(LanguageEnums.en_US.getLocal()); + } else if (detectLanguage.matches(split)) { + voiceLanguages = voiceLanguageMapper.listByLocal(detectLanguage); + } else { + voiceLanguages = voiceLanguageMapper.listByLanguage(detectLanguage); + } + + return voiceLanguages.get(0); + } } diff --git a/suimangService/src/main/resources/mapper/VoiceLanguageMapper.xml b/suimangService/src/main/resources/mapper/VoiceLanguageMapper.xml index 039a587..9f8c8d0 100644 --- a/suimangService/src/main/resources/mapper/VoiceLanguageMapper.xml +++ b/suimangService/src/main/resources/mapper/VoiceLanguageMapper.xml @@ -18,4 +18,22 @@ ) + + diff --git a/suimangVideo/src/main/java/com/iformall/constant/LanguageEnums.java b/suimangVideo/src/main/java/com/iformall/constant/LanguageEnums.java new file mode 100644 index 0000000..b4320a2 --- /dev/null +++ b/suimangVideo/src/main/java/com/iformall/constant/LanguageEnums.java @@ -0,0 +1,52 @@ +package com.iformall.constant; + + +/** + * 地区语言列表(具体到地区语言) + * 主要用来做默认语言常量 + * + * @author xmzhao71 + * @date 2023-10-13 + */ +public enum LanguageEnums { + en_US("en", "US", "en-US", "英语(美国)"), + zh_CN("zh", "CN", "zh-CN", "中文(普通话,简体)"), + ; + + LanguageEnums(String language, String country, String local, String desc) { + this.language = language; + this.country = country; + this.local = local; + this.desc = desc; + } + + private String language; + private String country; + private String local; + private String desc; + + public String getLanguage() { + return language; + } + + public String getCountry() { + return country; + } + + public String getLocal() { + return local; + } + + public String getDesc() { + return desc; + } + + public static LanguageEnums getEnum(String language) { + for (LanguageEnums value : values()) { + if (value.getLanguage().equals(language)) { + return value; + } + } + return null; + } +} diff --git a/suimangVideo/src/main/java/com/iformall/language/LanguageDetect.java b/suimangVideo/src/main/java/com/iformall/language/LanguageDetect.java index c9f75a2..5d0973c 100644 --- a/suimangVideo/src/main/java/com/iformall/language/LanguageDetect.java +++ b/suimangVideo/src/main/java/com/iformall/language/LanguageDetect.java @@ -1,105 +1,113 @@ package com.iformall.language; -import com.alibaba.fastjson.JSONObject; import com.cybozu.labs.langdetect.Detector; import com.cybozu.labs.langdetect.DetectorFactory; import com.cybozu.labs.langdetect.LangDetectException; -import com.google.common.io.ByteStreams; import lombok.extern.slf4j.Slf4j; -import lombok.var; +import org.apache.commons.io.IOUtils; import org.springframework.core.io.Resource; import org.springframework.core.io.support.PathMatchingResourcePatternResolver; -import org.springframework.util.StreamUtils; -import sun.misc.IOUtils; import java.io.IOException; import java.io.InputStream; -import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; @Slf4j public class LanguageDetect { - private static Detector initDetector(){ + private static Detector initDetector() { try { -// DetectorFactory.loadProfile(Thread.currentThread().getContextClassLoader().getResource("profiles").getPath()); -// return DetectorFactory.create(); List profile = new ArrayList<>(); - try { PathMatchingResourcePatternResolver resolver = new PathMatchingResourcePatternResolver(); Resource[] resources = resolver.getResources("classpath:profiles/*"); - for (Resource resource:resources) { - InputStream inputStream = resource.getInputStream(); -// String str = StreamUtils.copyToString(inputStream, StandardCharsets.UTF_8); - String str = new String(ByteStreams.toByteArray(inputStream)); - profile.add(str); -// InputStream inputStream = resource.getInputStream(); -// byte[] bytes = new byte[(int) resource.contentLength()]; -// inputStream.read(bytes); -// profile.add(new String(bytes)); + for (Resource resource : resources) { + InputStream inputStream = resource.getInputStream(); + String str = new String(IOUtils.toByteArray(inputStream)); + profile.add(str); } } catch (IOException e) { e.printStackTrace(); } - if(!profile.isEmpty()){ + if (!profile.isEmpty()) { DetectorFactory.clear(); DetectorFactory.loadProfile(profile); return DetectorFactory.create(); } } catch (LangDetectException e) { e.printStackTrace(); - log.error("语言检测初始化错误。"+e.getMessage()); + log.error("语言检测初始化错误。" + e.getMessage()); } return null; } - public static String detect(String str){ + public static String detect(String str) { Detector detector = initDetector(); - if(detector == null){ + if (detector == null) { return null; } detector.append(str); -// try { -// detect.getProbabilities(); -// } catch (LangDetectException e) { -// e.printStackTrace(); -// } - try { String language = detector.detect(); log.info("语种识别(" + str + ")-----" + language); return language; } catch (LangDetectException e) { - e.printStackTrace(); - log.error("语言检测错误。"); + log.error("语言检测错误", e); } return null; } public static void main(String[] args) { -// var resolver = new PathMatchingResourcePatternResolver(); -// Resource[] resources = new Resource[0]; -// try { -// resources = resolver.getResources("classpath:profiles/*"); -// } catch (IOException e) { -// e.printStackTrace(); -// } -// for (var resource:resources) { -// System.out.println(resource.getFilename()); -// try { -// byte[] bytes = new byte[0]; -// bytes = new byte[resource.getInputStream().available()]; -// resource.getInputStream().read(bytes); -// String str = new String(bytes); -// System.out.println(str); -// } catch (IOException e) { -// e.printStackTrace(); -// } -// -// } + PathMatchingResourcePatternResolver resolver = new PathMatchingResourcePatternResolver(); + Resource[] resources = new Resource[0]; + try { + resources = resolver.getResources("classpath:profiles/*"); + } catch (IOException e) { + e.printStackTrace(); + } + for (Resource resource : resources) { + System.out.println(resource.getFilename()); + try { + byte[] bytes = new byte[0]; + bytes = new byte[resource.getInputStream().available()]; + resource.getInputStream().read(bytes); + String str = new String(bytes); + System.out.println(str); + } catch (IOException e) { + e.printStackTrace(); + } + + } System.out.println(detect("科技公司")); + System.out.println(detect("科技公司有哪几家做的还不错呢,其中就有我们这一家")); + System.out.println(detect("中国科技还行吧")); + System.out.println(detect("中国")); + System.out.println(detect("梦想")); + System.out.println(detect("I am a teacher")); + System.out.println(detect("I am a 老师")); + System.out.println(detect("行人從非機動車道變換車道到機動車道,激動的叫起來了")); + System.out.println(detect("おはよう")); + + System.out.println("======================================="); + String regex = ".*[\u4e00-\u9fa5]+.*"; +// String regex2 = "\b.*[a-zA-Z]+.*\b"; + + String regex3 = ".*-.*"; + System.out.println("I am a 老师".matches(regex)); + System.out.println("I am a teacher".matches(regex)); + System.out.println("I am a 我們".matches(regex)); + System.out.println("行人從非機動車道變換車道到機動車道,激動的叫起來了".matches(regex)); + + System.out.println("======================================="); +// System.out.println("I am a 老师".matches(regex2)); +// System.out.println("I am a teacher".matches(regex2)); +// System.out.println("abcdefg".matches(regex2)); +// System.out.println("abc d e fg".matches(regex2)); + + System.out.println("======================================="); + System.out.println("abc".matches(regex3)); + System.out.println("a-bc".matches(regex3)); } } diff --git a/suimangVideo/src/main/java/com/iformall/util/DetectUtils.java b/suimangVideo/src/main/java/com/iformall/util/DetectUtils.java new file mode 100644 index 0000000..e3a11e6 --- /dev/null +++ b/suimangVideo/src/main/java/com/iformall/util/DetectUtils.java @@ -0,0 +1,45 @@ +package com.iformall.util; + +import com.iformall.constant.LanguageEnums; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * 语种检测类 + * + * @author xmzhao71 + * @date 2023-10-13 + */ +public class DetectUtils { + + /** + * 检测语言是否包含指定语种 + * + * @param text 文案 + * @return {@link String} 指定地区语种 + */ + public static String detectLanguage(String text) { + String zhCNRegex = ".*[\u4e00-\u9fa5]+.*"; + if (text.matches(zhCNRegex)) { + return LanguageEnums.zh_CN.getLocal(); + } + return null; + } + + /** + * 获取地区语言 + * + * @param detectLanguage 检测语言 + * @return {@link String} 指定地区语言 + */ + public static String getLocalLanguage(String detectLanguage) { + if (detectLanguage == null) { + return null; + } + if (detectLanguage.equalsIgnoreCase(LanguageEnums.en_US.getLanguage())) { + return LanguageEnums.en_US.getLocal(); + } + return detectLanguage; + } +} \ No newline at end of file