From fea4151fb3f24ceb2779f5a7af0688cd428c5165 Mon Sep 17 00:00:00 2001 From: Gary Date: Sun, 3 Mar 2024 20:10:48 +0800 Subject: [PATCH] 1.add cron job for journal tag update and auto complete regenerate --- .../luoo/music/config/SaticScheduleTask.java | 139 +++++++++++++++ .../com/luoo/music/dao/JournalSongDao.java | 2 + .../com/luoo/music/service/SearchService.java | 160 ------------------ 3 files changed, 141 insertions(+), 160 deletions(-) create mode 100644 luoo_music/src/main/java/com/luoo/music/config/SaticScheduleTask.java diff --git a/luoo_music/src/main/java/com/luoo/music/config/SaticScheduleTask.java b/luoo_music/src/main/java/com/luoo/music/config/SaticScheduleTask.java new file mode 100644 index 0000000..2f1b2ff --- /dev/null +++ b/luoo_music/src/main/java/com/luoo/music/config/SaticScheduleTask.java @@ -0,0 +1,139 @@ +package com.luoo.music.config; + +import java.io.IOException; +import java.io.StringReader; +import java.time.LocalDateTime; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; +import org.apache.lucene.analysis.tokenattributes.TypeAttribute; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.scheduling.annotation.Scheduled; + +import com.luoo.music.dao.JournalDao; +import com.luoo.music.dao.JournalSongDao; +import com.luoo.music.dao.TagDao; +import com.luoo.music.ik.lucene.IKAnalyzer; +import com.luoo.music.pojo.Journal; +import com.luoo.music.util.Constants; + +import lombok.SneakyThrows; +import net.oschina.j2cache.CacheChannel; +import util.StringTools; + +@Configuration // 1.主要用于标记配置类，兼备Component的效果。 +@EnableScheduling // 2.开启定时任务 +public class SaticScheduleTask { + private static final String REDIS_AUTO_COMPLETE = "redis_auto_complete:"; + @Autowired + private JournalDao journalDao; + @Autowired + private JournalSongDao journalSongDao; + @Autowired + private CacheChannel cacheChannel; + @Autowired + private RedisTemplate redisTemplate; + @Autowired + private TagDao tagDao; + + // 1.凌晨4点执行更新期刊tag + @Scheduled(cron="0 0 4 * * ?") + private void updateJouranlTag() { + AtomicLong updateCounter = new AtomicLong(); + journalDao.findValidJournals().parallelStream().forEach(j -> updateJouranlTag(j, updateCounter)); + if (updateCounter.get() > 0) { + cacheChannel.clear(constants.Constants.J2CACHE_REGION_JOURNAL_QUERY_PAGE); + } + } + + // 2.凌晨4:20点执行更新分词 + @Scheduled(cron="0 20 4 * * ?") + private void updateAutoComplete() { + List validJournals = journalDao.findValidJournals(); + validJournals.parallelStream().forEach(j -> { + addIKKeyWord(j.getJournalNo()); + addIKKeyWord(j.getTitle()); + }); + List journalNos = validJournals.stream().map(Journal::getJournalNo).collect(Collectors.toList()); + journalSongDao.findByJournalNoIn(journalNos).parallelStream().forEach(s -> { + addIKKeyWord(s.getName()); + addIKKeyWord(s.getArtist()); + addIKKeyWord(s.getAlbum()); + }); + } + + private void updateJouranlTag(Journal journal, AtomicLong updateCounter) { + List tags = tagDao.getTagNameChByJournalId(journal.getId()); + if (!tags.isEmpty()) { + String tagStr = tags.stream().distinct().sorted().collect(Collectors.joining(Constants.COMMA)); + if (!tagStr.equals(journal.getNameChTags())) { + journal.setNameChTags(tagStr); + journalDao.save(journal); + this.cacheChannel.evict(constants.Constants.J2CACHE_REGION_JOURNAL_ID, journal.getId()); + updateCounter.incrementAndGet(); + } + } + } + + @SneakyThrows + private void addIKKeyWord(String keyword) { + if (!StringTools.isEmpty(keyword)) { + keyword = keyword.trim(); + // 构建IK分词器，使用smart分词模式 + Analyzer analyzer = new IKAnalyzer(true); + // 获取Lucene的TokenStream对象 + TokenStream ts = null; + try { + ts = analyzer.tokenStream("myfield", new StringReader(keyword)); + // 获取词元位置属性 + OffsetAttribute offset = ts.addAttribute(OffsetAttribute.class); + // 获取词元文本属性 + CharTermAttribute term = ts.addAttribute(CharTermAttribute.class); + // 获取词元文本属性 + TypeAttribute type = ts.addAttribute(TypeAttribute.class); + + // 重置TokenStream（重置StringReader） + ts.reset(); + // 迭代获取分词结果 + while (ts.incrementToken()) { + String token = new String(term.toString().getBytes(), "UTF-8"); + + for (int i = 1; i < token.length() + 1; i++) { + String sub = token.substring(0, i); + String encodedString = new String(sub.getBytes(), "UTF-8"); + + redisTemplate.opsForZSet().add(REDIS_AUTO_COMPLETE + encodedString, keyword, 0); + + String encodedStringLowerCase = encodedString.toLowerCase(); + if (!encodedString.equals(encodedStringLowerCase)) { + redisTemplate.opsForZSet().add(REDIS_AUTO_COMPLETE + encodedStringLowerCase, keyword, 0); + } + } + } + // 关闭TokenStream（关闭StringReader） + ts.end(); // Perform end-of-stream operations, e.g. set the final offset. + + } catch (IOException e) { + e.printStackTrace(); + } finally { + // 释放TokenStream的所有资源 + if (ts != null) { + try { + ts.close(); + } catch (IOException e) { + e.printStackTrace(); + } + } + } + } + } +} diff --git a/luoo_music/src/main/java/com/luoo/music/dao/JournalSongDao.java b/luoo_music/src/main/java/com/luoo/music/dao/JournalSongDao.java index 8495d1b..63e13c1 100644 --- a/luoo_music/src/main/java/com/luoo/music/dao/JournalSongDao.java +++ b/luoo_music/src/main/java/com/luoo/music/dao/JournalSongDao.java @@ -51,4 +51,6 @@ public interface JournalSongDao extends JpaRepository, JpaSp @Query(value = "select tbjs.song_id from tb_journal_song as tbjs,tb_journal as tbj where tbjs.journal_no=tbj.journal_no and tbj.id in ?1 order by field(tbj.id,?1), tbjs.song_no", nativeQuery = true) List orderByJournalIdField(List jouranlIds); + + List findByJournalNoIn(List journalNos); } diff --git a/luoo_music/src/main/java/com/luoo/music/service/SearchService.java b/luoo_music/src/main/java/com/luoo/music/service/SearchService.java index 3591b09..5eddc01 100644 --- a/luoo_music/src/main/java/com/luoo/music/service/SearchService.java +++ b/luoo_music/src/main/java/com/luoo/music/service/SearchService.java @@ -1,42 +1,13 @@ package com.luoo.music.service; -import java.io.IOException; -import java.io.StringReader; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.time.LocalDateTime; -import java.time.format.DateTimeFormatter; import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.ListIterator; -import java.util.Map; import java.util.Set; -import java.util.UUID; - import javax.annotation.PostConstruct; -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; -import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; -import org.apache.lucene.analysis.tokenattributes.TypeAttribute; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.redis.connection.RedisZSetCommands.Limit; -import org.springframework.data.redis.connection.RedisZSetCommands.Range; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.stereotype.Service; -import org.springframework.util.StringUtils; - -import com.luoo.music.dao.JournalDao; -import com.luoo.music.dao.JournalSongDao; -import com.luoo.music.dao.SongInfoDao; -import com.luoo.music.ik.lucene.IKAnalyzer; - -import enums.DateTimePatternEnum; -import lombok.SneakyThrows; -import util.StringTools; @Service public class SearchService { @@ -44,148 +15,17 @@ public class SearchService { @Autowired private RedisTemplate redisTemplate; - @Autowired - private JournalDao journalDao; - @Autowired - private JournalSongDao journalSongDao; - - @Autowired - private SongInfoDao songInfoDao; - @PostConstruct private void init() { - //todo: 添加定时任务 填充搜索补全 zset - - - /* - * journalSongDao.findAll().parallelStream().forEach(s->{ String - * name=s.getName(); String artist=s.getArtist(); String album=s.getAlbum(); - * if(isNeedReplace(name)||isNeedReplace(artist)||isNeedReplace(album)) { - * s.setName(name.replaceAll("'", "'").replaceAll("&", "&")); - * s.setArtist(artist.replaceAll("'", "'").replaceAll("&", "&")); - * s.setAlbum(album.replaceAll("'", "'").replaceAll("&", "&")); - * journalSongDao.save(s); } - * - * }); - */ - - /* - * journalDao.findValidJournals().parallelStream().forEach(j->{ - * addIKKeyWord(j.getJournalNo()); addIKKeyWord(j.getTitle()); }); - * - * - * - * journalSongDao.findAll().parallelStream().forEach(s->{ - * - * addIKKeyWord(s.getName()); addIKKeyWord(s.getArtist()); - * addIKKeyWord(s.getAlbum()); }); - * - * - * System.exit(-1); - */ - - /* - * DateTimeFormatter formatter = - * DateTimeFormatter.ofPattern(DateTimePatternEnum.YYYY_MM_DD_HH_MM_SS. - * getPattern()); - * - * Map map=getMap(); journalDao.findAll().forEach(j->{ - * if(map.containsKey(j.getJournalNo())) { - * j.setPubTime(LocalDateTime.parse(map.get(j.getJournalNo()), formatter)); - * journalDao.save(j); } - * - * }); System.exit(-1); - */ - //addKeyWord("中国"); - //searchKey("中"); - //System.exit(-1); - //searchKey("绵"); - //System.exit(-1); - /* - * journalDao.findValidJournals().forEach(j->{ addKeyWord(j.getTitle()); - * addKeyWord(j.getJournalNo()); }); System.exit(-1); - */ - /* - * journalDao.findEnableJournals().parallelStream().forEach(j->{ - * //addKeyWord(j.getTitle()); addKeyWord(j.getJournalNo()); }); - */ - - /* - * journalSongDao.findAll().parallelStream().forEach(s->{ - * addKeyWord(s.getName()); addKeyWord(s.getArtist()); addKeyWord(s.getAlbum()); - * }); System.exit(-1); - */ } - private boolean isNeedReplace(String name) { - return name.contains("'")||name.contains("&"); - } - - - - - public List autoComplete(String query, int limit) { Set values=redisTemplate.opsForZSet().rangeByScore(REDIS_AUTO_COMPLETE+query, 0, Double.MAX_VALUE, 0, limit); return new ArrayList<>(values); } - - @SneakyThrows - private void addIKKeyWord(String keyword) { - if(!StringTools.isEmpty(keyword)) { - keyword=keyword.trim(); - //构建IK分词器，使用smart分词模式 - Analyzer analyzer = new IKAnalyzer(true); - //获取Lucene的TokenStream对象 - TokenStream ts = null; - try { - ts = analyzer.tokenStream("myfield", new StringReader(keyword)); - //获取词元位置属性 - OffsetAttribute offset = ts.addAttribute(OffsetAttribute.class); - //获取词元文本属性 - CharTermAttribute term = ts.addAttribute(CharTermAttribute.class); - //获取词元文本属性 - TypeAttribute type = ts.addAttribute(TypeAttribute.class); - - - //重置TokenStream（重置StringReader） - ts.reset(); - //迭代获取分词结果 - while (ts.incrementToken()) { - String token=new String(term.toString().getBytes(),"UTF-8"); - - for(int i=1;i