queyue-backend
/
luoo_parent
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

1.add cron job for journal tag update and auto complete regenerate

Browse Source
main
Gary 9 months ago
parent 271fe44881
commit fea4151fb3
3 changed files with 141 additions and 160 deletions
Show Stats Download Patch File Download Diff File

139
luoo_music/src/main/java/com/luoo/music/config/SaticScheduleTask.java
Unescape View File

@ -0,0 +1,139 @@
package com.luoo.music.config;
import java.io.IOException;
import java.io.StringReader;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Optional;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import com.luoo.music.dao.JournalDao;
import com.luoo.music.dao.JournalSongDao;
import com.luoo.music.dao.TagDao;
import com.luoo.music.ik.lucene.IKAnalyzer;
import com.luoo.music.pojo.Journal;
import com.luoo.music.util.Constants;
import lombok.SneakyThrows;
import net.oschina.j2cache.CacheChannel;
import util.StringTools;
@Configuration // 1.主要用于标记配置类，兼备Component的效果。
@EnableScheduling // 2.开启定时任务
public class SaticScheduleTask {
private static final String REDIS_AUTO_COMPLETE = "redis_auto_complete:";
@Autowired
private JournalDao journalDao;
@Autowired
private JournalSongDao journalSongDao;
@Autowired
private CacheChannel cacheChannel;
@Autowired
private RedisTemplate redisTemplate;
@Autowired
private TagDao tagDao;
// 1.凌晨4点执行更新期刊tag
@Scheduled(cron="0 0 4 * * ?")
private void updateJouranlTag() {
AtomicLong updateCounter = new AtomicLong();
journalDao.findValidJournals().parallelStream().forEach(j -> updateJouranlTag(j, updateCounter));
if (updateCounter.get() > 0) {
cacheChannel.clear(constants.Constants.J2CACHE_REGION_JOURNAL_QUERY_PAGE);
}
}
// 2.凌晨4:20点执行更新分词
@Scheduled(cron="0 20 4 * * ?")
private void updateAutoComplete() {
List<Journal> validJournals = journalDao.findValidJournals();
validJournals.parallelStream().forEach(j -> {
addIKKeyWord(j.getJournalNo());
addIKKeyWord(j.getTitle());
});
List<String> journalNos = validJournals.stream().map(Journal::getJournalNo).collect(Collectors.toList());
journalSongDao.findByJournalNoIn(journalNos).parallelStream().forEach(s -> {
addIKKeyWord(s.getName());
addIKKeyWord(s.getArtist());
addIKKeyWord(s.getAlbum());
});
}
private void updateJouranlTag(Journal journal, AtomicLong updateCounter) {
List<String> tags = tagDao.getTagNameChByJournalId(journal.getId());
if (!tags.isEmpty()) {
String tagStr = tags.stream().distinct().sorted().collect(Collectors.joining(Constants.COMMA));
if (!tagStr.equals(journal.getNameChTags())) {
journal.setNameChTags(tagStr);
journalDao.save(journal);
this.cacheChannel.evict(constants.Constants.J2CACHE_REGION_JOURNAL_ID, journal.getId());
updateCounter.incrementAndGet();
}
}
}
@SneakyThrows
private void addIKKeyWord(String keyword) {
if (!StringTools.isEmpty(keyword)) {
keyword = keyword.trim();
// 构建IK分词器，使用smart分词模式
Analyzer analyzer = new IKAnalyzer(true);
// 获取Lucene的TokenStream对象
TokenStream ts = null;
try {
ts = analyzer.tokenStream("myfield", new StringReader(keyword));
// 获取词元位置属性
OffsetAttribute offset = ts.addAttribute(OffsetAttribute.class);
// 获取词元文本属性
CharTermAttribute term = ts.addAttribute(CharTermAttribute.class);
// 获取词元文本属性
TypeAttribute type = ts.addAttribute(TypeAttribute.class);
// 重置TokenStream（重置StringReader）
ts.reset();
// 迭代获取分词结果
while (ts.incrementToken()) {
String token = new String(term.toString().getBytes(), "UTF-8");
for (int i = 1; i < token.length() + 1; i++) {
String sub = token.substring(0, i);
String encodedString = new String(sub.getBytes(), "UTF-8");
redisTemplate.opsForZSet().add(REDIS_AUTO_COMPLETE + encodedString, keyword, 0);
String encodedStringLowerCase = encodedString.toLowerCase();
if (!encodedString.equals(encodedStringLowerCase)) {
redisTemplate.opsForZSet().add(REDIS_AUTO_COMPLETE + encodedStringLowerCase, keyword, 0);
}
}
}
// 关闭TokenStream（关闭StringReader）
ts.end(); // Perform end-of-stream operations, e.g. set the final offset.
} catch (IOException e) {
e.printStackTrace();
} finally {
// 释放TokenStream的所有资源
if (ts != null) {
try {
ts.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
}
}

2
luoo_music/src/main/java/com/luoo/music/dao/JournalSongDao.java
Unescape View File

@ -51,4 +51,6 @@ public interface JournalSongDao extends JpaRepository<JournalSong,String>, JpaSp
@Query(value = "select tbjs.song_id from tb_journal_song as tbjs,tb_journal as tbj where tbjs.journal_no=tbj.journal_no and tbj.id in ?1 order by field(tbj.id,?1), tbjs.song_no", nativeQuery = true) @Query(value = "select tbjs.song_id from tb_journal_song as tbjs,tb_journal as tbj where tbjs.journal_no=tbj.journal_no and tbj.id in ?1 order by field(tbj.id,?1), tbjs.song_no", nativeQuery = true)
List<String> orderByJournalIdField(List<String> jouranlIds); List<String> orderByJournalIdField(List<String> jouranlIds);
List<JournalSong> findByJournalNoIn(List<String> journalNos);
} }

160
luoo_music/src/main/java/com/luoo/music/service/SearchService.java
Unescape View File

@ -1,42 +1,13 @@
package com.luoo.music.service; package com.luoo.music.service;
import java.io.IOException;
import java.io.StringReader;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.ListIterator;
import java.util.Map;
import java.util.Set; import java.util.Set;
import java.util.UUID;
import javax.annotation.PostConstruct; import javax.annotation.PostConstruct;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.connection.RedisZSetCommands.Limit;
import org.springframework.data.redis.connection.RedisZSetCommands.Range;
import org.springframework.data.redis.core.RedisTemplate; import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.util.StringUtils;
import com.luoo.music.dao.JournalDao;
import com.luoo.music.dao.JournalSongDao;
import com.luoo.music.dao.SongInfoDao;
import com.luoo.music.ik.lucene.IKAnalyzer;
import enums.DateTimePatternEnum;
import lombok.SneakyThrows;
import util.StringTools;
@Service @Service
public class SearchService { public class SearchService {
@ -44,86 +15,9 @@ public class SearchService {
@Autowired @Autowired
private RedisTemplate redisTemplate; private RedisTemplate redisTemplate;
@Autowired
private JournalDao journalDao;
@Autowired
private JournalSongDao journalSongDao;
@Autowired
private SongInfoDao songInfoDao;
@PostConstruct @PostConstruct
private void init() { private void init() {
//todo: 添加定时任务 填充搜索补全 zset
/*
* journalSongDao.findAll().parallelStream().forEach(s->{ String
* name=s.getName(); String artist=s.getArtist(); String album=s.getAlbum();
* if(isNeedReplace(name)||isNeedReplace(artist)||isNeedReplace(album)) {
* s.setName(name.replaceAll("&#39;", "'").replaceAll("&amp;", "&"));
* s.setArtist(artist.replaceAll("&#39;", "'").replaceAll("&amp;", "&"));
* s.setAlbum(album.replaceAll("&#39;", "'").replaceAll("&amp;", "&"));
* journalSongDao.save(s); }
*
* });
*/
/*
* journalDao.findValidJournals().parallelStream().forEach(j->{
* addIKKeyWord(j.getJournalNo()); addIKKeyWord(j.getTitle()); });
*
*
*
* journalSongDao.findAll().parallelStream().forEach(s->{
*
* addIKKeyWord(s.getName()); addIKKeyWord(s.getArtist());
* addIKKeyWord(s.getAlbum()); });
*
*
* System.exit(-1);
*/
/*
* DateTimeFormatter formatter =
* DateTimeFormatter.ofPattern(DateTimePatternEnum.YYYY_MM_DD_HH_MM_SS.
* getPattern());
*
* Map<String,String> map=getMap(); journalDao.findAll().forEach(j->{
* if(map.containsKey(j.getJournalNo())) {
* j.setPubTime(LocalDateTime.parse(map.get(j.getJournalNo()), formatter));
* journalDao.save(j); }
*
* }); System.exit(-1);
*/
//addKeyWord("中国");
//searchKey("中");
//System.exit(-1);
//searchKey("绵");
//System.exit(-1);
/*
* journalDao.findValidJournals().forEach(j->{ addKeyWord(j.getTitle());
* addKeyWord(j.getJournalNo()); }); System.exit(-1);
*/
/*
* journalDao.findEnableJournals().parallelStream().forEach(j->{
* //addKeyWord(j.getTitle()); addKeyWord(j.getJournalNo()); });
*/
/*
* journalSongDao.findAll().parallelStream().forEach(s->{
* addKeyWord(s.getName()); addKeyWord(s.getArtist()); addKeyWord(s.getAlbum());
* }); System.exit(-1);
*/
}
private boolean isNeedReplace(String name) {
return name.contains("&#39;")||name.contains("&amp;");
} }
@ -134,58 +28,4 @@ public class SearchService {
Set<String> values=redisTemplate.opsForZSet().rangeByScore(REDIS_AUTO_COMPLETE+query, 0, Double.MAX_VALUE, 0, limit); Set<String> values=redisTemplate.opsForZSet().rangeByScore(REDIS_AUTO_COMPLETE+query, 0, Double.MAX_VALUE, 0, limit);
return new ArrayList<>(values); return new ArrayList<>(values);
} }
@SneakyThrows
private void addIKKeyWord(String keyword) {
if(!StringTools.isEmpty(keyword)) {
keyword=keyword.trim();
//构建IK分词器，使用smart分词模式
Analyzer analyzer = new IKAnalyzer(true);
//获取Lucene的TokenStream对象
TokenStream ts = null;
try {
ts = analyzer.tokenStream("myfield", new StringReader(keyword));
//获取词元位置属性
OffsetAttribute offset = ts.addAttribute(OffsetAttribute.class);
//获取词元文本属性
CharTermAttribute term = ts.addAttribute(CharTermAttribute.class);
//获取词元文本属性
TypeAttribute type = ts.addAttribute(TypeAttribute.class);
//重置TokenStream（重置StringReader）
ts.reset();
//迭代获取分词结果
while (ts.incrementToken()) {
String token=new String(term.toString().getBytes(),"UTF-8");
for(int i=1;i<token.length()+1;i++) {
String sub=token.substring(0, i);
String encodedString=new String(sub.getBytes(),"UTF-8");
redisTemplate.opsForZSet().add(REDIS_AUTO_COMPLETE+encodedString,keyword,0);
String encodedStringLowerCase=encodedString.toLowerCase();
if(!encodedString.equals(encodedStringLowerCase)) {
redisTemplate.opsForZSet().add(REDIS_AUTO_COMPLETE+encodedStringLowerCase,keyword,0);
}
}
}
//关闭TokenStream（关闭StringReader）
ts.end(); // Perform end-of-stream operations, e.g. set the final offset.
} catch (IOException e) {
e.printStackTrace();
} finally {
//释放TokenStream的所有资源
if(ts != null){
try {
ts.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
}
} }

Write Preview
Loading…
Cancel
Save