queyue-backend
/
luoo_parent
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

1.change auto complete from zset to sorted list

Browse Source
main
Gary 11 months ago
parent b4cb6d72a4
commit a10a82a083
4 changed files with 92 additions and 48 deletions
Show Stats Download Patch File Download Diff File

1
luoo_common/src/main/java/constants/Constants.java
Unescape View File

@ -21,6 +21,7 @@ public class Constants {
public static final String J2CACHE_REGION_JOURNAL_COMMENT_PAGE_NEW = "journal_comment_page_new";
public static final String J2CACHE_REGION_JOURNAL_COMMENT_PAGE_CHILDREN = "journal_comment_page_children";
public static final String J2CACHE_REGION_SEARCH_AUTO_COMPLETE = "search_auto_complete";
public static final String RABBIT_MESSAGE_CLEAN_JOURANL_QUERY_PAGE = "clean_journal_query_page";

110
luoo_music/src/main/java/com/luoo/music/config/SaticScheduleTask.java
Unescape View File

@ -2,28 +2,31 @@ package com.luoo.music.config;
import java.io.IOException;
import java.io.StringReader;
import java.time.LocalDateTime;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Optional;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import javax.annotation.PostConstruct;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Configuration;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import com.luoo.music.dao.JournalDao;
import com.luoo.music.dao.JournalSongDao;
import com.luoo.music.dao.TagDao;
import com.luoo.music.ik.lucene.IKAnalyzer;
import com.luoo.music.pojo.Journal;
import com.luoo.music.pojo.JournalSong;
import com.luoo.music.util.Constants;
import lombok.SneakyThrows;
@ -33,7 +36,6 @@ import util.StringTools;
@Configuration // 1.主要用于标记配置类，兼备Component的效果。
@EnableScheduling // 2.开启定时任务
public class SaticScheduleTask {
private static final String REDIS_AUTO_COMPLETE = "redis_auto_complete:";
@Autowired
private JournalDao journalDao;
@Autowired
@ -41,12 +43,32 @@ public class SaticScheduleTask {
@Autowired
private CacheChannel cacheChannel;
@Autowired
private RedisTemplate redisTemplate;
@Autowired
private TagDao tagDao;
@PostConstruct
private void init() {
Stream<Entry<String, Set<String>>> journalSongStream = journalSongDao.findAll().parallelStream()
.map(j -> getJournalSongMap(j)).flatMap(m -> m.entrySet().stream());
Stream<Entry<String, Set<String>>> journalStream = journalDao.findValidJournals().parallelStream()
.map(j -> getJournalMap(j)).flatMap(m -> m.entrySet().stream());
Map<String, List<Entry<String, Set<String>>>> map = Stream.concat(journalSongStream, journalStream)
.collect(Collectors.groupingBy(e -> e.getKey()));
int keyCount = map.size();
int cacheKeyCount = cacheChannel.keys(constants.Constants.J2CACHE_REGION_SEARCH_AUTO_COMPLETE).size();
if (keyCount != cacheKeyCount) {
map.entrySet().forEach(e -> {
List<String> value = e.getValue().stream().flatMap(a -> a.getValue().stream()).distinct().sorted()
.collect(Collectors.toList());
cacheChannel.set(constants.Constants.J2CACHE_REGION_SEARCH_AUTO_COMPLETE, e.getKey(), value);
});
}
}
// 1.凌晨4点执行更新期刊tag
@Scheduled(cron="0 0 4 * * *")
@Scheduled(cron = "0 0 4 * * *")
private void updateJouranlTag() {
AtomicLong updateCounter = new AtomicLong();
journalDao.findValidJournals().parallelStream().forEach(j -> updateJouranlTag(j, updateCounter));
@ -56,18 +78,41 @@ public class SaticScheduleTask {
}
// 2.凌晨4:20点执行更新分词
@Scheduled(cron="0 20 4 * * *")
@Scheduled(cron = "0 20 4 * * *")
private void updateAutoComplete() {
List<Journal> validJournals = journalDao.findValidJournals();
validJournals.parallelStream().forEach(j -> {
addIKKeyWord(j.getJournalNo());
addIKKeyWord(j.getTitle());
});
journalSongDao.findAll().parallelStream().forEach(s -> {
addIKKeyWord(s.getName());
addIKKeyWord(s.getArtist());
addIKKeyWord(s.getAlbum());
});
Stream<Entry<String, Set<String>>> journalSongStream = journalSongDao.findAll().parallelStream()
.map(j -> getJournalSongMap(j)).flatMap(m -> m.entrySet().stream());
Stream<Entry<String, Set<String>>> journalStream = journalDao.findValidJournals().parallelStream()
.map(j -> getJournalMap(j)).flatMap(m -> m.entrySet().stream());
Map<String, List<Entry<String, Set<String>>>> map = Stream.concat(journalSongStream, journalStream)
.collect(Collectors.groupingBy(e -> e.getKey()));
int keyCount = map.size();
int cacheKeyCount = cacheChannel.keys(constants.Constants.J2CACHE_REGION_SEARCH_AUTO_COMPLETE).size();
if (keyCount != cacheKeyCount) {
map.entrySet().forEach(e -> {
List<String> value = e.getValue().stream().flatMap(a -> a.getValue().stream()).distinct().sorted()
.collect(Collectors.toList());
cacheChannel.set(constants.Constants.J2CACHE_REGION_SEARCH_AUTO_COMPLETE, e.getKey(), value);
});
}
}
private Map<String, Set<String>> getJournalMap(Journal j) {
Map<String, Set<String>> map = new HashMap<>();
addIKKeyWord(j.getJournalNo(), map);
addIKKeyWord(j.getTitle(), map);
return map;
}
private Map<String, Set<String>> getJournalSongMap(JournalSong s) {
Map<String, Set<String>> map = new HashMap<>();
addIKKeyWord(s.getName(), map);
addIKKeyWord(s.getArtist(), map);
addIKKeyWord(s.getAlbum(), map);
return map;
}
private void updateJouranlTag(Journal journal, AtomicLong updateCounter) {
@ -84,7 +129,7 @@ public class SaticScheduleTask {
}
@SneakyThrows
private void addIKKeyWord(String keyword) {
private void addIKKeyWord(String keyword, Map<String, Set<String>> map) {
if (!StringTools.isEmpty(keyword)) {
keyword = keyword.trim();
// 构建IK分词器，使用smart分词模式
@ -93,13 +138,8 @@ public class SaticScheduleTask {
TokenStream ts = null;
try {
ts = analyzer.tokenStream("myfield", new StringReader(keyword));
// 获取词元位置属性
OffsetAttribute offset = ts.addAttribute(OffsetAttribute.class);
// 获取词元文本属性
CharTermAttribute term = ts.addAttribute(CharTermAttribute.class);
// 获取词元文本属性
TypeAttribute type = ts.addAttribute(TypeAttribute.class);
// 重置TokenStream（重置StringReader）
ts.reset();
// 迭代获取分词结果
@ -109,12 +149,24 @@ public class SaticScheduleTask {
for (int i = 1; i < token.length() + 1; i++) {
String sub = token.substring(0, i);
String encodedString = new String(sub.getBytes(), "UTF-8");
if (!map.containsKey(encodedString)) {
map.put(encodedString, new HashSet<>());
}
map.get(encodedString).add(keyword);
redisTemplate.opsForZSet().add(REDIS_AUTO_COMPLETE + encodedString, keyword, 0);
// redisTemplate.opsForZSet().add(REDIS_AUTO_COMPLETE + encodedString, keyword,
// 0);
String encodedStringLowerCase = encodedString.toLowerCase();
if (!encodedString.equals(encodedStringLowerCase)) {
redisTemplate.opsForZSet().add(REDIS_AUTO_COMPLETE + encodedStringLowerCase, keyword, 0);
if (!map.containsKey(encodedStringLowerCase)) {
map.put(encodedStringLowerCase, new HashSet<>());
}
map.get(encodedStringLowerCase).add(keyword);
// redisTemplate.opsForZSet().add(REDIS_AUTO_COMPLETE + encodedStringLowerCase,
// keyword, 0);
}
}
}

26
luoo_music/src/main/java/com/luoo/music/service/SearchService.java
Unescape View File

@ -1,31 +1,21 @@
package com.luoo.music.service;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import javax.annotation.PostConstruct;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.stereotype.Service;
import constants.Constants;
import net.oschina.j2cache.CacheChannel;
@Service
public class SearchService {
private static final String REDIS_AUTO_COMPLETE = "redis_auto_complete:";
@Autowired
private RedisTemplate redisTemplate;
@PostConstruct
private void init() {
}
private CacheChannel cacheChannel;
@SuppressWarnings("unchecked")
public List<String> autoComplete(String query, int limit) {
Set<String> values=redisTemplate.opsForZSet().rangeByScore(REDIS_AUTO_COMPLETE+query, 0, Double.MAX_VALUE, 0, limit);
return new ArrayList<>(values);
List<String> values = (List<String>) cacheChannel.get(Constants.J2CACHE_REGION_SEARCH_AUTO_COMPLETE, query)
.getValue();
return limit < values.size() ? values.subList(0, limit) : values;
}
}

3
luoo_music/src/main/resources/caffeine.properties
Unescape View File

@ -3,4 +3,5 @@ journal_query_page: 10000, 6h
journal_id: 2000, 6h
journal_no: 2000, 6h
journal_song_key: 10000, 6h
journal_song_list: 2000, 6h
journal_song_list: 2000, 6h
search_auto_complete: 200000
Write Preview
Loading…
Cancel
Save