springboot+redis+mysql+quartz-使用pipeline+lua技术将缓存数据定时更新到数据库

阿里云国内75折回扣微信号：monov8

阿里云国际，腾讯云国际，低至75折。AWS 93折免费开户实名账号代冲值优惠多多微信号：monov8 飞机：@monov6

一、重点

代码讲解7.3点赞功能-定时持久化到数据库-Java程序整合pipeline+lua_哔哩哔哩_bilibili

https://www.bilibili.com/video/BV1Lg4y1w7U9

代码

blogLike_schedule/like08 · xin麒/XinQiUtilsOrDemo - 码云 - 开源中国 (gitee.com)

https://gitee.com/flowers-bloom-is-the-sea/XinQiUtilsOrDemo/tree/master/blogLike_schedule/like08

数据库表
blogLike_schedule · xin麒/XinQiUtilsOrDemo - 码云 - 开源中国 (gitee.com)

二、实现过程

这里主要是开2个定时任务一个是平时的缓存数据持久化操作就是要pipeline技术

另一个时间就使用lua脚本向redis缓存获取缓存数据再更新到数据库。

1、对lua脚本任务执行

@Slf4j
public class LikeTaskAtNight extends QuartzJobBean {
    @Autowired
    BlogService blogService;

    @Autowired
    StringRedisTemplate stringRedisTemplate;

    /**
     * 用来从redis中的zset中获取点赞缓存的键值对间隔时间存入mysql
     * 这里就夜间执行吧因为lua会阻塞redis其他操作选择夜间流量低时进行执行
     * @param context
     * @throws JobExecutionException
     */
    @Override
    protected void executeInternal(JobExecutionContext context) throws JobExecutionException {
        int hour = LocalTime.now().getHour();
//        int minute = LocalTime.now().getMinute();
        if (hour == 3){
            // 要执行的内容就写在这个函数中
//        blogService.updateAllLikeListToDatabase();//这个没使用pipeline技术效果应该是明显低于updateAllLikeListToDatabaseByPipeline1
            log.debug("LikeTaskAtNight - time is {}",System.currentTimeMillis());
//        blogService.updateAllLikeListToDatabaseByPipeline1();//这个使用了pipeline技术但是还有优化空间比如书对sql的更新可以说批量更新
            blogService.updateAllLikeListToDatabaseByLua();//这个使用了lua技术但是还有优化空间比如书对sql的更新可以说批量更新

        }
    }

}

配置文件

import com.xinqi.task.LikeTask;
import com.xinqi.task.LikeTaskAtNight;
import org.quartz.*;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
@Configuration
public class QuartzConfig {
    @Bean
    public JobDetail quartzDetail_1(){
        // withIdentity指定的是这个job的id
        return JobBuilder.newJob(LikeTask.class)
                .withIdentity("LIKE_TASK_IDENTITY")
                .storeDurably()
                .build();
    }


    @Bean
    public Trigger quartzTrigger_1(){ //触发器
        SimpleScheduleBuilder scheduleBuilder = SimpleScheduleBuilder.simpleSchedule()
                .withIntervalInSeconds(20)  //设置时间周期单位秒
//                .withIntervalInHours(2)  //两个小时执行一次
                .repeatForever();
        return TriggerBuilder.newTrigger().forJob(quartzDetail_1())
//                .forJob(quartzDetail_2())
                .withIdentity("LikeTask_TRENDS_TRIGGER")
                .withSchedule(scheduleBuilder)
                .build();
    }

    @Bean
    public JobDetail quartzDetail_LikeTaskAtNight(){
        // withIdentity指定的是这个job的id
        return JobBuilder.newJob(LikeTaskAtNight.class)
                .withIdentity("LikeTaskAtNight_IDENTITY")
                .storeDurably()
                .build();
    }


    @Bean
    public Trigger quartzTrigger_2(){ //触发器
        SimpleScheduleBuilder scheduleBuilder = SimpleScheduleBuilder.simpleSchedule()
//                .withIntervalInSeconds(35)  //设置时间周期单位秒
                .withIntervalInHours(1)  //两个小时执行一次
                .repeatForever();
        return TriggerBuilder.newTrigger().forJob(quartzDetail_LikeTaskAtNight())
//                .forJob(quartzDetail_2())
                .withIdentity("LikeTaskAtNight_USER_TRENDS_TRIGGER")
                .withSchedule(scheduleBuilder)
                .build();
    }

}

updateAllLikeListToDatabaseByLua执行的内容

@Override
public void updateAllLikeListToDatabaseByLua() {
    String prefix = "BLOG_LIKED_KEY";

    Map<Long, Map<String, String>> maps = getMapsByLuaUseJedis(prefix);
    if (maps == null || maps.size() == 0) return;

    for (Map.Entry<Long, Map<String, String>> entry : maps.entrySet()) {
        Long blogId = entry.getKey();
        Map<String, String> likeList = entry.getValue();
        updateLikeListByBlogId(blogId, likeList);
    }
}

public Map<Long, Map<String, String>> getMapsByLuaUseJedis(String prefix) {
    Map<Long, Map<String, String>> maps = null;
    String script = "local prefix = KEYS[1]; \n" +
            "local redisKeys = redis.call('keys',prefix ..'*');\n" +
            "\n" +
            "if(not redisKeys) \n" +
            "    then    \n" +
            "    \treturn (nil);\n" +
            "end;\n" +
            "\n" +
            "local maps = {};\n" +
            "\n" +
            "for i, v in pairs(redisKeys) do\n" +
            "    local blogId = string.sub(v,string.len(prefix) + 1,string.len(v));\n" +
            "    local zset = redis.call('zrange',v,'0','-1','withscores');\n" +
            "    table.insert(maps,blogId);\n" +
            "    table.insert(maps,zset);    \n" +
            "end;\n" +
            "\n" +
            "return maps;";

    List result = null;

    result = getResultByLuaUseJedis(script, prefix);

    maps = parseLuaResultToMaps(result);

    return maps;

}

    private List getResultByLuaUseJedis(String script, String prefix) {
        Jedis jedis = null;

        List result = null;
        int failTime = 0;//执行失败的次数
        boolean flag = true;//如果为true就说明报错

        while (flag && failTime < 2) {
            try {
                jedis = jedisPool.getResource();
                result = (List) jedis.eval(script, Arrays.asList(prefix), new ArrayList<>());//https://blog.csdn.net/EnjoyFight/article/details/127808971
                System.out.println("result is " + result);//result is [1, [1, 1688218686914]]
/*        log.debug("result is ",result);//不知道为什么无法显示
        log.debug("result is null?{}",result==null);
        String s = JSONUtil.toJsonStr(result);
        log.debug("s is ",s);
        log.debug("s is null? {}",s==null);
        log.debug("s .len is  {}",s.length());*/

                flag = false;
            } catch (Exception e) {
                e.printStackTrace();
                failTime++;
            } finally {
                if (jedis != null) jedis.close();
            }

        }

        return result;
    }