Rework video cache

config/config.example.yml

@@ -1132,7 +1132,7 @@ video_cache:
     enable: true
 
     ## The backend used for Video cache.
-    ## This can be PostgreSQL (0), Redis (1) or LRU (2).
+    ## This can be PostgreSQL, Redis or LRU.
     ##
     ## Redis is recommended when using more than one
     ## Invidious process for load balancing
@@ -1144,10 +1144,10 @@ video_cache:
     ## by Out-of-Memory. Altrough LRU cache is not saved
     ## across restarts.
     ##
-    ## Accepted values: 0, 1, 2
+    ## Accepted values: "lru", "redis", "postgres"
-    ## Default: 1
+    ## Default: "redis"
     ##
-    backend: 1
+    backend: "redis"
 
     ## Maximum amount of items that can be inside thecache.
     ##

shard.lock

@@ -28,6 +28,10 @@ shards:
     git: https://github.com/kemalcr/kemal.git
     version: 1.6.0
 
+  lru:
+    git: https://github.com/fijxu/crystal-lru.git
+    version: 1.0.4
+
   pg:
     git: https://github.com/will/crystal-pg.git
     version: 0.28.0

shard.yml

@@ -33,6 +33,8 @@ dependencies:
   http_proxy:
     github: mamantoha/http_proxy
     version: ~> 0.10.3
+  lru:
+    github: fijxu/crystal-lru
 
 development_dependencies:
   spectator:

src/invidious.cr

@@ -141,6 +141,7 @@ LOGGER = Invidious::LogHandler.new(OUTPUT, CONFIG.log_level, CONFIG.colorize_log
 
 # Check table integrity
 Invidious::Database.check_integrity(CONFIG)
+Invidious::Database::Videos.init
 
 # Minifies Invidious Javascript
 {% if flag?(:minify_debug) || (flag?(:release) || flag?(:production)) && !flag?(:skip_minified_js) %}

src/invidious/config.cr

@@ -232,7 +232,7 @@ class Config
     include YAML::Serializable
 
     property enabled : Bool = true
-    property backend : Int32 = 1
+    property backend : Invidious::Database::Videos::CacheType = Invidious::Database::Videos::CacheType::Redis
     # Max quantity of keys that can be held on the LRU cache
     property lru_max_size : Int32 = 18432 # ~512MB
     # Compress cache with Deflate
@@ -441,19 +441,6 @@ class Config
       end
     end
 
-    if config.video_cache.enabled
-      if !config.video_cache.backend.in?(0, 1, 2)
-        puts "Config: 'video_cache_storage', can only be:"
-        puts "0 (PostgreSQL)"
-        puts "1 (Redis compatible DB) (Default)"
-        puts "2 (In memory LRU)"
-      end
-      if config.video_cache.compress && config.video_cache.backend == 0
-        puts "Video Cache compression can only be enabled when using backend 1 (Redis) or 2 (LRU)"
-        exit(1)
-      end
-    end
-
     # Check if the socket configuration is valid
     if sb = config.socket_binding
       if sb.path.ends_with?("/") || File.directory?(sb.path)

src/invidious/database/videos.cr

@@ -1,15 +1,27 @@
 require "./base.cr"
 require "redis"
-
+require "lru"
-VideoCache = Invidious::Database::Videos::Cache.new
 
 module Invidious::Database::Videos
-  struct VideoCacheInfo
+  extend self
-    property info : String
-    property id : String
-    property updated : String
 
-    def initialize(@info, @id, @updated)
+  @@cache : (CacheMethods::LRU | CacheMethods::InternalRedis | CacheMethods::PostgresSQL)? = nil
+
+  enum CacheType
+    Postgres = 0
+    Redis    = 1
+    LRU      = 2
+  end
+
+  class VideoCacheInfo
+    include DB::Serializable
+    include JSON::Serializable
+
+    property id : String
+    property info : String
+    property updated : Time
+
+    def initialize(@id, @info, @updated)
     end
   end
 
@@ -28,134 +40,56 @@ module Invidious::Database::Videos
       return compressed.gets_to_end
     end
 
-    def decompress(video_info_compressed : String, id : String) : String?
+    def decompress(video_info_compressed : String) : String?
       compressed = IO::Memory.new
       compressed << video_info_compressed
       compressed.rewind
       decompressed = Compress::Deflate::Reader.new(compressed, sync_close: true)
-      begin
+      return decompressed.gets_to_end
-        return decompressed.gets_to_end
-      rescue Compress::Deflate::Error
-        # If there is an error when decompressing the video data,
-        # delete the video from the cache to fetch it again.
-        VideoCache.del(id)
-        return nil
-      end
-    end
-  end
-
-  class Cache
-    def initialize
-      case CONFIG.video_cache.backend
-      when 0
-        @cache = CacheMethods::PostgresSQL.new
-      when 1
-        @cache = CacheMethods::Redis_.new
-      when 2
-        @cache = CacheMethods::LRU.new
-      else
-        LOGGER.debug "Video Cache: Using default cache method to store video cache (PostgreSQL)"
-        @cache = CacheMethods::PostgresSQL.new
-      end
-    end
-
-    def set(video : VideoCacheInfo, expire_time)
-      @cache.set(video, expire_time)
-    end
-
-    def del(id : String)
-      @cache.del(id)
-    end
-
-    def get(id : String)
-      return @cache.get(id)
-    end
-  end
-
-  module CacheUtils
-    extend self
-
-    def to_video(info : String?, time : String?, id : String) : Video?
-      if info && time
-        # With the { we identify if it's a JSON or not
-        if info[0] != '{'
-          info = CacheCompression.decompress(info, id)
-          if info.nil?
-            return nil
-          end
-        end
-        return Video.new({
-          id:      id,
-          info:    JSON.parse(info).as_h,
-          updated: Time.parse(time, "%Y-%m-%d %H:%M:%S %z", Time::Location::UTC),
-        })
-      else
-        return nil
-      end
     end
   end
 
   module CacheMethods
-    # TODO: Save the cache on a file with a Job
     class LRU
       @max_size : Int32
-      @lru = {} of String => String
-      @access = [] of String
 
-      def initialize(@max_size = CONFIG.video_cache.lru_max_size)
+      def initialize(
+        @max_size = CONFIG.video_cache.lru_max_size,
+      )
+        @cache = LRUCache(VideoCacheInfo).new(max_size: @max_size, clean_interval: 1.second)
         LOGGER.info "Video Cache: Using in memory LRU to store video cache"
         LOGGER.info "Video Cache, LRU: LRU cache max size set to #{@max_size}"
       end
 
-      # TODO: Handle expire_time with a Job
       def set(video : VideoCacheInfo, expire_time)
-        self[video.id] = video.info
+        if CONFIG.video_cache.compress
-        self[video.id + ":time"] = "#{video.updated}"
+          video.info = CacheCompression.compress(video.info)
-      end
-
-      def del(id : String)
-        self.delete(id)
-        self.delete(id + ":time")
-      end
-
-      def get(id : String)
-        info = self[id]
-        time = self[id + ":time"]
-        return CacheUtils.to_video(info, time, id)
-      end
-
-      private def [](key)
-        if @lru[key]?
-          @access.delete(key)
-          @access.push(key)
-          @lru[key]
-        else
-          nil
         end
+
+        @cache.set(video.id, video, expire_time)
       end
 
-      private def []=(key, value)
+      def del(video_id : String)
-        if @lru.size >= @max_size
+        @cache.del(video_id)
-          lru_key = @access.shift
-          @lru.delete(lru_key)
-        end
-        @lru[key] = value
-        @access.push(key)
       end
 
-      private def delete(key)
+      def get(video_id : String)
-        if @lru[key]?
+        cached_video = @cache.get(video_id)
-          @lru.delete(key)
+        return if cached_video.nil?
-          @access.delete(key)
+
+        if CONFIG.video_cache.compress && (cached_video.info[0] != '{')
+          cached_video.info = CacheCompression.decompress(cached_video.info)
         end
+
+        cached_video
       end
     end
 
-    class Redis_
+    class InternalRedis
-      @redis : Redis::Client
+      @client : Redis::Client
 
       def initialize
-        @redis = begin
+        @client = begin
           Redis::Client.new(CONFIG.redis_url)
         rescue ex
           LOGGER.fatal "Video Cache: Failed to connect to redis database: '#{ex.message}'"
@@ -163,31 +97,47 @@ module Invidious::Database::Videos
         end
         LOGGER.info "Video Cache: Using Redis compatible DB to store video cache"
         LOGGER.info "Connecting to Redis compatible DB"
-        if @redis.ping
+        if @client.ping
           LOGGER.info "Connected to Redis compatible DB at '#{CONFIG.redis_url}'" if CONFIG.redis_url
         end
       end
 
       def set(video : VideoCacheInfo, expire_time)
-        @redis.set(video.id, video.info, ex: expire_time)
+        video_json = video.to_json
-        @redis.set(video.id + ":time", video.updated.to_s, ex: expire_time)
+
+        if CONFIG.video_cache.compress
+          video_json_compressed = CacheCompression.compress(video_json)
+          @client.set(video.id, video_json_compressed, ex: expire_time)
+        else
+          @client.set(video.id, video_json, ex: expire_time)
+        end
       end
 
-      def del(id : String)
+      def del(video_id : String)
-        @redis.del(id)
+        @client.del(video_id)
-        @redis.del(id + ":time")
       end
 
-      def get(id : String)
+      def get(video_id : String)
-        info = @redis.get(id)
+        cached_video = @client.get(video_id)
-        time = @redis.get(id + ":time")
+        return if cached_video.nil?
-        return CacheUtils.to_video(info, time, id)
+
+        # With the { we identify if it's a JSON or not
+        if CONFIG.video_cache.compress && (cached_video[0] != '{')
+          video_json_decompressed = CacheCompression.decompress(cached_video)
+          return VideoCacheInfo.from_json(video_json_decompressed)
+        else
+          return VideoCacheInfo.from_json(cached_video)
+        end
       end
     end
 
     class PostgresSQL
       def initialize
         LOGGER.info "Video Cache: Using PostgreSQL to store video cache"
+        if CONFIG.video_cache.compress
+          LOGGER.warn "Video Cache: PostgreSQL does not support cache compression, disabling cache compression"
+          CONFIG.video_cache.compress = false
+        end
       end
 
       def set(video : VideoCacheInfo, expire_time)
@@ -200,47 +150,76 @@ module Invidious::Database::Videos
         PG_DB.exec(request, video.id, video.info, video.updated)
       end
 
-      def del(id)
+      def del(video_id)
         request = <<-SQL
           DELETE FROM videos *
           WHERE id = $1
         SQL
 
-        PG_DB.exec(request, id)
+        PG_DB.exec(request, video_id)
       end
 
-      def get(id : String) : Video?
+      def get(video_id : String) : VideoCacheInfo?
         request = <<-SQL
           SELECT * FROM videos
           WHERE id = $1
         SQL
 
-        data = PG_DB.query_one?(request, id, as: VideoCacheInfo)
+        PG_DB.query_one?(request, video_id, as: VideoCacheInfo)
-
-        if data
-          return CacheUtils.to_video(data.info, data.updated, id)
-        end
       end
     end
   end
 
-  extend self
+  def init
+    if !CONFIG.video_cache.enabled
+      LOGGER.info "Video Cache: Cache is disabled, no videos will be cached"
+      return
+    end
+
+    case CONFIG.video_cache.backend
+    when CacheType::Postgres
+      @@cache = CacheMethods::PostgresSQL.new
+    when CacheType::Redis
+      @@cache = CacheMethods::InternalRedis.new
+    when CacheType::LRU
+      @@cache = CacheMethods::LRU.new
+    else
+      LOGGER.debug "Video Cache: Using default cache method to store video cache (PostgreSQL)"
+      @@cache = CacheMethods::PostgresSQL.new
+    end
+  end
 
   def insert(video : Video)
-    video_info = video.info.to_json
+    cache = @@cache
-    if CONFIG.video_cache.compress
+    return if cache.nil?
-      video_info = CacheCompression.compress(video_info)
+
-    end
+    video_cache_info = VideoCacheInfo.new(video.id, video.info.to_json, video.updated)
-    video_cache_info = VideoCacheInfo.new(video_info, video.id, video.updated.to_s)
+
-    VideoCache.set(video: video_cache_info, expire_time: 14400) if CONFIG.video_cache.enabled
+    # Videos expire after 6 hours, so we expire them before it expires in the
+    # youtube side
+    # 3600 * 5.95 = 21420
+    cache.set(video_cache_info, 21420)
   end
 
-  def delete(id)
+  def delete(video_id) : Nil
-    VideoCache.del(id)
+    cache = @@cache
+    return if cache.nil?
+
+    cache.del(video_id)
   end
 
-  def select(id : String) : Video?
+  def select(video_id : String) : Video?
-    VideoCache.get(id)
+    cache = @@cache
+    return if cache.nil?
+
+    cached_video = cache.get(video_id)
+    return if cached_video.nil?
+
+    video = Video.new({
+      id:      video_id,
+      info:    JSON.parse(cached_video.info).as_h,
+      updated: cached_video.updated,
+    })
   end
 
   def delete_expired