Merge remote-tracking branch 'upstream'

src/ext/kemal_static_file_handler.cr

@@ -1,3 +1,24 @@
+{% if compare_versions(Crystal::VERSION, "1.17.0-dev") >= 0 %}
+  # Strip StaticFileHandler from the binary
+  #
+  # This allows us to compile on 1.17.0 as the compiler won't try to
+  # semantically check the outdated upstream code.
+  class Kemal::Config
+    private def setup_static_file_handler
+    end
+  end
+
+  # Nullify `Kemal::StaticFileHandler`
+  #
+  # Needed until the next release of Kemal after 1.7
+  class Kemal::StaticFileHandler < HTTP::StaticFileHandler
+    def call(context : HTTP::Server::Context)
+    end
+  end
+
+  {% skip_file %}
+{% end %}
+
 # Since systems have a limit on number of open files (`ulimit -a`),
 # we serve them from memory to avoid 'Too many open files' without needing
 # to modify ulimit.

src/invidious.cr

@@ -86,6 +86,7 @@ HTTP_CHUNK_SIZE            = 10485760 # ~10MB
 CURRENT_BRANCH  = {{ "#{`git branch | sed -n '/* /s///p'`.strip}" }}
 CURRENT_COMMIT  = {{ "#{`git rev-list HEAD --max-count=1 --abbrev-commit`.strip}" }}
 CURRENT_VERSION = {{ "#{`git log -1 --format=%ci | awk '{print $1}' | sed s/-/./g`.strip}" }}
+CURRENT_TAG     = {{ "#{`git tag --points-at HEAD`.strip}" }}
 
 # This is used to determine the `?v=` on the end of file URLs (for cache busting). We
 # only need to expire modified assets, so we can use this to find the last commit that changes
@@ -229,19 +230,25 @@ error 500 do |env, exception|
   error_template(500, exception)
 end
 
-static_headers do |env|
-  env.response.headers.add("Cache-Control", "max-age=2629800")
-end
-
 # Init Kemal
 
-public_folder "assets"
-
 Kemal.config.powered_by_header = false
 add_handler FilteredCompressHandler.new
 add_handler APIHandler.new
 add_handler AuthHandler.new
 add_handler DenyFrame.new
+
+{% if compare_versions(Crystal::VERSION, "1.17.0-dev") >= 0 %}
+  Kemal.config.serve_static = false
+  add_handler Invidious::HttpServer::StaticAssetsHandler.new("assets", directory_listing: false)
+{% else %}
+  public_folder "assets"
+
+  static_headers do |env|
+    env.response.headers.add("Cache-Control", "max-age=2629800")
+  end
+{% end %}
+
 add_context_storage_type(Array(String))
 add_context_storage_type(Preferences)
 add_context_storage_type(Invidious::User)
@@ -256,11 +263,8 @@ Kemal.config.app_name = "Invidious"
 {% end %}
 
 Kemal.run do |config|
-  # Set max request line size if configured
-  if max_size = CONFIG.max_request_line_size
-    config.server.not_nil!.max_request_line_size = max_size
-  end
-  
+  config.server.not_nil!.max_request_line_size = 16384
+
   if socket_binding = CONFIG.socket_binding
     File.delete?(socket_binding.path)
     # Create a socket and set its desired permissions

src/invidious/helpers/crystal_class_overrides.cr

@@ -3,15 +3,28 @@
 # IPv6 addresses.
 #
 class TCPSocket
-  def initialize(host, port, dns_timeout = nil, connect_timeout = nil, blocking = false, family = Socket::Family::UNSPEC)
-    Addrinfo.tcp(host, port, timeout: dns_timeout, family: family) do |addrinfo|
-      super(addrinfo.family, addrinfo.type, addrinfo.protocol, blocking)
-      connect(addrinfo, timeout: connect_timeout) do |error|
-        close
-        error
+  {% if compare_versions(Crystal::VERSION, "1.18.0-dev") >= 0 %}
+    def initialize(host : String, port, dns_timeout = nil, connect_timeout = nil, blocking = false, family = Socket::Family::UNSPEC)
+      Addrinfo.tcp(host, port, timeout: dns_timeout, family: family) do |addrinfo|
+        super(family: addrinfo.family, type: addrinfo.type, protocol: addrinfo.protocol)
+        Socket.set_blocking(self.fd, blocking)
+        connect(addrinfo, timeout: connect_timeout) do |error|
+          close
+          error
+        end
       end
     end
-  end
+  {% else %}
+    def initialize(host : String, port, dns_timeout = nil, connect_timeout = nil, blocking = false, family = Socket::Family::UNSPEC)
+      Addrinfo.tcp(host, port, timeout: dns_timeout, family: family) do |addrinfo|
+        super(addrinfo.family, addrinfo.type, addrinfo.protocol, blocking)
+        connect(addrinfo, timeout: connect_timeout) do |error|
+          close
+          error
+        end
+      end
+    end
+  {% end %}
 end
 
 # :ditto:

src/invidious/http_server/static_assets_handler.cr

@@ -0,0 +1,120 @@
+{% skip_file if compare_versions(Crystal::VERSION, "1.17.0-dev") < 0 %}
+
+module Invidious::HttpServer
+  class StaticAssetsHandler < HTTP::StaticFileHandler
+    # In addition to storing the actual data of a file, it also implements the required
+    # getters needed for the object to imitate a `File::Stat` within `StaticFileHandler`.
+    #
+    # Since the `File::Stat` is created once in `#call` and then passed around to the
+    # rest of the class's methods, imitating the object allows us to only lookup
+    # the cache hash once for every request.
+    #
+    private record CachedFile, data : Bytes, size : Int64, modification_time : Time do
+      def directory?
+        false
+      end
+
+      def file?
+        true
+      end
+    end
+
+    CACHE_LIMIT = 5_000_000 # 5MB
+    @@current_cache_size = 0
+    @@cached_files = {} of Path => CachedFile
+
+    # Returns metadata for the requested file
+    #
+    # If the requested file is cached, a `CachedFile` is returned instead of a `File::Stat`.
+    # This represents the metadata info of a cached file and implements all the methods of `File::Stat` that
+    # is used by the `StaticAssetsHandler`.
+    #
+    # The `CachedFile` also stores the raw bytes of the cached file, and this method serves as the place where
+    # the cached file is retrieved if it exists. Though the data will only be read in `#serve_file`
+    private def file_info(expanded_path : Path)
+      file_path = @public_dir.join(expanded_path.to_kind(Path::Kind.native))
+      {@@cached_files[file_path]? || File.info?(file_path), file_path}
+    end
+
+    # Add "Cache-Control" header to the response
+    private def add_cache_headers(response_headers : HTTP::Headers, last_modified : Time) : Nil
+      super; response_headers["Cache-Control"] = "max-age=2629800"
+    end
+
+    # Serves and caches the file at the given path.
+    #
+    # This is an override of `serve_file` to allow serving a file from memory, and to cache it
+    # it as needed.
+    private def serve_file(context : HTTP::Server::Context, file_info, file_path : Path, original_file_path : Path, last_modified : Time)
+      context.response.content_type = MIME.from_filename(original_file_path.to_s, "application/octet-stream")
+
+      range_header = context.request.headers["Range"]?
+
+      # If the file is cached we can just directly serve it
+      if file_info.is_a? CachedFile
+        return dispatch_serve(context, file_info.data, file_info, range_header)
+      end
+
+      # Otherwise we'll need to read from disk and cache it
+      retrieve_bytes_from = IO::Memory.new
+      File.open(file_path) do |file|
+        # We cannot cache partial data so we'll rewind and read from the start
+        if range_header
+          dispatch_serve(context, file, file_info, range_header)
+          IO.copy(file.rewind, retrieve_bytes_from)
+        else
+          context.response.output = IO::MultiWriter.new(context.response.output, retrieve_bytes_from, sync_close: true)
+          dispatch_serve(context, file, file_info, range_header)
+        end
+      end
+
+      return flush_io_to_cache(retrieve_bytes_from, file_path, file_info)
+    end
+
+    # Writes file data to the cache
+    private def flush_io_to_cache(io, file_path, file_info)
+      if (@@current_cache_size += file_info.size) <= CACHE_LIMIT
+        @@cached_files[file_path] = CachedFile.new(io.to_slice, file_info.size, file_info.modification_time)
+      end
+    end
+
+    # Either send the file in full, or just fragments of it depending on the request
+    private def dispatch_serve(context, file, file_info, range_header)
+      if range_header
+        # an IO is needed for `serve_file_range`
+        file = file.is_a?(Bytes) ? IO::Memory.new(file, writeable: false) : file
+        serve_file_range(context, file, range_header, file_info)
+      else
+        context.response.headers["Accept-Ranges"] = "bytes"
+        serve_file_full(context, file, file_info)
+      end
+    end
+
+    # If we're serving the full file right away then there's no need for an IO at all.
+    private def serve_file_full(context : HTTP::Server::Context, file : Bytes, file_info)
+      context.response.status = :ok
+      context.response.content_length = file_info.size
+      context.response.write file
+    end
+
+    # Serves segments of a file based on the `Range header`
+    #
+    # An override of `serve_file_range` to allow using a generic IO rather than a `File`.
+    # Literally the same code as what we inherited but just with the `file` argument's type
+    # being set to `IO` rather than `File`
+    #
+    # Can be removed once https://github.com/crystal-lang/crystal/issues/15817 is fixed.
+    private def serve_file_range(context : HTTP::Server::Context, file : IO, range_header : String, file_info)
+      # Paste in the body of inherited serve_file_range
+      {{@type.superclass.methods.select(&.name.==("serve_file_range"))[0].body}}
+    end
+
+    # Clear cached files.
+    #
+    # This is only used in the specs to clear the cache before each handler test
+    def self.clear_cache
+      @@current_cache_size = 0
+      return @@cached_files.clear
+    end
+  end
+end

src/invidious/routes/companion.cr

@@ -1,5 +1,5 @@
 module Invidious::Routes::Companion
-  # /companion
+  # GET /companion
   def self.get_companion(env)
     current_companion = env.get("current_companion").as(Int32)
 
@@ -18,6 +18,24 @@ module Invidious::Routes::Companion
     end
   end
 
+  # POST /companion
+  def self.post_companion(env)
+    url = env.request.path
+    if env.request.query
+      url += "?#{env.request.query}"
+    end
+
+    begin
+      COMPANION_POOL.client do |wrapper|
+        wrapper.client.post(url, env.request.headers, env.request.body) do |resp|
+          return self.proxy_companion(env, resp)
+        end
+      end
+    rescue ex
+    end
+  end
+
+
   def self.options_companion(env)
     current_companion = env.get("current_companion").as(Int32)
 

src/invidious/routes/login.cr

@@ -98,6 +98,8 @@ module Invidious::Routes::Login
 
             begin
               validate_request(tokens[0], answer, env.request, HMAC_KEY, locale)
+            rescue ex : InfoException
+              return error_template(400, InfoException.new("Erroneous CAPTCHA"))
             rescue ex
               return error_template(400, ex)
             end

src/invidious/routing.cr

@@ -230,6 +230,7 @@ module Invidious::Routing
   def register_companion_routes
     if CONFIG.invidious_companion.present?
       get "/companion/*", Routes::Companion, :get_companion
+      post "/companion/*", Routes::Companion, :post_companion
       options "/companion/*", Routes::Companion, :options_companion
     end
   end

src/invidious/views/template.ecr

@@ -344,8 +344,21 @@
                     <% else %>
                         <%= translate(locale, "Current version: ") %>
                     <% end %>
-                    
-                    <%= CURRENT_VERSION %>-<%= CURRENT_COMMIT %> @ <%= CURRENT_BRANCH %>
+                    <% if CONFIG.modified_source_code_url %>
+                        <a href="<%= CONFIG.modified_source_code_url %>/commit/<%= CURRENT_COMMIT %>"><%= CURRENT_VERSION %>-<%= CURRENT_COMMIT %></a>
+                    <% else %>
+                        <a href="https://github.com/iv-org/invidious/commit/<%= CURRENT_COMMIT %>"><%= CURRENT_VERSION %>-<%= CURRENT_COMMIT %></a>
+                    <% end %>
+                    @ <%= CURRENT_BRANCH %>
+                    <% if CURRENT_TAG != "" %>
+                    (
+                        <% if CONFIG.modified_source_code_url %>
+                            <a href="<%= CONFIG.modified_source_code_url %>/releases/tag/<%= CURRENT_TAG %>"><%= CURRENT_TAG %></a>
+                        <% else %>
+                            <a href="https://github.com/iv-org/invidious/releases/tag/<%= CURRENT_TAG %>"><%= CURRENT_TAG %></a>
+                        <% end %>
+                    )
+                    <% end %>
                 </span>
                 <div class="right">
                     <a href="/privacy" title="<%= translate(locale, "footer_privacy_policy_link")%>"><%= translate(locale, "footer_privacy_policy_link") %></a>