app/lib/request.rb

   1 # frozen_string_literal: true
   2
   3 require 'ipaddr'
   4 require 'socket'
   5 require 'resolv'
   6
   7 # Monkey-patch the HTTP.rb timeout class to avoid using a timeout block
   8 # around the Socket#open method, since we use our own timeout blocks inside
   9 # that method
  10 class HTTP::Timeout::PerOperation
  11   def connect(socket_class, host, port, nodelay = false)
  12     @socket = socket_class.open(host, port)
  13     @socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1) if nodelay
  14   end
  15 end
  16
  17 class Request
  18   REQUEST_TARGET = '(request-target)'
  19
  20   # We enforce a 5s timeout on DNS resolving, 5s timeout on socket opening
  21   # and 5s timeout on the TLS handshake, meaning the worst case should take
  22   # about 15s in total
  23   TIMEOUT = { connect: 5, read: 10, write: 10 }.freeze
  24
  25   include RoutingHelper
  26
  27   def initialize(verb, url, **options)
  28     raise ArgumentError if url.blank?
  29
  30     @verb        = verb
  31     @url         = Addressable::URI.parse(url).normalize
  32     @http_client = options.delete(:http_client)
  33     @options     = options.merge(socket_class: use_proxy? ? ProxySocket : Socket)
  34     @options     = @options.merge(Rails.configuration.x.http_client_proxy) if use_proxy?
  35     @headers     = {}
  36
  37     raise Mastodon::HostValidationError, 'Instance does not support hidden service connections' if block_hidden_service?
  38
  39     set_common_headers!
  40     set_digest! if options.key?(:body)
  41   end
  42
  43   def on_behalf_of(account, key_id_format = :uri, sign_with: nil)
  44     raise ArgumentError, 'account must not be nil' if account.nil?
  45
  46     @account       = account
  47     @keypair       = sign_with.present? ? OpenSSL::PKey::RSA.new(sign_with) : @account.keypair
  48     @key_id_format = key_id_format
  49
  50     self
  51   end
  52
  53   def add_headers(new_headers)
  54     @headers.merge!(new_headers)
  55     self
  56   end
  57
  58   def perform
  59     begin
  60       response = http_client.public_send(@verb, @url.to_s, @options.merge(headers: headers))
  61     rescue => e
  62       raise e.class, "#{e.message} on #{@url}", e.backtrace[0]
  63     end
  64
  65     begin
  66       response = response.extend(ClientLimit)
  67
  68       # If we are using a persistent connection, we have to
  69       # read every response to be able to move forward at all.
  70       # However, simply calling #to_s or #flush may not be safe,
  71       # as the response body, if malicious, could be too big
  72       # for our memory. So we use the #body_with_limit method
  73       response.body_with_limit if http_client.persistent?
  74
  75       yield response if block_given?
  76     ensure
  77       http_client.close unless http_client.persistent?
  78     end
  79   end
  80
  81   def headers
  82     (@account ? @headers.merge('Signature' => signature) : @headers).without(REQUEST_TARGET)
  83   end
  84
  85   class << self
  86     def valid_url?(url)
  87       begin
  88         parsed_url = Addressable::URI.parse(url)
  89       rescue Addressable::URI::InvalidURIError
  90         return false
  91       end
  92
  93       %w(http https).include?(parsed_url.scheme) && parsed_url.host.present?
  94     end
  95
  96     def http_client
  97       HTTP.use(:auto_inflate).timeout(TIMEOUT.dup).follow(max_hops: 2)
  98     end
  99   end
 100
 101   private
 102
 103   def set_common_headers!
 104     @headers[REQUEST_TARGET]    = "#{@verb} #{@url.path}"
 105     @headers['User-Agent']      = Mastodon::Version.user_agent
 106     @headers['Host']            = @url.host
 107     @headers['Date']            = Time.now.utc.httpdate
 108     @headers['Accept-Encoding'] = 'gzip' if @verb != :head
 109   end
 110
 111   def set_digest!
 112     @headers['Digest'] = "SHA-256=#{Digest::SHA256.base64digest(@options[:body])}"
 113   end
 114
 115   def signature
 116     algorithm = 'rsa-sha256'
 117     signature = Base64.strict_encode64(@keypair.sign(OpenSSL::Digest.new('SHA256'), signed_string))
 118
 119     "keyId=\"#{key_id}\",algorithm=\"#{algorithm}\",headers=\"#{signed_headers.keys.join(' ').downcase}\",signature=\"#{signature}\""
 120   end
 121
 122   def signed_string
 123     signed_headers.map { |key, value| "#{key.downcase}: #{value}" }.join("\n")
 124   end
 125
 126   def signed_headers
 127     @headers.without('User-Agent', 'Accept-Encoding')
 128   end
 129
 130   def key_id
 131     case @key_id_format
 132     when :acct
 133       @account.to_webfinger_s
 134     when :uri
 135       [ActivityPub::TagManager.instance.uri_for(@account), '#main-key'].join
 136     end
 137   end
 138
 139   def http_client
 140     @http_client ||= Request.http_client
 141   end
 142
 143   def use_proxy?
 144     Rails.configuration.x.http_client_proxy.present?
 145   end
 146
 147   def block_hidden_service?
 148     !Rails.configuration.x.access_to_hidden_service && /\.(onion|i2p)$/.match?(@url.host)
 149   end
 150
 151   module ClientLimit
 152     def body_with_limit(limit = 1.megabyte)
 153       raise Mastodon::LengthValidationError if content_length.present? && content_length > limit
 154
 155       if charset.nil?
 156         encoding = Encoding::BINARY
 157       else
 158         begin
 159           encoding = Encoding.find(charset)
 160         rescue ArgumentError
 161           encoding = Encoding::BINARY
 162         end
 163       end
 164
 165       contents = String.new(encoding: encoding)
 166
 167       while (chunk = readpartial)
 168         contents << chunk
 169         chunk.clear
 170
 171         raise Mastodon::LengthValidationError if contents.bytesize > limit
 172       end
 173
 174       contents
 175     end
 176   end
 177
 178   class Socket < TCPSocket
 179     class << self
 180       def open(host, *args)
 181         outer_e = nil
 182         port    = args.first
 183
 184         addresses = []
 185         begin
 186           addresses = [IPAddr.new(host)]
 187         rescue IPAddr::InvalidAddressError
 188           Resolv::DNS.open do |dns|
 189             dns.timeouts = 5
 190             addresses = dns.getaddresses(host).take(2)
 191           end
 192         end
 193
 194         socks = []
 195         addr_by_socket = {}
 196
 197         addresses.each do |address|
 198           begin
 199             check_private_address(address)
 200
 201             sock     = ::Socket.new(address.is_a?(Resolv::IPv6) ? ::Socket::AF_INET6 : ::Socket::AF_INET, ::Socket::SOCK_STREAM, 0)
 202             sockaddr = ::Socket.pack_sockaddr_in(port, address.to_s)
 203
 204             sock.setsockopt(::Socket::IPPROTO_TCP, ::Socket::TCP_NODELAY, 1)
 205
 206             sock.connect_nonblock(sockaddr)
 207
 208             # If that hasn't raised an exception, we somehow managed to connect
 209             # immediately, close pending sockets and return immediately
 210             socks.each(&:close)
 211             return sock
 212           rescue IO::WaitWritable
 213             socks << sock
 214             addr_by_socket[sock] = sockaddr
 215           rescue => e
 216             outer_e = e
 217           end
 218         end
 219
 220         until socks.empty?
 221           _, available_socks, = IO.select(nil, socks, nil, Request::TIMEOUT[:connect])
 222
 223           if available_socks.nil?
 224             socks.each(&:close)
 225             raise HTTP::TimeoutError, "Connect timed out after #{Request::TIMEOUT[:connect]} seconds"
 226           end
 227
 228           available_socks.each do |sock|
 229             socks.delete(sock)
 230
 231             begin
 232               sock.connect_nonblock(addr_by_socket[sock])
 233             rescue Errno::EISCONN
 234               # Do nothing
 235             rescue => e
 236               sock.close
 237               outer_e = e
 238               next
 239             end
 240
 241             socks.each(&:close)
 242             return sock
 243           end
 244         end
 245
 246         if outer_e
 247           raise outer_e
 248         else
 249           raise SocketError, "No address for #{host}"
 250         end
 251       end
 252
 253       alias new open
 254
 255       def check_private_address(address)
 256         addr = IPAddr.new(address.to_s)
 257         return if private_address_exceptions.any? { |range| range.include?(addr) }
 258         raise Mastodon::HostValidationError if PrivateAddressCheck.private_address?(addr)
 259       end
 260
 261       def private_address_exceptions
 262         @private_address_exceptions = begin
 263           (ENV['ALLOWED_PRIVATE_ADDRESSES'] || '').split(',').map { |addr| IPAddr.new(addr) }
 264         end
 265       end
 266     end
 267   end
 268
 269   class ProxySocket < Socket
 270     class << self
 271       def check_private_address(_address)
 272         # Accept connections to private addresses as HTTP proxies will usually
 273         # be on local addresses
 274         nil
 275       end
 276     end
 277   end
 278
 279   private_constant :ClientLimit, :Socket, :ProxySocket
 280 end