Improve logging and error reporting around IP and location info (#386)

* Beef up error reporting of IP and location info

* Tidy up error reporting after some manual testing of it

* Don't cache erroneous locations; try again when asked again

* cargo fmt

backend/telemetry_shard/src/main.rs

@@ -134,7 +134,7 @@ async fn start_server(opts: Opts) -> anyhow::Result<()> {
                 (&Method::GET, "/health") => Ok(Response::new("OK".into())),
                 // Nodes send messages here:
                 (&Method::GET, "/submit") => {
-                    let real_addr = real_ip::real_ip(addr, req.headers());
+                    let (real_addr, real_addr_source) = real_ip::real_ip(addr, req.headers());
 
                     if let Some(reason) = block_list.blocked_reason(&real_addr) {
                         return Ok(Response::builder().status(403).body(reason.into()).unwrap());
@@ -143,7 +143,11 @@ async fn start_server(opts: Opts) -> anyhow::Result<()> {
                     Ok(http_utils::upgrade_to_websocket(
                         req,
                         move |ws_send, ws_recv| async move {
-                            log::info!("Opening /submit connection from {:?}", addr);
+                            log::info!(
+                                "Opening /submit connection from {:?} (address source: {})",
+                                real_addr,
+                                real_addr_source
+                            );
                             let tx_to_aggregator = aggregator.subscribe_node();
                             let (mut tx_to_aggregator, mut ws_send) =
                                 handle_node_websocket_connection(
@@ -156,7 +160,11 @@ async fn start_server(opts: Opts) -> anyhow::Result<()> {
                                     block_list,
                                 )
                                 .await;
-                            log::info!("Closing /submit connection from {:?}", addr);
+                            log::info!(
+                                "Closing /submit connection from {:?} (address source: {})",
+                                real_addr,
+                                real_addr_source
+                            );
                             // Tell the aggregator that this connection has closed, so it can tidy up.
                             let _ = tx_to_aggregator.send(FromWebsocket::Disconnected).await;
                             let _ = ws_send.close().await;

backend/telemetry_shard/src/real_ip.rs

@@ -35,13 +35,32 @@ If still no luck, look for the X-Real-IP header, which we expect to contain a si
 
 If that _still_ doesn't work, fall back to the socket address of the connection.
 */
-pub fn real_ip(addr: SocketAddr, headers: &hyper::HeaderMap) -> IpAddr {
+pub fn real_ip(addr: SocketAddr, headers: &hyper::HeaderMap) -> (IpAddr, Source) {
     let forwarded = headers.get("forwarded").and_then(header_as_str);
     let forwarded_for = headers.get("x-forwarded-for").and_then(header_as_str);
     let real_ip = headers.get("x-real-ip").and_then(header_as_str);
     pick_best_ip_from_options(forwarded, forwarded_for, real_ip, addr)
 }
 
+/// The source of the address returned
+pub enum Source {
+    ForwardedHeader,
+    XForwardedForHeader,
+    XRealIpHeader,
+    SocketAddr,
+}
+
+impl std::fmt::Display for Source {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Source::ForwardedHeader => write!(f, "'Forwarded' header"),
+            Source::XForwardedForHeader => write!(f, "'X-Forwarded-For' header"),
+            Source::XRealIpHeader => write!(f, "'X-Real-Ip' header"),
+            Source::SocketAddr => write!(f, "Socket address"),
+        }
+    }
+}
+
 fn header_as_str(value: &hyper::header::HeaderValue) -> Option<&str> {
     std::str::from_utf8(value.as_bytes()).ok()
 }
@@ -55,30 +74,39 @@ fn pick_best_ip_from_options(
     real_ip: Option<&str>,
     // socket address (if known)
     addr: SocketAddr,
-) -> IpAddr {
+) -> (IpAddr, Source) {
     let realip = forwarded
         .as_ref()
-        .and_then(|val| get_first_addr_from_forwarded_header(val))
+        .and_then(|val| {
+            let addr = get_first_addr_from_forwarded_header(val)?;
+            Some((addr, Source::ForwardedHeader))
+        })
         .or_else(|| {
             // fall back to X-Forwarded-For
-            forwarded_for
-                .as_ref()
-                .and_then(|val| get_first_addr_from_x_forwarded_for_header(val))
+            forwarded_for.as_ref().and_then(|val| {
+                let addr = get_first_addr_from_x_forwarded_for_header(val)?;
+                Some((addr, Source::XForwardedForHeader))
+            })
         })
         .or_else(|| {
             // fall back to X-Real-IP
-            real_ip.as_ref().map(|val| val.trim())
+            real_ip.as_ref().and_then(|val| {
+                let addr = val.trim();
+                Some((addr, Source::XRealIpHeader))
+            })
         })
-        .and_then(|ip| {
+        .and_then(|(ip, source)| {
             // Try parsing assuming it may have a port first,
             // and then assuming it doesn't.
-            ip.parse::<SocketAddr>()
+            let addr = ip
+                .parse::<SocketAddr>()
                 .map(|s| s.ip())
                 .or_else(|_| ip.parse::<IpAddr>())
-                .ok()
+                .ok()?;
+            Some((addr, source))
         })
         // Fall back to local IP address if the above fails
-        .unwrap_or(addr.ip());
+        .unwrap_or((addr.ip(), Source::SocketAddr));
 
     realip
 }