Add PII correlation tool

.rubocop.yml

@@ -12,6 +12,7 @@ AllCops:
     - 'bin/**/*'
     - 'lib/namespaced_env_cache.rb'
     - 'vendor/bundle/**/*'
+    - 'docker/**/*'
   NewCops: enable
   SuggestExtensions: false
 

Gemfile

@@ -2,6 +2,7 @@ source 'https://rubygems.org'
 ruby '>= 3.3', '< 3.5'
 
 # Essential gems: servers, adapters, Rails + Rails requirements
+gem 'bcrypt', '~> 3.1'
 gem 'coffee-rails', '~> 5.0.0'
 gem 'connection_pool', '< 3.0' # mperham/connection_pool#210
 gem 'counter_culture', '~> 3.2'
@@ -38,6 +39,7 @@ gem 'groupdate', '~> 6.1'
 
 # View stuff.
 gem 'diffy', '~> 3.4'
+gem 'ipaddress', '~> 0.8'
 gem 'jbuilder', '~> 2.11'
 gem 'rqrcode', '~> 2.1'
 gem 'will_paginate', '~> 3.3'

Gemfile.lock

@@ -181,6 +181,7 @@ GEM
       mini_magick (>= 4.9.5, < 6)
       ruby-vips (>= 2.0.17, < 3)
     io-console (0.8.2)
+    ipaddress (0.8.3)
     irb (1.17.0)
       pp (>= 0.6.0)
       prism (>= 1.3.0)
@@ -497,6 +498,7 @@ DEPENDENCIES
   aws-sdk-s3 (~> 1.208)
   aws-sdk-sns (~> 1.72)
   aws-ses-v4
+  bcrypt (~> 3.1)
   byebug (~> 11.1)
   capybara (~> 3.38)
   chartkick (~> 4.2)
@@ -512,6 +514,7 @@ DEPENDENCIES
   flamegraph (~> 0.9)
   groupdate (~> 6.1)
   image_processing (~> 1.12)
+  ipaddress (~> 0.8)
   jbuilder (~> 2.11)
   jquery-rails (~> 4.5.0)
   letter_opener_web (~> 2.0)

app/assets/javascripts/moderator.js

@@ -47,4 +47,14 @@ $(() => {
       checkbox.checked = action === 'all';
     });
   });
+
+  QPixel.DOM.addSelectorListener('submit', '#pii-correlation-form', async (ev) => {
+    ev.preventDefault();
+
+    const targetId = /** @type {HTMLInputElement}*/(document.querySelector('input[name="target_id"]')).value;
+    const resp = await QPixel.fetch(`${location.pathname}?format=template&target_id=${targetId}`);
+    const html = await resp.text();
+
+    document.querySelector('.js-correlation-container').innerHTML = html;
+  });
 });

app/controllers/moderator_controller.rb

@@ -89,6 +89,16 @@ def handle_spammy_users
     redirect_to mod_spammers_path
   end
 
+  def pii_correlation
+    @user = User.find(params[:id])
+    respond_to do |format|
+      format.html
+      format.template do
+        @target = User.find_by(id: params[:target_id])
+      end
+    end
+  end
+
   private
 
   def set_post

app/helpers/moderator_helper.rb

@@ -16,4 +16,29 @@ def text_bg(cls, content = nil, **opts, &block)
       tag.span content, class: ["has-background-color-#{cls}", opts[:class]].join(' ')
     end
   end
+
+  ##
+  # Split an IP address into an array of hashed octets (well, hexadecets for IPv6).
+  # @param ip [String] The IP address to process.
+  # @param salting_user [User] A user from which to source a salt for hashing. For hashes to be directly comparable, you
+  # must use the same user for each IP address you wish to compare, even if sourced from a different user.
+  # @return [[String, [String?]]] The IP address family, and an array of hashed octets.
+  def split_hash_ip(ip, salting_user)
+    begin
+      addr = IPAddress.parse(ip)
+    rescue ArgumentError
+      return ['', []]
+    end
+    splat = if addr.ipv6?
+              addr.hexs
+            else
+              addr.octets
+            end
+    salt = BCrypt::Password.new(salting_user.encrypted_password).salt
+    splat = splat.map { |p| Digest::SHA2.hexdigest(salt + p.to_s) }
+    [
+      addr.ipv6? ? 'IPv6' : 'IPv4',
+      splat
+    ]
+  end
 end

app/jobs/update_user_stats_job.rb

@@ -0,0 +1,11 @@
+class UpdateUserStatsJob < ApplicationJob
+  queue_as :default
+
+  def perform(*)
+    domains = User.all.select(:email)
+                  .group_by { |u| u.email&.split('@')&.[](1) }
+                  .transform_values(&:size)
+                  .reject { |d, _u| d.include? 'localhost' }
+    Rails.cache.hmset('user_email_domains', domains)
+  end
+end

app/views/moderator/pii_correlation.html.erb

@@ -0,0 +1,25 @@
+<h2>PII correlation for <%= user_link @user %></h2>
+<p>
+  This tool displays correlations between personally-identifying information across user accounts. This includes email
+  address and IP addresses. Please select a user against whom to compare.
+</p>
+
+<noscript>JavaScript is required to use this tool.</noscript>
+<form id="pii-correlation-form" action="#" class="form-horizontal">
+  <div class="form-group-horizontal">
+    <div class="form-group">
+      <label for="target-id" class="form-element">Target user ID</label>
+      <input type="number" id="target-id" name="target_id" class="form-element" required />
+    </div>
+    <div class="actions">
+      <button type="submit" class="button is-primary is-filled">Compare</button>
+    </div>
+  </div>
+</form>
+
+<p>
+  Information is hashed to protect users' privacy.
+  Text <span class="has-background-color-red">highlighted in red</span> indicates matching data.
+</p>
+
+<div class="js-correlation-container"></div>

app/views/moderator/pii_correlation.template.erb

@@ -0,0 +1,132 @@
+<h3>Comparing with <%= user_link @target %></h3>
+
+<h4>Email address</h4>
+<table class="table is-full-width is-striped is-with-hover">
+  <thead>
+  <tr>
+    <th></th>
+    <th>Handle</th>
+    <th>Domain</th>
+  </tr>
+  </thead>
+  <tbody>
+  <tr>
+    <td><strong><%= @user.rtl_safe_username %></strong></td>
+    <% user_handle = Digest::SHA2.hexdigest(@user.email.split('@')[0]) %>
+    <% target_handle = Digest::SHA2.hexdigest(@target.email.split('@')[0]) %>
+    <% user_domain = Digest::SHA2.hexdigest(@user.email.split('@')[1]) %>
+    <% target_domain = Digest::SHA2.hexdigest(@target.email.split('@')[1]) %>
+    <td>
+      <code class="<%= 'has-background-color-red' if user_handle == target_handle %>">
+        <%= user_handle[0..7] %>
+      </code>
+    </td>
+    <td>
+      <code class="<%= 'has-background-color-red' if user_domain == target_domain %>">
+        <%= user_domain[0..7] %>
+      </code><br/>
+      <% domain_users = Rails.cache.hget 'user_email_domains', @user.email.split('@')[1] %>
+      <% if domain_users.nil? %>
+        (unknown number of users)
+      <% else %>
+        Used by <%= pluralize(domain_users, 'user') %>
+      <% end %>
+    </td>
+  </tr>
+  <tr>
+    <td><strong><%= @target.rtl_safe_username %></strong></td>
+    <td>
+      <code class="<%= 'has-background-color-red' if user_handle == target_handle %>">
+        <%= target_handle[0..7] %>
+      </code>
+    </td>
+    <td>
+      <code class="<%= 'has-background-color-red' if user_domain == target_domain %>">
+        <%= target_domain[0..7] %>
+      </code><br/>
+      <% domain_users = Rails.cache.hget 'user_email_domains', @target.email.split('@')[1] %>
+      <% if domain_users.nil? %>
+        (unknown number of users)
+      <% else %>
+        Used by <%= pluralize(domain_users, 'user') %>
+      <% end %>
+    </td>
+  </tr>
+  </tbody>
+</table><br/>
+
+<h4>IP addresses</h4>
+
+<% user_current_family, user_current_ip = split_hash_ip(@user.current_sign_in_ip, @user) %>
+<% user_joiner = user_current_family == 'IPv4' ? '.' : ':' %>
+<% target_current_family, target_current_ip = split_hash_ip(@target.current_sign_in_ip, @user) %>
+<% target_joiner = target_current_family == 'IPv4' ? '.' : ':' %>
+<strong>Current sign-in</strong><br/>
+<table class="table is-striped is-with-hover">
+  <thead>
+    <tr>
+      <th>User</th>
+      <th>Family</th>
+      <th>Address</th>
+    </tr>
+  </thead>
+  <tbody>
+    <tr>
+      <td><%= @user.rtl_safe_username %></td>
+      <td><%= user_current_family %></td>
+      <td>
+        <% user_current_ip.map.with_index do |p, i| %>
+          <code class="<%= 'has-background-color-red' if p == target_current_ip[i] %>">
+            <%= p[0..3] %></code><%= user_joiner if i < user_current_ip.length - 1 %>
+        <% end %>
+      </td>
+    </tr>
+    <tr>
+      <td><%= @target.rtl_safe_username %></td>
+      <td><%= target_current_family %></td>
+      <td>
+        <% target_current_ip.map.with_index do |p, i| %>
+          <code class="<%= 'has-background-color-red' if p == user_current_ip[i] %>">
+            <%= p[0..3] %></code><%= target_joiner if i < target_current_ip.length - 1 %>
+        <% end %>
+      </td>
+    </tr>
+  </tbody>
+</table><br/>
+
+<% user_last_family, user_last_ip = split_hash_ip(@user.last_sign_in_ip, @user) %>
+<% user_joiner = user_last_family == 'IPv4' ? '.' : ':' %>
+<% target_last_family, target_last_ip = split_hash_ip(@target.last_sign_in_ip, @user) %>
+<% target_joiner = target_last_family == 'IPv4' ? '.' : ':' %>
+<strong>Last sign-in</strong><br/>
+<table class="table is-striped is-with-hover">
+  <thead>
+  <tr>
+    <th>User</th>
+    <th>Family</th>
+    <th>Address</th>
+  </tr>
+  </thead>
+  <tbody>
+  <tr>
+    <td><%= @user.rtl_safe_username %></td>
+    <td><%= user_last_family %></td>
+    <td>
+      <% user_last_ip.map.with_index do |p, i| %>
+        <code class="<%= 'has-background-color-red' if p == target_last_ip[i] %>">
+          <%= p[0..3] %></code><%= user_joiner if i < user_last_ip.length - 1 %>
+      <% end %>
+    </td>
+  </tr>
+  <tr>
+    <td><%= @target.rtl_safe_username %></td>
+    <td><%= target_last_family %></td>
+    <td>
+      <% target_last_ip.map.with_index do |p, i| %>
+        <code class="<%= 'has-background-color-red' if p == user_last_ip[i] %>">
+          <%= p[0..3] %></code><%= target_joiner if i < target_last_ip.length - 1 %>
+      <% end %>
+    </td>
+  </tr>
+  </tbody>
+</table>

app/views/users/mod.html.erb

@@ -12,6 +12,7 @@
       <li><a href="/warning/log/<%= @user.id %>">warnings and suspensions sent to user</a> <% if @user.community_user.suspended? %>(includes lifting the suspension)<% end %></li>
       <li><a href="/warning/new/<%= @user.id %>">warn or suspend user</a></li>
       <li><%= link_to 'vote summary', mod_vote_summary_path(@user) %></li>
+      <li><%= link_to 'compare PII', mod_pii_correlation_path(@user) %></li>
       <% if current_user.developer %>
        <li><%= link_to 'impersonate', start_impersonating_path(@user), class: 'is-yellow' %></li>
       <% end %>

config/initializers/mime_types.rb

@@ -2,3 +2,5 @@
 
 # Add new mime types for use in respond_to blocks:
 # Mime::Type.register "text/richtext", :rtf
+
+Mime::Type.register('text/html+template', :template)

config/routes.rb

@@ -226,6 +226,7 @@
     get    '/:id/mod/activity-log',     to: 'users#full_log', as: :full_user_log
     post   '/:id/hellban',              to: 'admin#hellban', as: :hellban_user
     get    '/:id/avatar/:size',         to: 'users#avatar', as: :user_auto_avatar
+    get    '/:id/mod/pii',              to: 'moderator#pii_correlation', as: :mod_pii_correlation
   end
 
   post   'notifications/:id/read',         to: 'notifications#read', as: :read_notifications

config/schedule.rb

@@ -34,6 +34,10 @@
   runner 'scripts/run_new_thread_followers_cleanup.rb'
 end
 
+every 7.days, at: '05:00' do
+  runner 'scripts/run_user_stats.rb'
+end
+
 every 6.hours do
   runner 'scripts/recalc_abilities.rb'
 end

lib/namespaced_env_cache.rb

@@ -1,5 +1,10 @@
+require_relative 'redis_cache_hash_methods'
+
 module QPixel
   class NamespacedEnvCache < ActiveSupport::Cache::Store
+    include RedisCacheHashMethods
+    attr_reader :underlying
+
     def initialize(underlying)
       @underlying = underlying
       @getters = {}
@@ -144,6 +149,14 @@ def self.supports_cache_versioning?
       true
     end
 
+    def method_missing(name, *args, **opts, &block)
+      @underlying.send(name, *args, **opts, &block)
+    end
+
+    def respond_to_missing?(name, *)
+      @underlying.respond_to?(name)
+    end
+
     private
 
     # Raises an error if a given collection is not cacheable