diff --git a/backend/app/controllers/admin/analytics_controller.rb b/backend/app/controllers/admin/analytics_controller.rb new file mode 100644 index 0000000..408c40f --- /dev/null +++ b/backend/app/controllers/admin/analytics_controller.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +module Admin + class AnalyticsController < Admin::BaseController + def index + @filters = { + from: parse_date(params[:from]) || 7.days.ago.to_date, + to: parse_date(params[:to]) || Time.zone.today, + device: params[:device].presence + } + + scope = AnalyticsPageStat.where(day: @filters[:from]..@filters[:to]) + scope = scope.where(device: @filters[:device]) if @filters[:device].present? && AnalyticsEvent::DEVICES.include?(@filters[:device]) + + click_scope = AnalyticsPageCell.where(day: @filters[:from]..@filters[:to]) + click_scope = click_scope.where(device: @filters[:device]) if @filters[:device].present? && AnalyticsEvent::DEVICES.include?(@filters[:device]) + + clicks_by_path = click_scope.group(:page_path).sum(:click_count) + pageviews_by_path = scope.group(:page_path).sum(:pageview_count) + scroll_samples_by_path = scope.group(:page_path).sum(:scroll_samples) + scroll_sum_by_path = scope.group(:page_path).sum(:scroll_sum_pct) + max_scroll_by_path = scope.group(:page_path).maximum(:max_scroll_pct) + + paths = (pageviews_by_path.keys + clicks_by_path.keys).uniq + @pages = paths.map do |path| + samples = scroll_samples_by_path[path].to_i + { + page_path: path, + pageviews: pageviews_by_path[path].to_i, + clicks: clicks_by_path[path].to_i, + scroll_samples: samples, + scroll_sum: scroll_sum_by_path[path].to_i, + max_scroll: max_scroll_by_path[path].to_i + } + end.sort_by { |r| [-r[:pageviews], -r[:clicks], r[:page_path]] } + end + + def show + @page_path = params[:page_path].to_s + redirect_to admin_analytics_path, alert: t("admin.analytics.missing_path") and return if @page_path.blank? + + @filters = { + from: parse_date(params[:from]) || 7.days.ago.to_date, + to: parse_date(params[:to]) || Time.zone.today, + device: params[:device].presence + } + + cells = AnalyticsPageCell.where(page_path: @page_path, day: @filters[:from]..@filters[:to]) + cells = cells.where(device: @filters[:device]) if @filters[:device].present? && AnalyticsEvent::DEVICES.include?(@filters[:device]) + @cells = cells.group(:cell_x, :cell_y).sum(:click_count) + @max_clicks = @cells.values.max.to_i + + stats = AnalyticsPageStat.where(page_path: @page_path, day: @filters[:from]..@filters[:to]) + stats = stats.where(device: @filters[:device]) if @filters[:device].present? && AnalyticsEvent::DEVICES.include?(@filters[:device]) + @pageviews = stats.sum(:pageview_count) + @scroll_samples = stats.sum(:scroll_samples) + @scroll_sum = stats.sum(:scroll_sum_pct) + @max_scroll = stats.maximum(:max_scroll_pct).to_i + @avg_scroll = @scroll_samples.positive? ? (@scroll_sum.to_f / @scroll_samples).round : 0 + @grid = AnalyticsPageCell::GRID_SIZE + end + + private + + def parse_date(value) + return nil if value.blank? + + Date.parse(value.to_s) + rescue ArgumentError, TypeError + nil + end + end +end diff --git a/backend/app/controllers/analytics/events_controller.rb b/backend/app/controllers/analytics/events_controller.rb new file mode 100644 index 0000000..33b8bba --- /dev/null +++ b/backend/app/controllers/analytics/events_controller.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Analytics + class EventsController < ActionController::API + def create + payload = parse_payload + result = Analytics::Ingest.new(events: payload, remote_ip: request.remote_ip).call + + if result.rate_limited + return head :too_many_requests + end + + render json: { accepted: result.accepted, rejected: result.rejected }, status: :accepted + end + + private + + def parse_payload + body = request.request_parameters + return body["events"] if body.is_a?(Hash) && body["events"].is_a?(Array) + return body if body.is_a?(Array) + + raw = request.raw_post + return [] if raw.blank? + + parsed = JSON.parse(raw) + return parsed["events"] if parsed.is_a?(Hash) && parsed["events"].is_a?(Array) + return parsed if parsed.is_a?(Array) + + [] + rescue JSON::ParserError + [] + end + end +end diff --git a/backend/app/jobs/analytics/aggregate_job.rb b/backend/app/jobs/analytics/aggregate_job.rb new file mode 100644 index 0000000..eafd9b2 --- /dev/null +++ b/backend/app/jobs/analytics/aggregate_job.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Analytics + class AggregateJob < ApplicationJob + queue_as :default + + def perform(purge: false) + redis = Redis.new(url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0")) + lock_key = "analytics:aggregate:lock" + acquired = redis.set(lock_key, "1", nx: true, ex: 120) + return unless acquired + + begin + service = Analytics::Aggregate.new + service.call + service.purge_old! if purge + ensure + redis.del(lock_key) + end + rescue StandardError => e + Rails.logger.error("[analytics] aggregate failed: #{e.class} #{e.message}") + raise + end + end +end diff --git a/backend/app/models/analytics_event.rb b/backend/app/models/analytics_event.rb new file mode 100644 index 0000000..a1bde6c --- /dev/null +++ b/backend/app/models/analytics_event.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class AnalyticsEvent < ApplicationRecord + self.table_name = "analytics_events" + + EVENT_TYPES = %w[click scroll pageview].freeze + DEVICES = %w[mobile tablet desktop].freeze + + validates :page_path, presence: true + validates :device, inclusion: { in: DEVICES } + validates :event_type, inclusion: { in: EVENT_TYPES } + validates :occurred_at, presence: true +end diff --git a/backend/app/models/analytics_page_cell.rb b/backend/app/models/analytics_page_cell.rb new file mode 100644 index 0000000..f590204 --- /dev/null +++ b/backend/app/models/analytics_page_cell.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +class AnalyticsPageCell < ApplicationRecord + self.table_name = "analytics_page_cells" + + GRID_SIZE = 40 + + validates :day, :page_path, :device, :cell_x, :cell_y, presence: true + validates :device, inclusion: { in: AnalyticsEvent::DEVICES } + validates :cell_x, :cell_y, inclusion: { in: 0...(GRID_SIZE) } + validates :click_count, numericality: { greater_than_or_equal_to: 0 } +end diff --git a/backend/app/models/analytics_page_stat.rb b/backend/app/models/analytics_page_stat.rb new file mode 100644 index 0000000..af8d421 --- /dev/null +++ b/backend/app/models/analytics_page_stat.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +class AnalyticsPageStat < ApplicationRecord + self.table_name = "analytics_page_stats" + + validates :day, :page_path, :device, presence: true + validates :device, inclusion: { in: AnalyticsEvent::DEVICES } + + def avg_scroll_pct + return 0 if scroll_samples.to_i <= 0 + + (scroll_sum_pct.to_f / scroll_samples).round + end +end diff --git a/backend/app/services/analytics/aggregate.rb b/backend/app/services/analytics/aggregate.rb new file mode 100644 index 0000000..d768f49 --- /dev/null +++ b/backend/app/services/analytics/aggregate.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +module Analytics + class Aggregate + BATCH = 500 + + def call + loop do + ids = AnalyticsEvent.order(:occurred_at).limit(BATCH).pluck(:id) + break if ids.empty? + + events = AnalyticsEvent.where(id: ids).to_a + apply_clicks(events.select { |e| e.event_type == "click" }) + apply_stats(events.reject { |e| e.event_type == "click" }) + AnalyticsEvent.where(id: ids).delete_all + end + end + + def purge_old!(retention: 30.days) + # Raw should already be empty after aggregate; keep as safety net. + AnalyticsEvent.where("occurred_at < ?", retention.ago).delete_all + AnalyticsPageCell.where("day < ?", retention.ago.to_date).delete_all + AnalyticsPageStat.where("day < ?", retention.ago.to_date).delete_all + end + + private + + def apply_clicks(events) + return if events.empty? + + grid = AnalyticsPageCell::GRID_SIZE + grouped = Hash.new(0) + events.each do |e| + next if e.x_pct.nil? || e.y_pct.nil? + + cell_x = [[(e.x_pct.to_f / 100 * grid).floor, grid - 1].min, 0].max + cell_y = [[(e.y_pct.to_f / 100 * grid).floor, grid - 1].min, 0].max + key = [e.occurred_at.in_time_zone.to_date, e.page_path, e.device, cell_x, cell_y] + grouped[key] += 1 + end + + now = Time.current + grouped.each do |(day, page_path, device, cell_x, cell_y), count| + cell = AnalyticsPageCell.find_or_initialize_by( + day: day, page_path: page_path, device: device, cell_x: cell_x, cell_y: cell_y + ) + cell.click_count = cell.click_count.to_i + count + cell.created_at ||= now + cell.updated_at = now + cell.save! + end + end + + def apply_stats(events) + return if events.empty? + + grouped = {} + events.each do |e| + day = e.occurred_at.in_time_zone.to_date + key = [day, e.page_path, e.device] + bucket = grouped[key] ||= { pageviews: 0, scroll_samples: 0, scroll_sum: 0, max_scroll: 0 } + case e.event_type + when "pageview" + bucket[:pageviews] += 1 + when "scroll" + pct = e.scroll_pct.to_f.round + bucket[:scroll_samples] += 1 + bucket[:scroll_sum] += pct + bucket[:max_scroll] = [bucket[:max_scroll], pct].max + end + end + + now = Time.current + grouped.each do |(day, page_path, device), vals| + stat = AnalyticsPageStat.find_or_initialize_by(day: day, page_path: page_path, device: device) + stat.pageview_count = stat.pageview_count.to_i + vals[:pageviews] + stat.scroll_samples = stat.scroll_samples.to_i + vals[:scroll_samples] + stat.scroll_sum_pct = stat.scroll_sum_pct.to_i + vals[:scroll_sum] + stat.max_scroll_pct = [stat.max_scroll_pct.to_i, vals[:max_scroll]].max + stat.created_at ||= now + stat.updated_at = now + stat.save! + end + end + end +end diff --git a/backend/app/services/analytics/ingest.rb b/backend/app/services/analytics/ingest.rb new file mode 100644 index 0000000..3139f4f --- /dev/null +++ b/backend/app/services/analytics/ingest.rb @@ -0,0 +1,118 @@ +# frozen_string_literal: true + +module Analytics + class Ingest + MAX_BATCH = 50 + RATE_LIMIT_PER_MINUTE = 60 + + Result = Struct.new(:accepted, :rejected, :rate_limited, keyword_init: true) + + def initialize(events:, remote_ip:) + @events = Array(events) + @remote_ip = remote_ip.to_s.presence || "unknown" + end + + def call + return Result.new(accepted: 0, rejected: 0, rate_limited: true) if rate_limited? + + slice = @events.first(MAX_BATCH) + accepted = 0 + rejected = 0 + rows = [] + + slice.each do |raw| + attrs = build_attrs(raw) + if attrs + rows << attrs + accepted += 1 + else + rejected += 1 + end + end + + AnalyticsEvent.insert_all(rows) if rows.any? + Analytics::Aggregate.new.call if rows.any? + + Result.new(accepted: accepted, rejected: rejected + (@events.size - slice.size), rate_limited: false) + end + + private + + def rate_limited? + key = "analytics:ingest:#{@remote_ip}" + redis = Redis.new(url: ENV.fetch("REDIS_URL", "redis://localhost:6379/0")) + count = redis.incr(key) + redis.expire(key, 60) if count == 1 + count > RATE_LIMIT_PER_MINUTE + rescue StandardError => e + Rails.logger.warn("[analytics] rate limit unavailable: #{e.class}") + false + end + + def build_attrs(raw) + data = raw.is_a?(Hash) ? raw.with_indifferent_access : {} + event_type = data[:type].to_s.presence || data[:event_type].to_s + return nil unless AnalyticsEvent::EVENT_TYPES.include?(event_type) + + device = data[:device].to_s + return nil unless AnalyticsEvent::DEVICES.include?(device) + + page_path = PathNormalizer.normalize(data[:path] || data[:page_path]) + return nil if PathNormalizer.excluded?(page_path) + return nil if page_path.length > 200 + + occurred_at = parse_time(data[:ts] || data[:occurred_at]) || Time.current + attrs = { + page_path: page_path, + device: device, + event_type: event_type, + occurred_at: occurred_at, + x_pct: nil, + y_pct: nil, + scroll_pct: nil, + created_at: Time.current, + updated_at: Time.current + } + + case event_type + when "click" + x = clamp_pct(data[:x] || data[:x_pct]) + y = clamp_pct(data[:y] || data[:y_pct]) + return nil if x.nil? || y.nil? + + attrs[:x_pct] = x + attrs[:y_pct] = y + when "scroll" + scroll = clamp_pct(data[:scroll] || data[:scroll_pct]) + return nil if scroll.nil? + + attrs[:scroll_pct] = scroll + when "pageview" + # no extra fields + end + + attrs + end + + def clamp_pct(value) + return nil if value.nil? + + n = Float(value) + return nil if n.nan? || n.infinite? + return nil if n.negative? || n > 100 + + n.round(2) + rescue ArgumentError, TypeError + nil + end + + def parse_time(value) + return value if value.is_a?(Time) || value.is_a?(ActiveSupport::TimeWithZone) + return Time.zone.at(value.to_i / 1000.0) if value.is_a?(Numeric) || value.to_s.match?(/\A\d{10,13}\z/) + + Time.zone.parse(value.to_s) + rescue ArgumentError, TypeError + nil + end + end +end diff --git a/backend/app/services/analytics/path_normalizer.rb b/backend/app/services/analytics/path_normalizer.rb new file mode 100644 index 0000000..9ae5ab6 --- /dev/null +++ b/backend/app/services/analytics/path_normalizer.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Analytics + class PathNormalizer + UUID_RE = /\A[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\z/i + NUMERIC_RE = /\A\d{6,}\z/ + TOKENISH_RE = /\A[A-Za-z0-9_\-]{22,}\z/ + + EXCLUDED_PREFIXES = %w[/admin /regia /analytics /cable /api /webhooks /internal].freeze + + class << self + def normalize(raw_path) + path = raw_path.to_s.split("?", 2).first.to_s + path = "/" if path.blank? + path = "/#{path}" unless path.start_with?("/") + path = path.gsub(%r{/{2,}}, "/").chomp("/") + path = "/" if path.blank? + + parts = path.split("/").map do |segment| + next segment if segment.blank? + next ":id" if UUID_RE.match?(segment) + next ":id" if NUMERIC_RE.match?(segment) + next ":token" if TOKENISH_RE.match?(segment) && segment.match?(/[A-Z]/) + + segment + end + normalized = parts.join("/") + normalized = "/" if normalized.blank? + normalized + end + + def excluded?(raw_or_normalized) + path = normalize(raw_or_normalized) + return true if EXCLUDED_PREFIXES.any? { |p| path == p || path.start_with?("#{p}/") } + return true if path.match?(%r{\A/live/[^/]+}) + return true if path.match?(%r{\A/replay/[^/]+}) + + false + end + end + end +end diff --git a/backend/app/views/admin/analytics/index.html.erb b/backend/app/views/admin/analytics/index.html.erb new file mode 100644 index 0000000..043d5eb --- /dev/null +++ b/backend/app/views/admin/analytics/index.html.erb @@ -0,0 +1,70 @@ +<% content_for :body_class, "admin-body" %> + +
<%= t("admin.analytics.index.lead") %>
+| <%= t("admin.analytics.index.table.path") %> | +<%= t("admin.analytics.index.table.pageviews") %> | +<%= t("admin.analytics.index.table.clicks") %> | +<%= t("admin.analytics.index.table.avg_scroll") %> | +<%= t("admin.analytics.index.table.max_scroll") %> | ++ |
|---|---|---|---|---|---|
<%= row[:page_path] %> |
+ <%= row[:pageviews] %> | +<%= row[:clicks] %> | +<%= avg %>% | +<%= row[:max_scroll] %>% | ++ <%= link_to t("admin.analytics.index.heatmap"), + admin_analytics_page_path(page_path: row[:page_path], from: @filters[:from], to: @filters[:to], device: @filters[:device]) %> + | +
<%= t("admin.analytics.index.none") %>
+ <% end %> ++ <%= link_to t("admin.analytics.show.back"), admin_analytics_path(from: @filters[:from], to: @filters[:to], device: @filters[:device]) %> +
+<%= @page_path %>
+ <%= t("admin.analytics.show.scroll_hint", avg: @avg_scroll, max: @max_scroll) %> +
+<%= t("admin.analytics.show.no_clicks") %>
+ <% end %> +