Feature/raw data archive

.app_version

@@ -1 +1 @@
-0.36.2
+0.36.3

CHANGELOG.md

@@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+# [0.36.3] - Unreleased
+
+## Added
+
+- Setting `ARCHIVE_RAW_DATA` env var to true will enable monthly raw data archiving for all users. It will look for points older than 2 months with `raw_data` column not empty and create a zip archive containing raw data files for each month. After successful archiving, raw data will be removed from the database to save space. Monthly archiving job is being run every day at 2:00 AM. Default env var value is false.
+
 
 # [0.36.2] - 2025-12-06
 

Gemfile

@@ -14,6 +14,7 @@ gem 'bootsnap', require: false
 gem 'chartkick'
 gem 'data_migrate'
 gem 'devise'
+gem 'foreman'
 gem 'geocoder', github: 'Freika/geocoder', branch: 'master'
 gem 'gpx'
 gem 'groupdate'
@@ -55,7 +56,7 @@ gem 'stimulus-rails'
 gem 'tailwindcss-rails', '= 3.3.2'
 gem 'turbo-rails', '>= 2.0.17'
 gem 'tzinfo-data', platforms: %i[mingw mswin x64_mingw jruby]
-gem 'foreman'
+gem 'with_advisory_lock'
 
 group :development, :test, :staging do
   gem 'brakeman', require: false

Gemfile.lock

@@ -620,6 +620,9 @@ GEM
       base64
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
+    with_advisory_lock (7.0.2)
+      activerecord (>= 7.2)
+      zeitwerk (>= 2.7)
     xpath (3.2.0)
       nokogiri (~> 1.8)
     zeitwerk (2.7.3)
@@ -703,6 +706,7 @@ DEPENDENCIES
   turbo-rails (>= 2.0.17)
   tzinfo-data
   webmock
+  with_advisory_lock
 
 RUBY VERSION
    ruby 3.4.6p54

app/jobs/family/invitations/cleanup_job.rb

@@ -4,6 +4,8 @@ class Family::Invitations::CleanupJob < ApplicationJob
   queue_as :families
 
   def perform
+    return unless DawarichSettings.family_feature_enabled?
+
     Rails.logger.info 'Starting family invitations cleanup'
 
     expired_count = Family::Invitation.where(status: :pending)

app/jobs/points/raw_data/archive_job.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Points
+  module RawData
+    class ArchiveJob < ApplicationJob
+      queue_as :archival
+
+      def perform
+        return unless ENV['ARCHIVE_RAW_DATA'] == 'true'
+
+        stats = Points::RawData::Archiver.new.call
+
+        Rails.logger.info("Archive job complete: #{stats}")
+      rescue StandardError => e
+        ExceptionReporter.call(e, 'Points raw data archival job failed')
+
+        raise
+      end
+    end
+  end
+end

app/jobs/points/raw_data/re_archive_month_job.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Points
+  module RawData
+    class ReArchiveMonthJob < ApplicationJob
+      queue_as :archival
+
+      def perform(user_id, year, month)
+        Rails.logger.info("Re-archiving #{user_id}/#{year}/#{month} (retrospective import)")
+
+        Points::RawData::Archiver.new.archive_specific_month(user_id, year, month)
+      rescue StandardError => e
+        ExceptionReporter.call(e, "Re-archival job failed for #{user_id}/#{year}/#{month}")
+
+        raise
+      end
+    end
+  end
+end

app/models/concerns/archivable.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module Archivable
+  extend ActiveSupport::Concern
+
+  included do
+    belongs_to :raw_data_archive,
+               class_name: 'Points::RawDataArchive',
+               optional: true
+
+    scope :archived, -> { where(raw_data_archived: true) }
+    scope :not_archived, -> { where(raw_data_archived: false) }
+    scope :with_archived_raw_data, lambda {
+      includes(raw_data_archive: { file_attachment: :blob })
+    }
+  end
+
+  # Main method: Get raw_data with fallback to archive
+  # Use this instead of point.raw_data when you need archived data
+  def raw_data_with_archive
+    return raw_data if raw_data.present? || !raw_data_archived?
+
+    fetch_archived_raw_data
+  end
+
+  # Restore archived data back to database column
+  def restore_raw_data!(value)
+    update!(
+      raw_data: value,
+      raw_data_archived: false,
+      raw_data_archive_id: nil
+    )
+  end
+
+  private
+
+  def fetch_archived_raw_data
+    # Check temporary restore cache first (for migrations)
+    cached = check_temporary_restore_cache
+    return cached if cached
+
+    fetch_from_archive_file
+  rescue StandardError => e
+    handle_archive_fetch_error(e)
+  end
+
+  def check_temporary_restore_cache
+    return nil unless respond_to?(:timestamp)
+
+    recorded_time = Time.at(timestamp)
+    cache_key = "raw_data:temp:#{user_id}:#{recorded_time.year}:#{recorded_time.month}:#{id}"
+    Rails.cache.read(cache_key)
+  end
+
+  def fetch_from_archive_file
+    return {} unless raw_data_archive&.file&.attached?
+
+    # Download and search through JSONL
+    compressed_content = raw_data_archive.file.blob.download
+    io = StringIO.new(compressed_content)
+    gz = Zlib::GzipReader.new(io)
+
+    result = nil
+    gz.each_line do |line|
+      data = JSON.parse(line)
+      if data['id'] == id
+        result = data['raw_data']
+        break
+      end
+    end
+
+    gz.close
+    result || {}
+  end
+
+  def handle_archive_fetch_error(error)
+    ExceptionReporter.call(error, "Failed to fetch archived raw_data for Point ID #{id}")
+
+    {} # Graceful degradation
+  end
+end

app/models/point.rb

@@ -3,6 +3,7 @@
 class Point < ApplicationRecord
   include Nearable
   include Distanceable
+  include Archivable
 
   belongs_to :import, optional: true, counter_cache: true
   belongs_to :visit, optional: true

app/models/points/raw_data_archive.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Points
+  class RawDataArchive < ApplicationRecord
+    self.table_name = 'points_raw_data_archives'
+
+    belongs_to :user
+    has_many :points, dependent: :nullify
+
+    has_one_attached :file
+
+    validates :year, :month, :chunk_number, :point_count, presence: true
+    validates :year, numericality: { greater_than: 1970, less_than: 2100 }
+    validates :month, numericality: { greater_than_or_equal_to: 1, less_than_or_equal_to: 12 }
+    validates :chunk_number, numericality: { greater_than: 0 }
+    validates :point_ids_checksum, presence: true
+
+    scope :for_month, lambda { |user_id, year, month|
+      where(user_id: user_id, year: year, month: month)
+        .order(:chunk_number)
+    }
+
+    scope :recent, -> { where('archived_at > ?', 30.days.ago) }
+    scope :old, -> { where('archived_at < ?', 1.year.ago) }
+
+    def month_display
+      Date.new(year, month, 1).strftime('%B %Y')
+    end
+
+    def filename
+      "raw_data_archives/#{user_id}/#{year}/#{format('%02d', month)}/#{format('%03d', chunk_number)}.jsonl.gz"
+    end
+
+    def size_mb
+      return 0 unless file.attached?
+
+      (file.blob.byte_size / 1024.0 / 1024.0).round(2)
+    end
+  end
+end

app/models/user.rb

@@ -20,6 +20,7 @@ class User < ApplicationRecord # rubocop:disable Metrics/ClassLength
   has_many :tags,           dependent: :destroy
   has_many :trips,  dependent: :destroy
   has_many :tracks, dependent: :destroy
+  has_many :raw_data_archives, class_name: 'Points::RawDataArchive', dependent: :destroy
 
   after_create :create_api_key
   after_commit :activate, on: :create, if: -> { DawarichSettings.self_hosted? }

app/services/exception_reporter.rb

@@ -2,10 +2,14 @@
 
 class ExceptionReporter
   def self.call(exception, human_message = 'Exception reported')
-    return unless DawarichSettings.self_hosted?
+    return if DawarichSettings.self_hosted?
 
-    Rails.logger.error "#{human_message}: #{exception.message}"
-
-    Sentry.capture_exception(exception)
+    if exception.is_a?(Exception)
+      Rails.logger.error "#{human_message}: #{exception.message}"
+      Sentry.capture_exception(exception)
+    else
+      Rails.logger.error "#{exception}: #{human_message}"
+      Sentry.capture_message("#{exception}: #{human_message}")
+    end
   end
 end