diff --git a/lung_cancer_screening/questions/migrations/0012_alter_whenyouquitsmokingresponse_value_incentivised.py b/lung_cancer_screening/questions/migrations/0012_alter_whenyouquitsmokingresponse_value_incentivised.py new file mode 100644 index 00000000..6256301e --- /dev/null +++ b/lung_cancer_screening/questions/migrations/0012_alter_whenyouquitsmokingresponse_value_incentivised.py @@ -0,0 +1,39 @@ +# Generated by Django 6.0.3 on 2026-04-13 15:15 + +import django.core.validators +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('questions', '0011_alter_whenyouquitsmokingresponse_response_set'), + ] + + operations = [ + migrations.AlterField( + model_name='whenyouquitsmokingresponse', + name='value', + field=models.IntegerField(validators=[django.core.validators.MinValueValidator(1)]), + ), + migrations.CreateModel( + name='Incentivised', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('updated_at', models.DateTimeField(auto_now=True)), + ('incentivised_at', models.DateTimeField(auto_now_add=True)), + ('response_set', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='incentivised_record', to='questions.responseset')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='incentivised_records', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'abstract': False, + }, + ), + migrations.AddConstraint( + model_name='incentivised', + constraint=models.UniqueConstraint(fields=('user',), name='questions_incentivised_unique_user'), + ), + ] diff --git a/lung_cancer_screening/questions/migrations/0013_remove_incentivised_questions_incentivised_unique_user_and_more.py b/lung_cancer_screening/questions/migrations/0013_remove_incentivised_questions_incentivised_unique_user_and_more.py new file mode 100644 index 00000000..8d1997b6 --- /dev/null +++ b/lung_cancer_screening/questions/migrations/0013_remove_incentivised_questions_incentivised_unique_user_and_more.py @@ -0,0 +1,21 @@ +# Generated by Django 6.0.4 on 2026-04-24 12:12 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('questions', '0012_alter_whenyouquitsmokingresponse_value_incentivised'), + ] + + operations = [ + migrations.RemoveConstraint( + model_name='incentivised', + name='questions_incentivised_unique_user', + ), + migrations.AddConstraint( + model_name='incentivised', + constraint=models.UniqueConstraint(fields=('user',), name='unique_incentive_per_user'), + ), + ] diff --git a/lung_cancer_screening/questions/models/__init__.py b/lung_cancer_screening/questions/models/__init__.py index 97c0536a..c08f3b0a 100644 --- a/lung_cancer_screening/questions/models/__init__.py +++ b/lung_cancer_screening/questions/models/__init__.py @@ -12,6 +12,7 @@ from .gender_response import GenderResponse # noqa: F401 from .have_you_ever_smoked_response import HaveYouEverSmokedResponse # noqa: F401 from .height_response import HeightResponse # noqa: F401 +from .incentivised import Incentivised # noqa: F401 from .periods_when_you_stopped_smoking_response import PeriodsWhenYouStoppedSmokingResponse # noqa: F401 from .relatives_age_when_diagnosed_response import RelativesAgeWhenDiagnosedResponse # noqa: F401 from .respiratory_conditions_response import RespiratoryConditionsResponse # noqa: F401 diff --git a/lung_cancer_screening/questions/models/incentivised.py b/lung_cancer_screening/questions/models/incentivised.py new file mode 100644 index 00000000..133712a9 --- /dev/null +++ b/lung_cancer_screening/questions/models/incentivised.py @@ -0,0 +1,20 @@ + +from django.db import models + +from .base import BaseModel +from .response_set import ResponseSet + + +class Incentivised(BaseModel): + user = models.ForeignKey('questions.User', on_delete=models.CASCADE, related_name='incentivised_records') + response_set = models.OneToOneField(ResponseSet, on_delete=models.CASCADE, related_name='incentivised_record') + + incentivised_at = models.DateTimeField(auto_now_add=True) + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["user"], + name="unique_incentive_per_user", + ) + ] diff --git a/scripts/sql/export_email_addresses_for_incentives.sql b/scripts/sql/export_email_addresses_for_incentives.sql new file mode 100644 index 00000000..3e46271e --- /dev/null +++ b/scripts/sql/export_email_addresses_for_incentives.sql @@ -0,0 +1,77 @@ +-- Commands are written on one line for psql execution, can be formatted for readability when running in a SQL client. + +-- This script performs the following steps: +-- 1 - imports new data from csv file. The csv file is expected to be a full export of all partner records, but only new records will be inserted into the permanent table. +-- 2 - creates a list of email addresses of participants eligible for incentives +-- 3 - updates participants that have been exported so that they are timestamped as having received an incentive + +-- Incentive eligibility is based on: +-- - having submitted the questionnaire +-- - not having already received an incentive +-- - Online submitted_at date is earlier than telephone questionnaire date_conducted. + +-- Partner import strategy: +-- The partner sends a full CSV export every week. Rather than replacing all data each run, we insert only +-- new records using a UNIQUE index on (nhs_number, conducted_at) and ON CONFLICT DO NOTHING. +-- Records removed from the partner's source will NOT be deleted. + + +-- Steps to follow: +-- 1. Log into AVD. +-- 2. Upload csv file to AVD +-- 3. Find file in RemoteVirtualDrive and copy to accessible location for psql COPY command. +-- 4. PATH_TO_FILE - search for this and replace with actual file path. +-- 5. Login to DB in AVD. + + +-- ============================================================ +-- RUN ONCE: Create permanent partner import table +-- Only run this block on first setup. The unique constraint on +-- (nhs_number, conducted_at) prevents duplicate rows being +-- inserted on subsequent weekly loads. +-- ============================================================ +CREATE TABLE IF NOT EXISTS inhealth_partner_data (nhs_number TEXT, date_of_birth TEXT, date_conducted TEXT, conducted_at TIMESTAMPTZ, smoking_status TEXT, average_cigarettes_per_day_while_smoking TEXT, duration_smoked_years TEXT, years_since_quitting_smoking TEXT, height_measurement_type TEXT, height_measurement_value_metric_cm TEXT, weight_measurement_type TEXT, weight_measurement_value_metric_kg TEXT, previous_respiratory_diagnosis TEXT, personal_history_of_previous_cancer TEXT, family_history_of_lung_cancer TEXT, personal_history_of_asthma TEXT, asbestos_exposure_from_job_or_activity TEXT, education TEXT, ethnicity TEXT, plco_lung_cancer_risk_score TEXT, llp_lung_cancer_risk_score TEXT, created_at TIMESTAMPTZ NOT NULL DEFAULT now(), CONSTRAINT uq_partner_nhs_conducted_at UNIQUE (nhs_number, conducted_at)); + + +-- ============================================================ +-- RUN WEEKLY: Load new partner records from CSV +-- Step 1: Import CSV into a temporary staging table. +-- Step 2: Insert only rows where (nhs_number, conducted_at) +-- do not already exist in the permanent table. +-- Existing rows are silently skipped (DO NOTHING). +-- ============================================================ + +-- Step 1: Create staging table and load CSV +CREATE TEMP TABLE tmp_incentive_partner_staging (nhs_number TEXT, date_of_birth TEXT, date_conducted TEXT, smoking_status TEXT, average_cigarettes_per_day_while_smoking TEXT, duration_smoked_years TEXT, years_since_quitting_smoking TEXT, height_measurement_type TEXT, height_measurement_value_metric_cm TEXT, weight_measurement_type TEXT, weight_measurement_value_metric_kg TEXT, previous_respiratory_diagnosis TEXT, personal_history_of_previous_cancer TEXT, family_history_of_lung_cancer TEXT, personal_history_of_asthma TEXT, asbestos_exposure_from_job_or_activity TEXT, education TEXT, ethnicity TEXT, plco_lung_cancer_risk_score TEXT, llp_lung_cancer_risk_score TEXT); + +-- Copy data from file into staging table - update PATH_TO_FILE before running + +\copy tmp_incentive_partner_staging (nhs_number, date_of_birth, date_conducted, smoking_status, average_cigarettes_per_day_while_smoking, duration_smoked_years, years_since_quitting_smoking, height_measurement_type, height_measurement_value_metric_cm, weight_measurement_type, weight_measurement_value_metric_kg, previous_respiratory_diagnosis, personal_history_of_previous_cancer, family_history_of_lung_cancer, personal_history_of_asthma, asbestos_exposure_from_job_or_activity, education, ethnicity, plco_lung_cancer_risk_score, llp_lung_cancer_risk_score) FROM 'PATH_TO_FILE' WITH (FORMAT csv, HEADER true); + +-- Step 2: Insert all rows from staging. Existing rows with the same (nhs_number, conducted_at) +-- are skipped. +INSERT INTO inhealth_partner_data (nhs_number, date_of_birth, date_conducted, conducted_at, smoking_status, average_cigarettes_per_day_while_smoking, duration_smoked_years, years_since_quitting_smoking, height_measurement_type, height_measurement_value_metric_cm, weight_measurement_type, weight_measurement_value_metric_kg, previous_respiratory_diagnosis, personal_history_of_previous_cancer, family_history_of_lung_cancer, personal_history_of_asthma, asbestos_exposure_from_job_or_activity, education, ethnicity, plco_lung_cancer_risk_score, llp_lung_cancer_risk_score) SELECT nhs_number, date_of_birth, date_conducted, to_timestamp(NULLIF(date_conducted, ''), 'DD/MM/YYYY HH24:MI')::timestamptz, smoking_status, average_cigarettes_per_day_while_smoking, duration_smoked_years, years_since_quitting_smoking, height_measurement_type, height_measurement_value_metric_cm, weight_measurement_type, weight_measurement_value_metric_kg, previous_respiratory_diagnosis, personal_history_of_previous_cancer, family_history_of_lung_cancer, personal_history_of_asthma, asbestos_exposure_from_job_or_activity, education, ethnicity, plco_lung_cancer_risk_score, llp_lung_cancer_risk_score FROM tmp_incentive_partner_staging ON CONFLICT (nhs_number, conducted_at) DO NOTHING; + +-- Delete temporary staging table +DROP TABLE IF EXISTS tmp_incentive_partner_staging; + + +-- TRANSACTION START for exporting eligible participants for incentives and updating incentivised table. +-- Update PATH_TO_EXPORT_FILE before running. + +\r +BEGIN; +CREATE TEMP TABLE tmp_eligible_incentive_export AS WITH canonical_users AS (SELECT DISTINCT ON (nhs_number) id, email, given_name, family_name, nhs_number FROM questions_user WHERE nhs_number IS NOT NULL ORDER BY nhs_number ASC, created_at DESC) SELECT DISTINCT ON (cu.nhs_number) cu.id AS user_id, qrs.id AS response_set_id, cu.email, cu.given_name, cu.family_name FROM canonical_users cu JOIN questions_responseset qrs ON qrs.user_id = cu.id JOIN inhealth_partner_data ipd ON ipd.nhs_number = cu.nhs_number WHERE ipd.conducted_at > qrs.submitted_at::timestamptz AND NOT EXISTS (SELECT 1 FROM questions_incentivised qi JOIN questions_user iu ON iu.id = qi.user_id WHERE iu.nhs_number = cu.nhs_number) ORDER BY cu.nhs_number ASC, qrs.submitted_at DESC, qrs.id DESC; +\copy (SELECT email, given_name, family_name FROM tmp_eligible_incentive_export ORDER BY family_name) TO 'PATH_TO_EXPORT_FILE' WITH (FORMAT csv, HEADER true); +INSERT INTO questions_incentivised (created_at, updated_at, incentivised_at, user_id, response_set_id) SELECT now(), now(), now(), user_id, response_set_id FROM tmp_eligible_incentive_export; +SELECT count(*) AS rows_exported_and_marked FROM tmp_eligible_incentive_export; + +-- If happy with the Select result type COMMIT; if not, ROLLBACK; to undo changes; + +-- COMMIT; +-- ROLLBACK; + +-- TRANSACTION END + +-- DELETE temporary export table +DROP TABLE IF EXISTS tmp_eligible_incentive_export;