From 67244a14eaea3158fdb1fcf8b4fd9bcb342e8e68 Mon Sep 17 00:00:00 2001 From: Mael Date: Tue, 21 Feb 2023 15:56:49 +0100 Subject: [PATCH 01/16] =?UTF-8?q?Test=20de=20la=20r=C3=A9cup=C3=A9ration?= =?UTF-8?q?=20de=20stats=20priv=C3=A9es=20en=20mode=20connect=C3=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../components/stats/content/IframeFigures.js | 6 ++++-- .../components/stats/content/ScoreFromURL.js | 1 + .../components/stats/content/sources/Table.js | 3 ++- source/components/stats/matomo.js | 21 ++++++++++++------- webpack.dev.js | 1 + webpack.prod.js | 1 + 6 files changed, 23 insertions(+), 10 deletions(-) diff --git a/source/components/stats/content/IframeFigures.js b/source/components/stats/content/IframeFigures.js index cc568f3a5e..7f9e8cc415 100644 --- a/source/components/stats/content/IframeFigures.js +++ b/source/components/stats/content/IframeFigures.js @@ -81,14 +81,16 @@ const Text = styled.p` ` export default function IframeFigures(props) { + const { i18n } = useTranslation() + const currentLangInfos = getCurrentLangInfos(i18n) + if (!props.pages.length || !props.activePages.length) return const [iframes, activeIframes] = props.pages && props.activePages && getIframeRate(props.pages, props.activePages) + const [iframePages, totalIframe] = props.pages && getIdentifiedIframes(props.pages) - const { i18n } = useTranslation() - const currentLangInfos = getCurrentLangInfos(i18n) return (
diff --git a/source/components/stats/content/ScoreFromURL.js b/source/components/stats/content/ScoreFromURL.js index db0a3ea7b9..3dfb8aa234 100644 --- a/source/components/stats/content/ScoreFromURL.js +++ b/source/components/stats/content/ScoreFromURL.js @@ -39,6 +39,7 @@ const Text = styled.p` ` export default function ScoreFromURL(props) { + if (!props.pages.length) return const scores = props.pages && getScores(props.pages) // we exclude high number of visits on same urls (corresponds to average test score ?) // pb : if a user goes to end page, come back to test, change test score, come back to end page, 2 score values are taken into account instead of one. diff --git a/source/components/stats/content/sources/Table.js b/source/components/stats/content/sources/Table.js index efbef4b751..a0c8fb583b 100644 --- a/source/components/stats/content/sources/Table.js +++ b/source/components/stats/content/sources/Table.js @@ -1,5 +1,5 @@ -import styled from 'styled-components' import { Trans } from 'react-i18next' +import styled from 'styled-components' import Tile from '../../utils/Tile' @@ -75,6 +75,7 @@ export default function Table(props) { % {props.data && + props.data.length > 0 && props.data.map( (line, index) => (!props.limit || index < props.limit) && ( diff --git a/source/components/stats/matomo.js b/source/components/stats/matomo.js index 7978ea65c5..ca3ea40087 100644 --- a/source/components/stats/matomo.js +++ b/source/components/stats/matomo.js @@ -1,15 +1,17 @@ -import { useQuery } from 'react-query' import axios from 'axios' +import { useQuery } from 'react-query' const idSite = 153 +const authToken = MATOMO_TOKEN + export const useChart = ({ chartPeriod, chartDate }) => useQuery( ['chart', chartPeriod, chartDate], () => axios .get( - `https://stats.data.gouv.fr/?module=API&date=last${chartDate}&period=${chartPeriod}&format=json&idSite=${idSite}&method=VisitsSummary.getVisits` + `https://stats.data.gouv.fr/?module=API&date=last${chartDate}&period=${chartPeriod}&format=json&idSite=${idSite}&method=VisitsSummary.getVisits&token_auth=${authToken}` ) .then((res) => res.data), { @@ -23,7 +25,8 @@ export const useSimulationsTerminees = () => () => axios .get( - `https://stats.data.gouv.fr/?module=API&method=Events.getAction&idSite=${idSite}&period=range&date=last6000&format=JSON` + `https://stats.data.gouv.fr/?module=API&method=Events.getAction&idSite=${idSite}&period=range&date=last6000&format=JSON&token_auth=${authToken} +` ) .then((res) => res.data.find((action) => action.label === 'A terminé la simulation') @@ -43,28 +46,32 @@ export const useX = (queryName, urlQuery, transformResult) => export const useVisitsDuration = () => useX( 'VisitsDuration', - `module=API&idSite=${idSite}&method=VisitorInterest.getNumberOfVisitsPerVisitDuration&segment=eventAction%3D%3DClic%252520CTA%252520accueil&period=range&date=last60&format=JSON`, + `module=API&idSite=${idSite}&method=VisitorInterest.getNumberOfVisitsPerVisitDuration&segment=eventAction%3D%3DClic%252520CTA%252520accueil&period=range&date=last60&format=JSON&token_auth=${authToken} +`, (res) => res.data ) export const useVisitsAvgDuration = () => useX( 'VisitsAvgDuration', - `module=API&idSite=${idSite}&method=VisitFrequency.get&period=range&date=last60&format=JSON&segment=eventAction%3D%3DClic%252520CTA%252520accueil;visitDuration>=60`, + `module=API&idSite=${idSite}&method=VisitFrequency.get&period=range&date=last60&format=JSON&segment=eventAction%3D%3DClic%252520CTA%252520accueil;visitDuration>=60&token_auth=${authToken} +`, (res) => res.data.avg_time_on_site_new / 60 ) export const useSimulationAvgDuration = () => useX( 'SimulationAvgDuration', - `module=API&idSite=${idSite}&method=Actions.getPageUrl&pageUrl=simulateur/bilan&period=range&date=last60&format=JSON&segment=eventAction%3D%3DA%252520termin%2525C3%2525A9%252520la%252520simulation;visitDuration>=60`, + `module=API&idSite=${idSite}&method=Actions.getPageUrl&pageUrl=simulateur/bilan&period=range&date=last60&format=JSON&segment=eventAction%3D%3DA%252520termin%2525C3%2525A9%252520la%252520simulation;visitDuration>=60&token_auth=${authToken} +`, (res) => res.data[0].sum_time_spent / res.data[0].nb_visits / 60 ) export const useTotal = () => useX( 'total', - `module=API&date=last30&period=range&format=json&idSite=${idSite}&method=VisitsSummary.getVisits`, + `module=API&date=last30&period=range&format=json&idSite=${idSite}&method=VisitsSummary.getVisits&token_auth=${authToken} +`, (res) => res.data ) diff --git a/webpack.dev.js b/webpack.dev.js index 9db5bdee0d..d58a0efe78 100644 --- a/webpack.dev.js +++ b/webpack.dev.js @@ -29,6 +29,7 @@ module.exports = { new webpack.DefinePlugin({ NODE_ENV: JSON.stringify('development'), SERVER_URL: JSON.stringify(process.env.SERVER_URL), + MATOMO_TOKEN: JSON.stringify(process.env.MATOMO_TOKEN), }), new ReactRefreshWebpackPlugin(), /* diff --git a/webpack.prod.js b/webpack.prod.js index 25194f13b0..d83662b433 100644 --- a/webpack.prod.js +++ b/webpack.prod.js @@ -41,6 +41,7 @@ module.exports = { new webpack.DefinePlugin({ NODE_ENV: JSON.stringify('production'), SERVER_URL: JSON.stringify(process.env.SERVER_URL), + MATOMO_TOKEN: JSON.stringify(process.env.MATOMO_TOKEN), }), ], } From d789a8aba0fd931bed4a87d1774c5086fc4cc1ad Mon Sep 17 00:00:00 2001 From: Mael Date: Tue, 21 Feb 2023 17:05:50 +0100 Subject: [PATCH 02/16] =?UTF-8?q?Exp=C3=A9rimentation=20d'une=20fonction?= =?UTF-8?q?=20Netlify;=20=C3=A7a=20marche=20pour=20un=20point=20de=20data?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- netlify/functions/get-stats.js | 13 +++++++++++++ source/components/stats/StatsContent.js | 1 + 2 files changed, 14 insertions(+) create mode 100644 netlify/functions/get-stats.js diff --git a/netlify/functions/get-stats.js b/netlify/functions/get-stats.js new file mode 100644 index 0000000000..36a0be4f03 --- /dev/null +++ b/netlify/functions/get-stats.js @@ -0,0 +1,13 @@ +import fetch from 'node-fetch' + +const idSite = 153 + +let { MATOMO_TOKEN } = process.env + +exports.handler = async (event, context) => { + const simulationsResponse = await fetch( + `https://stats.data.gouv.fr/?module=API&method=Events.getAction&idSite=${idSite}&period=range&date=last6000&format=JSON&token_auth=${MATOMO_TOKEN}` + ) + const simulations = await simulationsResponse.json() + return { statusCode: 200, body: JSON.stringify({ simulations }) } +} diff --git a/source/components/stats/StatsContent.js b/source/components/stats/StatsContent.js index 938de2dc5d..a628e25051 100644 --- a/source/components/stats/StatsContent.js +++ b/source/components/stats/StatsContent.js @@ -28,6 +28,7 @@ import { useVisitsDuration, useWebsites, } from './matomo' + import Section from './utils/Section' const Wrapper = styled.div` From 717fbab4df50706a6dd1d03b3a1cf3bfa4220ac0 Mon Sep 17 00:00:00 2001 From: Mael Date: Tue, 21 Feb 2023 17:14:14 +0100 Subject: [PATCH 03/16] Utilisation basique de la fonction --- source/components/stats/StatsContent.js | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/source/components/stats/StatsContent.js b/source/components/stats/StatsContent.js index a628e25051..20a73327de 100644 --- a/source/components/stats/StatsContent.js +++ b/source/components/stats/StatsContent.js @@ -1,3 +1,4 @@ +import { useEffect, useState } from 'react' import { Trans } from 'react-i18next' import styled from 'styled-components' @@ -107,6 +108,16 @@ export default function Data() { const simulationsfromhelp = useSimulationsfromKmHelp() const ridesnumber = useRidesNumber() + const [statsData, setStatsData] = useState(null) + + useEffect(async () => { + const response = await fetch('/.netlify/functions/get-stats') + const data = await response.json() + console.log('data', data) + setStatsData(data) + return undefined + }, []) + return (
@@ -116,6 +127,9 @@ export default function Data() { Stats générales +
+ DATA FROM FUNCTION {statsData && statsData.simulations[0].nb_visits} +
Date: Tue, 21 Feb 2023 17:39:46 +0100 Subject: [PATCH 04/16] =?UTF-8?q?It=C3=A9ration=20sur=20la=20fonction=20st?= =?UTF-8?q?ats?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- netlify/functions/get-stats.js | 28 +++++++++++++++++++++---- source/components/stats/StatsContent.js | 4 +++- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/netlify/functions/get-stats.js b/netlify/functions/get-stats.js index 36a0be4f03..841861142c 100644 --- a/netlify/functions/get-stats.js +++ b/netlify/functions/get-stats.js @@ -4,10 +4,30 @@ const idSite = 153 let { MATOMO_TOKEN } = process.env +const queries = (chartDate, chartPeriod) => [ + [ + 'chart', + `https://stats.data.gouv.fr/?module=API&date=last${chartDate}&period=${chartPeriod}&format=json&idSite=${idSite}&method=VisitsSummary.getVisits&token_auth=${MATOMO_TOKEN}`, + (res) => res.data, + ], + [ + 'simulations', + `https://stats.data.gouv.fr/?module=API&method=Events.getAction&idSite=${idSite}&period=range&date=last6000&format=JSON&token_auth=${MATOMO_TOKEN}`, + + (res) => + res.data.find((action) => action.label === 'A terminé la simulation'), + ], +] exports.handler = async (event, context) => { - const simulationsResponse = await fetch( - `https://stats.data.gouv.fr/?module=API&method=Events.getAction&idSite=${idSite}&period=range&date=last6000&format=JSON&token_auth=${MATOMO_TOKEN}` + const { chartDate, chartPeriod } = event.queryStringParameters + + const promises = queries(chartDate, chartPeriod).map( + ([key, URL, transform]) => + fetch(URL) + .then((res) => res.json()) + .then((json) => [key, transform(json)]) ) - const simulations = await simulationsResponse.json() - return { statusCode: 200, body: JSON.stringify({ simulations }) } + const data = await Promise.all(promises) + + return { statusCode: 200, body: JSON.stringify(data) } } diff --git a/source/components/stats/StatsContent.js b/source/components/stats/StatsContent.js index 20a73327de..7ebb1b1772 100644 --- a/source/components/stats/StatsContent.js +++ b/source/components/stats/StatsContent.js @@ -111,7 +111,9 @@ export default function Data() { const [statsData, setStatsData] = useState(null) useEffect(async () => { - const response = await fetch('/.netlify/functions/get-stats') + const response = await fetch( + `/.netlify/functions/get-stats?chartPeriod=${chartPeriod}&chartDate=${chartDate}` + ) const data = await response.json() console.log('data', data) setStatsData(data) From fc4cd3e505525fe1236db7dee22397c2c25537ec Mon Sep 17 00:00:00 2001 From: Mael Date: Tue, 21 Feb 2023 17:55:01 +0100 Subject: [PATCH 05/16] =?UTF-8?q?Drastique=20simplif=20de=20l'id=C3=A9e=20?= =?UTF-8?q?de=20d=C3=A9part=20:=20la=20fonction=20n'est=20qu'un=20proxy?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Qui ajoute le token à la requête et renvoie la réponse. --- netlify/functions/get-stats.js | 30 ++++--------- source/components/stats/StatsContent.js | 16 ------- source/components/stats/matomo.js | 56 ++++++++++--------------- 3 files changed, 32 insertions(+), 70 deletions(-) diff --git a/netlify/functions/get-stats.js b/netlify/functions/get-stats.js index 841861142c..7ae7346ca1 100644 --- a/netlify/functions/get-stats.js +++ b/netlify/functions/get-stats.js @@ -4,30 +4,18 @@ const idSite = 153 let { MATOMO_TOKEN } = process.env -const queries = (chartDate, chartPeriod) => [ - [ - 'chart', - `https://stats.data.gouv.fr/?module=API&date=last${chartDate}&period=${chartPeriod}&format=json&idSite=${idSite}&method=VisitsSummary.getVisits&token_auth=${MATOMO_TOKEN}`, - (res) => res.data, - ], - [ - 'simulations', - `https://stats.data.gouv.fr/?module=API&method=Events.getAction&idSite=${idSite}&period=range&date=last6000&format=JSON&token_auth=${MATOMO_TOKEN}`, - - (res) => - res.data.find((action) => action.label === 'A terminé la simulation'), - ], -] exports.handler = async (event, context) => { - const { chartDate, chartPeriod } = event.queryStringParameters + const requestParams = decodeURIComponent( + event.queryStringParameters.requestParams + ) - const promises = queries(chartDate, chartPeriod).map( - ([key, URL, transform]) => - fetch(URL) - .then((res) => res.json()) - .then((json) => [key, transform(json)]) + const response = await fetch( + 'https://stats.data.gouv.fr/?' + + requestParams + + '&token_auth=' + + MATOMO_TOKEN ) - const data = await Promise.all(promises) + const data = await response.json() return { statusCode: 200, body: JSON.stringify(data) } } diff --git a/source/components/stats/StatsContent.js b/source/components/stats/StatsContent.js index 7ebb1b1772..a628e25051 100644 --- a/source/components/stats/StatsContent.js +++ b/source/components/stats/StatsContent.js @@ -1,4 +1,3 @@ -import { useEffect, useState } from 'react' import { Trans } from 'react-i18next' import styled from 'styled-components' @@ -108,18 +107,6 @@ export default function Data() { const simulationsfromhelp = useSimulationsfromKmHelp() const ridesnumber = useRidesNumber() - const [statsData, setStatsData] = useState(null) - - useEffect(async () => { - const response = await fetch( - `/.netlify/functions/get-stats?chartPeriod=${chartPeriod}&chartDate=${chartDate}` - ) - const data = await response.json() - console.log('data', data) - setStatsData(data) - return undefined - }, []) - return (
@@ -129,9 +116,6 @@ export default function Data() { Stats générales -
- DATA FROM FUNCTION {statsData && statsData.simulations[0].nb_visits} -
+export const useX = (queryName, urlQuery, transformResult, keepPreviousData) => useQuery( - ['chart', chartPeriod, chartDate], + queryName, () => axios .get( - `https://stats.data.gouv.fr/?module=API&date=last${chartDate}&period=${chartPeriod}&format=json&idSite=${idSite}&method=VisitsSummary.getVisits&token_auth=${authToken}` + '/.netlify/functions/get-stats?requestParams=' + + encodeURIComponent(urlQuery) ) - .then((res) => res.data), - { - keepPreviousData: true, - } + .then((res) => transformResult(res)), + { keepPreviousData } ) -export const useSimulationsTerminees = () => - useQuery( - ['SimulationsTerminees'], - () => - axios - .get( - `https://stats.data.gouv.fr/?module=API&method=Events.getAction&idSite=${idSite}&period=range&date=last6000&format=JSON&token_auth=${authToken} -` - ) - .then((res) => - res.data.find((action) => action.label === 'A terminé la simulation') - ), - { - keepPreviousData: true, - } +export const useChart = ({ chartPeriod, chartDate }) => + useX( + ['chart', chartPeriod, chartDate], + `module=API&date=last${chartDate}&period=${chartPeriod}&format=json&idSite=${idSite}&method=VisitsSummary.getVisits`, + (res) => res.data, + true ) -export const useX = (queryName, urlQuery, transformResult) => - useQuery(queryName, () => - axios - .get('https://stats.data.gouv.fr/?' + urlQuery) - .then((res) => transformResult(res)) +export const useSimulationsTerminees = () => + useX( + ['SimulationsTerminees'], + `module=API&method=Events.getAction&idSite=${idSite}&period=range&date=last6000&format=JSON`, + (res) => + res.data.find((action) => action.label === 'A terminé la simulation'), + true ) export const useVisitsDuration = () => useX( 'VisitsDuration', - `module=API&idSite=${idSite}&method=VisitorInterest.getNumberOfVisitsPerVisitDuration&segment=eventAction%3D%3DClic%252520CTA%252520accueil&period=range&date=last60&format=JSON&token_auth=${authToken} + `module=API&idSite=${idSite}&method=VisitorInterest.getNumberOfVisitsPerVisitDuration&segment=eventAction%3D%3DClic%252520CTA%252520accueil&period=range&date=last60&format=JSON `, (res) => res.data ) @@ -54,7 +44,7 @@ export const useVisitsDuration = () => export const useVisitsAvgDuration = () => useX( 'VisitsAvgDuration', - `module=API&idSite=${idSite}&method=VisitFrequency.get&period=range&date=last60&format=JSON&segment=eventAction%3D%3DClic%252520CTA%252520accueil;visitDuration>=60&token_auth=${authToken} + `module=API&idSite=${idSite}&method=VisitFrequency.get&period=range&date=last60&format=JSON&segment=eventAction%3D%3DClic%252520CTA%252520accueil;visitDuration>=60 `, (res) => res.data.avg_time_on_site_new / 60 ) @@ -62,7 +52,7 @@ export const useVisitsAvgDuration = () => export const useSimulationAvgDuration = () => useX( 'SimulationAvgDuration', - `module=API&idSite=${idSite}&method=Actions.getPageUrl&pageUrl=simulateur/bilan&period=range&date=last60&format=JSON&segment=eventAction%3D%3DA%252520termin%2525C3%2525A9%252520la%252520simulation;visitDuration>=60&token_auth=${authToken} + `module=API&idSite=${idSite}&method=Actions.getPageUrl&pageUrl=simulateur/bilan&period=range&date=last60&format=JSON&segment=eventAction%3D%3DA%252520termin%2525C3%2525A9%252520la%252520simulation;visitDuration>=60 `, (res) => res.data[0].sum_time_spent / res.data[0].nb_visits / 60 ) @@ -70,7 +60,7 @@ export const useSimulationAvgDuration = () => export const useTotal = () => useX( 'total', - `module=API&date=last30&period=range&format=json&idSite=${idSite}&method=VisitsSummary.getVisits&token_auth=${authToken} + `module=API&date=last30&period=range&format=json&idSite=${idSite}&method=VisitsSummary.getVisits `, (res) => res.data ) From a7117d55a42640279106973fe6c42903e28e2b7c Mon Sep 17 00:00:00 2001 From: Mael Date: Tue, 21 Feb 2023 18:10:41 +0100 Subject: [PATCH 06/16] =?UTF-8?q?Filtrage=20des=20=C3=A9l=C3=A9ments=20ind?= =?UTF-8?q?=C3=A9sirables?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- netlify/functions/get-stats.js | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/netlify/functions/get-stats.js b/netlify/functions/get-stats.js index 7ae7346ca1..e4ce0b9c50 100644 --- a/netlify/functions/get-stats.js +++ b/netlify/functions/get-stats.js @@ -15,7 +15,19 @@ exports.handler = async (event, context) => { '&token_auth=' + MATOMO_TOKEN ) - const data = await response.json() + const json = await response.json() - return { statusCode: 200, body: JSON.stringify(data) } + //We have to filter the page URL data because of a security flaw that introduced secret data in some URLs + if (requestParams.includes('Page')) { + return success( + json.filter( + (el) => + !el.label.includes('conférence/') && !el.label.includes('sondage/') + ) + ) + } + + return success(json) } + +const success = (data) => ({ statusCode: 200, body: JSON.stringify(data) }) From a1a46e8af9e64660d0c3240b306df2a1c1d83dd3 Mon Sep 17 00:00:00 2001 From: Mael Date: Tue, 21 Feb 2023 18:33:05 +0100 Subject: [PATCH 07/16] =?UTF-8?q?Filtration=20correcte=20des=20donn=C3=A9e?= =?UTF-8?q?s=20stats=20de=20sondage=20et=20conf?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- netlify/functions/get-stats.js | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/netlify/functions/get-stats.js b/netlify/functions/get-stats.js index e4ce0b9c50..f39f76b739 100644 --- a/netlify/functions/get-stats.js +++ b/netlify/functions/get-stats.js @@ -17,12 +17,13 @@ exports.handler = async (event, context) => { ) const json = await response.json() - //We have to filter the page URL data because of a security flaw that introduced secret data in some URLs + // Remove secret pages that would reveal groupe names that should stay private if (requestParams.includes('Page')) { return success( json.filter( (el) => - !el.label.includes('conférence/') && !el.label.includes('sondage/') + !isPrivate(el.label) && + !(el.subtable && el.subtable.find((t) => isPrivate(t.url))) ) ) } @@ -30,4 +31,21 @@ exports.handler = async (event, context) => { return success(json) } -const success = (data) => ({ statusCode: 200, body: JSON.stringify(data) }) +const success = (data) => ({ + statusCode: 200, + body: JSON.stringify(data), + headers: { + 'Content-Type': 'application/json; charset=utf-8', + 'Access-Control-Allow-Origin': '*', + }, +}) + +const isPrivate = (rawString) => { + const string = decodeURIComponent(rawString) + return ( + string != undefined && + (string.includes('conférence/') || + string.includes('conference/') || + string.includes('sondage/')) + ) +} From 778bb66180efcc711fe7fae870e1e79125686802 Mon Sep 17 00:00:00 2001 From: Mael Date: Tue, 21 Feb 2023 18:38:29 +0100 Subject: [PATCH 08/16] Nettoyage --- netlify/functions/get-stats.js | 2 -- source/components/stats/matomo.js | 12 ++++-------- webpack.dev.js | 1 - webpack.prod.js | 1 - 4 files changed, 4 insertions(+), 12 deletions(-) diff --git a/netlify/functions/get-stats.js b/netlify/functions/get-stats.js index f39f76b739..b8fea96ec6 100644 --- a/netlify/functions/get-stats.js +++ b/netlify/functions/get-stats.js @@ -1,7 +1,5 @@ import fetch from 'node-fetch' -const idSite = 153 - let { MATOMO_TOKEN } = process.env exports.handler = async (event, context) => { diff --git a/source/components/stats/matomo.js b/source/components/stats/matomo.js index 4fe8e1e783..06f8a6f23a 100644 --- a/source/components/stats/matomo.js +++ b/source/components/stats/matomo.js @@ -36,32 +36,28 @@ export const useSimulationsTerminees = () => export const useVisitsDuration = () => useX( 'VisitsDuration', - `module=API&idSite=${idSite}&method=VisitorInterest.getNumberOfVisitsPerVisitDuration&segment=eventAction%3D%3DClic%252520CTA%252520accueil&period=range&date=last60&format=JSON -`, + `module=API&idSite=${idSite}&method=VisitorInterest.getNumberOfVisitsPerVisitDuration&segment=eventAction%3D%3DClic%252520CTA%252520accueil&period=range&date=last60&format=JSON`, (res) => res.data ) export const useVisitsAvgDuration = () => useX( 'VisitsAvgDuration', - `module=API&idSite=${idSite}&method=VisitFrequency.get&period=range&date=last60&format=JSON&segment=eventAction%3D%3DClic%252520CTA%252520accueil;visitDuration>=60 -`, + `module=API&idSite=${idSite}&method=VisitFrequency.get&period=range&date=last60&format=JSON&segment=eventAction%3D%3DClic%252520CTA%252520accueil;visitDuration>=60`, (res) => res.data.avg_time_on_site_new / 60 ) export const useSimulationAvgDuration = () => useX( 'SimulationAvgDuration', - `module=API&idSite=${idSite}&method=Actions.getPageUrl&pageUrl=simulateur/bilan&period=range&date=last60&format=JSON&segment=eventAction%3D%3DA%252520termin%2525C3%2525A9%252520la%252520simulation;visitDuration>=60 -`, + `module=API&idSite=${idSite}&method=Actions.getPageUrl&pageUrl=simulateur/bilan&period=range&date=last60&format=JSON&segment=eventAction%3D%3DA%252520termin%2525C3%2525A9%252520la%252520simulation;visitDuration>=60`, (res) => res.data[0].sum_time_spent / res.data[0].nb_visits / 60 ) export const useTotal = () => useX( 'total', - `module=API&date=last30&period=range&format=json&idSite=${idSite}&method=VisitsSummary.getVisits -`, + `module=API&date=last30&period=range&format=json&idSite=${idSite}&method=VisitsSummary.getVisits`, (res) => res.data ) diff --git a/webpack.dev.js b/webpack.dev.js index d58a0efe78..9db5bdee0d 100644 --- a/webpack.dev.js +++ b/webpack.dev.js @@ -29,7 +29,6 @@ module.exports = { new webpack.DefinePlugin({ NODE_ENV: JSON.stringify('development'), SERVER_URL: JSON.stringify(process.env.SERVER_URL), - MATOMO_TOKEN: JSON.stringify(process.env.MATOMO_TOKEN), }), new ReactRefreshWebpackPlugin(), /* diff --git a/webpack.prod.js b/webpack.prod.js index d83662b433..25194f13b0 100644 --- a/webpack.prod.js +++ b/webpack.prod.js @@ -41,7 +41,6 @@ module.exports = { new webpack.DefinePlugin({ NODE_ENV: JSON.stringify('production'), SERVER_URL: JSON.stringify(process.env.SERVER_URL), - MATOMO_TOKEN: JSON.stringify(process.env.MATOMO_TOKEN), }), ], } From 2fd895d30a81f3cce7501f51ed2c6841d7678af4 Mon Sep 17 00:00:00 2001 From: Mael Date: Tue, 21 Feb 2023 18:42:00 +0100 Subject: [PATCH 09/16] Commentaire pour expliquer la fonction netlify --- netlify/functions/get-stats.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/netlify/functions/get-stats.js b/netlify/functions/get-stats.js index b8fea96ec6..3a22cdb256 100644 --- a/netlify/functions/get-stats.js +++ b/netlify/functions/get-stats.js @@ -2,6 +2,8 @@ import fetch from 'node-fetch' let { MATOMO_TOKEN } = process.env +// This function authorizes requests made from our front-end to fetch stats properties +// Our full stats data are now private, since they could expose sensitive informations exports.handler = async (event, context) => { const requestParams = decodeURIComponent( event.queryStringParameters.requestParams From 3cd8e044581e4d41bde8b7db6b1328253c65a2a4 Mon Sep 17 00:00:00 2001 From: Mael Date: Tue, 21 Feb 2023 18:58:27 +0100 Subject: [PATCH 10/16] =?UTF-8?q?On=20traite=20une=20faille,=20la=20foncti?= =?UTF-8?q?on=20ne=20doit=20pas=20=C3=AAtre=20utilisable=20pour=20d'autres?= =?UTF-8?q?=20requ=C3=AAtes=20que=20l'on=20n'a=20pas=20filtr=C3=A9=20et=20?= =?UTF-8?q?qui=20pourraient=20r=C3=A9v=C3=A9ler=20d'autres=20donn=C3=A9es?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- netlify/functions/get-stats.js | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/netlify/functions/get-stats.js b/netlify/functions/get-stats.js index 3a22cdb256..d9c959672a 100644 --- a/netlify/functions/get-stats.js +++ b/netlify/functions/get-stats.js @@ -2,6 +2,20 @@ import fetch from 'node-fetch' let { MATOMO_TOKEN } = process.env +const authorizedMethods = [ + 'VisitsSummary.getVisits', + 'VisitsSummary.getVisits', + 'VisitorInterest.getNumberOfVisitsPerVisitDuration', + 'VisitFrequency.get', + 'Actions.getPageUrl', + 'Referrers.getWebsites', + 'Referrers.getSocials', + 'Referrers.getKeywords', + 'Actions.getEntryPageUrls', + 'Actions.getPageUrls', + 'Events.getAction', +] + // This function authorizes requests made from our front-end to fetch stats properties // Our full stats data are now private, since they could expose sensitive informations exports.handler = async (event, context) => { @@ -9,6 +23,14 @@ exports.handler = async (event, context) => { event.queryStringParameters.requestParams ) + const matomoMethod = new URLSearchParams(requestParams).get('method'), + authorizedMethod = authorizedMethods.includes(matomoMethod) + + if (!authorizedMethod) + return { + statusCode: 401, + } + const response = await fetch( 'https://stats.data.gouv.fr/?' + requestParams + @@ -36,7 +58,6 @@ const success = (data) => ({ body: JSON.stringify(data), headers: { 'Content-Type': 'application/json; charset=utf-8', - 'Access-Control-Allow-Origin': '*', }, }) From b787d63a5789136f882f19e2269eb18f6d576207 Mon Sep 17 00:00:00 2001 From: Mael Date: Wed, 22 Feb 2023 10:17:08 +0100 Subject: [PATCH 11/16] Faille sur l'id du site Matomo --- netlify/functions/get-stats.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/netlify/functions/get-stats.js b/netlify/functions/get-stats.js index d9c959672a..6bb70fe971 100644 --- a/netlify/functions/get-stats.js +++ b/netlify/functions/get-stats.js @@ -26,7 +26,9 @@ exports.handler = async (event, context) => { const matomoMethod = new URLSearchParams(requestParams).get('method'), authorizedMethod = authorizedMethods.includes(matomoMethod) - if (!authorizedMethod) + const authorizedSiteId = idSite === '153' + + if (!authorizedMethod || !authorizedSiteId) return { statusCode: 401, } From 8121ba73ca81c129676074527034c0dc8ee4ca3c Mon Sep 17 00:00:00 2001 From: Mael Date: Wed, 22 Feb 2023 17:20:21 +0100 Subject: [PATCH 12/16] =?UTF-8?q?Factoraisation=20du=20code=20de=20test=20?= =?UTF-8?q?d'URL=20priv=C3=A9e=20pour=20stats?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Emile Rolley <44124798+EmileRolley@users.noreply.github.com> --- netlify/functions/get-stats.js | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/netlify/functions/get-stats.js b/netlify/functions/get-stats.js index 6bb70fe971..3387890f46 100644 --- a/netlify/functions/get-stats.js +++ b/netlify/functions/get-stats.js @@ -63,12 +63,13 @@ const success = (data) => ({ }, }) +const privateURLs = [ 'conférence/', 'conference/', 'sondage/' ] + const isPrivate = (rawString) => { - const string = decodeURIComponent(rawString) - return ( - string != undefined && - (string.includes('conférence/') || - string.includes('conference/') || - string.includes('sondage/')) + const uriComponents = decodeURIComponent(rawString) + + return privateURLs.string?. + uriComponents != undefined && + privateURLs.some((url) => uriComponents.includes(url)) ) } From 9dc53444fcc8cd9da59644a62a4e37699fb5b950 Mon Sep 17 00:00:00 2001 From: Mael Date: Tue, 28 Feb 2023 15:43:42 +0100 Subject: [PATCH 13/16] Pas besoin de node-fetch dans la v18 de node --- netlify/functions/get-stats.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/netlify/functions/get-stats.js b/netlify/functions/get-stats.js index 3387890f46..51ca38314a 100644 --- a/netlify/functions/get-stats.js +++ b/netlify/functions/get-stats.js @@ -1,5 +1,3 @@ -import fetch from 'node-fetch' - let { MATOMO_TOKEN } = process.env const authorizedMethods = [ From 317663e26090283a51caaf38ead987ec92b11744 Mon Sep 17 00:00:00 2001 From: Mael Date: Tue, 28 Feb 2023 15:55:01 +0100 Subject: [PATCH 14/16] =?UTF-8?q?Erreur=20syntaxique;=20probl=C3=A8me=20de?= =?UTF-8?q?=20variable=20idSite=20introduite=20n'importe=20comment?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Autant pour moi --- netlify/functions/get-stats.js | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/netlify/functions/get-stats.js b/netlify/functions/get-stats.js index 51ca38314a..e93cafcee0 100644 --- a/netlify/functions/get-stats.js +++ b/netlify/functions/get-stats.js @@ -17,14 +17,15 @@ const authorizedMethods = [ // This function authorizes requests made from our front-end to fetch stats properties // Our full stats data are now private, since they could expose sensitive informations exports.handler = async (event, context) => { - const requestParams = decodeURIComponent( - event.queryStringParameters.requestParams - ) + const rawRequestParams = decodeURIComponent( + event.queryStringParameters.requestParams + ), + requestParams = new URLSearchParams(rawRequestParams) - const matomoMethod = new URLSearchParams(requestParams).get('method'), + const matomoMethod = requestParams.get('method'), authorizedMethod = authorizedMethods.includes(matomoMethod) - const authorizedSiteId = idSite === '153' + const authorizedSiteId = requestParams.get('idSite') === '153' if (!authorizedMethod || !authorizedSiteId) return { @@ -40,7 +41,7 @@ exports.handler = async (event, context) => { const json = await response.json() // Remove secret pages that would reveal groupe names that should stay private - if (requestParams.includes('Page')) { + if (rawRequestParams.includes('Page')) { return success( json.filter( (el) => @@ -61,12 +62,12 @@ const success = (data) => ({ }, }) -const privateURLs = [ 'conférence/', 'conference/', 'sondage/' ] +const privateURLs = ['conférence/', 'conference/', 'sondage/'] const isPrivate = (rawString) => { const uriComponents = decodeURIComponent(rawString) - - return privateURLs.string?. + + return ( uriComponents != undefined && privateURLs.some((url) => uriComponents.includes(url)) ) From b3d9457cc1043482b9c8dae3845b5193e63949cf Mon Sep 17 00:00:00 2001 From: Mael Date: Wed, 1 Mar 2023 11:18:30 +0100 Subject: [PATCH 15/16] Dans le but d'utiliser un cache, on migre la fonction netlify vers scalingo --- netlify/functions/get-stats.js | 74 ------------------- source/components/stats/matomo.js | 4 +- .../publicodes/conference/useDatabase.tsx | 7 +- 3 files changed, 6 insertions(+), 79 deletions(-) delete mode 100644 netlify/functions/get-stats.js diff --git a/netlify/functions/get-stats.js b/netlify/functions/get-stats.js deleted file mode 100644 index e93cafcee0..0000000000 --- a/netlify/functions/get-stats.js +++ /dev/null @@ -1,74 +0,0 @@ -let { MATOMO_TOKEN } = process.env - -const authorizedMethods = [ - 'VisitsSummary.getVisits', - 'VisitsSummary.getVisits', - 'VisitorInterest.getNumberOfVisitsPerVisitDuration', - 'VisitFrequency.get', - 'Actions.getPageUrl', - 'Referrers.getWebsites', - 'Referrers.getSocials', - 'Referrers.getKeywords', - 'Actions.getEntryPageUrls', - 'Actions.getPageUrls', - 'Events.getAction', -] - -// This function authorizes requests made from our front-end to fetch stats properties -// Our full stats data are now private, since they could expose sensitive informations -exports.handler = async (event, context) => { - const rawRequestParams = decodeURIComponent( - event.queryStringParameters.requestParams - ), - requestParams = new URLSearchParams(rawRequestParams) - - const matomoMethod = requestParams.get('method'), - authorizedMethod = authorizedMethods.includes(matomoMethod) - - const authorizedSiteId = requestParams.get('idSite') === '153' - - if (!authorizedMethod || !authorizedSiteId) - return { - statusCode: 401, - } - - const response = await fetch( - 'https://stats.data.gouv.fr/?' + - requestParams + - '&token_auth=' + - MATOMO_TOKEN - ) - const json = await response.json() - - // Remove secret pages that would reveal groupe names that should stay private - if (rawRequestParams.includes('Page')) { - return success( - json.filter( - (el) => - !isPrivate(el.label) && - !(el.subtable && el.subtable.find((t) => isPrivate(t.url))) - ) - ) - } - - return success(json) -} - -const success = (data) => ({ - statusCode: 200, - body: JSON.stringify(data), - headers: { - 'Content-Type': 'application/json; charset=utf-8', - }, -}) - -const privateURLs = ['conférence/', 'conference/', 'sondage/'] - -const isPrivate = (rawString) => { - const uriComponents = decodeURIComponent(rawString) - - return ( - uriComponents != undefined && - privateURLs.some((url) => uriComponents.includes(url)) - ) -} diff --git a/source/components/stats/matomo.js b/source/components/stats/matomo.js index 06f8a6f23a..6fef07c795 100644 --- a/source/components/stats/matomo.js +++ b/source/components/stats/matomo.js @@ -1,5 +1,6 @@ import axios from 'axios' import { useQuery } from 'react-query' +import { serverURL } from '../../sites/publicodes/conference/useDatabase' const idSite = 153 @@ -9,8 +10,7 @@ export const useX = (queryName, urlQuery, transformResult, keepPreviousData) => () => axios .get( - '/.netlify/functions/get-stats?requestParams=' + - encodeURIComponent(urlQuery) + serverURL + '/get-stats?requestParams=' + encodeURIComponent(urlQuery) ) .then((res) => transformResult(res)), { keepPreviousData } diff --git a/source/sites/publicodes/conference/useDatabase.tsx b/source/sites/publicodes/conference/useDatabase.tsx index a5917c69d2..1eddc1b393 100644 --- a/source/sites/publicodes/conference/useDatabase.tsx +++ b/source/sites/publicodes/conference/useDatabase.tsx @@ -3,12 +3,13 @@ import { io } from 'socket.io-client' const secure = NODE_ENV === 'development' ? '' : 's' const protocol = `http${secure}://` +export const serverURL = protocol + SERVER_URL -export const answersURL = protocol + SERVER_URL + '/answers/' +export const answersURL = serverURL + '/answers/' -export const surveysURL = protocol + SERVER_URL + '/surveys/' +export const surveysURL = serverURL + '/surveys/' -export const contextURL = protocol + SERVER_URL +export const contextURL = serverURL export default () => { const database = useMemo( From da9f3915b62c3f9840500e7dfcf3c8143378e0e9 Mon Sep 17 00:00:00 2001 From: Mael Date: Wed, 1 Mar 2023 11:20:29 +0100 Subject: [PATCH 16/16] =?UTF-8?q?Lien=20vers=20la=20page=20stats=20depuis?= =?UTF-8?q?=20/=C3=A0-propos?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- source/locales/pages/fr/about.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/source/locales/pages/fr/about.md b/source/locales/pages/fr/about.md index dc44a6431f..c57c78d792 100644 --- a/source/locales/pages/fr/about.md +++ b/source/locales/pages/fr/about.md @@ -34,6 +34,10 @@ Le simulateur est amélioré en continu. [✨️ Découvrez les dernières nouveautés et les notes de versions](/nouveautés). +## Statistiques + +Une sélection de statistiques d'audience est disponible publiquement [sur notre page stats](/stats). + ## Vie privée Nous collectons des données anonymisées uniquement pour améliorer ce