chore: Migration of Open Food Facts from off1 (bare metal) to off2 (p…

…roxmox) (#8827)

- Migration of Open Food Facts (site and pro platform) from off1 (bare metal) to off2 (proxmox)
- Modifications linked to moving off and off-pro to off2 server. 
- This includes a lot of path modifications:
  - some directories where removed from the repository and are created dynamically
  - we try to differentiate folders containing code (only this is in git), data, cache data and dist files
  - we control this by using a fixed set of base path (see Paths.pm)
- Most of server configurations are now in the git repository
- Import scripts where reworked 

See also: openfoodfacts/openfoodfacts-infrastructure#243

Part of:
- openfoodfacts/openfoodfacts-infrastructure#182

.github/workflows/generate-doc.yml

@@ -7,7 +7,6 @@ on:
     - "docs/**"
     - ".github/workflows/generate-doc.yml"
     - "mkdocs.yml"
-    - "html/files/doc/perl/**"
   push:
     # on merge to main, build and publish
     branches: [ "main" ]
@@ -38,10 +37,10 @@ jobs:
 
       - name: Run generate_perl_html_doc_from_pod.pl
         run: |
-          perl -CS -I lib scripts/generate_perl_html_doc_from_pod.pl && \
-          # and copy to target directory
           rm -rf gh_pages/dev/ref-perl-pod && \
-          mv html/files/doc/perl gh_pages/dev/ref-perl-pod
+          mkdir gh_pages/dev/ref-perl-pod && \
+          cp docs/assets/simple.min.css gh_pages/dev/ref-perl-pod && \
+          perl -CS -I lib scripts/generate_perl_html_doc_from_pod.pl gh_pages/dev/ref-perl-pod
 
       # DISABLED: in favor of rapidoc
       # generating OpenAPI documentation with redocly

.github/workflows/pull_request.yml

@@ -134,11 +134,16 @@ jobs:
       uses: ishworkh/docker-image-artifact-download@v1
       with:
         image: "openfoodfacts-server/backend:dev"
+    - name: set right UID and GID in .envrc
+      run: |
+        rm -f .envrc
+        echo "export USER_UID=$(id -u)" >> .envrc
+        echo "export USER_GID=$(id -g)" >> .envrc
     - name: Test make dev
       run: |
         make dev
         make status
     - name: Test all is running
-      run: make livecheck
+      run: make livecheck || ( tail -n 300 logs/apache2/*error*log; docker-compose logs; false )
     - name: test clean
       run: make hdown

.github/workflows/release-please.yml

@@ -70,6 +70,7 @@ jobs:
           id=$(docker create $image_name)
           docker cp -a $id:/opt/product-opener/html/css/dist/ /tmp/frontend-dist/css/
           docker cp -a $id:/opt/product-opener/html/images/icons/dist/ /tmp/frontend-dist/icons/
+          docker cp -a $id:/opt/product-opener/html/images/attributes/dist/ /tmp/frontend-dist/attributes/
           docker cp -a $id:/opt/product-opener/html/js/dist/ /tmp/frontend-dist/js/
           docker rm -v $id
           # make a tar

.gitignore

@@ -8,6 +8,37 @@ po/site-specific
 tests/unit/outputs/
 tests/integration/outputs/
 
+# local configs, most of them are link to files in conf/ depending on instance
+/html/.well-known/apple-*
+/log.conf
+/minion_log.conf
+
+# data dirs - we shall not mix data and git files, as it makes deployments/updates difficult
+/build-cache
+/debug
+/deleted.images
+/dist
+/logs
+/new_images
+/orgs
+/products
+/tmp
+/users
+/imports
+/lang
+/html/dump
+/html/files
+/html/data
+/html/exports
+/deleted_private_products
+/deleted_products
+/deleted_products_images
+/export_files
+
+# Images
+/html/images/products
+
+
 # Local files
 .DS_Store?
 ._*
@@ -46,13 +77,10 @@ html/bower_components
 local
 
 # Front build
-html/css/dist
-html/js/dist
-html/images/icons/dist
-html/images/attributes/*.svg
-
-# Images
-html/images/products/*
+/html/css/dist
+/html/js/dist
+/html/images/icons/dist
+/html/images/attributes/dist
 
 # Don't store helm dependencies in our git repo
 docker/productopener/charts

Dockerfile

@@ -17,17 +17,27 @@ RUN --mount=type=cache,id=apt-cache,target=/var/cache/apt set -x && \
         apache2 \
         apt-utils \
         cpanminus \
+        # being able to build things
         g++ \
         gcc \
         less \
         libapache2-mod-perl2 \
-        # libexpat1-dev \
         make \
         gettext \
         wget \
+        # images processing
         imagemagick \
         graphviz \
         tesseract-ocr \
+        # ftp client
+        lftp \
+        # some compression utils
+        gzip \
+        tar \
+        unzip \
+        zip \
+        # useful to send mail
+        mailutils \
         # perlmagick \
         #
         # Packages from ./cpanfile:
@@ -146,7 +156,9 @@ RUN --mount=type=cache,id=apt-cache,target=/var/cache/apt set -x && \
         # gnu readline
         libreadline-dev \
         # IO::AIO needed by Perl::LanguageServer
-        libperl-dev
+        libperl-dev \
+        # needed to build Apache2::Connection::XForwardedFor
+        libapache2-mod-perl2-dev
 
 # Run www-data user as host user 'off' or developper uid
 ARG USER_UID
@@ -166,7 +178,11 @@ WORKDIR /tmp
 COPY ./cpanfile* /tmp/
 # Add ProductOpener runtime dependencies from cpan
 RUN --mount=type=cache,id=cpanm-cache,target=/root/.cpanm \
-    cpanm $CPANMOPTS --notest --quiet --skip-satisfied --local-lib /tmp/local/ --installdeps .
+    # first install some dependencies that are not well handled
+    cpanm --notest --quiet --skip-satisfied --local-lib /tmp/local/ "Apache::Bootstrap" && \
+    cpanm $CPANMOPTS --notest --quiet --skip-satisfied --local-lib /tmp/local/ --installdeps . \
+    # in case of errors show build.log, but still, fail
+    || ( for f in /root/.cpanm/work/*/build.log;do echo $f"= start =============";cat $f; echo $f"= end ============="; done; false )
 
 ######################
 # backend production image stage
@@ -189,20 +205,25 @@ RUN \
 RUN \
     mkdir -p var/run/apache2/ && \
     chown www-data:www-data var/run/apache2/ && \
-    for path in data html_data users products product_images orgs new_images logs tmp; do \
+    for path in data html_data users products product_images orgs logs new_images deleted_products_images reverted_products deleted_private_products translate deleted_products deleted.images import_files tmp build-cache/taxonomies debug; do \
         mkdir -p /mnt/podata/${path}; \
     done && \
     chown www-data:www-data -R /mnt/podata && \
     # Create symlinks of data files that are indeed conf data in /mnt/podata (because we currently mix data and conf data)
     # NOTE: do not changes those links for they are in a volume, or handle migration in entry-point
-    for path in data-default external-data emb_codes ingredients madenearme packager-codes po taxonomies templates build-cache; do \
+    for path in data-default external-data emb_codes ingredients madenearme packager-codes po taxonomies templates; do \
         ln -sf /opt/product-opener/${path} /mnt/podata/${path}; \
     done && \
     # Create some necessary files to ensure permissions in volumes
     mkdir -p /opt/product-opener/html/data/ && \
     mkdir -p /opt/product-opener/html/data/taxonomies/ && \
-    mkdir -p /opt/product-opener/html/images/ && \
+    mkdir -p /opt/product-opener/html/images/products && \
     chown www-data:www-data -R /opt/product-opener/html/ && \
+    # inter services directories (until we get a real solution)
+    for service in obf off opf opff; do \
+        mkdir -p /srv/$service; \
+        chown www-data:www-data -R /srv/$service; \
+    done && \
     # logs dir
     mkdir -p /var/log/apache2/ && \
     chown www-data:www-data -R /var/log

Makefile

@@ -89,6 +89,11 @@ dev: hello build init_backend _up import_sample_data create_mongodb_indexes refr
 edit_etc_hosts:
 	@grep -qxF -- "${HOSTS}" /etc/hosts || echo "${HOSTS}" >> /etc/hosts
 
+create_folders:
+# create some folders to avoid having them owned by root (when created by docker compose)
+	@echo "🥫 Creating folders before docker-compose use them."
+	mkdir -p logs/apache2 logs/nginx debug || ( whoami; ls -l . ; false )
+
 # TODO: Figure out events => actions and implement live reload
 # live_reload:
 # 	@echo "🥫 Installing when-changed …"
@@ -119,7 +124,7 @@ _up:
 	${DOCKER_COMPOSE} up -d 2>&1
 	@echo "🥫 started service at http://openfoodfacts.localhost"
 
-up: build _up
+up: build create_folders _up
 
 down:
 	@echo "🥫 Bringing down containers …"
@@ -156,7 +161,7 @@ tail:
 	@echo "🥫 Reading logs (Apache2, Nginx) …"
 	tail -f logs/**/*
 
-codecov_prepare:
+codecov_prepare: create_folders
 	@echo "🥫 Preparing to run code coverage…"
 	mkdir -p cover_db
 	${DOCKER_COMPOSE_TEST} run --rm backend cover -delete
@@ -173,13 +178,13 @@ coverage_txt:
 #----------#
 # Services #
 #----------#
-build_lang:
+build_lang: create_folders
 	@echo "🥫 Rebuild language"
     # Run build_lang.pl
     # Languages may build taxonomies on-the-fly so include GITHUB_TOKEN so results can be cached
 	${DOCKER_COMPOSE} run --rm -e GITHUB_TOKEN=${GITHUB_TOKEN} backend perl -I/opt/product-opener/lib -I/opt/perl/local/lib/perl5 /opt/product-opener/scripts/build_lang.pl
 
-build_lang_test:
+build_lang_test: create_folders
 # Run build_lang.pl in test env
 	${DOCKER_COMPOSE_TEST} run --rm -e GITHUB_TOKEN=${GITHUB_TOKEN} backend perl -I/opt/product-opener/lib -I/opt/perl/local/lib/perl5 /opt/product-opener/scripts/build_lang.pl
 
@@ -243,14 +248,14 @@ lint: lint_perltidy
 tests: build_lang_test unit_test integration_test
 
 # add COVER_OPTS='-e HARNESS_PERL_SWITCHES="-MDevel::Cover"' if you want to trigger code coverage report generation
-unit_test:
+unit_test: create_folders
 	@echo "🥫 Running unit tests …"
 	${DOCKER_COMPOSE_TEST} up -d memcached postgres mongodb
 	${DOCKER_COMPOSE_TEST} run ${COVER_OPTS} -T --rm backend prove -l --jobs ${CPU_COUNT} -r tests/unit
 	${DOCKER_COMPOSE_TEST} stop
 	@echo "🥫 unit tests success"
 
-integration_test:
+integration_test: create_folders
 	@echo "🥫 Running integration tests …"
 # we launch the server and run tests within same container
 # we also need dynamicfront for some assets to exists
@@ -268,14 +273,15 @@ test-stop:
 
 # usage:  make test-unit test=test-name.t
 # you can add args= to pass options, like args="-d" to debug
-test-unit: guard-test
+test-unit: guard-test create_folders
 	@echo "🥫 Running test: 'tests/unit/${test}' …"
 	${DOCKER_COMPOSE_TEST} up -d memcached postgres mongodb
 	${DOCKER_COMPOSE_TEST} run --rm backend perl ${args} tests/unit/${test}
 
 # usage:  make test-int test=test-name.t
 # to update expected results: make test-int test="test-name.t --update-expected-results"
-test-int: guard-test # usage: make test-int test=test-file.t
+# you can add args= to pass options, like args="-d" to debug
+test-int: guard-test create_folders
 	@echo "🥫 Running test: 'tests/integration/${test}' …"
 	${DOCKER_COMPOSE_TEST} up -d memcached postgres mongodb backend dynamicfront incron minion
 	${DOCKER_COMPOSE_TEST} exec backend perl ${args} tests/integration/${test}
@@ -386,6 +392,9 @@ create_external_volumes:
 # local data
 	docker volume create --driver=local -o type=none -o o=bind -o device=${DOCKER_LOCAL_DATA}/data ${COMPOSE_PROJECT_NAME}_html_data
 	docker volume create --driver=local -o type=none -o o=bind -o device=${DOCKER_LOCAL_DATA}/podata ${COMPOSE_PROJECT_NAME}_podata
+# note for this one, it should be shared with pro instance in the future
+	docker volume create --driver=local -o type=none -o o=bind -o device=${DOCKER_LOCAL_DATA}/export_files ${COMPOSE_PROJECT_NAME}_export_files
+
 
 create_external_networks:
 	@echo "🥫 Creating external networks (production only) …"

cgi/change_password.pl

@@ -25,6 +25,7 @@
 use CGI::Carp qw(fatalsToBrowser);
 
 use ProductOpener::Config qw/:all/;
+use ProductOpener::Paths qw/:all/;
 use ProductOpener::Store qw/:all/;
 use ProductOpener::Display qw/:all/;
 use ProductOpener::Users qw/:all/;
@@ -51,7 +52,7 @@
 my @errors = ();
 
 if ($ENV{'REQUEST_METHOD'} eq 'POST') {
-	my $user_file = "$data_root/users/" . get_string_id_for_lang('no_language', $User_id) . '.sto';
+	my $user_file = "$BASE_DIRS{USERS}/" . get_string_id_for_lang('no_language', $User_id) . '.sto';
 	my $user_ref = retrieve($user_file);
 	if (not(defined $user_ref)) {
 		push @errors, 'undefined user';
@@ -83,7 +84,7 @@
 	}
 	else {
 		$user_ref->{encrypted_password} = create_password_hash(encode_utf8(decode utf8 => single_param('password')));
-		store("$data_root/users/$User_id.sto", $user_ref);
+		store("$BASE_DIRS{USERS}/$User_id.sto", $user_ref);
 		$template_data_ref->{success} = 1;
 	}
 }

cgi/display.pl

@@ -185,7 +185,13 @@
 	display_properties($request_ref);
 }
 elsif (defined $request_ref->{text}) {
-	display_text($request_ref);
+	if ($request_ref->{text} =~ /^products_stats/) {
+		display_stats($request_ref);
+	}
+	else {
+		# note: this also displays the index or index-pro pages
+		display_text($request_ref);
+	}
 }
 elsif (defined $request_ref->{mission}) {
 	display_mission($request_ref);

cgi/import_file_job_status.pl

@@ -28,6 +28,7 @@
 use CGI::Carp qw(fatalsToBrowser);
 
 use ProductOpener::Config qw/:all/;
+use ProductOpener::Paths qw/:all/;
 use ProductOpener::Store qw/:all/;
 use ProductOpener::Display qw/:all/;
 use ProductOpener::Users qw/:all/;
@@ -75,7 +76,7 @@
 }
 else {
 
-	$import_files_ref = retrieve("$data_root/import_files/${Owner_id}/import_files.sto");
+	$import_files_ref = retrieve("$BASE_DIRS{IMPORT_FILES}/${Owner_id}/import_files.sto");
 
 	if ((not defined $import_files_ref) or (not defined $import_files_ref->{$file_id})) {
 		$data{error} = "file_id_not_found";