Merge branch 'dev' into feature/make_project_desc_confrigurable_1137

.env.example

@@ -14,6 +14,7 @@ SITE_ID=4
 DB_USER=physionet
 DB_PASSWORD=password
 DB_HOST=localhost
+DB_PORT=5432
 DB_NAME=physionet
 
 # Emails
@@ -137,6 +138,10 @@ SSO_LOGIN_BUTTON_TEXT="Single Sign-On"
 ENABLE_FILE_DOWNLOADS_OPTION=true
 COPY_FILES_TO_NEW_VERSION=true
 
+# Allowed Access Policies [Optional] - This is a list of access policies that can be assigned to a newly submitted project on the 'Project Access' page.
+# The environment variable is a stringified list of access policies separated by commas
+ALLOWED_ACCESS_POLICIES="OPEN,RESTRICTED,CREDENTIALED,CONTRIBUTOR_REVIEW"
+
 # Used for bucket creation
 GCP_DOMAIN=
 

README.md

@@ -41,6 +41,12 @@ The code should dynamically reload in development, however, if there are any iss
 
 Docker-compose uses volumes to persist the database contents and data directories (media and static files). To clean up the created containers, networks and volumes stop `docker-compose up` and run `docker-compose down -v`. Do not run `docker-compose down -v` if you want to retain current database contents.
 
+## Background tasks
+
+Background tasks are managed by [Django Q2](https://django-q2.readthedocs.io/en/master/), "a native Django task queue, scheduler and worker application using Python multiprocessing".
+
+If you would like to run background tasks on your development server, you will need to start the task manager with `python manage.py qcluster`
+
 ## Using a debugger with Docker
 
 To access a debug prompt raised using `breakpoint()`:

deploy/MIGRATION-CHECKLIST.md

@@ -0,0 +1,41 @@
+Server migration checklist
+==========================
+
+[ ] **Target**: Check network (reachable, link speed)
+[ ] **Target**: Check disk (enough space, no RAID problems)
+[ ] **Target**: Pre-synchronize files
+[ ] **DNS**: Set minimum TTL
+[ ] **Target**: Check/upgrade system packages
+[ ] **Target**: Update `/physionet/physionet-build.git`
+[ ] **Target**: Update `/physionet/.env`
+[ ] **Target**: Update system config files
+[ ] **Target**: Enable `SYSTEM_MAINTENANCE_NO_CHANGES`
+[ ] **Source**: Dump and transfer database
+[ ] **Target**: Restore database (reload uwsgi)
+[ ] **Target**: Test nginx/uwsgi is working
+[ ] **Target**: Update `/etc/aliases`
+[ ] **Target**: Update certificates
+[ ] **Target**: Test postfix is working
+[ ] **Target**: Test haproxy is working
+[ ] **Target**: Configure iptables
+[ ] **Target**: Configure qdisc
+
+[ ] (proxy-backward) **Target**: Redirect connections to haproxy
+[ ] (proxy-backward) **DNS**: Update records
+
+[ ] **Target**: Stop background-tasks daemon
+[ ] **Source**: Stop background-tasks daemon
+[ ] **Source**: Enable `SYSTEM_MAINTENANCE_NO_UPLOAD` (reload uwsgi)
+[ ] **Target**: Synchronize files
+[ ] **Source**: Enable `SYSTEM_MAINTENANCE_NO_CHANGES` (reload uwsgi)
+[ ] **Source**: Dump and transfer database
+[ ] **Target**: Restore database (reload uwsgi)
+[ ] **Target**: Check new content is visible
+
+[ ] (proxy-forward) **Source**: Redirect connections to haproxy
+[ ] (proxy-forward) **DNS**: Update records
+
+[ ] **Target**: Disable `SYSTEM_MAINTENANCE_NO_CHANGES` (reload uwsgi)
+[ ] **Target**: Remove iptables redirections
+[ ] **Target**: Start background-tasks daemon
+[ ] **DNS**: Set normal TTL

deploy/common/etc/sudoers.d/physionet

@@ -16,3 +16,7 @@ pn ALL=(root:root) NOPASSWD: \
 # Restart django-background-tasks.service (if it is running)
 pn ALL=(root:root) NOPASSWD: \
     /bin/systemctl try-restart django-background-tasks.service
+
+# Restart django-q2-tasks.service (if it is running)
+pn ALL=(root:root) NOPASSWD: \
+    /bin/systemctl try-restart django-q2-tasks.service

deploy/post-receive

@@ -112,6 +112,7 @@ else
 fi
 
 # Restart the django-background-tasks server
+# Background tasks will soon be migrated to Django Q2
 (
     cd $working_dir/physionet-django
     if cmp -s OLD-TARGETS LATE-TARGETS; then
@@ -130,6 +131,25 @@ fi
     fi
 )
 
+# Restart the Django Q2 server
+(
+    cd $working_dir/physionet-django
+    if cmp -s OLD-TARGETS LATE-TARGETS; then
+        # assume that if there are no migrations, no need to restart
+        # background tasks (and don't even bother showing a message)
+        :
+    elif [ -n "$no_bgtasks" ]; then
+        echo "- SKIPPING restarting django-q2-tasks due to"
+        echo "  --push-option=no-bgtasks"
+    elif [ -n "$no_install" ]; then
+        echo "- SKIPPING restarting django-q2-tasks due to"
+        echo "  --push-option=no-install"
+    else
+        echo "* Restarting django-q2-tasks..."
+        sudo systemctl try-restart django-q2-tasks.service
+    fi
+)
+
 # Run late migrations (as well as early migrations, if they were
 # skipped before)
 (

deploy/production/etc/rsyslog.d/django_q.conf

@@ -0,0 +1,3 @@
+# Filter to get all background tasks and send them to a custom location
+:programname, isequal, "django-q2-tasks" -/data/log/background_tasks/django_q_tasks.log
+& stop

deploy/production/etc/systemd/system/django-q2-tasks.service

@@ -0,0 +1,18 @@
+[Unit]
+Description=Command that runs Django Q2 tasks
+After=emperor.uwsgi.service
+
+[Service]
+Environment=DJANGO_SETTINGS_MODULE=physionet.settings.production
+ExecStart=/physionet/python-env/physionet/bin/python /physionet/physionet-build/physionet-django/manage.py qcluster
+StandardError=syslog
+SyslogIdentifier=django-q2-tasks
+Restart=always
+KillSignal=SIGINT
+Type=simple
+NotifyAccess=all
+User=www-data
+Group=www-data
+
+[Install]
+WantedBy=multi-user.target

deploy/staging/etc/rsyslog.d/django_q.conf

@@ -0,0 +1,3 @@
+# Filter to get all background tasks and send them to a custom location
+:programname, isequal, "django-q2-tasks" -/data/log/background_tasks/django_q_tasks.log
+& stop

deploy/staging/etc/systemd/system/django-q2-tasks.service

@@ -0,0 +1,18 @@
+[Unit]
+Description=Command that runs Django Q2 tasks
+After=emperor.uwsgi.service
+
+[Service]
+Environment=DJANGO_SETTINGS_MODULE=physionet.settings.staging
+ExecStart=/physionet/python-env/physionet/bin/python /physionet/physionet-build/physionet-django/manage.py qcluster
+StandardError=syslog
+SyslogIdentifier=django-q2-tasks
+Restart=always
+KillSignal=SIGINT
+Type=simple
+NotifyAccess=all
+User=www-data
+Group=www-data
+
+[Install]
+WantedBy=multi-user.target

deploy/test-server/install-pn-test-server

@@ -49,6 +49,8 @@ apt-get install -y \
         virtualenv python3-dev libpq-dev libffi-dev \
         `# Requirements for building LightWAVE` \
         build-essential libcap-dev libflac-dev libseccomp-dev \
+        `# Logging system` \
+        rsyslog \
         `# Other tools required by PhysioNet` \
         zip unzip xfsprogs \
         `# Development and administration tools` \
@@ -130,9 +132,9 @@ su pn -c '
    cp -p /physionet/physionet-build/deploy/post-receive hooks/post-receive
 
    cd /physionet/physionet-build
-   grep -v ^SECRET_KEY= < .env.example > .env
+   grep -v -e ^SECRET_KEY= -e ^DB_PASSWORD= < .env.example > .env
    printf SECRET_KEY=%s\\n "$SECRET_KEY" >> .env
-   printf DATABASES_PASSWORD=%s\\n "$DBPASSWORD" >> .env
+   printf DB_PASSWORD=%s\\n "$DBPASSWORD" >> .env
    chgrp www-data .env
    chmod 640 .env
 
@@ -230,13 +232,21 @@ systemctl restart postfix
 systemctl disable systemd-timesyncd
 systemctl stop systemd-timesyncd
 
+systemctl enable rsyslog
+systemctl restart rsyslog
+
 systemctl daemon-reload
 systemctl enable emperor.uwsgi
 systemctl restart emperor.uwsgi
 systemctl restart nginx
+
+# django-background-tasks will be replaced with Django Q2 soon
 systemctl enable django-background-tasks
 systemctl restart django-background-tasks
 
+systemctl enable django-q2-tasks
+systemctl restart django-q2-tasks
+
 su pn -c '
    cd /physionet/physionet-build.git
    echo

physionet-django/console/navbar.py

@@ -168,6 +168,8 @@ def get_menu_items(self, request):
     NavLink(_('Training check'), 'training_list', 'school',
             view_args=['review']),
 
+    NavLink(_('Courses'), 'courses', 'chalkboard-teacher'),
+
     NavSubmenu(_('Events'), 'events', 'clipboard-list', [
         NavLink(_('Active'), 'event_active'),
         NavLink(_('Archived'), 'event_archive'),

physionet-django/console/templates/console/console_navbar.html

@@ -13,42 +13,37 @@
     <ul class="navbar-nav navbar-sidenav" id="sideAccordion">
       {% console_nav_menu_items request as nav_menu_items %}
       {% for item in nav_menu_items %}
-        {% if item.subitems %}
-          <li class="nav-item">
-            <a id="nav_{{ item.name }}_dropdown"
-               class="nav-link nav-link-collapse drop {{ item.active|yesno:',collapsed' }}"
-               data-toggle="collapse"
-               data-target="#nav_{{ item.name }}_components"
-               data-parent="#sideAccordion"
-               aria-expanded="{{ item.active|yesno:'true,false' }}"
-               href="#">
-              {% if item.icon %}
-                <span class="nav-link-icon fa fa-fw fa-{{ item.icon }}"></span>
-              {% endif %}
-              <span class="nav-link-text">{{ item.title }}</span>
-            </a>
-            <ul class="sidenav-second-level collapse {{ item.active|yesno:'show,' }}"
-                id="nav_{{ item.name }}_components">
-              {% for subitem in item.subitems %}
-                <li class="nav-item {{ subitem.active|yesno:'active,' }}">
-                  <a id="nav_{{ subitem.name }}" class="nav-link"
-                     href="{{ subitem.url }}">{{ subitem.title }}</a>
-                </li>
-              {% endfor %}
-            </ul>
-          </li>
-        {% else %}
-          <li class="nav-item {{ item.active|yesno:'active,' }}">
-            <a id="nav_{{ item.name }}" class="nav-link" href="{{ item.url }}">
-              {% if item.icon %}
-                <span class="nav-link-icon fa fa-fw fa-{{ item.icon }}"></span>
-              {% endif %}
-              <span class="nav-link-text">{{ item.title }}</span>
-            </a>
+      {% if item.subitems %}
+      <li class="nav-item">
+        <a id="nav_{{ item.name }}_dropdown"
+          class="nav-link nav-link-collapse drop {{ item.active|yesno:',collapsed' }}" data-toggle="collapse"
+          data-target="#nav_{{ item.name }}_components" data-parent="#sideAccordion"
+          aria-expanded="{{ item.active|yesno:'true,false' }}" href="#">
+          {% if item.icon %}
+          <span class="nav-link-icon fa fa-fw fa-{{ item.icon }}"></span>
+          {% endif %}
+          <span class="nav-link-text">{{ item.title }}</span>
+        </a>
+        <ul class="sidenav-second-level collapse {{ item.active|yesno:'show,' }}" id="nav_{{ item.name }}_components">
+          {% for subitem in item.subitems %}
+          <li class="nav-item {{ subitem.active|yesno:'active,' }}">
+            <a id="nav_{{ subitem.name }}" class="nav-link" href="{{ subitem.url }}">{{ subitem.title }}</a>
           </li>
-        {% endif %}
+          {% endfor %}
+        </ul>
+      </li>
+      {% else %}
+      <li class="nav-item {{ item.active|yesno:'active,' }}">
+        <a id="nav_{{ item.name }}" class="nav-link" href="{{ item.url }}">
+          {% if item.icon %}
+          <span class="nav-link-icon fa fa-fw fa-{{ item.icon }}"></span>
+          {% endif %}
+          <span class="nav-link-text">{{ item.title }}</span>
+        </a>
+      </li>
+      {% endif %}
       {% endfor %}
-    <!-- end of menu items -->
+      <!-- end of menu items -->
     </ul>
 
     <ul class="navbar-nav sidenav-toggler">
@@ -64,10 +59,11 @@
   </div>
   <div class="navbar-search">
     <form class="form-inline" action="{% url 'content_index' %}">
-        <input name="topic" class="search-input" type="text" placeholder="Search">
-        <span class="input-group-btn">
-          <button id="search-button" type="submit" class="btn-search my-2 my-sm-0" type="button"><i class="fa fa-search"></i></button>
-        </span>
+      <input name="topic" class="search-input" type="text" placeholder="Search">
+      <span class="input-group-btn">
+        <button id="search-button" type="submit" class="btn-search my-2 my-sm-0" type="button"><i
+            class="fa fa-search"></i></button>
+      </span>
     </form>
   </div>
 </nav>