Update to Python 3.7

.circleci/config.yml

@@ -0,0 +1,96 @@
+version: 2
+jobs:
+  build:
+    working_directory: ~/phovea
+    docker:
+      - image: circleci/python:3.7-buster-node-browsers # for node version see Dockerfile on https://hub.docker.com/r/circleci/python
+    steps:
+      - checkout
+      - run:
+          name: Show Node.js and npm version
+          command: |
+            node -v
+            npm -v
+      - run:
+          name: Show Python and pip version
+          command: |
+            python --version
+            pip --version
+      - run:
+          name: Install Docker packages from docker_packages.txt
+          command: |
+            (!(test -f docker_packages.txt) || (cat docker_packages.txt | xargs sudo apt-get install -y))
+      - restore_cache:
+          key: deps2-{{ .Branch }}-{{ checksum "package.json" }}
+      - run:
+          name: Install npm dependencies
+          command: npm install
+      - run:
+          name: Remove npm dependencies installed from git repositories (avoid caching of old commits)
+          command: |
+            (grep -l '._resolved.: .\(git[^:]*\|bitbucket\):' ./node_modules/*/package.json || true) | xargs -r dirname | xargs -r rm -rf
+      - save_cache:
+          key: deps2-{{ .Branch }}-{{ checksum "package.json" }}
+          paths:
+          - ./node_modules
+      - run:
+          name: Install npm dependencies from git repositories (always get latest commit)
+          command: npm install
+      - run:
+          name: Show installed npm dependencies
+          command: npm list --depth=1 || true
+      - restore_cache:
+          key: deps1-{{ .Branch }}-{{ checksum "requirements.txt" }}-{{ checksum "requirements_dev.txt" }}
+      - run:
+          name: Install pip requirements
+          command: |
+            virtualenv ~/venv
+            . ~/venv/bin/activate
+            pip install -r requirements_dev.txt
+            pip install -r requirements.txt
+      - save_cache:
+          key: deps1-{{ .Branch }}-{{ checksum "requirements.txt" }}-{{ checksum "requirements_dev.txt" }}
+          paths:
+          - ~/venv
+      - run:
+          name: Force an update of pip dependencies from git repositories # not sure if this is working ?
+          command: |
+            . ~/venv/bin/activate
+            pip install --upgrade --upgrade-strategy=only-if-needed -r requirements.txt
+      - run:
+          name: Show installed pip packages
+          command: pip list || true
+      - run:
+          name: Build
+          command: |
+            . ~/venv/bin/activate
+            npm run dist
+      - store_artifacts:
+          path: dist
+          destination: dist
+workflows:
+  version: 2
+#  build-nightly:
+#    triggers:
+#      - schedule:
+#          cron: "15 1 * * 1-5" # "At 01:15 on every day-of-week from Monday through Friday.”, see: https://crontab.guru/#15_1_*_*_1-5
+#          filters:
+#            branches:
+#              only:
+#                - develop
+#    jobs:
+#      - build
+  build-branch:
+    jobs:
+      - build:
+          filters:
+            tags:
+              ignore: /^v.*/
+  build-tag:
+    jobs:
+      - build:
+          filters:
+            branches:
+              ignore: /.*/
+            tags:
+              only: /^v.*/

.gitignore

@@ -22,4 +22,5 @@ node_modules/
 /tests/**/*.js
 *.map
 *.css
+/.cache-loader
 package-lock.json

.gitlab-ci.yml

@@ -0,0 +1,57 @@
+image: circleci/python:3.7-buster-node-browsers # for node version see Dockerfile on https://hub.docker.com/r/circleci/python
+
+variables:
+  GIT_DEPTH: "1"
+
+cache:
+  key: "$CI_REPOSITORY_URL-$CI_COMMIT_REF_NAME"
+  paths:
+  - node_modules
+  - ~/venv
+
+before_script:
+  # Install ssh-agent if not already installed, it is required by Docker.
+  # (change apt-get to yum if you use a CentOS-based image)
+  - 'which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )'
+
+  # Run ssh-agent (inside the build environment)
+  - eval $(ssh-agent -s)
+
+  # Add the SSH key stored in SSH_PRIVATE_KEY variable to the agent store
+  - ssh-add <(echo "$SSH_PRIVATE_KEY")
+
+  # For Docker builds disable host key checking. Be aware that by adding that
+  # you are suspectible to man-in-the-middle attacks.
+  # WARNING: Use this only with the Docker executor, if you use it with shell
+  # you will overwrite your user's SSH config.
+  - mkdir -p ~/.ssh
+  - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config'
+
+stages:
+  - install
+  - build
+
+install-npm-wee:
+  stage: install
+  script:
+  - npm install
+
+install-pip-wee:
+  stage: install
+  script: |
+    (!(test -f docker_packages.txt) || (cat docker_packages.txt | xargs sudo apt-get install -y))
+    virtualenv ~/venv
+    . ~/venv/bin/activate
+    pip install --upgrade --upgrade-strategy=only-if-needed -r requirements_dev.txt
+    pip install --upgrade --upgrade-strategy=only-if-needed -r requirements.txt
+
+dist:
+  stage: build
+  script: |
+    . ~/venv/bin/activate
+    npm run dist
+  allow_failure: false
+  artifacts:
+    expire_in: 1 week
+    paths:
+      - dist

.travis.yml

@@ -1,8 +1,7 @@
 language: python
 sudo: required
 env:
-- TOXENV=py27
-- TOXENV=py34
+- TOXENV=py37
 node_js:
 - 6
 addons:

.yo-rc.json

@@ -43,6 +43,12 @@
     "unknown": {
       "requirements": [],
       "dockerPackages": []
+    },
+    "promptValues": {
+      "authorName": "The Caleydo Team",
+      "authorEmail": "contact@caleydo.org",
+      "authorUrl": "https://caleydo.org",
+      "githubAccount": "phovea"
     }
   }
 }

buildInfo.js

@@ -2,13 +2,11 @@
  * Created by sam on 13.11.2016.
  */
 
-
 const spawnSync = require('child_process').spawnSync;
 const path = require('path');
 const resolve = path.resolve;
 const fs = require('fs');
 
-
 function dependencyGraph(cwd) {
   const npm = process.platform === 'win32' ? 'npm.cmd' : 'npm';
   const r = spawnSync(npm, ['ls', '--prod', '--json'], {
@@ -37,7 +35,7 @@ function gitHead(cwd) {
 
 function resolveModules() {
   const reg = fs.readFileSync('../phovea_registry.js').toString();
-  const regex = /import '(.*)\/phovea_registry.js'/g;
+  const regex = /^import '(.*)\/phovea_registry.js'/gm;
   const modules = [];
   let r;
   while ((r = regex.exec(reg)) !== null) {
@@ -55,19 +53,20 @@ function resolveWorkspace() {
   const workspaceDeps = dependencyGraph('..').dependencies;
   const modules = new Set(resolveModules());
 
+  let deps = null;
   const resolveModule = (m) => {
     console.log('resolve', m);
-    const pkg = require(`../${m}/package.json`);
+    const pkg = JSON.parse(fs.readFileSync(`../${m}/package.json`).toString());
     const head = gitHead('../' + m);
     const repo = pkg.repository.url;
     return {
       name: pkg.name,
       version: pkg.version,
-      resolved: head ? `${repo.endsWith('.git') ? repo.slice(0, repo.length-4) : repo}/commit/${head}` : pkg.version,
+      resolved: head ? `${repo.endsWith('.git') ? repo.slice(0, repo.length - 4) : repo}/commit/${head}` : pkg.version,
       dependencies: deps(pkg.dependencies)
     };
   };
-  const deps = (deps) => {
+  deps = (deps) => {
     const r = {};
     Object.keys(deps).forEach((d) => {
       if (d in workspaceDeps) {
@@ -121,59 +120,57 @@ function generate() {
   const isWorkspaceContext = fs.existsSync('../phovea_registry.js');
   if (isWorkspaceContext) {
     return resolveWorkspace();
-  } else {
-    return resolveSingle();
   }
+  return resolveSingle();
 }
 
-
 const IS_WINDOWS = process.platform === 'win32';
 
 function tmpdir() {
   if (IS_WINDOWS) {
     return process.env.TEMP || process.env.TMP ||
-           (process.env.SystemRoot || process.env.windir) + '\\temp';
-  } else {
-    return process.env.TMPDIR || process.env.TMP || process.env.TEMP || '/tmp';
+      (process.env.SystemRoot || process.env.windir) + '\\temp';
   }
+  return process.env.TMPDIR || process.env.TMP || process.env.TEMP || '/tmp';
 }
 
 function resolveScreenshot() {
   const f = resolve(__dirname, 'media/screenshot.png');
   if (!fs.existsSync(f)) {
     return null;
   }
-  const buffer = new Buffer(fs.readFileSync(f)).toString('base64');
+  const buffer = Buffer.from(fs.readFileSync(f)).toString('base64');
   return `data:image/png;base64,${buffer}`;
 }
 
 function metaData(pkg) {
   pkg = pkg || require(`./package.json`);
   return {
     name: pkg.name,
+    displayName: pkg.displayName,
     version: pkg.version,
     repository: pkg.repository.url,
+    homepage: pkg.homepage,
     description: pkg.description,
     screenshot: resolveScreenshot()
   };
 }
 
 module.exports.metaData = metaData;
-module.exports.metaDataTmpFile = function(pkg) {
+module.exports.metaDataTmpFile = function (pkg) {
   const s = metaData(pkg);
   const file = `${tmpdir()}/metaData${Math.random().toString(36).slice(-8)}.txt`;
   fs.writeFileSync(file, JSON.stringify(s, null, ' '));
   return file;
 };
 module.exports.generate = generate;
-module.exports.tmpFile = function() {
+module.exports.tmpFile = function () {
   const s = generate();
   const file = `${tmpdir()}/buildInfo${Math.random().toString(36).slice(-8)}.txt`;
   fs.writeFileSync(file, JSON.stringify(s, null, ' '));
   return file;
 };
 
-
 if (require.main === module) {
   fs.writeFile('deps.json', JSON.stringify(generate(), null, ' '));
 }

buildPython.js

@@ -0,0 +1,69 @@
+/**
+ * Created by sam on 13.11.2016.
+ */
+
+const spawnSync = require('child_process').spawnSync;
+const fs = require('fs');
+
+function gitHead(cwd) {
+  const r = spawnSync('git', ['rev-parse', '--verify', 'HEAD'], {
+    cwd: cwd
+  });
+  if (!r.stdout) {
+    console.error(cwd, r.error);
+    return 'error';
+  }
+  return r.stdout.toString().trim();
+}
+
+function resolvePlugin(repo, version) {
+  if (fs.lstatSync('.git').isDirectory() && repo) {
+    if (repo.endsWith('.git')) {
+      repo = repo.slice(0, repo.length - 4);
+      return repo + '/commit/' + gitHead('.');
+    }
+  }
+  // not a git repo
+  return version;
+}
+
+function toVersion(v) {
+  const now = new Date().toISOString();
+  // %Y%m%d-%H%M%S
+  const fmt = now
+    .replace(/T/, ' ')
+    .replace(/\..+/, '')
+    .replace(/[-:]/, '')
+    .replace(' ', '-');
+  return v.replace('SNAPSHOT', fmt);
+}
+
+function _main() {
+  const pkg = require('./package.json');
+  const name = pkg.name;
+  const version = toVersion(pkg.version);
+  const resolved = resolvePlugin((pkg.repository || {}).url, version);
+
+  const buildInfo = {
+    name,
+    version,
+    resolved,
+    description: pkg.description,
+    homepage: pkg.homepage,
+    repository: (pkg.repository || {}).url
+  };
+
+  const l = ('build/source/' + name.toLowerCase()).split('/');
+  l.forEach((_, i) => {
+    const path = l.slice(0, i + 1).join('/');
+    if (!fs.existsSync(path)) {
+      fs.mkdirSync(path);
+    }
+  });
+
+  fs.writeFileSync('build/source/' + name.toLowerCase() + '/buildInfo.json', JSON.stringify(buildInfo, null, ' '));
+}
+
+if (require.main === module) {
+  _main();
+}

deploy/Dockerfile

@@ -1,16 +1,20 @@
-FROM caleydo/phovea_python:v1.0
+FROM python:3.7
 
-MAINTAINER Samuel Gratzl <samuel.gratzl@datavisyn.io>
+LABEL maintainer="contact@caleydo.org"
 WORKDIR /phovea
-RUN printf "from phovea_server import launch\nlaunch.run()\n" > ./launch.py
+RUN printf "from phovea_server import launch\nlaunch.run()\n" > /phovea/launch.py
 
 # install dependencies last step such that everything before can be cached
-COPY requirements*.txt docker_packages.txt ./
-RUN (!(test -f docker_packages.txt) || (cat docker_packages.txt | xargs apt-get install -y)) && pip install --no-cache-dir -r requirements.txt
+COPY requirements*.txt docker_packages.txt docker_script*.sh ./
+RUN (!(test -s docker_packages.txt) || (apt-get update && \
+  (cat docker_packages.txt | xargs apt-get install -y))) && \
+  (pip install --no-cache-dir -r requirements.txt)
+RUN (!(test -f docker_script.sh) || (bash ./docker_script.sh))
 
 COPY ./build/source ./
 
 ENV PHOVEA_ENV=production
 ENV PHOVEA_SERVICE=celery
+ENV PHOVEA_CONFIG_PATH=config.json
 CMD python launch.py --env ${PHOVEA_ENV} ${PHOVEA_SERVICE}
 EXPOSE 80