Merge pull request #3389 from sergerdn/feature/travis-python38

improve Travis tests
2021-02-01 16:03:19 -08:00 · 2021-02-01 16:03:19 -08:00 · b699683fa3
parent 770a9539bb 9fe912892b
commit b699683fa3
5 changed files with 138 additions and 80 deletions
--- a/.travis.yml
+++ b/.travis.yml
@ -1,16 +1,50 @@
-language: python
-dist: bionic
-
 node_js:
-  - "6.2.0"
+  - "10"

-addons:
-  postgresql: "9.4"
-
-matrix:
+jobs:
  include:
-    - python: "3.7"
+    - name: "python3.7-postgresql-9.4-bionic"
+      dist: bionic
+      language: python
+      python: "3.7"
      env: TOXENV=py37
+      addons:
+        postgresql: "9.4"
+        chrome: stable
+      services:
+        - xvfb
+    - name: "python3.7-postgresql-10-bionic"
+      dist: bionic
+      language: python
+      python: "3.7"
+      env: TOXENV=py37
+      addons:
+        postgresql: '10'
+        chrome: stable
+        apt:
+          packages:
+            - postgresql-10
+            - postgresql-client-10
+            - postgresql-server-dev-10
+      services:
+        - postgresql
+        - xvfb
+    - name: "python3.8-postgresql-12-focal"
+      dist: focal
+      language: python
+      python: "3.8"
+      env: TOXENV=py38
+      addons:
+        postgresql: '12'
+        chrome: stable
+        apt:
+          packages:
+            - postgresql-12
+            - postgresql-client-12
+            - postgresql-server-dev-12
+      services:
+        - postgresql
+        - xvfb

 cache:
  directories:
@ -26,13 +60,23 @@ env:
    # https://github.com/travis-ci/travis-ci/issues/5246#issuecomment-166460882
    - BOTO_CONFIG=/doesnotexist

+before_install:
+  - export CHROME_BIN=/usr/bin/google-chrome
+
 before_script:
+  - sudo systemctl stop postgresql
+  # the port may have been auto-configured to use 5433 if it thought 5422 was already in use,
+  # for some reason it happens very often
+  # https://github.com/travis-ci/travis-build/blob/master/lib/travis/build/bash/travis_setup_postgresql.bash#L52
+  - sudo sed -i -e 's/5433/5432/' /etc/postgresql/*/main/postgresql.conf
+  - sudo systemctl restart postgresql
  - psql -c "create database lemur;" -U postgres
  - psql -c "create user lemur with password 'lemur;'" -U postgres
  - psql lemur -c "create extension IF NOT EXISTS pg_trgm;" -U postgres
  - npm config set registry https://registry.npmjs.org
-  - npm install -g bower
+  - npm install -g npm@latest bower
  - pip install --upgrade setuptools
+  - export DISPLAY=:99.0

 install:
  - pip install coveralls
@ -41,6 +85,7 @@ install:
 script:
  - make test
  - bandit -r . -ll -ii -x lemur/tests/,docs
+  - make test-js

 after_success:
  - coveralls
@ -51,4 +96,4 @@ notifications:
      - lemur@netflix.com
    on_success: never
    on_failure: always
-    on_cancel:  never # Dependbot cancels Travis before rebase and triggers too many emails
+    on_cancel: never # Dependbot cancels Travis before rebase and triggers too many emails
--- a/gulp/build.js
+++ b/gulp/build.js
@ -7,7 +7,7 @@ var gulp = require('gulp'),
  gulpif = require('gulp-if'),
  gutil = require('gulp-util'),
  foreach = require('gulp-foreach'),
-  path =require('path'),
+  path = require('path'),
  merge = require('merge-stream'),
  del = require('del'),
  size = require('gulp-size'),
@ -28,6 +28,23 @@ var gulp = require('gulp'),
  replace = require('gulp-replace'),
  argv = require('yargs').argv;

+// http://stackoverflow.com/questions/1144783/replacing-all-occurrences-of-a-string-in-javascript
+function escapeRegExp(string) {
+  return string.replace(/([.*+?^=!:${}()|\[\]\/\\])/g, '\\$1');
+}
+
+function replaceAll(string, find, replace) {
+  return string.replace(new RegExp(escapeRegExp(find), 'g'), replace);
+}
+
+function stringSrc(filename, string) {
+  let src = require('stream').Readable({objectMode: true});
+  src._read = function () {
+    this.push(new gutil.File({cwd: '', base: '', path: filename, contents: new Buffer(string)}));
+    this.push(null);
+  };
+  return src;
+}

 gulp.task('clean', function (done) {
  del(['.tmp', 'lemur/static/dist'], done);
@ -39,12 +56,18 @@ gulp.task('default', gulp.series(['clean'], function () {
 }));

 gulp.task('test', function (done) {
-  // returning the promise
-  return new karma.Server({
+  new karma.Server({
    configFile: __dirname + '/karma.conf.js',
    singleRun: true
-  }, function() {
-    done();
+  }, function (err) {
+    if (err === 0) {
+      done();
+    } else {
+      // if karma server failed to start raise error
+      done(new gutil.PluginError('karma', {
+        message: 'Karma Tests failed'
+      }));
+    }
  }).start();
 });

@ -73,7 +96,7 @@ gulp.task('dev:styles', function () {
    return fileName.indexOf(suffix) === 0;
  };

-  var fileList = [
+  let fileList = [
    'bower_components/bootswatch/sandstone/bootswatch.less',
    'bower_components/fontawesome/css/font-awesome.css',
    'bower_components/angular-chart.js/dist/angular-chart.css',
@ -89,10 +112,8 @@ gulp.task('dev:styles', function () {
  return gulp.src(fileList)
    .pipe(gulpif(isBootswatchFile, foreach(function (stream, file) {
      let themeName = path.basename(path.dirname(file.path)),
-        content = replaceAll(baseContent, '$theme$', themeName),
-        file2 = string_src('bootstrap-' +  themeName + '.less', content);
-
-      return file2;
+        content = replaceAll(baseContent, '$theme$', themeName);
+      return stringSrc('bootstrap-' + themeName + '.less', content);
    })))
    .pipe(less())
    .pipe(gulpif(isBootstrapFile, foreach(function (stream, file) {
@ -101,7 +122,7 @@ gulp.task('dev:styles', function () {

      // http://stackoverflow.com/questions/21719833/gulp-how-to-add-src-files-in-the-middle-of-a-pipe
      // https://github.com/gulpjs/gulp/blob/master/docs/recipes/using-multiple-sources-in-one-task.md
-      return merge(stream, gulp.src(['.tmp/styles/font-awesome.css', '.tmp/styles/lemur.css'], { allowEmpty: true }))
+      return merge(stream, gulp.src(['.tmp/styles/font-awesome.css', '.tmp/styles/lemur.css'], {allowEmpty: true}))
        .pipe(concat('style-' + themeName + '.css'));
    })))
    .pipe(plumber())
@ -112,24 +133,6 @@ gulp.task('dev:styles', function () {
    .pipe(size());
 });

-// http://stackoverflow.com/questions/1144783/replacing-all-occurrences-of-a-string-in-javascript
-function escapeRegExp(string) {
-  return string.replace(/([.*+?^=!:${}()|\[\]\/\\])/g, '\\$1');
-}
-
-function replaceAll(string, find, replace) {
-  return string.replace(new RegExp(escapeRegExp(find), 'g'), replace);
-}
-
-function string_src(filename, string) {
-  let src = require('stream').Readable({ objectMode: true });
-  src._read = function () {
-    this.push(new gutil.File({ cwd: '', base: '', path: filename, contents: new Buffer(string) }));
-    this.push(null);
-  };
-  return src;
-}
-
 gulp.task('dev:scripts', function () {
  return gulp.src(['lemur/static/app/angular/**/*.js'])
    .pipe(jshint())
@ -145,12 +148,12 @@ gulp.task('build:extras', function () {
 function injectHtml(isDev) {
  return gulp.src('lemur/static/app/index.html')
    .pipe(
-    inject(gulp.src(bowerFiles({ base: 'app' })), {
-      starttag: '<!-- inject:bower:{{ext}} -->',
-      addRootSlash: false,
-      ignorePath: isDev ? ['lemur/static/app/', '.tmp/'] : null
-    })
-  )
+      inject(gulp.src(bowerFiles({base: 'app'})), {
+        starttag: '<!-- inject:bower:{{ext}} -->',
+        addRootSlash: false,
+        ignorePath: isDev ? ['lemur/static/app/', '.tmp/'] : null
+      })
+    )
    .pipe(inject(gulp.src(['lemur/static/app/angular/**/*.js']), {
      starttag: '<!-- inject:{{ext}} -->',
      addRootSlash: false,
@ -162,13 +165,13 @@ function injectHtml(isDev) {
      ignorePath: isDev ? ['lemur/static/app/', '.tmp/'] : null
    }))
    .pipe(
-    gulpif(!isDev,
-      inject(gulp.src('lemur/static/dist/ngviews/ngviews.min.js', { allowEmpty: true }), {
-        starttag: '<!-- inject:ngviews -->',
-        addRootSlash: false
-      })
-    )
-  ).pipe(gulp.dest('.tmp/'));
+      gulpif(!isDev,
+        inject(gulp.src('lemur/static/dist/ngviews/ngviews.min.js', {allowEmpty: true}), {
+          starttag: '<!-- inject:ngviews -->',
+          addRootSlash: false
+        })
+      )
+    ).pipe(gulp.dest('.tmp/'));
 }

 gulp.task('dev:inject', gulp.series(['dev:styles', 'dev:scripts'], function () {
@ -231,27 +234,27 @@ gulp.task('package:strip', function () {
    .pipe(size());
 });

-gulp.task('addUrlContextPath:revision', function(){
-  return gulp.src(['lemur/static/dist/**/*.css','lemur/static/dist/**/*.js'])
+gulp.task('addUrlContextPath:revision', function () {
+  return gulp.src(['lemur/static/dist/**/*.css', 'lemur/static/dist/**/*.js'])
    .pipe(rev())
    .pipe(gulp.dest('lemur/static/dist'))
    .pipe(rev.manifest())
-    .pipe(gulp.dest('lemur/static/dist'))
+    .pipe(gulp.dest('lemur/static/dist'));
 });

-gulp.task('addUrlContextPath:revreplace', gulp.series(['addUrlContextPath:revision'], function(){
-  return gulp.src( "lemur/static/dist/index.html")
+gulp.task('addUrlContextPath:revreplace', gulp.series(['addUrlContextPath:revision'], function () {
+  return gulp.src('lemur/static/dist/index.html')
    .pipe(gulp.dest('lemur/static/dist'));
 }));

-gulp.task('addUrlContextPath', gulp.series(['addUrlContextPath:revreplace'], function(){
-  let urlContextPathExists = argv.urlContextPath ? true : false;
+gulp.task('addUrlContextPath', gulp.series(['addUrlContextPath:revreplace'], function () {
+  let urlContextPathExists = !!argv.urlContextPath;
  return gulp.src(['lemur/static/dist/scripts/main*.js', 'lemur/static/dist/angular/**/*.html'])
-      .pipe(gulpif(urlContextPathExists, replace('api/', argv.urlContextPath + '/api/')))
-      .pipe(gulpif(urlContextPathExists, replace('/angular/', '/' + argv.urlContextPath + '/angular/')))
-      .pipe(gulp.dest(function(file){
-        return file.base;
-      }))
+    .pipe(gulpif(urlContextPathExists, replace('api/', argv.urlContextPath + '/api/')))
+    .pipe(gulpif(urlContextPathExists, replace('/angular/', '/' + argv.urlContextPath + '/angular/')))
+    .pipe(gulp.dest(function (file) {
+      return file.base;
+    }));
 }));

 gulp.task('build', gulp.series(['build:images', 'build:fonts', 'build:html', 'build:extras']));
--- a/gulp/karma.conf.js
+++ b/gulp/karma.conf.js
@ -1,12 +1,14 @@
 // Contents of: config/karma.conf.js
+'use strict';
+
 module.exports = function (config) {
  config.set({
-    basePath : '../',
+    basePath: '../',

    // Fix for "JASMINE is not supported anymore" warning
-    frameworks : ["jasmine"],
+    frameworks: ['jasmine'],

-    files : [
+    files: [
      'app/lib/angular/angular.js',
      'app/lib/angular/angular-*.js',
      'test/lib/angular/angular-mocks.js',
@ -14,14 +16,22 @@ module.exports = function (config) {
      'test/unit/**/*.js'
    ],

-    autoWatch : true,
+    autoWatch: true,

-    browsers : ['Chrome'],
+    browsers: [process.env.TRAVIS ? 'Chrome_travis_ci' : 'Chrome'],
+    customLaunchers: {
+      'Chrome_travis_ci': {
+        base: 'Chrome',
+        flags: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-gpu',],
+      },
+    },

-    junitReporter : {
-      outputFile : 'test_out/unit.xml',
-      suite      : 'unit'
+    junitReporter: {
+      outputFile: 'test_out/unit.xml',
+      suite: 'unit'
      //...
-    }
+    },
+
+    failOnEmptyTestSuite: false,
  });
-};
+};
--- a/package.json
+++ b/package.json
@ -37,11 +37,10 @@
    "gulp-size": "^2.1.0",
    "gulp-uglify": "^2.0.0",
    "gulp-useref": "^3.1.2",
-    "gulp-util": "^3.0.1",
    "http-proxy": ">=1.18.1",
    "jshint-stylish": "^2.2.1",
-    "karma": "^5.2.3",
-    "karma-jasmine": "^1.1.0",
+    "karma": "^6.0.3",
+    "karma-jasmine": "^4.0.1",
    "main-bower-files": "^2.13.1",
    "merge-stream": "^1.0.1",
    "require-dir": "~0.3.0",
@ -59,7 +58,8 @@
  },
  "devDependencies": {
    "gulp": "^4.0.2",
+    "gulp-util": "^3.0.8",
    "jshint": "^2.11.0",
-    "karma-chrome-launcher": "^2.0.0"
+    "karma-chrome-launcher": "^3.1.0"
  }
 }
--- a/tox.ini
+++ b/tox.ini
@ -1,2 +1,2 @@
 [tox]
-envlist = py37
+envlist = py37,py38