Merge pull request #3379 from sergerdn/fix/draft-nodejs-version

multiple fixes docker alpine build

docker/Dockerfile

@@ -1,4 +1,4 @@
-FROM alpine:3.8
+FROM python:3.7.9-alpine3.12
 
 ARG VERSION
 ENV VERSION master
@@ -12,7 +12,7 @@ ENV group lemur
 
 RUN addgroup -S ${group} -g ${gid} && \
     adduser -D -S ${user} -G ${group} -u ${uid} && \
-    apk --update add python3 libldap postgresql-client nginx supervisor curl tzdata openssl bash && \
+    apk add --no-cache --update python3 py-pip libldap postgresql-client nginx supervisor curl tzdata openssl bash && \
     apk --update add --virtual build-dependencies \
                 git \
                 tar \
@@ -39,10 +39,12 @@ RUN addgroup -S ${group} -g ${gid} && \
     pip3 install --upgrade setuptools && \
     mkdir -p /run/nginx/ /etc/nginx/ssl/ && \
     chown -R $user:$group /opt/lemur/ /home/lemur/.lemur/
-    
+
 WORKDIR /opt/lemur
 
-RUN npm install --unsafe-perm && \
+RUN echo "Running with python:" && python -c 'import platform; print(platform.python_version())' && \
+    echo "Running with nodejs:" && node -v && \
+    npm install --unsafe-perm && \
     pip3 install -e . && \
     node_modules/.bin/gulp build && \
     node_modules/.bin/gulp package --urlContextPath=${URLCONTEXT} && \

docker/docker-compose.yml

@@ -1,9 +1,12 @@
 version: '3'
 
+volumes:
+  pg_data: { }
+
 services:
   postgres:
-    image: "postgres:10"
+    image: "postgres:13.1-alpine"
-    restart: always
+    restart: on-failure
     volumes:
       - pg_data:/var/lib/postgresql/data
     env_file:
@@ -11,7 +14,9 @@ services:
 
   lemur:
     # image: "netlix-lemur:latest"
-    build: .
+    restart: on-failure
+    build:
+      context: .
     depends_on:
       - postgres
       - redis
@@ -19,11 +24,9 @@ services:
       - lemur-env
       - pgsql-env
     ports:
-      - 80:80
+      - 87:80
-      - 443:443
+      - 447:443
 
   redis:
-    image: "redis:alpine"
+    image: "redis:alpine3.12"
-
+    restart: on-failure
-volumes:
-    pg_data: {}

docker/entrypoint

@@ -14,10 +14,10 @@ export LEMUR_ADMIN_PASSWORD="${LEMUR_ADMIN_PASSWORD:-admin}"
 export SQLALCHEMY_DATABASE_URI="postgresql://$POSTGRES_USER:$POSTGRES_PASSWORD@$POSTGRES_HOST:$POSTGRES_PORT/$POSTGRES_DB"
 
 
-PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'select 1;'
+PGPASSWORD=$POSTGRES_PASSWORD psql -h "$POSTGRES_HOST" -p "$POSTGRES_PORT" -U "$POSTGRES_USER" -d "$POSTGRES_DB" --command 'select 1;'
 
 echo " # Create Postgres trgm extension"
-PGPASSWORD=$POSTGRES_PASSWORD psql -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER -d $POSTGRES_DB --command 'CREATE EXTENSION IF NOT EXISTS pg_trgm;'
+PGPASSWORD=$POSTGRES_PASSWORD psql -h "$POSTGRES_HOST" -p "$POSTGRES_PORT" -U "$POSTGRES_USER" -d "$POSTGRES_DB" --command 'CREATE EXTENSION IF NOT EXISTS pg_trgm;'
 echo " # Done"
 
 if [ -z "${SKIP_SSL}" ]; then

docker/src/lemur.conf.py

@@ -1,11 +1,18 @@
-import os
+import os.path
 import random
 import string
+from celery.schedules import crontab
+
 import base64
-from ast import literal_eval
 
 _basedir = os.path.abspath(os.path.dirname(__file__))
 
+# See the Lemur docs (https://lemur.readthedocs.org) for more information on configuration
+
+LOG_LEVEL = str(os.environ.get('LOG_LEVEL', 'DEBUG'))
+LOG_FILE = str(os.environ.get('LOG_FILE', '/home/lemur/.lemur/lemur.log'))
+LOG_JSON = True
+
 CORS = os.environ.get("CORS") == "True"
 debug = os.environ.get("DEBUG") == "True"
 
@@ -17,44 +24,214 @@ def get_random_secret(length):
     return secret_key + ''.join(random.choice(string.digits) for x in range(round(length / 4)))
 
 
+# This is the secret key used by Flask session management
 SECRET_KEY = repr(os.environ.get('SECRET_KEY', get_random_secret(32).encode('utf8')))
 
+# You should consider storing these separately from your config
 LEMUR_TOKEN_SECRET = repr(os.environ.get('LEMUR_TOKEN_SECRET',
                                          base64.b64encode(get_random_secret(32).encode('utf8'))))
+# This must match the key for whichever DB the container is using - this could be a dump of dev or test, or a unique key
 LEMUR_ENCRYPTION_KEYS = repr(os.environ.get('LEMUR_ENCRYPTION_KEYS',
-                                            base64.b64encode(get_random_secret(32).encode('utf8'))))
+                                            base64.b64encode(get_random_secret(32).encode('utf8')).decode('utf8')))
 
-LEMUR_ALLOWED_DOMAINS = []
+REDIS_HOST = 'redis'
+REDIS_PORT = 6379
+REDIS_DB = 0
+CELERY_RESULT_BACKEND = f'redis://{REDIS_HOST}:{REDIS_PORT}'
+CELERY_BROKER_URL = f'redis://{REDIS_HOST}:{REDIS_PORT}'
+CELERY_IMPORTS = ('lemur.common.celery')
+CELERYBEAT_SCHEDULE = {
+    # All tasks are disabled by default. Enable any tasks you wish to run.
+    # 'fetch_all_pending_acme_certs': {
+    #     'task': 'lemur.common.celery.fetch_all_pending_acme_certs',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(minute="*"),
+    # },
+    # 'remove_old_acme_certs': {
+    #     'task': 'lemur.common.celery.remove_old_acme_certs',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour=8, minute=0, day_of_week=5),
+    # },
+    # 'clean_all_sources': {
+    #     'task': 'lemur.common.celery.clean_all_sources',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour=5, minute=0, day_of_week=5),
+    # },
+    # 'sync_all_sources': {
+    #     'task': 'lemur.common.celery.sync_all_sources',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour="*/2", minute=0),
+    #     # this job is running 30min before endpoints_expire which deletes endpoints which were not updated
+    # },
+    # 'sync_source_destination': {
+    #     'task': 'lemur.common.celery.sync_source_destination',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour="*/2", minute=15),
+    # },
+    # 'report_celery_last_success_metrics': {
+    #     'task': 'lemur.common.celery.report_celery_last_success_metrics',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(minute="*"),
+    # },
+    # 'certificate_reissue': {
+    #     'task': 'lemur.common.celery.certificate_reissue',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour=9, minute=0),
+    # },
+    # 'certificate_rotate': {
+    #     'task': 'lemur.common.celery.certificate_rotate',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour=10, minute=0),
+    # },
+    # 'endpoints_expire': {
+    #     'task': 'lemur.common.celery.endpoints_expire',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour="*/2", minute=30),
+    #     # this job is running 30min after sync_all_sources which updates endpoints
+    # },
+    # 'get_all_zones': {
+    #     'task': 'lemur.common.celery.get_all_zones',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(minute="*/30"),
+    # },
+    # 'check_revoked': {
+    #     'task': 'lemur.common.celery.check_revoked',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour=10, minute=0),
+    # }
+    # 'enable_autorotate_for_certs_attached_to_endpoint': {
+    #     'task': 'lemur.common.celery.enable_autorotate_for_certs_attached_to_endpoint',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour=10, minute=0),
+    # }
+    # 'notify_expirations': {
+    #     'task': 'lemur.common.celery.notify_expirations',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour=10, minute=0),
+    #  },
+    # 'notify_authority_expirations': {
+    #     'task': 'lemur.common.celery.notify_authority_expirations',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour=10, minute=0),
+    # },
+    # 'send_security_expiration_summary': {
+    #     'task': 'lemur.common.celery.send_security_expiration_summary',
+    #     'options': {
+    #         'expires': 180
+    #     },
+    #     'schedule': crontab(hour=10, minute=0, day_of_week='mon-fri'),
+    # }
+}
+CELERY_TIMEZONE = 'UTC'
 
-LEMUR_EMAIL = ''
+SQLALCHEMY_ENABLE_FLASK_REPLICATED = False
-LEMUR_SECURITY_TEAM_EMAIL = []
+SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI', 'postgresql://lemur:lemur@localhost:5432/lemur')
 
-ALLOW_CERT_DELETION = os.environ.get('ALLOW_CERT_DELETION') == "True"
+SQLALCHEMY_TRACK_MODIFICATIONS = False
+SQLALCHEMY_ECHO = True
+SQLALCHEMY_POOL_RECYCLE = 499
+SQLALCHEMY_POOL_TIMEOUT = 20
 
-LEMUR_DEFAULT_COUNTRY = str(os.environ.get('LEMUR_DEFAULT_COUNTRY',''))
+LEMUR_EMAIL = 'lemur@example.com'
-LEMUR_DEFAULT_STATE = str(os.environ.get('LEMUR_DEFAULT_STATE',''))
+LEMUR_SECURITY_TEAM_EMAIL = ['security@example.com']
-LEMUR_DEFAULT_LOCATION = str(os.environ.get('LEMUR_DEFAULT_LOCATION',''))
+LEMUR_SECURITY_TEAM_EMAIL_INTERVALS = [15, 2]
-LEMUR_DEFAULT_ORGANIZATION = str(os.environ.get('LEMUR_DEFAULT_ORGANIZATION',''))
+LEMUR_DEFAULT_EXPIRATION_NOTIFICATION_INTERVALS = [30, 15, 2]
-LEMUR_DEFAULT_ORGANIZATIONAL_UNIT = str(os.environ.get('LEMUR_DEFAULT_ORGANIZATIONAL_UNIT',''))
+LEMUR_EMAIL_SENDER = 'smtp'
 
-LEMUR_DEFAULT_ISSUER_PLUGIN = str(os.environ.get('LEMUR_DEFAULT_ISSUER_PLUGIN',''))
+# mail configuration
-LEMUR_DEFAULT_AUTHORITY = str(os.environ.get('LEMUR_DEFAULT_AUTHORITY',''))
+# MAIL_SERVER = 'mail.example.com'
+
+PUBLIC_CA_MAX_VALIDITY_DAYS = 397
+DEFAULT_VALIDITY_DAYS = 365
+
+LEMUR_OWNER_EMAIL_IN_SUBJECT = False
+
+LEMUR_DEFAULT_COUNTRY = str(os.environ.get('LEMUR_DEFAULT_COUNTRY', 'US'))
+LEMUR_DEFAULT_STATE = str(os.environ.get('LEMUR_DEFAULT_STATE', 'California'))
+LEMUR_DEFAULT_LOCATION = str(os.environ.get('LEMUR_DEFAULT_LOCATION', 'Los Gatos'))
+LEMUR_DEFAULT_ORGANIZATION = str(os.environ.get('LEMUR_DEFAULT_ORGANIZATION', 'Example, Inc.'))
+LEMUR_DEFAULT_ORGANIZATIONAL_UNIT = str(os.environ.get('LEMUR_DEFAULT_ORGANIZATIONAL_UNIT', ''))
+
+LEMUR_DEFAULT_AUTHORITY = str(os.environ.get('LEMUR_DEFAULT_AUTHORITY', 'ExampleCa'))
+
+LEMUR_DEFAULT_ROLE = 'operator'
 
 ACTIVE_PROVIDERS = []
-
 METRIC_PROVIDERS = []
 
-LOG_LEVEL = str(os.environ.get('LOG_LEVEL','DEBUG'))
+# Authority Settings - These will change depending on which authorities you are
-LOG_FILE = str(os.environ.get('LOG_FILE','/home/lemur/.lemur/lemur.log'))
+# using
+current_path = os.path.dirname(os.path.realpath(__file__))
 
-SQLALCHEMY_DATABASE_URI = os.environ.get('SQLALCHEMY_DATABASE_URI','postgresql://lemur:lemur@localhost:5432/lemur')
+# DNS Settings
 
-LDAP_DEBUG = os.environ.get('LDAP_DEBUG') == "True"
+# exclude logging missing SAN, since we can have certs from private CAs with only cn, prod parity
-LDAP_AUTH = os.environ.get('LDAP_AUTH') == "True"
+LOG_SSL_SUBJ_ALT_NAME_ERRORS = False
-LDAP_IS_ACTIVE_DIRECTORY = os.environ.get('LDAP_IS_ACTIVE_DIRECTORY') == "True"
+
-LDAP_BIND_URI = str(os.environ.get('LDAP_BIND_URI',''))
+ACME_DNS_PROVIDER_TYPES = {"items": [
-LDAP_BASE_DN = str(os.environ.get('LDAP_BASE_DN',''))
+    {
-LDAP_EMAIL_DOMAIN = str(os.environ.get('LDAP_EMAIL_DOMAIN',''))
+        'name': 'route53',
-LDAP_USE_TLS = str(os.environ.get('LDAP_USE_TLS',''))
+        'requirements': [
-LDAP_REQUIRED_GROUP = str(os.environ.get('LDAP_REQUIRED_GROUP',''))
+            {
-LDAP_GROUPS_TO_ROLES = literal_eval(os.environ.get('LDAP_GROUPS_TO_ROLES') or "{}")
+                'name': 'account_id',
+                'type': 'int',
+                'required': True,
+                'helpMessage': 'AWS Account number'
+            },
+        ]
+    },
+    {
+        'name': 'cloudflare',
+        'requirements': [
+            {
+                'name': 'email',
+                'type': 'str',
+                'required': True,
+                'helpMessage': 'Cloudflare Email'
+            },
+            {
+                'name': 'key',
+                'type': 'str',
+                'required': True,
+                'helpMessage': 'Cloudflare Key'
+            },
+        ]
+    },
+    {
+        'name': 'dyn',
+    },
+    {
+        'name': 'ultradns',
+    },
+]}
+
+# Authority plugins which support revocation
+SUPPORTED_REVOCATION_AUTHORITY_PLUGINS = ['acme-issuer']