Infra/lemur
5
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Merge branch 'github' into get_by_attributes

Browse Source
This commit is contained in:
Non Sequitur 2018-10-17 12:00:36 -04:00
commit 81d114092e
15 changed files with 214 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
lemur/certificates/models.py
View File

@ -5,41 +5,32 @@
:license: Apache, see LICENSE for more details.
.. moduleauthor:: Kevin Glisson <kglisson@netflix.com>
"""
import arrow
from datetime import timedelta
from flask import current_app
import arrow
from cryptography import x509
from cryptography.hazmat.primitives.asymmetric import rsa
from flask import current_app
from idna.core import InvalidCodepoint
from sqlalchemy import event, Integer, ForeignKey, String, PassiveDefault, func, Column, Text, Boolean, Index
from sqlalchemy.ext.hybrid import hybrid_property
from sqlalchemy.orm import relationship
from sqlalchemy.sql.expression import case, extract
from sqlalchemy.ext.hybrid import hybrid_property
from sqlalchemy import event, Integer, ForeignKey, String, PassiveDefault, func, Column, Text, Boolean
from sqlalchemy_utils.types.arrow import ArrowType
from werkzeug.utils import cached_property
from lemur.database import db
from lemur.extensions import sentry
from lemur.utils import Vault
from lemur.common import defaults, utils
from lemur.plugins.base import plugins
from lemur.extensions import metrics
from lemur.constants import SUCCESS_METRIC_STATUS, FAILURE_METRIC_STATUS
from lemur.database import db
from lemur.domains.models import Domain
from lemur.extensions import metrics
from lemur.extensions import sentry
from lemur.models import certificate_associations, certificate_source_associations, \
certificate_destination_associations, certificate_notification_associations, \
certificate_replacement_associations, roles_certificates, pending_cert_replacement_associations
from lemur.domains.models import Domain
from lemur.plugins.base import plugins
from lemur.policies.models import RotationPolicy
from lemur.utils import Vault
def get_sequence(name):
@ -87,6 +78,7 @@ def get_or_increase_name(name, serial):
class Certificate(db.Model):
__tablename__ = 'certificates'
id = Column(Integer, primary_key=True)
ix = Index('ix_certificates_id_desc', id.desc(), postgresql_using='btree', unique=True)
external_id = Column(String(128))
owner = Column(String(128), nullable=False)
name = Column(String(256), unique=True)

2
lemur/certificates/schemas.py
View File

@ -193,6 +193,8 @@ class CertificateOutputSchema(LemurOutputSchema):
name = fields.String()
dns_provider_id = fields.Integer(required=False, allow_none=True)
date_created = ArrowDateTime()
resolved = fields.Boolean(required=False, allow_none=True)
resolved_cert_id = fields.Integer(required=False, allow_none=True)
rotation = fields.Boolean()

55
lemur/common/celery.py
View File

@ -8,9 +8,8 @@ command: celery -A lemur.common.celery worker --loglevel=info -l DEBUG -B
"""
import copy
import datetime
import sys
from datetime import timezone
from datetime import datetime, timezone, timedelta
from celery import Celery
from flask import current_app
@ -20,7 +19,6 @@ from lemur.factory import create_app
from lemur.notifications.messaging import send_pending_failure_notification
from lemur.pending_certificates import service as pending_certificate_service
from lemur.plugins.base import plugins
from lemur.users import service as user_service
flask_app = create_app()
@ -57,7 +55,6 @@ def fetch_acme_cert(id):
"function": "{}.{}".format(__name__, sys._getframe().f_code.co_name)
}
pending_certs = pending_certificate_service.get_pending_certs([id])
user = user_service.get_by_username('lemur')
new = 0
failed = 0
wrong_issuer = 0
@ -78,12 +75,22 @@ def fetch_acme_cert(id):
real_cert = cert.get("cert")
# It's necessary to reload the pending cert due to detached instance: http://sqlalche.me/e/bhk3
pending_cert = pending_certificate_service.get(cert.get("pending_cert").id)
if not pending_cert:
log_data["message"] = "Pending certificate doesn't exist anymore. Was it resolved by another process?"
current_app.logger.error(log_data)
continue
if real_cert:
# If a real certificate was returned from issuer, then create it in Lemur and delete
# the pending certificate
pending_certificate_service.create_certificate(pending_cert, real_cert, user)
pending_certificate_service.delete_by_id(pending_cert.id)
# If a real certificate was returned from issuer, then create it in Lemur and mark
# the pending certificate as resolved
final_cert = pending_certificate_service.create_certificate(pending_cert, real_cert, pending_cert.user)
pending_certificate_service.update(
cert.get("pending_cert").id,
resolved=True
)
pending_certificate_service.update(
cert.get("pending_cert").id,
resolved_cert_id=final_cert.id
)
# add metrics to metrics extension
new += 1
else:
@ -97,7 +104,11 @@ def fetch_acme_cert(id):
if pending_cert.number_attempts > 4:
error_log["message"] = "Deleting pending certificate"
send_pending_failure_notification(pending_cert, notify_owner=pending_cert.notify)
pending_certificate_service.delete(pending_certificate_service.cancel(pending_cert))
# Mark the pending cert as resolved
pending_certificate_service.update(
cert.get("pending_cert").id,
resolved=True
)
else:
pending_certificate_service.increment_attempt(pending_cert)
pending_certificate_service.update(
@ -124,12 +135,30 @@ def fetch_acme_cert(id):
@celery.task()
def fetch_all_pending_acme_certs():
"""Instantiate celery workers to resolve all pending Acme certificates"""
pending_certs = pending_certificate_service.get_pending_certs('all')
pending_certs = pending_certificate_service.get_unresolved_pending_certs()
# We only care about certs using the acme-issuer plugin
for cert in pending_certs:
cert_authority = get_authority(cert.authority_id)
if cert_authority.plugin_name == 'acme-issuer':
if cert.last_updated == cert.date_created or datetime.datetime.now(
timezone.utc) - cert.last_updated > datetime.timedelta(minutes=3):
if cert.last_updated == cert.date_created or datetime.now(
timezone.utc) - cert.last_updated > timedelta(minutes=3):
fetch_acme_cert.delay(cert.id)
@celery.task()
def remove_old_acme_certs():
"""Prune old pending acme certificates from the database"""
log_data = {
"function": "{}.{}".format(__name__, sys._getframe().f_code.co_name)
}
pending_certs = pending_certificate_service.get_pending_certs('all')
# Delete pending certs more than a week old
for cert in pending_certs:
if datetime.now(timezone.utc) - cert.last_updated > timedelta(days=7):
log_data['pending_cert_id'] = cert.id
log_data['pending_cert_name'] = cert.name
log_data['message'] = "Deleting pending certificate"
current_app.logger.debug(log_data)
pending_certificate_service.delete(cert.id)

2
lemur/domains/models.py
View File

@ -15,7 +15,7 @@ from lemur.database import db
class Domain(db.Model):
__tablename__ = 'domains'
id = Column(Integer, primary_key=True)
name = Column(String(256))
name = Column(String(256), index=True)
sensitive = Column(Boolean, default=False)
def __repr__(self):

24
lemur/migrations/versions/984178255c83_.py Normal file
View File

@ -0,0 +1,24 @@
"""Add status to pending certificate, and store resolved cert id
Revision ID: 984178255c83
Revises: f2383bf08fbc
Create Date: 2018-10-11 20:49:12.704563
"""
# revision identifiers, used by Alembic.
revision = '984178255c83'
down_revision = 'f2383bf08fbc'
from alembic import op
import sqlalchemy as sa
def upgrade():
op.add_column('pending_certs', sa.Column('resolved', sa.Boolean(), nullable=True))
op.add_column('pending_certs', sa.Column('resolved_cert_id', sa.Integer(), nullable=True))
def downgrade():
op.drop_column('pending_certs', 'resolved_cert_id')
op.drop_column('pending_certs', 'resolved')

19
lemur/migrations/versions/c87cb989af04_.py Normal file
View File

@ -0,0 +1,19 @@
"""Create an index on the domains table for the domain name
Revision ID: c87cb989af04
Revises: 9392b9f9a805
Create Date: 2018-10-11 09:44:57.099854
"""
revision = 'c87cb989af04'
down_revision = '9392b9f9a805'
from alembic import op
def upgrade():
op.create_index(op.f('ix_domains_name'), 'domains', ['name'], unique=False)
def downgrade():
op.drop_index(op.f('ix_domains_name'), table_name='domains')

23
lemur/migrations/versions/f2383bf08fbc_.py Normal file
View File

@ -0,0 +1,23 @@
"""Create index on certificates table for id desc
Revision ID: f2383bf08fbc
Revises: c87cb989af04
Create Date: 2018-10-11 11:23:31.195471
"""
revision = 'f2383bf08fbc'
down_revision = 'c87cb989af04'
import sqlalchemy as sa
from alembic import op
def upgrade():
op.create_index('ix_certificates_id_desc', 'certificates', [sa.text('id DESC')], unique=True,
postgresql_using='btree')
def downgrade():
op.drop_index('ix_certificates_id_desc', table_name='certificates')

46
lemur/pending_certificates/cli.py
View File

@ -15,7 +15,6 @@ from lemur.authorities.service import get as get_authority
from lemur.notifications.messaging import send_pending_failure_notification
from lemur.pending_certificates import service as pending_certificate_service
from lemur.plugins.base import plugins
from lemur.users import service as user_service
manager = Manager(usage="Handles pending certificate related tasks.")
@ -23,14 +22,14 @@ manager = Manager(usage="Handles pending certificate related tasks.")
@manager.option('-i', dest='ids', action='append', help='IDs of pending certificates to fetch')
def fetch(ids):
"""
Attempt to get full certificates for each pending certificate listed.
Attempt to get full certificate for each pending certificate listed.
Args:
ids: a list of ids of PendingCertificates (passed in by manager options when run as CLI)
`python manager.py pending_certs fetch -i 123 321 all`
"""
pending_certs = pending_certificate_service.get_pending_certs(ids)
user = user_service.get_by_username('lemur')
new = 0
failed = 0
@ -38,10 +37,17 @@ def fetch(ids):
authority = plugins.get(cert.authority.plugin_name)
real_cert = authority.get_ordered_certificate(cert)
if real_cert:
# If a real certificate was returned from issuer, then create it in Lemur and delete
# the pending certificate
pending_certificate_service.create_certificate(cert, real_cert, user)
pending_certificate_service.delete(cert)
# If a real certificate was returned from issuer, then create it in Lemur and mark
# the pending certificate as resolved
final_cert = pending_certificate_service.create_certificate(cert, real_cert, cert.user)
pending_certificate_service.update(
cert.id,
resolved=True
)
pending_certificate_service.update(
cert.id,
resolved_cert_id=final_cert.id
)
# add metrics to metrics extension
new += 1
else:
@ -66,8 +72,7 @@ def fetch_all_acme():
log_data = {
"function": "{}.{}".format(__name__, sys._getframe().f_code.co_name)
}
pending_certs = pending_certificate_service.get_pending_certs('all')
user = user_service.get_by_username('lemur')
pending_certs = pending_certificate_service.get_unresolved_pending_certs()
new = 0
failed = 0
wrong_issuer = 0
@ -90,10 +95,17 @@ def fetch_all_acme():
pending_cert = pending_certificate_service.get(cert.get("pending_cert").id)
if real_cert:
# If a real certificate was returned from issuer, then create it in Lemur and delete
# the pending certificate
pending_certificate_service.create_certificate(pending_cert, real_cert, user)
pending_certificate_service.delete_by_id(pending_cert.id)
# If a real certificate was returned from issuer, then create it in Lemur and mark
# the pending certificate as resolved
final_cert = pending_certificate_service.create_certificate(pending_cert, real_cert, pending_cert.user)
pending_certificate_service.update(
pending_cert.id,
resolved=True
)
pending_certificate_service.update(
pending_cert.id,
resolved_cert_id=final_cert.id
)
# add metrics to metrics extension
new += 1
else:
@ -105,9 +117,13 @@ def fetch_all_acme():
error_log["cn"] = pending_cert.cn
if pending_cert.number_attempts > 4:
error_log["message"] = "Deleting pending certificate"
error_log["message"] = "Marking pending certificate as resolved"
send_pending_failure_notification(pending_cert, notify_owner=pending_cert.notify)
pending_certificate_service.delete(pending_certificate_service.cancel(pending_cert))
# Mark "resolved" as True
pending_certificate_service.update(
cert.id,
resolved=True
)
else:
pending_certificate_service.increment_attempt(pending_cert)
pending_certificate_service.update(

2
lemur/pending_certificates/models.py
View File

@ -29,6 +29,8 @@ class PendingCertificate(db.Model):
notify = Column(Boolean, default=True)
number_attempts = Column(Integer)
rename = Column(Boolean, default=True)
resolved = Column(Boolean, default=False)
resolved_cert_id = Column(Integer, nullable=True)
cn = Column(String(128))
csr = Column(Text(), nullable=False)

2
lemur/pending_certificates/schemas.py
View File

@ -37,6 +37,8 @@ class PendingCertificateOutputSchema(LemurOutputSchema):
number_attempts = fields.Integer()
date_created = fields.Date()
last_updated = fields.Date()
resolved = fields.Boolean(required=False)
resolved_cert_id = fields.Integer(required=False)
rotation = fields.Boolean()

32
lemur/pending_certificates/service.py
View File

@ -4,25 +4,21 @@
.. moduleauthor:: James Chuong <jchuong@instartlogic.com>
"""
import arrow
from sqlalchemy import or_, cast, Integer
from lemur import database
from lemur.common.utils import truthiness
from lemur.plugins.base import plugins
from lemur.roles.models import Role
from lemur.domains.models import Domain
from lemur.authorities.models import Authority
from lemur.certificates import service as certificate_service
from lemur.certificates.schemas import CertificateUploadInputSchema
from lemur.common.utils import truthiness
from lemur.destinations.models import Destination
from lemur.domains.models import Domain
from lemur.notifications.models import Notification
from lemur.pending_certificates.models import PendingCertificate
from lemur.certificates import service as certificate_service
from lemur.plugins.base import plugins
from lemur.roles.models import Role
from lemur.users import service as user_service
from lemur.certificates.schemas import CertificateUploadInputSchema
def get(pending_cert_id):
"""
@ -63,6 +59,15 @@ def delete_by_id(id):
database.delete(get(id))
def get_unresolved_pending_certs():
"""
Retrieve a list of unresolved pending certs given a list of ids
Filters out non-existing pending certs
"""
query = database.session_query(PendingCertificate).filter(PendingCertificate.resolved.is_(False))
return database.find_all(query, PendingCertificate, {}).all()
def get_pending_certs(pending_ids):
"""
Retrieve a list of pending certs given a list of ids
@ -116,6 +121,7 @@ def create_certificate(pending_certificate, certificate, user):
# If generating name from certificate, remove the one from pending certificate
del data['name']
data['creator'] = creator
cert = certificate_service.import_certificate(**data)
database.update(cert)
return cert
@ -172,8 +178,8 @@ def render(args):
if 'issuer' in terms:
# we can't rely on issuer being correct in the cert directly so we combine queries
sub_query = database.session_query(Authority.id)\
.filter(Authority.name.ilike('%{0}%'.format(terms[1])))\
sub_query = database.session_query(Authority.id) \
.filter(Authority.name.ilike('%{0}%'.format(terms[1]))) \
.subquery()
query = query.filter(
@ -221,4 +227,6 @@ def render(args):
now = arrow.now().format('YYYY-MM-DD')
query = query.filter(PendingCertificate.not_after <= to).filter(PendingCertificate.not_after >= now)
# Only show unresolved certificates in the UI
query = query.filter(PendingCertificate.resolved.is_(False))
return database.sort_and_page(query, PendingCertificate, args)

19
requirements-dev.txt
View File

@ -5,26 +5,35 @@
# pip-compile --no-index --output-file requirements-dev.txt requirements-dev.in
#
aspy.yaml==1.1.1 # via pre-commit
bleach==3.0.2 # via readme-renderer
cached-property==1.5.1 # via pre-commit
certifi==2018.8.24 # via requests
cffi==1.11.5 # via cmarkgfm
cfgv==1.1.0 # via pre-commit
chardet==3.0.4 # via requests
cmarkgfm==0.4.2 # via readme-renderer
docutils==0.14 # via readme-renderer
flake8==3.5.0
identify==1.1.6 # via pre-commit
future==0.16.0 # via readme-renderer
identify==1.1.7 # via pre-commit
idna==2.7 # via requests
invoke==1.2.0
mccabe==0.6.1 # via flake8
nodeenv==1.3.2
pkginfo==1.4.2 # via twine
pre-commit==1.11.0
pre-commit==1.11.2
pycodestyle==2.3.1 # via flake8
pycparser==2.19 # via cffi
pyflakes==1.6.0 # via flake8
pygments==2.2.0 # via readme-renderer
pyyaml==3.13 # via aspy.yaml, pre-commit
readme-renderer==22.0 # via twine
requests-toolbelt==0.8.0 # via twine
requests==2.19.1 # via requests-toolbelt, twine
six==1.11.0 # via cfgv, pre-commit
toml==0.9.6 # via pre-commit
six==1.11.0 # via bleach, cfgv, pre-commit, readme-renderer
toml==0.10.0 # via pre-commit
tqdm==4.26.0 # via twine
twine==1.11.0
twine==1.12.1
urllib3==1.23 # via requests
virtualenv==16.0.0 # via pre-commit
webencodings==0.5.1 # via bleach

12
requirements-docs.txt
View File

@ -5,7 +5,7 @@
# pip-compile --no-index --output-file requirements-docs.txt requirements-docs.in
#
acme==0.27.1
alabaster==0.7.11 # via sphinx
alabaster==0.7.12 # via sphinx
alembic-autogenerate-enums==0.0.2
alembic==1.0.0
amqp==2.3.2
@ -54,10 +54,10 @@ lockfile==0.12.2
mako==1.0.7
markupsafe==1.0
marshmallow-sqlalchemy==0.14.1
marshmallow==2.15.4
marshmallow==2.15.5
mock==2.0.0
ndg-httpsclient==0.5.1
packaging==17.1 # via sphinx
packaging==18.0 # via sphinx
paramiko==2.4.1
pbr==4.2.0
pem==18.1.0
@ -69,7 +69,7 @@ pygments==2.2.0 # via sphinx
pyjwt==1.6.4
pynacl==1.2.1
pyopenssl==18.0.0
pyparsing==2.2.0 # via packaging
pyparsing==2.2.2 # via packaging
pyrfc3339==1.1
python-dateutil==2.7.3
python-editor==1.0.3
@ -83,8 +83,8 @@ retrying==1.3.3
s3transfer==0.1.13
six==1.11.0
snowballstemmer==1.2.1 # via sphinx
sphinx-rtd-theme==0.4.1
sphinx==1.8.0
sphinx-rtd-theme==0.4.2
sphinx==1.8.1
sphinxcontrib-httpdomain==1.7.0
sphinxcontrib-websupport==1.1.0 # via sphinx
sqlalchemy-utils==0.33.4

16
requirements-tests.txt
View File

@ -8,13 +8,13 @@ asn1crypto==0.24.0 # via cryptography
atomicwrites==1.2.1 # via pytest
attrs==18.2.0 # via pytest
aws-xray-sdk==0.95 # via moto
boto3==1.9.4 # via moto
boto3==1.9.21 # via moto
boto==2.49.0 # via moto
botocore==1.12.4 # via boto3, moto, s3transfer
botocore==1.12.21 # via boto3, moto, s3transfer
certifi==2018.8.24 # via requests
cffi==1.11.5 # via cryptography
chardet==3.0.4 # via requests
click==6.7 # via flask
click==7.0 # via flask
cookies==2.2.1 # via moto, responses
coverage==4.5.1
cryptography==2.3.1 # via moto
@ -32,22 +32,22 @@ itsdangerous==0.24 # via flask
jinja2==2.10 # via flask, moto
jmespath==0.9.3 # via boto3, botocore
jsondiff==1.1.1 # via moto
jsonpickle==0.9.6 # via aws-xray-sdk
jsonpickle==1.0 # via aws-xray-sdk
markupsafe==1.0 # via jinja2
mock==2.0.0 # via moto
more-itertools==4.3.0 # via pytest
moto==1.3.4
nose==1.3.7
pbr==4.2.0 # via mock
pbr==4.3.0 # via mock
pluggy==0.7.1 # via pytest
py==1.6.0 # via pytest
pyaml==17.12.1 # via moto
pycparser==2.18 # via cffi
pycparser==2.19 # via cffi
pycryptodome==3.6.6 # via python-jose
pyflakes==2.0.0
pytest-flask==0.12.0
pytest-flask==0.13.0
pytest-mock==1.10.0
pytest==3.8.0
pytest==3.8.2
python-dateutil==2.7.3 # via botocore, faker, freezegun, moto
python-jose==2.0.2 # via moto
pytz==2018.5 # via moto

18
requirements.txt
View File

@ -21,7 +21,7 @@ celery[redis]==4.2.1
certifi==2018.8.24
cffi==1.11.5 # via bcrypt, cryptography, pynacl
chardet==3.0.4 # via requests
click==6.7 # via flask
click==7.0 # via flask
cloudflare==2.1.0
cryptography==2.3.1
dnspython3==1.15.0
@ -51,18 +51,18 @@ lockfile==0.12.2
mako==1.0.7 # via alembic
markupsafe==1.0 # via jinja2, mako
marshmallow-sqlalchemy==0.14.1
marshmallow==2.15.5
marshmallow==2.16.0
mock==2.0.0 # via acme
ndg-httpsclient==0.5.1
paramiko==2.4.1
pbr==4.2.0 # via mock
pem==18.1.0
paramiko==2.4.2
pbr==4.3.0 # via mock
pem==18.2.0
psycopg2==2.7.5
pyasn1-modules==0.2.2 # via python-ldap
pyasn1==0.4.4 # via ndg-httpsclient, paramiko, pyasn1-modules, python-ldap
pycparser==2.18 # via cffi
pycparser==2.19 # via cffi
pyjwt==1.6.4
pynacl==1.2.1 # via paramiko
pynacl==1.3.0 # via paramiko
pyopenssl==18.0.0
pyrfc3339==1.1 # via acme
python-dateutil==2.7.3 # via alembic, arrow, botocore
@ -77,8 +77,8 @@ requests[security]==2.19.1
retrying==1.3.3
s3transfer==0.1.13 # via boto3
six==1.11.0
sqlalchemy-utils==0.33.4
sqlalchemy==1.2.11 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
sqlalchemy-utils==0.33.5
sqlalchemy==1.2.12 # via alembic, flask-sqlalchemy, marshmallow-sqlalchemy, sqlalchemy-utils
tabulate==0.8.2
urllib3==1.23 # via requests
vine==1.1.4 # via amqp