From 9cd443bef68ae26d184831c617d3d0b3151f4a73 Mon Sep 17 00:00:00 2001 From: William Petit Date: Wed, 16 May 2018 15:08:27 +0200 Subject: [PATCH] =?UTF-8?q?Mise=20=C3=A0=20jour=20des=20templates=20de=20V?= =?UTF-8?q?M=20et=20nettoyage=20des=20images=20inutilis=C3=A9es?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../image-template/image_template.go | 387 ++++++++++++++---- stepper/stepper.go | 40 ++ stepper/stepper_test.go | 146 +++++++ 3 files changed, 497 insertions(+), 76 deletions(-) create mode 100644 stepper/stepper.go create mode 100644 stepper/stepper_test.go diff --git a/cmd/post-processor/image-template/image_template.go b/cmd/post-processor/image-template/image_template.go index c87e215..019cd9e 100644 --- a/cmd/post-processor/image-template/image_template.go +++ b/cmd/post-processor/image-template/image_template.go @@ -1,10 +1,14 @@ package main import ( + "context" "fmt" + "regexp" + "strconv" "strings" "time" + "forge.cadoles.com/wpetit/packer-opennebula/stepper" "github.com/Cadoles/goca" "github.com/hashicorp/packer/common" "github.com/hashicorp/packer/helper/config" @@ -28,7 +32,9 @@ type Config struct { User string Group string } `mapstructure:"owner"` - ctx interpolate.Context + AutoUpdateVMTemplates bool `mapstructure:"auto_update_vm_templates"` + CleanupUnusedImages bool `mapstructure:"cleanup_unused_images"` + ctx interpolate.Context } // PostProcessor is an OpenNebula post processor for packer @@ -53,139 +59,368 @@ func (pp *PostProcessor) Configure(raws ...interface{}) error { // PostProcess creates/updates your configured image to OpenNebula using the XML-RPC API func (pp *PostProcessor) PostProcess(ui packer.Ui, a packer.Artifact) (packer.Artifact, bool, error) { - // Initialize OpenNebula XML-RPC API client - config := goca.NewConfig(pp.Conf.User, pp.Conf.Password, pp.Conf.Endpoint) - if err := goca.SetClient(config); err != nil { + ctx := context.Background() + state := &postProcessorState{ + Config: pp.Conf, + UI: ui, + } + status, err := stepper.Run( + ctx, state, + stepInitializeGocaClient, + stepGenerateFullImageName, + stepRetrieveImage, + stepUpsertImage, + stepUpdateImageOwnership, + stepUpdateImageACL, + stepUpdateVMTemplates, + stepCleanupUnusedImages, + ) + + if err != nil { return a, true, err } - ui.Say(fmt.Sprintf("Connecting to OpenNebula RPC endpoint '%s' with provided credentials...", pp.Conf.Endpoint)) + switch status { + case stepper.Completed: + ui.Say("Operation completed.") + case stepper.Canceled: + ui.Say("Operation canceled.") + } + + return a, true, nil + +} + +type postProcessorState struct { + Config *Config + UI packer.Ui + Image *goca.Image + FullImageName string +} + +func stepInitializeGocaClient(ctx context.Context, cancelFunc context.CancelFunc, st interface{}) error { + + state := toPostProcessorState(st) + + // Initialize OpenNebula XML-RPC API client + config := goca.NewConfig(state.Config.User, state.Config.Password, state.Config.Endpoint) + if err := goca.SetClient(config); err != nil { + return err + } + + return nil + +} + +func stepGenerateFullImageName(ctx context.Context, cancelFunc context.CancelFunc, st interface{}) error { + + state := toPostProcessorState(st) + + if !state.Config.AutoUpdateVMTemplates { + state.FullImageName = state.Config.ImageName + return nil + } + + state.UI.Say("VM templates auto update activated. Adding timestamp to image's name.") + + now := time.Now().Format("200601021504") + state.FullImageName = fmt.Sprintf("%s-%s", state.Config.ImageName, now) + + return nil + +} + +func stepRetrieveImage(ctx context.Context, cancelFunc context.CancelFunc, st interface{}) error { + + state := toPostProcessorState(st) + + state.UI.Say(fmt.Sprintf("Connecting to OpenNebula RPC endpoint '%s' with provided credentials...", state.Config.Endpoint)) // Search image template by its name - img, err := goca.NewImageFromName(pp.Conf.ImageName) + img, err := goca.NewImageFromName(state.FullImageName) if err != nil && err.Error() != "resource not found" { - return a, true, err + return err } if img != nil { // Retreive info about the template if err := img.Info(); err != nil { - return a, true, err + return err } - state, err := img.State() + imgState, err := img.State() if err != nil { - return a, true, err + return err } - inUse := state == goca.ImageUsed || state == goca.ImageLockUsed + inUse := isImageUsed(imgState) if inUse { - ui.Say(fmt.Sprintf("Template '%s' is in use. Cannot delete it.", pp.Conf.ImageName)) + state.UI.Say(fmt.Sprintf("Image '%s' is in use. Cannot delete it.", state.FullImageName)) } - if pp.Conf.DeleteIfExists && !inUse { - - ui.Say(fmt.Sprintf("Deleting template '%s'...", pp.Conf.ImageName)) - + if state.Config.DeleteIfExists && !inUse { + state.UI.Say(fmt.Sprintf("Deleting image '%s'...", state.FullImageName)) if err := img.Delete(); err != nil { - return a, true, err + return err } time.Sleep(1 * time.Second) img = nil - } } + state.Image = img + + return nil + +} + +func stepUpsertImage(ctx context.Context, cancelFunc context.CancelFunc, st interface{}) error { + + state := toPostProcessorState(st) + // Generate image template - tmplStr := serializeImageTemplate(pp.Conf.ImageTemplate) + tmplStr := serializeImageTemplate(state.FullImageName, state.Config.ImageTemplate) // If the image template can not be found, we create it - if img == nil { + if state.Image == nil { - ui.Say(fmt.Sprintf("Creating template '%s'...", pp.Conf.ImageName)) + state.UI.Say(fmt.Sprintf("Creating image '%s'...", state.FullImageName)) // Search image datastore's ID - datastore, err := goca.NewDatastoreFromName(pp.Conf.DatastoreName) + datastore, err := goca.NewDatastoreFromName(state.Config.DatastoreName) if err != nil { - return a, true, err + return err } // Create image template imageID, err := goca.CreateImage(tmplStr, datastore.ID) if err != nil { - return a, true, err + return err } - img = goca.NewImage(imageID) + state.Image = goca.NewImage(imageID) // Retreive info about the template - if err := img.Info(); err != nil { - return a, true, err + if err := state.Image.Info(); err != nil { + return err } } else { - ui.Say(fmt.Sprintf("Updating template '%s'...", pp.Conf.ImageName)) + state.UI.Say(fmt.Sprintf("Updating image '%s'...", state.FullImageName)) // Update image template - if err := img.Update(tmplStr, pp.Conf.MergeTemplate); err != nil { - return a, true, err + if err := state.Image.Update(tmplStr, state.Config.MergeTemplate); err != nil { + return err } } - if pp.Conf.Owner != nil { - - currentUserName, userFound := img.XPath("/IMAGE/UNAME") - currentGroupName, groupFound := img.XPath("/IMAGE/GNAME") - - isSameUser := userFound && currentUserName == pp.Conf.Owner.User - isSameGroup := groupFound && currentGroupName == pp.Conf.Owner.Group - - userID := -1 - groupID := -1 - - if pp.Conf.Owner.User != "" && !isSameUser { - user, err := goca.NewUserFromName(pp.Conf.Owner.User) - if err != nil { - return a, true, err - } - userID = int(user.ID) - } - - if pp.Conf.Owner.Group != "" && !isSameGroup { - group, err := goca.NewGroupFromName(pp.Conf.Owner.Group) - if err != nil { - return a, true, err - } - groupID = int(group.ID) - } - - ui.Say(fmt.Sprintf("Updating template '%s' owner...", pp.Conf.ImageName)) - if err := img.Chown(userID, groupID); err != nil { - return a, true, err - } - - } - - if pp.Conf.ACL != nil { - ui.Say(fmt.Sprintf("Updating template '%s' ACL...", pp.Conf.ImageName)) - chmodOptions := aclToChmodOptions(pp.Conf.ACL) - if err := img.Chmod(chmodOptions); err != nil { - return a, true, err - } - } - - ui.Say("Operation completed.") - - return a, true, nil + return nil } -func serializeImageTemplate(tmpl []string) string { +func stepUpdateImageOwnership(ctx context.Context, cancelFunc context.CancelFunc, st interface{}) error { + + state := toPostProcessorState(st) + + if state.Config.Owner == nil { + return nil + } + + owner := state.Config.Owner + + currentUserName, userFound := state.Image.XPath("/IMAGE/UNAME") + currentGroupName, groupFound := state.Image.XPath("/IMAGE/GNAME") + + isSameUser := userFound && currentUserName == owner.User + isSameGroup := groupFound && currentGroupName == owner.Group + + userID := -1 + groupID := -1 + + if owner.User != "" && !isSameUser { + user, err := goca.NewUserFromName(owner.User) + if err != nil { + return err + } + userID = int(user.ID) + } + + if owner.Group != "" && !isSameGroup { + group, err := goca.NewGroupFromName(owner.Group) + if err != nil { + return err + } + groupID = int(group.ID) + } + + state.UI.Say(fmt.Sprintf("Updating image '%s' owner...", state.FullImageName)) + if err := state.Image.Chown(userID, groupID); err != nil { + return err + } + + return nil + +} + +func stepUpdateImageACL(ctx context.Context, cancelFunc context.CancelFunc, st interface{}) error { + + state := toPostProcessorState(st) + + if state.Config.ACL == nil { + return nil + } + + state.UI.Say(fmt.Sprintf("Updating template '%s' ACL...", state.FullImageName)) + chmodOptions := aclToChmodOptions(state.Config.ACL) + if err := state.Image.Chmod(chmodOptions); err != nil { + return err + } + + return nil + +} + +func stepUpdateVMTemplates(ctx context.Context, cancel context.CancelFunc, st interface{}) error { + + state := toPostProcessorState(st) + + if !state.Config.AutoUpdateVMTemplates { + return nil + } + + state.UI.Say("Searching VM templates using old images...") + + templatePool, err := goca.NewTemplatePool(goca.PoolWhoAll, -1, -1) + if err != nil { + return err + } + + imageNameRegExp, err := getFullImageNamePattern(state.Config.ImageName) + if err != nil { + return err + } + + xPathIter := templatePool.XPathIter("/VMTEMPLATE_POOL/VMTEMPLATE") + for xPathIter.Next() { + + node := xPathIter.Node() + imageName, _ := node.XPath("TEMPLATE/DISK/IMAGE") + + if !imageNameRegExp.MatchString(imageName) || imageName == state.FullImageName { + continue + } + + templateName, _ := node.XPath("NAME") + state.UI.Say(fmt.Sprintf("Found VM template '%s' using image '%s' ", templateName, imageName)) + + rawTemplateID, _ := node.XPath("ID") + templateID, err := strconv.ParseUint(rawTemplateID, 10, 32) + if err != nil { + return err + } + + template := goca.NewTemplate(uint(templateID)) + imageUser, _ := state.Image.XPath("/IMAGE/UNAME") + + templateUpdate := fmt.Sprintf(` + DISK = [ + IMAGE = "%s", + IMAGE_UNAME = "%s" ] + `, state.FullImageName, imageUser) + + state.UI.Say(fmt.Sprintf("Updating VM template '%s'", templateName)) + if err := template.Update(templateUpdate, 1); err != nil { + return err + } + + } + + return nil + +} + +func stepCleanupUnusedImages(ctx context.Context, cancel context.CancelFunc, st interface{}) error { + + state := toPostProcessorState(st) + + if !state.Config.CleanupUnusedImages { + return nil + } + + state.UI.Say("Cleaning old unused images...") + + imageNameRegExp, err := getFullImageNamePattern(state.Config.ImageName) + if err != nil { + return err + } + + imagePool, err := goca.NewImagePool(goca.PoolWhoAll, -1, -1) + if err != nil { + return err + } + + xPathIter := imagePool.XPathIter("/IMAGE_POOL/IMAGE") + for xPathIter.Next() { + + node := xPathIter.Node() + imageName, _ := node.XPath("NAME") + + if !imageNameRegExp.MatchString(imageName) || imageName == state.FullImageName { + continue + } + + rawImgState, _ := node.XPath("STATE") + imgState, err := strconv.Atoi(rawImgState) + if err != nil { + return err + } + + if isImageUsed(goca.ImageState(imgState)) { + continue + } + + state.UI.Say(fmt.Sprintf("Found unused image '%s'. Deleting it.", imageName)) + + rawImageID, _ := node.XPath("ID") + imageID, err := strconv.ParseUint(rawImageID, 10, 32) + if err != nil { + return err + } + + image := goca.NewImage(uint(imageID)) + if err := image.Delete(); err != nil { + return err + } + + } + + return nil + +} + +func toPostProcessorState(st interface{}) *postProcessorState { + state, ok := st.(*postProcessorState) + if !ok { + panic("couldnt cast state to the expected type") + } + return state +} + +func isImageUsed(imgState goca.ImageState) bool { + return imgState == goca.ImageUsed || imgState == goca.ImageLockUsed +} + +func getFullImageNamePattern(imageName string) (*regexp.Regexp, error) { + return regexp.Compile(fmt.Sprintf(`^%s-\d+$`, imageName)) +} + +func serializeImageTemplate(imageName string, tmpl []string) string { + tmpl = append(tmpl, fmt.Sprintf("NAME = \"%s\"", imageName)) return strings.Join(tmpl, "\n") } diff --git a/stepper/stepper.go b/stepper/stepper.go new file mode 100644 index 0000000..85d7e29 --- /dev/null +++ b/stepper/stepper.go @@ -0,0 +1,40 @@ +package stepper + +import ( + "context" +) + +const ( + Error Status = iota + Completed Status = iota + Canceled Status = iota +) + +var ( + ErrUnsupportedState = "unsupported state" +) + +type Status int + +type Step func(ctx context.Context, cancelFunc context.CancelFunc, state interface{}) error + +func Run(ctx context.Context, state interface{}, steps ...Step) (Status, error) { + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + for _, s := range steps { + select { + case <-ctx.Done(): + return Canceled, nil + default: + err := s(ctx, cancel, state) + if err != nil { + return Error, err + } + } + } + + return Completed, nil + +} diff --git a/stepper/stepper_test.go b/stepper/stepper_test.go new file mode 100644 index 0000000..336deeb --- /dev/null +++ b/stepper/stepper_test.go @@ -0,0 +1,146 @@ +package stepper + +import ( + "context" + "errors" + "testing" +) + +func TestBasicSteps(t *testing.T) { + + ctx := context.Background() + state := make(map[string]bool) + + step1 := func(ctx context.Context, cancel context.CancelFunc, state interface{}) error { + st, ok := state.(map[string]bool) + if !ok { + t.Error(ErrUnsupportedState) + } + st["step1"] = true + return nil + } + + step2 := func(ctx context.Context, cancel context.CancelFunc, state interface{}) error { + st, ok := state.(map[string]bool) + if !ok { + t.Error(ErrUnsupportedState) + } + st["step2"] = true + return nil + } + + status, err := Run(ctx, state, step1, step2) + if err != nil { + t.Error(err) + } + + if g, e := status, Completed; g != e { + t.Errorf("status: expected '%v', got '%v'", e, g) + } + + if g, e := state["step1"], true; g != e { + t.Errorf("state[\"step1\"]: expected '%v', got '%v'", e, g) + } + + if g, e := state["step2"], true; g != e { + t.Errorf("state[\"step2\"]: expected '%v', got '%v'", e, g) + } + +} + +func TestStepsCancel(t *testing.T) { + + ctx := context.Background() + state := make(map[string]bool) + + step1 := func(ctx context.Context, cancel context.CancelFunc, state interface{}) error { + st, ok := state.(map[string]bool) + if !ok { + t.Error(ErrUnsupportedState) + } + st["step1"] = true + return nil + } + + step2 := func(ctx context.Context, cancel context.CancelFunc, state interface{}) error { + cancel() + return nil + } + + step3 := func(ctx context.Context, cancel context.CancelFunc, state interface{}) error { + st, ok := state.(map[string]bool) + if !ok { + t.Error(ErrUnsupportedState) + } + st["step3"] = true + return nil + } + + status, err := Run(ctx, state, step1, step2, step3) + if err != nil { + t.Error(err) + } + + if g, e := status, Canceled; g != e { + t.Errorf("status: expected '%v', got '%v'", e, g) + } + + if g, e := state["step1"], true; g != e { + t.Errorf("state[\"step1\"]: expected '%v', got '%v'", e, g) + } + + if g, e := state["step3"], false; g != e { + t.Errorf("state[\"step3\"]: expected '%v', got '%v'", e, g) + } + +} + +func TestStepsError(t *testing.T) { + + stepErr := errors.New("test error") + ctx := context.Background() + state := make(map[string]bool) + + step1 := func(ctx context.Context, cancel context.CancelFunc, state interface{}) error { + st, ok := state.(map[string]bool) + if !ok { + t.Error(ErrUnsupportedState) + } + st["step1"] = true + return nil + } + + step2 := func(ctx context.Context, cancel context.CancelFunc, state interface{}) error { + return stepErr + } + + step3 := func(ctx context.Context, cancel context.CancelFunc, state interface{}) error { + st, ok := state.(map[string]bool) + if !ok { + t.Error(ErrUnsupportedState) + } + st["step3"] = true + return nil + } + + status, err := Run(ctx, state, step1, step2, step3) + if err == nil { + t.Errorf("err should not be nil") + } + if g, e := err, stepErr; g != e { + t.Errorf("err: expected '%v', got '%v'", e, g) + } + + if g, e := status, Error; g != e { + t.Errorf("status: expected '%v', got '%v'", e, g) + } + + if g, e := state["step1"], true; g != e { + t.Errorf("state[\"step1\"]: expected '%v', got '%v'", e, g) + } + + if g, e := state["step3"], false; g != e { + t.Errorf("state[\"step3\"]: expected '%v', got '%v'", e, g) + } + +}