Implement sops publish command (#473)

* Implement `sops publish` command

Publishes a file to a pre-configured destination (this lives in the sops
config file). Additionally, support re-encryption rules that work
just like the creation rules. Initial support for S3/GCS.

This is a part of the sops-workspace v2.0 project

Includes the addition of a new dependency:
  github.com/googleapis/gax-go/v2

* code review changes; support global --verbose flag

* Switch to recreation_rule with full support

Reencryption rule is now recreation rule and supports everything that a
creation rule does. Now, when you load a config for a file, you load
either the creation rule or the destination rule. I'm not sure about
this style long term, but it allows for support to be added for the
recreation rules without a bigger refactor of how the config file works.

* split loadForFileFromBytes into two functions

remove branching based on destination rule or not, create one for
creation rules and one for destination rules

* pretty diff for keygroup updates in sops publish

Author: ajvb

README.rst

@@ -791,6 +791,31 @@ By default ``sops`` just dumps all the output to the standard output. We can use
 Beware using both ``--in-place`` and ``--output`` flags will result in an error.
 
 
+Using the publish command
+~~~~~~~~~~~~~~~~~~~~~~~~~
+``sops publish $file`` publishes a file to a pre-configured destination (this lives in the sops
+config file). Additionally, support re-encryption rules that work just like the creation rules.
+
+This command requires a ``.sops.yaml`` configuration file. Below is an example:
+
+.. code:: yaml
+
+   destination_rules:
+      - s3_bucket: "sops-secrets"
+        path_regex: s3/*
+        recreation_rule:
+           pgp: F69E4901EDBAD2D1753F8C67A64535C4163FB307
+      - gcs_bucket: "sops-secrets"
+        path_regex: gcs/*
+        recreation_rule:
+           pgp: F69E4901EDBAD2D1753F8C67A64535C4163FB307
+
+The above configuration will place all files under ``s3/*`` into the S3 bucket ``sops-secrets`` and
+will place all files under ``gcs/*`` into the GCS bucket ``sops-secrets``. As well, it will decrypt
+these files and re-encrypt them using the ``F69E4901EDBAD2D1753F8C67A64535C4163FB307`` pgp key.
+
+You would deploy a file to S3 with a command like: ``sops publish s3/app.yaml``
+
 Important information on types
 ------------------------------
 

cmd/sops/common/common.go

@@ -8,9 +8,11 @@ import (
 	"strings"
 	"time"
 
+	"github.com/fatih/color"
 	wordwrap "github.com/mitchellh/go-wordwrap"
 	"go.mozilla.org/sops"
 	"go.mozilla.org/sops/cmd/sops/codes"
+	"go.mozilla.org/sops/keys"
 	"go.mozilla.org/sops/keyservice"
 	"go.mozilla.org/sops/kms"
 	"go.mozilla.org/sops/stores/dotenv"
@@ -339,3 +341,65 @@ func RecoverDataKeyFromBuggyKMS(opts GenericDecryptOpts, tree *sops.Tree) []byte
 
 	return nil
 }
+
+type Diff struct {
+	Common  []keys.MasterKey
+	Added   []keys.MasterKey
+	Removed []keys.MasterKey
+}
+
+func max(a, b int) int {
+	if a > b {
+		return a
+	}
+	return b
+}
+
+func DiffKeyGroups(ours, theirs []sops.KeyGroup) []Diff {
+	var diffs []Diff
+	for i := 0; i < max(len(ours), len(theirs)); i++ {
+		var diff Diff
+		var ourGroup, theirGroup sops.KeyGroup
+		if len(ours) > i {
+			ourGroup = ours[i]
+		}
+		if len(theirs) > i {
+			theirGroup = theirs[i]
+		}
+		ourKeys := make(map[string]struct{})
+		theirKeys := make(map[string]struct{})
+		for _, key := range ourGroup {
+			ourKeys[key.ToString()] = struct{}{}
+		}
+		for _, key := range theirGroup {
+			if _, ok := ourKeys[key.ToString()]; ok {
+				diff.Common = append(diff.Common, key)
+			} else {
+				diff.Added = append(diff.Added, key)
+			}
+			theirKeys[key.ToString()] = struct{}{}
+		}
+		for _, key := range ourGroup {
+			if _, ok := theirKeys[key.ToString()]; !ok {
+				diff.Removed = append(diff.Removed, key)
+			}
+		}
+		diffs = append(diffs, diff)
+	}
+	return diffs
+}
+
+func PrettyPrintDiffs(diffs []Diff) {
+	for i, diff := range diffs {
+		color.New(color.Underline).Printf("Group %d\n", i+1)
+		for _, c := range diff.Common {
+			fmt.Printf("    %s\n", c.ToString())
+		}
+		for _, c := range diff.Added {
+			color.New(color.FgGreen).Printf("+++ %s\n", c.ToString())
+		}
+		for _, c := range diff.Removed {
+			color.New(color.FgRed).Printf("--- %s\n", c.ToString())
+		}
+	}
+}

cmd/sops/main.go

@@ -21,6 +21,7 @@ import (
 	"go.mozilla.org/sops/cmd/sops/common"
 	"go.mozilla.org/sops/cmd/sops/subcommand/groups"
 	keyservicecmd "go.mozilla.org/sops/cmd/sops/subcommand/keyservice"
+	publishcmd "go.mozilla.org/sops/cmd/sops/subcommand/publish"
 	"go.mozilla.org/sops/cmd/sops/subcommand/updatekeys"
 	"go.mozilla.org/sops/config"
 	"go.mozilla.org/sops/gcpkms"
@@ -105,6 +106,49 @@ func main() {
    For more information, see the README at github.com/mozilla/sops`
 	app.EnableBashCompletion = true
 	app.Commands = []cli.Command{
+		{
+			Name:      "publish",
+			Usage:     "Publish sops file to a configured destination",
+			ArgsUsage: `file`,
+			Flags: append([]cli.Flag{
+				cli.BoolFlag{
+					Name:  "yes, y",
+					Usage: `pre-approve all changes and run non-interactively`,
+				},
+				cli.BoolFlag{
+					Name:  "verbose",
+					Usage: "Enable verbose logging output",
+				},
+			}, keyserviceFlags...),
+			Action: func(c *cli.Context) error {
+				if c.Bool("verbose") || c.GlobalBool("verbose") {
+					logging.SetLevel(logrus.DebugLevel)
+				}
+				configPath, err := config.FindConfigFile(".")
+				if err != nil {
+					return common.NewExitError(err, codes.ErrorGeneric)
+				}
+				if c.NArg() < 1 {
+					return common.NewExitError("Error: no file specified", codes.NoFileSpecified)
+				}
+				fileName := c.Args()[0]
+				inputStore := inputStore(c, fileName)
+				err = publishcmd.Run(publishcmd.Opts{
+					ConfigPath:  configPath,
+					InputPath:   fileName,
+					InputStore:  inputStore,
+					Cipher:      aes.NewCipher(),
+					KeyServices: keyservices(c),
+					Interactive: !c.Bool("yes"),
+				})
+				if cliErr, ok := err.(*cli.ExitError); ok && cliErr != nil {
+					return cliErr
+				} else if err != nil {
+					return common.NewExitError(err, codes.ErrorGeneric)
+				}
+				return nil
+			},
+		},
 		{
 			Name:  "keyservice",
 			Usage: "start a SOPS key service server",
@@ -129,7 +173,7 @@ func main() {
 				},
 			},
 			Action: func(c *cli.Context) error {
-				if c.Bool("verbose") {
+				if c.Bool("verbose") || c.GlobalBool("verbose") {
 					logging.SetLevel(logrus.DebugLevel)
 				}
 				err := keyservicecmd.Run(keyservicecmd.Opts{

cmd/sops/subcommand/publish/publish.go

@@ -0,0 +1,151 @@
+package publish
+
+import (
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"go.mozilla.org/sops"
+	"go.mozilla.org/sops/cmd/sops/codes"
+	"go.mozilla.org/sops/cmd/sops/common"
+	"go.mozilla.org/sops/config"
+	"go.mozilla.org/sops/keyservice"
+	"go.mozilla.org/sops/logging"
+	"go.mozilla.org/sops/version"
+
+	"github.com/sirupsen/logrus"
+)
+
+var log *logrus.Logger
+
+func init() {
+	log = logging.NewLogger("PUBLISH")
+}
+
+type Opts struct {
+	Interactive bool
+	Cipher      sops.Cipher
+	ConfigPath  string
+	InputPath   string
+	KeyServices []keyservice.KeyServiceClient
+	InputStore  sops.Store
+}
+
+func Run(opts Opts) error {
+	var fileContents []byte
+	path, err := filepath.Abs(opts.InputPath)
+	if err != nil {
+		return err
+	}
+	info, err := os.Stat(path)
+	if err != nil {
+		return err
+	}
+	if info.IsDir() {
+		return fmt.Errorf("can't operate on a directory")
+	}
+	_, fileName := filepath.Split(path)
+
+	conf, err := config.LoadDestinationRuleForFile(opts.ConfigPath, opts.InputPath, make(map[string]*string))
+	if err != nil {
+		return err
+	}
+	if conf.Destination == nil {
+		return errors.New("no destination configured for this file")
+	}
+
+	// Check that this is a sops-encrypted file
+	tree, err := common.LoadEncryptedFile(opts.InputStore, opts.InputPath)
+	if err != nil {
+		return err
+	}
+
+	// Re-encrypt if settings exist to do so
+	if len(conf.KeyGroups[0]) != 0 {
+		log.Debug("Re-encrypting tree before publishing")
+		_, err = common.DecryptTree(common.DecryptTreeOpts{
+			Cipher:      opts.Cipher,
+			IgnoreMac:   false,
+			Tree:        tree,
+			KeyServices: opts.KeyServices,
+		})
+		if err != nil {
+			return err
+		}
+
+		diffs := common.DiffKeyGroups(tree.Metadata.KeyGroups, conf.KeyGroups)
+		keysWillChange := false
+		for _, diff := range diffs {
+			if len(diff.Added) > 0 || len(diff.Removed) > 0 {
+				keysWillChange = true
+			}
+		}
+		if keysWillChange {
+			fmt.Printf("The following changes will be made to the file's key groups:\n")
+			common.PrettyPrintDiffs(diffs)
+		}
+
+		tree.Metadata = sops.Metadata{
+			KeyGroups:         conf.KeyGroups,
+			UnencryptedSuffix: conf.UnencryptedSuffix,
+			EncryptedSuffix:   conf.EncryptedSuffix,
+			Version:           version.Version,
+			ShamirThreshold:   conf.ShamirThreshold,
+		}
+
+		dataKey, errs := tree.GenerateDataKeyWithKeyServices(opts.KeyServices)
+		if len(errs) > 0 {
+			err = fmt.Errorf("Could not generate data key: %s", errs)
+			return err
+		}
+
+		err = common.EncryptTree(common.EncryptTreeOpts{
+			DataKey: dataKey,
+			Tree:    tree,
+			Cipher:  opts.Cipher,
+		})
+		if err != nil {
+			return err
+		}
+
+		fileContents, err = opts.InputStore.EmitEncryptedFile(*tree)
+		if err != nil {
+			return common.NewExitError(fmt.Sprintf("Could not marshal tree: %s", err), codes.ErrorDumpingTree)
+		}
+	} else {
+		fileContents, err = ioutil.ReadFile(path)
+		if err != nil {
+			return fmt.Errorf("could not read file: %s", err)
+		}
+	}
+
+	if opts.Interactive {
+		var response string
+		for response != "y" && response != "n" {
+			fmt.Printf("uploading %s to %s ? (y/n): ", path, conf.Destination.Path(fileName))
+			_, err := fmt.Scanln(&response)
+			if err != nil {
+				return err
+			}
+		}
+		if response == "n" {
+			return errors.New("Publish canceled")
+		}
+	}
+
+	err = conf.Destination.Upload(fileContents, fileName)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func min(a, b int) int {
+	if a < b {
+		return a
+	}
+	return b
+}